Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,12 @@ jobs:
- name: Lint with Ruff
run: uv run ruff check .

- name: Check formatting with Ruff
run: uv run ruff format --check .

- name: Check spelling with codespell
run: uv run codespell

- name: Type check with ty
run: uv run ty check

Expand Down
44 changes: 22 additions & 22 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -59,31 +59,31 @@ v = vsparse.VCSCArray.from_scipy(csc)

# Or build from an AnnData object or SciPy sparse array
adata = ... # an AnnData object
v = vsparse.from_anndata(adata) # VCSCArray (column-compressed) from adata.X
vr = vsparse.from_anndata(adata, format="csr") # VCSRArray (row-compressed)
v = vsparse.from_anndata(adata) # VCSCArray (column-compressed) from adata.X
vr = vsparse.from_anndata(adata, format="csr") # VCSRArray (row-compressed)

# Transposition is zero-copy (swaps major/minor axes and shares buffers)
vr = v.T # VCSRArray
vr = v.T # VCSRArray

# Scalar arithmetic & math
v2 = v * 2.0 # Scalar multiplication
v_div = v / 2.0 # Scalar division
v_neg = -v # Negation
v_log = v.log1p() # Elementwise log1p
v2 = v * 2.0 # Scalar multiplication
v_div = v / 2.0 # Scalar division
v_neg = -v # Negation
v_log = v.log1p() # Elementwise log1p

# Matrix & vector products (Numba-parallelized)
y = v @ x # Matrix-vector: (n_rows, n_cols) @ (n_cols,) -> (n_rows,)
y_left = x @ v # Vector-matrix: (n_rows,) @ (n_rows, n_cols) -> (n_cols,)
Y = v @ B # Matrix-matrix: (n_rows, n_cols) @ (n_cols, k) -> (n_rows, k)
Y_left = B @ v # Matrix-matrix: (k, n_rows) @ (n_rows, n_cols) -> (k, n_cols)
y = v @ x # Matrix-vector: (n_rows, n_cols) @ (n_cols,) -> (n_rows,)
y_left = x @ v # Vector-matrix: (n_rows,) @ (n_rows, n_cols) -> (n_cols,)
Y = v @ B # Matrix-matrix: (n_rows, n_cols) @ (n_cols, k) -> (n_rows, k)
Y_left = B @ v # Matrix-matrix: (k, n_rows) @ (n_rows, n_cols) -> (k, n_cols)

# Slicing
col_slice = v[:, [1, 3, 5]] # Fast major-axis slicing (returns VCSCArray)
sub = v[0:10, 0:10] # 2D slicing (falls back to scipy)
col_slice = v[:, [1, 3, 5]] # Fast major-axis slicing (returns VCSCArray)
sub = v[0:10, 0:10] # 2D slicing (falls back to scipy)

# Conversion & layers
sp_csc = v.to_scipy() # -> scipy.sparse.csc_array (or to_csr())
dense = v.toarray() # -> numpy.ndarray
sp_csc = v.to_scipy() # -> scipy.sparse.csc_array (or to_csr())
dense = v.toarray() # -> numpy.ndarray
vsparse.to_layer(adata, v, key="counts_vcsc") # Attach to AnnData layer
```

Expand All @@ -96,15 +96,15 @@ is an `AnnData` subclass whose `X` (and optionally `raw_X`) is backed directly b
```python
import vsparse

va = vsparse.VCSCAnnData.from_anndata(adata) # Compresses X and raw.X
va.X # VCSCArray
va.raw_X # VCSCArray (separate from anndata's .raw)
va = vsparse.VCSCAnnData.from_anndata(adata) # Compresses X and raw.X
va.X # VCSCArray
va.raw_X # VCSCArray (separate from anndata's .raw)

# Persist to HDF5 (.h5ad) or Zarr with default Blosc2+LZ4 compression
va.write_h5ad("compressed.h5ad") # Read back with VCSCAnnData.read_h5ad
va.write_h5ad("compressed.h5ad") # Read back with VCSCAnnData.read_h5ad
va2 = vsparse.VCSCAnnData.read_h5ad("compressed.h5ad")

va.write_zarr("compressed.zarr") # Read back with VCSCAnnData.read_zarr
va.write_zarr("compressed.zarr") # Read back with VCSCAnnData.read_zarr
va3 = vsparse.VCSCAnnData.read_zarr("compressed.zarr")

# Escape hatch back to standard AnnData
Expand Down Expand Up @@ -135,8 +135,8 @@ For IVCSR-backed datasets, `vsparse.load_and_normalize` bypasses full array deco
```python
adata_norm = vsparse.load_and_normalize(
"archived.h5ad",
min_cell_counts=10.0, # Filter cells with counts <= 10
gene_threshold=0.05, # Filter genes with counts <= 0.05 * n_cells
min_cell_counts=10.0, # Filter cells with counts <= 10
gene_threshold=0.05, # Filter genes with counts <= 0.05 * n_cells
)
```

Expand Down
28 changes: 14 additions & 14 deletions docs/usage.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,23 +31,23 @@ from vsparse import VCSCArray
csc = sp.csc_array(dense_or_sparse_matrix)
v = VCSCArray.from_scipy(csc)

back = v.to_scipy() # -> scipy.sparse.csc_array
dense = v.toarray() # -> numpy.ndarray
back = v.to_scipy() # -> scipy.sparse.csc_array
dense = v.toarray() # -> numpy.ndarray
```

## Supported operations

```python
v.T # transpose (free: returns a VCSRArray sharing buffers)
v * 2.0 # scalar multiplication
v / 2.0 # scalar division
v.log1p() # elementwise log1p
v @ x # matrix-vector product, x: 1-D array of length n_cols
x @ v # vector-matrix product, x: 1-D array of length n_rows
v @ B # matrix-matrix product, B: 2-D array with B.shape[0] == n_cols
B @ v # matrix-matrix product, B: 2-D array with B.shape[1] == n_rows
v[:, [1, 3, 5]] # major-axis (column, for VCSCArray) indexing
v[0:2, 0:2] # general 2-D indexing (falls back to scipy internally)
v.T # transpose (free: returns a VCSRArray sharing buffers)
v * 2.0 # scalar multiplication
v / 2.0 # scalar division
v.log1p() # elementwise log1p
v @ x # matrix-vector product, x: 1-D array of length n_cols
x @ v # vector-matrix product, x: 1-D array of length n_rows
v @ B # matrix-matrix product, B: 2-D array with B.shape[0] == n_cols
B @ v # matrix-matrix product, B: 2-D array with B.shape[1] == n_rows
v[:, [1, 3, 5]] # major-axis (column, for VCSCArray) indexing
v[0:2, 0:2] # general 2-D indexing (falls back to scipy internally)
```

Attach a decompressed result back onto an `AnnData` object as a layer:
Expand All @@ -65,8 +65,8 @@ vsparse.to_layer(adata, v, key="vcsc_roundtrip")
import vsparse

va = vsparse.VCSCAnnData.from_anndata(adata) # compresses X and raw.X
va.X # a VCSCArray
va.raw_X # a VCSCArray (kept separately from anndata's own `.raw`)
va.X # a VCSCArray
va.raw_X # a VCSCArray (kept separately from anndata's own `.raw`)

# Persist -- read back with the matching classmethod, not anndata.read_h5ad/read_zarr
va.write_h5ad("compressed.h5ad")
Expand Down
7 changes: 6 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ dev = [
"pytest-cov>=5.0",
"ruff>=0.6",
"ty>=0.0.1a1",
"codespell>=2.3",
]

[build-system]
Expand All @@ -55,12 +56,16 @@ line-length = 100
target-version = "py312"

[tool.ruff.lint]
select = ["E", "F", "I", "UP", "B", "NPY", "RUF"]
select = ["E", "F", "B", "NPY", "PD", "SIM", "RET", "PIE", "ERA", "PERF", "UP", "C4", "RUF"]
ignore = ["E501"]

[tool.ruff.lint.isort]
known-first-party = ["vsparse"]

[tool.codespell]
skip = "*.lock,uv.lock,.venv,docs/_build,benchmarks/baselines.json"
ignore-words-list = "coo"

[tool.ty.environment]
python-version = "3.12"

Expand Down
2 changes: 1 addition & 1 deletion src/vsparse/_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ def _read(elem: GroupStorageType, *, _reader: Reader) -> _VCSBase:
values = cast(np.ndarray, _reader.read_elem(elem["values"]))
value_ptr = cast(np.ndarray, _reader.read_elem(elem["value_ptr"]))
packed = cast(np.ndarray, _reader.read_elem(elem["packed_indices"]))
dtype = np.dtype(elem.attrs["indices_dtype"])
dtype = np.dtype(cast(str, elem.attrs["indices_dtype"]))
indices = _ivcsc.unpack_indices(value_ptr, packed, dtype)
return cls(shape, major_ptr, values, value_ptr, indices)

Expand Down
1 change: 0 additions & 1 deletion tests/test_anndata.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,4 +76,3 @@ def test_to_layer_shape_mismatch_raises(adata, dense):
mismatched_adata = ad.AnnData(X=np.zeros((dense.shape[0] + 1, dense.shape[1])))
with pytest.raises(ValueError, match="shape mismatch"):
vsparse.to_layer(mismatched_adata, v, key="layer_key")

5 changes: 3 additions & 2 deletions tests/test_anndata_class.py
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,9 @@ def test_uns_embedding_roundtrips_via_registry(base_adata, dense, tmp_path):
def test_x_setter_validation(base_adata, dense):
"""Verify that assigning a valid X matrix updates the X property and shape mismatch raises ValueError."""
va = VCSCAnnData.from_anndata(base_adata)
mismatched_v = VCSCArray.from_scipy(sp.csc_array(np.zeros((dense.shape[0] + 1, dense.shape[1]))))
mismatched_v = VCSCArray.from_scipy(
sp.csc_array(np.zeros((dense.shape[0] + 1, dense.shape[1])))
)
with pytest.raises(ValueError, match="does not match adata shape"):
va.X = mismatched_v

Expand Down Expand Up @@ -249,4 +251,3 @@ def test_getitem_single_int_row(base_adata, dense):
assert sub.shape == (1, dense.shape[1])
assert isinstance(sub.X, VCSCArray)
np.testing.assert_allclose(sub.X.toarray(), dense[0:1])

8 changes: 5 additions & 3 deletions tests/test_compression.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,11 @@ def test_zarr_dataset_kwargs_options():

def test_numeric_only_compression_invalid_backend():
"""Verify that numeric_only_compression raises ValueError for an unsupported format."""
with pytest.raises(ValueError, match="store_kind must be 'h5' or 'zarr'"):
with _compression.numeric_only_compression("invalid_format"):
pass
with (
pytest.raises(ValueError, match="store_kind must be 'h5' or 'zarr'"),
_compression.numeric_only_compression("invalid_format"),
):
pass


def test_is_string_like_detection():
Expand Down
4 changes: 1 addition & 3 deletions tests/test_index_dtypes.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,7 @@ def _with_int64_indices(mat):
return out


@pytest.mark.parametrize(
("n", "expected"), [(INT32_MAX, np.int32), (INT32_MAX + 1, np.int64)]
)
@pytest.mark.parametrize(("n", "expected"), [(INT32_MAX, np.int32), (INT32_MAX + 1, np.int64)])
def test_dtype_switches_at_the_int32_boundary(n, expected):
assert smallest_index_dtype(n) == np.dtype(expected)

Expand Down
1 change: 0 additions & 1 deletion tests/test_indexing.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,4 +123,3 @@ def test_invalid_tuple_indexing_dimensions_raises(dense, vcls):
v = vcls.from_scipy(sp.csr_array(dense))
with pytest.raises(IndexError, match="arrays are 2-D"):
_ = v[0, 0, 0]

4 changes: 3 additions & 1 deletion tests/test_ivcsc.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,9 @@ def test_unpack_parallel_matches_serial():
v = VCSCArray.from_scipy(sp.csc_array(dense))

packed = _ivcsc.pack_indices(v.value_ptr, v.indices)
assert packed.nbytes > _ivcsc._PARALLEL_MIN_BYTES, "test data too small to hit the parallel path"
assert packed.nbytes > _ivcsc._PARALLEL_MIN_BYTES, (
"test data too small to hit the parallel path"
)

out = np.empty(v.indices.shape[0], dtype=v.indices.dtype)
_ivcsc._unpack(v.value_ptr, packed, out)
Expand Down
4 changes: 3 additions & 1 deletion tests/test_metadata_encoding.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,4 +109,6 @@ def test_zarr_write_converts_too(tmp_path):
back = VCSCAnnData.read_zarr(store)

assert isinstance(back.obs["cell_type"].dtype, pd.CategoricalDtype)
np.testing.assert_array_equal(back.obs["cell_type"].astype(str), va.obs["cell_type"].astype(str))
np.testing.assert_array_equal(
back.obs["cell_type"].astype(str), va.obs["cell_type"].astype(str)
)
1 change: 0 additions & 1 deletion tests/test_ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,4 +163,3 @@ def test_matmul_does_not_overflow_narrow_value_dtype(vcls):

ones_mat2 = np.ones((2000, 3), dtype=np.float64)
np.testing.assert_array_equal(v2 @ ones_mat2, np.tile(expected_rows, (3, 1)).T)

22 changes: 14 additions & 8 deletions tests/test_rapid_load.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,7 @@ def test_min_cells_uses_obs_filtered_cohort(tmp_path):
min_cells=2,
obs_filter=lambda obs: obs["condition"] == "control",
)
ref_X, _, ref_genes = _reference_prepare(
dense[obs_mask], -1.0, 0.0, min_cells=2
)
ref_X, _, ref_genes = _reference_prepare(dense[obs_mask], -1.0, 0.0, min_cells=2)

assert list(result.var_names) == [str(i) for i in ref_genes]
assert list(result.var_names) == ["1", "2"]
Expand Down Expand Up @@ -340,11 +338,18 @@ def test_metadata_sliced_correctly(tmp_path, rng):

# Check obsm/varm
np.testing.assert_allclose(np.asarray(result.obsm["pca"]), np.asarray(obsm["pca"])[ref_cells])
np.testing.assert_allclose(np.asarray(result.varm["loadings"]), np.asarray(varm["loadings"])[ref_genes])
np.testing.assert_allclose(
np.asarray(result.varm["loadings"]), np.asarray(varm["loadings"])[ref_genes]
)

# Check obsp/varp
np.testing.assert_allclose(np.asarray(result.obsp["distances"]), np.asarray(obsp["distances"])[ref_cells][:, ref_cells])
np.testing.assert_allclose(np.asarray(result.varp["correlations"]), np.asarray(varp["correlations"])[ref_genes][:, ref_genes])
np.testing.assert_allclose(
np.asarray(result.obsp["distances"]), np.asarray(obsp["distances"])[ref_cells][:, ref_cells]
)
np.testing.assert_allclose(
np.asarray(result.varp["correlations"]),
np.asarray(varp["correlations"])[ref_genes][:, ref_genes],
)

# Check uns
assert result.uns == uns
Expand Down Expand Up @@ -388,7 +393,9 @@ def test_rapid_load_custom_x_key(tmp_path, rng):
with h5py.File(path, "w") as f:
_io.write_ivcs_elem(f, "custom_matrix", vcsr)

result = load_and_normalize(path, x_key="custom_matrix", min_cell_counts=-1.0, gene_threshold=0.0)
result = load_and_normalize(
path, x_key="custom_matrix", min_cell_counts=-1.0, gene_threshold=0.0
)
assert isinstance(result, ad.AnnData)
assert result.shape == dense.shape

Expand Down Expand Up @@ -422,4 +429,3 @@ def test_load_packed_metadata_preserved(tmp_path, rng):
pd.testing.assert_index_equal(result.obs.index, adata.obs.index)
assert list(result.obs["grp"]) == list(adata.obs["grp"])
assert list(result.var["gene"]) == list(adata.var["gene"])

4 changes: 1 addition & 3 deletions tests/test_select_minor_fanout.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,7 @@ def test_matches_scipy_for_a_random_selection_with_repeats(vcls, rng):

for _ in range(25):
cols = rng.integers(0, dense.shape[1], size=rng.integers(1, 15)).tolist()
np.testing.assert_allclose(
v[:, cols].toarray(), np.asarray(reference[:, cols].todense())
)
np.testing.assert_allclose(v[:, cols].toarray(), np.asarray(reference[:, cols].todense()))


# -- selections that already worked, kept working ----------------------------
Expand Down
5 changes: 1 addition & 4 deletions tests/test_vcs_norm_recipes.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,7 @@ def _reference(dense: np.ndarray, recipe: str) -> np.ndarray:
else:
g = x

if recipe in ("parafac2", "scanpy", "pearson"):
c = g.mean(axis=0)
else:
c = np.zeros(dense.shape[1])
c = g.mean(axis=0) if recipe in ("parafac2", "scanpy", "pearson") else np.zeros(dense.shape[1])

if recipe in ("scanpy", "pearson"):
std = g.std(axis=0)
Expand Down
Loading