Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions HISTORY.rst
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
History
=======

unreleased
-----------

1.2.3 (2026-02-03)
------------------
* Fix: Restrict netCDF4 version to not equal 1.7.4 due to a bug related to strings and string variables (#133)
Expand Down
157 changes: 156 additions & 1 deletion sofar/sofastream.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,34 @@
from netCDF4 import Dataset
import numpy as np

import sofar as sf

from .io import _format_value_from_netcdf


class _LazyArray(np.ndarray):
"""Represent a NetCDF numeric variable without reading its data."""

def __new__(cls, shape, dtype):
data = np.array(0, dtype=dtype)
strides = (0, ) * len(shape)
return np.lib.stride_tricks.as_strided(
data, shape=shape, strides=strides).view(cls)

def copy(self, order='C'):
"""Return a view instead of materializing the full array."""
_ = order
return self


class SofaStream():
"""
Read desired data from SOFA-file directly from disk without loading entire
file into memory.

:class:`SofaStream` opens a SOFA-file and retrieves only the requested
data.
data. It can also verify the file against the SOFA convention using the
same checks as :py:func:`~sofar.Sofa.verify`.

If you want to use all the data from a SOFA-file use :class:`Sofa`
class and :func:`read_sofa` function instead.
Expand Down Expand Up @@ -83,6 +103,141 @@ def __exit__(self, *args):
"""
self._file.close()

def verify(self, issue_handling="raise", mode="write"):
"""
Verify a SOFA file against the SOFA standard.

See :py:func:`~sofar.Sofa.verify` for details.

This method checks whether mandatory data are present, names, data
types, and dimensions follow the SOFA standard, and attribute values
are consistent with the SOFA standard. Missing mandatory data raise an
error when ``issue_handling='raise'``; otherwise, they are added with
their default value and a warning is given.

Numeric variables are represented by lazy placeholders, so only their
type and dimensions are checked without loading their contents into
memory. String variables are loaded because their values and lengths
can be relevant for convention verification.

A detailed set of validation rules can be found at
https://github.com/pyfar/sofar/tree/main/sofar/verification_rules

Parameters
----------
issue_handling : str, optional
Defines how detected issues are handled.

``'raise'``
Warnings and errors are raised if issues are detected.
``'print'``
Issues are printed without raising warnings and errors.
``'return'``
Issues are returned as a string but neither raised nor printed.

The default is ``'raise'``.
mode : str, optional
The SOFA standard is more strict for writing data than for reading
data.

``'write'``
All units (e.g., ``'meter'``) must be lower case.
``'read'``
Units can contain upper case letters (e.g., ``'Meter'``).

The default is ``'write'``.

Comment thread
artpelling marked this conversation as resolved.
Returns
-------
issues : str, None
Detected issues as a string. None if no issues were detected. Note
that this is only returned if ``issue_handling='return'``.

Examples
--------
>>> import sofar
>>>
>>> with sofar.SofaStream("path/to/your.sofa") as file:
>>> file.verify()
"""
return self._verify_open_file(issue_handling, mode)

def _verify_open_file(self, issue_handling, mode):
"""Verify the currently opened NetCDF file."""
return self._read_open_file_for_verification().verify(
issue_handling=issue_handling, mode=mode)

def _read_open_file_for_verification(self):
"""Read the currently opened file using lazy numeric variables."""
# get SOFA object with default values and convention metadata
sofa = sf.Sofa(
self._file.SOFAConventions,
version=self._file.SOFAConventionsVersion,
verify=False)
sofa.protected = False

# NetCDF attribute _Encoding is skipped when reading SOFA files
skip = ["_Encoding"]
all_attr = []

# load global attributes
for attr in self._file.ncattrs():
value = getattr(self._file, attr)
key = f"GLOBAL_{attr}"
all_attr.append(key)

if not hasattr(sofa, key):
sofa._add_custom_api_entry(key, value, None, None, "attribute")
sofa.protected = False
else:
setattr(sofa, key, value)

# load variable metadata and string values
for var in self._file.variables.keys():
netcdf_variable = self._file[var]
key = var.replace(".", "_")
all_attr.append(key)

if netcdf_variable.datatype == "S1":
value = _format_value_from_netcdf(netcdf_variable[:], var)
dtype = "string"
else:
value = _LazyArray(
netcdf_variable.shape, netcdf_variable.dtype)
dtype = "double"

if hasattr(sofa, key):
setattr(sofa, key, value)
else:
dimensions = "".join(list(netcdf_variable.dimensions))
sofa._add_custom_api_entry(
key, value, None, dimensions, dtype)
sofa.protected = False

# load variable attributes
for attr in [a for a in netcdf_variable.ncattrs()
if a not in skip]:
value = getattr(netcdf_variable, attr)
attr_key = key + "_" + attr
all_attr.append(attr_key)

if not hasattr(sofa, attr_key):
sofa._add_custom_api_entry(
attr_key, value, None, None, "attribute")
sofa.protected = False
else:
setattr(sofa, attr_key, value)

# remove default entries that were not contained in the file
attrs = [attr for attr in sofa.__dict__.keys()
if not attr.startswith("_")]
for attr in attrs:
if attr not in all_attr:
delattr(sofa, attr)

sofa.protected = True
return sofa

def __getattr__(self, name):
"""
Executed when accessing data within a with statement
Expand Down
2 changes: 1 addition & 1 deletion sofar/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,7 @@ def _atleast_nd(array, ndim):

def _nd_newaxis(array, ndim):
"""Append dimensions to the end of an array until array.ndim == ndim."""
array = np.array(array)
array = np.asarray(array)

for _ in range(ndim - array.ndim):
array = array[..., np.newaxis]
Expand Down
39 changes: 39 additions & 0 deletions tests/test_sofastream.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,45 @@ def test_sofastream_attribute_error(temp_sofa_file):
_ = file.Wrong_Attribute


def test_sofastream_verify_does_not_call_read_sofa(
temp_sofa_file, monkeypatch):
"""Verify SofaStream does not load the full SOFA file."""

def fail_read_sofa(*_args, **_kwargs):
raise AssertionError("SofaStream.verify must not call read_sofa")

monkeypatch.setattr(sf, "read_sofa", fail_read_sofa)

with SofaStream(temp_sofa_file) as file:
file.verify()


def test_sofastream_read_open_file_for_verification(temp_sofa_file):
"""Verify the lazy SOFA object retains file metadata."""
expected = sf.read_sofa(temp_sofa_file, verify=False)

with SofaStream(temp_sofa_file) as file:
actual = file._read_open_file_for_verification()

expected_keys = {key for key in expected.__dict__
if not key.startswith("_")}
actual_keys = {key for key in actual.__dict__
if not key.startswith("_")}
assert actual_keys == expected_keys

for key in expected_keys:
expected_value = getattr(expected, key)
actual_value = getattr(actual, key)
if isinstance(actual_value, np.ndarray):
expected_array = np.atleast_1d(expected_value)
assert actual_value.shape == expected_array.shape
assert actual_value.dtype == expected_array.dtype
if np.issubdtype(expected_array.dtype, np.str_):
np.testing.assert_array_equal(actual_value, expected_array)
else:
assert actual_value == expected_value


def test_sofastream_inspect(capfd, temp_sofa_file):

tempdir = TemporaryDirectory()
Expand Down