From 2e19844fe2d716763f880f2a850ac13c858271ba Mon Sep 17 00:00:00 2001 From: dance858 Date: Sat, 5 Sep 2026 09:12:56 -0700 Subject: [PATCH 1/6] Build free-threaded (cp313t/cp314t) wheels and declare GIL-free support Closes #18. The extension now calls PyUnstable_Module_SetGIL(Py_MOD_GIL_NOT_USED) on free-threaded CPython, so importing it no longer re-enables the GIL. The build matrix gains an `abi` axis ("" / "t") for 3.13 and 3.14, opts in via CIBW_ENABLE=cpython-freethreading, and the wheel test asserts the GIL stays disabled after import on the t builds. Audit backing the declaration (engine d8cb9b8 + these bindings): - nm on libdnlp_diff.a shows no writable data/bss symbols; the only globals in the engine are the SP_TRACK_MEMORY byte counters, which the wheel build does not enable. No function-local statics, no non-reentrant libc. - All version counters (values_version, *_seen) are per-matrix/per-expr. - Every problem/expr owns its buffers; forward, update_params and the problem constructor memcpy their inputs, and every wrapper copies results into fresh NumPy arrays, so no Python memory is aliased across calls. - The only static state in the bindings is the NumPy-init flag and the module def, both written once at import under the import lock. - expr refcounts are plain ints, so a single expr/problem capsule must not be used from two threads at once -- the same contract as with the GIL. Verified locally on Homebrew python3.14t: import keeps the GIL disabled and 16 threads x 200 evaluations on distinct problems are bit-identical to a single-threaded reference (forward, jacobian, hessian). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_013aQs1u37EG6XTHFrcaBRkA --- .github/workflows/build-and-publish.yml | 19 ++++++++++++++++--- README.md | 15 +++++++++++++++ sparsediffpy/_bindings/bindings.c | 14 +++++++++++++- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 8d33d33..e9cda89 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -32,6 +32,14 @@ jobs: matrix: os: [ ubuntu-22.04, macos-14, windows-2022 ] python-version: [ "3.11", "3.12", "3.13", "3.14" ] + # "" builds the default (GIL) interpreter, "t" the free-threaded one. + # Free-threaded CPython exists from 3.13 onwards. + abi: [ "", "t" ] + exclude: + - python-version: "3.11" + abi: "t" + - python-version: "3.12" + abi: "t" env: PYTHON_VERSION: ${{ matrix.python-version }} @@ -60,7 +68,10 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v3.3.0 env: - CIBW_BUILD: "cp3${{ env.PYTHON_SUBVERSION }}-*" + CIBW_BUILD: "cp3${{ env.PYTHON_SUBVERSION }}${{ matrix.abi }}-*" + # Opt in to cp3XXt build identifiers; harmless for the GIL builds since + # CIBW_BUILD already selects a single ABI. + CIBW_ENABLE: "cpython-freethreading" CIBW_SKIP: "*-win32 *-manylinux_i686 *-musllinux*" CIBW_ARCHS_MACOS: "x86_64 universal2" CIBW_ARCHS_LINUX: "auto aarch64" @@ -69,7 +80,9 @@ jobs: CIBW_ENVIRONMENT_WINDOWS: "CMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake" # Bundle openblas.dll into the wheel (auditwheel/delocate handle this on Linux/macOS) CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel} --add-path C:/vcpkg/installed/x64-windows/bin" - CIBW_TEST_COMMAND: python -c "import sparsediffpy" + # On free-threaded builds also assert the import did not re-enable the GIL, + # i.e. the extension really declares Py_MOD_GIL_NOT_USED. + CIBW_TEST_COMMAND: python -c "import sys, sysconfig, sparsediffpy; ft = sysconfig.get_config_var('Py_GIL_DISABLED') == 1; assert not ft or not sys._is_gil_enabled(), 'GIL re-enabled on import'" - name: Check wheels shell: bash run: | @@ -78,7 +91,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.os }}-${{ matrix.python-version }} + name: wheels-${{ matrix.os }}-${{ matrix.python-version }}${{ matrix.abi }} path: ./wheelhouse/*.whl build_sdist: diff --git a/README.md b/README.md index f515061..d8e2b2b 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,21 @@ pip install sparsediffpy from sparsediffpy import _sparsediffengine ``` +## Free-threaded Python + +Wheels are published for the free-threaded CPython builds (`cp313t`, `cp314t`) as +well as the default ones. The extension declares that it does not need the GIL: +the engine keeps no global mutable state, every problem and expression owns its +own buffers, and all inputs and outputs are copied at the boundary. Distinct +problems can therefore be built and evaluated concurrently from different +threads. + +A single problem or expression capsule is not thread-safe. Do not call into the +same problem from two threads at once, and do not share expression capsules +between problems that are evaluated concurrently. This is the same contract as +under the GIL, which only serialized individual calls and never protected +against interleaved use of one object. + ## License Apache License 2.0 diff --git a/sparsediffpy/_bindings/bindings.c b/sparsediffpy/_bindings/bindings.c index 7eaac62..b08e5b4 100644 --- a/sparsediffpy/_bindings/bindings.c +++ b/sparsediffpy/_bindings/bindings.c @@ -187,5 +187,17 @@ static struct PyModuleDef sparsediffpy_module = { PyMODINIT_FUNC PyInit__sparsediffengine(void) { if (ensure_numpy() < 0) return NULL; - return PyModule_Create(&sparsediffpy_module); + PyObject *module = PyModule_Create(&sparsediffpy_module); + if (!module) return NULL; +#ifdef Py_GIL_DISABLED + /* Free-threaded CPython (3.13t+): declare that this module does not need + the GIL. The engine keeps no global mutable state -- every problem and + expression owns its own buffers, and the wrappers copy inputs and + outputs -- so distinct problems may be used concurrently from different + threads. A single problem or expression capsule is not thread-safe and + must not be used from two threads at once (same contract as with the + GIL, which never protected against interleaved calls on one object). */ + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); +#endif + return module; } From 8406c344780abc784f777de7ca11b5acdec2ab4d Mon Sep 17 00:00:00 2001 From: dance858 Date: Sat, 5 Sep 2026 09:36:29 -0700 Subject: [PATCH 2/6] Drop 3.13t from the wheel matrix; run twine via python -m NumPy >= 2.5 ships no cp313t wheels on any platform, so the 3.13t builds had to compile NumPy from source: macOS failed outright, Windows took six minutes for the NumPy build alone, and Linux aarch64 under QEMU was still running after an hour. Only 3.14t is built now. The macOS 3.14t job built and tested both wheels fine (GIL stays disabled on x86_64 and arm64) but then failed in the twine step with "pip: command not found". Call pip and twine via python -m so the step does not depend on a pip launcher being on PATH. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_013aQs1u37EG6XTHFrcaBRkA --- .github/workflows/build-and-publish.yml | 9 ++++++--- README.md | 5 +++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index e9cda89..3b629a0 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -33,13 +33,16 @@ jobs: os: [ ubuntu-22.04, macos-14, windows-2022 ] python-version: [ "3.11", "3.12", "3.13", "3.14" ] # "" builds the default (GIL) interpreter, "t" the free-threaded one. - # Free-threaded CPython exists from 3.13 onwards. + # Only 3.14t: NumPy >= 2.5 ships no cp313t wheels on any platform, so a + # 3.13t build would have to compile NumPy from source. abi: [ "", "t" ] exclude: - python-version: "3.11" abi: "t" - python-version: "3.12" abi: "t" + - python-version: "3.13" + abi: "t" env: PYTHON_VERSION: ${{ matrix.python-version }} @@ -86,8 +89,8 @@ jobs: - name: Check wheels shell: bash run: | - pip install --upgrade twine - twine check wheelhouse/* + python -m pip install --upgrade twine + python -m twine check wheelhouse/* - uses: actions/upload-artifact@v4 with: diff --git a/README.md b/README.md index d8e2b2b..b168067 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,9 @@ from sparsediffpy import _sparsediffengine ## Free-threaded Python -Wheels are published for the free-threaded CPython builds (`cp313t`, `cp314t`) as -well as the default ones. The extension declares that it does not need the GIL: +Wheels are published for free-threaded CPython 3.14 (`cp314t`) as well as the +default builds. There is no 3.13t wheel because NumPy 2.5 and later ship no +`cp313t` wheels. The extension declares that it does not need the GIL: the engine keeps no global mutable state, every problem and expression owns its own buffers, and all inputs and outputs are copied at the boundary. Distinct problems can therefore be built and evaluated concurrently from different From ae8799c2a43cb39d189319935cf673ee200ae40a Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 6 Sep 2026 22:04:59 -0400 Subject: [PATCH 3/6] Bump engine to d8cb9b8 so the GIL-free declaration actually holds The audit in 2e19844 was performed against engine d8cb9b8, but the submodule pointer was left at 4172c5e from #21, nine commits behind. The two revisions differ in exactly the way the audit depended on. Engine 6e64401 ("Make peak-memory tracking a compile-time option (SP_TRACK_MEMORY)", merged one day before this branch) puts the g_allocated_bytes / g_peak_bytes counters behind an option that defaults to OFF, precisely so the library can be called from several threads at once. At 4172c5e those counters are unconditional, and every sp_malloc / sp_free updates them non-atomically. So the audit's "no writable data/bss symbols" finding was true of the engine it inspected and false of the engine this branch ships: nm on the cp313t extension built from 4172c5e lists _g_allocated_bytes and _g_peak_bytes as its only external writable data. That race is reachable from the usage the README blesses, since two threads building or evaluating *distinct* problems both allocate. Bumping the pointer removes it: nm on the rebuilt extension shows no mutable globals at all, and the engine's own ctest suite passes at d8cb9b8. Verified on cpython-3.13.5+freethreaded, 8 threads: * independent problem per thread, 300 evaluations each, every result bit-identical to a single-threaded reference: 10/10 runs clean. * sharing one expression capsule across threads, which the README forbids: 10/10 runs die with SIGSEGV / SIGBUS / SIGABRT / SIGTRAP. The second number is why the README and the bindings.c comment no longer describe that contract as "the same contract as with the GIL". expr::refcount is a plain int updated non-atomically by expr_retain() / free_expr(), and capsule destructors run on whichever thread drops the last Python reference, so breaking the rule now corrupts the heap instead of interleaving calls. The same misuse is harmless under the GIL. Making that count atomic upstream would turn it back into ordinary unsupported usage; until then the docs say plainly how sharp the edge is. Both documents also note that SP_TRACK_MEMORY must stay off in a wheel build, since turning it on silently reintroduces the global-counter race. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 40 +++++++++++++++++++++---------- SparseDiffEngine | 2 +- sparsediffpy/_bindings/bindings.c | 25 ++++++++++++++----- 3 files changed, 48 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index b168067..415c7fd 100644 --- a/README.md +++ b/README.md @@ -17,18 +17,34 @@ from sparsediffpy import _sparsediffengine ## Free-threaded Python Wheels are published for free-threaded CPython 3.14 (`cp314t`) as well as the -default builds. There is no 3.13t wheel because NumPy 2.5 and later ship no -`cp313t` wheels. The extension declares that it does not need the GIL: -the engine keeps no global mutable state, every problem and expression owns its -own buffers, and all inputs and outputs are copied at the boundary. Distinct -problems can therefore be built and evaluated concurrently from different -threads. - -A single problem or expression capsule is not thread-safe. Do not call into the -same problem from two threads at once, and do not share expression capsules -between problems that are evaluated concurrently. This is the same contract as -under the GIL, which only serialized individual calls and never protected -against interleaved use of one object. +default builds. There is no `cp313t` wheel: NumPy ships none from 2.5 onwards +(2.4.6 is the last release with one), so a 3.13t build would have to compile +NumPy from source. + +The extension declares that it does not need the GIL, so importing it leaves +free threading enabled. Distinct problems can be built and evaluated +concurrently from different threads: the engine keeps no mutable global state, +every problem and expression owns its own buffers, and all inputs and outputs +are copied at the boundary. + +This depends on the engine being built without `SP_TRACK_MEMORY`, which is the +default. That option makes every allocation update the `g_allocated_bytes` and +`g_peak_bytes` process globals non-atomically, so two threads that merely +allocate race on them. It is a development switch; do not turn it on for a +wheel build. + +A single problem or expression capsule is **not** thread-safe. Do not call into +the same problem from two threads at once, and do not share expression capsules +between problems that are evaluated concurrently. + +Without the GIL, breaking that rule is worse than it used to be. `expr::refcount` +is a plain `int` that `expr_retain()` and `free_expr()` update non-atomically, +and capsule destructors run on whichever thread drops the last Python reference. +So sharing one capsule across threads corrupts the heap instead of merely +interleaving calls: an eight-thread loop building and dropping `exp()` nodes over +one shared variable node crashes on every run here, while the same loop is +harmless under the GIL. Making that reference count atomic upstream would turn +this back into an ordinary "unsupported usage" rather than a crash. ## License diff --git a/SparseDiffEngine b/SparseDiffEngine index 4172c5e..d8cb9b8 160000 --- a/SparseDiffEngine +++ b/SparseDiffEngine @@ -1 +1 @@ -Subproject commit 4172c5ece836bc8d8ebd3fa59b89e019874ea681 +Subproject commit d8cb9b88453dae339c74755f7c3082dafaccbecc diff --git a/sparsediffpy/_bindings/bindings.c b/sparsediffpy/_bindings/bindings.c index b08e5b4..71de8c5 100644 --- a/sparsediffpy/_bindings/bindings.c +++ b/sparsediffpy/_bindings/bindings.c @@ -191,12 +191,25 @@ PyMODINIT_FUNC PyInit__sparsediffengine(void) if (!module) return NULL; #ifdef Py_GIL_DISABLED /* Free-threaded CPython (3.13t+): declare that this module does not need - the GIL. The engine keeps no global mutable state -- every problem and - expression owns its own buffers, and the wrappers copy inputs and - outputs -- so distinct problems may be used concurrently from different - threads. A single problem or expression capsule is not thread-safe and - must not be used from two threads at once (same contract as with the - GIL, which never protected against interleaved calls on one object). */ + the GIL. + + This holds only for an engine built WITHOUT SP_TRACK_MEMORY, which is + the default. With that option on, every sp_malloc / sp_free updates the + g_allocated_bytes and g_peak_bytes process globals non-atomically, so + two threads that merely allocate would race. Never enable it for a + wheel build. + + Otherwise the engine keeps no mutable global state: every problem and + expression owns its buffers, and the wrappers copy inputs and outputs, + so distinct problems may be built and evaluated concurrently. + + A single problem or expression capsule remains NOT thread-safe, and + without the GIL the consequence is harsher than it used to be. + expr::refcount is a plain int that expr_retain() and free_expr() update + non-atomically, and capsule destructors run on whichever thread drops + the last Python reference. Sharing one capsule across threads therefore + corrupts the heap rather than merely interleaving calls. Making that + count atomic upstream would remove the sharp edge. */ PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); #endif return module; From 3ca649b8ffd858d8bde645248e7fdaf38261cdae Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 6 Sep 2026 22:04:59 -0400 Subject: [PATCH 4/6] Tidy the free-threaded wheel build configuration No functional change to the default builds. CIBW_ENABLE=cpython-freethreading was a no-op here. cibuildwheel 3.3.0 gates only cp313t behind that group (selector.py checks fnmatch(build_id, "cp313t-*") before consulting the enable set); cp314t is selectable by default, and this matrix builds no cp313t. The option is also deprecated in 3.4.1 and removed in 4.0, so leaving it set would break a later cibuildwheel bump for no benefit. The matrix excludes now say why each one is there. 3.11t and 3.12t are not build identifiers at all, since free threading starts at 3.13, so without the exclude cibuildwheel selects nothing and the job fails on an empty wheelhouse. 3.13t is a deliberate choice about NumPy instead: no cp313t wheels are published from 2.5 onwards, 2.4.6 being the last, which was confirmed by installing numpy on a free-threaded 3.13 and watching 2.5.3 build from source. CIBW_TEST_COMMAND uses bool(...) rather than == 1 on the Py_GIL_DISABLED config var. The var is always an int on 3.13+, on Windows too, so this is not a fix, just the idiom the free-threading porting guide uses. It also prints the flag so the log shows which ABI the job exercised. cmake.version moves to >=3.30.3. CMake 3.30 is the first whose FindPython3 understands the free-threaded "t" ABI, and scikit-build-core only sends the Python3_FIND_ABI hint when the configured specifier admits 3.30+, so at >=3.15 a 3.15-3.29 system CMake would silently skip it. 3.30.3 adds the Python3_DEFINITIONS that a Windows free-threaded build needs, since PC/pyconfig.h relies on Py_GIL_DISABLED being defined to auto-link python3XXt.lib. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build-and-publish.yml | 30 ++++++++++++------------- pyproject.toml | 6 ++++- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 3b629a0..7f458ca 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -32,17 +32,18 @@ jobs: matrix: os: [ ubuntu-22.04, macos-14, windows-2022 ] python-version: [ "3.11", "3.12", "3.13", "3.14" ] - # "" builds the default (GIL) interpreter, "t" the free-threaded one. - # Only 3.14t: NumPy >= 2.5 ships no cp313t wheels on any platform, so a - # 3.13t build would have to compile NumPy from source. + # "" builds the default interpreter, "t" the free-threaded one. abi: [ "", "t" ] exclude: - - python-version: "3.11" - abi: "t" - - python-version: "3.12" - abi: "t" - - python-version: "3.13" - abi: "t" + # Free threading does not exist before 3.13: cp311t and cp312t are + # not build identifiers at all, so cibuildwheel would select nothing + # and the job would fail on an empty wheelhouse. + - { python-version: "3.11", abi: "t" } + - { python-version: "3.12", abi: "t" } + # NumPy ships no cp313t wheels from 2.5 onwards (2.4.6 is the last + # release with one), so a 3.13t build would compile NumPy from + # source: that timed out under QEMU and failed outright on macOS. + - { python-version: "3.13", abi: "t" } env: PYTHON_VERSION: ${{ matrix.python-version }} @@ -71,10 +72,9 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v3.3.0 env: + # No CIBW_ENABLE needed: cibuildwheel 3.x gates only cp313t behind + # cpython-freethreading, and cp314t is selectable by default. CIBW_BUILD: "cp3${{ env.PYTHON_SUBVERSION }}${{ matrix.abi }}-*" - # Opt in to cp3XXt build identifiers; harmless for the GIL builds since - # CIBW_BUILD already selects a single ABI. - CIBW_ENABLE: "cpython-freethreading" CIBW_SKIP: "*-win32 *-manylinux_i686 *-musllinux*" CIBW_ARCHS_MACOS: "x86_64 universal2" CIBW_ARCHS_LINUX: "auto aarch64" @@ -83,9 +83,9 @@ jobs: CIBW_ENVIRONMENT_WINDOWS: "CMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake" # Bundle openblas.dll into the wheel (auditwheel/delocate handle this on Linux/macOS) CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel} --add-path C:/vcpkg/installed/x64-windows/bin" - # On free-threaded builds also assert the import did not re-enable the GIL, - # i.e. the extension really declares Py_MOD_GIL_NOT_USED. - CIBW_TEST_COMMAND: python -c "import sys, sysconfig, sparsediffpy; ft = sysconfig.get_config_var('Py_GIL_DISABLED') == 1; assert not ft or not sys._is_gil_enabled(), 'GIL re-enabled on import'" + # On free-threaded builds also assert the import did not re-enable the + # GIL, i.e. the extension really declares Py_MOD_GIL_NOT_USED. + CIBW_TEST_COMMAND: python -c "import sys, sysconfig, sparsediffpy; ft = bool(sysconfig.get_config_var('Py_GIL_DISABLED')); assert not ft or not sys._is_gil_enabled(), 'GIL re-enabled on import'; print('freethreaded_build=', ft)" - name: Check wheels shell: bash run: | diff --git a/pyproject.toml b/pyproject.toml index 677defb..3d1ccdb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,11 @@ file = "README.md" content-type = "text/markdown" [tool.scikit-build] -cmake.version = ">=3.15" +# CMake 3.30 is the first whose FindPython3 understands the free-threaded ("t") +# ABI, and scikit-build-core only sends the Python3_FIND_ABI hint when this +# specifier admits 3.30+. 3.30.3 adds the Python3_DEFINITIONS a Windows +# free-threaded build needs to link python3XXt.lib. +cmake.version = ">=3.30.3" cmake.build-type = "Release" wheel.packages = ["sparsediffpy"] # Persistent build dir: avoids scikit-build-core's TemporaryDirectory, whose cleanup From 75d51a5524a574a2b36c27623854a1a32dc4e8c3 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 6 Sep 2026 22:19:56 -0400 Subject: [PATCH 5/6] Trim the comments added in the previous two commits The rationale belongs in these commit messages, not inline. Cuts the Py_MOD_GIL_NOT_USED block from 23 lines to 5, the cmake.version note from 4 lines to 1, and the free-threading README section back to roughly its original length, keeping the two facts a reader needs: SP_TRACK_MEMORY must stay off, and expr::refcount being a plain int makes a shared capsule fatal rather than merely unsupported. No functional change. Rebuilt on cpython-3.13.5+freethreaded: the import still leaves the GIL disabled and 8 threads on independent problems stay bit-identical to a single-threaded reference. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build-and-publish.yml | 14 ++++--------- README.md | 27 +++++++------------------ pyproject.toml | 5 +---- sparsediffpy/_bindings/bindings.c | 25 +++++------------------ 4 files changed, 17 insertions(+), 54 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 7f458ca..1ab1195 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -35,14 +35,10 @@ jobs: # "" builds the default interpreter, "t" the free-threaded one. abi: [ "", "t" ] exclude: - # Free threading does not exist before 3.13: cp311t and cp312t are - # not build identifiers at all, so cibuildwheel would select nothing - # and the job would fail on an empty wheelhouse. + # Free threading starts at 3.13, so cp311t / cp312t do not exist. - { python-version: "3.11", abi: "t" } - { python-version: "3.12", abi: "t" } - # NumPy ships no cp313t wheels from 2.5 onwards (2.4.6 is the last - # release with one), so a 3.13t build would compile NumPy from - # source: that timed out under QEMU and failed outright on macOS. + # No cp313t NumPy wheels from 2.5 on; it would build from source. - { python-version: "3.13", abi: "t" } env: @@ -72,8 +68,7 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v3.3.0 env: - # No CIBW_ENABLE needed: cibuildwheel 3.x gates only cp313t behind - # cpython-freethreading, and cp314t is selectable by default. + # cp314t needs no CIBW_ENABLE; cibuildwheel gates only cp313t. CIBW_BUILD: "cp3${{ env.PYTHON_SUBVERSION }}${{ matrix.abi }}-*" CIBW_SKIP: "*-win32 *-manylinux_i686 *-musllinux*" CIBW_ARCHS_MACOS: "x86_64 universal2" @@ -83,8 +78,7 @@ jobs: CIBW_ENVIRONMENT_WINDOWS: "CMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake" # Bundle openblas.dll into the wheel (auditwheel/delocate handle this on Linux/macOS) CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel} --add-path C:/vcpkg/installed/x64-windows/bin" - # On free-threaded builds also assert the import did not re-enable the - # GIL, i.e. the extension really declares Py_MOD_GIL_NOT_USED. + # On free-threaded builds, assert the import did not re-enable the GIL. CIBW_TEST_COMMAND: python -c "import sys, sysconfig, sparsediffpy; ft = bool(sysconfig.get_config_var('Py_GIL_DISABLED')); assert not ft or not sys._is_gil_enabled(), 'GIL re-enabled on import'; print('freethreaded_build=', ft)" - name: Check wheels shell: bash diff --git a/README.md b/README.md index 415c7fd..3a02ca6 100644 --- a/README.md +++ b/README.md @@ -17,34 +17,21 @@ from sparsediffpy import _sparsediffengine ## Free-threaded Python Wheels are published for free-threaded CPython 3.14 (`cp314t`) as well as the -default builds. There is no `cp313t` wheel: NumPy ships none from 2.5 onwards -(2.4.6 is the last release with one), so a 3.13t build would have to compile -NumPy from source. +default builds. There is no `cp313t` wheel because NumPy ships none from 2.5 +onwards, so that build would have to compile NumPy from source. The extension declares that it does not need the GIL, so importing it leaves free threading enabled. Distinct problems can be built and evaluated concurrently from different threads: the engine keeps no mutable global state, every problem and expression owns its own buffers, and all inputs and outputs -are copied at the boundary. - -This depends on the engine being built without `SP_TRACK_MEMORY`, which is the -default. That option makes every allocation update the `g_allocated_bytes` and -`g_peak_bytes` process globals non-atomically, so two threads that merely -allocate race on them. It is a development switch; do not turn it on for a -wheel build. +are copied at the boundary. This assumes a default build, since +`SP_TRACK_MEMORY` adds global allocation counters that are not thread-safe. A single problem or expression capsule is **not** thread-safe. Do not call into the same problem from two threads at once, and do not share expression capsules -between problems that are evaluated concurrently. - -Without the GIL, breaking that rule is worse than it used to be. `expr::refcount` -is a plain `int` that `expr_retain()` and `free_expr()` update non-atomically, -and capsule destructors run on whichever thread drops the last Python reference. -So sharing one capsule across threads corrupts the heap instead of merely -interleaving calls: an eight-thread loop building and dropping `exp()` nodes over -one shared variable node crashes on every run here, while the same loop is -harmless under the GIL. Making that reference count atomic upstream would turn -this back into an ordinary "unsupported usage" rather than a crash. +between problems that are evaluated concurrently. Note that `expr::refcount` is +a plain `int`, so breaking this rule corrupts the heap rather than merely +interleaving calls, as it would under the GIL. ## License diff --git a/pyproject.toml b/pyproject.toml index 3d1ccdb..379eca4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,10 +15,7 @@ file = "README.md" content-type = "text/markdown" [tool.scikit-build] -# CMake 3.30 is the first whose FindPython3 understands the free-threaded ("t") -# ABI, and scikit-build-core only sends the Python3_FIND_ABI hint when this -# specifier admits 3.30+. 3.30.3 adds the Python3_DEFINITIONS a Windows -# free-threaded build needs to link python3XXt.lib. +# 3.30.3+ so FindPython3 detects the free-threaded ABI on all platforms. cmake.version = ">=3.30.3" cmake.build-type = "Release" wheel.packages = ["sparsediffpy"] diff --git a/sparsediffpy/_bindings/bindings.c b/sparsediffpy/_bindings/bindings.c index 71de8c5..4011d1c 100644 --- a/sparsediffpy/_bindings/bindings.c +++ b/sparsediffpy/_bindings/bindings.c @@ -190,26 +190,11 @@ PyMODINIT_FUNC PyInit__sparsediffengine(void) PyObject *module = PyModule_Create(&sparsediffpy_module); if (!module) return NULL; #ifdef Py_GIL_DISABLED - /* Free-threaded CPython (3.13t+): declare that this module does not need - the GIL. - - This holds only for an engine built WITHOUT SP_TRACK_MEMORY, which is - the default. With that option on, every sp_malloc / sp_free updates the - g_allocated_bytes and g_peak_bytes process globals non-atomically, so - two threads that merely allocate would race. Never enable it for a - wheel build. - - Otherwise the engine keeps no mutable global state: every problem and - expression owns its buffers, and the wrappers copy inputs and outputs, - so distinct problems may be built and evaluated concurrently. - - A single problem or expression capsule remains NOT thread-safe, and - without the GIL the consequence is harsher than it used to be. - expr::refcount is a plain int that expr_retain() and free_expr() update - non-atomically, and capsule destructors run on whichever thread drops - the last Python reference. Sharing one capsule across threads therefore - corrupts the heap rather than merely interleaving calls. Making that - count atomic upstream would remove the sharp edge. */ + /* Requires an engine built without SP_TRACK_MEMORY (the default), which + leaves no mutable global state, so distinct problems are independent. + A single problem or expression capsule is still single-threaded: + expr::refcount is a plain int, so sharing one across threads corrupts + the heap. See README.md. */ PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); #endif return module; From 1798e115e858b369a720f2aba267b693ab51ed80 Mon Sep 17 00:00:00 2001 From: William Zijie Zhang Date: Sun, 6 Sep 2026 22:42:05 -0400 Subject: [PATCH 6/6] Collapse the wheel matrix back to one python-version axis Drop the separate abi axis and its three excludes; "3.14t" goes straight into python-version instead. The existing subversion step already handles it, since cut -c 3- of "3.14t" is "14t" and CIBW_BUILD becomes cp314t-*. Artifact names stay unique without the suffix, and the job count is unchanged at 15. Two excludes only existed because free threading starts at 3.13, so cp311t and cp312t were dead combinations the axis generated; listing the ABIs that do exist avoids generating them in the first place. The host interpreter for the 3.14t job is now free-threaded as well. That only affects the twine step, since pypa/cibuildwheel runs its own setup-python pinned to "3.11 - 3.13" with update-environment false, and actions/python-versions publishes free-threaded 3.14 builds. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build-and-publish.yml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index 1ab1195..24edc51 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -31,15 +31,9 @@ jobs: fail-fast: false matrix: os: [ ubuntu-22.04, macos-14, windows-2022 ] - python-version: [ "3.11", "3.12", "3.13", "3.14" ] - # "" builds the default interpreter, "t" the free-threaded one. - abi: [ "", "t" ] - exclude: - # Free threading starts at 3.13, so cp311t / cp312t do not exist. - - { python-version: "3.11", abi: "t" } - - { python-version: "3.12", abi: "t" } - # No cp313t NumPy wheels from 2.5 on; it would build from source. - - { python-version: "3.13", abi: "t" } + # 3.14t is the free-threaded build. No 3.13t: NumPy ships no cp313t + # wheels from 2.5 on, so it would compile NumPy from source. + python-version: [ "3.11", "3.12", "3.13", "3.14", "3.14t" ] env: PYTHON_VERSION: ${{ matrix.python-version }} @@ -69,7 +63,7 @@ jobs: uses: pypa/cibuildwheel@v3.3.0 env: # cp314t needs no CIBW_ENABLE; cibuildwheel gates only cp313t. - CIBW_BUILD: "cp3${{ env.PYTHON_SUBVERSION }}${{ matrix.abi }}-*" + CIBW_BUILD: "cp3${{ env.PYTHON_SUBVERSION }}-*" CIBW_SKIP: "*-win32 *-manylinux_i686 *-musllinux*" CIBW_ARCHS_MACOS: "x86_64 universal2" CIBW_ARCHS_LINUX: "auto aarch64" @@ -88,7 +82,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.os }}-${{ matrix.python-version }}${{ matrix.abi }} + name: wheels-${{ matrix.os }}-${{ matrix.python-version }} path: ./wheelhouse/*.whl build_sdist: