From be6ca7fc7fbc858161d21c7e35685a54224c7d16 Mon Sep 17 00:00:00 2001 From: Joel Dice Date: Thu, 20 Aug 2026 14:50:34 -0700 Subject: [PATCH] add support for the `wasm32-wasip3` target and ABI This paves the way for supporting cooperative mulithreading. Currently, we embed two copies of each library: one for p2 and one for p3, selecting which to use based on whether the target world uses any async features. In order to avoid even longer build times, I've published a pre-built CPython WASI tarball which we use by default instead of building from source. This also adds support for testing WIT maps in the test generator, which I meant to include in an earlier commit. Co-authored-by: Alex Crichton --- .github/workflows/release.yaml | 48 --- .github/workflows/test.yaml | 48 --- CONTRIBUTING.md | 8 +- Cargo.lock | 68 +++-- Cargo.toml | 3 +- build.rs | 274 ++++++++++-------- .../componentize_py_async_support/__init__.py | 41 +-- runtime/Cargo.toml | 2 +- runtime/src/lib.rs | 88 +++--- src/lib.rs | 108 +++++-- src/link.rs | 1 + src/prelink.rs | 109 ++++++- src/stubwasi.rs | 1 + src/test/echoes.rs | 1 + src/test/python_source/app.py | 2 +- src/test/tests.rs | 17 +- src/test/wit/tests.wit | 2 +- test-generator/src/lib.rs | 46 ++- wit/init.wit | 2 +- 19 files changed, 511 insertions(+), 358 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2d9b0bd5..1a87dd13 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -19,48 +19,6 @@ permissions: pull-requests: write jobs: - linux: - name: Populate cache - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: "recursive" - - - uses: actions/setup-python@v5 - with: - python-version: "3.14.0" - - - name: Install latest Rust stable toolchain - uses: dtolnay/rust-toolchain@stable - with: - targets: wasm32-wasip2 - components: clippy, rustfmt - - - uses: Swatinem/rust-cache@v2 - with: - shared-key: "rust-cache-${{ hashFiles('./Cargo.lock') }}" - cache-on-failure: false - - - name: Install WASI-SDK - shell: bash - run: | - cd /tmp - curl -LO https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz - tar xf wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz - mv wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux /opt/wasi-sdk - - - name: Build - shell: bash - run: cargo build --release - - - name: Upload CPython builddir - uses: actions/upload-artifact@v4 - with: - name: cpython-wasi - path: cpython/builddir/wasi - release: name: Build and release needs: linux @@ -174,12 +132,6 @@ jobs: shell: bash run: echo "WASI_SDK_PATH=$(cygpath -m /tmp/wasi-sdk-${WASI_SDK_VERSION}.0-${{ matrix.config.wasiSDK }})" >> ${GITHUB_ENV} - - name: Restore CPython - uses: actions/download-artifact@v5 - with: - name: cpython-wasi - path: cpython/builddir/wasi - - name: Build wheels uses: PyO3/maturin-action@v1 with: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4be7148e..6e787f3c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -17,48 +17,6 @@ env: WASI_SDK_VERSION: 34 jobs: - linux: - name: Populate cache - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: "recursive" - - - uses: actions/setup-python@v5 - with: - python-version: "3.14.0" - - - name: Install latest Rust stable toolchain - uses: dtolnay/rust-toolchain@stable - with: - targets: wasm32-wasip2 - components: clippy, rustfmt - - - uses: Swatinem/rust-cache@v2 - with: - shared-key: "rust-cache-${{ hashFiles('./Cargo.lock') }}" - cache-on-failure: false - - - name: Install WASI-SDK - shell: bash - run: | - cd /tmp - curl -LO https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz - tar xf wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz - mv wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux /opt/wasi-sdk - - - name: Build - shell: bash - run: cargo build --release - - - name: Upload CPython builddir - uses: actions/upload-artifact@v4 - with: - name: cpython-wasi - path: cpython/builddir/wasi - test: name: Test needs: linux @@ -104,12 +62,6 @@ jobs: shell: bash run: echo "WASI_SDK_PATH=$(cygpath -m /tmp/wasi-sdk-${WASI_SDK_VERSION}.0-${{ matrix.config.wasiSDK }})" >> ${GITHUB_ENV} - - name: Restore CPython - uses: actions/download-artifact@v5 - with: - name: cpython-wasi - path: cpython/builddir/wasi - - name: Lint shell: bash run: bash .github/workflows/lint.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0e2975f3..172aad92 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,16 +34,16 @@ First, make sure you've got all the submodules cloned. git submodule update --init --recursive ``` -Next, install WASI-SDK 33 to `/opt/wasi-sdk` (alternatively, you can specify a +Next, install WASI-SDK 34 to `/opt/wasi-sdk` (alternatively, you can specify a different location and reference it later using the `WASI_SDK_PATH` environment variable). Replace `arm64-linux` with `x86_64-linux`, `arm64-macos`, `x86_64-macos`, `arm64-windows`, or `x86_64-windows` below depending on your architecure and OS, if necessary. ```shell -curl -LO https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-33/wasi-sdk-33.0-arm64-linux.tar.gz -tar xf wasi-sdk-33.0-arm64-linux.tar.gz -sudo mv wasi-sdk-33.0-arm64-linux /opt/wasi-sdk +curl -LO https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-34/wasi-sdk-34.0-arm64-linux.tar.gz +tar xf wasi-sdk-34.0-arm64-linux.tar.gz +sudo mv wasi-sdk-34.0-arm64-linux /opt/wasi-sdk export WASI_SDK_PATH=/opt/wasi-sdk ``` diff --git a/Cargo.lock b/Cargo.lock index c8842db8..1b61fdf2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -161,9 +161,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "auditable-serde" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7bf8143dfc3c0258df908843e169b5cc5fcf76c7718bd66135ef4a9cd558c5" +checksum = "d026218ae25ba5c72834245412dd1338f6d270d2c5109ee03a4badec288d4056" dependencies = [ "semver", "serde", @@ -443,15 +443,14 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "component-init-transform" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "905e0d94b9dce0ad90bf44b5dedaab3f1a8275964d5012fec8e91656ad91d81e" +source = "git+https://github.com/dicej/component-init?rev=d8bad1bf#d8bad1bfd93390f502a0e2315491ce0b65961499" dependencies = [ "anyhow", "async-trait", "futures", - "wasm-encoder 0.244.0", - "wasm-metadata 0.244.0", - "wasmparser 0.244.0", + "wasm-encoder 0.257.1", + "wasm-metadata 0.257.1", + "wasmparser 0.257.1", ] [[package]] @@ -1132,7 +1131,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "foldhash 0.1.5", - "serde", ] [[package]] @@ -2426,9 +2424,9 @@ dependencies = [ [[package]] name = "spdx" -version = "0.10.9" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3e17e880bafaeb362a7b751ec46bdc5b61445a188f80e0606e68167cd540fa3" +checksum = "081670c233dfbed55690cc0cd38424e0e24ac1b2673d0b408b3f7b684738dfa9" dependencies = [ "smallvec", ] @@ -3024,6 +3022,16 @@ dependencies = [ "wasmparser 0.254.0", ] +[[package]] +name = "wasm-encoder" +version = "0.257.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8ad9f0a39050867bda22e6486c316e2e52d20a42154d5bc433934bf738d085" +dependencies = [ + "leb128fmt", + "wasmparser 0.257.1", +] + [[package]] name = "wasm-encoder" version = "0.258.0" @@ -3041,14 +3049,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "auditable-serde", - "flate2", "indexmap", - "serde", - "serde_derive", - "serde_json", - "spdx", - "url", "wasm-encoder 0.244.0", "wasmparser 0.244.0", ] @@ -3065,6 +3066,25 @@ dependencies = [ "wasmparser 0.254.0", ] +[[package]] +name = "wasm-metadata" +version = "0.257.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57460ad9bce753e8a80d47a445cb87c8724ea57f463d9c906a947c41032ce1ac" +dependencies = [ + "anyhow", + "auditable-serde", + "flate2", + "indexmap", + "serde", + "serde_derive", + "serde_json", + "spdx", + "url", + "wasm-encoder 0.257.1", + "wasmparser 0.257.1", +] + [[package]] name = "wasm-metadata" version = "0.258.0" @@ -3087,7 +3107,6 @@ dependencies = [ "hashbrown 0.15.5", "indexmap", "semver", - "serde", ] [[package]] @@ -3103,6 +3122,19 @@ dependencies = [ "serde", ] +[[package]] +name = "wasmparser" +version = "0.257.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92fc335fb6d48f46bda1d8b26b69e28320c15ac3272208333833d6e217e2b4a" +dependencies = [ + "bitflags", + "hashbrown 0.17.1", + "indexmap", + "semver", + "serde", +] + [[package]] name = "wasmparser" version = "0.258.0" diff --git a/Cargo.toml b/Cargo.toml index 7804fb50..f382b326 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,8 @@ pyo3 = { version = "0.28.3", features = [ wasmtime = "48.0.0" wasmtime-wasi = { version = "48.0.0", features = [ "p3" ] } once_cell = "1.20.2" -component-init-transform = "0.2.0" +# TODO: publish a release to crates.io and use it: +component-init-transform = { git = "https://github.com/dicej/component-init", rev = "d8bad1bf" } async-trait = "0.1.83" futures = "0.3.31" tokio = { version = "1.41.0", features = [ diff --git a/build.rs b/build.rs index 5c2e65c6..158cbe3e 100644 --- a/build.rs +++ b/build.rs @@ -38,8 +38,13 @@ const CLANG_EXECUTABLE: &str = "clang"; // TODO: switch to upstream release per // https://github.com/bytecodealliance/componentize-py/issues/215 -const CPYTHON_TARBALL_URL: &str = "https://github.com/dicej/cpython/tarball/v3.14.0-wasi-sdk-30"; -const CPYTHON_TARBALL_BASE_DIR: &str = "dicej-cpython-0e13686"; +const CPYTHON_SOURCE_TARBALL_URL: &str = + "https://github.com/dicej/cpython/tarball/v3.14.0-wasi-sdk-34"; +const CPYTHON_SOURCE_TARBALL_BASE_DIR: &str = "dicej-cpython-1cf1514"; + +const CPYTHON_BINARY_TARBALL_URL: &str = "https://github.com/dicej/cpython/releases/download/v3.14.0-wasi-sdk-34/cpython-wasi-v3.14.0-wasi-sdk-34.tar.zst"; + +static TARGETS: &[&str] = &["wasip2", "wasip3"]; fn main() -> Result<()> { println!("cargo:rerun-if-changed=build.rs"); @@ -61,13 +66,10 @@ fn main() -> Result<()> { } fn stubs_for_clippy(out_dir: &Path) -> Result<()> { - println!( - "cargo:warning=using stubbed runtime, core library, and adapter for static analysis purposes..." - ); + println!("cargo:warning=using dummy embedded files for static analysis purposes..."); - let files = [ - "libcomponentize_py_runtime_sync.so.zst", - "libcomponentize_py_runtime_async.so.zst", + let libraries = [ + "libcomponentize_py_runtime.so.zst", "libpython3.14.so.zst", "libc.so.zst", "libwasi-emulated-mman.so.zst", @@ -76,10 +78,24 @@ fn stubs_for_clippy(out_dir: &Path) -> Result<()> { "libwasi-emulated-signal.so.zst", "libc++.so.zst", "libc++abi.so.zst", - "wasi_snapshot_preview1.reactor.wasm.zst", ]; - for file in files { + for file in libraries { + for target in TARGETS { + let target_dir = out_dir.join(target); + fs::create_dir_all(&target_dir)?; + + let path = target_dir.join(file); + + if !path.exists() { + Encoder::new(File::create(path)?, ZSTD_COMPRESSION_LEVEL)?.do_finish()?; + } + } + } + + let adapters = ["wasi_snapshot_preview1.reactor.wasm.zst"]; + + for file in adapters { let path = out_dir.join(file); if !path.exists() { @@ -87,20 +103,16 @@ fn stubs_for_clippy(out_dir: &Path) -> Result<()> { } } - let path = out_dir.join("python-lib.tar.zst"); - - if !path.exists() { - Builder::new(Encoder::new(File::create(path)?, ZSTD_COMPRESSION_LEVEL)?) - .into_inner()? - .do_finish()?; - } + let tarballs = ["python-lib.tar.zst", "bundled.tar.zst"]; - let path = out_dir.join("bundled.tar.zst"); + for file in tarballs { + let path = out_dir.join(file); - if !path.exists() { - Builder::new(Encoder::new(File::create(path)?, ZSTD_COMPRESSION_LEVEL)?) - .into_inner()? - .do_finish()?; + if !path.exists() { + Builder::new(Encoder::new(File::create(path)?, ZSTD_COMPRESSION_LEVEL)?) + .into_inner()? + .do_finish()?; + } } Ok(()) @@ -154,7 +166,7 @@ fn find_wasi_sdk(out_dir: &Path) -> Result { url, wasi_sdk.display() ); - fetch_extract(&url, out_dir)?; + fetch_extract_gzip(&url, out_dir)?; Ok(wasi_sdk) } @@ -164,60 +176,56 @@ fn package_all_the_things(out_dir: &Path) -> Result<()> { let wasi_sdk = find_wasi_sdk(out_dir)?; - maybe_make_cpython(&repo_dir, &wasi_sdk)?; + for target in TARGETS { + maybe_make_cpython(target, &repo_dir, &wasi_sdk)?; - let cpython_wasi_dir = repo_dir.join("cpython/builddir/wasi"); + let cpython_wasi_dir = repo_dir.join("cpython/builddir").join(target); - make_pyo3_config(&repo_dir)?; + let library_dir = out_dir.join(target); + fs::create_dir_all(&library_dir)?; - make_runtime( - &repo_dir, - out_dir, - &wasi_sdk, - &cpython_wasi_dir, - false, - "libcomponentize_py_runtime_sync.so", - )?; - make_runtime( - &repo_dir, - out_dir, - &wasi_sdk, - &cpython_wasi_dir, - true, - "libcomponentize_py_runtime_async.so", - )?; - - let libraries = [ - "libc.so", - "libwasi-emulated-mman.so", - "libwasi-emulated-process-clocks.so", - "libwasi-emulated-getpid.so", - "libwasi-emulated-signal.so", - ]; + make_pyo3_config(target, &repo_dir)?; - for library in libraries { - compress( - &wasi_sdk.join("share/wasi-sysroot/lib/wasm32-wasip2"), - library, - out_dir, - true, + make_runtime( + target, + &repo_dir, + &library_dir, + &wasi_sdk, + &cpython_wasi_dir, )?; - } - let libraries = ["libc++.so", "libc++abi.so"]; + let libraries = [ + "libc.so", + "libwasi-emulated-mman.so", + "libwasi-emulated-process-clocks.so", + "libwasi-emulated-getpid.so", + "libwasi-emulated-signal.so", + ]; + + for library in libraries { + compress( + &wasi_sdk.join(format!("share/wasi-sysroot/lib/wasm32-{target}")), + library, + &library_dir, + true, + )?; + } - for library in libraries { - compress( - &wasi_sdk.join("share/wasi-sysroot/lib/wasm32-wasip2/noeh"), - library, - out_dir, - true, - )?; - } + let libraries = ["libc++.so", "libc++abi.so"]; - compress(&cpython_wasi_dir, "libpython3.14.so", out_dir, true)?; + for library in libraries { + compress( + &wasi_sdk.join(format!("share/wasi-sysroot/lib/wasm32-{target}/noeh")), + library, + &library_dir, + true, + )?; + } - let path = repo_dir.join("cpython/builddir/wasi/install/lib/python3.14"); + compress(&cpython_wasi_dir, "libpython3.14.so", &library_dir, true)?; + } + + let path = repo_dir.join("cpython/builddir/wasip2/install/lib/python3.14"); if path.exists() { let mut builder = Builder::new(Encoder::new( @@ -305,18 +313,29 @@ fn add(builder: &mut Builder, root: &Path, path: &Path) -> Result<() Ok(()) } -fn maybe_make_cpython(repo_dir: &Path, wasi_sdk: &Path) -> Result<()> { +fn maybe_make_cpython(target: &str, repo_dir: &Path, wasi_sdk: &Path) -> Result<()> { let cpython_dir = repo_dir.join("cpython"); if !cpython_dir.exists() { - let url = &env::var("CPYTHON_TARBALL_URL").unwrap_or_else(|_| CPYTHON_TARBALL_URL.into()); - let base_dir = &env::var_os("CPYTHON_TARBALL_BASE_DIR") - .unwrap_or_else(|| CPYTHON_TARBALL_BASE_DIR.into()); - println!("cargo:warning=downloading CPython source code from {url}"); - fetch_extract(url, repo_dir)?; - fs::rename(repo_dir.join(base_dir), &cpython_dir)?; + if env::var_os("CPYTHON_BUILD_FROM_SOURCE").is_some() { + let url = &env::var("CPYTHON_SOURCE_TARBALL_URL") + .unwrap_or_else(|_| CPYTHON_SOURCE_TARBALL_URL.into()); + let base_dir = &env::var_os("CPYTHON_SOURCE_TARBALL_BASE_DIR") + .unwrap_or_else(|| CPYTHON_SOURCE_TARBALL_BASE_DIR.into()); + println!("cargo:warning=downloading CPython source code from {url}"); + fetch_extract_gzip(url, repo_dir)?; + fs::rename(repo_dir.join(base_dir), &cpython_dir)?; + } else { + let url = &env::var("CPYTHON_BINARY_TARBALL_URL") + .unwrap_or_else(|_| CPYTHON_BINARY_TARBALL_URL.into()); + println!("cargo:warning=downloading CPython build from {url}"); + let dir = cpython_dir.join("builddir"); + fs::create_dir_all(&dir)?; + fetch_extract_zstd(url, &dir)?; + return Ok(()); + } } - let cpython_wasi_dir = cpython_dir.join("builddir/wasi"); + let cpython_wasi_dir = cpython_dir.join("builddir").join(target); if !cpython_wasi_dir.join("libpython3.14.so").exists() { fs::create_dir_all(&cpython_wasi_dir)?; if !cpython_wasi_dir.join("libpython3.14.a").exists() { @@ -338,9 +357,9 @@ fn maybe_make_cpython(repo_dir: &Path, wasi_sdk: &Path) -> Result<()> { } let lib_install_dir = cpython_wasi_dir.join("deps"); - build_zlib(wasi_sdk, &lib_install_dir)?; + build_zlib(target, wasi_sdk, &lib_install_dir)?; - build_sqlite(wasi_sdk, &lib_install_dir)?; + build_sqlite(target, wasi_sdk, &lib_install_dir)?; let config_guess = run(Command::new("../../config.guess").current_dir(&cpython_wasi_dir))?; @@ -358,12 +377,12 @@ fn maybe_make_cpython(repo_dir: &Path, wasi_sdk: &Path) -> Result<()> { ) .env( "CFLAGS", - format!("--target=wasm32-wasip2 -fPIC -I{dir}/deps/include"), + format!("--target=wasm32-{target} -fPIC -I{dir}/deps/include"), ) .env("WASI_SDK_PATH", wasi_sdk) .env( "LDFLAGS", - format!("--target=wasm32-wasip2 -L{dir}/deps/lib"), + format!("--target=wasm32-{target} -L{dir}/deps/lib"), ) .current_dir(&cpython_wasi_dir) .args([ @@ -388,7 +407,7 @@ fn maybe_make_cpython(repo_dir: &Path, wasi_sdk: &Path) -> Result<()> { // Link libpython3.14.so - now includes libsqlite3.a run(Command::new(wasi_sdk.join("bin/clang")) - .arg("--target=wasm32-wasip2") + .arg(format!("--target=wasm32-{target}")) .arg("-shared") .arg("-o") .arg(cpython_wasi_dir.join("libpython3.14.so")) @@ -475,9 +494,9 @@ fn run(command: &mut Command) -> Result> { } } -fn make_pyo3_config(repo_dir: &Path) -> Result<()> { +fn make_pyo3_config(target: &str, repo_dir: &Path) -> Result<()> { let out_dir = env::var("OUT_DIR")?; - let mut cpython_wasi_dir = repo_dir.join("cpython/builddir/wasi"); + let mut cpython_wasi_dir = repo_dir.join("cpython/builddir").join(target); let mut cygpath = Command::new("cygpath"); cygpath.arg("-w").arg(&cpython_wasi_dir); if let Ok(output) = cygpath.output() { @@ -497,7 +516,10 @@ fn make_pyo3_config(repo_dir: &Path) -> Result<()> { "lib_dir={}", cpython_wasi_dir.to_str().unwrap() )?; - fs::write(Path::new(&out_dir).join("pyo3-config.txt"), pyo3_config)?; + fs::write( + Path::new(&out_dir).join(target).join("pyo3-config.txt"), + pyo3_config, + )?; println!("cargo:rerun-if-changed=pyo3-config.txt"); @@ -505,37 +527,26 @@ fn make_pyo3_config(repo_dir: &Path) -> Result<()> { } fn make_runtime( + target: &str, repo_dir: &Path, out_dir: &Path, wasi_sdk: &Path, cpython_wasi_dir: &Path, - async_: bool, - name: &str, ) -> Result<()> { let mut cmd = Command::new("cargo"); cmd.current_dir(repo_dir.join("runtime")) .arg("build") - .arg("--target=wasm32-wasip2"); + .arg(format!("--target=wasm32-{target}")); if !DEBUG_RUNTIME { cmd.arg("--release"); } - if async_ { + if target == "wasip3" { cmd.arg("--features=async"); } - for (key, _) in env::vars_os() { - if key - .to_str() - .map(|key| key.starts_with("RUST") || key.starts_with("CARGO")) - .unwrap_or(false) - { - cmd.env_remove(&key); - } - } - - let target = if async_ { "async" } else { "sync" }; + cmd.env_remove("CARGO_ENCODED_RUSTFLAGS"); let clang = wasi_sdk.join(format!("bin/{CLANG_EXECUTABLE}")); cmd.env( @@ -543,6 +554,7 @@ fn make_runtime( format!( "--cfg pyo3_disable_reference_pool \ -Clink-args=-Wl,--skip-wit-component \ + -Clink-args=-Wl,--export-if-defined=__wasm_library_tls_info \ -Clink-args=-shared \ -Clink-args=-L{} \ -Clink-args=-lpython3.14 \ @@ -550,7 +562,10 @@ fn make_runtime( cpython_wasi_dir.to_str().unwrap() ), ) - .env("CARGO_TARGET_WASM32_WASIP2_LINKER", clang) + .env( + format!("CARGO_TARGET_WASM32_{}_LINKER", target.to_uppercase()), + clang, + ) .env("CARGO_TARGET_DIR", out_dir.join(target)) .env("PYO3_CONFIG_FILE", out_dir.join("pyo3-config.txt")); @@ -562,10 +577,11 @@ fn make_runtime( let build = if DEBUG_RUNTIME { "debug" } else { "release" }; let path = out_dir.join(target).join(format!( - "wasm32-wasip2/{build}/componentize_py_runtime.wasm" + "wasm32-{target}/{build}/componentize_py_runtime.wasm" )); if path.exists() { + let name = "libcomponentize_py_runtime.so"; fs::copy(&path, out_dir.join(name))?; compress(out_dir, name, out_dir, false)?; } else { @@ -575,7 +591,7 @@ fn make_runtime( Ok(()) } -fn fetch_extract(url: &str, out_dir: &Path) -> Result<()> { +fn fetch_extract_gzip(url: &str, out_dir: &Path) -> Result<()> { let response = reqwest::blocking::get(url)?; let decoder = flate2::read::GzDecoder::new(response); let mut archive = tar::Archive::new(decoder); @@ -583,7 +599,15 @@ fn fetch_extract(url: &str, out_dir: &Path) -> Result<()> { Ok(()) } -fn add_compile_envs(wasi_sdk: &Path, command: &mut Command) { +fn fetch_extract_zstd(url: &str, out_dir: &Path) -> Result<()> { + let response = reqwest::blocking::get(url)?; + let decoder = zstd::Decoder::new(response)?; + let mut archive = tar::Archive::new(decoder); + archive.unpack(out_dir)?; + Ok(()) +} + +fn add_compile_envs(target: &str, wasi_sdk: &Path, command: &mut Command) { let sysroot = wasi_sdk.join("share/wasi-sysroot"); let sysroot = sysroot.to_string_lossy(); command @@ -592,17 +616,17 @@ fn add_compile_envs(wasi_sdk: &Path, command: &mut Command) { .env("RANLIB", wasi_sdk.join("bin/ranlib")) .env( "CFLAGS", - format!("--target=wasm32-wasip2 --sysroot={sysroot} -I{sysroot}/include/wasm32-wasip2 -D_WASI_EMULATED_SIGNAL -fPIC"), + format!("--target=wasm32-{target} --sysroot={sysroot} -I{sysroot}/include/wasm32-{target} -D_WASI_EMULATED_SIGNAL -fPIC"), ) .env( "LDFLAGS", - format!("--target=wasm32-wasip2 --sysroot={sysroot} -L{sysroot}/lib -lwasi-emulated-signal") + format!("--target=wasm32-{target} --sysroot={sysroot} -L{sysroot}/lib -lwasi-emulated-signal") ); } -fn build_zlib(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { +fn build_zlib(target: &str, wasi_sdk: &Path, install_dir: &Path) -> Result<()> { let out_dir = PathBuf::from(env::var("OUT_DIR")?); - fetch_extract( + fetch_extract_gzip( "https://github.com/madler/zlib/releases/download/v1.3.1/zlib-1.3.1.tar.gz", &out_dir, )?; @@ -613,7 +637,7 @@ fn build_zlib(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { .ok_or_else(|| anyhow!("non-UTF8 path: {}", install_dir.display()))?; let mut configure = Command::new("./configure"); - add_compile_envs(wasi_sdk, &mut configure); + add_compile_envs(target, wasi_sdk, &mut configure); configure .current_dir(&src_dir) .arg("--static") @@ -631,7 +655,7 @@ fn build_zlib(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { .ok_or_else(|| anyhow!("non-UTF8 path: {}", clang_dir.display()))?; let mut make = Command::new("make"); - add_compile_envs(wasi_sdk, &mut make); + add_compile_envs(target, wasi_sdk, &mut make); make.current_dir(src_dir) .arg(format!("AR={ar_dir}")) .arg("ARFLAGS=rcs") @@ -650,7 +674,7 @@ fn build_zlib(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { /// - SQLITE_OMIT_WAL: WAL requires mmap which isn't available in WASI preview1 /// - SQLITE_OMIT_LOAD_EXTENSION: No dlopen in WASI /// - SQLITE_THREADSAFE=0: Single-threaded for WASM -fn build_sqlite(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { +fn build_sqlite(target: &str, wasi_sdk: &Path, install_dir: &Path) -> Result<()> { let out_dir = PathBuf::from(env::var("OUT_DIR")?); // Check if already built @@ -661,12 +685,18 @@ fn build_sqlite(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { println!("cargo:warning=Building SQLite {SQLITE_VERSION} for WASI..."); - // Download SQLite amalgamation - let url = format!("https://sqlite.org/{SQLITE_YEAR}/sqlite-autoconf-{SQLITE_VERSION}.tar.gz"); - fetch_extract(&url, &out_dir)?; - let src_dir = out_dir.join(format!("sqlite-autoconf-{SQLITE_VERSION}")); + if !src_dir.exists() { + // Download SQLite amalgamation + let url = + format!("https://sqlite.org/{SQLITE_YEAR}/sqlite-autoconf-{SQLITE_VERSION}.tar.gz"); + fetch_extract_gzip(&url, &out_dir)?; + } + + let build_dir = src_dir.join(target); + fs::create_dir_all(&build_dir)?; + // Ensure install directories exist fs::create_dir_all(install_dir.join("lib"))?; fs::create_dir_all(install_dir.join("include"))?; @@ -687,9 +717,9 @@ fn build_sqlite(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { // Note: Don't set SQLITE_THREADSAFE here - let --disable-threadsafe handle it // to avoid macro redefinition warnings let sqlite_cflags = format!( - "--target=wasm32-wasip2 \ + "--target=wasm32-{target} \ --sysroot={sysroot_str} \ - -I{sysroot_str}/include/wasm32-wasip2 \ + -I{sysroot_str}/include/wasm32-{target} \ -D_WASI_EMULATED_SIGNAL \ -D_WASI_EMULATED_PROCESS_CLOCKS \ -fPIC \ @@ -702,18 +732,18 @@ fn build_sqlite(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { ); // Configure SQLite - let mut configure = Command::new("./configure"); + let mut configure = Command::new("../configure"); configure - .current_dir(&src_dir) + .current_dir(&build_dir) .env("AR", wasi_sdk.join("bin/ar")) .env("CC", wasi_sdk.join("bin/clang")) .env("RANLIB", wasi_sdk.join("bin/ranlib")) .env("CFLAGS", &sqlite_cflags) .env( "LDFLAGS", - format!("--target=wasm32-wasip2 --sysroot={sysroot_str} -L{sysroot_str}/lib",), + format!("--target=wasm32-{target} --sysroot={sysroot_str} -L{sysroot_str}/lib",), ) - .arg("--host=wasm32-wasip2") + .arg(format!("--host=wasm32-{target}")) .arg(format!("--prefix={install_dir_str}")) .arg("--disable-shared") .arg("--enable-static") @@ -725,7 +755,7 @@ fn build_sqlite(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { // Build only the static library (not the shell, which fails to link on WASI) let mut make = Command::new("make"); - make.current_dir(&src_dir) + make.current_dir(&build_dir) .env("AR", wasi_sdk.join("bin/ar")) .env("CC", wasi_sdk.join("bin/clang")) .env("RANLIB", wasi_sdk.join("bin/ranlib")) @@ -738,7 +768,7 @@ fn build_sqlite(wasi_sdk: &Path, install_dir: &Path) -> Result<()> { // Manual install since we didn't build everything // Copy the library fs::copy( - src_dir.join("libsqlite3.a"), + build_dir.join("libsqlite3.a"), install_dir.join("lib/libsqlite3.a"), )?; // Copy the headers diff --git a/bundled/componentize_py_async_support/__init__.py b/bundled/componentize_py_async_support/__init__.py index d4025667..6095b6f9 100644 --- a/bundled/componentize_py_async_support/__init__.py +++ b/bundled/componentize_py_async_support/__init__.py @@ -10,6 +10,7 @@ import componentize_py_runtime import subprocess +from threading import local from os import PathLike from socket import AddressFamily, AddressInfo, SocketKind, socket from concurrent.futures import Executor @@ -67,7 +68,7 @@ class _Status: async def _noop() -> None: pass - + class _Loop(asyncio.AbstractEventLoop): def __init__(self) -> None: self.running: bool = False @@ -80,13 +81,13 @@ def poll(self, future_state: _FutureState) -> None: for handle in handles: if not handle._cancelled: handle._run() - + if self.exception is not None: raise self.exception if len(handles) == 0 and len(future_state.handles) == 0: return - + def get_debug(self) -> bool: return False @@ -187,7 +188,7 @@ async def getnameinfo(self, sockaddr: tuple[str, int] | tuple[str, int, int, int], flags: int = 0) -> tuple[str, str]: raise NotImplementedError - + async def create_connection( self, protocol_factory: Callable[[], _ProtocolT], @@ -207,7 +208,7 @@ async def create_connection( interleave: int | None = None, ) -> tuple[Transport, _ProtocolT]: raise NotImplementedError - + async def create_server( self, protocol_factory: Callable[[], BaseProtocol], @@ -420,12 +421,9 @@ def default_exception_handler(self, context: dict[str, Any]) -> None: def set_debug(self, enabled: bool) -> None: raise NotImplementedError - + _future_state: ContextVar[_FutureState] = ContextVar("_future_state") _loop = _Loop() -asyncio.set_event_loop(_loop) -_loop.running = True -asyncio.events._set_running_loop(_loop) def _set_future_state(future_state: _FutureState) -> None: global _future_state @@ -448,12 +446,17 @@ async def _return_result(export_index: int, borrows: int, coroutine: Any) -> Non assert _future_state.get().pending_count > 0 _future_state.get().pending_count -= 1 +_thread_local = local() + def first_poll(export_index: int, borrows: int, coroutine: Any) -> int: """Internal function called by generated code for exported functions. This is not meant to be called by application code. """ + asyncio.set_event_loop(_loop) + _loop.running = True + asyncio.events._set_running_loop(_loop) context = Context() future_state = _FutureState(None, {}, [], 1) context.run(_set_future_state, future_state) @@ -468,12 +471,12 @@ def _poll(future_state: _FutureState) -> int: if future_state.pending_count == 0: if future_state.waitable_set is not None: componentize_py_runtime.waitable_set_drop(future_state.waitable_set) - + return _CallbackCode.EXIT else: waitable_set = future_state.waitable_set assert waitable_set is not None - componentize_py_runtime.context_set(future_state) + _thread_local.state = future_state return _CallbackCode.WAIT | (waitable_set << 4) def callback(event0: int, event1: int, event2: int) -> int: @@ -482,9 +485,9 @@ def callback(event0: int, event1: int, event2: int) -> int: This is not meant to be called by application code. """ - future_state = componentize_py_runtime.context_get() - componentize_py_runtime.context_set(None) - + future_state = _thread_local.state + _thread_local.state = None + match event0: case _Event.NONE: pass @@ -509,7 +512,7 @@ def callback(event0: int, event1: int, event2: int) -> int: raise NotImplementedError return _poll(future_state) - + async def await_result[T](result: Result[T, tuple[int, int]]) -> T: """Internal function called by generated code for imported functions. @@ -518,7 +521,7 @@ async def await_result[T](result: Result[T, tuple[int, int]]) -> T: """ global _loop global _future_state - + if isinstance(result, Ok): return result.value else: @@ -526,11 +529,11 @@ async def await_result[T](result: Result[T, tuple[int, int]]) -> T: waitable, promise = result.value future = _loop.create_future() future_state.futures[waitable] = future - + if future_state.waitable_set is None: future_state.waitable_set = componentize_py_runtime.waitable_set_new() componentize_py_runtime.waitable_join(waitable, future_state.waitable_set) - + return cast(T, componentize_py_runtime.promise_get_result(await future, promise)) async def _wrap_spawned(coroutine: Any) -> None: @@ -566,5 +569,5 @@ def spawn(coroutine: Any) -> None: global _future_state _future_state.get().pending_count += 1 - + asyncio.create_task(_wrap_spawned(coroutine)) diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index e2d8576c..7f8469f4 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -13,7 +13,7 @@ anyhow = "1.0.91" once_cell = "1.20.2" pyo3 = { version="0.28.3", features = ["abi3-py312", "num-bigint"] } num-bigint = "0.4.6" -wit-bindgen = { version = "0.61.0", default-features = false, features = ["macros", "realloc"] } +wit-bindgen = { version = "0.61.0", default-features = false, features = ["macros", "realloc", "async"] } wit-dylib-ffi = { git = "https://github.com/bytecodealliance/wasm-tools", tag = "v1.258.0", features = ["async-raw"] } [features] diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 4eb7c4e3..9d43e27c 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -22,7 +22,7 @@ use { pyo3::{ Bound, IntoPyObject, Py, PyAny, PyErr, PyResult, Python, exceptions::PyAssertionError, - intern, + ffi, intern, types::{ PyAnyMethods, PyBool, PyBytes, PyBytesMethods, PyDict, PyDictMethods, PyList, PyListMethods, PyMapping, PyMappingMethods, PyModule, PyModuleMethods, PyString, @@ -31,9 +31,10 @@ use { }, std::{ alloc::{self, Layout}, + cell::Cell, iter, marker::PhantomData, - mem, slice, str, + mem, ptr, slice, str, sync::Once, }, wit_dylib_ffi::{ @@ -53,6 +54,8 @@ mod bindings { export!(MyExports); } +const CALLBACK_CODE_EXIT: u32 = 0; + static WIT: OnceCell = OnceCell::new(); static STUB_WASI: OnceCell = OnceCell::new(); static EXPORTS: OnceCell> = OnceCell::new(); @@ -1167,7 +1170,7 @@ fn do_init(app_name: String, symbols: Symbols, stub_wasi: bool) -> Result<(), St struct MyExports; impl Guest for MyExports { - fn init(app_name: String, symbols: Symbols, stub_wasi: bool) -> Result<(), String> { + async fn init(app_name: String, symbols: Symbols, stub_wasi: bool) -> Result<(), String> { let result = do_init(app_name, symbols, stub_wasi); // This tells the WASI Preview 1 component adapter to reset its state. @@ -1201,7 +1204,9 @@ struct MyInterpreter; impl MyInterpreter { fn export_call_(func: ExportFunction, cx: &mut MyCall<'_>, async_: bool) -> u32 { - Python::attach(|py| { + let gil_state = unsafe { ffi::PyGILState_Ensure() }; + + let result = Python::attach(|py| { if !*STUB_WASI.get().unwrap() { static ONCE: Once = Once::new(); ONCE.call_once(|| { @@ -1299,12 +1304,24 @@ impl MyInterpreter { release_borrows(py, mem::take(&mut cx.borrows)); - 0 + CALLBACK_CODE_EXIT } - }) + }); + + if result == CALLBACK_CODE_EXIT { + unsafe { ffi::PyGILState_Release(gil_state) }; + } else { + THREAD_STATE.with(|v| { + v.set((Some(gil_state), unsafe { ffi::PyEval_SaveThread() })); + }); + } + + result } } +std::thread_local!(static THREAD_STATE: Cell<(Option, *mut ffi::PyThreadState)> = const { Cell::new((None, ptr::null_mut())) } ); + impl Interpreter for MyInterpreter { type CallCx<'a> = MyCall<'a>; @@ -1335,7 +1352,13 @@ impl Interpreter for MyInterpreter { fn export_async_callback(event0: u32, event1: u32, event2: u32) -> u32 { #[cfg(feature = "async")] { - Python::attach(|py| { + let gil_state = THREAD_STATE.with(|v| { + let (gil_state, thread_state) = v.get(); + unsafe { ffi::PyEval_RestoreThread(thread_state) }; + gil_state + }); + + let result = Python::attach(|py| { async_::CALLBACK .get() .unwrap() @@ -1343,7 +1366,17 @@ impl Interpreter for MyInterpreter { .unwrap() .extract(py) .unwrap() - }) + }); + + if result == CALLBACK_CODE_EXIT { + unsafe { ffi::PyGILState_Release(gil_state.unwrap()) }; + } else { + THREAD_STATE.with(|v| { + v.set((gil_state, unsafe { ffi::PyEval_SaveThread() })); + }); + } + + result } #[cfg(not(feature = "async"))] { @@ -1427,7 +1460,9 @@ impl Drop for MyCall<'_> { fn drop(&mut self) { for &(ptr, layout) in &self.deferred_deallocations { unsafe { - alloc::dealloc(ptr, layout); + if layout.size() > 0 { + alloc::dealloc(ptr, layout); + } } } } @@ -2081,7 +2116,9 @@ impl Call for MyCall<'_> { .to_owned() .into_any() .unbind(); - alloc::dealloc(src, Layout::from_size_align(len, 1).unwrap()); + if len > 0 { + alloc::dealloc(src, Layout::from_size_align(len, 1).unwrap()); + } value })); true @@ -2209,34 +2246,3 @@ fn exported_resource_to_canon( } wit_dylib_ffi::export!(MyInterpreter); - -// As of this writing, recent Rust `nightly` builds include a version of the -// `libc` crate that expects `wasi-libc` to define the following global -// variables, but `wasi-libc` defines them as preprocessor constants which -// aren't visible at link time, so we need to define them somewhere. Ideally, -// we should fix this upstream, but for now we work around it: - -#[unsafe(no_mangle)] -static _CLOCK_PROCESS_CPUTIME_ID: u8 = 2; -#[unsafe(no_mangle)] -static _CLOCK_THREAD_CPUTIME_ID: u8 = 3; - -// Traditionally, `wit-bindgen` would provide a `cabi_realloc` implementation, -// but recent versions use a weak symbol trick to avoid conflicts when more than -// one `wit-bindgen` version is used, and that trick does not currently play -// nice with how we build this library. So for now, we just define it ourselves -// here: -/// # Safety -/// TODO -#[unsafe(export_name = "cabi_realloc")] -pub unsafe extern "C" fn cabi_realloc( - old_ptr: *mut u8, - old_len: usize, - align: usize, - new_size: usize, -) -> *mut u8 { - assert!(old_ptr.is_null()); - assert!(old_len == 0); - - unsafe { alloc::alloc(Layout::from_size_align(new_size, align).unwrap()) } -} diff --git a/src/lib.rs b/src/lib.rs index 6ccb33df..53e06e0b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -50,6 +50,7 @@ mod test; mod util; const DEBUG_PYTHON_BINDINGS: bool = false; +const DEBUG_PRE_INIT: bool = false; /// The default name of the Python module containing code generated from the /// specified WIT world. This may be overriden programatically or via the CLI @@ -76,7 +77,14 @@ impl WasiView for Ctx { } } +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Target { + Wasip2, + Wasip3, +} + pub struct Library { + target: Target, name: String, module: Vec, dl_openable: bool, @@ -555,13 +563,49 @@ impl ComponentGenerator<'_> { // Extract relevant metadata from the `Resolve` into a `Summary` instance, // which we'll use to generate Wasm- and Python-level bindings. + // Determine whether to use the WASIp2 or WASIp3 target based on whether + // the world uses any async features. + // + // TODO: Allow the user to explicitly specify the target instead of + // inferring it here, e.g. if they want to use WASIp3 despite the target + // world not using any async features. + // + // TODO #2: Creating a temporary `Summary` is a heavyweight way to + // determine whether the world uses async features, especially since we + // will create the real one down below, but otherwise we'd have an + // ordering problem because we need to know the target before we call + // `wit_dylib::create_with_metadata` (which produces the metadata we'll + // need to create the real `Summary`), and we can't call that until we + // know the target. We should be able to extract the code that + // `Summary::try_new` uses to check for async features and use it + // without the rest of the things `Summary::try_new` does. + let need_async = Summary::try_new( + &resolve, + &iter::once((world, Naming::from_full(self.full_names))).collect(), + &import_interface_names, + &export_interface_names, + &HashMap::new(), + &HashMap::new(), + &HashMap::new(), + )? + .need_async(); + + let target = if need_async { + Target::Wasip3 + } else { + Target::Wasip2 + }; + let (mut bindings, metadata) = wit_dylib::create_with_metadata( &resolve, world, Some(&mut DylibOpts { - stack_pointer: wit_dylib::StackPointer::Global, interpreter: Some("libcomponentize_py_runtime.so".into()), async_: Default::default(), + stack_pointer: match target { + Target::Wasip2 => wit_dylib::StackPointer::Global, + Target::Wasip3 => wit_dylib::StackPointer::TaskContext, + }, }), ); @@ -627,29 +671,18 @@ impl ComponentGenerator<'_> { &stream_and_future_indexes, )?; - let need_async = summary.need_async(); - - // Now that we know whether to use the sync or async version of - // `libcomponentize_py_runtime.so`, update `libraries` accordingly. + // Now that we know which target to use, update `libraries` accordingly. // - // Note that we have two separate versions because older runtimes don't - // understand the new async ABI, so we only use the async version if it's - // actually needed. + // Note that we must only use libraries which match the target because + // the targets have mutually incompatible ABIs, plus users may wish to + // target runtimes which do not support WASIp3 or async features. let mut libraries = libraries .into_iter() - .filter_map(|library| match (need_async, library.name.as_str()) { - (true, "libcomponentize_py_runtime_sync.so") - | (false, "libcomponentize_py_runtime_async.so") => None, - (true, "libcomponentize_py_runtime_async.so") - | (false, "libcomponentize_py_runtime_sync.so") => Some(Library { - name: "libcomponentize_py_runtime.so".into(), - ..library - }), - _ => Some(library), - }) + .filter(|library| library.target == target) .collect::>(); libraries.push(Library { + target, name: "libcomponentize_py_bindings.so".into(), module: bindings, dl_openable: false, @@ -673,10 +706,14 @@ impl ComponentGenerator<'_> { let stderr = MemoryOutputPipe::new(10000); let mut wasi = WasiCtxBuilder::new(); - wasi.stdin(MemoryInputPipe::new(Bytes::new())) - .stdout(stdout.clone()) - .stderr(stderr.clone()) - .env("PYTHONUNBUFFERED", "1") + if DEBUG_PRE_INIT { + wasi.inherit_stdio(); + } else { + wasi.stdin(MemoryInputPipe::new(Bytes::new())) + .stdout(stdout.clone()) + .stderr(stderr.clone()); + } + wasi.env("PYTHONUNBUFFERED", "1") .env("PYTHONHOME", "/python") .preopened_dir( embedded_python_standard_lib.path(), @@ -849,10 +886,16 @@ impl ComponentGenerator<'_> { let instance = pre.instance_pre.instantiate_async(&mut store).await?; let guest = pre.indices.interface0.load(&mut store, &instance)?; - guest - .call_init(&mut store, &app_name, &symbols, stub_wasi) - .await? - .map_err(|e| anyhow!("{e}"))?; + store + .run_concurrent(async |store| { + guest + .call_init(store, app_name, symbols, stub_wasi) + .await? + .map_err(|e| anyhow!("{e}"))?; + + anyhow::Ok(()) + }) + .await??; Ok(Box::new(MyInvoker { store, instance }) as Box) } @@ -1038,6 +1081,7 @@ fn add_wasi_and_stubs( linker: &mut Linker, ) -> Result<()> { wasmtime_wasi::p2::add_to_linker_async(linker)?; + wasmtime_wasi::p3::add_to_linker(linker)?; enum Stub<'a> { Function(&'a String, &'a FunctionKind), @@ -1093,10 +1137,10 @@ fn add_wasi_and_stubs( for (interface_name, stubs) in stubs { if let Some(interface_name) = interface_name { // Note that we do _not_ stub interfaces which appear to be part of - // WASIp2 since those should be provided by the - // `wasmtime_wasi::add_to_linker_async` call above, and adding stubs - // to those same interfaces would just cause trouble. - if !is_wasip2_cli(&interface_name) + // WASIp2/p3 since those should be provided by the + // `add_to_linker{_async}` calls above, and adding stubs to those + // same interfaces would just cause trouble. + if !is_wasi_cli(&interface_name) && let Ok(mut instance) = linker.instance(&interface_name) { for stub in stubs { @@ -1178,12 +1222,12 @@ fn add_wasi_and_stubs( Ok(()) } -fn is_wasip2_cli(interface_name: &str) -> bool { +fn is_wasi_cli(interface_name: &str) -> bool { (interface_name.starts_with("wasi:cli/") || interface_name.starts_with("wasi:clocks/") || interface_name.starts_with("wasi:random/") || interface_name.starts_with("wasi:io/") || interface_name.starts_with("wasi:filesystem/") || interface_name.starts_with("wasi:sockets/")) - && interface_name.contains("@0.2.") + && (interface_name.contains("@0.2.") || interface_name.contains("@0.3.")) } diff --git a/src/link.rs b/src/link.rs index 86793864..3425e8ad 100644 --- a/src/link.rs +++ b/src/link.rs @@ -9,6 +9,7 @@ pub fn link_libraries(libraries: &[Library]) -> Result> { linker.use_built_in_libdl(true).encoder().validate(true); for Library { + target: _, name, module, dl_openable, diff --git a/src/prelink.rs b/src/prelink.rs index 3aac57af..a89d6e1b 100644 --- a/src/prelink.rs +++ b/src/prelink.rs @@ -14,7 +14,7 @@ use tar::Archive; use tempfile::TempDir; use zstd::Decoder; -use crate::{ComponentizePyConfig, ConfigContext, Library, RawComponentizePyConfig}; +use crate::{ComponentizePyConfig, ConfigContext, Library, RawComponentizePyConfig, Target}; static NATIVE_EXTENSION_SUFFIXES: &[&str] = &[".cpython-314-wasm32-wasi.so", ".abi3.so"]; @@ -60,82 +60,164 @@ pub fn embedded_helper_utils() -> Result { pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> Result> { let mut libraries = vec![ Library { - name: "libcomponentize_py_runtime_sync.so".into(), + target: Target::Wasip2, + name: "libcomponentize_py_runtime.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/libcomponentize_py_runtime_sync.so.zst" + "/wasip2/libcomponentize_py_runtime.so.zst" ))))?, dl_openable: false, }, Library { - name: "libcomponentize_py_runtime_async.so".into(), + target: Target::Wasip3, + name: "libcomponentize_py_runtime.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/libcomponentize_py_runtime_async.so.zst" + "/wasip3/libcomponentize_py_runtime.so.zst" ))))?, dl_openable: false, }, Library { + target: Target::Wasip2, name: "libpython3.14.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/libpython3.14.so.zst" + "/wasip2/libpython3.14.so.zst" ))))?, dl_openable: false, }, Library { + target: Target::Wasip3, + name: "libpython3.14.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip3/libpython3.14.so.zst" + ))))?, + dl_openable: false, + }, + Library { + target: Target::Wasip2, + name: "libc.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip2/libc.so.zst" + ))))?, + dl_openable: false, + }, + Library { + target: Target::Wasip3, name: "libc.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/libc.so.zst" + "/wasip3/libc.so.zst" ))))?, dl_openable: false, }, Library { + target: Target::Wasip2, name: "libwasi-emulated-mman.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/libwasi-emulated-mman.so.zst" + "/wasip2/libwasi-emulated-mman.so.zst" + ))))?, + dl_openable: false, + }, + Library { + target: Target::Wasip3, + name: "libwasi-emulated-mman.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip3/libwasi-emulated-mman.so.zst" + ))))?, + dl_openable: false, + }, + Library { + target: Target::Wasip2, + name: "libwasi-emulated-process-clocks.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip2/libwasi-emulated-process-clocks.so.zst" ))))?, dl_openable: false, }, Library { + target: Target::Wasip3, name: "libwasi-emulated-process-clocks.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/libwasi-emulated-process-clocks.so.zst" + "/wasip3/libwasi-emulated-process-clocks.so.zst" ))))?, dl_openable: false, }, Library { + target: Target::Wasip2, name: "libwasi-emulated-getpid.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/libwasi-emulated-getpid.so.zst" + "/wasip2/libwasi-emulated-getpid.so.zst" ))))?, dl_openable: false, }, Library { + target: Target::Wasip3, + name: "libwasi-emulated-getpid.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip3/libwasi-emulated-getpid.so.zst" + ))))?, + dl_openable: false, + }, + Library { + target: Target::Wasip2, + name: "libwasi-emulated-signal.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip2/libwasi-emulated-signal.so.zst" + ))))?, + dl_openable: false, + }, + Library { + target: Target::Wasip3, name: "libwasi-emulated-signal.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/libwasi-emulated-signal.so.zst" + "/wasip3/libwasi-emulated-signal.so.zst" ))))?, dl_openable: false, }, Library { + target: Target::Wasip2, name: "libc++.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/libc++.so.zst" + "/wasip2/libc++.so.zst" + ))))?, + dl_openable: false, + }, + Library { + target: Target::Wasip3, + name: "libc++.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip3/libc++.so.zst" + ))))?, + dl_openable: false, + }, + Library { + target: Target::Wasip2, + name: "libc++abi.so".into(), + module: zstd::decode_all(Cursor::new(include_bytes!(concat!( + env!("OUT_DIR"), + "/wasip2/libc++abi.so.zst" ))))?, dl_openable: false, }, Library { + target: Target::Wasip3, name: "libc++abi.so".into(), module: zstd::decode_all(Cursor::new(include_bytes!(concat!( env!("OUT_DIR"), - "/libc++abi.so.zst" + "/wasip3/libc++abi.so.zst" ))))?, dl_openable: false, }, @@ -152,6 +234,7 @@ pub fn bundle_libraries(library_path: Vec<(&str, Vec)>) -> Result) -> Result) -> Result<()> { wasmtime_wasi::p2::add_to_linker_async(&mut *linker)?; + wasmtime_wasi::p3::add_to_linker(&mut *linker)?; componentize_py::test::echoes::add_to_linker::<_, HasSelf<_>>(linker, |ctx| ctx)?; Ok(()) } diff --git a/src/test/python_source/app.py b/src/test/python_source/app.py index 7eded4e6..e3edfbfd 100644 --- a/src/test/python_source/app.py +++ b/src/test/python_source/app.py @@ -229,7 +229,7 @@ def test_resource_alias(self, things: List[imports.resource_alias1.Thing]) -> Li def add(self, a: imports.resource_floats.Float, b: imports.resource_floats.Float) -> imports.resource_floats.Float: return imports.resource_floats.Float(a.get() + b.get() + 5) - def read_file(self, path: str) -> bytes: + async def read_file(self, path: str) -> bytes: try: with open(file=path, mode="rb") as f: return f.read() diff --git a/src/test/tests.rs b/src/test/tests.rs index 8dd48bd3..06e2c685 100644 --- a/src/test/tests.rs +++ b/src/test/tests.rs @@ -119,6 +119,7 @@ impl super::Host for Host { fn add_to_linker(linker: &mut Linker) -> Result<()> { wasmtime_wasi::p2::add_to_linker_async(linker)?; + wasmtime_wasi::p3::add_to_linker(linker)?; Tests::add_to_linker::<_, HasSelf<_>>(linker, |ctx| ctx)?; foo_sdk::FooWorldUnion::add_to_linker::<_, HasSelf<_>>(linker, |ctx| ctx)?; Ok(()) @@ -977,14 +978,18 @@ fn filesystem() -> Result<()> { TESTER.test_with_wasi::(wasi, |world, store, runtime| { runtime.block_on(async { - let value = world - .call_read_file(store, filename) - .await? - .map_err(|s| anyhow!("{s}"))?; + store + .run_concurrent(async |store| { + let value = world + .call_read_file(store, filename.to_string()) + .await? + .map_err(|s| anyhow!("{s}"))?; - assert_eq!(&value, message); + assert_eq!(&value, message); - Ok(()) + Ok(()) + }) + .await? }) }) } diff --git a/src/test/wit/tests.wit b/src/test/wit/tests.wit index c645cfac..6e610cee 100644 --- a/src/test/wit/tests.wit +++ b/src/test/wit/tests.wit @@ -213,7 +213,7 @@ world tests { export add: func(a: borrow, b: borrow) -> own; - export read-file: func(path: string) -> result, string>; + export read-file: async func(path: string) -> result, string>; export test-refcounts: func(); diff --git a/test-generator/src/lib.rs b/test-generator/src/lib.rs index b58a191c..9d7889db 100644 --- a/test-generator/src/lib.rs +++ b/test-generator/src/lib.rs @@ -1,7 +1,7 @@ #![deny(warnings)] use { - anyhow::{Result, anyhow}, + anyhow::{anyhow, Result}, proptest::{ strategy::{Just, Strategy, ValueTree}, test_runner::{Config, TestRng, TestRunner}, @@ -60,10 +60,28 @@ enum Type { }, Tuple(Vec), List(Box), + Map(Box, Box), +} + +fn any_key_type() -> impl Strategy { + (0..11).prop_flat_map(move |index| match index { + 0 => Just(Type::Bool).boxed(), + 1 => Just(Type::U8).boxed(), + 2 => Just(Type::S8).boxed(), + 3 => Just(Type::U16).boxed(), + 4 => Just(Type::S16).boxed(), + 5 => Just(Type::U32).boxed(), + 6 => Just(Type::S32).boxed(), + 7 => Just(Type::U64).boxed(), + 8 => Just(Type::S64).boxed(), + 9 => Just(Type::Char).boxed(), + 10 => Just(Type::String).boxed(), + _ => unreachable!(), + }) } fn any_type(max_size: usize, next_id: Rc>) -> impl Strategy { - (0..21).prop_flat_map(move |index| match index { + (0..22).prop_flat_map(move |index| match index { 0 => Just(Type::Bool).boxed(), 1 => Just(Type::U8).boxed(), 2 => Just(Type::S8).boxed(), @@ -142,6 +160,9 @@ fn any_type(max_size: usize, next_id: Rc>) -> impl Strategy any_type(max_size, next_id.clone()) .prop_map(|ty| Type::List(Box::new(ty))) .boxed(), + 21 => (any_key_type(), any_type(max_size, next_id.clone())) + .prop_map(|(k, v)| Type::Map(Box::new(k), Box::new(v))) + .boxed(), _ => unreachable!(), }) } @@ -271,6 +292,9 @@ fn wit_type_name(wit: &mut String, ty: &Type) -> String { Type::List(ty) => { format!("list<{}>", wit_type_name(wit, ty)) } + Type::Map(k, v) => { + format!("map<{}, {}>", wit_type_name(wit, k), wit_type_name(wit, v)) + } } } @@ -326,6 +350,9 @@ fn rust_type_name(ty: &Type) -> String { Type::List(ty) => { format!("Vec<{}>", rust_type_name(ty)) } + Type::Map(k, v) => { + format!("HashMap<{}, {}>", rust_type_name(k), rust_type_name(v)) + } } } @@ -418,6 +445,12 @@ fn equality(a: &str, b: &str, ty: &Type) -> String { "{a}.len() == {b}.len() && {a}.iter().zip({b}.iter()).all(|(a, b)| {})", equality("a", "b", ty) ), + Type::Map(k, v) => format!( + "{a}.len() == {b}.len() && {a}.iter().collect::>().into_iter()\ + .zip({b}.iter().collect::>().into_iter()).all(|((ak, av), (bk, bv))| {} && {})", + equality("ak", "bk", k), + equality("av", "bv", v) + ), } } @@ -547,6 +580,13 @@ fn strategy(ty: &Type, max_list_size: usize) -> String { strategy(ty, max_list_size / 2) ) } + Type::Map(k, v) => { + format!( + "proptest::collection::hash_map({}, {}, 0..{max_list_size}.max(1))", + strategy(k, max_list_size / 2), + strategy(v, max_list_size / 2) + ) + } } } @@ -814,6 +854,7 @@ use {{ component::{{Instance, InstancePre, Linker, TypedFunc, HasSelf}}, Store, }}, + std::collections::{{HashMap, BTreeMap}}, }}; wasmtime::component::bindgen!({{ @@ -838,6 +879,7 @@ impl super::Host for Host {{ fn add_to_linker(linker: &mut Linker) -> Result<()> {{ wasmtime_wasi::p2::add_to_linker_async(&mut *linker)?; + wasmtime_wasi::p3::add_to_linker(&mut *linker)?; {PREFIX}::add_to_linker::<_, HasSelf<_>>(linker, |ctx| ctx)?; Ok(()) }} diff --git a/wit/init.wit b/wit/init.wit index 2f5de1ae..919ba99e 100644 --- a/wit/init.wit +++ b/wit/init.wit @@ -98,6 +98,6 @@ world init { results: list, } - init: func(app-name: string, symbols: symbols, stub-wasi: bool) -> result<_, string>; + init: async func(app-name: string, symbols: symbols, stub-wasi: bool) -> result<_, string>; } }