diff --git a/.cargo/config.toml b/.cargo/config.toml index 7b7b2eb4e1f..e0b12c98bca 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -4,6 +4,7 @@ rustflags = ["--cfg", "tokio_unstable"] [alias] llm = "run --package xtask-llm-benchmark --bin llm_benchmark --" ci = "run -p ci --" +spacetime = "ci other-workflows run-spacetime" regen = "run -p regen --" smoketest = "ci smoketests --" smoketests = "smoketest" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b56dd41aa0c..0dac88c7699 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -314,7 +314,6 @@ jobs: SMOKETEST_SUITE: standalone steps: &smoketest-build-steps - *find-git-ref - - *checkout-sources - uses: dsherret/rust-toolchain-file@v1 @@ -400,7 +399,6 @@ jobs: PARTITION_COUNT: 1 steps: &smoketest-partition-steps - *find-git-ref - - *checkout-sources - uses: dsherret/rust-toolchain-file@v1 @@ -472,13 +470,15 @@ jobs: - name: Install cargo-nextest uses: taiki-e/install-action@nextest - - name: Download build artifacts + - &download-build-artifacts + name: Download build artifacts uses: actions/download-artifact@v4 with: name: build-artifacts-${{ env.ARTIFACT_SUFFIX }} path: build-artifacts - - name: Extract build artifacts + - &extract-build-artifacts + name: Extract build artifacts shell: bash run: | tar -xzf build-artifacts/build-support.tar.gz @@ -570,7 +570,7 @@ jobs: run: echo "Merge queue commit has the same tree as the PR head; smoketest already ran for the PR." test: - needs: [merge_queue_noop, lints] + needs: [merge_queue_noop, lints, upload-build-artifacts-linux] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Test Suite runs-on: spacetimedb-new-runner-2 @@ -578,6 +578,8 @@ jobs: env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full + ARTIFACT_SUFFIX: linux + SPACETIME_BIN: ${{ github.workspace }}/target/release/spacetimedb-cli steps: - name: Find Git ref env: @@ -627,6 +629,9 @@ jobs: - *restore-openssl - *configure-cached-openssl + - *download-build-artifacts + - *extract-build-artifacts + - uses: actions/setup-dotnet@v3 with: global-json-file: global.json @@ -678,7 +683,7 @@ jobs: - name: Run tests run: | source ~/emsdk/emsdk_env.sh - cargo ci test + cargo ci test --no-build - name: Upload timing reports if: always() @@ -820,47 +825,29 @@ jobs: path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ retention-days: 14 - wasm_bindings: - needs: [merge_queue_noop] + module_latest_deps: + needs: [merge_queue_noop, upload-build-artifacts-linux] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: Build and test wasm bindings runs-on: spacetimedb-new-runner-2 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full + ARTIFACT_SUFFIX: linux + SPACETIME_BIN: ${{ github.workspace }}/target/release/spacetimedb-cli steps: - - uses: actions/checkout@v3 + - *find-git-ref + - *checkout-sources - uses: dsherret/rust-toolchain-file@v1 - - name: Set default rust toolchain - run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) + - *set-default-rust-toolchain - run: echo ::add-matcher::.github/workflows/rust_matcher.json - - *set-native-cache-keys - - *restore-jemalloc - - *configure-cached-jemalloc - - *restore-zstd - - *configure-cached-zstd - - *restore-openssl - - *configure-cached-openssl - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 22 - - - uses: ./.github/actions/setup-pnpm + - *download-build-artifacts + - *extract-build-artifacts - - name: Run bindgen tests - run: cargo ci wasm-bindings - - - name: Upload timing reports - if: always() - uses: actions/upload-artifact@v4 - with: - name: cargo-timings-wasm-bindings - path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ - retention-days: 14 + - name: Build module with latest compatible dependencies + run: cargo ci module-latest-deps --no-build publish_checks: needs: [merge_queue_noop] @@ -1103,7 +1090,7 @@ jobs: retention-days: 14 unity-testsuite: - needs: [merge_queue_noop, lints] + needs: [merge_queue_noop, lints, upload-build-artifacts-linux] # Skip if this is an external contribution. # The license secrets will be empty, so the step would fail anyway. if: ${{ needs.merge_queue_noop.outputs.skip != 'true' && (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) }} @@ -1115,11 +1102,11 @@ jobs: env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full + ARTIFACT_SUFFIX: linux UNITY_VERSION: 2022.3.32f1 steps: - - name: Checkout repository - id: checkout-stdb - uses: actions/checkout@v4 + - *find-git-ref + - *checkout-sources # Keep this before generation/hydration so ignored build outputs do not affect hashFiles. - name: Restore Unity Library @@ -1177,14 +1164,15 @@ jobs: - *restore-openssl - *configure-cached-openssl - - name: Install SpacetimeDB CLI from the local checkout + - *download-build-artifacts + - *extract-build-artifacts + + - &expose-spacetime-cli + name: Expose SpacetimeDB CLI on PATH + shell: bash run: | - export CARGO_HOME="$HOME/.cargo" - echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" - cargo install --force --path crates/cli --locked --message-format=short - cargo install --force --path crates/standalone --locked --message-format=short - # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules). - ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime + ln -sf spacetimedb-cli target/release/spacetime + echo "$GITHUB_WORKSPACE/target/release" >> "$GITHUB_PATH" - name: Generate client bindings working-directory: demo/Blackholio/server-rust @@ -1247,18 +1235,18 @@ jobs: key: ${{ steps.restore-unity-library.outputs.cache-primary-key }} godot-testsuite: - needs: [merge_queue_noop, lints] + needs: [merge_queue_noop, lints, upload-build-artifacts-linux] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} permissions: contents: read runs-on: spacetimedb-new-runner-2 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target + ARTIFACT_SUFFIX: linux UseLocalBsatnRuntime: true steps: - - name: Checkout repository - id: checkout-stdb - uses: actions/checkout@v4 + - *find-git-ref + - *checkout-sources - name: Setup dotnet uses: actions/setup-dotnet@v3 @@ -1304,14 +1292,9 @@ jobs: - *restore-openssl - *configure-cached-openssl - - name: Install SpacetimeDB CLI from the local checkout - run: | - export CARGO_HOME="$HOME/.cargo" - echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" - cargo install --force --path crates/cli --locked --message-format=short - cargo install --force --path crates/standalone --locked --message-format=short - # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules). - ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime + - *download-build-artifacts + - *extract-build-artifacts + - *expose-spacetime-cli - name: Generate client bindings working-directory: demo/Blackholio/server-rust @@ -1361,17 +1344,18 @@ jobs: run: godot --headless --scene res://tests/GodotPlayModeTests.tscn csharp-testsuite: - needs: [merge_queue_noop, lints] + needs: [merge_queue_noop, lints, upload-build-artifacts-linux] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} runs-on: spacetimedb-new-runner-2 timeout-minutes: 30 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full + ARTIFACT_SUFFIX: linux + SPACETIME_BIN: ${{ github.workspace }}/target/release/spacetimedb-cli steps: - - name: Checkout repository - id: checkout-stdb - uses: actions/checkout@v4 + - *find-git-ref + - *checkout-sources - name: Setup dotnet uses: actions/setup-dotnet@v3 @@ -1429,14 +1413,9 @@ jobs: - *restore-openssl - *configure-cached-openssl - - name: Install SpacetimeDB CLI from the local checkout - run: | - export CARGO_HOME="$HOME/.cargo" - echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" - cargo install --force --path crates/cli --locked --message-format=short - cargo install --force --path crates/standalone --features allow_loopback_http_for_tests --locked --message-format=short - # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules). - ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime + - *download-build-artifacts + - *extract-build-artifacts + - *expose-spacetime-cli - name: Check quickstart-chat bindings are up to date run: | @@ -1597,16 +1576,18 @@ jobs: retention-days: 14 typescript-test: - needs: [merge_queue_noop] + needs: [merge_queue_noop, upload-build-artifacts-linux] if: ${{ needs.merge_queue_noop.outputs.skip != 'true' }} name: TypeScript - Tests runs-on: spacetimedb-new-runner-2 env: CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full + ARTIFACT_SUFFIX: linux + SPACETIME_BIN: ${{ github.workspace }}/target/release/spacetimedb-cli steps: - - name: Checkout repository - uses: actions/checkout@v4 + - *find-git-ref + - *checkout-sources - name: Set up Node.js uses: actions/setup-node@v4 @@ -1617,31 +1598,6 @@ jobs: with: run_install: true - # - name: Extract SpacetimeDB branch name from file - # id: extract-branch - # run: | - # # Define the path to the branch file - # BRANCH_FILE=".github/spacetimedb-branch.txt" - - # # Default to master if file doesn't exist - # if [ ! -f "$BRANCH_FILE" ]; then - # echo "::notice::No SpacetimeDB branch file found, using 'master'" - # echo "branch=master" >> $GITHUB_OUTPUT - # exit 0 - # fi - - # # Read and trim whitespace from the file - # branch=$(cat "$BRANCH_FILE" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') - - # # Fallback to master if empty - # if [ -z "$branch" ]; then - # echo "::warning::SpacetimeDB branch file is empty, using 'master'" - # branch="master" - # fi - - # echo "branch=$branch" >> $GITHUB_OUTPUT - # echo "Using SpacetimeDB branch from file: $branch" - - name: Install Rust toolchain uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain @@ -1656,19 +1612,11 @@ jobs: - *restore-openssl - *configure-cached-openssl - # - name: Install SpacetimeDB CLI from the local checkout - # run: | - # export CARGO_HOME="$HOME/.cargo" - # echo "$CARGO_HOME/bin" >> "$GITHUB_PATH" - # cargo install --force --path crates/cli --locked --message-format=short - # cargo install --force --path crates/standalone --locked --message-format=short - # # Add a handy alias using the old binary name, so that we don't have to rewrite all scripts (incl. in submodules). - # ln -sf $CARGO_HOME/bin/spacetimedb-cli $CARGO_HOME/bin/spacetime - # # Clear any existing information - # spacetime server clear -y + - *download-build-artifacts + - *extract-build-artifacts - name: Run TypeScript tests - run: cargo ci typescript-test + run: cargo ci typescript-test --no-build - name: Upload timing reports if: always() diff --git a/Cargo.lock b/Cargo.lock index 22cd022fb09..d16dc421a4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -961,6 +961,16 @@ dependencies = [ "serde_json", ] +[[package]] +name = "ci-module-latest-deps" +version = "0.1.0" +dependencies = [ + "anyhow", + "ci-common", + "clap 4.5.50", + "duct", +] + [[package]] name = "ci-publish-checks" version = "0.1.0" @@ -970,6 +980,14 @@ dependencies = [ "duct", ] +[[package]] +name = "ci-run-spacetime" +version = "0.1.0" +dependencies = [ + "anyhow", + "duct", +] + [[package]] name = "ci-smoketest-checks" version = "0.1.0" @@ -1025,16 +1043,6 @@ dependencies = [ "duct", ] -[[package]] -name = "ci-wasm-bindings" -version = "0.1.0" -dependencies = [ - "anyhow", - "ci-common", - "clap 4.5.50", - "duct", -] - [[package]] name = "ci-workflow-watch" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 44858f71b9c..28529189980 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,7 +70,7 @@ members = [ "tools/ci", "tools/ci/commands/test", "tools/ci/commands/lint", - "tools/ci/commands/wasm-bindings", + "tools/ci/commands/module-latest-deps", "tools/ci/commands/smoketests", "tools/ci/commands/smoketest-checks", "tools/ci/commands/keynote-bench", @@ -78,6 +78,7 @@ members = [ "tools/ci/commands/cli-docs", "tools/ci/commands/global-json-policy", "tools/ci/commands/publish-checks", + "tools/ci/commands/run-spacetime", "tools/ci/commands/typescript-test", "tools/ci/commands/version-upgrade-check", "tools/ci/commands/docs", diff --git a/modules/benchmarks-ts/package.json b/modules/benchmarks-ts/package.json index 27c251622dc..3dd18348bb2 100644 --- a/modules/benchmarks-ts/package.json +++ b/modules/benchmarks-ts/package.json @@ -4,9 +4,9 @@ "description": "", "main": "index.js", "scripts": { - "build": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- build", - "generate-ts": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- generate --lang typescript --out-dir ts-codegen", - "publish": "cargo run -p spacetimedb-cli -- publish" + "build": "cargo spacetime build", + "generate-ts": "cargo spacetime generate --lang typescript --out-dir ts-codegen", + "publish": "cargo spacetime publish" }, "keywords": [], "author": "", diff --git a/modules/module-test-ts/package.json b/modules/module-test-ts/package.json index 4e964319a75..c5ed04bbe20 100644 --- a/modules/module-test-ts/package.json +++ b/modules/module-test-ts/package.json @@ -3,9 +3,9 @@ "license": "ISC", "type": "module", "scripts": { - "build": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- build", - "generate-ts": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- generate --lang typescript --out-dir ts-codegen", - "publish": "cargo run -p spacetimedb-cli -- publish" + "build": "cargo spacetime build", + "generate-ts": "cargo spacetime generate --lang typescript --out-dir ts-codegen", + "publish": "cargo spacetime publish" }, "dependencies": { "spacetimedb": "workspace:^" diff --git a/modules/sdk-test-case-conversion-ts/package.json b/modules/sdk-test-case-conversion-ts/package.json index 326c97ec7fc..f18f60c3e5c 100644 --- a/modules/sdk-test-case-conversion-ts/package.json +++ b/modules/sdk-test-case-conversion-ts/package.json @@ -4,9 +4,9 @@ "description": "", "main": "index.js", "scripts": { - "build": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- build", - "generate-ts": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- generate --lang typescript --out-dir ts-codegen", - "publish": "cargo run -p spacetimedb-cli -- publish" + "build": "cargo spacetime build", + "generate-ts": "cargo spacetime generate --lang typescript --out-dir ts-codegen", + "publish": "cargo spacetime publish" }, "keywords": [], "author": "", diff --git a/modules/sdk-test-connect-disconnect-ts/package.json b/modules/sdk-test-connect-disconnect-ts/package.json index cd7b60b0e47..10b1b1d76d6 100644 --- a/modules/sdk-test-connect-disconnect-ts/package.json +++ b/modules/sdk-test-connect-disconnect-ts/package.json @@ -4,9 +4,9 @@ "description": "", "main": "index.js", "scripts": { - "build": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- build", - "generate-ts": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- generate --lang typescript --out-dir ts-codegen", - "publish": "cargo run -p spacetimedb-cli -- publish" + "build": "cargo spacetime build", + "generate-ts": "cargo spacetime generate --lang typescript --out-dir ts-codegen", + "publish": "cargo spacetime publish" }, "keywords": [], "author": "", diff --git a/modules/sdk-test-procedural-view-pk-ts/package.json b/modules/sdk-test-procedural-view-pk-ts/package.json index ed3cefd9de5..c9ec6933973 100644 --- a/modules/sdk-test-procedural-view-pk-ts/package.json +++ b/modules/sdk-test-procedural-view-pk-ts/package.json @@ -4,9 +4,9 @@ "license": "ISC", "type": "module", "scripts": { - "build": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- build", - "generate-ts": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- generate --lang typescript --out-dir ts-codegen", - "publish": "cargo run -p spacetimedb-cli -- publish" + "build": "cargo spacetime build", + "generate-ts": "cargo spacetime generate --lang typescript --out-dir ts-codegen", + "publish": "cargo spacetime publish" }, "dependencies": { "spacetimedb": "workspace:^" diff --git a/modules/sdk-test-procedure-ts/package.json b/modules/sdk-test-procedure-ts/package.json index c8b4bc4ba89..97f104e94cc 100644 --- a/modules/sdk-test-procedure-ts/package.json +++ b/modules/sdk-test-procedure-ts/package.json @@ -3,9 +3,9 @@ "license": "ISC", "type": "module", "scripts": { - "build": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- build", - "generate-ts": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- generate --lang typescript --out-dir ts-codegen", - "publish": "cargo run -p spacetimedb-cli -- publish" + "build": "cargo spacetime build", + "generate-ts": "cargo spacetime generate --lang typescript --out-dir ts-codegen", + "publish": "cargo spacetime publish" }, "dependencies": { "spacetimedb": "workspace:^" diff --git a/modules/sdk-test-ts/package.json b/modules/sdk-test-ts/package.json index 4e964319a75..c5ed04bbe20 100644 --- a/modules/sdk-test-ts/package.json +++ b/modules/sdk-test-ts/package.json @@ -3,9 +3,9 @@ "license": "ISC", "type": "module", "scripts": { - "build": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- build", - "generate-ts": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- generate --lang typescript --out-dir ts-codegen", - "publish": "cargo run -p spacetimedb-cli -- publish" + "build": "cargo spacetime build", + "generate-ts": "cargo spacetime generate --lang typescript --out-dir ts-codegen", + "publish": "cargo spacetime publish" }, "dependencies": { "spacetimedb": "workspace:^" diff --git a/modules/sdk-test-view-pk-ts/package.json b/modules/sdk-test-view-pk-ts/package.json index a03509611e5..28918205874 100644 --- a/modules/sdk-test-view-pk-ts/package.json +++ b/modules/sdk-test-view-pk-ts/package.json @@ -4,9 +4,9 @@ "license": "ISC", "type": "module", "scripts": { - "build": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- build", - "generate-ts": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- generate --lang typescript --out-dir ts-codegen", - "publish": "cargo run -p spacetimedb-cli -- publish" + "build": "cargo spacetime build", + "generate-ts": "cargo spacetime generate --lang typescript --out-dir ts-codegen", + "publish": "cargo spacetime publish" }, "dependencies": { "spacetimedb": "workspace:^" diff --git a/sdks/csharp/tools~/gen-quickstart.sh b/sdks/csharp/tools~/gen-quickstart.sh index e683271bf53..834dcbcd111 100755 --- a/sdks/csharp/tools~/gen-quickstart.sh +++ b/sdks/csharp/tools~/gen-quickstart.sh @@ -7,6 +7,8 @@ SDK_PATH="$(realpath "$SDK_PATH")" STDB_PATH="$SDK_PATH/../.." DOTNET_VERSION="${1:-}" +cd "$STDB_PATH" + GLOBAL_JSON_BACKUPS=() backup_global_json_once() { @@ -66,5 +68,4 @@ if [ -n "$DOTNET_VERSION" ]; then BUILD_OPTIONS+=("--build-options=--dotnet-version $DOTNET_VERSION") fi -cargo build --manifest-path "$STDB_PATH/crates/standalone/Cargo.toml" -cargo run --manifest-path "$STDB_PATH/crates/cli/Cargo.toml" -- generate -y -l csharp -o "$STDB_PATH/templates/chat-console-cs/module_bindings" --module-path "$STDB_PATH/templates/chat-console-cs/spacetimedb" "${BUILD_OPTIONS[@]}" +cargo spacetime generate -y -l csharp -o "$STDB_PATH/templates/chat-console-cs/module_bindings" --module-path "$STDB_PATH/templates/chat-console-cs/spacetimedb" "${BUILD_OPTIONS[@]}" diff --git a/sdks/csharp/tools~/gen-regression-tests.sh b/sdks/csharp/tools~/gen-regression-tests.sh index 54852ff03fb..d7bd005cf45 100755 --- a/sdks/csharp/tools~/gen-regression-tests.sh +++ b/sdks/csharp/tools~/gen-regression-tests.sh @@ -7,6 +7,8 @@ SDK_PATH="$(realpath "$SDK_PATH")" STDB_PATH="$SDK_PATH/../.." DOTNET_VERSION="${1:-}" +cd "$STDB_PATH" + GLOBAL_JSON_BACKUPS=() expected_global_json_symlink_target() { @@ -81,7 +83,6 @@ if [ -n "$DOTNET_VERSION" ]; then BUILD_OPTIONS+=("--build-options=--dotnet-version $DOTNET_VERSION") fi -cargo build --manifest-path "$STDB_PATH/crates/standalone/Cargo.toml" -cargo run --manifest-path "$STDB_PATH/crates/cli/Cargo.toml" -- generate -y -l csharp -o "$SDK_PATH/examples~/regression-tests/client/module_bindings" --module-path "$SDK_PATH/examples~/regression-tests/server" "${BUILD_OPTIONS[@]}" -cargo run --manifest-path "$STDB_PATH/crates/cli/Cargo.toml" -- generate -y -l csharp -o "$SDK_PATH/examples~/regression-tests/republishing/client/module_bindings" --module-path "$SDK_PATH/examples~/regression-tests/republishing/server-republish" "${BUILD_OPTIONS[@]}" -cargo run --manifest-path "$STDB_PATH/crates/cli/Cargo.toml" -- generate -y -l csharp -o "$SDK_PATH/examples~/regression-tests/procedure-client/module_bindings" --module-path "$STDB_PATH/modules/sdk-test-procedure" "${BUILD_OPTIONS[@]}" +cargo spacetime generate -y -l csharp -o "$SDK_PATH/examples~/regression-tests/client/module_bindings" --module-path "$SDK_PATH/examples~/regression-tests/server" "${BUILD_OPTIONS[@]}" +cargo spacetime generate -y -l csharp -o "$SDK_PATH/examples~/regression-tests/republishing/client/module_bindings" --module-path "$SDK_PATH/examples~/regression-tests/republishing/server-republish" "${BUILD_OPTIONS[@]}" +cargo spacetime generate -y -l csharp -o "$SDK_PATH/examples~/regression-tests/procedure-client/module_bindings" --module-path "$STDB_PATH/modules/sdk-test-procedure" "${BUILD_OPTIONS[@]}" diff --git a/sdks/csharp/tools~/run-regression-tests.sh b/sdks/csharp/tools~/run-regression-tests.sh index fb3aca90f4d..8415f319ee7 100644 --- a/sdks/csharp/tools~/run-regression-tests.sh +++ b/sdks/csharp/tools~/run-regression-tests.sh @@ -9,6 +9,8 @@ SDK_PATH="$(realpath "$SDK_PATH")" STDB_PATH="$SDK_PATH/../.." SPACETIMEDB_SERVER_URL="${SPACETIMEDB_SERVER_URL:-local}" +cd "$STDB_PATH" + DOTNET_VERSIONS=("$@") if [ ${#DOTNET_VERSIONS[@]} -eq 0 ]; then DOTNET_VERSIONS=(8 10) @@ -99,9 +101,6 @@ run_client() { trap restore_global_jsons EXIT -# Build and run SpacetimeDB server -cargo build --manifest-path "$STDB_PATH/crates/standalone/Cargo.toml" - for dotnet_version in "${DOTNET_VERSIONS[@]}"; do echo "Running C# regression tests with .NET $dotnet_version" @@ -111,13 +110,13 @@ for dotnet_version in "${DOTNET_VERSIONS[@]}"; do "$SDK_PATH/tools~/gen-regression-tests.sh" "$dotnet_version" # Publish module for btree test - cargo run --manifest-path "$STDB_PATH/crates/cli/Cargo.toml" -- publish --dotnet-version "$dotnet_version" -c -y --server "$SPACETIMEDB_SERVER_URL" -p "$SDK_PATH/examples~/regression-tests/server" btree-repro + cargo spacetime publish --dotnet-version "$dotnet_version" -c -y --server "$SPACETIMEDB_SERVER_URL" -p "$SDK_PATH/examples~/regression-tests/server" btree-repro # Publish module for republishing module test - cargo run --manifest-path "$STDB_PATH/crates/cli/Cargo.toml" -- publish --dotnet-version "$dotnet_version" -c -y --server "$SPACETIMEDB_SERVER_URL" -p "$SDK_PATH/examples~/regression-tests/republishing/server-initial" republish-test - cargo run --manifest-path "$STDB_PATH/crates/cli/Cargo.toml" call --server "$SPACETIMEDB_SERVER_URL" republish-test insert 1 - cargo run --manifest-path "$STDB_PATH/crates/cli/Cargo.toml" -- publish --dotnet-version "$dotnet_version" --server "$SPACETIMEDB_SERVER_URL" -p "$SDK_PATH/examples~/regression-tests/republishing/server-republish" --break-clients republish-test - cargo run --manifest-path "$STDB_PATH/crates/cli/Cargo.toml" call --server "$SPACETIMEDB_SERVER_URL" republish-test insert 2 + cargo spacetime publish --dotnet-version "$dotnet_version" -c -y --server "$SPACETIMEDB_SERVER_URL" -p "$SDK_PATH/examples~/regression-tests/republishing/server-initial" republish-test + cargo spacetime call --server "$SPACETIMEDB_SERVER_URL" republish-test insert 1 + cargo spacetime publish --dotnet-version "$dotnet_version" --server "$SPACETIMEDB_SERVER_URL" -p "$SDK_PATH/examples~/regression-tests/republishing/server-republish" --break-clients republish-test + cargo spacetime call --server "$SPACETIMEDB_SERVER_URL" republish-test insert 2 echo "Cleanup obj~ folders generated in $SDK_PATH/examples~/regression-tests/procedure-client" # There is a bug in the code generator that creates obj~ folders in the output directory using a Rust project. @@ -125,7 +124,7 @@ for dotnet_version in "${DOTNET_VERSIONS[@]}"; do rm -rf "$SDK_PATH/examples~/regression-tests/procedure-client/module_bindings"/*/obj~ # Publish module for procedure tests - cargo run --manifest-path "$STDB_PATH/crates/cli/Cargo.toml" -- publish --dotnet-version "$dotnet_version" -c -y --server "$SPACETIMEDB_SERVER_URL" -p "$STDB_PATH/modules/sdk-test-procedure" procedure-tests + cargo spacetime publish --dotnet-version "$dotnet_version" -c -y --server "$SPACETIMEDB_SERVER_URL" -p "$STDB_PATH/modules/sdk-test-procedure" procedure-tests # Run clients against the modules published with this .NET version. run_client "$SDK_PATH/examples~/regression-tests/client" "$dotnet_version" diff --git a/templates/chat-react-ts/spacetimedb/package.json b/templates/chat-react-ts/spacetimedb/package.json index c8a3cdbdaa4..1662934e7a3 100644 --- a/templates/chat-react-ts/spacetimedb/package.json +++ b/templates/chat-react-ts/spacetimedb/package.json @@ -3,9 +3,9 @@ "license": "ISC", "type": "module", "scripts": { - "build": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- build", - "generate-ts": "cargo build -p spacetimedb-standalone && cargo run -p spacetimedb-cli -- generate --lang typescript --out-dir ts-codegen", - "publish": "cargo run -p spacetimedb-cli -- publish" + "build": "cargo spacetime build", + "generate-ts": "cargo spacetime generate --lang typescript --out-dir ts-codegen", + "publish": "cargo spacetime publish" }, "dependencies": { "spacetimedb": "workspace:^" diff --git a/tools/ci/commands/wasm-bindings/Cargo.toml b/tools/ci/commands/module-latest-deps/Cargo.toml similarity index 84% rename from tools/ci/commands/wasm-bindings/Cargo.toml rename to tools/ci/commands/module-latest-deps/Cargo.toml index e807f25ea7e..c360db5b600 100644 --- a/tools/ci/commands/wasm-bindings/Cargo.toml +++ b/tools/ci/commands/module-latest-deps/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "ci-wasm-bindings" +name = "ci-module-latest-deps" version = "0.1.0" edition.workspace = true diff --git a/tools/ci/commands/module-latest-deps/src/main.rs b/tools/ci/commands/module-latest-deps/src/main.rs new file mode 100644 index 00000000000..f07af82da33 --- /dev/null +++ b/tools/ci/commands/module-latest-deps/src/main.rs @@ -0,0 +1,48 @@ +#![allow(clippy::disallowed_macros)] + +use anyhow::{ensure, Result}; +use clap::Parser; +use duct::cmd; +use std::{env, path::PathBuf}; + +/// Checks that a module builds with the latest compatible dependencies. +#[derive(Parser)] +struct Cli { + /// Do not build the CLI; use the binary selected by SPACETIME_BIN. + #[arg(long)] + no_build: bool, +} + +fn target_dir() -> PathBuf { + match env::var_os("CARGO_TARGET_DIR").map(PathBuf::from) { + Some(path) if path.is_absolute() => path, + Some(path) => ci_common::repo_root().join(path), + None => ci_common::repo_root().join("target"), + } +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + + // Build the CLI before updating the lockfile so a newly published incompatible dependency + // cannot prevent us from exercising the fresh module dependency graph. + let cli_path = if cli.no_build { + ci_common::require_spacetime_bin()? + } else { + ensure!( + env::var_os("SPACETIME_BIN").is_none(), + "SPACETIME_BIN requires --no-build" + ); + cmd!("cargo", "build", "-p", "spacetimedb-cli").run()?; + target_dir() + .join("debug/spacetimedb-cli") + .with_extension(env::consts::EXE_EXTENSION) + }; + + // A fresh module gets the latest versions permitted by its dependency constraints rather + // than the exact versions pinned in this repository's committed lockfile. + cmd!("cargo", "update").run()?; + cmd!(cli_path, "build", "--module-path", "modules/module-test").run()?; + + Ok(()) +} diff --git a/tools/ci/commands/run-spacetime/Cargo.toml b/tools/ci/commands/run-spacetime/Cargo.toml new file mode 100644 index 00000000000..3ec38ddf291 --- /dev/null +++ b/tools/ci/commands/run-spacetime/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "ci-run-spacetime" +version = "0.1.0" +edition.workspace = true + +[dependencies] +anyhow.workspace = true +duct.workspace = true + +[lints] +workspace = true diff --git a/tools/ci/commands/run-spacetime/src/main.rs b/tools/ci/commands/run-spacetime/src/main.rs new file mode 100644 index 00000000000..8a387c18fb7 --- /dev/null +++ b/tools/ci/commands/run-spacetime/src/main.rs @@ -0,0 +1,78 @@ +#![allow(clippy::disallowed_macros)] + +use anyhow::{bail, ensure, Result}; +use duct::cmd; +use std::{ + env, + ffi::OsString, + path::{Path, PathBuf}, +}; + +const CLI_NAME: &str = "spacetimedb-cli"; +const STANDALONE_NAME: &str = "spacetimedb-standalone"; + +fn executable_name(name: &str) -> OsString { + if env::consts::EXE_EXTENSION.is_empty() { + name.into() + } else { + format!("{name}.{}", env::consts::EXE_EXTENSION).into() + } +} + +fn validate_runtime(cli: &Path) -> Result<()> { + ensure!( + cli.is_file(), + "SpacetimeDB CLI binary does not exist at {}", + cli.display() + ); + + let standalone = cli.with_file_name(executable_name(STANDALONE_NAME)); + ensure!( + standalone.is_file(), + "SpacetimeDB standalone binary does not exist beside the CLI at {}", + standalone.display() + ); + + Ok(()) +} + +fn repo_root() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(4) + .expect("run-spacetime package should be nested beneath the workspace root") + .to_owned() +} + +fn local_cli() -> Result { + cmd!("cargo", "build", "-p", CLI_NAME, "-p", STANDALONE_NAME).run()?; + + let target_dir = env::var_os("CARGO_TARGET_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| repo_root().join("target")); + Ok(target_dir.join("debug").join(executable_name(CLI_NAME))) +} + +fn main() -> Result<()> { + let args = env::args_os().skip(1).collect::>(); + if args.is_empty() { + bail!("Usage: cargo spacetime [args...]"); + } + + let cli = match env::var_os("SPACETIME_BIN") { + Some(path) => { + let path = PathBuf::from(path); + ensure!( + path.is_absolute(), + "SPACETIME_BIN must be an absolute path, got {}", + path.display() + ); + path + } + None => local_cli()?, + }; + + validate_runtime(&cli)?; + cmd(cli, args).run()?; + Ok(()) +} diff --git a/tools/ci/commands/test/src/main.rs b/tools/ci/commands/test/src/main.rs index 373a30ef49a..a8b4b55fd16 100644 --- a/tools/ci/commands/test/src/main.rs +++ b/tools/ci/commands/test/src/main.rs @@ -1,5 +1,5 @@ #![allow(clippy::disallowed_macros)] -use anyhow::Result; +use anyhow::{ensure, Result}; use ci_common::pnpm; use clap::Parser; use duct::cmd; @@ -10,10 +10,23 @@ use duct::cmd; /// This does not include Unreal tests. /// This expects to run in a clean git state. #[derive(Parser)] -struct Cli {} +struct Cli { + /// Do not build CLI and standalone; use the binaries selected by SPACETIME_BIN. + #[arg(long)] + no_build: bool, +} fn main() -> Result<()> { - Cli::parse(); + let cli = Cli::parse(); + + if cli.no_build { + ci_common::require_runtime()?; + } else { + ensure!( + std::env::var_os("SPACETIME_BIN").is_none(), + "SPACETIME_BIN requires --no-build" + ); + } pnpm(["build"]).dir("crates/bindings-typescript").run()?; @@ -53,18 +66,20 @@ fn main() -> Result<()> { .run()?; // The SDK test harness uses the same child-process server guard as smoketests, // which expects release CLI/standalone binaries to already exist. - cmd!( - "cargo", - "build", - "--release", - "-p", - "spacetimedb-cli", - "-p", - "spacetimedb-standalone", - "--features", - "spacetimedb-standalone/allow_loopback_http_for_tests", - ) - .run()?; + if !cli.no_build { + cmd!( + "cargo", + "build", + "--release", + "-p", + "spacetimedb-cli", + "-p", + "spacetimedb-standalone", + "--features", + "spacetimedb-standalone/allow_loopback_http_for_tests", + ) + .run()?; + } // SDK procedure tests intentionally make localhost HTTP requests. cmd!( "cargo", diff --git a/tools/ci/commands/typescript-test/src/main.rs b/tools/ci/commands/typescript-test/src/main.rs index f18a3b82bfb..8fc8d4ce34f 100644 --- a/tools/ci/commands/typescript-test/src/main.rs +++ b/tools/ci/commands/typescript-test/src/main.rs @@ -1,20 +1,31 @@ #![allow(clippy::disallowed_macros)] -use anyhow::{bail, Result}; +use anyhow::{bail, ensure, Result}; use ci_common::pnpm; use clap::Parser; -use duct::cmd; /// Runs TypeScript workspace tests and template build checks. #[derive(Parser)] -struct Cli {} +struct Cli { + /// Do not build CLI and standalone; use the binaries selected by SPACETIME_BIN. + #[arg(long)] + no_build: bool, +} fn main() -> Result<()> { - Cli::parse(); + let cli = Cli::parse(); + if cli.no_build { + ci_common::require_runtime()?; + } else { + ensure!( + std::env::var_os("SPACETIME_BIN").is_none(), + "SPACETIME_BIN requires --no-build" + ); + } pnpm(["build"]).dir("crates/bindings-typescript").run()?; pnpm(["test"]).dir("crates/bindings-typescript").run()?; pnpm(["generate"]).dir("templates/chat-react-ts").run()?; - let diff_status = cmd!( + let diff_status = duct::cmd!( "bash", "tools/check-diff.sh", "templates/chat-react-ts/src/module_bindings" diff --git a/tools/ci/commands/wasm-bindings/src/main.rs b/tools/ci/commands/wasm-bindings/src/main.rs deleted file mode 100644 index aa1946d11ac..00000000000 --- a/tools/ci/commands/wasm-bindings/src/main.rs +++ /dev/null @@ -1,41 +0,0 @@ -#![allow(clippy::disallowed_macros)] -use anyhow::Result; -use ci_common::pnpm; -use clap::Parser; -use duct::cmd; - -/// Tests Wasm bindings -/// -/// Runs tests for the codegen crate and builds a test module with the wasm bindings. -#[derive(Parser)] -struct Cli {} - -fn main() -> Result<()> { - Cli::parse(); - - pnpm([ - "install", - "--filter", - "./crates/bindings-typescript...", - "--filter", - "./modules/module-test-ts...", - ]) - .run()?; - pnpm(["build"]).dir("crates/bindings-typescript").run()?; - cmd!("cargo", "test", "-p", "spacetimedb-codegen").run()?; - // Pre-build the CLI so that it _doesn't_ get `cargo update`d, since that may break the build. - cmd!("cargo", "build", "-p", "spacetimedb-cli").run()?; - // Make sure the `Cargo.lock` file reflects the latest available versions. - // This is what users would end up with on a fresh module, so we want to - // catch any compile errors arising from a different transitive closure - // of dependencies than what is in the workspace lock file. - // - // For context see also: https://github.com/clockworklabs/SpacetimeDB/pull/2714 - cmd!("cargo", "update").run()?; - let cli_path = ci_common::repo_root() - .join("target/debug/spacetimedb-cli") - .with_extension(std::env::consts::EXE_EXTENSION); - cmd!(cli_path, "build", "--module-path", "modules/module-test",).run()?; - - Ok(()) -} diff --git a/tools/ci/common/src/lib.rs b/tools/ci/common/src/lib.rs index 4dfafe8e03b..063383cd0ba 100644 --- a/tools/ci/common/src/lib.rs +++ b/tools/ci/common/src/lib.rs @@ -1,5 +1,6 @@ -use anyhow::{bail, Result}; +use anyhow::{bail, ensure, Context, Result}; use duct::{cmd, Expression}; +use std::env; use std::ffi::OsStr; use std::path::Path; use std::path::PathBuf; @@ -20,6 +21,36 @@ pub fn repo_root() -> PathBuf { .to_path_buf() } +pub fn require_spacetime_bin() -> Result { + let path = env::var_os("SPACETIME_BIN") + .map(PathBuf::from) + .context("--no-build requires SPACETIME_BIN")?; + ensure!( + path.is_absolute(), + "SPACETIME_BIN must be an absolute path, got {}", + path.display() + ); + ensure!( + path.is_file(), + "SpacetimeDB CLI binary does not exist at {}", + path.display() + ); + Ok(path) +} + +pub fn require_runtime() -> Result<()> { + let cli = require_spacetime_bin()?; + let standalone = cli + .with_file_name("spacetimedb-standalone") + .with_extension(env::consts::EXE_EXTENSION); + ensure!( + standalone.is_file(), + "SpacetimeDB standalone binary does not exist beside the CLI at {}", + standalone.display() + ); + Ok(()) +} + pub fn pnpm(args: I) -> Expression where I: IntoIterator, diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index bb0878277d9..a6592c4a6cd 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -21,8 +21,8 @@ const COMMANDS: &[Command] = &[ package: "ci-lint", }, Command { - path: &["wasm-bindings"], - package: "ci-wasm-bindings", + path: &["module-latest-deps"], + package: "ci-module-latest-deps", }, Command { path: &["smoketests"], @@ -80,6 +80,10 @@ const COMMANDS: &[Command] = &[ path: &["other-workflows", "watch"], package: "ci-workflow-watch", }, + Command { + path: &["other-workflows", "run-spacetime"], + package: "ci-run-spacetime", + }, ]; fn print_help() { diff --git a/tools/gen-bindings/src/main.rs b/tools/gen-bindings/src/main.rs index 88fe7c7cc51..c5cbcf665d2 100644 --- a/tools/gen-bindings/src/main.rs +++ b/tools/gen-bindings/src/main.rs @@ -46,26 +46,16 @@ fn run_inherit(cmd: impl AsRef, args: &[&str], cwd: Option<&Path>) -> Res fn main() -> Result<()> { let args = Cli::parse(); - let workspace_dir = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .parent() - .unwrap(); - - // 1) Build prerequisite - run_inherit("cargo", &["build"], Some(workspace_dir))?; - - // 2) Ensure output directory exists + // 1) Ensure output directory exists if !Path::new(&args.out_dir).exists() { fs::create_dir_all(&args.out_dir).context("create output directory")?; } - // 3) Generate TS client from project + // 2) Generate TS client from project run_inherit( - workspace_dir - .join("target/debug/spacetimedb-cli") - .with_extension(std::env::consts::EXE_EXTENSION), + "cargo", &[ + "spacetime", "generate", "-y", "--lang",