diff --git a/.github/actions/setup-copilot/action.yml b/.github/actions/setup-copilot/action.yml index 506f472ca2..a9c39a2a0a 100644 --- a/.github/actions/setup-copilot/action.yml +++ b/.github/actions/setup-copilot/action.yml @@ -4,6 +4,9 @@ outputs: cli-path: description: "Path to the Copilot CLI" value: ${{ steps.cli-path.outputs.path }} + javascript-cli-path: + description: "Path to the JavaScript Copilot CLI entrypoint" + value: ${{ steps.cli-path.outputs.javascript-path }} runs: using: "composite" steps: @@ -28,10 +31,14 @@ runs: echo "Could not prepare the Copilot CLI runtime" >&2 exit 1 fi + javascript_cli_path=$(npm --prefix "$(pwd)/nodejs" run --silent prepare:runtime -- --print-legacy-path) + if [ -z "$javascript_cli_path" ]; then + echo "Could not prepare the Copilot CLI JavaScript entrypoint" >&2 + exit 1 + fi echo "path=$cli_path" >> $GITHUB_OUTPUT + echo "javascript-path=$javascript_cli_path" >> $GITHUB_OUTPUT shell: bash - name: Verify CLI works - run: | - legacy_cli=$(npm --prefix "$(pwd)/nodejs" run --silent prepare:runtime -- --print-legacy-path) - node "$legacy_cli" --version + run: node "${{ steps.cli-path.outputs.javascript-path }}" --version shell: bash diff --git a/.github/workflows/java-adapt-handwritten-code-to-accept-upgrade-changes.lock.yml b/.github/workflows/java-adapt-handwritten-code-to-accept-upgrade-changes.lock.yml index 7b7d524fe7..7554b0bb76 100644 --- a/.github/workflows/java-adapt-handwritten-code-to-accept-upgrade-changes.lock.yml +++ b/.github/workflows/java-adapt-handwritten-code-to-accept-upgrade-changes.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"a5f19a89f89b0693f86ca89ea90e3a633fe19c17bf4d27214fa9124429cdc156","body_hash":"8db09798070cbcba22c42c50a316ae45c8e8c650eeb23c556b44fde8d519550a","compiler_version":"v0.88.2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"03beaef805d9bdf4b878b9fd1dd47c62793f30f179373406e081b88a9817b182","body_hash":"f535cb24328c6e9b3963e72de09404b08a4f4580a8174bc6ad580c0f005837ae","compiler_version":"v0.88.2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_CI_TRIGGER_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"9271a1804551c0dc4fb0085a97979950aa2f8489","version":"v0.88.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12","digest":"sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12@sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12","digest":"sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12@sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12","digest":"sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12@sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.15","digest":"sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.15@sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e","pinned_image":"ghcr.io/github/gh-aw-node@sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_release_by_tag","get_tag","list_branches","list_commits","list_releases","list_starred_repositories","list_tags","search_code","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","missing_data","missing_tool","noop","push_to_pull_request_branch"]}]} # This file was automatically generated by gh-aw (v0.88.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -24,7 +24,7 @@ # For more information: https://github.github.com/gh-aw/introduction/overview/ # # Adapt handwritten Java SDK code to work with regenerated types after a -# @github/copilot version bump. Assumes codegen succeeded and generated code +# Copilot CLI release update. Assumes codegen succeeded and generated code # compiles. Fixes handwritten source and tests only. # # Secrets used: @@ -1516,7 +1516,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: WORKFLOW_NAME: "Java Handwritten Code Adaptation After CLI Upgrade" - WORKFLOW_DESCRIPTION: "Adapt handwritten Java SDK code to work with regenerated types after a\n@github/copilot version bump. Assumes codegen succeeded and generated code\ncompiles. Fixes handwritten source and tests only." + WORKFLOW_DESCRIPTION: "Adapt handwritten Java SDK code to work with regenerated types after a\nCopilot CLI release update. Assumes codegen succeeded and generated code\ncompiles. Fixes handwritten source and tests only." HAS_PATCH: ${{ needs.agent.outputs.has_patch }} GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" GH_AW_DETECTION_SKIP_PROMPT_SUMMARY: "true" @@ -1580,7 +1580,7 @@ jobs: S2STOKENS: true TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} WORKFLOW_NAME: "Java Handwritten Code Adaptation After CLI Upgrade" - WORKFLOW_DESCRIPTION: "Adapt handwritten Java SDK code to work with regenerated types after a\n@github/copilot version bump. Assumes codegen succeeded and generated code\ncompiles. Fixes handwritten source and tests only." + WORKFLOW_DESCRIPTION: "Adapt handwritten Java SDK code to work with regenerated types after a\nCopilot CLI release update. Assumes codegen succeeded and generated code\ncompiles. Fixes handwritten source and tests only." HAS_PATCH: ${{ needs.agent.outputs.has_patch }} GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" run: | diff --git a/.github/workflows/java-adapt-handwritten-code-to-accept-upgrade-changes.md b/.github/workflows/java-adapt-handwritten-code-to-accept-upgrade-changes.md index dd1bfe2bbc..91f11d1f86 100644 --- a/.github/workflows/java-adapt-handwritten-code-to-accept-upgrade-changes.md +++ b/.github/workflows/java-adapt-handwritten-code-to-accept-upgrade-changes.md @@ -1,7 +1,7 @@ --- description: | Adapt handwritten Java SDK code to work with regenerated types after a - @github/copilot version bump. Assumes codegen succeeded and generated code + Copilot CLI release update. Assumes codegen succeeded and generated code compiles. Fixes handwritten source and tests only. on: @@ -45,14 +45,13 @@ safe-outputs: # Java Handwritten Code Adaptation After CLI Upgrade -You are an automation agent that fixes handwritten Java SDK source and test code after a `@github/copilot` version bump has regenerated the typed schemas. +You are an automation agent that fixes handwritten Java SDK source and test code after a Copilot CLI release update has regenerated the typed schemas. ## Assumptions - The branch `${{ inputs.branch }}` already has: - - Updated `java/scripts/codegen/package.json` with the new version + - Updated the shared CLI release pin in `nodejs/package.json` - Regenerated `java/sdk/src/generated/java/` code that compiles successfully - - Updated the Java POM CLI/version pin property - Your job is ONLY to fix **handwritten** code, NOT generated code. ## Boundaries @@ -147,7 +146,7 @@ If this passes, commit and push: ```bash git add java/sdk/src/main/java java/sdk/src/test/java -git commit -m "Fix handwritten Java code for @github/copilot schema changes +git commit -m "Fix handwritten Java code for CLI schema changes Adapt constructor calls, enum references, and test assertions to match regenerated types after CLI version bump." diff --git a/.github/workflows/java-codegen-check.yml b/.github/workflows/java-codegen-check.yml index f2f4527966..e490a5cf4e 100644 --- a/.github/workflows/java-codegen-check.yml +++ b/.github/workflows/java-codegen-check.yml @@ -5,11 +5,13 @@ on: branches: - main paths: + - 'nodejs/package.json' - 'java/scripts/codegen/**' - 'java/sdk/src/generated/**' - '.github/workflows/java-codegen-check.yml' pull_request: paths: + - 'nodejs/package.json' - 'java/scripts/codegen/**' - 'java/sdk/src/generated/**' - '.github/workflows/java-codegen-check.yml' @@ -48,9 +50,13 @@ jobs: working-directory: ./java/scripts/codegen run: npm ci + - name: Test schema fetcher + working-directory: ./java/scripts/codegen + run: npm test + - name: Run codegen working-directory: ./java/scripts/codegen - run: npx tsx java.ts + run: npm run generate - name: Check for uncommitted changes id: check-changes @@ -68,7 +74,7 @@ jobs: - name: Fail on stale generated files (push to main) if: steps.check-changes.outputs.changed == 'true' && github.event_name != 'pull_request' run: | - echo "::error::Generated files are out of date. Run 'cd java/scripts/codegen && npx tsx java.ts' and commit the changes." + echo "::error::Generated files are out of date. Run 'cd java/scripts/codegen && npm run generate' and commit the changes." git diff exit 1 @@ -93,7 +99,7 @@ jobs: if: steps.push-regen.outcome == 'failure' run: | echo "::error::Could not push regenerated files to the PR branch. This is expected for Dependabot PRs (read-only token) and fork PRs." - echo "To fix: check out this PR branch locally, run 'cd java/scripts/codegen && npx tsx java.ts', commit, and push." + echo "To fix: check out this PR branch locally, run 'cd java/scripts/codegen && npm run generate', commit, and push." exit 1 - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 diff --git a/.github/workflows/java-codegen-fix.lock.yml b/.github/workflows/java-codegen-fix.lock.yml index 91c4ea1816..fa64a0fe88 100644 --- a/.github/workflows/java-codegen-fix.lock.yml +++ b/.github/workflows/java-codegen-fix.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"b0390c9ab9beb0d7e106314299e89e486269ab7f64d8489d5021132d79aa6b9b","body_hash":"63d6ce13a5131b158ddffb10a469aa59e0fdc2278eec4d8de7f6763e0b6f2ea2","compiler_version":"v0.88.2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"b0390c9ab9beb0d7e106314299e89e486269ab7f64d8489d5021132d79aa6b9b","body_hash":"c7cc7984b1d512e871371a7fd99bc4f7e4615d4192348170eca763cf584f4855","compiler_version":"v0.88.2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_CI_TRIGGER_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"9271a1804551c0dc4fb0085a97979950aa2f8489","version":"v0.88.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12","digest":"sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12@sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12","digest":"sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12@sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12","digest":"sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12@sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.15","digest":"sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.15@sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e","pinned_image":"ghcr.io/github/gh-aw-node@sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_release_by_tag","get_tag","list_branches","list_commits","list_releases","list_starred_repositories","list_tags","search_code","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","missing_data","missing_tool","noop","push_to_pull_request_branch"]}]} # This file was automatically generated by gh-aw (v0.88.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # diff --git a/.github/workflows/java-codegen-fix.md b/.github/workflows/java-codegen-fix.md index b1dcb1f636..8f6a845433 100644 --- a/.github/workflows/java-codegen-fix.md +++ b/.github/workflows/java-codegen-fix.md @@ -52,7 +52,7 @@ You are an automation agent that fixes Java compilation and test failures caused ## Context -A Dependabot PR bumped the `@github/copilot` npm dependency in `java/scripts/codegen/package.json`. The `java-codegen-check` workflow ran the code generator (`java/scripts/codegen/java.ts`) against the new schemas and `mvn verify` subsequently failed. Your job is to fix **both** the code generator script (if needed) and the handwritten SDK/test source code so the build passes. +A Copilot CLI release pin update fetched new schemas from GitHub Releases. The `java-codegen-check` workflow ran the code generator (`java/scripts/codegen/java.ts`) against those schemas and `mvn verify` subsequently failed. Your job is to fix **both** the code generator script (if needed) and the handwritten SDK/test source code so the build passes. **❌❌❌ YOU MUST NEVER EDIT any of the java source code in `java/sdk/src/generated/` directly.** ✅✅Rather, the way to affect changes in these files is to change the code generator script and re-generate the classes in `java/sdk/src/generated`. @@ -66,9 +66,9 @@ ${{ inputs.error_summary }} ## Architecture overview -The code generator (`java/scripts/codegen/java.ts`) reads JSON schemas from `node_modules/@github/copilot/schemas/` and produces Java source files under `java/sdk/src/generated/java/`. These generated types are consumed by handwritten code in `java/sdk/src/main/java/` (primarily `CopilotSession.java`) and tested by handwritten tests in `java/sdk/src/test/java/`. +The code generator (`java/scripts/codegen/java.ts`) reads JSON schemas from `java/scripts/codegen/target/schemas/`. The schemas are extracted from the pinned `github-copilot--linux-x64.tgz` GitHub Release asset by `fetch-schemas.mjs`. The generator produces Java source files under `java/sdk/src/generated/java/`. These generated types are consumed by handwritten code in `java/sdk/src/main/java/` (primarily `CopilotSession.java`) and tested by handwritten tests in `java/sdk/src/test/java/`. -When `@github/copilot` is bumped, the schemas may change in ways the code generator does not yet handle. Common schema changes include: +When the Copilot CLI release pin is bumped, the schemas may change in ways the code generator does not yet handle. Common schema changes include: - **`$ref` references**: Inline nested type definitions replaced with `$ref` pointers to `#/definitions/` entries. The code generator must resolve these references and emit standalone Java types instead of nested records. - **Field type changes**: Numeric fields changing between `double`, `Long`, `int`, etc. @@ -97,10 +97,10 @@ mvn --version node --version ``` -Install codegen dependencies: +Install codegen dependencies and fetch the pinned release schemas: ```bash -cd java/scripts/codegen && npm ci && cd ../../.. +cd java/scripts/codegen && npm ci && npm run fetch:schemas && cd ../../.. ``` ### Step 1: Reproduce the failure @@ -135,13 +135,13 @@ To diagnose, compare the current schemas with the generated output: ```bash # List available schemas -ls java/scripts/codegen/node_modules/@github/copilot/schemas/ +ls java/scripts/codegen/target/schemas/ # Check for $ref usage in schemas (indicates the codegen may need $ref resolution) -grep -r '"$ref"' java/scripts/codegen/node_modules/@github/copilot/schemas/ | head -20 +grep -r '"$ref"' java/scripts/codegen/target/schemas/ | head -20 # Look at a specific schema that relates to failing types -cat java/scripts/codegen/node_modules/@github/copilot/schemas/.json | head -80 +head -80 java/scripts/codegen/target/schemas/.json ``` ### Step 3: Fix the code generator (if needed) @@ -157,7 +157,7 @@ If the diagnosis shows the code generator does not handle the new schema format: 3. **Re-run code generation** to produce updated generated files: ```bash - cd java/scripts/codegen && npx tsx java.ts && cd ../../.. + cd java/scripts/codegen && npm run generate && cd ../../.. ``` 4. **Verify the generated output** looks reasonable: @@ -213,7 +213,7 @@ After `mvn verify` passes, commit all changes and use the `push-to-pull-request- ```bash git add -A -git commit -m "Fix Java codegen and build failures after @github/copilot update +git commit -m "Fix Java codegen and build failures after CLI update Automated fix applied by java-codegen-fix workflow." ``` @@ -236,7 +236,7 @@ Do **NOT** push broken code. ## Important constraints -- **NEVER** hand-edit files under `java/sdk/src/generated/java/` — these are auto-generated. They are updated by running `cd java/scripts/codegen && npx tsx java.ts`. +- **NEVER** hand-edit files under `java/sdk/src/generated/java/` — these are auto-generated. They are updated by running `cd java/scripts/codegen && npm run generate`. - **NEVER** modify `java/sdk/pom.xml` — build config is not in scope - **NEVER** modify `java/scripts/codegen/package.json` or `java/scripts/codegen/package-lock.json` — dependency versions are not in scope - **NEVER** modify files under `.github/` — workflow files are not in scope diff --git a/.github/workflows/java-smoke-test.yml b/.github/workflows/java-smoke-test.yml index e7e9a417d2..5f808f8d0d 100644 --- a/.github/workflows/java-smoke-test.yml +++ b/.github/workflows/java-smoke-test.yml @@ -29,27 +29,8 @@ jobs: distribution: "microsoft" cache: "maven" - - uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v6 - with: - node-version: 22 - - - name: Read pinned @github/copilot version from pom.xml - id: cli-version - run: | - PROP="readonly-copilot-sdk-ref-impl-version-from-lastmerge-file-updated-by-reference-impl-sync" - VERSION=$(sed -n "s|.*<${PROP}>\(.*\).*|\1|p" pom.xml | head -n 1 | tr -d '[:space:]') - if [[ -z "$VERSION" || "$VERSION" == "PRIMER_TO_REPLACE" ]]; then - echo "::error::Could not read pinned @github/copilot version from pom.xml property <${PROP}>" >&2 - exit 1 - fi - echo "Pinned @github/copilot version: $VERSION" - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - - - name: Install Copilot CLI globally (pinned to pom.xml version) - run: npm install -g "@github/copilot@${{ steps.cli-version.outputs.version }}" - - - name: Verify CLI works - run: copilot --version + - uses: ./.github/actions/setup-copilot + id: setup-copilot - name: Build SDK and install to local repo run: mvn -DskipTests -Pskip-test-harness clean install @@ -57,6 +38,7 @@ jobs: - name: Create and run smoke test via Copilot CLI env: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_CLI_JS: ${{ steps.setup-copilot.outputs.javascript-cli-path }} run: | cat > /tmp/smoke-test-prompt.txt << 'PROMPT_EOF' You are running inside the copilot-sdk monorepo, in the java/ subdirectory. @@ -74,7 +56,7 @@ jobs: If any step fails, exit with a non-zero exit code. Do not silently fix errors. PROMPT_EOF - copilot --yolo --prompt "$(cat /tmp/smoke-test-prompt.txt)" + node "$COPILOT_CLI_JS" --yolo --prompt "$(cat /tmp/smoke-test-prompt.txt)" - name: Run smoke test jar env: @@ -102,27 +84,8 @@ jobs: distribution: "microsoft" cache: "maven" - - uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v6 - with: - node-version: 22 - - - name: Read pinned @github/copilot version from pom.xml - id: cli-version - run: | - PROP="readonly-copilot-sdk-ref-impl-version-from-lastmerge-file-updated-by-reference-impl-sync" - VERSION=$(sed -n "s|.*<${PROP}>\(.*\).*|\1|p" pom.xml | head -n 1 | tr -d '[:space:]') - if [[ -z "$VERSION" || "$VERSION" == "PRIMER_TO_REPLACE" ]]; then - echo "::error::Could not read pinned @github/copilot version from pom.xml property <${PROP}>" >&2 - exit 1 - fi - echo "Pinned @github/copilot version: $VERSION" - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - - - name: Install Copilot CLI globally (pinned to pom.xml version) - run: npm install -g "@github/copilot@${{ steps.cli-version.outputs.version }}" - - - name: Verify CLI works - run: copilot --version + - uses: ./.github/actions/setup-copilot + id: setup-copilot - name: Build SDK and install to local repo run: mvn -DskipTests -Pskip-test-harness clean install @@ -130,6 +93,7 @@ jobs: - name: Create and run smoke test via Copilot CLI env: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_CLI_JS: ${{ steps.setup-copilot.outputs.javascript-cli-path }} run: | cat > /tmp/smoke-test-prompt.txt << 'PROMPT_EOF' You are running inside the copilot-sdk monorepo, in the java/ subdirectory. @@ -150,7 +114,7 @@ jobs: If any step fails, exit with a non-zero exit code. Do not silently fix errors. PROMPT_EOF - copilot --yolo --prompt "$(cat /tmp/smoke-test-prompt.txt)" + node "$COPILOT_CLI_JS" --yolo --prompt "$(cat /tmp/smoke-test-prompt.txt)" - name: Run smoke test jar env: diff --git a/.github/workflows/update-copilot-dependency.yml b/.github/workflows/update-copilot-dependency.yml index 2870ad27f7..f1e4d9fca9 100644 --- a/.github/workflows/update-copilot-dependency.yml +++ b/.github/workflows/update-copilot-dependency.yml @@ -87,25 +87,6 @@ jobs: java-version: "25" distribution: "microsoft" - - name: Update @github/copilot in Java codegen - env: - VERSION: ${{ inputs.version }} - working-directory: ./java/scripts/codegen - run: npm install "@github/copilot@$VERSION" - - - name: Update Java POM CLI version property - env: - VERSION: ${{ inputs.version }} - working-directory: ./java - run: | - PROP="readonly-copilot-sdk-ref-impl-version-from-lastmerge-file-updated-by-reference-impl-sync" - sed -i -E "s|(<${PROP}>)[^<]*()|\1^${VERSION}\2|" pom.xml - # Use fixed-string matching (-F) because npm versions contain regex - # metacharacters: '^' (caret ranges) and '.' (dots in semver) would - # otherwise be interpreted as start-of-line and any-char respectively, - # causing false negatives or spurious matches. - grep -qF "<${PROP}>^${VERSION}" pom.xml - - name: Run Java codegen working-directory: ./java run: mvn generate-sources -Pcodegen @@ -163,7 +144,7 @@ jobs: git commit -m "Update Copilot CLI to $VERSION - - Updated the Node.js CLI release pin + - Updated the shared CLI release pin - Re-ran code generators - Formatted generated code" @@ -173,11 +154,11 @@ jobs: Automated update of the Copilot CLI release to version `PLACEHOLDER_VERSION`. ### Changes - - Updated the release pin in `nodejs/package.json` + - Updated the shared release pin in `nodejs/package.json` - Validated the release assets listed in `SHA256SUMS.txt` - Re-ran all code generators (`scripts/codegen`) - Formatted generated output - - Updated Java codegen dependency, POM property, and regenerated Java types + - Regenerated Java types from the pinned CLI release schemas ### Java Handwritten Code Adaptation Plan @@ -221,7 +202,7 @@ jobs: else gh pr create \ --draft \ - --title "Update @github/copilot to $VERSION" \ + --title "Update Copilot CLI to $VERSION" \ --body "$PR_BODY" \ --base main \ --head "$BRANCH" diff --git a/dotnet/README.md b/dotnet/README.md index 23a78030b4..40c7a0bdae 100644 --- a/dotnet/README.md +++ b/dotnet/README.md @@ -14,6 +14,13 @@ To use the SDK, you'll need: dotnet add package GitHub.Copilot.SDK ``` +The package downloads the pinned Copilot CLI runtime for the build RID from the +matching `github/copilot-cli` GitHub release and verifies the archive against +that release's `SHA256SUMS.txt`. Set `CopilotCliReleaseBaseUrl` in MSBuild (or +`COPILOT_CLI_DOWNLOAD_BASE_URL` in the environment) to use a release mirror. +Set `CopilotCliBinaryPath` to copy a preinstalled binary instead, or set +`CopilotSkipCliDownload=true` to omit runtime acquisition. + ## Run the Samples Try the interactive chat sample (from the repo root): diff --git a/dotnet/src/build/GitHub.Copilot.SDK.targets b/dotnet/src/build/GitHub.Copilot.SDK.targets index aca299dd27..a5aba612ab 100644 --- a/dotnet/src/build/GitHub.Copilot.SDK.targets +++ b/dotnet/src/build/GitHub.Copilot.SDK.targets @@ -26,7 +26,7 @@ - + <_CopilotPlatform Condition="'$(_CopilotRid)' == 'win-x64'">win32-x64 <_CopilotPlatform Condition="'$(_CopilotRid)' == 'win-arm64'">win32-arm64 @@ -49,15 +49,16 @@ <_CopilotRuntimeLib Condition="'$(_CopilotRuntimeLib)' == ''">libcopilot_runtime.so - + COPILOT_CLI_DOWNLOAD_BASE_URL is also honored. --> - https://registry.npmjs.org + $(COPILOT_CLI_DOWNLOAD_BASE_URL) + https://github.com/github/copilot-cli/releases/download @@ -93,23 +94,58 @@ <_CopilotCacheDir>$(IntermediateOutputPath)copilot-cli\$(CopilotCliVersion)\$(_CopilotPlatform) - <_CopilotCliBinaryPath Condition="'$(_CopilotCliBinaryPath)' == ''">$(_CopilotCacheDir)\$(_CopilotBinary) + + <_CopilotCliBinaryPath Condition="'$(_CopilotCliBinaryPath)' == ''">$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\$(_CopilotRuntimeWrapper) + <_CopilotRuntimeNodePath>$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\runtime.node + <_CopilotRuntimeBundleCompleteMarker>$(_CopilotCacheDir)\.copilot-runtime-complete <_CopilotArchivePath>$(_CopilotCacheDir)\copilot.tgz - <_CopilotNormalizedRegistryUrl>$([System.String]::Copy('$(CopilotNpmRegistryUrl)').TrimEnd('/')) - <_CopilotDownloadUrl>$(_CopilotNormalizedRegistryUrl)/@github/copilot-$(_CopilotPlatform)/-/copilot-$(_CopilotPlatform)-$(CopilotCliVersion).tgz + <_CopilotChecksumPath>$(_CopilotCacheDir)\SHA256SUMS.txt + <_CopilotAssetName>github-copilot-$(CopilotCliVersion)-$(_CopilotPlatform).tgz + <_CopilotAssetNameRegex>$([System.Text.RegularExpressions.Regex]::Escape('$(_CopilotAssetName)')) + <_CopilotNormalizedReleaseBaseUrl>$([System.String]::Copy('$(CopilotCliReleaseBaseUrl)').TrimEnd('/')) + <_CopilotReleaseUrl>$(_CopilotNormalizedReleaseBaseUrl)/v$(CopilotCliVersion) + <_CopilotDownloadUrl>$(_CopilotReleaseUrl)/$(_CopilotAssetName) + <_CopilotChecksumsUrl>$(_CopilotReleaseUrl)/SHA256SUMS.txt + <_CopilotRuntimeBundleMissing Condition="!Exists('$(_CopilotCliBinaryPath)') Or !Exists('$(_CopilotRuntimeNodePath)') Or !Exists('$(_CopilotRuntimeBundleCompleteMarker)')">true <_CopilotCliDownloadTimeoutMs>$([System.Convert]::ToInt32($([MSBuild]::Multiply($(CopilotCliDownloadTimeout), 1000)))) - - + + - - + + + + + + + + <_CopilotChecksumMatch Include="@(_CopilotChecksumLine)" + Condition="$([System.Text.RegularExpressions.Regex]::IsMatch('%(Identity)', '^[0-9a-fA-F]{64}[\t ]+\*?$(_CopilotAssetNameRegex)[\t ]*$'))" /> + + + Condition="'$(_CopilotRuntimeBundleMissing)' == 'true'" /> + + + + + <_CopilotChecksumLineValue>@(_CopilotChecksumMatch) + <_CopilotArchiveHashValue>@(_CopilotArchiveHash->'%(FileHash)') + <_CopilotExpectedChecksum>$([System.String]::Copy('$(_CopilotChecksumLineValue)').Substring(0, 64).ToUpperInvariant()) + <_CopilotActualChecksum>$([System.String]::Copy('$(_CopilotArchiveHashValue)').ToUpperInvariant()) + <_CopilotChecksumMismatch Condition="'$(_CopilotExpectedChecksum)' != '$(_CopilotActualChecksum)'">true + + + @@ -117,23 +153,26 @@ <_TarCommand Condition="'$(_TarCommand)' == ''">tar + Condition="'$(_CopilotRuntimeBundleMissing)' == 'true'" /> - + + + - + <_CopilotCacheDir Condition="'$(_CopilotCacheDir)' == ''">$(IntermediateOutputPath)copilot-cli\$(CopilotCliVersion)\$(_CopilotPlatform) - <_CopilotCliBinaryPath Condition="'$(_CopilotCliBinaryPath)' == ''">$(_CopilotCacheDir)\$(_CopilotBinary) + <_CopilotCliBinaryPath Condition="'$(_CopilotCliBinaryPath)' == ''">$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\$(_CopilotRuntimeWrapper) <_CopilotOutputDir>$(OutDir)runtimes\$(_CopilotRid)\native <_CopilotRuntimeNodePath>$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\runtime.node <_CopilotRuntimeWrapperPath Condition="'$(_CopilotRuntimeWrapperPath)' == ''">$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\$(_CopilotRuntimeWrapper) @@ -153,7 +192,7 @@ <_CopilotRuntimeRootAsset Include="$(_CopilotCacheDir)\**\*" - Exclude="$(_CopilotCacheDir)\app.js;$(_CopilotCacheDir)\assets\**\*;$(_CopilotCacheDir)\changelog.json;$(_CopilotCacheDir)\copilot;$(_CopilotCacheDir)\copilot.exe;$(_CopilotCacheDir)\copilot.tgz;$(_CopilotCacheDir)\foundry-local-sdk\**\*;$(_CopilotCacheDir)\index.js;$(_CopilotCacheDir)\LICENSE.md;$(_CopilotCacheDir)\napi-oop-runtime\**\*;$(_CopilotCacheDir)\npm-loader.js;$(_CopilotCacheDir)\package.json;$(_CopilotCacheDir)\prebuilds\**\*;$(_CopilotCacheDir)\pvrecorder\**\*;$(_CopilotCacheDir)\queries\**\*;$(_CopilotCacheDir)\README.md;$(_CopilotCacheDir)\sea-loader.js;$(_CopilotCacheDir)\tree-sitter*.wasm;$(_CopilotCacheDir)\voice-*.js;$(_CopilotCacheDir)\webview\**\*" /> + Exclude="$(_CopilotCacheDir)\.copilot-runtime-complete;$(_CopilotCacheDir)\app.js;$(_CopilotCacheDir)\assets\**\*;$(_CopilotCacheDir)\changelog.json;$(_CopilotCacheDir)\copilot;$(_CopilotCacheDir)\copilot.exe;$(_CopilotCacheDir)\copilot.tgz;$(_CopilotCacheDir)\foundry-local-sdk\**\*;$(_CopilotCacheDir)\index.js;$(_CopilotCacheDir)\LICENSE.md;$(_CopilotCacheDir)\napi-oop-runtime\**\*;$(_CopilotCacheDir)\npm-loader.js;$(_CopilotCacheDir)\package.json;$(_CopilotCacheDir)\prebuilds\**\*;$(_CopilotCacheDir)\pvrecorder\**\*;$(_CopilotCacheDir)\queries\**\*;$(_CopilotCacheDir)\README.md;$(_CopilotCacheDir)\sea-loader.js;$(_CopilotCacheDir)\SHA256SUMS.txt;$(_CopilotCacheDir)\tree-sitter*.wasm;$(_CopilotCacheDir)\voice-*.js;$(_CopilotCacheDir)\webview\**\*" /> <_CopilotRuntimePrebuildAsset Include="$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\**\*" Exclude="$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\cli-native.node;$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\mediaremote-adapter\**\*;$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\copilot-runtime-bin*" /> @@ -176,12 +215,12 @@ - <_CopilotCacheDir Condition="'$(_CopilotCacheDir)' == ''">$(IntermediateOutputPath)copilot-cli\$(CopilotCliVersion)\$(_CopilotPlatform) - <_CopilotCliBinaryPath Condition="'$(_CopilotCliBinaryPath)' == ''">$(_CopilotCacheDir)\$(_CopilotBinary) + <_CopilotCliBinaryPath Condition="'$(_CopilotCliBinaryPath)' == ''">$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\$(_CopilotRuntimeWrapper) <_CopilotRuntimeNodePath>$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\runtime.node <_CopilotRuntimeWrapperPath Condition="'$(_CopilotRuntimeWrapperPath)' == ''">$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\$(_CopilotRuntimeWrapper) <_CopilotExplicitCliMarker>$(_CopilotCacheDir)\.copilot-explicit-cli @@ -193,7 +232,7 @@ Condition="'$(CopilotCliBinaryPath)' != ''" /> <_CopilotRuntimeRootAsset Include="$(_CopilotCacheDir)\**\*" - Exclude="$(_CopilotCacheDir)\app.js;$(_CopilotCacheDir)\assets\**\*;$(_CopilotCacheDir)\changelog.json;$(_CopilotCacheDir)\copilot;$(_CopilotCacheDir)\copilot.exe;$(_CopilotCacheDir)\copilot.tgz;$(_CopilotCacheDir)\foundry-local-sdk\**\*;$(_CopilotCacheDir)\index.js;$(_CopilotCacheDir)\LICENSE.md;$(_CopilotCacheDir)\napi-oop-runtime\**\*;$(_CopilotCacheDir)\npm-loader.js;$(_CopilotCacheDir)\package.json;$(_CopilotCacheDir)\prebuilds\**\*;$(_CopilotCacheDir)\pvrecorder\**\*;$(_CopilotCacheDir)\queries\**\*;$(_CopilotCacheDir)\README.md;$(_CopilotCacheDir)\sea-loader.js;$(_CopilotCacheDir)\tree-sitter*.wasm;$(_CopilotCacheDir)\voice-*.js;$(_CopilotCacheDir)\webview\**\*" + Exclude="$(_CopilotCacheDir)\.copilot-runtime-complete;$(_CopilotCacheDir)\app.js;$(_CopilotCacheDir)\assets\**\*;$(_CopilotCacheDir)\changelog.json;$(_CopilotCacheDir)\copilot;$(_CopilotCacheDir)\copilot.exe;$(_CopilotCacheDir)\copilot.tgz;$(_CopilotCacheDir)\foundry-local-sdk\**\*;$(_CopilotCacheDir)\index.js;$(_CopilotCacheDir)\LICENSE.md;$(_CopilotCacheDir)\napi-oop-runtime\**\*;$(_CopilotCacheDir)\npm-loader.js;$(_CopilotCacheDir)\package.json;$(_CopilotCacheDir)\prebuilds\**\*;$(_CopilotCacheDir)\pvrecorder\**\*;$(_CopilotCacheDir)\queries\**\*;$(_CopilotCacheDir)\README.md;$(_CopilotCacheDir)\sea-loader.js;$(_CopilotCacheDir)\SHA256SUMS.txt;$(_CopilotCacheDir)\tree-sitter*.wasm;$(_CopilotCacheDir)\voice-*.js;$(_CopilotCacheDir)\webview\**\*" Condition="Exists('$(_CopilotRuntimeWrapperPath)') And Exists('$(_CopilotRuntimeNodePath)')" /> <_CopilotRuntimePrebuildAsset Include="$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\**\*" Exclude="$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\cli-native.node;$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\mediaremote-adapter\**\*;$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\copilot-runtime-bin*" diff --git a/dotnet/test/Unit/MSBuildTargetsTests.cs b/dotnet/test/Unit/MSBuildTargetsTests.cs index 0dd3073956..a6e3bb5660 100644 --- a/dotnet/test/Unit/MSBuildTargetsTests.cs +++ b/dotnet/test/Unit/MSBuildTargetsTests.cs @@ -2,8 +2,12 @@ * Copyright (c) Microsoft Corporation. All rights reserved. *--------------------------------------------------------------------------------------------*/ +using System.Collections.Concurrent; using System.Diagnostics; +using System.Net; +using System.Net.Sockets; using System.Runtime.CompilerServices; +using System.Security.Cryptography; using System.Text; using Xunit; @@ -16,11 +20,7 @@ namespace GitHub.Copilot.Test.Unit; /// a subprocess so we exercise real MSBuild evaluation. /// /// -/// These tests deliberately do not exercise the network-bound default download path; they -/// pin a fake CopilotCliVersion and supply a fake CLI binary via -/// CopilotCliBinaryPath. That is sufficient to cover the regression in issue -/// #921 ("preinstalled CLI is ignored and copy/register are skipped when -/// CopilotSkipCliDownload=true"). +/// Download tests use a loopback release server; they never access the default GitHub URL. /// public class MSBuildTargetsTests { @@ -28,6 +28,9 @@ public class MSBuildTargetsTests private static readonly string BinaryName = OperatingSystem.IsWindows() ? "copilot.exe" : "copilot"; + private static readonly string RuntimeWrapperName = + OperatingSystem.IsWindows() ? "copilot-runtime.exe" : "copilot-runtime"; + [Fact] public async Task PreinstalledCliBinaryPath_IsHonored_DownloadSkipped_AndCopiedToOutput() { @@ -106,15 +109,107 @@ public async Task PreinstalledCliBinaryPath_WithSkipCliDownload_StillCopiesToOut Assert.True(File.Exists(sandbox.ExpectedOutputBinary()), result.FailureMessage()); } + [Fact] + public async Task ReleaseAsset_IsDownloadedVerifiedExtractedAndCached() + { + using var sandbox = MSBuildSandbox.Create(); + var archive = sandbox.CreateReleaseArchive("release-runtime-wrapper"); + var assetName = $"github-copilot-0.0.0-test-{GetReleasePlatform()}.tgz"; + var assetPath = $"/v0.0.0-test/{assetName}"; + var checksumsPath = "/v0.0.0-test/SHA256SUMS.txt"; + var checksum = ComputeSha256(archive); + using var server = new ReleaseServer(new Dictionary + { + [checksumsPath] = Encoding.UTF8.GetBytes($"{checksum} {assetName}\n"), + [assetPath] = archive, + }); + + var properties = new Dictionary + { + ["CopilotCliReleaseBaseUrl"] = server.BaseUrl, + }; + var firstBuild = await sandbox.BuildAsync(properties); + + Assert.True(firstBuild.Succeeded, firstBuild.FailureMessage()); + Assert.Equal("release-runtime-wrapper", File.ReadAllText(sandbox.ExpectedOutputBinary())); + Assert.Equal("release-runtime-wrapper", File.ReadAllText(sandbox.ExpectedRuntimeAsset(RuntimeWrapperName))); + Assert.Equal("runtime", File.ReadAllText(sandbox.ExpectedRuntimeAsset("runtime.node"))); + Assert.True(File.Exists(sandbox.ExpectedCacheAsset(".copilot-runtime-complete"))); + Assert.False(File.Exists(sandbox.ExpectedRuntimeAsset(".copilot-runtime-complete"))); + Assert.Equal(1, server.RequestPaths.Count(path => path == checksumsPath)); + Assert.Equal(1, server.RequestPaths.Count(path => path == assetPath)); + Assert.False(File.Exists(sandbox.ExpectedRuntimeAsset("SHA256SUMS.txt"))); + + var secondBuild = await sandbox.BuildAsync(properties); + + Assert.True(secondBuild.Succeeded, secondBuild.FailureMessage()); + Assert.Equal(2, server.RequestPaths.Count); + } + + [Fact] + public async Task IncompleteCache_WithRuntimePairButNoMarker_IsReacquired() + { + using var sandbox = MSBuildSandbox.Create(); + sandbox.WriteRuntimeCacheAsset("prebuilds", GetReleasePlatform(), RuntimeWrapperName, "partial-wrapper"); + sandbox.WriteRuntimeCacheAsset("prebuilds", GetReleasePlatform(), "runtime.node", "partial-runtime"); + sandbox.WriteRuntimeCacheAsset("definitions", "stale.json", "stale"); + var archive = sandbox.CreateReleaseArchive("complete-wrapper"); + var assetName = $"github-copilot-0.0.0-test-{GetReleasePlatform()}.tgz"; + var assetPath = $"/v0.0.0-test/{assetName}"; + var checksumsPath = "/v0.0.0-test/SHA256SUMS.txt"; + using var server = new ReleaseServer(new Dictionary + { + [checksumsPath] = Encoding.UTF8.GetBytes($"{ComputeSha256(archive)} {assetName}\n"), + [assetPath] = archive, + }); + + var result = await sandbox.BuildAsync(new Dictionary + { + ["CopilotCliReleaseBaseUrl"] = server.BaseUrl, + }); + + Assert.True(result.Succeeded, result.FailureMessage()); + Assert.Equal(1, server.RequestPaths.Count(path => path == checksumsPath)); + Assert.Equal(1, server.RequestPaths.Count(path => path == assetPath)); + Assert.Equal("complete-wrapper", File.ReadAllText(sandbox.ExpectedRuntimeAsset(RuntimeWrapperName))); + Assert.Equal("runtime", File.ReadAllText(sandbox.ExpectedRuntimeAsset("runtime.node"))); + Assert.True(File.Exists(sandbox.ExpectedCacheAsset(".copilot-runtime-complete"))); + Assert.False(File.Exists(sandbox.ExpectedCacheAsset("definitions", "stale.json"))); + Assert.False(File.Exists(sandbox.ExpectedRuntimeAsset("definitions", "stale.json"))); + } + + [Fact] + public async Task ReleaseAsset_WithChecksumMismatch_FailsBeforeExtraction() + { + using var sandbox = MSBuildSandbox.Create(); + var archive = Encoding.UTF8.GetBytes("not the expected archive"); + var assetName = $"github-copilot-0.0.0-test-{GetReleasePlatform()}.tgz"; + using var server = new ReleaseServer(new Dictionary + { + ["/v0.0.0-test/SHA256SUMS.txt"] = + Encoding.UTF8.GetBytes($"{new string('0', 64)} *{assetName}\n"), + [$"/v0.0.0-test/{assetName}"] = archive, + }); + + var result = await sandbox.BuildAsync(new Dictionary + { + ["CopilotCliReleaseBaseUrl"] = server.BaseUrl, + }); + + Assert.False(result.Succeeded, "Build should fail when the release checksum does not match."); + Assert.Contains($"Checksum mismatch for {assetName}", result.StandardOutput, StringComparison.Ordinal); + Assert.False(File.Exists(sandbox.ExpectedOutputBinary())); + } + [Fact] public async Task RuntimePackageAssets_AreFilteredAndCopiedToOutput() { using var sandbox = MSBuildSandbox.Create(); var preinstalled = sandbox.WritePreinstalledBinary("fake-cli-contents"); - sandbox.WriteRuntimeCacheAsset("prebuilds", GetNpmPlatform(), "runtime.node", "runtime"); - sandbox.WriteRuntimeCacheAsset("prebuilds", GetNpmPlatform(), - OperatingSystem.IsWindows() ? "copilot-runtime.exe" : "copilot-runtime", "wrapper"); - sandbox.WriteRuntimeCacheAsset("ripgrep", "bin", GetNpmPlatform(), "rg", "ripgrep"); + sandbox.WriteRuntimeCacheAsset("prebuilds", GetReleasePlatform(), "runtime.node", "runtime"); + sandbox.WriteRuntimeCacheAsset("prebuilds", GetReleasePlatform(), + RuntimeWrapperName, "wrapper"); + sandbox.WriteRuntimeCacheAsset("ripgrep", "bin", GetReleasePlatform(), "rg", "ripgrep"); sandbox.WriteRuntimeCacheAsset("definitions", "future.json", "{}"); sandbox.WriteRuntimeCacheAsset("copilot-sdk", "extension.js", "extension"); sandbox.WriteRuntimeCacheAsset("preloads", "extension_bootstrap.mjs", "preload"); @@ -130,7 +225,7 @@ public async Task RuntimePackageAssets_AreFilteredAndCopiedToOutput() }); Assert.True(result.Succeeded, result.FailureMessage()); - Assert.Equal("ripgrep", File.ReadAllText(sandbox.ExpectedRuntimeAsset("ripgrep", "bin", GetNpmPlatform(), "rg"))); + Assert.Equal("ripgrep", File.ReadAllText(sandbox.ExpectedRuntimeAsset("ripgrep", "bin", GetReleasePlatform(), "rg"))); Assert.Equal("{}", File.ReadAllText(sandbox.ExpectedRuntimeAsset("definitions", "future.json"))); Assert.Equal("extension", File.ReadAllText(sandbox.ExpectedRuntimeAsset("copilot-sdk", "extension.js"))); Assert.Equal("preload", File.ReadAllText(sandbox.ExpectedRuntimeAsset("preloads", "extension_bootstrap.mjs"))); @@ -186,7 +281,7 @@ private static string FindTargetsFile([CallerFilePath] string? thisFile = null) "Could not locate GitHub.Copilot.SDK.targets relative to test assembly or source file."); } - private static string GetNpmPlatform() + private static string GetReleasePlatform() { var arch = System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture == System.Runtime.InteropServices.Architecture.Arm64 @@ -197,6 +292,16 @@ private static string GetNpmPlatform() return $"linux-{arch}"; } + private static string ComputeSha256(byte[] contents) + { +#if NETFRAMEWORK + using var sha256 = SHA256.Create(); + return BitConverter.ToString(sha256.ComputeHash(contents)).Replace("-", "").ToLowerInvariant(); +#else + return Convert.ToHexString(SHA256.HashData(contents)).ToLowerInvariant(); +#endif + } + /// /// A throwaway directory containing a minimal csproj that imports the SDK targets /// file. Disposing removes the directory tree. @@ -244,6 +349,36 @@ public string WritePreinstalledBinary(string contents, string? fileName = null) return path; } + public byte[] CreateReleaseArchive(string runtimeWrapperContents) + { + var sourceDir = Path.Combine(ProjectDir, "release-source"); + var packageDir = Path.Combine(sourceDir, "package"); + var prebuildDir = Path.Combine(packageDir, "prebuilds", GetReleasePlatform()); + Directory.CreateDirectory(prebuildDir); + File.WriteAllText(Path.Combine(prebuildDir, "runtime.node"), "runtime"); + File.WriteAllText(Path.Combine(prebuildDir, RuntimeWrapperName), runtimeWrapperContents); + + var archivePath = Path.Combine(ProjectDir, "release-asset.tgz"); + var tarPath = OperatingSystem.IsWindows() + ? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Windows), "System32", "tar.exe") + : "tar"; + var startInfo = new ProcessStartInfo(tarPath) + { + Arguments = $"-czf \"{archivePath}\" -C \"{sourceDir}\" package", + RedirectStandardError = true, + UseShellExecute = false, + }; + using var process = Process.Start(startInfo) ?? + throw new InvalidOperationException("Failed to start tar while creating a release test asset."); + var standardError = process.StandardError.ReadToEnd(); + process.WaitForExit(); + if (process.ExitCode != 0) + { + throw new InvalidOperationException($"tar failed while creating a release test asset: {standardError}"); + } + return File.ReadAllBytes(archivePath); + } + public string ExpectedOutputBinary() { var rid = GetPortableRid(); @@ -253,14 +388,20 @@ public string ExpectedOutputBinary() public void WriteRuntimeCacheAsset(params string[] pathAndContents) { var pathParts = pathAndContents.Take(pathAndContents.Length - 1).ToArray(); + var path = ExpectedCacheAsset(pathParts); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, pathAndContents[^1]); + } + + public string ExpectedCacheAsset(params string[] pathParts) + { var path = Path.Combine(ProjectDir, "obj", "Debug", "net8.0", "copilot-cli", "0.0.0-test", - GetNpmPlatform()); + GetReleasePlatform()); foreach (var part in pathParts) { path = Path.Combine(path, part); } - Directory.CreateDirectory(Path.GetDirectoryName(path)!); - File.WriteAllText(path, pathAndContents[^1]); + return path; } public string ExpectedRuntimeAsset(params string[] pathParts) @@ -372,6 +513,92 @@ private static string GetPortableRid() } } + private sealed class ReleaseServer : IDisposable + { + private readonly IReadOnlyDictionary _responses; + private readonly TcpListener _listener = new(IPAddress.Loopback, 0); + private readonly CancellationTokenSource _cancellation = new(); + private readonly Task _serverTask; + + public ReleaseServer(IReadOnlyDictionary responses) + { + _responses = responses; + _listener.Start(); + var endpoint = (IPEndPoint)_listener.LocalEndpoint; + BaseUrl = $"http://127.0.0.1:{endpoint.Port}"; + _serverTask = ServeAsync(); + } + + public string BaseUrl { get; } + + public ConcurrentQueue RequestPaths { get; } = new(); + + public void Dispose() + { + _cancellation.Cancel(); + _listener.Stop(); + try { _serverTask.GetAwaiter().GetResult(); } + catch (OperationCanceledException) { } + _cancellation.Dispose(); + } + + private async Task ServeAsync() + { + while (!_cancellation.IsCancellationRequested) + { + TcpClient client; + try + { + client = await _listener.AcceptTcpClientAsync(); + } + catch (ObjectDisposedException) when (_cancellation.IsCancellationRequested) + { + break; + } + catch (SocketException) when (_cancellation.IsCancellationRequested) + { + break; + } + await RespondAsync(client); + } + } + + private async Task RespondAsync(TcpClient client) + { + using (client) + { + var stream = client.GetStream(); + using var reader = new StreamReader(stream, Encoding.ASCII, false, 1024, leaveOpen: true); + var requestLine = await reader.ReadLineAsync(); + string? header; + do + { + header = await reader.ReadLineAsync(); + } + while (!string.IsNullOrEmpty(header)); + + var path = requestLine?.Split(' ', StringSplitOptions.RemoveEmptyEntries).ElementAtOrDefault(1) ?? ""; + RequestPaths.Enqueue(path); + var found = _responses.TryGetValue(path, out var body); + body ??= Encoding.UTF8.GetBytes("Not found"); + var status = found ? "200 OK" : "404 Not Found"; + var responseHeaders = Encoding.ASCII.GetBytes( + $"HTTP/1.1 {status}\r\nContent-Length: {body.Length}\r\nConnection: close\r\n\r\n"); + await WriteBytesAsync(stream, responseHeaders); + await WriteBytesAsync(stream, body); + } + } + + private static Task WriteBytesAsync(Stream stream, byte[] contents) + { +#if NETFRAMEWORK + return stream.WriteAsync(contents, 0, contents.Length); +#else + return stream.WriteAsync(contents).AsTask(); +#endif + } + } + private sealed record BuildResult(int ExitCode, string StandardOutput, string StandardError, string CommandLine) { public bool Succeeded => ExitCode == 0; diff --git a/go/README.md b/go/README.md index 801ad8556c..29ecc69210 100644 --- a/go/README.md +++ b/go/README.md @@ -101,6 +101,9 @@ tool name is `-`. For `AvailableTools` and The SDK supports bundling, using Go's `embed` package, the Copilot CLI binary within your application's distribution. This allows you to bundle a specific CLI version and avoid external dependencies on the user's system. +The bundler downloads the matching `github-copilot--.tgz` +asset from the `github/copilot-cli` release and verifies it against that +release's `SHA256SUMS.txt`. Follow these steps to embed the CLI: diff --git a/go/cmd/bundler/main.go b/go/cmd/bundler/main.go index 7ee078eea0..41b5863347 100644 --- a/go/cmd/bundler/main.go +++ b/go/cmd/bundler/main.go @@ -30,34 +30,35 @@ import ( "regexp" "runtime" "strings" + "time" "github.com/klauspost/compress/zstd" ) const ( // Keep these URLs centralized so reviewers can verify all outbound calls in one place. - sdkModule = "github.com/github/copilot-sdk/go" - packageJSONURLFmt = "https://raw.githubusercontent.com/github/copilot-sdk/%s/nodejs/package.json" - packageLockURLFmt = "https://raw.githubusercontent.com/github/copilot-sdk/%s/nodejs/package-lock.json" - tarballURLFmt = "https://registry.npmjs.org/@github/copilot-%s/-/copilot-%s-%s.tgz" - licenseTarballFmt = "https://registry.npmjs.org/@github/copilot/-/copilot-%s.tgz" - defaultPackageName = "main" + sdkModule = "github.com/github/copilot-sdk/go" + packageJSONURLFmt = "https://raw.githubusercontent.com/github/copilot-sdk/%s/nodejs/package.json" + packageLockURLFmt = "https://raw.githubusercontent.com/github/copilot-sdk/%s/nodejs/package-lock.json" + defaultCLIDownloadBaseURL = "https://github.com/github/copilot-cli/releases/download" + cliDownloadBaseURLEnvironment = "COPILOT_CLI_DOWNLOAD_BASE_URL" + defaultPackageName = "main" ) -// Platform info: npm package suffix, binary name +// Platform info: release asset platform suffix, binary name type platformInfo struct { - npmPlatform string - binaryName string + runtimePlatform string + binaryName string } -// Map from GOOS/GOARCH to npm platform info +// Map from GOOS/GOARCH to release asset platform info. var platforms = map[string]platformInfo{ - "linux/amd64": {npmPlatform: "linux-x64", binaryName: "copilot"}, - "linux/arm64": {npmPlatform: "linux-arm64", binaryName: "copilot"}, - "darwin/amd64": {npmPlatform: "darwin-x64", binaryName: "copilot"}, - "darwin/arm64": {npmPlatform: "darwin-arm64", binaryName: "copilot"}, - "windows/amd64": {npmPlatform: "win32-x64", binaryName: "copilot.exe"}, - "windows/arm64": {npmPlatform: "win32-arm64", binaryName: "copilot.exe"}, + "linux/amd64": {runtimePlatform: "linux-x64", binaryName: "copilot"}, + "linux/arm64": {runtimePlatform: "linux-arm64", binaryName: "copilot"}, + "darwin/amd64": {runtimePlatform: "darwin-x64", binaryName: "copilot"}, + "darwin/arm64": {runtimePlatform: "darwin-arm64", binaryName: "copilot"}, + "windows/amd64": {runtimePlatform: "win32-x64", binaryName: "copilot.exe"}, + "windows/arm64": {runtimePlatform: "win32-arm64", binaryName: "copilot.exe"}, } // main is the CLI entry point. @@ -70,7 +71,7 @@ func main() { // Resolve version first so the default output name can include it. version := resolveCLIVersion(*cliVersion) - // Resolve platform once to validate input and get the npm package mapping. + // Resolve platform once to validate input and get the release asset mapping. goos, goarch, info, err := resolvePlatform(*platform) if err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) @@ -101,7 +102,7 @@ func main() { fmt.Printf("Building bundle for %s (CLI version %s)\n", *platform, version) - bundle, err := buildBundle(info, version, outputPath, goos) + bundle, err := buildBundle(info, version, outputPath, goos, true) if err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) os.Exit(1) @@ -110,8 +111,8 @@ func main() { var muslBundle bundleArtifacts if goos == "linux" { muslInfo := platformInfo{ - npmPlatform: strings.Replace(info.npmPlatform, "linux-", "linuxmusl-", 1), - binaryName: info.binaryName, + runtimePlatform: strings.Replace(info.runtimePlatform, "linux-", "linuxmusl-", 1), + binaryName: info.binaryName, } muslOutputPath := filepath.Join(*output, defaultOutputFileName(version, "linuxmusl", goarch, info.binaryName)) muslBundle, err = buildBundle( @@ -119,17 +120,13 @@ func main() { version, muslOutputPath, goos, + false, ) if err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) os.Exit(1) } } - if err := downloadCLILicense(version, outputPath); err != nil { - fmt.Fprintf(os.Stderr, "Error: failed to download CLI license: %v\n", err) - os.Exit(1) - } - // Generate the Go file with embed directive if err := generateGoFile( goos, @@ -397,19 +394,23 @@ type bundleArtifacts struct { assetsHash []byte } -// buildBundle downloads the CLI and native runtime artifacts from one platform package. -func buildBundle(info platformInfo, cliVersion, outputPath, goos string) (bundleArtifacts, error) { +// buildBundle downloads the CLI and native runtime artifacts from one release package. +func buildBundle(info platformInfo, cliVersion, outputPath, goos string, includeLicense bool) (bundleArtifacts, error) { outputDir := filepath.Dir(outputPath) if outputDir == "" { outputDir = "." } - runtimeArtifactPath := filepath.Join(outputDir, runtimeLibArtifactName(cliVersion, info.npmPlatform, goos)) - wrapperArtifactPath := filepath.Join(outputDir, runtimeWrapperArtifactName(cliVersion, info.npmPlatform, info.binaryName)) - assetsArtifactPath := filepath.Join(outputDir, runtimeAssetsArtifactName(cliVersion, info.npmPlatform)) + runtimeArtifactPath := filepath.Join(outputDir, runtimeLibArtifactName(cliVersion, info.runtimePlatform, goos)) + wrapperArtifactPath := filepath.Join(outputDir, runtimeWrapperArtifactName(cliVersion, info.runtimePlatform, info.binaryName)) + assetsArtifactPath := filepath.Join(outputDir, runtimeAssetsArtifactName(cliVersion, info.runtimePlatform)) + requiredPaths := []string{outputPath, runtimeArtifactPath, wrapperArtifactPath, assetsArtifactPath} + if includeLicense { + requiredPaths = append(requiredPaths, licensePathForOutput(outputPath)) + } - if filesExist(outputPath, runtimeArtifactPath, wrapperArtifactPath, assetsArtifactPath) { + if filesExist(requiredPaths...) { // Idempotent output avoids re-downloading in CI or local rebuilds. - fmt.Printf("Output runtime bundle for %s already exists, skipping download\n", info.npmPlatform) + fmt.Printf("Output runtime bundle for %s already exists, skipping download\n", info.runtimePlatform) binaryHash, err := sha256FileFromCompressed(outputPath) if err != nil { return bundleArtifacts{}, fmt.Errorf("failed to hash existing output: %w", err) @@ -436,7 +437,7 @@ func buildBundle(info platformInfo, cliVersion, outputPath, goos string) (bundle } defer os.RemoveAll(tempDir) - binaryPath, tarballPath, err := downloadCLIBinary(info.npmPlatform, info.binaryName, cliVersion, tempDir) + binaryPath, tarballPath, err := downloadCLIBinary(info.runtimePlatform, info.binaryName, cliVersion, tempDir) if err != nil { return bundleArtifacts{}, fmt.Errorf("failed to download CLI binary: %w", err) } @@ -446,6 +447,11 @@ func buildBundle(info platformInfo, cliVersion, outputPath, goos string) (bundle return bundleArtifacts{}, fmt.Errorf("failed to create output directory: %w", err) } } + if includeLicense { + if err := extractCLILicense(tarballPath, outputPath); err != nil { + return bundleArtifacts{}, fmt.Errorf("failed to extract CLI license: %w", err) + } + } binaryHash, err := sha256File(binaryPath) if err != nil { @@ -459,10 +465,10 @@ func buildBundle(info platformInfo, cliVersion, outputPath, goos string) (bundle if err := extractFileFromTarball( tarballPath, tempDir, - "package/prebuilds/"+info.npmPlatform+"/runtime.node", + "package/prebuilds/"+info.runtimePlatform+"/runtime.node", "runtime.node", ); err != nil { - return bundleArtifacts{}, fmt.Errorf("runtime package is missing prebuilds/%s/runtime.node: %w", info.npmPlatform, err) + return bundleArtifacts{}, fmt.Errorf("runtime package is missing prebuilds/%s/runtime.node: %w", info.runtimePlatform, err) } runtimeHash, err := sha256File(rawLibPath) if err != nil { @@ -477,10 +483,10 @@ func buildBundle(info platformInfo, cliVersion, outputPath, goos string) (bundle if err := extractFileFromTarball( tarballPath, tempDir, - "package/prebuilds/"+info.npmPlatform+"/"+wrapperName, + "package/prebuilds/"+info.runtimePlatform+"/"+wrapperName, wrapperName, ); err != nil { - return bundleArtifacts{}, fmt.Errorf("runtime package is missing prebuilds/%s/%s: %w", info.npmPlatform, wrapperName, err) + return bundleArtifacts{}, fmt.Errorf("runtime package is missing prebuilds/%s/%s: %w", info.runtimePlatform, wrapperName, err) } wrapperHash, err := sha256File(rawWrapperPath) if err != nil { @@ -514,16 +520,16 @@ func filesExist(paths ...string) bool { } // runtimeLibArtifactName builds the compressed runtime-library artifact filename. -func runtimeLibArtifactName(version, npmPlatform, goos string) string { - return fmt.Sprintf("zcopilotruntime_%s_%s.%s.zst", version, npmPlatform, runtimeLibExt(goos)) +func runtimeLibArtifactName(version, runtimePlatform, goos string) string { + return fmt.Sprintf("zcopilotruntime_%s_%s.%s.zst", version, runtimePlatform, runtimeLibExt(goos)) } -func runtimeWrapperArtifactName(version, npmPlatform, binaryName string) string { - return fmt.Sprintf("zcopilotruntimewrapper_%s_%s_%s.zst", version, npmPlatform, runtimeWrapperName(binaryName)) +func runtimeWrapperArtifactName(version, runtimePlatform, binaryName string) string { + return fmt.Sprintf("zcopilotruntimewrapper_%s_%s_%s.zst", version, runtimePlatform, runtimeWrapperName(binaryName)) } -func runtimeAssetsArtifactName(version, npmPlatform string) string { - return fmt.Sprintf("zcopilotruntimeassets_%s_%s.tgz", version, npmPlatform) +func runtimeAssetsArtifactName(version, runtimePlatform string) string { + return fmt.Sprintf("zcopilotruntimeassets_%s_%s.tgz", version, runtimePlatform) } func runtimeWrapperName(binaryName string) string { @@ -540,7 +546,7 @@ var hostlessExcludedTopLevel = map[string]bool{ "sea-loader.js": true, "webview": true, } -func hostlessRuntimePath(name, npmPlatform, wrapperName string) (string, bool) { +func hostlessRuntimePath(name, runtimePlatform, wrapperName string) (string, bool) { relative, ok := strings.CutPrefix(name, "package/") if !ok { return "", false @@ -561,7 +567,7 @@ func hostlessRuntimePath(name, npmPlatform, wrapperName string) (string, bool) { } } if topLevel == "prebuilds" { - if len(parts) < 3 || parts[1] != npmPlatform { + if len(parts) < 3 || parts[1] != runtimePlatform { return "", false } return strings.Join(parts[2:], "/"), true @@ -602,7 +608,7 @@ func createRuntimeAssetsArchive(tarballPath, outputPath string, info platformInf } destination, include := hostlessRuntimePath( header.Name, - info.npmPlatform, + info.runtimePlatform, runtimeWrapperName(info.binaryName), ) if !include { @@ -918,15 +924,82 @@ func mustDecodeBase64(s string) []byte { `, buildConstraint, pkgName, binaryName, licenseName, runtimeEmbed, muslEmbed, cliVersion, hashBase64, runtimeConfig, muslConfig, runtimeReader, muslReaders) } -// downloadCLIBinary downloads the npm tarball and extracts the CLI binary. It +var ( + releaseChecksumCache = map[string]map[string]string{} + releaseHTTPClient = &http.Client{Timeout: 10 * time.Minute} +) + +func cliDownloadBaseURL() string { + if override := strings.TrimRight(os.Getenv(cliDownloadBaseURLEnvironment), "/"); override != "" { + return override + } + return defaultCLIDownloadBaseURL +} + +func releaseAssetName(version, runtimePlatform string) string { + return fmt.Sprintf("github-copilot-%s-%s.tgz", version, runtimePlatform) +} + +func releaseDownloadURL(version, assetName string) string { + return fmt.Sprintf("%s/v%s/%s", cliDownloadBaseURL(), version, assetName) +} + +func parseReleaseChecksums(contents string) map[string]string { + checksums := make(map[string]string) + hashPattern := regexp.MustCompile(`^[0-9a-fA-F]{64}$`) + for _, line := range strings.Split(contents, "\n") { + fields := strings.Fields(line) + if len(fields) != 2 || !hashPattern.MatchString(fields[0]) { + continue + } + checksums[strings.TrimPrefix(fields[1], "*")] = strings.ToLower(fields[0]) + } + return checksums +} + +func getReleaseChecksum(version, assetName string) (string, error) { + baseURL := cliDownloadBaseURL() + cacheKey := baseURL + "\x00" + version + checksums, ok := releaseChecksumCache[cacheKey] + if !ok { + checksumsURL := fmt.Sprintf("%s/v%s/SHA256SUMS.txt", baseURL, version) + fmt.Printf("Downloading checksums from %s...\n", checksumsURL) + resp, err := releaseHTTPClient.Get(checksumsURL) + if err != nil { + return "", fmt.Errorf("failed to download checksums: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to download checksums: %s", resp.Status) + } + contents, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read checksums: %w", err) + } + checksums = parseReleaseChecksums(string(contents)) + releaseChecksumCache[cacheKey] = checksums + } + checksum, ok := checksums[assetName] + if !ok { + return "", fmt.Errorf("SHA256SUMS.txt does not contain %s", assetName) + } + return checksum, nil +} + +// downloadCLIBinary downloads the verified release package and extracts the CLI binary. It // returns the extracted binary path and the downloaded tarball path (retained so // callers can extract additional files, such as the runtime library). -func downloadCLIBinary(npmPlatform, binaryName, cliVersion, destDir string) (string, string, error) { - tarballURL := fmt.Sprintf(tarballURLFmt, npmPlatform, npmPlatform, cliVersion) +func downloadCLIBinary(runtimePlatform, binaryName, cliVersion, destDir string) (string, string, error) { + assetName := releaseAssetName(cliVersion, runtimePlatform) + expectedChecksum, err := getReleaseChecksum(cliVersion, assetName) + if err != nil { + return "", "", err + } + tarballURL := releaseDownloadURL(cliVersion, assetName) fmt.Printf("Downloading from %s...\n", tarballURL) - resp, err := http.Get(tarballURL) + resp, err := releaseHTTPClient.Get(tarballURL) if err != nil { return "", "", fmt.Errorf("failed to download: %w", err) } @@ -937,24 +1010,43 @@ func downloadCLIBinary(npmPlatform, binaryName, cliVersion, destDir string) (str } // Save tarball to temp file - tarballPath := filepath.Join(destDir, fmt.Sprintf("copilot-%s-%s.tgz", npmPlatform, cliVersion)) + tarballPath := filepath.Join(destDir, assetName) tarballFile, err := os.Create(tarballPath) if err != nil { return "", "", fmt.Errorf("failed to create tarball file: %w", err) } - if _, err := io.Copy(tarballFile, resp.Body); err != nil { + hasher := sha256.New() + if _, err := io.Copy(io.MultiWriter(tarballFile, hasher), resp.Body); err != nil { tarballFile.Close() return "", "", fmt.Errorf("failed to save tarball: %w", err) } if err := tarballFile.Close(); err != nil { return "", "", fmt.Errorf("failed to close tarball file: %w", err) } + actualChecksum := fmt.Sprintf("%x", hasher.Sum(nil)) + if actualChecksum != expectedChecksum { + return "", "", fmt.Errorf( + "checksum mismatch for %s: expected %s, got %s", + assetName, + expectedChecksum, + actualChecksum, + ) + } - // Extract only the CLI binary to avoid unpacking the full package tree. + // The SDK release package intentionally omits the legacy SEA binary. Preserve + // embeddedcli.Path compatibility by installing the runtime wrapper under the + // historical copilot[.exe] name; the normal client path uses the adjacent + // wrapper/runtime.node pair directly. binaryPath := filepath.Join(destDir, binaryName) - if err := extractFileFromTarball(tarballPath, destDir, "package/"+binaryName, binaryName); err != nil { - return "", "", fmt.Errorf("failed to extract binary: %w", err) + wrapperName := runtimeWrapperName(binaryName) + if err := extractFileFromTarball( + tarballPath, + destDir, + "package/prebuilds/"+runtimePlatform+"/"+wrapperName, + binaryName, + ); err != nil { + return "", "", fmt.Errorf("failed to extract runtime wrapper compatibility entrypoint: %w", err) } // Verify binary exists @@ -979,8 +1071,8 @@ func downloadCLIBinary(npmPlatform, binaryName, cliVersion, destDir string) (str return binaryPath, tarballPath, nil } -// downloadCLILicense downloads the @github/copilot package and writes its license next to outputPath. -func downloadCLILicense(cliVersion, outputPath string) error { +// extractCLILicense writes the license from the verified release package next to outputPath. +func extractCLILicense(tarballPath, outputPath string) error { outputDir := filepath.Dir(outputPath) if outputDir == "" { outputDir = "." @@ -990,18 +1082,13 @@ func downloadCLILicense(cliVersion, outputPath string) error { return nil } - licenseURL := fmt.Sprintf(licenseTarballFmt, cliVersion) - resp, err := http.Get(licenseURL) + source, err := os.Open(tarballPath) if err != nil { - return fmt.Errorf("failed to download license tarball: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("failed to download license tarball: %s", resp.Status) + return fmt.Errorf("failed to open release package: %w", err) } + defer source.Close() - gzReader, err := gzip.NewReader(resp.Body) + gzReader, err := gzip.NewReader(source) if err != nil { return fmt.Errorf("failed to create gzip reader: %w", err) } diff --git a/go/cmd/bundler/main_test.go b/go/cmd/bundler/main_test.go index 9c09559b66..7dd376d40b 100644 --- a/go/cmd/bundler/main_test.go +++ b/go/cmd/bundler/main_test.go @@ -4,9 +4,13 @@ import ( "archive/tar" "bytes" "compress/gzip" + "crypto/sha256" + "fmt" "go/parser" "go/token" "io" + "net/http" + "net/http/httptest" "os" "path/filepath" "strings" @@ -31,8 +35,8 @@ func TestCreateRuntimeAssetsArchiveRetainsUnknownAssetsAndFiltersCLIContent(t *t }) if err := createRuntimeAssetsArchive(source, output, platformInfo{ - npmPlatform: "linux-x64", - binaryName: "copilot", + runtimePlatform: "linux-x64", + binaryName: "copilot", }); err != nil { t.Fatal(err) } @@ -54,6 +58,93 @@ func TestCreateRuntimeAssetsArchiveRetainsUnknownAssetsAndFiltersCLIContent(t *t } } +func TestDownloadCLIBinaryUsesVerifiedReleasePackage(t *testing.T) { + dir := t.TempDir() + archivePath := filepath.Join(dir, "source.tgz") + writeTarGz(t, archivePath, map[string]string{ + "package/prebuilds/linux-x64/copilot-runtime": "runtime wrapper", + }) + archive, err := os.ReadFile(archivePath) + if err != nil { + t.Fatal(err) + } + checksum := fmt.Sprintf("%x", sha256.Sum256(archive)) + version := "1.2.3" + assetName := releaseAssetName(version, "linux-x64") + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/v1.2.3/SHA256SUMS.txt": + fmt.Fprintf(writer, "%s %s\n", checksum, assetName) + case "/v1.2.3/" + assetName: + writer.Write(archive) + default: + http.NotFound(writer, request) + } + })) + defer server.Close() + t.Setenv(cliDownloadBaseURLEnvironment, server.URL) + releaseChecksumCache = map[string]map[string]string{} + + binaryPath, downloadedArchive, err := downloadCLIBinary( + "linux-x64", + "copilot", + version, + t.TempDir(), + ) + if err != nil { + t.Fatal(err) + } + if got, err := os.ReadFile(binaryPath); err != nil || string(got) != "runtime wrapper" { + t.Fatalf("downloaded CLI = %q, %v", got, err) + } + if filepath.Base(downloadedArchive) != assetName { + t.Fatalf("downloaded archive = %q, want basename %q", downloadedArchive, assetName) + } +} + +func TestDownloadCLIBinaryRejectsChecksumMismatch(t *testing.T) { + dir := t.TempDir() + archivePath := filepath.Join(dir, "source.tgz") + writeTarGz(t, archivePath, map[string]string{ + "package/prebuilds/linux-x64/copilot-runtime": "runtime wrapper", + }) + archive, err := os.ReadFile(archivePath) + if err != nil { + t.Fatal(err) + } + version := "1.2.3" + assetName := releaseAssetName(version, "linux-x64") + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/v1.2.3/SHA256SUMS.txt": + fmt.Fprintf(writer, "%s %s\n", strings.Repeat("0", 64), assetName) + case "/v1.2.3/" + assetName: + _, _ = writer.Write(archive) + default: + http.NotFound(writer, request) + } + })) + defer server.Close() + t.Setenv(cliDownloadBaseURLEnvironment, server.URL) + releaseChecksumCache = map[string]map[string]string{} + + _, _, err = downloadCLIBinary("linux-x64", "copilot", version, t.TempDir()) + if err == nil || !strings.Contains(err.Error(), "checksum mismatch") { + t.Fatalf("downloadCLIBinary() error = %v, want checksum mismatch", err) + } +} + +func TestParseReleaseChecksums(t *testing.T) { + hash := strings.Repeat("a", 64) + checksums := parseReleaseChecksums( + "invalid\n" + + strings.ToUpper(hash) + " *github-copilot-1.2.3-linux-x64.tgz\n", + ) + if got := checksums["github-copilot-1.2.3-linux-x64.tgz"]; got != hash { + t.Fatalf("checksum = %q, want %q", got, hash) + } +} + func writeTarGz(t *testing.T, path string, files map[string]string) { t.Helper() var buffer bytes.Buffer diff --git a/go/internal/embeddedcli/embeddedcli.go b/go/internal/embeddedcli/embeddedcli.go index 2535cf5f20..2e3b8add16 100644 --- a/go/internal/embeddedcli/embeddedcli.go +++ b/go/internal/embeddedcli/embeddedcli.go @@ -26,7 +26,7 @@ import ( // when provided, is written next to the installed binary. // // RuntimeExecutable and RuntimeNode form the adjacent out-of-process runtime -// pair. RuntimeAssets is a filtered npm package archive containing auxiliary +// pair. RuntimeAssets is a filtered release package archive containing auxiliary // binaries and resources. RuntimeLib is the same cdylib bytes installed under // the natural platform name for the optional in-process transport. type Config struct { @@ -264,6 +264,13 @@ func installAt(installDir string) (string, error) { if !bytes.Equal(existingHash, config.CliHash) { return "", fmt.Errorf("existing binary hash mismatch") } + if config.RuntimeExecutable != nil { + path, err := installRuntimePair(installDir) + if err != nil { + return "", err + } + runtimePath = path + } if config.RuntimeLib != nil { libPath, err := installRuntimeLib(installDir) if err != nil { @@ -298,6 +305,14 @@ func installAt(installDir string) (string, error) { } } + if config.RuntimeExecutable != nil { + path, err := installRuntimePair(installDir) + if err != nil { + return "", err + } + runtimePath = path + } + // Install the native in-process runtime library (if bundled) next to the CLI. // Fail closed on any hash mismatch; never place unverified native code. if config.RuntimeLib != nil { @@ -305,11 +320,11 @@ func installAt(installDir string) (string, error) { if err != nil { return "", err } - if err := installRuntimeAssets(installDir); err != nil { - return "", err - } runtimeLibPath = libPath } + if err := installRuntimeAssets(installDir); err != nil { + return "", err + } return finalPath, nil } diff --git a/go/internal/embeddedcli/embeddedcli_test.go b/go/internal/embeddedcli/embeddedcli_test.go index 159b6e1505..a56dd8106e 100644 --- a/go/internal/embeddedcli/embeddedcli_test.go +++ b/go/internal/embeddedcli/embeddedcli_test.go @@ -259,6 +259,46 @@ func TestInstallAtWritesBinaryAndLicense(t *testing.T) { } } +func TestInstallAtInstallsRuntimePairAndAssetsWithoutRuntimeLib(t *testing.T) { + resetGlobals() + tempDir := t.TempDir() + wrapper := []byte("wrapper") + node := []byte("runtime") + assets := runtimeAssetsArchive(t, map[string]assetFixture{ + "definitions/future.json": {content: []byte("{}"), mode: 0644}, + }) + wrapperHash := sha256.Sum256(wrapper) + nodeHash := sha256.Sum256(node) + assetsHash := sha256.Sum256(assets) + Setup(Config{ + Cli: bytes.NewReader(wrapper), + CliHash: wrapperHash[:], + RuntimeExecutable: bytes.NewReader(wrapper), + RuntimeExecutableHash: wrapperHash[:], + RuntimeNode: bytes.NewReader(node), + RuntimeNodeHash: nodeHash[:], + RuntimeAssets: bytes.NewReader(assets), + RuntimeAssetsHash: assetsHash[:], + Version: "1.2.3", + Dir: tempDir, + }) + + path, err := installAt(tempDir) + if err != nil { + t.Fatal(err) + } + installDir := filepath.Dir(path) + if got, err := os.ReadFile(filepath.Join(installDir, runtimeExecutableName())); err != nil || !bytes.Equal(got, wrapper) { + t.Fatalf("runtime wrapper content=%q err=%v", got, err) + } + if got, err := os.ReadFile(filepath.Join(installDir, "runtime.node")); err != nil || !bytes.Equal(got, node) { + t.Fatalf("runtime.node content=%q err=%v", got, err) + } + if got, err := os.ReadFile(filepath.Join(installDir, "definitions", "future.json")); err != nil || string(got) != "{}" { + t.Fatalf("definition content=%q err=%v", got, err) + } +} + func TestInstallAtExistingBinaryHashMismatch(t *testing.T) { resetGlobals() tempDir := t.TempDir() diff --git a/java/copilot-native/pom.xml b/java/copilot-native/pom.xml index 0abe03ad18..4270d238d4 100644 --- a/java/copilot-native/pom.xml +++ b/java/copilot-native/pom.xml @@ -59,7 +59,7 @@ ${project.basedir}/.. - - ^1.0.83-5 true diff --git a/java/scripts/codegen/fetch-schemas.mjs b/java/scripts/codegen/fetch-schemas.mjs new file mode 100644 index 0000000000..d1bcb4c0b7 --- /dev/null +++ b/java/scripts/codegen/fetch-schemas.mjs @@ -0,0 +1,127 @@ +#!/usr/bin/env node + +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import { createHash } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const scriptDir = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(scriptDir, '../../..'); +const packagePath = path.join(repoRoot, 'nodejs', 'package.json'); +const outputDir = path.resolve( + process.env.COPILOT_CLI_SCHEMA_OUTPUT ?? path.join(scriptDir, 'target', 'schemas'), +); +// Schemas are platform-independent; use one asset consistently on every codegen host. +const platform = process.env.COPILOT_CLI_SCHEMA_PLATFORM ?? 'linux-x64'; +const version = + process.env.COPILOT_CLI_VERSION ?? + JSON.parse(fs.readFileSync(packagePath, 'utf8')).copilotCliVersion; + +if (!version) { + throw new Error(`Could not find copilotCliVersion in ${packagePath}`); +} + +const assetName = `github-copilot-${version}-${platform}.tgz`; +const releaseBase = ( + process.env.COPILOT_CLI_DOWNLOAD_BASE_URL ?? + 'https://github.com/github/copilot-cli/releases/download' +).replace(/\/+$/, ''); + +let archive; +let expectedHash; +if (process.env.COPILOT_CLI_RELEASE_TARBALL) { + archive = fs.readFileSync(process.env.COPILOT_CLI_RELEASE_TARBALL); + expectedHash = process.env.COPILOT_CLI_RELEASE_SHA256; +} else { + const releaseUrl = `${releaseBase}/v${version}`; + const checksums = (await download(`${releaseUrl}/SHA256SUMS.txt`)).toString('utf8'); + expectedHash = findChecksum(checksums, assetName); + archive = await download(`${releaseUrl}/${assetName}`); +} + +if (!expectedHash || !/^[a-fA-F0-9]{64}$/.test(expectedHash)) { + throw new Error(`Missing or invalid SHA-256 for ${assetName}`); +} +const actualHash = createHash('sha256').update(archive).digest('hex'); +if (actualHash !== expectedHash.toLowerCase()) { + throw new Error( + `Integrity verification failed for ${assetName}: expected ${expectedHash}, got ${actualHash}`, + ); +} + +const schemaNames = ['api.schema.json', 'session-events.schema.json']; +const members = execFileSync('tar', ['-tzf', '-'], { + encoding: 'utf8', + input: archive, + maxBuffer: 512 * 1024 * 1024, +}) + .split(/\r?\n/) + .filter(Boolean); +const outputParent = path.dirname(outputDir); +fs.mkdirSync(outputParent, { recursive: true }); +const stagingDir = fs.mkdtempSync(path.join(outputParent, '.schemas-')); + +try { + for (const schemaName of schemaNames) { + const member = `package/schemas/${schemaName}`; + if (members.filter((candidate) => candidate === member).length !== 1) { + throw new Error(`${assetName} must contain exactly one ${member}`); + } + const contents = execFileSync('tar', ['-xOzf', '-', member], { + encoding: null, + input: archive, + maxBuffer: 512 * 1024 * 1024, + }); + JSON.parse(contents.toString('utf8')); + fs.writeFileSync(path.join(stagingDir, schemaName), contents); + } + + fs.rmSync(outputDir, { recursive: true, force: true }); + fs.renameSync(stagingDir, outputDir); +} finally { + fs.rmSync(stagingDir, { recursive: true, force: true }); +} + +console.log(`Staged Copilot CLI ${version} schemas at ${outputDir}`); + +async function download(url) { + let lastError; + for (let attempt = 0; attempt < 3; attempt++) { + try { + // lgtm[js/file-access-to-http] The repository-pinned CLI version selects the release asset. + const response = await fetch(url, { signal: AbortSignal.timeout(600_000) }); + if (response.ok) { + return Buffer.from(await response.arrayBuffer()); + } + await response.body?.cancel(); + lastError = new Error(`${response.status} ${response.statusText}`); + if (response.status >= 400 && response.status < 500 && response.status !== 408 && response.status !== 429) { + break; + } + } catch (error) { + lastError = error; + } + if (attempt < 2) { + await new Promise((resolve) => setTimeout(resolve, 2 ** attempt * 1000)); + } + } + throw new Error(`Failed to download ${url}: ${lastError}`); +} + +function findChecksum(checksums, expectedAssetName) { + for (const line of checksums.split(/\r?\n/)) { + const [hash, name] = line.trim().split(/\s+/, 2); + if ( + name?.replace(/^\*/, '') === expectedAssetName && + /^[a-fA-F0-9]{64}$/.test(hash) + ) { + return hash.toLowerCase(); + } + } + throw new Error(`SHA256SUMS.txt does not contain ${expectedAssetName}`); +} diff --git a/java/scripts/codegen/fetch-schemas.test.mjs b/java/scripts/codegen/fetch-schemas.test.mjs new file mode 100644 index 0000000000..19d7ba6715 --- /dev/null +++ b/java/scripts/codegen/fetch-schemas.test.mjs @@ -0,0 +1,77 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; + +const scriptPath = path.join(path.dirname(fileURLToPath(import.meta.url)), 'fetch-schemas.mjs'); + +test('extracts schemas from a verified release archive', (t) => { + const fixture = createFixture(t); + const outputDir = path.join(fixture.root, 'output'); + const result = runFetch(fixture, outputDir); + + assert.equal(result.status, 0, result.stderr); + assert.deepEqual( + JSON.parse(fs.readFileSync(path.join(outputDir, 'api.schema.json'), 'utf8')), + { title: 'API' }, + ); + assert.deepEqual( + JSON.parse(fs.readFileSync(path.join(outputDir, 'session-events.schema.json'), 'utf8')), + { title: 'Events' }, + ); +}); + +test('rejects an archive with the wrong checksum', (t) => { + const fixture = createFixture(t); + const result = runFetch( + { ...fixture, hash: '0'.repeat(64) }, + path.join(fixture.root, 'output'), + ); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Integrity verification failed/); +}); + +test('requires both schema files', (t) => { + const fixture = createFixture(t, { includeEvents: false }); + const result = runFetch(fixture, path.join(fixture.root, 'output')); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /must contain exactly one package\/schemas\/session-events\.schema\.json/); +}); + +function createFixture(t, { includeEvents = true } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'copilot-java-schemas-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const packageDir = path.join(root, 'package'); + const schemasDir = path.join(packageDir, 'schemas'); + fs.mkdirSync(schemasDir, { recursive: true }); + fs.writeFileSync(path.join(schemasDir, 'api.schema.json'), '{"title":"API"}\n'); + if (includeEvents) { + fs.writeFileSync(path.join(schemasDir, 'session-events.schema.json'), '{"title":"Events"}\n'); + } + const archivePath = path.join(root, 'release.tgz'); + execFileSync('tar', ['-czf', archivePath, '-C', root, 'package']); + const hash = createHash('sha256').update(fs.readFileSync(archivePath)).digest('hex'); + return { root, archivePath, hash }; +} + +function runFetch(fixture, outputDir) { + return spawnSync(process.execPath, [scriptPath], { + encoding: 'utf8', + env: { + ...process.env, + COPILOT_CLI_RELEASE_TARBALL: fixture.archivePath, + COPILOT_CLI_RELEASE_SHA256: fixture.hash, + COPILOT_CLI_SCHEMA_OUTPUT: outputDir, + }, + }); +} diff --git a/java/scripts/codegen/java.ts b/java/scripts/codegen/java.ts index 785049afa1..abcd555338 100644 --- a/java/scripts/codegen/java.ts +++ b/java/scripts/codegen/java.ts @@ -174,53 +174,15 @@ function toEnumConstant(value: string): string { // ── Schema path resolution ─────────────────────────────────────────────────── -/** - * Resolve a JSON schema shipped by the `@github/copilot` CLI package. - * - * The CLI package layout changed in 1.0.64-1: the umbrella `@github/copilot` - * package became a thin loader and its bundled assets (including the JSON - * schemas) moved into the platform-specific packages installed as optional - * dependencies, e.g. `@github/copilot-linux-x64` or `@github/copilot-win32-x64`. - * - * We search both the Java codegen install (`scripts/codegen/node_modules`) and - * the Node SDK install (`nodejs/node_modules`), checking the umbrella package - * first (older versions) and then whichever platform package is present. - */ +/** Resolve a JSON schema staged from the pinned GitHub Release artifact. */ async function resolveCopilotSchemaPath(fileName: string): Promise { - const nodeModulesDirs = [ - path.join(REPO_ROOT, "scripts/codegen/node_modules"), - path.join(REPO_ROOT, "nodejs/node_modules"), - ]; - - const candidates: string[] = []; - for (const nodeModulesDir of nodeModulesDirs) { - candidates.push(path.join(nodeModulesDir, "@github/copilot/schemas", fileName)); - const githubScopeDir = path.join(nodeModulesDir, "@github"); - try { - for (const entry of await fs.readdir(githubScopeDir)) { - if (entry.startsWith("copilot-")) { - candidates.push(path.join(githubScopeDir, entry, "schemas", fileName)); - } - } - } catch (err) { - const code = (err as NodeJS.ErrnoException).code; - if (code !== "ENOENT" && code !== "ENOTDIR") { - throw err; - } - // @github scope directory may not exist; try the next location. - } - } - - for (const candidate of candidates) { - try { - await fs.access(candidate); - return candidate; - } catch { - // Try the next candidate. - } + const schemaPath = path.join(REPO_ROOT, "scripts/codegen/target/schemas", fileName); + try { + await fs.access(schemaPath); + return schemaPath; + } catch { + throw new Error(`${fileName} not found. Run 'npm run fetch:schemas' in java/scripts/codegen.`); } - - throw new Error(`${fileName} not found. Run 'npm ci' in java/scripts/codegen or java/nodejs first.`); } async function getSessionEventsSchemaPath(): Promise { diff --git a/java/scripts/codegen/package-lock.json b/java/scripts/codegen/package-lock.json index 5e10cd839b..a92322d340 100644 --- a/java/scripts/codegen/package-lock.json +++ b/java/scripts/codegen/package-lock.json @@ -6,7 +6,6 @@ "": { "name": "copilot-sdk-java-codegen", "dependencies": { - "@github/copilot": "^1.0.83-5", "json-schema": "^0.4.0", "tsx": "^4.23.13" } @@ -427,165 +426,6 @@ "node": ">=18" } }, - "node_modules/@github/copilot": { - "version": "1.0.83-5", - "resolved": "https://registry.npmjs.org/@github/copilot/-/copilot-1.0.83-5.tgz", - "integrity": "sha512-WtyV+Uom2EHR9agUeshRH9aUgvp4zEvAgY0k+DcD9qJKJ8nzPfGXpMrcOJF37CsvxQ3p8Djcvg4aY2ys3XM9CA==", - "license": "SEE LICENSE IN LICENSE.md", - "dependencies": { - "detect-libc": "^2.1.2" - }, - "bin": { - "copilot": "npm-loader.js" - }, - "optionalDependencies": { - "@github/copilot-darwin-arm64": "1.0.83-5", - "@github/copilot-darwin-x64": "1.0.83-5", - "@github/copilot-linux-arm64": "1.0.83-5", - "@github/copilot-linux-x64": "1.0.83-5", - "@github/copilot-linuxmusl-arm64": "1.0.83-5", - "@github/copilot-linuxmusl-x64": "1.0.83-5", - "@github/copilot-win32-arm64": "1.0.83-5", - "@github/copilot-win32-x64": "1.0.83-5" - } - }, - "node_modules/@github/copilot-darwin-arm64": { - "version": "1.0.83-5", - "resolved": "https://registry.npmjs.org/@github/copilot-darwin-arm64/-/copilot-darwin-arm64-1.0.83-5.tgz", - "integrity": "sha512-pj4yCrsVs7Nj16ogROYSKVyMDsmNMt4DRaiQEKUVhR9/Vs5n5DG+N55M2B1chT5wSk5MKtB9bov5Ep+x3jMU7g==", - "cpu": [ - "arm64" - ], - "license": "SEE LICENSE IN LICENSE.md", - "optional": true, - "os": [ - "darwin" - ], - "bin": { - "copilot-darwin-arm64": "copilot" - } - }, - "node_modules/@github/copilot-darwin-x64": { - "version": "1.0.83-5", - "resolved": "https://registry.npmjs.org/@github/copilot-darwin-x64/-/copilot-darwin-x64-1.0.83-5.tgz", - "integrity": "sha512-mU2TBR7zW9kIfM6+r2W+empF0O3J+M/EOKja04IWx4EMc/XQVRnRRdeDaQAPfVF5prEBLxxqe90Ctq6a14IHEA==", - "cpu": [ - "x64" - ], - "license": "SEE LICENSE IN LICENSE.md", - "optional": true, - "os": [ - "darwin" - ], - "bin": { - "copilot-darwin-x64": "copilot" - } - }, - "node_modules/@github/copilot-linux-arm64": { - "version": "1.0.83-5", - "resolved": "https://registry.npmjs.org/@github/copilot-linux-arm64/-/copilot-linux-arm64-1.0.83-5.tgz", - "integrity": "sha512-UG5X20iRyV6A/7tKiVjh/CcqoXDEKvvKd3zcH0l8LEkXhHNGmSY6WiF3FOvp3R5tFhwr2L97UIq8JoKJwUG9TA==", - "cpu": [ - "arm64" - ], - "license": "SEE LICENSE IN LICENSE.md", - "optional": true, - "os": [ - "linux" - ], - "bin": { - "copilot-linux-arm64": "copilot" - } - }, - "node_modules/@github/copilot-linux-x64": { - "version": "1.0.83-5", - "resolved": "https://registry.npmjs.org/@github/copilot-linux-x64/-/copilot-linux-x64-1.0.83-5.tgz", - "integrity": "sha512-sv0CwXtP1gyjDo2JbXC0OCIdMhKgK1ngaVc6HNIdIJpNZebMNTbe8QRxuV7DjXu9tSGP+tL9Kq3nPp/RipZq1Q==", - "cpu": [ - "x64" - ], - "license": "SEE LICENSE IN LICENSE.md", - "optional": true, - "os": [ - "linux" - ], - "bin": { - "copilot-linux-x64": "copilot" - } - }, - "node_modules/@github/copilot-linuxmusl-arm64": { - "version": "1.0.83-5", - "resolved": "https://registry.npmjs.org/@github/copilot-linuxmusl-arm64/-/copilot-linuxmusl-arm64-1.0.83-5.tgz", - "integrity": "sha512-orid/rhuGwypXypYCYDvbffY9CI0JPHntMUzuTP8lUQg45UsSUv3AtJOQPH3Rh3IGr7WJxG5lSxv/oKVv/+H7A==", - "cpu": [ - "arm64" - ], - "license": "SEE LICENSE IN LICENSE.md", - "optional": true, - "os": [ - "linux" - ], - "bin": { - "copilot-linuxmusl-arm64": "copilot" - } - }, - "node_modules/@github/copilot-linuxmusl-x64": { - "version": "1.0.83-5", - "resolved": "https://registry.npmjs.org/@github/copilot-linuxmusl-x64/-/copilot-linuxmusl-x64-1.0.83-5.tgz", - "integrity": "sha512-bcc5seLXLitC3+bH6QTzgFYBZPOmhiKHfkehPGwSdJ5S0l+D5GfuwVfNp7x+W0DDSF/2wXWKF2j6iTCruA45Zg==", - "cpu": [ - "x64" - ], - "license": "SEE LICENSE IN LICENSE.md", - "optional": true, - "os": [ - "linux" - ], - "bin": { - "copilot-linuxmusl-x64": "copilot" - } - }, - "node_modules/@github/copilot-win32-arm64": { - "version": "1.0.83-5", - "resolved": "https://registry.npmjs.org/@github/copilot-win32-arm64/-/copilot-win32-arm64-1.0.83-5.tgz", - "integrity": "sha512-nBFaP31NfWh787eLra3QqgSoEmd7wJGSKg7ANvsY8XDOC5lns8yQBm5y4WVvWNBCVwb196NEwInwGUHDXbj2cw==", - "cpu": [ - "arm64" - ], - "license": "SEE LICENSE IN LICENSE.md", - "optional": true, - "os": [ - "win32" - ], - "bin": { - "copilot-win32-arm64": "copilot.exe" - } - }, - "node_modules/@github/copilot-win32-x64": { - "version": "1.0.83-5", - "resolved": "https://registry.npmjs.org/@github/copilot-win32-x64/-/copilot-win32-x64-1.0.83-5.tgz", - "integrity": "sha512-DlxPHBRal+5oHyQ7f/ChB8zrc71eIpqxd7CzXFU5kcU0YzskitnIp9CLsymqWTEdSoWSNq9+7bhKemIUHWTkfA==", - "cpu": [ - "x64" - ], - "license": "SEE LICENSE IN LICENSE.md", - "optional": true, - "os": [ - "win32" - ], - "bin": { - "copilot-win32-x64": "copilot.exe" - } - }, - "node_modules/detect-libc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", - "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=8" - } - }, "node_modules/esbuild": { "version": "0.28.1", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", diff --git a/java/scripts/codegen/package.json b/java/scripts/codegen/package.json index 5ef4484fb7..a5de56c29f 100644 --- a/java/scripts/codegen/package.json +++ b/java/scripts/codegen/package.json @@ -3,11 +3,12 @@ "private": true, "type": "module", "scripts": { - "generate": "tsx java.ts", - "generate:java": "tsx java.ts" + "fetch:schemas": "node fetch-schemas.mjs", + "generate": "npm run fetch:schemas && tsx java.ts", + "generate:java": "npm run generate", + "test": "node --test fetch-schemas.test.mjs" }, "dependencies": { - "@github/copilot": "^1.0.83-5", "json-schema": "^0.4.0", "tsx": "^4.23.13" } diff --git a/java/sdk/pom.xml b/java/sdk/pom.xml index aa40b25189..693df17eb6 100644 --- a/java/sdk/pom.xml +++ b/java/sdk/pom.xml @@ -775,57 +775,6 @@ did not produce the multi-release output. Re-build on JDK 25+ and verify the - - - update-schemas-from-npm-artifact - - - - org.codehaus.mojo - exec-maven-plugin - - - update-copilot-schema-version - generate-sources - - exec - - - npm - ${project.parent.basedir}/scripts/codegen - - install - @github/copilot@${copilot.schema.version} - - - - - - - org.apache.maven.plugins - maven-enforcer-plugin - - - require-schema-version - validate - - enforce - - - - - copilot.schema.version - You must specify -Dcopilot.schema.version=VERSION (e.g. 1.0.25) - - - - - - - - - diff --git a/java/sdk/src/main/java/com/github/copilot/ffi/NativeRuntimeLoader.java b/java/sdk/src/main/java/com/github/copilot/ffi/NativeRuntimeLoader.java index bd4b185a07..3a1c6b35bc 100644 --- a/java/sdk/src/main/java/com/github/copilot/ffi/NativeRuntimeLoader.java +++ b/java/sdk/src/main/java/com/github/copilot/ffi/NativeRuntimeLoader.java @@ -318,9 +318,8 @@ static Path resolve(String cliPathEnv, Path cacheBase, ClassLoader loader, Strin * *

* Checks, in order, the flat bundled layout ({@code runtime.node} directly next - * to the CLI) and the npm package layout - * ({@code prebuilds//runtime.node} next to the CLI), matching the - * two layouts the {@code @github/copilot-} packages may ship. + * to the CLI) and the release package layout + * ({@code prebuilds//runtime.node} next to the CLI). */ static Path resolveFromCliPath(String cliPathStr) throws IOException { if (cliPathStr == null || cliPathStr.isBlank()) { diff --git a/python/README.md b/python/README.md index 359026df41..c45847806f 100644 --- a/python/README.md +++ b/python/README.md @@ -29,9 +29,11 @@ runtime: python -m copilot download-runtime ``` -This caches `copilot-runtime`, its adjacent `runtime.node`, and the compatible -`copilot` host locally. If you skip this step, the SDK downloads the bundle -automatically on first managed stdio/TCP use. +This downloads the platform release package, verifies it against the release's +`SHA256SUMS.txt`, and directly stages `copilot-runtime`, its adjacent `runtime.node`, +and the filtered hostless runtime assets locally without retaining the downloaded +archive. If you skip this step, the SDK performs the same staging automatically on +first managed stdio/TCP use. To pre-provision the native library required by the in-process (FFI) transport (see [In-process (FFI) transport](#in-process-ffi-transport)), pass `--in-process`: @@ -40,9 +42,10 @@ To pre-provision the native library required by the in-process (FFI) transport python -m copilot download-runtime --in-process ``` -This instead provisions the compatible CLI artifact and native runtime library -used by in-process hosting. When omitted, they are downloaded lazily on first -use of the in-process transport. +This also creates a `copilot` compatibility entrypoint from `copilot-runtime` +inside the complete materialized bundle. Its adjacent `runtime.node` can then be +used for in-process hosting. That canonical staged library is reused, so this does +not download a second runtime artifact. | Platform | Cache path | |----------|-----------| @@ -55,10 +58,9 @@ use of the in-process transport. | Variable | Description | |----------|-------------| | `COPILOT_CLI_PATH` | Use this specific binary instead of downloading | -| `COPILOT_CLI_EXTRACT_DIR` | Override the cache directory (binary placed directly here) | +| `COPILOT_CLI_EXTRACT_DIR` | Override the version-specific cache directory | | `COPILOT_SKIP_CLI_DOWNLOAD` | Set to `1` to disable auto-download | -| `COPILOT_NPM_REGISTRY_URL` | Override the npm registry used for managed out-of-process and in-process runtime downloads | -| `COPILOT_CLI_DOWNLOAD_BASE_URL` | Override the GitHub Releases download URL used for the root CLI | +| `COPILOT_CLI_DOWNLOAD_BASE_URL` | Override the GitHub Releases download URL used for the runtime package and checksums | ## Run the Sample diff --git a/python/copilot/_cli_download.py b/python/copilot/_cli_download.py index 2d9076f6b8..72424cb650 100644 --- a/python/copilot/_cli_download.py +++ b/python/copilot/_cli_download.py @@ -1,22 +1,24 @@ -"""Download and cache the Copilot CLI binary. +"""Download and cache the Copilot CLI runtime package. -This module implements a download-at-first-use strategy for the Copilot CLI -binary, similar to the Rust SDK's build.rs approach but triggered at runtime. -The binary is cached in a shared directory compatible with the Rust SDK: +The platform-specific GitHub release package contains the out-of-process runtime +wrapper, native runtime library, and runtime assets, but omits the legacy SEA +``copilot[.exe]``. Its bytes are downloaded and verified, then the filtered hostless +bundle is materialized directly into the SDK's existing cache layout. ``download_cli`` +preserves the historical CLI filename by creating a compatibility alias from the +runtime wrapper inside the complete materialized bundle: -- Linux: ~/.cache/github-copilot-sdk/cli/{version}/copilot -- macOS: ~/Library/Caches/github-copilot-sdk/cli/{version}/copilot -- Windows: %LOCALAPPDATA%/github-copilot-sdk/cli/{version}/copilot.exe +- Linux: ~/.cache/github-copilot-sdk/cli/{version}/prebuilds/{platform}/copilot +- macOS: ~/Library/Caches/github-copilot-sdk/cli/{version}/prebuilds/{platform}/copilot +- Windows: %LOCALAPPDATA%/github-copilot-sdk/cli/{version}/prebuilds/{platform}/copilot.exe Environment variables: -- COPILOT_CLI_EXTRACT_DIR: Override the cache directory (binary placed directly here). +- COPILOT_CLI_EXTRACT_DIR: Override the runtime bundle cache root. - COPILOT_SKIP_CLI_DOWNLOAD: Set to "1" or "true" to disable auto-download. - COPILOT_CLI_DOWNLOAD_BASE_URL: Override the GitHub Releases base URL. """ from __future__ import annotations -import base64 import hashlib import io import os @@ -26,7 +28,6 @@ import tarfile import tempfile import time -import zipfile from http.client import IncompleteRead from pathlib import Path, PurePosixPath from urllib.error import HTTPError, URLError @@ -34,17 +35,17 @@ from ._cli_version import ( CLI_VERSION, - get_asset_info, get_checksums_url, + get_cli_binary_name, get_download_url, - get_npm_platform, - get_runtime_lib_packument_url, - get_runtime_lib_url, + get_release_asset_name, + get_runtime_platform, ) _CACHE_DIR_NAME = "github-copilot-sdk" _MAX_RETRIES = 3 _RETRIABLE_DOWNLOAD_ERRORS = (HTTPError, URLError, IncompleteRead) +_HOSTLESS_ASSETS_MARKER = ".hostless-runtime-assets-v2" def _sanitize_version(version: str) -> str: @@ -57,13 +58,12 @@ def _sanitize_version(version: str) -> str: def get_cache_dir(version: str | None = None) -> Path: - """Return the cache directory for CLI binaries. + """Return the cache directory for runtime bundles. Args: version: CLI version string. If None, returns the root cache dir. """ - # COPILOT_CLI_EXTRACT_DIR overrides the entire version-specific directory - # (binary lives directly at $dir/, no version subdir). Matches Rust SDK. + # COPILOT_CLI_EXTRACT_DIR overrides the entire version-specific directory. extract_override = os.environ.get("COPILOT_CLI_EXTRACT_DIR") if extract_override: return Path(extract_override) @@ -89,7 +89,7 @@ def get_cache_dir(version: str | None = None) -> Path: def get_cached_cli_path(version: str | None = None) -> str | None: - """Return the path to the cached CLI binary if it exists. + """Return the cached compatibility entrypoint for a complete runtime bundle. Args: version: CLI version. Defaults to the pinned CLI_VERSION. @@ -102,12 +102,21 @@ def get_cached_cli_path(version: str | None = None) -> str | None: return None try: - _, binary_name = get_asset_info() + runtime_platform = get_runtime_platform() except RuntimeError: return None - binary_path = get_cache_dir(ver) / binary_name + binary_name = get_cli_binary_name() + wrapper_name = "copilot-runtime.exe" if sys.platform == "win32" else "copilot-runtime" + pair_dir = get_cache_dir(ver) / "prebuilds" / runtime_platform + binary_path = pair_dir / binary_name + required = ( + binary_path, + pair_dir / wrapper_name, + pair_dir / "runtime.node", + pair_dir / _HOSTLESS_ASSETS_MARKER, + ) - if binary_path.exists(): + if all(path.is_file() and path.stat().st_size > 0 for path in required): return str(binary_path) return None @@ -124,30 +133,22 @@ def _fetch_checksums(version: str) -> dict[str, str]: Returns a dict mapping filename → sha256 hex digest. """ url = get_checksums_url(version) - last_exc: Exception | None = None - for attempt in range(_MAX_RETRIES): - try: - with urlopen(url, timeout=30) as response: - text = response.read().decode("utf-8") - break - except _RETRIABLE_DOWNLOAD_ERRORS as exc: - last_exc = exc - if attempt < _MAX_RETRIES - 1: - time.sleep(2**attempt) - else: + try: + text = _fetch_url_bytes(url, timeout=30).decode("utf-8") + except (RuntimeError, UnicodeDecodeError) as exc: raise RuntimeError( - f"Failed to download checksums from {url}: {last_exc}\n\n" + f"Failed to download checksums from {url}: {exc}\n\n" "If you are in an offline or firewalled environment, set " "COPILOT_CLI_PATH to point to a manually-installed binary." - ) from last_exc + ) from exc checksums: dict[str, str] = {} for line in text.strip().splitlines(): parts = line.split() - if len(parts) == 2: + if len(parts) == 2 and re.fullmatch(r"[a-fA-F0-9]{64}", parts[0]): digest, filename = parts # Some formats use *filename (binary mode indicator) - checksums[filename.lstrip("*")] = digest + checksums[filename.lstrip("*")] = digest.lower() return checksums @@ -160,65 +161,36 @@ def _verify_checksum(data: bytes, expected_hash: str, filename: str) -> None: ) -def _extract_tar_gz(data: bytes, binary_name: str, dest_dir: Path) -> Path: - """Extract the CLI binary from a .tar.gz archive.""" - with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tf: - # Find the binary in the archive (may be at top level or in a subdirectory) - members = tf.getnames() - target_member = None - for name in members: - if name == binary_name or name.endswith(f"/{binary_name}"): - target_member = name - break - - if target_member is None: - raise RuntimeError( - f"Binary '{binary_name}' not found in archive. Archive contains: {members}" - ) - - member = tf.getmember(target_member) - f = tf.extractfile(member) - if f is None: - raise RuntimeError(f"Could not extract '{target_member}' from archive") - - dest_path = dest_dir / binary_name - with open(dest_path, "wb") as out: - out.write(f.read()) - - return dest_path - - -def _extract_zip(data: bytes, binary_name: str, dest_dir: Path) -> Path: - """Extract the CLI binary from a .zip archive.""" - with zipfile.ZipFile(io.BytesIO(data)) as zf: - names = zf.namelist() - target_member = None - for name in names: - if name == binary_name or name.endswith(f"/{binary_name}"): - target_member = name - break - - if target_member is None: - raise RuntimeError( - f"Binary '{binary_name}' not found in archive. Archive contains: {names}" - ) +def _fetch_verified_release_package(version: str, runtime_platform: str) -> bytes: + """Download and verify the unified platform release package.""" + asset_name = get_release_asset_name(version, runtime_platform) + expected_hash = _fetch_checksums(version).get(asset_name) + if not expected_hash: + raise RuntimeError(f"SHA256SUMS.txt does not contain {asset_name}.") + url = get_download_url(version, asset_name) + data = _fetch_url_bytes(url, timeout=600) + _verify_checksum(data, expected_hash, asset_name) + return data - dest_path = dest_dir / binary_name - with zf.open(target_member) as src, open(dest_path, "wb") as out: - out.write(src.read()) - return dest_path +def _runtime_bundle_is_complete(pair_dir: Path, wrapper_name: str) -> bool: + required = ( + pair_dir / wrapper_name, + pair_dir / "runtime.node", + pair_dir / _HOSTLESS_ASSETS_MARKER, + ) + return all(path.is_file() and path.stat().st_size > 0 for path in required) def download_cli(version: str | None = None, *, force: bool = False) -> str: - """Download the Copilot CLI binary and cache it. + """Provision a complete runtime bundle with a ``copilot[.exe]`` alias. Args: version: CLI version to download. Defaults to the pinned CLI_VERSION. force: If True, re-download even if already cached. Returns: - Path to the cached binary. + Path to the compatibility entrypoint adjacent to the complete runtime bundle. Raises: RuntimeError: If the version is not set, download fails, or @@ -231,81 +203,33 @@ def download_cli(version: str | None = None, *, force: bool = False) -> str: "set COPILOT_CLI_PATH or install a published wheel." ) - archive_name, binary_name = get_asset_info() - cache_dir = get_cache_dir(ver) - binary_path = cache_dir / binary_name + binary_name = get_cli_binary_name() - # Return cached binary if available (unless force) - if not force and binary_path.exists(): - return str(binary_path) - - # Fetch checksums - checksums = _fetch_checksums(ver) - expected_hash = checksums.get(archive_name) - if not expected_hash: - raise RuntimeError( - f"No checksum found for '{archive_name}' in SHA256SUMS.txt. " - f"Available files: {list(checksums.keys())}" - ) + if not force: + cached = get_cached_cli_path(ver) + if cached is not None: + return cached - # Download archive with retries - url = get_download_url(ver, archive_name) - last_exc: Exception | None = None - data: bytes | None = None - for attempt in range(_MAX_RETRIES): - try: - with urlopen(url, timeout=120) as response: - data = response.read() - break - except _RETRIABLE_DOWNLOAD_ERRORS as exc: - last_exc = exc - if attempt < _MAX_RETRIES - 1: - time.sleep(2**attempt) - if data is None: - raise RuntimeError( - f"Failed to download runtime from {url}: {last_exc}\n\n" - "If you are in an offline or firewalled environment, you can:\n" - f"1. Manually download the archive from: {url}\n" - f"2. Extract the '{binary_name}' binary to: {binary_path}\n" - "Or set COPILOT_CLI_PATH to point to an existing binary." - ) from last_exc - - # Verify checksum - _verify_checksum(data, expected_hash, archive_name) - - # Extract to a temporary directory, then atomically move into place. - # This prevents partial/corrupt cache entries if the process is interrupted. - cache_dir.mkdir(parents=True, exist_ok=True) - staging_dir = Path(tempfile.mkdtemp(dir=cache_dir, prefix=".download-")) + wrapper_path = Path(ensure_runtime_wrapper(ver, force=force)) + binary_path = wrapper_path.with_name(binary_name) + fd, temp_name = tempfile.mkstemp(dir=wrapper_path.parent, prefix=".cli-") try: - if archive_name.endswith(".tar.gz"): - extracted = _extract_tar_gz(data, binary_name, staging_dir) - elif archive_name.endswith(".zip"): - extracted = _extract_zip(data, binary_name, staging_dir) - else: - raise RuntimeError(f"Unknown archive format: {archive_name}") - - # Make executable on Unix + with os.fdopen(fd, "wb") as destination: + destination.write(wrapper_path.read_bytes()) + staged = Path(temp_name) if sys.platform != "win32": - extracted.chmod(extracted.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - - # Atomic rename into final location. Handle concurrent processes: - # another process may have written the file while we were downloading. - try: - extracted.replace(binary_path) - except OSError: - if not force and binary_path.exists(): - return str(binary_path) - raise - finally: - # Clean up staging directory + staged.chmod(staged.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + os.replace(staged, binary_path) + except OSError: try: - staging_dir.rmdir() + os.unlink(temp_name) except OSError: - # May not be empty if rename failed or other files were extracted - import shutil - - shutil.rmtree(staging_dir, ignore_errors=True) + pass + if not force: + cached = get_cached_cli_path(ver) + if cached is not None: + return cached + raise return str(binary_path) @@ -324,71 +248,6 @@ def _fetch_url_bytes(url: str, *, timeout: int) -> bytes: raise RuntimeError(f"Failed to download from {url}: {last_exc}") from last_exc -def _fetch_runtime_integrity(npm_platform: str, version: str) -> str | None: - """Return the npm ``dist.integrity`` (Subresource Integrity) for the tarball. - - Best-effort: returns None if the packument can't be fetched or parsed. - """ - import json - - url = get_runtime_lib_packument_url(npm_platform) - try: - raw = _fetch_url_bytes(url, timeout=30) - packument = json.loads(raw) - dist = packument.get("versions", {}).get(version, {}).get("dist", {}) - integrity = dist.get("integrity") - return integrity if isinstance(integrity, str) else None - except (RuntimeError, ValueError, KeyError): - return None - - -def _verify_integrity(data: bytes, integrity: str) -> None: - """Verify data against an npm Subresource Integrity string (e.g. ``sha512-``).""" - algo, _, b64 = integrity.partition("-") - algo = algo.lower() - if algo not in ("sha512", "sha384", "sha256"): - # Fail closed: an unrecognized algorithm means we cannot verify this native - # library, so refuse rather than loading unverified native code. - raise RuntimeError( - f"Unsupported integrity algorithm '{algo}' for the in-process runtime " - "library; refusing to load unverified native code." - ) - expected = base64.b64decode(b64) - actual = hashlib.new(algo, data).digest() - if actual != expected: - raise RuntimeError( - f"Integrity mismatch for runtime library ({algo}): " - "downloaded tarball does not match the npm registry checksum." - ) - - -def _extract_runtime_node(data: bytes, npm_platform: str) -> bytes: - """Extract ``package/prebuilds//runtime.node`` from an npm tarball.""" - target = f"package/prebuilds/{npm_platform}/runtime.node" - with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tf: - for name in tf.getnames(): - if name == target or name.endswith(f"/prebuilds/{npm_platform}/runtime.node"): - member = tf.getmember(name) - extracted = tf.extractfile(member) - if extracted is not None: - return extracted.read() - raise RuntimeError(f"'{target}' not found in runtime package for {npm_platform}.") - - -def _extract_runtime_wrapper(data: bytes, npm_platform: str) -> bytes: - """Extract the SDK out-of-process wrapper from an npm platform tarball.""" - wrapper_name = "copilot-runtime.exe" if sys.platform == "win32" else "copilot-runtime" - target = f"package/prebuilds/{npm_platform}/{wrapper_name}" - with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tf: - for name in tf.getnames(): - if name == target or name.endswith(f"/prebuilds/{npm_platform}/{wrapper_name}"): - member = tf.getmember(name) - extracted = tf.extractfile(member) - if extracted is not None: - return extracted.read() - raise RuntimeError(f"'{target}' not found in runtime package for {npm_platform}.") - - _HOSTLESS_EXCLUDED_TOP_LEVEL = { "app.js", "assets", @@ -412,7 +271,9 @@ def _extract_runtime_wrapper(data: bytes, npm_platform: str) -> bytes: } -def _hostless_runtime_path(member_name: str, npm_platform: str) -> Path | None: +def _hostless_runtime_path(member_name: str, runtime_platform: str) -> Path | None: + if "\\" in member_name: + raise RuntimeError(f"Unsafe runtime package path: {member_name}") parts = PurePosixPath(member_name).parts if not parts or parts[0] != "package" or len(parts) < 2: return None @@ -429,7 +290,7 @@ def _hostless_runtime_path(member_name: str, npm_platform: str) -> Path | None: ): return None if top_level == "prebuilds": - if len(relative) < 3 or relative[1] != npm_platform: + if len(relative) < 3 or relative[1] != runtime_platform: return None relative = relative[2:] destination = Path(*relative) @@ -438,11 +299,11 @@ def _hostless_runtime_path(member_name: str, npm_platform: str) -> Path | None: return destination -def _extract_runtime_bundle(data: bytes, npm_platform: str, destination: Path) -> None: +def _materialize_runtime_bundle(data: bytes, runtime_platform: str, destination: Path) -> None: """Extract the hostless runtime tree, retaining unknown package assets by default.""" with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as archive: for member in archive: - relative = _hostless_runtime_path(member.name, npm_platform) + relative = _hostless_runtime_path(member.name, runtime_platform) if relative is None or member.isdir(): continue if not member.isfile(): @@ -458,20 +319,20 @@ def _extract_runtime_bundle(data: bytes, npm_platform: str, destination: Path) - def ensure_runtime_wrapper(version: str | None = None, force: bool = False) -> str: - """Provision the runtime pair and its retained npm package assets.""" + """Provision the runtime pair and retained assets from the release package.""" ver = version or CLI_VERSION if not ver: raise RuntimeError("No runtime version is pinned.") - npm_platform = get_npm_platform() + runtime_platform = get_runtime_platform() wrapper_name = "copilot-runtime.exe" if sys.platform == "win32" else "copilot-runtime" - pair_dir = get_cache_dir(ver) / "prebuilds" / npm_platform + pair_dir = get_cache_dir(ver) / "prebuilds" / runtime_platform wrapper_path = pair_dir / wrapper_name runtime_path = pair_dir / "runtime.node" - assets_marker = pair_dir / ".hostless-runtime-assets-v2" + assets_marker = pair_dir / _HOSTLESS_ASSETS_MARKER wrapper_exists = wrapper_path.is_file() and wrapper_path.stat().st_size > 0 runtime_exists = runtime_path.is_file() and runtime_path.stat().st_size > 0 - if wrapper_exists and runtime_exists and assets_marker.is_file() and not force: + if _runtime_bundle_is_complete(pair_dir, wrapper_name) and not force: return str(wrapper_path) if not force and wrapper_exists != runtime_exists: raise RuntimeError( @@ -484,20 +345,13 @@ def ensure_runtime_wrapper(version: str | None = None, force: bool = False) -> s "and automatic downloads are disabled." ) - data = _fetch_url_bytes(get_runtime_lib_url(ver, npm_platform), timeout=600) - integrity = _fetch_runtime_integrity(npm_platform, ver) - if not integrity: - raise RuntimeError( - "No Subresource Integrity value available for the Copilot runtime " - f"package ({npm_platform}@{ver}); refusing to stage unverified native code." - ) - _verify_integrity(data, integrity) + data = _fetch_verified_release_package(ver, runtime_platform) import shutil pair_dir.parent.mkdir(parents=True, exist_ok=True) staging_dir = Path(tempfile.mkdtemp(dir=pair_dir.parent, prefix=".runtime-bundle-")) try: - _extract_runtime_bundle(data, npm_platform, staging_dir) + _materialize_runtime_bundle(data, runtime_platform, staging_dir) staged_wrapper = staging_dir / wrapper_name staged_runtime = staging_dir / "runtime.node" if ( @@ -536,15 +390,15 @@ def ensure_runtime_wrapper(version: str | None = None, force: bool = False) -> s def ensure_runtime_library(cli_path: str, version: str | None = None) -> str | None: """Ensure the native in-process (FFI) runtime library sits next to ``cli_path``. - The library is NOT part of the GitHub Releases CLI archive; it ships in the npm - platform package ``@github/copilot-`` under - ``package/prebuilds//runtime.node``. This helper downloads that tarball - and writes the library next to the CLI binary under its natural platform name - (``libcopilot_runtime.so`` / ``.dylib`` / ``copilot_runtime.dll``). + The canonical staged bundle contains ``prebuilds//runtime.node``. + This helper reuses that verified library and copies it next to the CLI binary + under its natural platform name (``libcopilot_runtime.so`` / ``.dylib`` / + ``copilot_runtime.dll``). - This is opt-in — only invoked when the in-process transport is actually selected - (lazy) or via ``python -m copilot download-runtime --in-process`` (explicit). The - default stdio download path never fetches these extra bytes. + Copying the library next to an external CLI is opt-in — this is only invoked when + the in-process transport is selected (lazy) or via + ``python -m copilot download-runtime --in-process`` (explicit). The default stdio + path leaves the library in the canonical staged bundle. Returns the absolute path to the library, or None if it could not be provisioned (e.g. download disabled or unsupported platform). Raises RuntimeError on @@ -558,15 +412,12 @@ def ensure_runtime_library(cli_path: str, version: str | None = None) -> str | N if existing is not None: return existing - if _should_skip_download(): - return None - ver = version or CLI_VERSION if not ver: return None try: - npm_platform = get_npm_platform() + runtime_platform = get_runtime_platform() except RuntimeError: return None @@ -575,31 +426,22 @@ def ensure_runtime_library(cli_path: str, version: str | None = None) -> str | N if lib_path.exists(): return str(lib_path) - url = get_runtime_lib_url(ver, npm_platform) - data = _fetch_url_bytes(url, timeout=600) - - integrity = _fetch_runtime_integrity(npm_platform, ver) - if not integrity: - # Fail closed: this native library is loaded into the host process, so it must - # be verified before use. The npm packument (which carries dist.integrity) was - # unavailable, so refuse rather than loading unverified native code — mirroring - # the CLI download, which requires a checksum. Retry when the registry is - # reachable, or install a runtime package that ships the library. - raise RuntimeError( - "No Subresource Integrity value available for the in-process runtime " - f"library ({npm_platform}@{ver}); refusing to load unverified native code." - ) - _verify_integrity(data, integrity) - - lib_bytes = _extract_runtime_node(data, npm_platform) + pair_dir = get_cache_dir(ver) / "prebuilds" / runtime_platform + wrapper_name = "copilot-runtime.exe" if sys.platform == "win32" else "copilot-runtime" + if _should_skip_download() and not _runtime_bundle_is_complete(pair_dir, wrapper_name): + return None + wrapper_path = Path(ensure_runtime_wrapper(ver)) + canonical_runtime = wrapper_path.with_name("runtime.node") # Write atomically next to the CLI so concurrent starts don't observe a partial # library. A rename within the same directory is atomic on POSIX and Windows. + import shutil + cli_dir.mkdir(parents=True, exist_ok=True) fd, tmp_name = tempfile.mkstemp(dir=cli_dir, prefix=".runtime-lib-") try: - with os.fdopen(fd, "wb") as out: - out.write(lib_bytes) + with os.fdopen(fd, "wb") as out, canonical_runtime.open("rb") as source: + shutil.copyfileobj(source, out) os.replace(tmp_name, lib_path) except OSError: try: @@ -634,16 +476,18 @@ def get_or_download_cli(version: str | None = None) -> str | None: if cached: return cached - # Check if download is disabled - if _should_skip_download(): - return None - # Check platform support before attempting download try: - get_asset_info() + runtime_platform = get_runtime_platform() except RuntimeError: return None + if _should_skip_download(): + pair_dir = get_cache_dir(ver) / "prebuilds" / runtime_platform + wrapper_name = "copilot-runtime.exe" if sys.platform == "win32" else "copilot-runtime" + if not _runtime_bundle_is_complete(pair_dir, wrapper_name): + return None + # Download return download_cli(ver) diff --git a/python/copilot/_cli_version.py b/python/copilot/_cli_version.py index cb5939820a..4fd0921c2d 100644 --- a/python/copilot/_cli_version.py +++ b/python/copilot/_cli_version.py @@ -16,35 +16,10 @@ # DO NOT reformat this line — the inject script matches it exactly. CLI_VERSION: str | None = None -# Maps (sys.platform, platform.machine()) → (archive filename, binary name inside archive). -PLATFORM_ASSETS: dict[tuple[str, str], tuple[str, str]] = { - ("linux", "x86_64"): ("copilot-linux-x64.tar.gz", "copilot"), - ("linux", "aarch64"): ("copilot-linux-arm64.tar.gz", "copilot"), - ("linux", "arm64"): ("copilot-linux-arm64.tar.gz", "copilot"), - ("darwin", "x86_64"): ("copilot-darwin-x64.tar.gz", "copilot"), - ("darwin", "arm64"): ("copilot-darwin-arm64.tar.gz", "copilot"), - ("win32", "AMD64"): ("copilot-win32-x64.zip", "copilot.exe"), - ("win32", "ARM64"): ("copilot-win32-arm64.zip", "copilot.exe"), -} - -# Musl (Alpine) variants — detected at runtime via _is_musl(). -_MUSL_ASSETS: dict[str, tuple[str, str]] = { - "x86_64": ("copilot-linuxmusl-x64.tar.gz", "copilot"), - "aarch64": ("copilot-linuxmusl-arm64.tar.gz", "copilot"), - "arm64": ("copilot-linuxmusl-arm64.tar.gz", "copilot"), -} - _DOWNLOAD_BASE_URL = "https://github.com/github/copilot-cli/releases/download" -# The native in-process (FFI) runtime library (`runtime.node`) is NOT part of the -# GitHub Releases `copilot-` archive (that ships only the CLI binary). It -# lives in the npm platform package `@github/copilot-`, under -# `package/prebuilds//runtime.node`. Mirrors the .NET SDK targets, -# which download the same npm tarball. -_NPM_REGISTRY_BASE_URL = "https://registry.npmjs.org" - -# Maps (sys.platform, platform.machine()) → npm platform name (glibc Linux/macOS/Windows). -NPM_PLATFORMS: dict[tuple[str, str], str] = { +# Maps (sys.platform, platform.machine()) to the platform segment used by release assets. +RUNTIME_PLATFORMS: dict[tuple[str, str], str] = { ("linux", "x86_64"): "linux-x64", ("linux", "aarch64"): "linux-arm64", ("linux", "arm64"): "linux-arm64", @@ -54,8 +29,8 @@ ("win32", "ARM64"): "win32-arm64", } -# Musl (Alpine) npm platform variants — detected at runtime via _is_musl(). -_MUSL_NPM_PLATFORMS: dict[str, str] = { +# Musl (Alpine) runtime platform variants — detected at runtime via _is_musl(). +_MUSL_RUNTIME_PLATFORMS: dict[str, str] = { "x86_64": "linuxmusl-x64", "aarch64": "linuxmusl-arm64", "arm64": "linuxmusl-arm64", @@ -82,33 +57,11 @@ def get_platform_key() -> tuple[str, str]: return (sys.platform, platform.machine()) -def get_asset_info() -> tuple[str, str]: - """Return (archive_filename, binary_name) for the current platform. - - Raises RuntimeError if the platform is not supported. - """ - key = get_platform_key() - - # On Linux, check for musl/Alpine first - if key[0] == "linux" and _is_musl(): - musl_info = _MUSL_ASSETS.get(key[1]) - if musl_info: - return musl_info - - info = PLATFORM_ASSETS.get(key) - if info is None: - raise RuntimeError( - f"Unsupported platform: {key[0]}/{key[1]}. " - f"Supported platforms: {', '.join(f'{p}/{m}' for p, m in PLATFORM_ASSETS)}" - ) - return info - - def get_download_url(version: str, archive_name: str) -> str: """Return the download URL for a given version and archive.""" import os - base = os.environ.get("COPILOT_CLI_DOWNLOAD_BASE_URL", _DOWNLOAD_BASE_URL) + base = os.environ.get("COPILOT_CLI_DOWNLOAD_BASE_URL", _DOWNLOAD_BASE_URL).rstrip("/") return f"{base}/v{version}/{archive_name}" @@ -116,47 +69,38 @@ def get_checksums_url(version: str) -> str: """Return the URL for the SHA256SUMS.txt file.""" import os - base = os.environ.get("COPILOT_CLI_DOWNLOAD_BASE_URL", _DOWNLOAD_BASE_URL) + base = os.environ.get("COPILOT_CLI_DOWNLOAD_BASE_URL", _DOWNLOAD_BASE_URL).rstrip("/") return f"{base}/v{version}/SHA256SUMS.txt" -def get_npm_platform() -> str: - """Return the npm platform name (e.g. ``linux-x64``) for the current host. +def get_runtime_platform() -> str: + """Return the release asset platform name (e.g. ``linux-x64``) for this host. - Used to locate the native in-process runtime library. Raises RuntimeError if - the platform is not supported. + The name matches the ``prebuilds`` folder embedded in the release package. + Raises RuntimeError if the platform is not supported. """ key = get_platform_key() if key[0] == "linux" and _is_musl(): - musl = _MUSL_NPM_PLATFORMS.get(key[1]) + musl = _MUSL_RUNTIME_PLATFORMS.get(key[1]) if musl: return musl - npm_platform = NPM_PLATFORMS.get(key) - if npm_platform is None: + runtime_platform = RUNTIME_PLATFORMS.get(key) + if runtime_platform is None: raise RuntimeError( - f"Unsupported platform for in-process runtime: {key[0]}/{key[1]}. " - f"Supported platforms: {', '.join(f'{p}/{m}' for p, m in NPM_PLATFORMS)}" + f"Unsupported Copilot runtime platform: {key[0]}/{key[1]}. " + f"Supported platforms: {', '.join(f'{p}/{m}' for p, m in RUNTIME_PLATFORMS)}" ) - return npm_platform + return runtime_platform -def get_runtime_lib_packument_url(npm_platform: str) -> str: - """Return the npm packument URL for the platform runtime package.""" - import os +def get_release_asset_name(version: str, runtime_platform: str | None = None) -> str: + """Return the unified runtime package asset name for a version and platform.""" + platform_name = runtime_platform or get_runtime_platform() + return f"github-copilot-{version}-{platform_name}.tgz" - base = os.environ.get("COPILOT_NPM_REGISTRY_URL", _NPM_REGISTRY_BASE_URL).rstrip("/") - return f"{base}/@github/copilot-{npm_platform}" - - -def get_runtime_lib_url(version: str, npm_platform: str) -> str: - """Return the download URL for the platform runtime tarball. - - Mirrors the .NET targets' URL layout - ``/@github/copilot-/-/copilot--.tgz``. - """ - import os - base = os.environ.get("COPILOT_NPM_REGISTRY_URL", _NPM_REGISTRY_BASE_URL).rstrip("/") - return f"{base}/@github/copilot-{npm_platform}/-/copilot-{npm_platform}-{version}.tgz" +def get_cli_binary_name() -> str: + """Return the CLI executable name inside the release package.""" + return "copilot.exe" if sys.platform == "win32" else "copilot" diff --git a/python/copilot/_ffi_runtime_host.py b/python/copilot/_ffi_runtime_host.py index 5665f8fba7..511cbae9c4 100644 --- a/python/copilot/_ffi_runtime_host.py +++ b/python/copilot/_ffi_runtime_host.py @@ -121,7 +121,7 @@ def resolve_library_path(runtime_entrypoint: str) -> str | None: 1. The natural platform library name next to the CLI (bundled/flat layout, what the Python download-at-first-use path writes). - 2. ``runtime.node`` next to the CLI (prepared npm runtime layout). + 2. ``runtime.node`` next to the CLI (prepared release-package layout). 3. ``prebuilds//runtime.node`` next to the CLI (package-root layout). Returns the absolute path, or ``None`` when neither exists. @@ -211,7 +211,7 @@ def _load_library(library_path: str) -> _FfiLibrary: return _FfiLibrary(_loaded_library) # Load with immediate binding (RTLD_NOW) on POSIX, matching the .NET/Rust - # hosts. The runtime cdylib from the npm platform package is self-contained; + # hosts. The runtime cdylib from the platform release package is self-contained; # eager binding surfaces any load problem here rather than at first call. if sys.platform == "win32": lib = ctypes.WinDLL(library_path) diff --git a/python/e2e/testharness/context.py b/python/e2e/testharness/context.py index 616fa9bb0a..12eb9466fa 100644 --- a/python/e2e/testharness/context.py +++ b/python/e2e/testharness/context.py @@ -147,6 +147,7 @@ def _apply_inprocess_environment(self) -> None: { "GH_TOKEN": DEFAULT_GITHUB_TOKEN, "GITHUB_TOKEN": DEFAULT_GITHUB_TOKEN, + "COPILOT_CLI_PATH": self.cli_path, "COPILOT_HMAC_KEY": "", "CAPI_HMAC_KEY": "", } diff --git a/python/test_cli_download.py b/python/test_cli_download.py index 4ae4fe6bc3..fcf65df41d 100644 --- a/python/test_cli_download.py +++ b/python/test_cli_download.py @@ -1,42 +1,36 @@ -"""Tests for the in-process runtime library download integrity checks.""" +"""Tests for unified Copilot release-package provisioning.""" from __future__ import annotations -import base64 import hashlib import io import os import tarfile +from concurrent.futures import ThreadPoolExecutor from http.client import IncompleteRead +from threading import Barrier from unittest.mock import MagicMock, patch import pytest -from copilot import _cli_download, _ffi_runtime_host +from copilot import _cli_download, _cli_version, _ffi_runtime_host -def _integrity(data: bytes, algo: str = "sha512") -> str: - digest = hashlib.new(algo, data).digest() - return f"{algo}-{base64.b64encode(digest).decode('ascii')}" - - -def _runtime_package(npm_platform: str) -> bytes: +def _release_package(runtime_platform: str) -> bytes: wrapper_name = "copilot-runtime.exe" if os.name == "nt" else "copilot-runtime" members = { - f"package/prebuilds/{npm_platform}/{wrapper_name}": b"wrapper", - f"package/prebuilds/{npm_platform}/runtime.node": b"runtime", - "package/copilot": b"excluded", - "package/copilot.exe": b"excluded", - f"package/ripgrep/bin/{npm_platform}/rg": b"ripgrep", + f"package/prebuilds/{runtime_platform}/{wrapper_name}": b"wrapper", + f"package/prebuilds/{runtime_platform}/runtime.node": b"runtime", + f"package/ripgrep/bin/{runtime_platform}/rg": b"ripgrep", "package/definitions/future.json": b"{}", "package/app.js": b"excluded", "package/LICENSE.md": b"excluded", - "package/README.md": b"excluded", } buffer = io.BytesIO() with tarfile.open(fileobj=buffer, mode="w:gz") as archive: for name, content in members.items(): info = tarfile.TarInfo(name) + info.mode = 0o755 if name.endswith((wrapper_name, "/rg")) else 0o644 info.size = len(content) archive.addfile(info, io.BytesIO(content)) return buffer.getvalue() @@ -62,41 +56,243 @@ def test_fetch_url_bytes_retries_truncated_response(): sleep.assert_called_once_with(1) -class TestVerifyIntegrity: - def test_accepts_matching_checksum(self): - data = b"native-library-bytes" - _cli_download._verify_integrity(data, _integrity(data)) +def _release_fetches(version: str, runtime_platform: str, data: bytes): + asset_name = f"github-copilot-{version}-{runtime_platform}.tgz" + checksum = hashlib.sha256(data).hexdigest() + + def fetch(url: str, *, timeout: int) -> bytes: + del timeout + if url.endswith("/SHA256SUMS.txt"): + return f"{checksum} {asset_name}\n".encode() + assert url.endswith(f"/{asset_name}") + return data + + return fetch + + +def test_release_asset_uses_platform_package_name(monkeypatch): + monkeypatch.setenv("COPILOT_CLI_DOWNLOAD_BASE_URL", "https://mirror.example/releases/") + + name = _cli_version.get_release_asset_name("1.2.3-4", "linux-x64") + + assert name == "github-copilot-1.2.3-4-linux-x64.tgz" + assert ( + _cli_version.get_download_url("1.2.3-4", name) + == "https://mirror.example/releases/v1.2.3-4/github-copilot-1.2.3-4-linux-x64.tgz" + ) + + +@pytest.mark.parametrize( + "member_name", + ["package/../outside", r"package\..\outside"], +) +def test_hostless_runtime_path_rejects_traversal(member_name): + with pytest.raises(RuntimeError, match="Unsafe runtime package path"): + _cli_download._hostless_runtime_path(member_name, "linux-x64") + + +def test_rejects_release_package_checksum_mismatch(tmp_path): + runtime_platform = "linux-x64" + data = _release_package(runtime_platform) + asset_name = f"github-copilot-1.2.3-{runtime_platform}.tgz" + + def fetch(url: str, *, timeout: int) -> bytes: + del timeout + if url.endswith("/SHA256SUMS.txt"): + return f"{'0' * 64} {asset_name}\n".encode() + return data + + with ( + patch.object(_cli_download, "get_cache_dir", return_value=tmp_path / "cache"), + patch.object(_cli_download, "get_runtime_platform", return_value=runtime_platform), + patch.object(_cli_download, "_fetch_url_bytes", side_effect=fetch), + ): + with pytest.raises(RuntimeError, match="Checksum mismatch"): + _cli_download.ensure_runtime_wrapper(version="1.2.3") + + +def test_rejects_release_package_without_checksum(tmp_path): + runtime_platform = "linux-x64" + + def fetch(url: str, *, timeout: int) -> bytes: + del timeout + assert url.endswith("/SHA256SUMS.txt") + return f"{'0' * 64} another-file.tgz\n".encode() + + with ( + patch.object(_cli_download, "get_cache_dir", return_value=tmp_path / "cache"), + patch.object(_cli_download, "get_runtime_platform", return_value=runtime_platform), + patch.object(_cli_download, "_fetch_url_bytes", side_effect=fetch), + ): + with pytest.raises(RuntimeError, match="SHA256SUMS.txt does not contain"): + _cli_download.ensure_runtime_wrapper(version="1.2.3") + + +def test_cli_and_runtime_share_one_staged_bundle(tmp_path, monkeypatch): + version = "1.2.3" + runtime_platform = "linux-x64" + cli_name = "copilot.exe" if os.name == "nt" else "copilot" + wrapper_name = "copilot-runtime.exe" if os.name == "nt" else "copilot-runtime" + data = _release_package(runtime_platform) + cache_dir = tmp_path / "cache" + install_dir = cache_dir / "prebuilds" / runtime_platform + fetch = _release_fetches(version, runtime_platform, data) + + with ( + patch.object(_cli_download, "get_cache_dir", return_value=cache_dir), + patch.object(_cli_download, "get_runtime_platform", return_value=runtime_platform), + patch.object(_cli_download, "_fetch_url_bytes", side_effect=fetch) as fetch_mock, + ): + cli = _cli_download.download_cli(version) + monkeypatch.setenv("COPILOT_SKIP_CLI_DOWNLOAD", "1") + wrapper = _cli_download.ensure_runtime_wrapper(version) + assert _cli_download.get_cached_cli_path(version) == str(install_dir / cli_name) + + assert cli == str(install_dir / cli_name) + assert wrapper == str(install_dir / wrapper_name) + assert (install_dir / cli_name).read_bytes() == b"wrapper" + assert not (cache_dir / cli_name).exists() + assert not (cache_dir / "packages").exists() + assert (install_dir / wrapper_name).read_bytes() == b"wrapper" + assert (install_dir / "runtime.node").read_bytes() == b"runtime" + assert (install_dir / "ripgrep" / "bin" / runtime_platform / "rg").read_bytes() == b"ripgrep" + assert (install_dir / "definitions" / "future.json").read_bytes() == b"{}" + assert not (install_dir / "app.js").exists() + assert (install_dir / ".hostless-runtime-assets-v2").is_file() + assert fetch_mock.call_count == 2 + if os.name != "nt": + assert (install_dir / cli_name).stat().st_mode & 0o111 + assert (install_dir / wrapper_name).stat().st_mode & 0o111 + + +def test_concurrent_staging_materializes_one_complete_bundle(tmp_path): + version = "1.2.3" + runtime_platform = "linux-x64" + wrapper_name = "copilot-runtime.exe" if os.name == "nt" else "copilot-runtime" + data = _release_package(runtime_platform) + cache_dir = tmp_path / "cache" + fetch = _release_fetches(version, runtime_platform, data) + fetch_barrier = Barrier(2) - def test_rejects_mismatched_checksum(self): - with pytest.raises(RuntimeError, match="Integrity mismatch"): - _cli_download._verify_integrity(b"tampered", _integrity(b"original")) + def concurrent_fetch(url: str, *, timeout: int) -> bytes: + fetch_barrier.wait(timeout=10) + return fetch(url, timeout=timeout) - def test_rejects_unsupported_algorithm(self): - # Fail closed rather than silently skipping verification of native code. - with pytest.raises(RuntimeError, match="Unsupported integrity algorithm"): - _cli_download._verify_integrity(b"bytes", "md5-deadbeef") + with ( + patch.object(_cli_download, "get_cache_dir", return_value=cache_dir), + patch.object(_cli_download, "get_runtime_platform", return_value=runtime_platform), + patch.object(_cli_download, "_fetch_url_bytes", side_effect=concurrent_fetch) as fetch_mock, + ThreadPoolExecutor(max_workers=2) as executor, + ): + futures = [executor.submit(_cli_download.ensure_runtime_wrapper, version) for _ in range(2)] + wrappers = [future.result() for future in futures] + + install_dir = cache_dir / "prebuilds" / runtime_platform + expected_wrapper = str(install_dir / wrapper_name) + assert wrappers == [expected_wrapper, expected_wrapper] + assert (install_dir / wrapper_name).read_bytes() == b"wrapper" + assert (install_dir / "runtime.node").read_bytes() == b"runtime" + assert (install_dir / ".hostless-runtime-assets-v2").is_file() + assert not list((cache_dir / "prebuilds").glob(".runtime-bundle-*")) + assert not (cache_dir / "packages").exists() + assert fetch_mock.call_count == 4 + + +def test_force_restages_complete_bundle_and_compatibility_alias(tmp_path): + version = "1.2.3" + runtime_platform = "linux-x64" + cli_name = "copilot.exe" if os.name == "nt" else "copilot" + wrapper_name = "copilot-runtime.exe" if os.name == "nt" else "copilot-runtime" + data = _release_package(runtime_platform) + cache_dir = tmp_path / "cache" + install_dir = cache_dir / "prebuilds" / runtime_platform + install_dir.mkdir(parents=True) + (install_dir / cli_name).write_bytes(b"old-alias") + (install_dir / wrapper_name).write_bytes(b"old-wrapper") + (install_dir / "runtime.node").write_bytes(b"old-runtime") + (install_dir / ".hostless-runtime-assets-v2").write_text("1\n", encoding="ascii") + fetch = _release_fetches(version, runtime_platform, data) + with ( + patch.object(_cli_download, "get_cache_dir", return_value=cache_dir), + patch.object(_cli_download, "get_runtime_platform", return_value=runtime_platform), + patch.object(_cli_download, "_fetch_url_bytes", side_effect=fetch) as fetch_mock, + ): + cli = _cli_download.download_cli(version, force=True) + + assert cli == str(install_dir / cli_name) + assert (install_dir / cli_name).read_bytes() == b"wrapper" + assert (install_dir / wrapper_name).read_bytes() == b"wrapper" + assert (install_dir / "runtime.node").read_bytes() == b"runtime" + assert fetch_mock.call_count == 2 -class TestEnsureRuntimeLibraryFailsClosed: - def test_raises_when_integrity_unavailable(self, tmp_path): - """A missing npm integrity value must abort the download, not load unverified code.""" - cli_path = tmp_path / "copilot" - cli_path.write_bytes(b"#!/bin/sh\n") - with ( - patch("copilot._ffi_runtime_host.resolve_library_path", return_value=None), - patch.object(_cli_download, "_should_skip_download", return_value=False), - patch.object(_cli_download, "get_npm_platform", return_value="linux-x64"), - patch.object(_cli_download, "get_runtime_lib_url", return_value="https://example/lib"), - patch.object(_cli_download, "_fetch_url_bytes", return_value=b"tarball-bytes"), - patch.object(_cli_download, "_fetch_runtime_integrity", return_value=None), - patch.object(_cli_download, "_extract_runtime_node") as extract, - ): - with pytest.raises(RuntimeError, match="refusing to load unverified native code"): - _cli_download.ensure_runtime_library(str(cli_path), version="1.2.3") +def test_skip_download_returns_none_without_cached_bundle(tmp_path, monkeypatch): + monkeypatch.setenv("COPILOT_SKIP_CLI_DOWNLOAD", "true") + runtime_platform = "linux-x64" + + with ( + patch.object(_cli_download, "get_cache_dir", return_value=tmp_path / "cache"), + patch.object(_cli_download, "get_runtime_platform", return_value=runtime_platform), + patch.object(_cli_download, "_fetch_url_bytes") as fetch_mock, + ): + assert _cli_download.get_or_download_cli("1.2.3") is None - # The library bytes must never be extracted/written when verification is impossible. - extract.assert_not_called() + fetch_mock.assert_not_called() + + +def test_cached_cli_rejects_alias_from_incomplete_bundle(tmp_path): + version = "1.2.3" + runtime_platform = "linux-x64" + cli_name = "copilot.exe" if os.name == "nt" else "copilot" + wrapper_name = "copilot-runtime.exe" if os.name == "nt" else "copilot-runtime" + cache_dir = tmp_path / "cache" + install_dir = cache_dir / "prebuilds" / runtime_platform + install_dir.mkdir(parents=True) + (install_dir / cli_name).write_bytes(b"stale-wrapper") + (install_dir / wrapper_name).write_bytes(b"wrapper") + + with ( + patch.object(_cli_download, "get_cache_dir", return_value=cache_dir), + patch.object(_cli_download, "get_runtime_platform", return_value=runtime_platform), + ): + assert _cli_download.get_cached_cli_path(version) is None + with pytest.raises(RuntimeError, match="Incomplete Copilot runtime bundle"): + _cli_download.download_cli(version) + + +def test_explicit_cli_reuses_library_from_canonical_staged_bundle(tmp_path): + version = "1.2.3" + runtime_platform = "linux-x64" + data = _release_package(runtime_platform) + cache_dir = tmp_path / "cache" + cli_dir = tmp_path / "external" + cli_dir.mkdir() + cli_path = cli_dir / ("copilot.exe" if os.name == "nt" else "copilot") + cli_path.write_bytes(b"external") + fetch = _release_fetches(version, runtime_platform, data) + + with ( + patch.object(_cli_download, "get_cache_dir", return_value=cache_dir), + patch.object(_cli_download, "get_runtime_platform", return_value=runtime_platform), + patch.object(_cli_download, "_fetch_url_bytes", side_effect=fetch) as fetch_mock, + patch("copilot._ffi_runtime_host.resolve_library_path", return_value=None), + ): + library = _cli_download.ensure_runtime_library(str(cli_path), version) + wrapper = _cli_download.ensure_runtime_wrapper(version) + + assert library == str(cli_dir / _ffi_runtime_host._natural_library_name()) + assert (cli_dir / _ffi_runtime_host._natural_library_name()).read_bytes() == b"runtime" + assert (cache_dir / "prebuilds" / runtime_platform / "runtime.node").read_bytes() == b"runtime" + assert not (cache_dir / "packages").exists() + assert wrapper.endswith( + os.path.join( + "prebuilds", + runtime_platform, + "copilot-runtime.exe" if os.name == "nt" else "copilot-runtime", + ) + ) + assert fetch_mock.call_count == 2 def test_resolve_library_path_accepts_adjacent_runtime_node(tmp_path): @@ -108,90 +304,17 @@ def test_resolve_library_path_accepts_adjacent_runtime_node(tmp_path): assert _ffi_runtime_host.resolve_library_path(str(wrapper)) == str(runtime_node) -class TestEnsureRuntimeWrapper: - def test_materializes_pair_from_absent_cache_with_stripped_environment( - self, tmp_path, monkeypatch +def test_rejects_cached_wrapper_without_runtime_node(tmp_path): + runtime_platform = "linux-x64" + wrapper_name = "copilot-runtime.exe" if os.name == "nt" else "copilot-runtime" + cache_dir = tmp_path / "cache" + install_dir = cache_dir / "prebuilds" / runtime_platform + install_dir.mkdir(parents=True) + (install_dir / wrapper_name).write_bytes(b"wrapper") + + with ( + patch.object(_cli_download, "get_cache_dir", return_value=cache_dir), + patch.object(_cli_download, "get_runtime_platform", return_value=runtime_platform), ): - npm_platform = "win32-x64" if os.name == "nt" else "linux-x64" - wrapper_name = "copilot-runtime.exe" if os.name == "nt" else "copilot-runtime" - data = _runtime_package(npm_platform) - cache_dir = tmp_path / "cache" - empty_path = tmp_path / "empty-path" - empty_path.mkdir() - assert not cache_dir.exists() - - for name in ( - "COPILOT_CLI_PATH", - "COPILOT_RUNTIME_HOST_COMMAND", - "COPILOT_RUNTIME_PROVIDER_LIB", - ): - monkeypatch.delenv(name, raising=False) - monkeypatch.setenv("PATH", str(empty_path)) - - with ( - patch.object(_cli_download, "get_cache_dir", return_value=cache_dir), - patch.object(_cli_download, "get_npm_platform", return_value=npm_platform), - patch.object(_cli_download, "_should_skip_download", return_value=False), - patch.object(_cli_download, "_fetch_url_bytes", return_value=data), - patch.object( - _cli_download, - "_fetch_runtime_integrity", - return_value=_integrity(data), - ), - ): - wrapper = _cli_download.ensure_runtime_wrapper(version="1.2.3") - - install_dir = cache_dir / "prebuilds" / npm_platform - assert wrapper == str(install_dir / wrapper_name) - assert (install_dir / wrapper_name).read_bytes() == b"wrapper" - assert (install_dir / "runtime.node").read_bytes() == b"runtime" - assert (install_dir / "ripgrep" / "bin" / npm_platform / "rg").read_bytes() == b"ripgrep" - assert (install_dir / "definitions" / "future.json").read_bytes() == b"{}" - assert not (install_dir / "app.js").exists() - assert not (install_dir / "copilot").exists() - assert not (install_dir / "copilot.exe").exists() - assert (install_dir / ".hostless-runtime-assets-v2").is_file() - if os.name != "nt": - assert (install_dir / wrapper_name).stat().st_mode & 0o111 - - def test_rejects_cached_wrapper_without_runtime_node(self, tmp_path): - npm_platform = "win32-x64" if os.name == "nt" else "linux-x64" - wrapper_name = "copilot-runtime.exe" if os.name == "nt" else "copilot-runtime" - cache_dir = tmp_path / "cache" - install_dir = cache_dir / "prebuilds" / npm_platform - install_dir.mkdir(parents=True) - (install_dir / wrapper_name).write_bytes(b"wrapper") - - with ( - patch.object(_cli_download, "get_cache_dir", return_value=cache_dir), - patch.object(_cli_download, "get_npm_platform", return_value=npm_platform), - ): - with pytest.raises(RuntimeError, match="Incomplete Copilot runtime bundle"): - _cli_download.ensure_runtime_wrapper(version="1.2.3") - - def test_upgrades_pair_only_cache_with_retained_assets(self, tmp_path): - npm_platform = "win32-x64" if os.name == "nt" else "linux-x64" - wrapper_name = "copilot-runtime.exe" if os.name == "nt" else "copilot-runtime" - cache_dir = tmp_path / "cache" - install_dir = cache_dir / "prebuilds" / npm_platform - install_dir.mkdir(parents=True) - (install_dir / wrapper_name).write_bytes(b"old-wrapper") - (install_dir / "runtime.node").write_bytes(b"old-runtime") - (install_dir / "copilot").write_bytes(b"legacy-sea") - (install_dir / ".hostless-runtime-assets-v1").write_text("1\n", encoding="ascii") - data = _runtime_package(npm_platform) - - with ( - patch.object(_cli_download, "get_cache_dir", return_value=cache_dir), - patch.object(_cli_download, "get_npm_platform", return_value=npm_platform), - patch.object(_cli_download, "_should_skip_download", return_value=False), - patch.object(_cli_download, "_fetch_url_bytes", return_value=data), - patch.object(_cli_download, "_fetch_runtime_integrity", return_value=_integrity(data)), - ): - wrapper = _cli_download.ensure_runtime_wrapper(version="1.2.3") - - assert wrapper == str(install_dir / wrapper_name) - assert (install_dir / wrapper_name).read_bytes() == b"wrapper" - assert not (install_dir / "copilot").exists() - assert (install_dir / ".hostless-runtime-assets-v2").is_file() - assert (install_dir / "ripgrep" / "bin" / npm_platform / "rg").is_file() + with pytest.raises(RuntimeError, match="Incomplete Copilot runtime bundle"): + _cli_download.ensure_runtime_wrapper(version="1.2.3") diff --git a/python/test_e2e_harness_cli_path.py b/python/test_e2e_harness_cli_path.py index 83167ffd33..c81fb48d05 100644 --- a/python/test_e2e_harness_cli_path.py +++ b/python/test_e2e_harness_cli_path.py @@ -37,3 +37,20 @@ class Result: with pytest.raises(RuntimeError) as excinfo: context._prepare_pinned_cli(tmp_path) assert "download failed" in str(excinfo.value) + + +def test_inprocess_environment_reuses_prepared_runtime(tmp_path, monkeypatch): + cli = tmp_path / "copilot-runtime" + cli.write_text("runtime\n") + + test_context = context.E2ETestContext() + test_context.cli_path = str(cli) + test_context.work_dir = str(tmp_path) + monkeypatch.setattr(test_context, "get_env", lambda: {"HTTPS_PROXY": "https://proxy"}) + monkeypatch.delenv("COPILOT_CLI_PATH", raising=False) + + try: + test_context._apply_inprocess_environment() + assert context.os.environ["COPILOT_CLI_PATH"] == str(cli) + finally: + test_context._restore_inprocess_environment()