From af0c340f051e71981cd7b7ce227e46888a947cfa Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Thu, 3 Sep 2026 14:51:46 +0200 Subject: [PATCH 1/6] Use CLI release artifacts across SDKs Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: daec7b3b-799c-4396-b372-6eb638d0faf8 --- .github/workflows/publish.yml | 16 +- .../workflows/update-copilot-dependency.yml | 6 +- dotnet/README.md | 7 + dotnet/src/build/GitHub.Copilot.SDK.targets | 102 ++- dotnet/test/Unit/MSBuildTargetsTests.cs | 252 ++++++- go/README.md | 3 + go/cmd/bundler/main.go | 213 ++++-- go/cmd/bundler/main_test.go | 95 ++- go/internal/embeddedcli/embeddedcli.go | 17 +- go/internal/embeddedcli/embeddedcli_test.go | 28 + java/README.md | 4 +- java/copilot-native/pom.xml | 11 +- java/copilot-native/scripts/fetch-native.mjs | 27 +- .../scripts/fetch-native.test.mjs | 181 ++++- .../adr/adr-007-native-bundling-strategy.md | 2 +- .../copilot/ffi/NativeRuntimeLoader.java | 5 +- python/README.md | 19 +- python/copilot/_cli_download.py | 455 +++++------- python/copilot/_cli_version.py | 102 +-- python/copilot/_ffi_runtime_host.py | 4 +- python/test_cli_download.py | 306 ++++---- rust/.gitignore | 1 - rust/Cargo.lock | 82 +-- rust/Cargo.toml | 5 +- rust/README.md | 7 +- rust/build.rs | 2 +- rust/build/out_of_process.rs | 692 ------------------ rust/build/{in_process.rs => runtime.rs} | 137 ++-- rust/scripts/snapshot-bundled-cli-version.sh | 14 +- .../snapshot-bundled-in-process-version.sh | 58 -- rust/src/embeddedcli.rs | 11 +- rust/tests/cli_resolution_test.rs | 33 +- 32 files changed, 1288 insertions(+), 1609 deletions(-) delete mode 100644 rust/build/out_of_process.rs rename rust/build/{in_process.rs => runtime.rs} (88%) delete mode 100755 rust/scripts/snapshot-bundled-in-process-version.sh diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5e1d277259..7a30ba2b19 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -316,17 +316,13 @@ jobs: - name: Set version run: sed -i -E 's/^version = ".*"$/version = "${{ needs.version.outputs.version }}"/' Cargo.toml - name: Snapshot CLI version + hashes for build.rs + run: bash scripts/snapshot-bundled-cli-version.sh + - name: Verify CLI version snapshot exists run: | - bash scripts/snapshot-bundled-cli-version.sh - bash scripts/snapshot-bundled-in-process-version.sh - - name: Verify CLI version snapshots exist - run: | - for snapshot in cli-version.txt cli-version-in-process.txt; do - if [[ ! -f "${snapshot}" ]]; then - echo "::error::${snapshot} was not generated. The Snapshot step must run before packaging." - exit 1 - fi - done + if [[ ! -f cli-version.txt ]]; then + echo "::error::cli-version.txt was not generated. The Snapshot step must run before packaging." + exit 1 + fi - name: Package (dry run) run: cargo publish --dry-run --allow-dirty - name: Upload artifact diff --git a/.github/workflows/update-copilot-dependency.yml b/.github/workflows/update-copilot-dependency.yml index 2870ad27f7..fb4a51f43f 100644 --- a/.github/workflows/update-copilot-dependency.yml +++ b/.github/workflows/update-copilot-dependency.yml @@ -163,7 +163,7 @@ jobs: git commit -m "Update Copilot CLI to $VERSION - - Updated the Node.js CLI release pin + - Updated the shared CLI release pin - Re-ran code generators - Formatted generated code" @@ -173,7 +173,7 @@ jobs: Automated update of the Copilot CLI release to version `PLACEHOLDER_VERSION`. ### Changes - - Updated the release pin in `nodejs/package.json` + - Updated the shared release pin in `nodejs/package.json` - Validated the release assets listed in `SHA256SUMS.txt` - Re-ran all code generators (`scripts/codegen`) - Formatted generated output @@ -221,7 +221,7 @@ jobs: else gh pr create \ --draft \ - --title "Update @github/copilot to $VERSION" \ + --title "Update Copilot CLI to $VERSION" \ --body "$PR_BODY" \ --base main \ --head "$BRANCH" diff --git a/dotnet/README.md b/dotnet/README.md index 23a78030b4..40c7a0bdae 100644 --- a/dotnet/README.md +++ b/dotnet/README.md @@ -14,6 +14,13 @@ To use the SDK, you'll need: dotnet add package GitHub.Copilot.SDK ``` +The package downloads the pinned Copilot CLI runtime for the build RID from the +matching `github/copilot-cli` GitHub release and verifies the archive against +that release's `SHA256SUMS.txt`. Set `CopilotCliReleaseBaseUrl` in MSBuild (or +`COPILOT_CLI_DOWNLOAD_BASE_URL` in the environment) to use a release mirror. +Set `CopilotCliBinaryPath` to copy a preinstalled binary instead, or set +`CopilotSkipCliDownload=true` to omit runtime acquisition. + ## Run the Samples Try the interactive chat sample (from the repo root): diff --git a/dotnet/src/build/GitHub.Copilot.SDK.targets b/dotnet/src/build/GitHub.Copilot.SDK.targets index aca299dd27..d5258480c6 100644 --- a/dotnet/src/build/GitHub.Copilot.SDK.targets +++ b/dotnet/src/build/GitHub.Copilot.SDK.targets @@ -26,7 +26,7 @@ - + <_CopilotPlatform Condition="'$(_CopilotRid)' == 'win-x64'">win32-x64 <_CopilotPlatform Condition="'$(_CopilotRid)' == 'win-arm64'">win32-arm64 @@ -49,15 +49,19 @@ <_CopilotRuntimeLib Condition="'$(_CopilotRuntimeLib)' == ''">libcopilot_runtime.so - + COPILOT_CLI_DOWNLOAD_BASE_URL is also honored. CopilotNpmRegistryUrl remains + a compatibility alias, but its value is interpreted as a release base URL; + these targets never fall back to npm. --> - https://registry.npmjs.org + $(COPILOT_CLI_DOWNLOAD_BASE_URL) + $(CopilotNpmRegistryUrl) + https://github.com/github/copilot-cli/releases/download @@ -93,23 +97,58 @@ <_CopilotCacheDir>$(IntermediateOutputPath)copilot-cli\$(CopilotCliVersion)\$(_CopilotPlatform) - <_CopilotCliBinaryPath Condition="'$(_CopilotCliBinaryPath)' == ''">$(_CopilotCacheDir)\$(_CopilotBinary) + + <_CopilotCliBinaryPath Condition="'$(_CopilotCliBinaryPath)' == ''">$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\$(_CopilotRuntimeWrapper) + <_CopilotRuntimeNodePath>$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\runtime.node + <_CopilotRuntimeBundleCompleteMarker>$(_CopilotCacheDir)\.copilot-runtime-complete <_CopilotArchivePath>$(_CopilotCacheDir)\copilot.tgz - <_CopilotNormalizedRegistryUrl>$([System.String]::Copy('$(CopilotNpmRegistryUrl)').TrimEnd('/')) - <_CopilotDownloadUrl>$(_CopilotNormalizedRegistryUrl)/@github/copilot-$(_CopilotPlatform)/-/copilot-$(_CopilotPlatform)-$(CopilotCliVersion).tgz + <_CopilotChecksumPath>$(_CopilotCacheDir)\SHA256SUMS.txt + <_CopilotAssetName>github-copilot-$(CopilotCliVersion)-$(_CopilotPlatform).tgz + <_CopilotAssetNameRegex>$([System.Text.RegularExpressions.Regex]::Escape('$(_CopilotAssetName)')) + <_CopilotNormalizedReleaseBaseUrl>$([System.String]::Copy('$(CopilotCliReleaseBaseUrl)').TrimEnd('/')) + <_CopilotReleaseUrl>$(_CopilotNormalizedReleaseBaseUrl)/v$(CopilotCliVersion) + <_CopilotDownloadUrl>$(_CopilotReleaseUrl)/$(_CopilotAssetName) + <_CopilotChecksumsUrl>$(_CopilotReleaseUrl)/SHA256SUMS.txt + <_CopilotRuntimeBundleMissing Condition="!Exists('$(_CopilotCliBinaryPath)') Or !Exists('$(_CopilotRuntimeNodePath)') Or !Exists('$(_CopilotRuntimeBundleCompleteMarker)')">true <_CopilotCliDownloadTimeoutMs>$([System.Convert]::ToInt32($([MSBuild]::Multiply($(CopilotCliDownloadTimeout), 1000)))) - - + + - - + + + + + + + + <_CopilotChecksumMatch Include="@(_CopilotChecksumLine)" + Condition="$([System.Text.RegularExpressions.Regex]::IsMatch('%(Identity)', '^[0-9a-fA-F]{64}[\t ]+\*?$(_CopilotAssetNameRegex)[\t ]*$'))" /> + + + Condition="'$(_CopilotRuntimeBundleMissing)' == 'true'" /> + + + + + <_CopilotChecksumLineValue>@(_CopilotChecksumMatch) + <_CopilotArchiveHashValue>@(_CopilotArchiveHash->'%(FileHash)') + <_CopilotExpectedChecksum>$([System.String]::Copy('$(_CopilotChecksumLineValue)').Substring(0, 64).ToUpperInvariant()) + <_CopilotActualChecksum>$([System.String]::Copy('$(_CopilotArchiveHashValue)').ToUpperInvariant()) + <_CopilotChecksumMismatch Condition="'$(_CopilotExpectedChecksum)' != '$(_CopilotActualChecksum)'">true + + + @@ -117,23 +156,26 @@ <_TarCommand Condition="'$(_TarCommand)' == ''">tar + Condition="'$(_CopilotRuntimeBundleMissing)' == 'true'" /> - + + + - + <_CopilotCacheDir Condition="'$(_CopilotCacheDir)' == ''">$(IntermediateOutputPath)copilot-cli\$(CopilotCliVersion)\$(_CopilotPlatform) - <_CopilotCliBinaryPath Condition="'$(_CopilotCliBinaryPath)' == ''">$(_CopilotCacheDir)\$(_CopilotBinary) + <_CopilotCliBinaryPath Condition="'$(_CopilotCliBinaryPath)' == ''">$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\$(_CopilotRuntimeWrapper) <_CopilotOutputDir>$(OutDir)runtimes\$(_CopilotRid)\native <_CopilotRuntimeNodePath>$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\runtime.node <_CopilotRuntimeWrapperPath Condition="'$(_CopilotRuntimeWrapperPath)' == ''">$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\$(_CopilotRuntimeWrapper) @@ -153,7 +195,7 @@ <_CopilotRuntimeRootAsset Include="$(_CopilotCacheDir)\**\*" - Exclude="$(_CopilotCacheDir)\app.js;$(_CopilotCacheDir)\assets\**\*;$(_CopilotCacheDir)\changelog.json;$(_CopilotCacheDir)\copilot;$(_CopilotCacheDir)\copilot.exe;$(_CopilotCacheDir)\copilot.tgz;$(_CopilotCacheDir)\foundry-local-sdk\**\*;$(_CopilotCacheDir)\index.js;$(_CopilotCacheDir)\LICENSE.md;$(_CopilotCacheDir)\napi-oop-runtime\**\*;$(_CopilotCacheDir)\npm-loader.js;$(_CopilotCacheDir)\package.json;$(_CopilotCacheDir)\prebuilds\**\*;$(_CopilotCacheDir)\pvrecorder\**\*;$(_CopilotCacheDir)\queries\**\*;$(_CopilotCacheDir)\README.md;$(_CopilotCacheDir)\sea-loader.js;$(_CopilotCacheDir)\tree-sitter*.wasm;$(_CopilotCacheDir)\voice-*.js;$(_CopilotCacheDir)\webview\**\*" /> + Exclude="$(_CopilotCacheDir)\.copilot-runtime-complete;$(_CopilotCacheDir)\app.js;$(_CopilotCacheDir)\assets\**\*;$(_CopilotCacheDir)\changelog.json;$(_CopilotCacheDir)\copilot;$(_CopilotCacheDir)\copilot.exe;$(_CopilotCacheDir)\copilot.tgz;$(_CopilotCacheDir)\foundry-local-sdk\**\*;$(_CopilotCacheDir)\index.js;$(_CopilotCacheDir)\LICENSE.md;$(_CopilotCacheDir)\napi-oop-runtime\**\*;$(_CopilotCacheDir)\npm-loader.js;$(_CopilotCacheDir)\package.json;$(_CopilotCacheDir)\prebuilds\**\*;$(_CopilotCacheDir)\pvrecorder\**\*;$(_CopilotCacheDir)\queries\**\*;$(_CopilotCacheDir)\README.md;$(_CopilotCacheDir)\sea-loader.js;$(_CopilotCacheDir)\SHA256SUMS.txt;$(_CopilotCacheDir)\tree-sitter*.wasm;$(_CopilotCacheDir)\voice-*.js;$(_CopilotCacheDir)\webview\**\*" /> <_CopilotRuntimePrebuildAsset Include="$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\**\*" Exclude="$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\cli-native.node;$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\mediaremote-adapter\**\*;$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\copilot-runtime-bin*" /> @@ -176,12 +218,12 @@ - <_CopilotCacheDir Condition="'$(_CopilotCacheDir)' == ''">$(IntermediateOutputPath)copilot-cli\$(CopilotCliVersion)\$(_CopilotPlatform) - <_CopilotCliBinaryPath Condition="'$(_CopilotCliBinaryPath)' == ''">$(_CopilotCacheDir)\$(_CopilotBinary) + <_CopilotCliBinaryPath Condition="'$(_CopilotCliBinaryPath)' == ''">$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\$(_CopilotRuntimeWrapper) <_CopilotRuntimeNodePath>$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\runtime.node <_CopilotRuntimeWrapperPath Condition="'$(_CopilotRuntimeWrapperPath)' == ''">$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\$(_CopilotRuntimeWrapper) <_CopilotExplicitCliMarker>$(_CopilotCacheDir)\.copilot-explicit-cli @@ -193,7 +235,7 @@ Condition="'$(CopilotCliBinaryPath)' != ''" /> <_CopilotRuntimeRootAsset Include="$(_CopilotCacheDir)\**\*" - Exclude="$(_CopilotCacheDir)\app.js;$(_CopilotCacheDir)\assets\**\*;$(_CopilotCacheDir)\changelog.json;$(_CopilotCacheDir)\copilot;$(_CopilotCacheDir)\copilot.exe;$(_CopilotCacheDir)\copilot.tgz;$(_CopilotCacheDir)\foundry-local-sdk\**\*;$(_CopilotCacheDir)\index.js;$(_CopilotCacheDir)\LICENSE.md;$(_CopilotCacheDir)\napi-oop-runtime\**\*;$(_CopilotCacheDir)\npm-loader.js;$(_CopilotCacheDir)\package.json;$(_CopilotCacheDir)\prebuilds\**\*;$(_CopilotCacheDir)\pvrecorder\**\*;$(_CopilotCacheDir)\queries\**\*;$(_CopilotCacheDir)\README.md;$(_CopilotCacheDir)\sea-loader.js;$(_CopilotCacheDir)\tree-sitter*.wasm;$(_CopilotCacheDir)\voice-*.js;$(_CopilotCacheDir)\webview\**\*" + Exclude="$(_CopilotCacheDir)\.copilot-runtime-complete;$(_CopilotCacheDir)\app.js;$(_CopilotCacheDir)\assets\**\*;$(_CopilotCacheDir)\changelog.json;$(_CopilotCacheDir)\copilot;$(_CopilotCacheDir)\copilot.exe;$(_CopilotCacheDir)\copilot.tgz;$(_CopilotCacheDir)\foundry-local-sdk\**\*;$(_CopilotCacheDir)\index.js;$(_CopilotCacheDir)\LICENSE.md;$(_CopilotCacheDir)\napi-oop-runtime\**\*;$(_CopilotCacheDir)\npm-loader.js;$(_CopilotCacheDir)\package.json;$(_CopilotCacheDir)\prebuilds\**\*;$(_CopilotCacheDir)\pvrecorder\**\*;$(_CopilotCacheDir)\queries\**\*;$(_CopilotCacheDir)\README.md;$(_CopilotCacheDir)\sea-loader.js;$(_CopilotCacheDir)\SHA256SUMS.txt;$(_CopilotCacheDir)\tree-sitter*.wasm;$(_CopilotCacheDir)\voice-*.js;$(_CopilotCacheDir)\webview\**\*" Condition="Exists('$(_CopilotRuntimeWrapperPath)') And Exists('$(_CopilotRuntimeNodePath)')" /> <_CopilotRuntimePrebuildAsset Include="$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\**\*" Exclude="$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\cli-native.node;$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\mediaremote-adapter\**\*;$(_CopilotCacheDir)\prebuilds\$(_CopilotPlatform)\copilot-runtime-bin*" diff --git a/dotnet/test/Unit/MSBuildTargetsTests.cs b/dotnet/test/Unit/MSBuildTargetsTests.cs index 0dd3073956..fa610006b1 100644 --- a/dotnet/test/Unit/MSBuildTargetsTests.cs +++ b/dotnet/test/Unit/MSBuildTargetsTests.cs @@ -2,8 +2,12 @@ * Copyright (c) Microsoft Corporation. All rights reserved. *--------------------------------------------------------------------------------------------*/ +using System.Collections.Concurrent; using System.Diagnostics; +using System.Net; +using System.Net.Sockets; using System.Runtime.CompilerServices; +using System.Security.Cryptography; using System.Text; using Xunit; @@ -16,11 +20,7 @@ namespace GitHub.Copilot.Test.Unit; /// a subprocess so we exercise real MSBuild evaluation. /// /// -/// These tests deliberately do not exercise the network-bound default download path; they -/// pin a fake CopilotCliVersion and supply a fake CLI binary via -/// CopilotCliBinaryPath. That is sufficient to cover the regression in issue -/// #921 ("preinstalled CLI is ignored and copy/register are skipped when -/// CopilotSkipCliDownload=true"). +/// Download tests use a loopback release server; they never access the default GitHub URL. /// public class MSBuildTargetsTests { @@ -28,6 +28,9 @@ public class MSBuildTargetsTests private static readonly string BinaryName = OperatingSystem.IsWindows() ? "copilot.exe" : "copilot"; + private static readonly string RuntimeWrapperName = + OperatingSystem.IsWindows() ? "copilot-runtime.exe" : "copilot-runtime"; + [Fact] public async Task PreinstalledCliBinaryPath_IsHonored_DownloadSkipped_AndCopiedToOutput() { @@ -106,15 +109,104 @@ public async Task PreinstalledCliBinaryPath_WithSkipCliDownload_StillCopiesToOut Assert.True(File.Exists(sandbox.ExpectedOutputBinary()), result.FailureMessage()); } + [Fact] + public async Task ReleaseAsset_IsDownloadedVerifiedExtractedAndCached() + { + using var sandbox = MSBuildSandbox.Create(); + var archive = sandbox.CreateReleaseArchive("release-runtime-wrapper"); + var assetName = $"github-copilot-0.0.0-test-{GetReleasePlatform()}.tgz"; + var assetPath = $"/v0.0.0-test/{assetName}"; + var checksumsPath = "/v0.0.0-test/SHA256SUMS.txt"; + var checksum = ComputeSha256(archive); + using var server = new ReleaseServer(new Dictionary + { + [checksumsPath] = Encoding.UTF8.GetBytes($"{checksum} {assetName}\n"), + [assetPath] = archive, + }); + + var properties = new Dictionary + { + ["CopilotCliReleaseBaseUrl"] = server.BaseUrl, + }; + var firstBuild = await sandbox.BuildAsync(properties); + + Assert.True(firstBuild.Succeeded, firstBuild.FailureMessage()); + Assert.Equal("release-runtime-wrapper", File.ReadAllText(sandbox.ExpectedOutputBinary())); + Assert.Equal("release-runtime-wrapper", File.ReadAllText(sandbox.ExpectedRuntimeAsset(RuntimeWrapperName))); + Assert.Equal("runtime", File.ReadAllText(sandbox.ExpectedRuntimeAsset("runtime.node"))); + Assert.True(File.Exists(sandbox.ExpectedCacheAsset(".copilot-runtime-complete"))); + Assert.False(File.Exists(sandbox.ExpectedRuntimeAsset(".copilot-runtime-complete"))); + Assert.Equal(1, server.RequestPaths.Count(path => path == checksumsPath)); + Assert.Equal(1, server.RequestPaths.Count(path => path == assetPath)); + Assert.False(File.Exists(sandbox.ExpectedRuntimeAsset("SHA256SUMS.txt"))); + + var secondBuild = await sandbox.BuildAsync(properties); + + Assert.True(secondBuild.Succeeded, secondBuild.FailureMessage()); + Assert.Equal(2, server.RequestPaths.Count); + } + + [Fact] + public async Task IncompleteCache_WithRuntimePairButNoMarker_IsReacquired() + { + using var sandbox = MSBuildSandbox.Create(); + sandbox.WriteRuntimeCacheAsset("prebuilds", GetReleasePlatform(), RuntimeWrapperName, "partial-wrapper"); + sandbox.WriteRuntimeCacheAsset("prebuilds", GetReleasePlatform(), "runtime.node", "partial-runtime"); + var archive = sandbox.CreateReleaseArchive("complete-wrapper"); + var assetName = $"github-copilot-0.0.0-test-{GetReleasePlatform()}.tgz"; + var assetPath = $"/v0.0.0-test/{assetName}"; + var checksumsPath = "/v0.0.0-test/SHA256SUMS.txt"; + using var server = new ReleaseServer(new Dictionary + { + [checksumsPath] = Encoding.UTF8.GetBytes($"{ComputeSha256(archive)} {assetName}\n"), + [assetPath] = archive, + }); + + var result = await sandbox.BuildAsync(new Dictionary + { + ["CopilotCliReleaseBaseUrl"] = server.BaseUrl, + }); + + Assert.True(result.Succeeded, result.FailureMessage()); + Assert.Equal(1, server.RequestPaths.Count(path => path == checksumsPath)); + Assert.Equal(1, server.RequestPaths.Count(path => path == assetPath)); + Assert.Equal("complete-wrapper", File.ReadAllText(sandbox.ExpectedRuntimeAsset(RuntimeWrapperName))); + Assert.Equal("runtime", File.ReadAllText(sandbox.ExpectedRuntimeAsset("runtime.node"))); + Assert.True(File.Exists(sandbox.ExpectedCacheAsset(".copilot-runtime-complete"))); + } + + [Fact] + public async Task ReleaseAsset_WithChecksumMismatch_FailsBeforeExtraction() + { + using var sandbox = MSBuildSandbox.Create(); + var archive = Encoding.UTF8.GetBytes("not the expected archive"); + var assetName = $"github-copilot-0.0.0-test-{GetReleasePlatform()}.tgz"; + using var server = new ReleaseServer(new Dictionary + { + ["/v0.0.0-test/SHA256SUMS.txt"] = + Encoding.UTF8.GetBytes($"{new string('0', 64)} *{assetName}\n"), + [$"/v0.0.0-test/{assetName}"] = archive, + }); + + var result = await sandbox.BuildAsync(new Dictionary + { + ["CopilotCliReleaseBaseUrl"] = server.BaseUrl, + }); + + Assert.False(result.Succeeded, "Build should fail when the release checksum does not match."); + Assert.Contains($"Checksum mismatch for {assetName}", result.StandardOutput, StringComparison.Ordinal); + Assert.False(File.Exists(sandbox.ExpectedOutputBinary())); + } + [Fact] public async Task RuntimePackageAssets_AreFilteredAndCopiedToOutput() { using var sandbox = MSBuildSandbox.Create(); var preinstalled = sandbox.WritePreinstalledBinary("fake-cli-contents"); - sandbox.WriteRuntimeCacheAsset("prebuilds", GetNpmPlatform(), "runtime.node", "runtime"); - sandbox.WriteRuntimeCacheAsset("prebuilds", GetNpmPlatform(), - OperatingSystem.IsWindows() ? "copilot-runtime.exe" : "copilot-runtime", "wrapper"); - sandbox.WriteRuntimeCacheAsset("ripgrep", "bin", GetNpmPlatform(), "rg", "ripgrep"); + sandbox.WriteRuntimeCacheAsset("prebuilds", GetReleasePlatform(), "runtime.node", "runtime"); + sandbox.WriteRuntimeCacheAsset("prebuilds", GetReleasePlatform(), + RuntimeWrapperName, "wrapper"); + sandbox.WriteRuntimeCacheAsset("ripgrep", "bin", GetReleasePlatform(), "rg", "ripgrep"); sandbox.WriteRuntimeCacheAsset("definitions", "future.json", "{}"); sandbox.WriteRuntimeCacheAsset("copilot-sdk", "extension.js", "extension"); sandbox.WriteRuntimeCacheAsset("preloads", "extension_bootstrap.mjs", "preload"); @@ -130,7 +222,7 @@ public async Task RuntimePackageAssets_AreFilteredAndCopiedToOutput() }); Assert.True(result.Succeeded, result.FailureMessage()); - Assert.Equal("ripgrep", File.ReadAllText(sandbox.ExpectedRuntimeAsset("ripgrep", "bin", GetNpmPlatform(), "rg"))); + Assert.Equal("ripgrep", File.ReadAllText(sandbox.ExpectedRuntimeAsset("ripgrep", "bin", GetReleasePlatform(), "rg"))); Assert.Equal("{}", File.ReadAllText(sandbox.ExpectedRuntimeAsset("definitions", "future.json"))); Assert.Equal("extension", File.ReadAllText(sandbox.ExpectedRuntimeAsset("copilot-sdk", "extension.js"))); Assert.Equal("preload", File.ReadAllText(sandbox.ExpectedRuntimeAsset("preloads", "extension_bootstrap.mjs"))); @@ -186,7 +278,7 @@ private static string FindTargetsFile([CallerFilePath] string? thisFile = null) "Could not locate GitHub.Copilot.SDK.targets relative to test assembly or source file."); } - private static string GetNpmPlatform() + private static string GetReleasePlatform() { var arch = System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture == System.Runtime.InteropServices.Architecture.Arm64 @@ -197,6 +289,16 @@ private static string GetNpmPlatform() return $"linux-{arch}"; } + private static string ComputeSha256(byte[] contents) + { +#if NETFRAMEWORK + using var sha256 = SHA256.Create(); + return BitConverter.ToString(sha256.ComputeHash(contents)).Replace("-", "").ToLowerInvariant(); +#else + return Convert.ToHexString(SHA256.HashData(contents)).ToLowerInvariant(); +#endif + } + /// /// A throwaway directory containing a minimal csproj that imports the SDK targets /// file. Disposing removes the directory tree. @@ -244,6 +346,36 @@ public string WritePreinstalledBinary(string contents, string? fileName = null) return path; } + public byte[] CreateReleaseArchive(string runtimeWrapperContents) + { + var sourceDir = Path.Combine(ProjectDir, "release-source"); + var packageDir = Path.Combine(sourceDir, "package"); + var prebuildDir = Path.Combine(packageDir, "prebuilds", GetReleasePlatform()); + Directory.CreateDirectory(prebuildDir); + File.WriteAllText(Path.Combine(prebuildDir, "runtime.node"), "runtime"); + File.WriteAllText(Path.Combine(prebuildDir, RuntimeWrapperName), runtimeWrapperContents); + + var archivePath = Path.Combine(ProjectDir, "release-asset.tgz"); + var tarPath = OperatingSystem.IsWindows() + ? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Windows), "System32", "tar.exe") + : "tar"; + var startInfo = new ProcessStartInfo(tarPath) + { + Arguments = $"-czf \"{archivePath}\" -C \"{sourceDir}\" package", + RedirectStandardError = true, + UseShellExecute = false, + }; + using var process = Process.Start(startInfo) ?? + throw new InvalidOperationException("Failed to start tar while creating a release test asset."); + var standardError = process.StandardError.ReadToEnd(); + process.WaitForExit(); + if (process.ExitCode != 0) + { + throw new InvalidOperationException($"tar failed while creating a release test asset: {standardError}"); + } + return File.ReadAllBytes(archivePath); + } + public string ExpectedOutputBinary() { var rid = GetPortableRid(); @@ -253,14 +385,20 @@ public string ExpectedOutputBinary() public void WriteRuntimeCacheAsset(params string[] pathAndContents) { var pathParts = pathAndContents.Take(pathAndContents.Length - 1).ToArray(); + var path = ExpectedCacheAsset(pathParts); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, pathAndContents[^1]); + } + + public string ExpectedCacheAsset(params string[] pathParts) + { var path = Path.Combine(ProjectDir, "obj", "Debug", "net8.0", "copilot-cli", "0.0.0-test", - GetNpmPlatform()); + GetReleasePlatform()); foreach (var part in pathParts) { path = Path.Combine(path, part); } - Directory.CreateDirectory(Path.GetDirectoryName(path)!); - File.WriteAllText(path, pathAndContents[^1]); + return path; } public string ExpectedRuntimeAsset(params string[] pathParts) @@ -372,6 +510,92 @@ private static string GetPortableRid() } } + private sealed class ReleaseServer : IDisposable + { + private readonly IReadOnlyDictionary _responses; + private readonly TcpListener _listener = new(IPAddress.Loopback, 0); + private readonly CancellationTokenSource _cancellation = new(); + private readonly Task _serverTask; + + public ReleaseServer(IReadOnlyDictionary responses) + { + _responses = responses; + _listener.Start(); + var endpoint = (IPEndPoint)_listener.LocalEndpoint; + BaseUrl = $"http://127.0.0.1:{endpoint.Port}"; + _serverTask = ServeAsync(); + } + + public string BaseUrl { get; } + + public ConcurrentQueue RequestPaths { get; } = new(); + + public void Dispose() + { + _cancellation.Cancel(); + _listener.Stop(); + try { _serverTask.GetAwaiter().GetResult(); } + catch (OperationCanceledException) { } + _cancellation.Dispose(); + } + + private async Task ServeAsync() + { + while (!_cancellation.IsCancellationRequested) + { + TcpClient client; + try + { + client = await _listener.AcceptTcpClientAsync(); + } + catch (ObjectDisposedException) when (_cancellation.IsCancellationRequested) + { + break; + } + catch (SocketException) when (_cancellation.IsCancellationRequested) + { + break; + } + await RespondAsync(client); + } + } + + private async Task RespondAsync(TcpClient client) + { + using (client) + { + var stream = client.GetStream(); + using var reader = new StreamReader(stream, Encoding.ASCII, false, 1024, leaveOpen: true); + var requestLine = await reader.ReadLineAsync(); + string? header; + do + { + header = await reader.ReadLineAsync(); + } + while (!string.IsNullOrEmpty(header)); + + var path = requestLine?.Split(' ', StringSplitOptions.RemoveEmptyEntries).ElementAtOrDefault(1) ?? ""; + RequestPaths.Enqueue(path); + var found = _responses.TryGetValue(path, out var body); + body ??= Encoding.UTF8.GetBytes("Not found"); + var status = found ? "200 OK" : "404 Not Found"; + var responseHeaders = Encoding.ASCII.GetBytes( + $"HTTP/1.1 {status}\r\nContent-Length: {body.Length}\r\nConnection: close\r\n\r\n"); + await WriteBytesAsync(stream, responseHeaders); + await WriteBytesAsync(stream, body); + } + } + + private static Task WriteBytesAsync(Stream stream, byte[] contents) + { +#if NETFRAMEWORK + return stream.WriteAsync(contents, 0, contents.Length); +#else + return stream.WriteAsync(contents).AsTask(); +#endif + } + } + private sealed record BuildResult(int ExitCode, string StandardOutput, string StandardError, string CommandLine) { public bool Succeeded => ExitCode == 0; diff --git a/go/README.md b/go/README.md index 801ad8556c..29ecc69210 100644 --- a/go/README.md +++ b/go/README.md @@ -101,6 +101,9 @@ tool name is `-`. For `AvailableTools` and The SDK supports bundling, using Go's `embed` package, the Copilot CLI binary within your application's distribution. This allows you to bundle a specific CLI version and avoid external dependencies on the user's system. +The bundler downloads the matching `github-copilot--.tgz` +asset from the `github/copilot-cli` release and verifies it against that +release's `SHA256SUMS.txt`. Follow these steps to embed the CLI: diff --git a/go/cmd/bundler/main.go b/go/cmd/bundler/main.go index 7ee078eea0..0d79243715 100644 --- a/go/cmd/bundler/main.go +++ b/go/cmd/bundler/main.go @@ -36,28 +36,28 @@ import ( const ( // Keep these URLs centralized so reviewers can verify all outbound calls in one place. - sdkModule = "github.com/github/copilot-sdk/go" - packageJSONURLFmt = "https://raw.githubusercontent.com/github/copilot-sdk/%s/nodejs/package.json" - packageLockURLFmt = "https://raw.githubusercontent.com/github/copilot-sdk/%s/nodejs/package-lock.json" - tarballURLFmt = "https://registry.npmjs.org/@github/copilot-%s/-/copilot-%s-%s.tgz" - licenseTarballFmt = "https://registry.npmjs.org/@github/copilot/-/copilot-%s.tgz" - defaultPackageName = "main" + sdkModule = "github.com/github/copilot-sdk/go" + packageJSONURLFmt = "https://raw.githubusercontent.com/github/copilot-sdk/%s/nodejs/package.json" + packageLockURLFmt = "https://raw.githubusercontent.com/github/copilot-sdk/%s/nodejs/package-lock.json" + defaultCLIDownloadBaseURL = "https://github.com/github/copilot-cli/releases/download" + cliDownloadBaseURLEnvironment = "COPILOT_CLI_DOWNLOAD_BASE_URL" + defaultPackageName = "main" ) -// Platform info: npm package suffix, binary name +// Platform info: release asset platform suffix, binary name type platformInfo struct { - npmPlatform string - binaryName string + runtimePlatform string + binaryName string } -// Map from GOOS/GOARCH to npm platform info +// Map from GOOS/GOARCH to release asset platform info. var platforms = map[string]platformInfo{ - "linux/amd64": {npmPlatform: "linux-x64", binaryName: "copilot"}, - "linux/arm64": {npmPlatform: "linux-arm64", binaryName: "copilot"}, - "darwin/amd64": {npmPlatform: "darwin-x64", binaryName: "copilot"}, - "darwin/arm64": {npmPlatform: "darwin-arm64", binaryName: "copilot"}, - "windows/amd64": {npmPlatform: "win32-x64", binaryName: "copilot.exe"}, - "windows/arm64": {npmPlatform: "win32-arm64", binaryName: "copilot.exe"}, + "linux/amd64": {runtimePlatform: "linux-x64", binaryName: "copilot"}, + "linux/arm64": {runtimePlatform: "linux-arm64", binaryName: "copilot"}, + "darwin/amd64": {runtimePlatform: "darwin-x64", binaryName: "copilot"}, + "darwin/arm64": {runtimePlatform: "darwin-arm64", binaryName: "copilot"}, + "windows/amd64": {runtimePlatform: "win32-x64", binaryName: "copilot.exe"}, + "windows/arm64": {runtimePlatform: "win32-arm64", binaryName: "copilot.exe"}, } // main is the CLI entry point. @@ -70,7 +70,7 @@ func main() { // Resolve version first so the default output name can include it. version := resolveCLIVersion(*cliVersion) - // Resolve platform once to validate input and get the npm package mapping. + // Resolve platform once to validate input and get the release asset mapping. goos, goarch, info, err := resolvePlatform(*platform) if err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) @@ -101,7 +101,7 @@ func main() { fmt.Printf("Building bundle for %s (CLI version %s)\n", *platform, version) - bundle, err := buildBundle(info, version, outputPath, goos) + bundle, err := buildBundle(info, version, outputPath, goos, true) if err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) os.Exit(1) @@ -110,8 +110,8 @@ func main() { var muslBundle bundleArtifacts if goos == "linux" { muslInfo := platformInfo{ - npmPlatform: strings.Replace(info.npmPlatform, "linux-", "linuxmusl-", 1), - binaryName: info.binaryName, + runtimePlatform: strings.Replace(info.runtimePlatform, "linux-", "linuxmusl-", 1), + binaryName: info.binaryName, } muslOutputPath := filepath.Join(*output, defaultOutputFileName(version, "linuxmusl", goarch, info.binaryName)) muslBundle, err = buildBundle( @@ -119,17 +119,13 @@ func main() { version, muslOutputPath, goos, + false, ) if err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) os.Exit(1) } } - if err := downloadCLILicense(version, outputPath); err != nil { - fmt.Fprintf(os.Stderr, "Error: failed to download CLI license: %v\n", err) - os.Exit(1) - } - // Generate the Go file with embed directive if err := generateGoFile( goos, @@ -397,19 +393,23 @@ type bundleArtifacts struct { assetsHash []byte } -// buildBundle downloads the CLI and native runtime artifacts from one platform package. -func buildBundle(info platformInfo, cliVersion, outputPath, goos string) (bundleArtifacts, error) { +// buildBundle downloads the CLI and native runtime artifacts from one release package. +func buildBundle(info platformInfo, cliVersion, outputPath, goos string, includeLicense bool) (bundleArtifacts, error) { outputDir := filepath.Dir(outputPath) if outputDir == "" { outputDir = "." } - runtimeArtifactPath := filepath.Join(outputDir, runtimeLibArtifactName(cliVersion, info.npmPlatform, goos)) - wrapperArtifactPath := filepath.Join(outputDir, runtimeWrapperArtifactName(cliVersion, info.npmPlatform, info.binaryName)) - assetsArtifactPath := filepath.Join(outputDir, runtimeAssetsArtifactName(cliVersion, info.npmPlatform)) + runtimeArtifactPath := filepath.Join(outputDir, runtimeLibArtifactName(cliVersion, info.runtimePlatform, goos)) + wrapperArtifactPath := filepath.Join(outputDir, runtimeWrapperArtifactName(cliVersion, info.runtimePlatform, info.binaryName)) + assetsArtifactPath := filepath.Join(outputDir, runtimeAssetsArtifactName(cliVersion, info.runtimePlatform)) + requiredPaths := []string{outputPath, runtimeArtifactPath, wrapperArtifactPath, assetsArtifactPath} + if includeLicense { + requiredPaths = append(requiredPaths, licensePathForOutput(outputPath)) + } - if filesExist(outputPath, runtimeArtifactPath, wrapperArtifactPath, assetsArtifactPath) { + if filesExist(requiredPaths...) { // Idempotent output avoids re-downloading in CI or local rebuilds. - fmt.Printf("Output runtime bundle for %s already exists, skipping download\n", info.npmPlatform) + fmt.Printf("Output runtime bundle for %s already exists, skipping download\n", info.runtimePlatform) binaryHash, err := sha256FileFromCompressed(outputPath) if err != nil { return bundleArtifacts{}, fmt.Errorf("failed to hash existing output: %w", err) @@ -436,7 +436,7 @@ func buildBundle(info platformInfo, cliVersion, outputPath, goos string) (bundle } defer os.RemoveAll(tempDir) - binaryPath, tarballPath, err := downloadCLIBinary(info.npmPlatform, info.binaryName, cliVersion, tempDir) + binaryPath, tarballPath, err := downloadCLIBinary(info.runtimePlatform, info.binaryName, cliVersion, tempDir) if err != nil { return bundleArtifacts{}, fmt.Errorf("failed to download CLI binary: %w", err) } @@ -446,6 +446,11 @@ func buildBundle(info platformInfo, cliVersion, outputPath, goos string) (bundle return bundleArtifacts{}, fmt.Errorf("failed to create output directory: %w", err) } } + if includeLicense { + if err := extractCLILicense(tarballPath, outputPath); err != nil { + return bundleArtifacts{}, fmt.Errorf("failed to extract CLI license: %w", err) + } + } binaryHash, err := sha256File(binaryPath) if err != nil { @@ -459,10 +464,10 @@ func buildBundle(info platformInfo, cliVersion, outputPath, goos string) (bundle if err := extractFileFromTarball( tarballPath, tempDir, - "package/prebuilds/"+info.npmPlatform+"/runtime.node", + "package/prebuilds/"+info.runtimePlatform+"/runtime.node", "runtime.node", ); err != nil { - return bundleArtifacts{}, fmt.Errorf("runtime package is missing prebuilds/%s/runtime.node: %w", info.npmPlatform, err) + return bundleArtifacts{}, fmt.Errorf("runtime package is missing prebuilds/%s/runtime.node: %w", info.runtimePlatform, err) } runtimeHash, err := sha256File(rawLibPath) if err != nil { @@ -477,10 +482,10 @@ func buildBundle(info platformInfo, cliVersion, outputPath, goos string) (bundle if err := extractFileFromTarball( tarballPath, tempDir, - "package/prebuilds/"+info.npmPlatform+"/"+wrapperName, + "package/prebuilds/"+info.runtimePlatform+"/"+wrapperName, wrapperName, ); err != nil { - return bundleArtifacts{}, fmt.Errorf("runtime package is missing prebuilds/%s/%s: %w", info.npmPlatform, wrapperName, err) + return bundleArtifacts{}, fmt.Errorf("runtime package is missing prebuilds/%s/%s: %w", info.runtimePlatform, wrapperName, err) } wrapperHash, err := sha256File(rawWrapperPath) if err != nil { @@ -514,16 +519,16 @@ func filesExist(paths ...string) bool { } // runtimeLibArtifactName builds the compressed runtime-library artifact filename. -func runtimeLibArtifactName(version, npmPlatform, goos string) string { - return fmt.Sprintf("zcopilotruntime_%s_%s.%s.zst", version, npmPlatform, runtimeLibExt(goos)) +func runtimeLibArtifactName(version, runtimePlatform, goos string) string { + return fmt.Sprintf("zcopilotruntime_%s_%s.%s.zst", version, runtimePlatform, runtimeLibExt(goos)) } -func runtimeWrapperArtifactName(version, npmPlatform, binaryName string) string { - return fmt.Sprintf("zcopilotruntimewrapper_%s_%s_%s.zst", version, npmPlatform, runtimeWrapperName(binaryName)) +func runtimeWrapperArtifactName(version, runtimePlatform, binaryName string) string { + return fmt.Sprintf("zcopilotruntimewrapper_%s_%s_%s.zst", version, runtimePlatform, runtimeWrapperName(binaryName)) } -func runtimeAssetsArtifactName(version, npmPlatform string) string { - return fmt.Sprintf("zcopilotruntimeassets_%s_%s.tgz", version, npmPlatform) +func runtimeAssetsArtifactName(version, runtimePlatform string) string { + return fmt.Sprintf("zcopilotruntimeassets_%s_%s.tgz", version, runtimePlatform) } func runtimeWrapperName(binaryName string) string { @@ -540,7 +545,7 @@ var hostlessExcludedTopLevel = map[string]bool{ "sea-loader.js": true, "webview": true, } -func hostlessRuntimePath(name, npmPlatform, wrapperName string) (string, bool) { +func hostlessRuntimePath(name, runtimePlatform, wrapperName string) (string, bool) { relative, ok := strings.CutPrefix(name, "package/") if !ok { return "", false @@ -561,7 +566,7 @@ func hostlessRuntimePath(name, npmPlatform, wrapperName string) (string, bool) { } } if topLevel == "prebuilds" { - if len(parts) < 3 || parts[1] != npmPlatform { + if len(parts) < 3 || parts[1] != runtimePlatform { return "", false } return strings.Join(parts[2:], "/"), true @@ -602,7 +607,7 @@ func createRuntimeAssetsArchive(tarballPath, outputPath string, info platformInf } destination, include := hostlessRuntimePath( header.Name, - info.npmPlatform, + info.runtimePlatform, runtimeWrapperName(info.binaryName), ) if !include { @@ -918,11 +923,75 @@ func mustDecodeBase64(s string) []byte { `, buildConstraint, pkgName, binaryName, licenseName, runtimeEmbed, muslEmbed, cliVersion, hashBase64, runtimeConfig, muslConfig, runtimeReader, muslReaders) } -// downloadCLIBinary downloads the npm tarball and extracts the CLI binary. It +var releaseChecksumCache = map[string]map[string]string{} + +func cliDownloadBaseURL() string { + if override := strings.TrimRight(os.Getenv(cliDownloadBaseURLEnvironment), "/"); override != "" { + return override + } + return defaultCLIDownloadBaseURL +} + +func releaseAssetName(version, runtimePlatform string) string { + return fmt.Sprintf("github-copilot-%s-%s.tgz", version, runtimePlatform) +} + +func releaseDownloadURL(version, assetName string) string { + return fmt.Sprintf("%s/v%s/%s", cliDownloadBaseURL(), version, assetName) +} + +func parseReleaseChecksums(contents string) map[string]string { + checksums := make(map[string]string) + hashPattern := regexp.MustCompile(`^[0-9a-fA-F]{64}$`) + for _, line := range strings.Split(contents, "\n") { + fields := strings.Fields(line) + if len(fields) != 2 || !hashPattern.MatchString(fields[0]) { + continue + } + checksums[strings.TrimPrefix(fields[1], "*")] = strings.ToLower(fields[0]) + } + return checksums +} + +func getReleaseChecksum(version, assetName string) (string, error) { + baseURL := cliDownloadBaseURL() + cacheKey := baseURL + "\x00" + version + checksums, ok := releaseChecksumCache[cacheKey] + if !ok { + checksumsURL := fmt.Sprintf("%s/v%s/SHA256SUMS.txt", baseURL, version) + fmt.Printf("Downloading checksums from %s...\n", checksumsURL) + resp, err := http.Get(checksumsURL) + if err != nil { + return "", fmt.Errorf("failed to download checksums: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to download checksums: %s", resp.Status) + } + contents, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read checksums: %w", err) + } + checksums = parseReleaseChecksums(string(contents)) + releaseChecksumCache[cacheKey] = checksums + } + checksum, ok := checksums[assetName] + if !ok { + return "", fmt.Errorf("SHA256SUMS.txt does not contain %s", assetName) + } + return checksum, nil +} + +// downloadCLIBinary downloads the verified release package and extracts the CLI binary. It // returns the extracted binary path and the downloaded tarball path (retained so // callers can extract additional files, such as the runtime library). -func downloadCLIBinary(npmPlatform, binaryName, cliVersion, destDir string) (string, string, error) { - tarballURL := fmt.Sprintf(tarballURLFmt, npmPlatform, npmPlatform, cliVersion) +func downloadCLIBinary(runtimePlatform, binaryName, cliVersion, destDir string) (string, string, error) { + assetName := releaseAssetName(cliVersion, runtimePlatform) + expectedChecksum, err := getReleaseChecksum(cliVersion, assetName) + if err != nil { + return "", "", err + } + tarballURL := releaseDownloadURL(cliVersion, assetName) fmt.Printf("Downloading from %s...\n", tarballURL) @@ -937,24 +1006,43 @@ func downloadCLIBinary(npmPlatform, binaryName, cliVersion, destDir string) (str } // Save tarball to temp file - tarballPath := filepath.Join(destDir, fmt.Sprintf("copilot-%s-%s.tgz", npmPlatform, cliVersion)) + tarballPath := filepath.Join(destDir, assetName) tarballFile, err := os.Create(tarballPath) if err != nil { return "", "", fmt.Errorf("failed to create tarball file: %w", err) } - if _, err := io.Copy(tarballFile, resp.Body); err != nil { + hasher := sha256.New() + if _, err := io.Copy(io.MultiWriter(tarballFile, hasher), resp.Body); err != nil { tarballFile.Close() return "", "", fmt.Errorf("failed to save tarball: %w", err) } if err := tarballFile.Close(); err != nil { return "", "", fmt.Errorf("failed to close tarball file: %w", err) } + actualChecksum := fmt.Sprintf("%x", hasher.Sum(nil)) + if actualChecksum != expectedChecksum { + return "", "", fmt.Errorf( + "checksum mismatch for %s: expected %s, got %s", + assetName, + expectedChecksum, + actualChecksum, + ) + } - // Extract only the CLI binary to avoid unpacking the full package tree. + // The SDK release package intentionally omits the legacy SEA binary. Preserve + // embeddedcli.Path compatibility by installing the runtime wrapper under the + // historical copilot[.exe] name; the normal client path uses the adjacent + // wrapper/runtime.node pair directly. binaryPath := filepath.Join(destDir, binaryName) - if err := extractFileFromTarball(tarballPath, destDir, "package/"+binaryName, binaryName); err != nil { - return "", "", fmt.Errorf("failed to extract binary: %w", err) + wrapperName := runtimeWrapperName(binaryName) + if err := extractFileFromTarball( + tarballPath, + destDir, + "package/prebuilds/"+runtimePlatform+"/"+wrapperName, + binaryName, + ); err != nil { + return "", "", fmt.Errorf("failed to extract runtime wrapper compatibility entrypoint: %w", err) } // Verify binary exists @@ -979,8 +1067,8 @@ func downloadCLIBinary(npmPlatform, binaryName, cliVersion, destDir string) (str return binaryPath, tarballPath, nil } -// downloadCLILicense downloads the @github/copilot package and writes its license next to outputPath. -func downloadCLILicense(cliVersion, outputPath string) error { +// extractCLILicense writes the license from the verified release package next to outputPath. +func extractCLILicense(tarballPath, outputPath string) error { outputDir := filepath.Dir(outputPath) if outputDir == "" { outputDir = "." @@ -990,18 +1078,13 @@ func downloadCLILicense(cliVersion, outputPath string) error { return nil } - licenseURL := fmt.Sprintf(licenseTarballFmt, cliVersion) - resp, err := http.Get(licenseURL) + source, err := os.Open(tarballPath) if err != nil { - return fmt.Errorf("failed to download license tarball: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("failed to download license tarball: %s", resp.Status) + return fmt.Errorf("failed to open release package: %w", err) } + defer source.Close() - gzReader, err := gzip.NewReader(resp.Body) + gzReader, err := gzip.NewReader(source) if err != nil { return fmt.Errorf("failed to create gzip reader: %w", err) } diff --git a/go/cmd/bundler/main_test.go b/go/cmd/bundler/main_test.go index 9c09559b66..7dd376d40b 100644 --- a/go/cmd/bundler/main_test.go +++ b/go/cmd/bundler/main_test.go @@ -4,9 +4,13 @@ import ( "archive/tar" "bytes" "compress/gzip" + "crypto/sha256" + "fmt" "go/parser" "go/token" "io" + "net/http" + "net/http/httptest" "os" "path/filepath" "strings" @@ -31,8 +35,8 @@ func TestCreateRuntimeAssetsArchiveRetainsUnknownAssetsAndFiltersCLIContent(t *t }) if err := createRuntimeAssetsArchive(source, output, platformInfo{ - npmPlatform: "linux-x64", - binaryName: "copilot", + runtimePlatform: "linux-x64", + binaryName: "copilot", }); err != nil { t.Fatal(err) } @@ -54,6 +58,93 @@ func TestCreateRuntimeAssetsArchiveRetainsUnknownAssetsAndFiltersCLIContent(t *t } } +func TestDownloadCLIBinaryUsesVerifiedReleasePackage(t *testing.T) { + dir := t.TempDir() + archivePath := filepath.Join(dir, "source.tgz") + writeTarGz(t, archivePath, map[string]string{ + "package/prebuilds/linux-x64/copilot-runtime": "runtime wrapper", + }) + archive, err := os.ReadFile(archivePath) + if err != nil { + t.Fatal(err) + } + checksum := fmt.Sprintf("%x", sha256.Sum256(archive)) + version := "1.2.3" + assetName := releaseAssetName(version, "linux-x64") + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/v1.2.3/SHA256SUMS.txt": + fmt.Fprintf(writer, "%s %s\n", checksum, assetName) + case "/v1.2.3/" + assetName: + writer.Write(archive) + default: + http.NotFound(writer, request) + } + })) + defer server.Close() + t.Setenv(cliDownloadBaseURLEnvironment, server.URL) + releaseChecksumCache = map[string]map[string]string{} + + binaryPath, downloadedArchive, err := downloadCLIBinary( + "linux-x64", + "copilot", + version, + t.TempDir(), + ) + if err != nil { + t.Fatal(err) + } + if got, err := os.ReadFile(binaryPath); err != nil || string(got) != "runtime wrapper" { + t.Fatalf("downloaded CLI = %q, %v", got, err) + } + if filepath.Base(downloadedArchive) != assetName { + t.Fatalf("downloaded archive = %q, want basename %q", downloadedArchive, assetName) + } +} + +func TestDownloadCLIBinaryRejectsChecksumMismatch(t *testing.T) { + dir := t.TempDir() + archivePath := filepath.Join(dir, "source.tgz") + writeTarGz(t, archivePath, map[string]string{ + "package/prebuilds/linux-x64/copilot-runtime": "runtime wrapper", + }) + archive, err := os.ReadFile(archivePath) + if err != nil { + t.Fatal(err) + } + version := "1.2.3" + assetName := releaseAssetName(version, "linux-x64") + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/v1.2.3/SHA256SUMS.txt": + fmt.Fprintf(writer, "%s %s\n", strings.Repeat("0", 64), assetName) + case "/v1.2.3/" + assetName: + _, _ = writer.Write(archive) + default: + http.NotFound(writer, request) + } + })) + defer server.Close() + t.Setenv(cliDownloadBaseURLEnvironment, server.URL) + releaseChecksumCache = map[string]map[string]string{} + + _, _, err = downloadCLIBinary("linux-x64", "copilot", version, t.TempDir()) + if err == nil || !strings.Contains(err.Error(), "checksum mismatch") { + t.Fatalf("downloadCLIBinary() error = %v, want checksum mismatch", err) + } +} + +func TestParseReleaseChecksums(t *testing.T) { + hash := strings.Repeat("a", 64) + checksums := parseReleaseChecksums( + "invalid\n" + + strings.ToUpper(hash) + " *github-copilot-1.2.3-linux-x64.tgz\n", + ) + if got := checksums["github-copilot-1.2.3-linux-x64.tgz"]; got != hash { + t.Fatalf("checksum = %q, want %q", got, hash) + } +} + func writeTarGz(t *testing.T, path string, files map[string]string) { t.Helper() var buffer bytes.Buffer diff --git a/go/internal/embeddedcli/embeddedcli.go b/go/internal/embeddedcli/embeddedcli.go index 2535cf5f20..5d9eea39de 100644 --- a/go/internal/embeddedcli/embeddedcli.go +++ b/go/internal/embeddedcli/embeddedcli.go @@ -26,7 +26,7 @@ import ( // when provided, is written next to the installed binary. // // RuntimeExecutable and RuntimeNode form the adjacent out-of-process runtime -// pair. RuntimeAssets is a filtered npm package archive containing auxiliary +// pair. RuntimeAssets is a filtered release package archive containing auxiliary // binaries and resources. RuntimeLib is the same cdylib bytes installed under // the natural platform name for the optional in-process transport. type Config struct { @@ -264,6 +264,13 @@ func installAt(installDir string) (string, error) { if !bytes.Equal(existingHash, config.CliHash) { return "", fmt.Errorf("existing binary hash mismatch") } + if config.RuntimeExecutable != nil { + path, err := installRuntimePair(installDir) + if err != nil { + return "", err + } + runtimePath = path + } if config.RuntimeLib != nil { libPath, err := installRuntimeLib(installDir) if err != nil { @@ -298,6 +305,14 @@ func installAt(installDir string) (string, error) { } } + if config.RuntimeExecutable != nil { + path, err := installRuntimePair(installDir) + if err != nil { + return "", err + } + runtimePath = path + } + // Install the native in-process runtime library (if bundled) next to the CLI. // Fail closed on any hash mismatch; never place unverified native code. if config.RuntimeLib != nil { diff --git a/go/internal/embeddedcli/embeddedcli_test.go b/go/internal/embeddedcli/embeddedcli_test.go index 159b6e1505..50b6a29085 100644 --- a/go/internal/embeddedcli/embeddedcli_test.go +++ b/go/internal/embeddedcli/embeddedcli_test.go @@ -259,6 +259,34 @@ func TestInstallAtWritesBinaryAndLicense(t *testing.T) { } } +func TestPathInstallsAdjacentRuntimePair(t *testing.T) { + resetGlobals() + tempDir := t.TempDir() + wrapper := []byte("wrapper") + node := []byte("runtime") + wrapperHash := sha256.Sum256(wrapper) + nodeHash := sha256.Sum256(node) + Setup(Config{ + Cli: bytes.NewReader(wrapper), + CliHash: wrapperHash[:], + RuntimeExecutable: bytes.NewReader(wrapper), + RuntimeExecutableHash: wrapperHash[:], + RuntimeNode: bytes.NewReader(node), + RuntimeNodeHash: nodeHash[:], + Version: "1.2.3", + Dir: tempDir, + }) + + runtimePath := RuntimePath() + installDir := filepath.Dir(runtimePath) + if got, err := os.ReadFile(filepath.Join(installDir, runtimeExecutableName())); err != nil || !bytes.Equal(got, wrapper) { + t.Fatalf("runtime wrapper content=%q err=%v", got, err) + } + if got, err := os.ReadFile(filepath.Join(installDir, "runtime.node")); err != nil || !bytes.Equal(got, node) { + t.Fatalf("runtime.node content=%q err=%v", got, err) + } +} + func TestInstallAtExistingBinaryHashMismatch(t *testing.T) { resetGlobals() tempDir := t.TempDir() diff --git a/java/README.md b/java/README.md index bb71d7ab86..c179983ffe 100644 --- a/java/README.md +++ b/java/README.md @@ -548,9 +548,9 @@ mvn jacoco:prepare-agent@wire-up-coverage-instrumentation antrun:run@print-test- #### Development Setup for native embedding -Run native-runtime Maven commands from the `java` directory. Native packaging requires Node.js in addition to JDK 25 and Maven because `copilot-native/scripts/fetch-native.mjs` retrieves the pinned runtime package from the corresponding GitHub release. +Run native-runtime Maven commands from the `java` directory. Native packaging requires Node.js in addition to JDK 25 and Maven because `copilot-native/scripts/fetch-native.mjs` retrieves the pinned runtime asset from the corresponding GitHub release. It downloads `github-copilot--.tgz`, verifies the asset against that release's `SHA256SUMS.txt`, and does not fall back to npm runtime packages. Set `COPILOT_CLI_DOWNLOAD_BASE_URL` to use a release mirror. -On a native Linux glibc host, Maven activates `native-linux-x64` or `native-linux-arm64` for the matching architecture when `copilot.native.libc=glibc` is set. On Windows x64, Windows ARM64, and Apple Silicon macOS, Maven activates `native-win32-x64`, `native-win32-arm64`, or `native-darwin-arm64` automatically. The matching profile validates the host, runs the native script tests, fetches the pinned platform package from the corresponding `github/copilot-cli` release during `generate-resources`, packages the classifier JAR during `package`, and verifies its native contents. +On a native Linux glibc host, Maven activates `native-linux-x64` or `native-linux-arm64` for the matching architecture when `copilot.native.libc=glibc` is set. On Windows x64, Windows ARM64, and Apple Silicon macOS, Maven activates `native-win32-x64`, `native-win32-arm64`, or `native-darwin-arm64` automatically. The matching profile validates the host, runs the native script tests, fetches the pinned platform release asset from `github/copilot-cli` during `generate-resources`, packages the classifier JAR during `package`, and verifies its native contents. Before opting in, validate that Node.js reports glibc for the build host: diff --git a/java/copilot-native/pom.xml b/java/copilot-native/pom.xml index 0abe03ad18..aeaafc076b 100644 --- a/java/copilot-native/pom.xml +++ b/java/copilot-native/pom.xml @@ -59,10 +59,11 @@ org.codehaus.mojo @@ -909,7 +910,7 @@ diff --git a/java/copilot-native/scripts/fetch-native.mjs b/java/copilot-native/scripts/fetch-native.mjs index 49d2180a73..4e355333e1 100644 --- a/java/copilot-native/scripts/fetch-native.mjs +++ b/java/copilot-native/scripts/fetch-native.mjs @@ -7,10 +7,10 @@ * * Steps: * 1. Read the pinned version from `nodejs/package.json`. - * 2. Download the platform npm tarball and `SHA256SUMS.txt` from the matching release. + * 2. Download the platform release asset and `SHA256SUMS.txt` from the matching release. * 3. Verify the downloaded tarball against the release checksum. * 4. Stage the hostless runtime tree, flattening the selected prebuild directory - * beside the package's retained top-level runtime assets. + * beside the release archive's retained top-level runtime assets. * 5. Write an inventory consumed by the SDK's generic classpath extractor. * 6. Write `//native//platform.properties`. * @@ -49,13 +49,18 @@ if (!repoRoot || !stagingDir || !classifier) { } const packagePath = path.join(repoRoot, 'nodejs', 'package.json'); -const packageName = `@github/copilot-${classifier}`; -const packageJson = JSON.parse(fs.readFileSync(packagePath, 'utf8')); +let packageJson; +try { + packageJson = JSON.parse(fs.readFileSync(packagePath, 'utf8')); +} catch (error) { + throw new Error(`Could not read pinned Copilot CLI version from ${packagePath}: ${error.message}`); +} const version = packageJson.copilotCliVersion; if (!version) { console.error(`Could not find copilotCliVersion in ${packagePath}`); process.exit(1); } +const assetName = `github-copilot-${version}-${classifier}.tgz`; const outDir = path.join(stagingDir, classifier); const resourceDir = path.join(outDir, 'native', classifier); @@ -70,7 +75,7 @@ const stagingSchema = 'hostless-runtime-v3'; const stampPath = path.join(outDir, '.version'); // Idempotence: skip the download only when every required staged artifact -// matches the package identity recorded in the stamp. +// matches the release identity recorded in the stamp. if ( fs.existsSync(runtimePath) && fs.existsSync(wrapperPath) && @@ -90,7 +95,7 @@ if ( stampTreeDigest === currentTreeDigest && currentPlatformProperties === expectedPlatformProperties ) { - console.log(`${packageName}@${version} already staged at ${runtimePath}`); + console.log(`${assetName} already staged at ${runtimePath}`); process.exit(0); } } @@ -98,8 +103,7 @@ if ( fs.rmSync(outDir, { recursive: true, force: true }); fs.mkdirSync(resourceDir, { recursive: true }); -console.log(`Downloading ${packageName}@${version} ...`); -const assetName = `github-copilot-${version}-${classifier}.tgz`; +console.log(`Downloading ${assetName} ...`); const releaseBase = ( process.env.COPILOT_CLI_DOWNLOAD_BASE_URL ?? 'https://github.com/github/copilot-cli/releases/download' @@ -118,8 +122,9 @@ if (process.env.COPILOT_CLI_RELEASE_TARBALL) { if (!expectedHash || !/^[a-fA-F0-9]{64}$/.test(expectedHash)) { throw new Error(`Missing or invalid SHA-256 for ${assetName}`); } +expectedHash = expectedHash.toLowerCase(); const actual = createHash('sha256').update(archive).digest('hex'); -if (actual !== expectedHash.toLowerCase()) { +if (actual !== expectedHash) { console.error(`Integrity verification failed for ${assetName}`); console.error(` expected: ${expectedHash}`); console.error(` actual: ${actual}`); @@ -166,7 +171,7 @@ inventory.sort(); fs.writeFileSync(inventoryPath, `${inventory.join('\n')}\n`); if (!fs.existsSync(runtimePath) || !fs.existsSync(wrapperPath)) { - throw new Error(`Package ${packageName}@${version} is missing the runtime wrapper pair`); + throw new Error(`Release asset ${assetName} is missing the runtime wrapper pair`); } fs.writeFileSync(platformPropertiesPath, expectedPlatformProperties); const treeDigest = digestTree(resourceDir); @@ -231,7 +236,7 @@ async function download(url) { for (let attempt = 0; attempt < 3; attempt++) { try { // lgtm[js/file-access-to-http] The repository-pinned CLI version intentionally selects the release asset. - const response = await fetch(url); + const response = await fetch(url, { signal: AbortSignal.timeout(60_000) }); if (response.ok) { return Buffer.from(await response.arrayBuffer()); } diff --git a/java/copilot-native/scripts/fetch-native.test.mjs b/java/copilot-native/scripts/fetch-native.test.mjs index 3213d416f0..f3537d6a52 100644 --- a/java/copilot-native/scripts/fetch-native.test.mjs +++ b/java/copilot-native/scripts/fetch-native.test.mjs @@ -4,12 +4,12 @@ import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; +import { execFileSync, spawn } from 'node:child_process'; import fs from 'node:fs'; -import os from 'node:os'; +import http from 'node:http'; import path from 'node:path'; -import { execFileSync, spawnSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; -import test from 'node:test'; +import test, { after, before } from 'node:test'; const version = '1.0.79'; const checksum = '0'.repeat(64); @@ -17,65 +17,94 @@ const runtimeContent = 'runtime content'; const wrapperContent = 'wrapper content'; const stagingSchema = 'hostless-runtime-v3'; const scriptPath = fileURLToPath(new URL('./fetch-native.mjs', import.meta.url)); +const testRoot = fileURLToPath(new URL('../target/fetch-native-tests/', import.meta.url)); +const releaseFiles = new Map(); +const releaseRequests = []; +let releaseBase; +let releaseServer; + +before(async () => { + releaseServer = http.createServer((request, response) => { + releaseRequests.push(request.url); + const content = releaseFiles.get(request.url); + if (content === undefined) { + response.writeHead(404).end(); + } else { + response.writeHead(200, { 'Content-Length': content.length }).end(content); + } + }); + await new Promise((resolve, reject) => { + releaseServer.once('error', reject); + releaseServer.listen(0, '127.0.0.1', resolve); + }); + const address = releaseServer.address(); + releaseBase = `http://127.0.0.1:${address.port}`; +}); + +after(async () => { + await new Promise((resolve, reject) => { + releaseServer.close((error) => (error ? reject(error) : resolve())); + }); +}); for (const classifier of ['linux-x64', 'linux-arm64', 'win32-x64', 'win32-arm64', 'darwin-arm64']) { - test(`${classifier}: complete hostless artifacts use incremental fast path without a CLI`, (t) => { + test(`${classifier}: complete hostless artifacts use incremental fast path without a CLI`, async (t) => { const fixture = createFixture(t, classifier); - const result = runScript(fixture); + const result = await runScript(fixture); assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /already staged/); }); - test(`${classifier}: missing runtime wrapper does not use incremental fast path`, (t) => { + test(`${classifier}: missing runtime wrapper does not use incremental fast path`, async (t) => { const fixture = createFixture(t, classifier); fs.rmSync(fixture.wrapperPath); - const result = runScript(fixture); + const result = await runScript(fixture); assertRestagingAttempted(fixture, result); }); - test(`${classifier}: v2 staging schema does not use incremental fast path`, (t) => { + test(`${classifier}: v2 staging schema does not use incremental fast path`, async (t) => { const fixture = createFixture(t, classifier); const stampPath = path.join(fixture.stagingDir, classifier, '.version'); fs.writeFileSync(stampPath, fs.readFileSync(stampPath, 'utf8').replace(stagingSchema, 'hostless-runtime-v2')); - const result = runScript(fixture); + const result = await runScript(fixture); assertRestagingAttempted(fixture, result); }); - test(`${classifier}: missing platform metadata does not use incremental fast path`, (t) => { + test(`${classifier}: missing platform metadata does not use incremental fast path`, async (t) => { const fixture = createFixture(t, classifier); fs.rmSync(fixture.platformPropertiesPath); - const result = runScript(fixture); + const result = await runScript(fixture); assertRestagingAttempted(fixture, result); }); - test(`${classifier}: missing retained runtime asset does not use incremental fast path`, (t) => { + test(`${classifier}: missing retained runtime asset does not use incremental fast path`, async (t) => { const fixture = createFixture(t, classifier); fs.rmSync(fixture.ripgrepPath); - const result = runScript(fixture); + const result = await runScript(fixture); assertRestagingAttempted(fixture, result); }); - test(`${classifier}: complete matching artifacts use incremental fast path`, (t) => { + test(`${classifier}: complete matching artifacts use incremental fast path`, async (t) => { const fixture = createFixture(t, classifier); - const result = runScript(fixture); + const result = await runScript(fixture); assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /already staged/); }); } -test('stages retained package assets and excludes CLI-only content', (t) => { +test('downloads the exact release asset, verifies its manifest checksum, and stages runtime assets', async (t) => { const classifier = 'linux-x64'; const fixture = createFixture(t, classifier); const packageRoot = path.join(fixture.repoRoot, 'package-root', 'package'); @@ -85,6 +114,7 @@ test('stages retained package assets and excludes CLI-only content', (t) => { fs.writeFileSync(path.join(packageRoot, 'copilot'), 'excluded'); fs.writeFileSync(path.join(packageRoot, 'prebuilds', classifier, 'runtime.node'), runtimeContent); fs.writeFileSync(path.join(packageRoot, 'prebuilds', classifier, 'copilot-runtime'), wrapperContent); + fs.chmodSync(path.join(packageRoot, 'prebuilds', classifier, 'copilot-runtime'), 0o755); fs.writeFileSync(path.join(packageRoot, 'ripgrep', 'bin', classifier, 'rg'), 'ripgrep content'); fs.chmodSync(path.join(packageRoot, 'ripgrep', 'bin', classifier, 'rg'), 0o755); fs.writeFileSync(path.join(packageRoot, 'definitions', 'future.json'), '{}'); @@ -100,12 +130,23 @@ test('stages retained package assets and excludes CLI-only content', (t) => { ); fs.rmSync(path.join(fixture.stagingDir, classifier), { recursive: true, force: true }); - const result = runScript(fixture, { - COPILOT_CLI_RELEASE_TARBALL: tarball, - COPILOT_CLI_RELEASE_SHA256: packageChecksum, + const assetName = `github-copilot-${version}-${classifier}.tgz`; + const releasePrefix = `/v${version}`; + releaseFiles.set(`${releasePrefix}/SHA256SUMS.txt`, Buffer.from(`${packageChecksum} ${assetName}\n`)); + releaseFiles.set(`${releasePrefix}/${assetName}`, fs.readFileSync(tarball)); + const requestOffset = releaseRequests.length; + t.after(() => { + releaseFiles.delete(`${releasePrefix}/SHA256SUMS.txt`); + releaseFiles.delete(`${releasePrefix}/${assetName}`); }); + const result = await runScript(fixture); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(releaseRequests.slice(requestOffset), [ + `${releasePrefix}/SHA256SUMS.txt`, + `${releasePrefix}/${assetName}`, + ]); const resourceDir = path.join(fixture.stagingDir, classifier, 'native', classifier); assert.equal(fs.readFileSync(path.join(resourceDir, 'ripgrep', 'bin', classifier, 'rg'), 'utf8'), 'ripgrep content'); assert.equal(fs.readFileSync(path.join(resourceDir, 'definitions', 'future.json'), 'utf8'), '{}'); @@ -113,11 +154,79 @@ test('stages retained package assets and excludes CLI-only content', (t) => { assert.equal(fs.existsSync(path.join(resourceDir, 'copilot')), false); assert.equal(fs.existsSync(path.join(resourceDir, 'LICENSE.md')), false); assert.equal(fs.existsSync(path.join(resourceDir, 'README.md')), false); - assert.match(fs.readFileSync(path.join(resourceDir, 'runtime-assets.list'), 'utf8'), /ripgrep\/bin\/linux-x64\/rg/); + const inventory = fs.readFileSync(path.join(resourceDir, 'runtime-assets.list'), 'utf8'); + assert.match(inventory, /^644\truntime\.node$/m); + assert.match(inventory, /^755\tcopilot-runtime$/m); + assert.match(inventory, /^755\tripgrep\/bin\/linux-x64\/rg$/m); +}); + +test('stages a pre-downloaded release asset with its supplied SHA-256 without contacting a release server', async (t) => { + const classifier = 'linux-x64'; + const fixture = createFixture(t, classifier); + const packageRoot = path.join(fixture.repoRoot, 'local-package', 'package'); + const prebuildRoot = path.join(packageRoot, 'prebuilds', classifier); + fs.mkdirSync(prebuildRoot, { recursive: true }); + fs.writeFileSync(path.join(prebuildRoot, 'runtime.node'), runtimeContent); + fs.writeFileSync(path.join(prebuildRoot, 'copilot-runtime'), wrapperContent); + fs.chmodSync(path.join(prebuildRoot, 'copilot-runtime'), 0o755); + const tarball = path.join(fixture.repoRoot, 'pre-downloaded-release.tgz'); + execFileSync('tar', ['-czf', tarball, '-C', path.dirname(packageRoot), 'package']); + const expectedHash = createHash('sha256').update(fs.readFileSync(tarball)).digest('hex'); + fs.rmSync(path.join(fixture.stagingDir, classifier), { recursive: true, force: true }); + const requestOffset = releaseRequests.length; + + const result = await runScript(fixture, { + COPILOT_CLI_RELEASE_TARBALL: tarball, + COPILOT_CLI_RELEASE_SHA256: expectedHash, + }); + + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(releaseRequests.slice(requestOffset), []); + const resourceDir = path.join(fixture.stagingDir, classifier, 'native', classifier); + assert.equal(fs.readFileSync(path.join(resourceDir, 'runtime.node'), 'utf8'), runtimeContent); + assert.equal(fs.readFileSync(path.join(resourceDir, 'copilot-runtime'), 'utf8'), wrapperContent); +}); + +test('rejects a release asset that does not match SHA256SUMS.txt without npm fallback', async (t) => { + const classifier = 'linux-x64'; + const fixture = createFixture(t, classifier); + const assetName = `github-copilot-${version}-${classifier}.tgz`; + const releasePrefix = `/v${version}`; + releaseFiles.set(`${releasePrefix}/SHA256SUMS.txt`, Buffer.from(`${checksum} ${assetName}\n`)); + releaseFiles.set(`${releasePrefix}/${assetName}`, Buffer.from('not the release archive')); + t.after(() => { + releaseFiles.delete(`${releasePrefix}/SHA256SUMS.txt`); + releaseFiles.delete(`${releasePrefix}/${assetName}`); + }); + fs.rmSync(path.join(fixture.stagingDir, classifier), { recursive: true, force: true }); + + const result = await runScript(fixture); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Integrity verification failed/); + assert.doesNotMatch(result.stderr, /npm pack/); +}); + +test('fails when SHA256SUMS.txt does not list the exact release asset', async (t) => { + const classifier = 'linux-x64'; + const fixture = createFixture(t, classifier); + const releasePrefix = `/v${version}`; + releaseFiles.set(`${releasePrefix}/SHA256SUMS.txt`, Buffer.from(`${checksum} another-asset.tgz\n`)); + t.after(() => releaseFiles.delete(`${releasePrefix}/SHA256SUMS.txt`)); + fs.rmSync(path.join(fixture.stagingDir, classifier), { recursive: true, force: true }); + + const result = await runScript(fixture); + + assert.notEqual(result.status, 0); + assert.match( + result.stderr, + new RegExp(`SHA256SUMS\\.txt does not contain github-copilot-${version}-${classifier}\\.tgz`), + ); }); function createFixture(t, classifier) { - const root = fs.mkdtempSync(path.join(os.tmpdir(), 'fetch-native-test-')); + fs.mkdirSync(testRoot, { recursive: true }); + const root = fs.mkdtempSync(path.join(testRoot, 'fixture-')); t.after(() => fs.rmSync(root, { recursive: true, force: true })); const repoRoot = path.join(root, 'repo'); @@ -166,14 +275,28 @@ function createFixture(t, classifier) { } function runScript(fixture, extraEnv = {}) { - return spawnSync(process.execPath, [scriptPath, fixture.repoRoot, fixture.stagingDir, fixture.classifier], { - encoding: 'utf8', - env: { - ...process.env, - COPILOT_CLI_RELEASE_TARBALL: path.join(fixture.root, 'missing.tgz'), - COPILOT_CLI_RELEASE_SHA256: checksum, - ...extraEnv, - }, + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [scriptPath, fixture.repoRoot, fixture.stagingDir, fixture.classifier], { + env: { + ...process.env, + COPILOT_CLI_DOWNLOAD_BASE_URL: releaseBase, + ...extraEnv, + }, + }); + let stdout = ''; + let stderr = ''; + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', (chunk) => { + stdout += chunk; + }); + child.stderr.on('data', (chunk) => { + stderr += chunk; + }); + child.once('error', reject); + child.once('close', (status, signal) => { + resolve({ status, signal, stdout, stderr }); + }); }); } diff --git a/java/docs/adr/adr-007-native-bundling-strategy.md b/java/docs/adr/adr-007-native-bundling-strategy.md index 3c13d451cf..e5d5c85836 100644 --- a/java/docs/adr/adr-007-native-bundling-strategy.md +++ b/java/docs/adr/adr-007-native-bundling-strategy.md @@ -361,7 +361,7 @@ The pattern follows DJL's `LibUtils.loadLibrary()` approach: detect the platform 2. Locates the matching `runtime.node` binary on the classpath (via `getResourceAsStream` from the classifier JAR). 3. Extracts `runtime.node` and the transitional CLI entrypoint into `~/.copilot/runtime-cache/` if valid cached files are not already present. 4. Loads it via [JNA](#references) using the C ABI entry points, per the [binding technology decision](#binding-technology-jna-over-panama-ffm) above. The JNA-specific code is confined behind an internal binding interface to preserve a future FFM migration path. -* A validated supported-host profile fetches the matching platform tarball from the pinned `github/copilot-cli` release, verifies its release SHA-256, and packages the version-matched runtime files. +* A validated supported-host profile fetches `github-copilot--.tgz` from the pinned `github/copilot-cli` release, verifies it against the release's `SHA256SUMS.txt`, and packages the version-matched runtime files without an npm runtime-package fallback. * The current release work publishes the `linux-x64`, `linux-arm64`, `win32-x64`, `win32-arm64`, and `darwin-arm64` classifiers. The planned classifier set expands to the other detected platforms. * Adding an implemented platform requires validated host activation, a profile that supplies the classifier and platform CLI filename, and lifecycle bindings for the shared host validation, fetch, script test, package, and verification executions. * `cli-native.node` is not bundled. It provides terminal UI features that are irrelevant to the Java SDK's programmatic API surface. diff --git a/java/sdk/src/main/java/com/github/copilot/ffi/NativeRuntimeLoader.java b/java/sdk/src/main/java/com/github/copilot/ffi/NativeRuntimeLoader.java index bd4b185a07..dd5f91cf28 100644 --- a/java/sdk/src/main/java/com/github/copilot/ffi/NativeRuntimeLoader.java +++ b/java/sdk/src/main/java/com/github/copilot/ffi/NativeRuntimeLoader.java @@ -318,9 +318,8 @@ static Path resolve(String cliPathEnv, Path cacheBase, ClassLoader loader, Strin * *

* Checks, in order, the flat bundled layout ({@code runtime.node} directly next - * to the CLI) and the npm package layout - * ({@code prebuilds//runtime.node} next to the CLI), matching the - * two layouts the {@code @github/copilot-} packages may ship. + * to the CLI) and the release-package layout + * ({@code prebuilds//runtime.node} next to the CLI). */ static Path resolveFromCliPath(String cliPathStr) throws IOException { if (cliPathStr == null || cliPathStr.isBlank()) { diff --git a/python/README.md b/python/README.md index 359026df41..c857145b96 100644 --- a/python/README.md +++ b/python/README.md @@ -29,9 +29,10 @@ runtime: python -m copilot download-runtime ``` -This caches `copilot-runtime`, its adjacent `runtime.node`, and the compatible -`copilot` host locally. If you skip this step, the SDK downloads the bundle -automatically on first managed stdio/TCP use. +This downloads the platform release package, verifies it against the release's +`SHA256SUMS.txt`, and caches `copilot-runtime`, its adjacent `runtime.node`, and +the hostless runtime assets locally. If you skip this step, the SDK downloads the +same package automatically on first managed stdio/TCP use. To pre-provision the native library required by the in-process (FFI) transport (see [In-process (FFI) transport](#in-process-ffi-transport)), pass `--in-process`: @@ -40,9 +41,10 @@ To pre-provision the native library required by the in-process (FFI) transport python -m copilot download-runtime --in-process ``` -This instead provisions the compatible CLI artifact and native runtime library -used by in-process hosting. When omitted, they are downloaded lazily on first -use of the in-process transport. +This also creates a `copilot` compatibility entrypoint from `copilot-runtime` +inside the complete materialized bundle. Its adjacent `runtime.node` can then be +used for in-process hosting. The cached release package is reused, so this does +not download a second runtime artifact. | Platform | Cache path | |----------|-----------| @@ -55,10 +57,9 @@ use of the in-process transport. | Variable | Description | |----------|-------------| | `COPILOT_CLI_PATH` | Use this specific binary instead of downloading | -| `COPILOT_CLI_EXTRACT_DIR` | Override the cache directory (binary placed directly here) | +| `COPILOT_CLI_EXTRACT_DIR` | Override the version-specific cache directory | | `COPILOT_SKIP_CLI_DOWNLOAD` | Set to `1` to disable auto-download | -| `COPILOT_NPM_REGISTRY_URL` | Override the npm registry used for managed out-of-process and in-process runtime downloads | -| `COPILOT_CLI_DOWNLOAD_BASE_URL` | Override the GitHub Releases download URL used for the root CLI | +| `COPILOT_CLI_DOWNLOAD_BASE_URL` | Override the GitHub Releases download URL used for the runtime package and checksums | ## Run the Sample diff --git a/python/copilot/_cli_download.py b/python/copilot/_cli_download.py index 2d9076f6b8..e485a923c5 100644 --- a/python/copilot/_cli_download.py +++ b/python/copilot/_cli_download.py @@ -1,22 +1,24 @@ -"""Download and cache the Copilot CLI binary. +"""Download and cache the Copilot CLI runtime package. -This module implements a download-at-first-use strategy for the Copilot CLI -binary, similar to the Rust SDK's build.rs approach but triggered at runtime. -The binary is cached in a shared directory compatible with the Rust SDK: +The platform-specific GitHub release package contains the out-of-process runtime +wrapper, native runtime library, and runtime assets, but omits the legacy SEA +``copilot[.exe]``. It is downloaded and verified once, then materialized into the +SDK's existing cache layout. ``download_cli`` preserves the historical CLI filename +by creating a compatibility alias from the runtime wrapper inside the complete +materialized bundle: -- Linux: ~/.cache/github-copilot-sdk/cli/{version}/copilot -- macOS: ~/Library/Caches/github-copilot-sdk/cli/{version}/copilot -- Windows: %LOCALAPPDATA%/github-copilot-sdk/cli/{version}/copilot.exe +- Linux: ~/.cache/github-copilot-sdk/cli/{version}/prebuilds/{platform}/copilot +- macOS: ~/Library/Caches/github-copilot-sdk/cli/{version}/prebuilds/{platform}/copilot +- Windows: %LOCALAPPDATA%/github-copilot-sdk/cli/{version}/prebuilds/{platform}/copilot.exe Environment variables: -- COPILOT_CLI_EXTRACT_DIR: Override the cache directory (binary placed directly here). +- COPILOT_CLI_EXTRACT_DIR: Override the runtime bundle cache root. - COPILOT_SKIP_CLI_DOWNLOAD: Set to "1" or "true" to disable auto-download. - COPILOT_CLI_DOWNLOAD_BASE_URL: Override the GitHub Releases base URL. """ from __future__ import annotations -import base64 import hashlib import io import os @@ -26,7 +28,6 @@ import tarfile import tempfile import time -import zipfile from http.client import IncompleteRead from pathlib import Path, PurePosixPath from urllib.error import HTTPError, URLError @@ -34,17 +35,17 @@ from ._cli_version import ( CLI_VERSION, - get_asset_info, get_checksums_url, + get_cli_binary_name, get_download_url, - get_npm_platform, - get_runtime_lib_packument_url, - get_runtime_lib_url, + get_release_asset_name, + get_runtime_platform, ) _CACHE_DIR_NAME = "github-copilot-sdk" _MAX_RETRIES = 3 _RETRIABLE_DOWNLOAD_ERRORS = (HTTPError, URLError, IncompleteRead) +_HOSTLESS_ASSETS_MARKER = ".hostless-runtime-assets-v2" def _sanitize_version(version: str) -> str: @@ -57,13 +58,12 @@ def _sanitize_version(version: str) -> str: def get_cache_dir(version: str | None = None) -> Path: - """Return the cache directory for CLI binaries. + """Return the cache directory for runtime bundles. Args: version: CLI version string. If None, returns the root cache dir. """ - # COPILOT_CLI_EXTRACT_DIR overrides the entire version-specific directory - # (binary lives directly at $dir/, no version subdir). Matches Rust SDK. + # COPILOT_CLI_EXTRACT_DIR overrides the entire version-specific directory. extract_override = os.environ.get("COPILOT_CLI_EXTRACT_DIR") if extract_override: return Path(extract_override) @@ -89,7 +89,7 @@ def get_cache_dir(version: str | None = None) -> Path: def get_cached_cli_path(version: str | None = None) -> str | None: - """Return the path to the cached CLI binary if it exists. + """Return the cached compatibility entrypoint for a complete runtime bundle. Args: version: CLI version. Defaults to the pinned CLI_VERSION. @@ -102,12 +102,21 @@ def get_cached_cli_path(version: str | None = None) -> str | None: return None try: - _, binary_name = get_asset_info() + runtime_platform = get_runtime_platform() except RuntimeError: return None - binary_path = get_cache_dir(ver) / binary_name + binary_name = get_cli_binary_name() + wrapper_name = "copilot-runtime.exe" if sys.platform == "win32" else "copilot-runtime" + pair_dir = get_cache_dir(ver) / "prebuilds" / runtime_platform + binary_path = pair_dir / binary_name + required = ( + binary_path, + pair_dir / wrapper_name, + pair_dir / "runtime.node", + pair_dir / _HOSTLESS_ASSETS_MARKER, + ) - if binary_path.exists(): + if all(path.is_file() and path.stat().st_size > 0 for path in required): return str(binary_path) return None @@ -124,30 +133,22 @@ def _fetch_checksums(version: str) -> dict[str, str]: Returns a dict mapping filename → sha256 hex digest. """ url = get_checksums_url(version) - last_exc: Exception | None = None - for attempt in range(_MAX_RETRIES): - try: - with urlopen(url, timeout=30) as response: - text = response.read().decode("utf-8") - break - except _RETRIABLE_DOWNLOAD_ERRORS as exc: - last_exc = exc - if attempt < _MAX_RETRIES - 1: - time.sleep(2**attempt) - else: + try: + text = _fetch_url_bytes(url, timeout=30).decode("utf-8") + except (RuntimeError, UnicodeDecodeError) as exc: raise RuntimeError( - f"Failed to download checksums from {url}: {last_exc}\n\n" + f"Failed to download checksums from {url}: {exc}\n\n" "If you are in an offline or firewalled environment, set " "COPILOT_CLI_PATH to point to a manually-installed binary." - ) from last_exc + ) from exc checksums: dict[str, str] = {} for line in text.strip().splitlines(): parts = line.split() - if len(parts) == 2: + if len(parts) == 2 and re.fullmatch(r"[a-fA-F0-9]{64}", parts[0]): digest, filename = parts # Some formats use *filename (binary mode indicator) - checksums[filename.lstrip("*")] = digest + checksums[filename.lstrip("*")] = digest.lower() return checksums @@ -160,65 +161,99 @@ def _verify_checksum(data: bytes, expected_hash: str, filename: str) -> None: ) -def _extract_tar_gz(data: bytes, binary_name: str, dest_dir: Path) -> Path: - """Extract the CLI binary from a .tar.gz archive.""" - with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tf: - # Find the binary in the archive (may be at top level or in a subdirectory) - members = tf.getnames() - target_member = None - for name in members: - if name == binary_name or name.endswith(f"/{binary_name}"): - target_member = name - break - - if target_member is None: - raise RuntimeError( - f"Binary '{binary_name}' not found in archive. Archive contains: {members}" - ) +def _validate_file(path: Path, label: str) -> None: + if not path.is_file() or path.stat().st_size == 0: + raise RuntimeError(f"{label} not found or empty at {path}.") - member = tf.getmember(target_member) - f = tf.extractfile(member) - if f is None: - raise RuntimeError(f"Could not extract '{target_member}' from archive") - dest_path = dest_dir / binary_name - with open(dest_path, "wb") as out: - out.write(f.read()) +def _extract_release_package(data: bytes, destination: Path) -> None: + """Safely extract the npm-style ``package/`` tree from a release tarball.""" + with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as archive: + for member in archive: + parts = PurePosixPath(member.name).parts + if len(parts) < 2 or parts[0] != "package": + continue + relative = Path(*parts[1:]) + if relative.is_absolute() or ".." in relative.parts: + raise RuntimeError(f"Unsafe release package path: {member.name}") + target = destination / relative + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + continue + if not member.isfile(): + raise RuntimeError(f"Unsupported release package entry: {member.name}") + extracted = archive.extractfile(member) + if extracted is None: + raise RuntimeError(f"Failed to read release package entry: {member.name}") + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(extracted.read()) + if sys.platform != "win32": + target.chmod(member.mode & 0o777) - return dest_path +def _release_package_dir(version: str, runtime_platform: str) -> Path: + return get_cache_dir(version) / "packages" / runtime_platform -def _extract_zip(data: bytes, binary_name: str, dest_dir: Path) -> Path: - """Extract the CLI binary from a .zip archive.""" - with zipfile.ZipFile(io.BytesIO(data)) as zf: - names = zf.namelist() - target_member = None - for name in names: - if name == binary_name or name.endswith(f"/{binary_name}"): - target_member = name - break - if target_member is None: - raise RuntimeError( - f"Binary '{binary_name}' not found in archive. Archive contains: {names}" - ) +def _validate_release_package(package_dir: Path, runtime_platform: str) -> None: + prebuilds = package_dir / "prebuilds" / runtime_platform + wrapper_name = "copilot-runtime.exe" if sys.platform == "win32" else "copilot-runtime" + _validate_file(prebuilds / wrapper_name, "Copilot runtime wrapper") + _validate_file(prebuilds / "runtime.node", "Copilot runtime.node") - dest_path = dest_dir / binary_name - with zf.open(target_member) as src, open(dest_path, "wb") as out: - out.write(src.read()) - return dest_path +def _ensure_release_package(version: str, *, force: bool = False) -> Path: + """Download, verify, and cache the unified platform release package.""" + runtime_platform = get_runtime_platform() + package_dir = _release_package_dir(version, runtime_platform) + if package_dir.exists() and not force: + _validate_release_package(package_dir, runtime_platform) + return package_dir + if _should_skip_download(): + raise RuntimeError( + f"Copilot runtime release package is not cached in {package_dir} " + "and automatic downloads are disabled." + ) + + asset_name = get_release_asset_name(version, runtime_platform) + expected_hash = _fetch_checksums(version).get(asset_name) + if not expected_hash: + raise RuntimeError(f"SHA256SUMS.txt does not contain {asset_name}.") + url = get_download_url(version, asset_name) + data = _fetch_url_bytes(url, timeout=600) + _verify_checksum(data, expected_hash, asset_name) + + import shutil + + package_dir.parent.mkdir(parents=True, exist_ok=True) + staging_dir = Path(tempfile.mkdtemp(dir=package_dir.parent, prefix=".release-package-")) + staged_package = staging_dir / "package" + try: + staged_package.mkdir() + _extract_release_package(data, staged_package) + _validate_release_package(staged_package, runtime_platform) + if force and package_dir.exists(): + shutil.rmtree(package_dir) + try: + staged_package.replace(package_dir) + except OSError: + if not package_dir.exists(): + raise + _validate_release_package(package_dir, runtime_platform) + finally: + shutil.rmtree(staging_dir, ignore_errors=True) + return package_dir def download_cli(version: str | None = None, *, force: bool = False) -> str: - """Download the Copilot CLI binary and cache it. + """Provision a complete runtime bundle with a ``copilot[.exe]`` alias. Args: version: CLI version to download. Defaults to the pinned CLI_VERSION. force: If True, re-download even if already cached. Returns: - Path to the cached binary. + Path to the compatibility entrypoint adjacent to the complete runtime bundle. Raises: RuntimeError: If the version is not set, download fails, or @@ -231,81 +266,33 @@ def download_cli(version: str | None = None, *, force: bool = False) -> str: "set COPILOT_CLI_PATH or install a published wheel." ) - archive_name, binary_name = get_asset_info() - cache_dir = get_cache_dir(ver) - binary_path = cache_dir / binary_name - - # Return cached binary if available (unless force) - if not force and binary_path.exists(): - return str(binary_path) + binary_name = get_cli_binary_name() - # Fetch checksums - checksums = _fetch_checksums(ver) - expected_hash = checksums.get(archive_name) - if not expected_hash: - raise RuntimeError( - f"No checksum found for '{archive_name}' in SHA256SUMS.txt. " - f"Available files: {list(checksums.keys())}" - ) + if not force: + cached = get_cached_cli_path(ver) + if cached is not None: + return cached - # Download archive with retries - url = get_download_url(ver, archive_name) - last_exc: Exception | None = None - data: bytes | None = None - for attempt in range(_MAX_RETRIES): - try: - with urlopen(url, timeout=120) as response: - data = response.read() - break - except _RETRIABLE_DOWNLOAD_ERRORS as exc: - last_exc = exc - if attempt < _MAX_RETRIES - 1: - time.sleep(2**attempt) - if data is None: - raise RuntimeError( - f"Failed to download runtime from {url}: {last_exc}\n\n" - "If you are in an offline or firewalled environment, you can:\n" - f"1. Manually download the archive from: {url}\n" - f"2. Extract the '{binary_name}' binary to: {binary_path}\n" - "Or set COPILOT_CLI_PATH to point to an existing binary." - ) from last_exc - - # Verify checksum - _verify_checksum(data, expected_hash, archive_name) - - # Extract to a temporary directory, then atomically move into place. - # This prevents partial/corrupt cache entries if the process is interrupted. - cache_dir.mkdir(parents=True, exist_ok=True) - staging_dir = Path(tempfile.mkdtemp(dir=cache_dir, prefix=".download-")) + wrapper_path = Path(ensure_runtime_wrapper(ver, force=force)) + binary_path = wrapper_path.with_name(binary_name) + fd, temp_name = tempfile.mkstemp(dir=wrapper_path.parent, prefix=".cli-") try: - if archive_name.endswith(".tar.gz"): - extracted = _extract_tar_gz(data, binary_name, staging_dir) - elif archive_name.endswith(".zip"): - extracted = _extract_zip(data, binary_name, staging_dir) - else: - raise RuntimeError(f"Unknown archive format: {archive_name}") - - # Make executable on Unix + with os.fdopen(fd, "wb") as destination: + destination.write(wrapper_path.read_bytes()) + staged = Path(temp_name) if sys.platform != "win32": - extracted.chmod(extracted.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - - # Atomic rename into final location. Handle concurrent processes: - # another process may have written the file while we were downloading. - try: - extracted.replace(binary_path) - except OSError: - if not force and binary_path.exists(): - return str(binary_path) - raise - finally: - # Clean up staging directory + staged.chmod(staged.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + os.replace(staged, binary_path) + except OSError: try: - staging_dir.rmdir() + os.unlink(temp_name) except OSError: - # May not be empty if rename failed or other files were extracted - import shutil - - shutil.rmtree(staging_dir, ignore_errors=True) + pass + if not force: + cached = get_cached_cli_path(ver) + if cached is not None: + return cached + raise return str(binary_path) @@ -324,71 +311,6 @@ def _fetch_url_bytes(url: str, *, timeout: int) -> bytes: raise RuntimeError(f"Failed to download from {url}: {last_exc}") from last_exc -def _fetch_runtime_integrity(npm_platform: str, version: str) -> str | None: - """Return the npm ``dist.integrity`` (Subresource Integrity) for the tarball. - - Best-effort: returns None if the packument can't be fetched or parsed. - """ - import json - - url = get_runtime_lib_packument_url(npm_platform) - try: - raw = _fetch_url_bytes(url, timeout=30) - packument = json.loads(raw) - dist = packument.get("versions", {}).get(version, {}).get("dist", {}) - integrity = dist.get("integrity") - return integrity if isinstance(integrity, str) else None - except (RuntimeError, ValueError, KeyError): - return None - - -def _verify_integrity(data: bytes, integrity: str) -> None: - """Verify data against an npm Subresource Integrity string (e.g. ``sha512-``).""" - algo, _, b64 = integrity.partition("-") - algo = algo.lower() - if algo not in ("sha512", "sha384", "sha256"): - # Fail closed: an unrecognized algorithm means we cannot verify this native - # library, so refuse rather than loading unverified native code. - raise RuntimeError( - f"Unsupported integrity algorithm '{algo}' for the in-process runtime " - "library; refusing to load unverified native code." - ) - expected = base64.b64decode(b64) - actual = hashlib.new(algo, data).digest() - if actual != expected: - raise RuntimeError( - f"Integrity mismatch for runtime library ({algo}): " - "downloaded tarball does not match the npm registry checksum." - ) - - -def _extract_runtime_node(data: bytes, npm_platform: str) -> bytes: - """Extract ``package/prebuilds//runtime.node`` from an npm tarball.""" - target = f"package/prebuilds/{npm_platform}/runtime.node" - with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tf: - for name in tf.getnames(): - if name == target or name.endswith(f"/prebuilds/{npm_platform}/runtime.node"): - member = tf.getmember(name) - extracted = tf.extractfile(member) - if extracted is not None: - return extracted.read() - raise RuntimeError(f"'{target}' not found in runtime package for {npm_platform}.") - - -def _extract_runtime_wrapper(data: bytes, npm_platform: str) -> bytes: - """Extract the SDK out-of-process wrapper from an npm platform tarball.""" - wrapper_name = "copilot-runtime.exe" if sys.platform == "win32" else "copilot-runtime" - target = f"package/prebuilds/{npm_platform}/{wrapper_name}" - with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tf: - for name in tf.getnames(): - if name == target or name.endswith(f"/prebuilds/{npm_platform}/{wrapper_name}"): - member = tf.getmember(name) - extracted = tf.extractfile(member) - if extracted is not None: - return extracted.read() - raise RuntimeError(f"'{target}' not found in runtime package for {npm_platform}.") - - _HOSTLESS_EXCLUDED_TOP_LEVEL = { "app.js", "assets", @@ -412,7 +334,7 @@ def _extract_runtime_wrapper(data: bytes, npm_platform: str) -> bytes: } -def _hostless_runtime_path(member_name: str, npm_platform: str) -> Path | None: +def _hostless_runtime_path(member_name: str, runtime_platform: str) -> Path | None: parts = PurePosixPath(member_name).parts if not parts or parts[0] != "package" or len(parts) < 2: return None @@ -429,7 +351,7 @@ def _hostless_runtime_path(member_name: str, npm_platform: str) -> Path | None: ): return None if top_level == "prebuilds": - if len(relative) < 3 or relative[1] != npm_platform: + if len(relative) < 3 or relative[1] != runtime_platform: return None relative = relative[2:] destination = Path(*relative) @@ -438,36 +360,37 @@ def _hostless_runtime_path(member_name: str, npm_platform: str) -> Path | None: return destination -def _extract_runtime_bundle(data: bytes, npm_platform: str, destination: Path) -> None: - """Extract the hostless runtime tree, retaining unknown package assets by default.""" - with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as archive: - for member in archive: - relative = _hostless_runtime_path(member.name, npm_platform) - if relative is None or member.isdir(): - continue - if not member.isfile(): - raise RuntimeError(f"Unsupported runtime package entry: {member.name}") - extracted = archive.extractfile(member) - if extracted is None: - raise RuntimeError(f"Failed to read runtime package entry: {member.name}") - target = destination / relative - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(extracted.read()) - if sys.platform != "win32": - target.chmod(member.mode & 0o777) +def _materialize_runtime_bundle( + package_dir: Path, runtime_platform: str, destination: Path +) -> None: + """Copy the hostless runtime tree, retaining unknown package assets by default.""" + import shutil + + for source in package_dir.rglob("*"): + if source.is_dir(): + continue + member_name = PurePosixPath("package", *source.relative_to(package_dir).parts).as_posix() + relative = _hostless_runtime_path(member_name, runtime_platform) + if relative is None: + continue + if not source.is_file(): + raise RuntimeError(f"Unsupported runtime package entry: {member_name}") + target = destination / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) def ensure_runtime_wrapper(version: str | None = None, force: bool = False) -> str: - """Provision the runtime pair and its retained npm package assets.""" + """Provision the runtime pair and retained assets from the release package.""" ver = version or CLI_VERSION if not ver: raise RuntimeError("No runtime version is pinned.") - npm_platform = get_npm_platform() + runtime_platform = get_runtime_platform() wrapper_name = "copilot-runtime.exe" if sys.platform == "win32" else "copilot-runtime" - pair_dir = get_cache_dir(ver) / "prebuilds" / npm_platform + pair_dir = get_cache_dir(ver) / "prebuilds" / runtime_platform wrapper_path = pair_dir / wrapper_name runtime_path = pair_dir / "runtime.node" - assets_marker = pair_dir / ".hostless-runtime-assets-v2" + assets_marker = pair_dir / _HOSTLESS_ASSETS_MARKER wrapper_exists = wrapper_path.is_file() and wrapper_path.stat().st_size > 0 runtime_exists = runtime_path.is_file() and runtime_path.stat().st_size > 0 @@ -478,26 +401,13 @@ def ensure_runtime_wrapper(version: str | None = None, force: bool = False) -> s f"Incomplete Copilot runtime bundle in {pair_dir}: " f"{wrapper_name} and runtime.node are required." ) - if _should_skip_download(): - raise RuntimeError( - f"Copilot runtime bundle is not cached in {pair_dir} " - "and automatic downloads are disabled." - ) - - data = _fetch_url_bytes(get_runtime_lib_url(ver, npm_platform), timeout=600) - integrity = _fetch_runtime_integrity(npm_platform, ver) - if not integrity: - raise RuntimeError( - "No Subresource Integrity value available for the Copilot runtime " - f"package ({npm_platform}@{ver}); refusing to stage unverified native code." - ) - _verify_integrity(data, integrity) + package_dir = _ensure_release_package(ver, force=force) import shutil pair_dir.parent.mkdir(parents=True, exist_ok=True) staging_dir = Path(tempfile.mkdtemp(dir=pair_dir.parent, prefix=".runtime-bundle-")) try: - _extract_runtime_bundle(data, npm_platform, staging_dir) + _materialize_runtime_bundle(package_dir, runtime_platform, staging_dir) staged_wrapper = staging_dir / wrapper_name staged_runtime = staging_dir / "runtime.node" if ( @@ -536,11 +446,10 @@ def ensure_runtime_wrapper(version: str | None = None, force: bool = False) -> s def ensure_runtime_library(cli_path: str, version: str | None = None) -> str | None: """Ensure the native in-process (FFI) runtime library sits next to ``cli_path``. - The library is NOT part of the GitHub Releases CLI archive; it ships in the npm - platform package ``@github/copilot-`` under - ``package/prebuilds//runtime.node``. This helper downloads that tarball - and writes the library next to the CLI binary under its natural platform name - (``libcopilot_runtime.so`` / ``.dylib`` / ``copilot_runtime.dll``). + The unified platform release package contains ``runtime.node`` under + ``package/prebuilds/``. This helper copies that verified library next + to the CLI binary under its natural platform name (``libcopilot_runtime.so`` / + ``.dylib`` / ``copilot_runtime.dll``). This is opt-in — only invoked when the in-process transport is actually selected (lazy) or via ``python -m copilot download-runtime --in-process`` (explicit). The @@ -558,15 +467,12 @@ def ensure_runtime_library(cli_path: str, version: str | None = None) -> str | N if existing is not None: return existing - if _should_skip_download(): - return None - ver = version or CLI_VERSION if not ver: return None try: - npm_platform = get_npm_platform() + runtime_platform = get_runtime_platform() except RuntimeError: return None @@ -575,23 +481,11 @@ def ensure_runtime_library(cli_path: str, version: str | None = None) -> str | N if lib_path.exists(): return str(lib_path) - url = get_runtime_lib_url(ver, npm_platform) - data = _fetch_url_bytes(url, timeout=600) - - integrity = _fetch_runtime_integrity(npm_platform, ver) - if not integrity: - # Fail closed: this native library is loaded into the host process, so it must - # be verified before use. The npm packument (which carries dist.integrity) was - # unavailable, so refuse rather than loading unverified native code — mirroring - # the CLI download, which requires a checksum. Retry when the registry is - # reachable, or install a runtime package that ships the library. - raise RuntimeError( - "No Subresource Integrity value available for the in-process runtime " - f"library ({npm_platform}@{ver}); refusing to load unverified native code." - ) - _verify_integrity(data, integrity) - - lib_bytes = _extract_runtime_node(data, npm_platform) + package_dir = _release_package_dir(ver, runtime_platform) + if _should_skip_download() and not package_dir.exists(): + return None + package_dir = _ensure_release_package(ver) + lib_bytes = (package_dir / "prebuilds" / runtime_platform / "runtime.node").read_bytes() # Write atomically next to the CLI so concurrent starts don't observe a partial # library. A rename within the same directory is atomic on POSIX and Windows. @@ -634,16 +528,15 @@ def get_or_download_cli(version: str | None = None) -> str | None: if cached: return cached - # Check if download is disabled - if _should_skip_download(): - return None - # Check platform support before attempting download try: - get_asset_info() + runtime_platform = get_runtime_platform() except RuntimeError: return None + if _should_skip_download() and not _release_package_dir(ver, runtime_platform).exists(): + return None + # Download return download_cli(ver) diff --git a/python/copilot/_cli_version.py b/python/copilot/_cli_version.py index cb5939820a..4fd0921c2d 100644 --- a/python/copilot/_cli_version.py +++ b/python/copilot/_cli_version.py @@ -16,35 +16,10 @@ # DO NOT reformat this line — the inject script matches it exactly. CLI_VERSION: str | None = None -# Maps (sys.platform, platform.machine()) → (archive filename, binary name inside archive). -PLATFORM_ASSETS: dict[tuple[str, str], tuple[str, str]] = { - ("linux", "x86_64"): ("copilot-linux-x64.tar.gz", "copilot"), - ("linux", "aarch64"): ("copilot-linux-arm64.tar.gz", "copilot"), - ("linux", "arm64"): ("copilot-linux-arm64.tar.gz", "copilot"), - ("darwin", "x86_64"): ("copilot-darwin-x64.tar.gz", "copilot"), - ("darwin", "arm64"): ("copilot-darwin-arm64.tar.gz", "copilot"), - ("win32", "AMD64"): ("copilot-win32-x64.zip", "copilot.exe"), - ("win32", "ARM64"): ("copilot-win32-arm64.zip", "copilot.exe"), -} - -# Musl (Alpine) variants — detected at runtime via _is_musl(). -_MUSL_ASSETS: dict[str, tuple[str, str]] = { - "x86_64": ("copilot-linuxmusl-x64.tar.gz", "copilot"), - "aarch64": ("copilot-linuxmusl-arm64.tar.gz", "copilot"), - "arm64": ("copilot-linuxmusl-arm64.tar.gz", "copilot"), -} - _DOWNLOAD_BASE_URL = "https://github.com/github/copilot-cli/releases/download" -# The native in-process (FFI) runtime library (`runtime.node`) is NOT part of the -# GitHub Releases `copilot-` archive (that ships only the CLI binary). It -# lives in the npm platform package `@github/copilot-`, under -# `package/prebuilds//runtime.node`. Mirrors the .NET SDK targets, -# which download the same npm tarball. -_NPM_REGISTRY_BASE_URL = "https://registry.npmjs.org" - -# Maps (sys.platform, platform.machine()) → npm platform name (glibc Linux/macOS/Windows). -NPM_PLATFORMS: dict[tuple[str, str], str] = { +# Maps (sys.platform, platform.machine()) to the platform segment used by release assets. +RUNTIME_PLATFORMS: dict[tuple[str, str], str] = { ("linux", "x86_64"): "linux-x64", ("linux", "aarch64"): "linux-arm64", ("linux", "arm64"): "linux-arm64", @@ -54,8 +29,8 @@ ("win32", "ARM64"): "win32-arm64", } -# Musl (Alpine) npm platform variants — detected at runtime via _is_musl(). -_MUSL_NPM_PLATFORMS: dict[str, str] = { +# Musl (Alpine) runtime platform variants — detected at runtime via _is_musl(). +_MUSL_RUNTIME_PLATFORMS: dict[str, str] = { "x86_64": "linuxmusl-x64", "aarch64": "linuxmusl-arm64", "arm64": "linuxmusl-arm64", @@ -82,33 +57,11 @@ def get_platform_key() -> tuple[str, str]: return (sys.platform, platform.machine()) -def get_asset_info() -> tuple[str, str]: - """Return (archive_filename, binary_name) for the current platform. - - Raises RuntimeError if the platform is not supported. - """ - key = get_platform_key() - - # On Linux, check for musl/Alpine first - if key[0] == "linux" and _is_musl(): - musl_info = _MUSL_ASSETS.get(key[1]) - if musl_info: - return musl_info - - info = PLATFORM_ASSETS.get(key) - if info is None: - raise RuntimeError( - f"Unsupported platform: {key[0]}/{key[1]}. " - f"Supported platforms: {', '.join(f'{p}/{m}' for p, m in PLATFORM_ASSETS)}" - ) - return info - - def get_download_url(version: str, archive_name: str) -> str: """Return the download URL for a given version and archive.""" import os - base = os.environ.get("COPILOT_CLI_DOWNLOAD_BASE_URL", _DOWNLOAD_BASE_URL) + base = os.environ.get("COPILOT_CLI_DOWNLOAD_BASE_URL", _DOWNLOAD_BASE_URL).rstrip("/") return f"{base}/v{version}/{archive_name}" @@ -116,47 +69,38 @@ def get_checksums_url(version: str) -> str: """Return the URL for the SHA256SUMS.txt file.""" import os - base = os.environ.get("COPILOT_CLI_DOWNLOAD_BASE_URL", _DOWNLOAD_BASE_URL) + base = os.environ.get("COPILOT_CLI_DOWNLOAD_BASE_URL", _DOWNLOAD_BASE_URL).rstrip("/") return f"{base}/v{version}/SHA256SUMS.txt" -def get_npm_platform() -> str: - """Return the npm platform name (e.g. ``linux-x64``) for the current host. +def get_runtime_platform() -> str: + """Return the release asset platform name (e.g. ``linux-x64``) for this host. - Used to locate the native in-process runtime library. Raises RuntimeError if - the platform is not supported. + The name matches the ``prebuilds`` folder embedded in the release package. + Raises RuntimeError if the platform is not supported. """ key = get_platform_key() if key[0] == "linux" and _is_musl(): - musl = _MUSL_NPM_PLATFORMS.get(key[1]) + musl = _MUSL_RUNTIME_PLATFORMS.get(key[1]) if musl: return musl - npm_platform = NPM_PLATFORMS.get(key) - if npm_platform is None: + runtime_platform = RUNTIME_PLATFORMS.get(key) + if runtime_platform is None: raise RuntimeError( - f"Unsupported platform for in-process runtime: {key[0]}/{key[1]}. " - f"Supported platforms: {', '.join(f'{p}/{m}' for p, m in NPM_PLATFORMS)}" + f"Unsupported Copilot runtime platform: {key[0]}/{key[1]}. " + f"Supported platforms: {', '.join(f'{p}/{m}' for p, m in RUNTIME_PLATFORMS)}" ) - return npm_platform + return runtime_platform -def get_runtime_lib_packument_url(npm_platform: str) -> str: - """Return the npm packument URL for the platform runtime package.""" - import os +def get_release_asset_name(version: str, runtime_platform: str | None = None) -> str: + """Return the unified runtime package asset name for a version and platform.""" + platform_name = runtime_platform or get_runtime_platform() + return f"github-copilot-{version}-{platform_name}.tgz" - base = os.environ.get("COPILOT_NPM_REGISTRY_URL", _NPM_REGISTRY_BASE_URL).rstrip("/") - return f"{base}/@github/copilot-{npm_platform}" - - -def get_runtime_lib_url(version: str, npm_platform: str) -> str: - """Return the download URL for the platform runtime tarball. - - Mirrors the .NET targets' URL layout - ``/@github/copilot-/-/copilot--.tgz``. - """ - import os - base = os.environ.get("COPILOT_NPM_REGISTRY_URL", _NPM_REGISTRY_BASE_URL).rstrip("/") - return f"{base}/@github/copilot-{npm_platform}/-/copilot-{npm_platform}-{version}.tgz" +def get_cli_binary_name() -> str: + """Return the CLI executable name inside the release package.""" + return "copilot.exe" if sys.platform == "win32" else "copilot" diff --git a/python/copilot/_ffi_runtime_host.py b/python/copilot/_ffi_runtime_host.py index 5665f8fba7..511cbae9c4 100644 --- a/python/copilot/_ffi_runtime_host.py +++ b/python/copilot/_ffi_runtime_host.py @@ -121,7 +121,7 @@ def resolve_library_path(runtime_entrypoint: str) -> str | None: 1. The natural platform library name next to the CLI (bundled/flat layout, what the Python download-at-first-use path writes). - 2. ``runtime.node`` next to the CLI (prepared npm runtime layout). + 2. ``runtime.node`` next to the CLI (prepared release-package layout). 3. ``prebuilds//runtime.node`` next to the CLI (package-root layout). Returns the absolute path, or ``None`` when neither exists. @@ -211,7 +211,7 @@ def _load_library(library_path: str) -> _FfiLibrary: return _FfiLibrary(_loaded_library) # Load with immediate binding (RTLD_NOW) on POSIX, matching the .NET/Rust - # hosts. The runtime cdylib from the npm platform package is self-contained; + # hosts. The runtime cdylib from the platform release package is self-contained; # eager binding surfaces any load problem here rather than at first call. if sys.platform == "win32": lib = ctypes.WinDLL(library_path) diff --git a/python/test_cli_download.py b/python/test_cli_download.py index 4ae4fe6bc3..24433d862a 100644 --- a/python/test_cli_download.py +++ b/python/test_cli_download.py @@ -1,8 +1,7 @@ -"""Tests for the in-process runtime library download integrity checks.""" +"""Tests for unified Copilot release-package provisioning.""" from __future__ import annotations -import base64 import hashlib import io import os @@ -12,31 +11,24 @@ import pytest -from copilot import _cli_download, _ffi_runtime_host +from copilot import _cli_download, _cli_version, _ffi_runtime_host -def _integrity(data: bytes, algo: str = "sha512") -> str: - digest = hashlib.new(algo, data).digest() - return f"{algo}-{base64.b64encode(digest).decode('ascii')}" - - -def _runtime_package(npm_platform: str) -> bytes: +def _release_package(runtime_platform: str) -> bytes: wrapper_name = "copilot-runtime.exe" if os.name == "nt" else "copilot-runtime" members = { - f"package/prebuilds/{npm_platform}/{wrapper_name}": b"wrapper", - f"package/prebuilds/{npm_platform}/runtime.node": b"runtime", - "package/copilot": b"excluded", - "package/copilot.exe": b"excluded", - f"package/ripgrep/bin/{npm_platform}/rg": b"ripgrep", + f"package/prebuilds/{runtime_platform}/{wrapper_name}": b"wrapper", + f"package/prebuilds/{runtime_platform}/runtime.node": b"runtime", + f"package/ripgrep/bin/{runtime_platform}/rg": b"ripgrep", "package/definitions/future.json": b"{}", "package/app.js": b"excluded", "package/LICENSE.md": b"excluded", - "package/README.md": b"excluded", } buffer = io.BytesIO() with tarfile.open(fileobj=buffer, mode="w:gz") as archive: for name, content in members.items(): info = tarfile.TarInfo(name) + info.mode = 0o755 if name.endswith((wrapper_name, "/rg")) else 0o644 info.size = len(content) archive.addfile(info, io.BytesIO(content)) return buffer.getvalue() @@ -62,41 +54,168 @@ def test_fetch_url_bytes_retries_truncated_response(): sleep.assert_called_once_with(1) -class TestVerifyIntegrity: - def test_accepts_matching_checksum(self): - data = b"native-library-bytes" - _cli_download._verify_integrity(data, _integrity(data)) +def _release_fetches(version: str, runtime_platform: str, data: bytes): + asset_name = f"github-copilot-{version}-{runtime_platform}.tgz" + checksum = hashlib.sha256(data).hexdigest() + + def fetch(url: str, *, timeout: int) -> bytes: + del timeout + if url.endswith("/SHA256SUMS.txt"): + return f"{checksum} {asset_name}\n".encode() + assert url.endswith(f"/{asset_name}") + return data + + return fetch + - def test_rejects_mismatched_checksum(self): - with pytest.raises(RuntimeError, match="Integrity mismatch"): - _cli_download._verify_integrity(b"tampered", _integrity(b"original")) +def test_release_asset_uses_platform_package_name(monkeypatch): + monkeypatch.setenv("COPILOT_CLI_DOWNLOAD_BASE_URL", "https://mirror.example/releases/") - def test_rejects_unsupported_algorithm(self): - # Fail closed rather than silently skipping verification of native code. - with pytest.raises(RuntimeError, match="Unsupported integrity algorithm"): - _cli_download._verify_integrity(b"bytes", "md5-deadbeef") + name = _cli_version.get_release_asset_name("1.2.3-4", "linux-x64") + assert name == "github-copilot-1.2.3-4-linux-x64.tgz" + assert ( + _cli_version.get_download_url("1.2.3-4", name) + == "https://mirror.example/releases/v1.2.3-4/github-copilot-1.2.3-4-linux-x64.tgz" + ) -class TestEnsureRuntimeLibraryFailsClosed: - def test_raises_when_integrity_unavailable(self, tmp_path): - """A missing npm integrity value must abort the download, not load unverified code.""" - cli_path = tmp_path / "copilot" - cli_path.write_bytes(b"#!/bin/sh\n") - with ( - patch("copilot._ffi_runtime_host.resolve_library_path", return_value=None), - patch.object(_cli_download, "_should_skip_download", return_value=False), - patch.object(_cli_download, "get_npm_platform", return_value="linux-x64"), - patch.object(_cli_download, "get_runtime_lib_url", return_value="https://example/lib"), - patch.object(_cli_download, "_fetch_url_bytes", return_value=b"tarball-bytes"), - patch.object(_cli_download, "_fetch_runtime_integrity", return_value=None), - patch.object(_cli_download, "_extract_runtime_node") as extract, - ): - with pytest.raises(RuntimeError, match="refusing to load unverified native code"): - _cli_download.ensure_runtime_library(str(cli_path), version="1.2.3") +def test_rejects_release_package_checksum_mismatch(tmp_path): + runtime_platform = "linux-x64" + data = _release_package(runtime_platform) + asset_name = f"github-copilot-1.2.3-{runtime_platform}.tgz" + + def fetch(url: str, *, timeout: int) -> bytes: + del timeout + if url.endswith("/SHA256SUMS.txt"): + return f"{'0' * 64} {asset_name}\n".encode() + return data + + with ( + patch.object(_cli_download, "get_cache_dir", return_value=tmp_path / "cache"), + patch.object(_cli_download, "get_runtime_platform", return_value=runtime_platform), + patch.object(_cli_download, "_fetch_url_bytes", side_effect=fetch), + ): + with pytest.raises(RuntimeError, match="Checksum mismatch"): + _cli_download.ensure_runtime_wrapper(version="1.2.3") + - # The library bytes must never be extracted/written when verification is impossible. - extract.assert_not_called() +def test_rejects_release_package_without_checksum(tmp_path): + runtime_platform = "linux-x64" + + def fetch(url: str, *, timeout: int) -> bytes: + del timeout + assert url.endswith("/SHA256SUMS.txt") + return f"{'0' * 64} another-file.tgz\n".encode() + + with ( + patch.object(_cli_download, "get_cache_dir", return_value=tmp_path / "cache"), + patch.object(_cli_download, "get_runtime_platform", return_value=runtime_platform), + patch.object(_cli_download, "_fetch_url_bytes", side_effect=fetch), + ): + with pytest.raises(RuntimeError, match="SHA256SUMS.txt does not contain"): + _cli_download.ensure_runtime_wrapper(version="1.2.3") + + +def test_cli_and_runtime_share_one_cached_release_package(tmp_path, monkeypatch): + version = "1.2.3" + runtime_platform = "linux-x64" + cli_name = "copilot.exe" if os.name == "nt" else "copilot" + wrapper_name = "copilot-runtime.exe" if os.name == "nt" else "copilot-runtime" + data = _release_package(runtime_platform) + cache_dir = tmp_path / "cache" + install_dir = cache_dir / "prebuilds" / runtime_platform + package_dir = cache_dir / "packages" / runtime_platform + fetch = _release_fetches(version, runtime_platform, data) + + with ( + patch.object(_cli_download, "get_cache_dir", return_value=cache_dir), + patch.object(_cli_download, "get_runtime_platform", return_value=runtime_platform), + patch.object(_cli_download, "_fetch_url_bytes", side_effect=fetch) as fetch_mock, + ): + cli = _cli_download.download_cli(version) + monkeypatch.setenv("COPILOT_SKIP_CLI_DOWNLOAD", "1") + wrapper = _cli_download.ensure_runtime_wrapper(version) + assert _cli_download.get_cached_cli_path(version) == str(install_dir / cli_name) + + assert cli == str(install_dir / cli_name) + assert wrapper == str(install_dir / wrapper_name) + assert (install_dir / cli_name).read_bytes() == b"wrapper" + assert not (cache_dir / cli_name).exists() + assert not (package_dir / cli_name).exists() + assert (install_dir / wrapper_name).read_bytes() == b"wrapper" + assert (install_dir / "runtime.node").read_bytes() == b"runtime" + assert (install_dir / "ripgrep" / "bin" / runtime_platform / "rg").read_bytes() == b"ripgrep" + assert (install_dir / "definitions" / "future.json").read_bytes() == b"{}" + assert not (install_dir / "app.js").exists() + assert (install_dir / ".hostless-runtime-assets-v2").is_file() + assert fetch_mock.call_count == 2 + if os.name != "nt": + assert (install_dir / cli_name).stat().st_mode & 0o111 + assert (install_dir / wrapper_name).stat().st_mode & 0o111 + + +def test_skip_download_returns_none_without_cached_package(tmp_path, monkeypatch): + monkeypatch.setenv("COPILOT_SKIP_CLI_DOWNLOAD", "true") + runtime_platform = "linux-x64" + + with ( + patch.object(_cli_download, "get_cache_dir", return_value=tmp_path / "cache"), + patch.object(_cli_download, "get_runtime_platform", return_value=runtime_platform), + patch.object(_cli_download, "_fetch_url_bytes") as fetch_mock, + ): + assert _cli_download.get_or_download_cli("1.2.3") is None + + fetch_mock.assert_not_called() + + +def test_cached_cli_rejects_alias_from_incomplete_bundle(tmp_path): + version = "1.2.3" + runtime_platform = "linux-x64" + cli_name = "copilot.exe" if os.name == "nt" else "copilot" + wrapper_name = "copilot-runtime.exe" if os.name == "nt" else "copilot-runtime" + cache_dir = tmp_path / "cache" + install_dir = cache_dir / "prebuilds" / runtime_platform + install_dir.mkdir(parents=True) + (install_dir / cli_name).write_bytes(b"stale-wrapper") + (install_dir / wrapper_name).write_bytes(b"wrapper") + + with ( + patch.object(_cli_download, "get_cache_dir", return_value=cache_dir), + patch.object(_cli_download, "get_runtime_platform", return_value=runtime_platform), + ): + assert _cli_download.get_cached_cli_path(version) is None + with pytest.raises(RuntimeError, match="Incomplete Copilot runtime bundle"): + _cli_download.download_cli(version) + + +def test_explicit_cli_gets_library_from_cached_release_package(tmp_path): + version = "1.2.3" + runtime_platform = "linux-x64" + data = _release_package(runtime_platform) + cache_dir = tmp_path / "cache" + cli_dir = tmp_path / "external" + cli_dir.mkdir() + cli_path = cli_dir / ("copilot.exe" if os.name == "nt" else "copilot") + cli_path.write_bytes(b"external") + fetch = _release_fetches(version, runtime_platform, data) + + with ( + patch.object(_cli_download, "get_cache_dir", return_value=cache_dir), + patch.object(_cli_download, "get_runtime_platform", return_value=runtime_platform), + patch.object(_cli_download, "_fetch_url_bytes", side_effect=fetch) as fetch_mock, + patch("copilot._ffi_runtime_host.resolve_library_path", return_value=None), + ): + library = _cli_download.ensure_runtime_library(str(cli_path), version) + wrapper = _cli_download.ensure_runtime_wrapper(version) + + assert library == str(cli_dir / _ffi_runtime_host._natural_library_name()) + assert (cli_dir / _ffi_runtime_host._natural_library_name()).read_bytes() == b"runtime" + assert wrapper.endswith( + f"prebuilds/{runtime_platform}/" + f"{'copilot-runtime.exe' if os.name == 'nt' else 'copilot-runtime'}" + ) + assert fetch_mock.call_count == 2 def test_resolve_library_path_accepts_adjacent_runtime_node(tmp_path): @@ -108,90 +227,17 @@ def test_resolve_library_path_accepts_adjacent_runtime_node(tmp_path): assert _ffi_runtime_host.resolve_library_path(str(wrapper)) == str(runtime_node) -class TestEnsureRuntimeWrapper: - def test_materializes_pair_from_absent_cache_with_stripped_environment( - self, tmp_path, monkeypatch +def test_rejects_cached_wrapper_without_runtime_node(tmp_path): + runtime_platform = "linux-x64" + wrapper_name = "copilot-runtime.exe" if os.name == "nt" else "copilot-runtime" + cache_dir = tmp_path / "cache" + install_dir = cache_dir / "prebuilds" / runtime_platform + install_dir.mkdir(parents=True) + (install_dir / wrapper_name).write_bytes(b"wrapper") + + with ( + patch.object(_cli_download, "get_cache_dir", return_value=cache_dir), + patch.object(_cli_download, "get_runtime_platform", return_value=runtime_platform), ): - npm_platform = "win32-x64" if os.name == "nt" else "linux-x64" - wrapper_name = "copilot-runtime.exe" if os.name == "nt" else "copilot-runtime" - data = _runtime_package(npm_platform) - cache_dir = tmp_path / "cache" - empty_path = tmp_path / "empty-path" - empty_path.mkdir() - assert not cache_dir.exists() - - for name in ( - "COPILOT_CLI_PATH", - "COPILOT_RUNTIME_HOST_COMMAND", - "COPILOT_RUNTIME_PROVIDER_LIB", - ): - monkeypatch.delenv(name, raising=False) - monkeypatch.setenv("PATH", str(empty_path)) - - with ( - patch.object(_cli_download, "get_cache_dir", return_value=cache_dir), - patch.object(_cli_download, "get_npm_platform", return_value=npm_platform), - patch.object(_cli_download, "_should_skip_download", return_value=False), - patch.object(_cli_download, "_fetch_url_bytes", return_value=data), - patch.object( - _cli_download, - "_fetch_runtime_integrity", - return_value=_integrity(data), - ), - ): - wrapper = _cli_download.ensure_runtime_wrapper(version="1.2.3") - - install_dir = cache_dir / "prebuilds" / npm_platform - assert wrapper == str(install_dir / wrapper_name) - assert (install_dir / wrapper_name).read_bytes() == b"wrapper" - assert (install_dir / "runtime.node").read_bytes() == b"runtime" - assert (install_dir / "ripgrep" / "bin" / npm_platform / "rg").read_bytes() == b"ripgrep" - assert (install_dir / "definitions" / "future.json").read_bytes() == b"{}" - assert not (install_dir / "app.js").exists() - assert not (install_dir / "copilot").exists() - assert not (install_dir / "copilot.exe").exists() - assert (install_dir / ".hostless-runtime-assets-v2").is_file() - if os.name != "nt": - assert (install_dir / wrapper_name).stat().st_mode & 0o111 - - def test_rejects_cached_wrapper_without_runtime_node(self, tmp_path): - npm_platform = "win32-x64" if os.name == "nt" else "linux-x64" - wrapper_name = "copilot-runtime.exe" if os.name == "nt" else "copilot-runtime" - cache_dir = tmp_path / "cache" - install_dir = cache_dir / "prebuilds" / npm_platform - install_dir.mkdir(parents=True) - (install_dir / wrapper_name).write_bytes(b"wrapper") - - with ( - patch.object(_cli_download, "get_cache_dir", return_value=cache_dir), - patch.object(_cli_download, "get_npm_platform", return_value=npm_platform), - ): - with pytest.raises(RuntimeError, match="Incomplete Copilot runtime bundle"): - _cli_download.ensure_runtime_wrapper(version="1.2.3") - - def test_upgrades_pair_only_cache_with_retained_assets(self, tmp_path): - npm_platform = "win32-x64" if os.name == "nt" else "linux-x64" - wrapper_name = "copilot-runtime.exe" if os.name == "nt" else "copilot-runtime" - cache_dir = tmp_path / "cache" - install_dir = cache_dir / "prebuilds" / npm_platform - install_dir.mkdir(parents=True) - (install_dir / wrapper_name).write_bytes(b"old-wrapper") - (install_dir / "runtime.node").write_bytes(b"old-runtime") - (install_dir / "copilot").write_bytes(b"legacy-sea") - (install_dir / ".hostless-runtime-assets-v1").write_text("1\n", encoding="ascii") - data = _runtime_package(npm_platform) - - with ( - patch.object(_cli_download, "get_cache_dir", return_value=cache_dir), - patch.object(_cli_download, "get_npm_platform", return_value=npm_platform), - patch.object(_cli_download, "_should_skip_download", return_value=False), - patch.object(_cli_download, "_fetch_url_bytes", return_value=data), - patch.object(_cli_download, "_fetch_runtime_integrity", return_value=_integrity(data)), - ): - wrapper = _cli_download.ensure_runtime_wrapper(version="1.2.3") - - assert wrapper == str(install_dir / wrapper_name) - assert (install_dir / wrapper_name).read_bytes() == b"wrapper" - assert not (install_dir / "copilot").exists() - assert (install_dir / ".hostless-runtime-assets-v2").is_file() - assert (install_dir / "ripgrep" / "bin" / npm_platform / "rg").is_file() + with pytest.raises(RuntimeError, match="Incomplete Copilot runtime bundle"): + _cli_download.ensure_runtime_wrapper(version="1.2.3") diff --git a/rust/.gitignore b/rust/.gitignore index c149fa3946..c4095ffc0f 100644 --- a/rust/.gitignore +++ b/rust/.gitignore @@ -1,4 +1,3 @@ /target Cargo.lock.bak cli-version.txt -cli-version-in-process.txt diff --git a/rust/Cargo.lock b/rust/Cargo.lock index b91eebd06c..8a9188832d 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -23,15 +23,6 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" -[[package]] -name = "arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" -dependencies = [ - "derive_arbitrary", -] - [[package]] name = "async-trait" version = "0.1.89" @@ -138,12 +129,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - [[package]] name = "crypto-common" version = "0.1.7" @@ -160,17 +145,6 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" -[[package]] -name = "derive_arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "digest" version = "0.10.7" @@ -455,7 +429,6 @@ dependencies = [ "ureq", "uuid", "windows-sys 0.61.2", - "zip", ] [[package]] @@ -1091,7 +1064,7 @@ checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 1.0.69", + "thiserror", ] [[package]] @@ -1585,16 +1558,7 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl 1.0.69", -] - -[[package]] -name = "thiserror" -version = "2.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" -dependencies = [ - "thiserror-impl 2.0.18", + "thiserror-impl", ] [[package]] @@ -1608,17 +1572,6 @@ dependencies = [ "syn", ] -[[package]] -name = "thiserror-impl" -version = "2.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tinystr" version = "0.8.3" @@ -1812,7 +1765,7 @@ dependencies = [ "native-tls", "rand", "sha1", - "thiserror 1.0.69", + "thiserror", "utf-8", ] @@ -2410,37 +2363,8 @@ dependencies = [ "syn", ] -[[package]] -name = "zip" -version = "2.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" -dependencies = [ - "arbitrary", - "crc32fast", - "crossbeam-utils", - "displaydoc", - "flate2", - "indexmap", - "memchr", - "thiserror 2.0.18", - "zopfli", -] - [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" - -[[package]] -name = "zopfli" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" -dependencies = [ - "bumpalo", - "crc32fast", - "log", - "simd-adler32", -] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index c66480d511..af6f7e28ff 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -21,7 +21,6 @@ include = [ "README.md", "LICENSE", "cli-version.txt", - "cli-version-in-process.txt", ] [lib] @@ -29,7 +28,7 @@ name = "github_copilot_sdk" [features] default = ["bundled-cli"] -bundled-cli = ["dep:tar", "dep:flate2", "dep:zip"] +bundled-cli = ["dep:tar", "dep:flate2"] bundled-in-process = ["bundled-cli", "dep:libloading"] derive = ["dep:schemars"] test-support = [] @@ -69,7 +68,6 @@ reqwest = { version = "0.12", default-features = false, features = ["stream", "h tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "native-tls"] } [target.'cfg(windows)'.dependencies] -zip = { version = "2", default-features = false, features = ["deflate"], optional = true } windows-sys = { version = "0.61", default-features = false, features = [ "Win32_Foundation", "Win32_System_Diagnostics_ToolHelp", @@ -126,4 +124,3 @@ sha2 = "0.10" tar = "0.4" ureq = { version = "2", default-features = false, features = ["native-tls"] } native-tls = "0.2" -zip = { version = "2", default-features = false, features = ["deflate"] } diff --git a/rust/README.md b/rust/README.md index a561e6da09..02035193a0 100644 --- a/rust/README.md +++ b/rust/README.md @@ -965,8 +965,9 @@ github-copilot-sdk = { version = "0.1", default-features = false } The resolved version is baked into the crate via `cargo:rustc-env=COPILOT_SDK_CLI_VERSION` regardless of mode. The runtime resolver consumes it to recompute the on-disk path by convention, so no absolute paths leak into the rlib. -2. **Build time:** `build.rs` downloads the platform-specific release archive and - verifies its SHA-256 against the release's `SHA256SUMS.txt` or the publish snapshot. +2. **Build time:** `build.rs` downloads + `github-copilot--.tgz` and verifies its SHA-256 against + the release's `SHA256SUMS.txt` or the publish snapshot. Then: - **`bundled-cli` on (default):** creates and embeds a minimal archive containing the CLI executable, `copilot-runtime[.exe]`, and `runtime.node`. - **`bundled-in-process` on:** the archive additionally contains the platform-native runtime library (`.dll`, `.so`, or `.dylib`). @@ -1065,7 +1066,7 @@ In embed mode `build.rs` re-downloads on every clean build by default. Set `BUND ### Platforms -Supported: `darwin-arm64`, `darwin-x64`, `linux-x64`, `linux-arm64`, `win32-x64`, `win32-arm64`. The target platform is auto-detected from `CARGO_CFG_TARGET_OS` and `CARGO_CFG_TARGET_ARCH` (cross-compilation works). +Supported: `darwin-arm64`, `darwin-x64`, `linux-x64`, `linux-arm64`, `linuxmusl-x64`, `linuxmusl-arm64`, `win32-x64`, `win32-arm64`. The target platform is auto-detected from Cargo's target OS, architecture, and environment (cross-compilation works). ## Features diff --git a/rust/build.rs b/rust/build.rs index c01464bb4a..2d8eb9992d 100644 --- a/rust/build.rs +++ b/rust/build.rs @@ -1,4 +1,4 @@ -#[path = "build/in_process.rs"] +#[path = "build/runtime.rs"] mod implementation; fn main() { diff --git a/rust/build/out_of_process.rs b/rust/build/out_of_process.rs deleted file mode 100644 index c0a9dd3050..0000000000 --- a/rust/build/out_of_process.rs +++ /dev/null @@ -1,692 +0,0 @@ -use std::io::{Read, Write}; -use std::path::{Path, PathBuf}; -use std::time::Duration; - -use sha2::Digest; - -pub(crate) fn main() { - println!("cargo:rerun-if-env-changed=DOCS_RS"); - println!("cargo:rerun-if-env-changed=COPILOT_SKIP_CLI_DOWNLOAD"); - println!("cargo:rerun-if-env-changed=COPILOT_CLI_EXTRACT_DIR"); - println!("cargo:rerun-if-env-changed=BUNDLED_CLI_CACHE_DIR"); - println!("cargo::rustc-check-cfg=cfg(has_bundled_cli)"); - println!("cargo::rustc-check-cfg=cfg(has_extracted_cli)"); - println!("cargo:rerun-if-changed=cli-version.txt"); - - // Only declare the package metadata rerun when it actually exists. - // Cargo treats `rerun-if-changed` for a missing path as "always rerun" - // — so unconditionally declaring this on consumers without a sibling - // `nodejs/` (vendored slots, published crates) would force build.rs - // to re-run on every `cargo build` even when nothing has changed. - // The package file is only the source-of-truth in this repo's - // contributor builds; everywhere else `cli-version.txt` is canonical. - let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is set"); - let package_json = Path::new(&manifest_dir) - .join("..") - .join("nodejs") - .join("package.json"); - if package_json.is_file() { - println!("cargo:rerun-if-changed={}", package_json.display()); - } - - // Hard opt-out: disable the entire download / bundle / cache mechanism - // in one step. For consumers who always supply the CLI via - // `CliProgram::Path` or `COPILOT_CLI_PATH` and don't want build.rs to - // touch the network (offline builds, locked-down CI, etc.). Works - // regardless of the `bundled-cli` cargo feature state — with neither - // `has_bundled_cli` nor `has_extracted_cli` emitted, runtime resolution - // falls straight through to `Error::BinaryNotFound` unless an explicit - // path source resolves first. - if std::env::var_os("COPILOT_SKIP_CLI_DOWNLOAD").is_some() { - println!( - "cargo:warning=COPILOT_SKIP_CLI_DOWNLOAD is set — skipping CLI download/bundle/cache" - ); - return; - } - - // docs.rs builds in a sandboxed environment without network access. - // Skip the CLI download so documentation can be generated successfully. - if std::env::var_os("DOCS_RS").is_some() { - println!("cargo:warning=DOCS_RS is set — skipping CLI download/bundle/cache"); - return; - } - - let Some(platform) = target_platform() else { - println!("cargo:warning=Unsupported target platform for Copilot CLI bundling — skipping"); - return; - }; - - let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR is always set by cargo"); - let out = Path::new(&out_dir); - - // Resolve version + per-asset SHA-256 from one of two sources, in order: - // 1. `cli-version.txt` snapshot at the crate root (published-crate - // consumer; generated by the publish workflow from SHA256SUMS.txt). - // 2. Sibling `../nodejs/package.json` plus the release SHA256SUMS.txt - // (contributor build inside the github/copilot-sdk repo). - let (version, expected_hash) = resolve_version_and_hash(platform.asset_name); - - // Bake the version into the crate regardless of mode. This is the - // single source of truth for "what CLI version did build.rs target", - // consumed by both the embed-mode path computation in embeddedcli.rs - // and the runtime path computation in resolve.rs (when `bundled-cli` - // is off). It's a small, machine-independent datum: no absolute - // paths, no username/home leakage, so sccache / cross-machine - // `target/` reuse stays cache-coherent. - println!("cargo:rustc-env=COPILOT_SDK_CLI_VERSION={version}"); - - let base_url = format!("https://github.com/github/copilot-cli/releases/download/v{version}"); - let cache_dir = std::env::var("BUNDLED_CLI_CACHE_DIR") - .ok() - .map(std::path::PathBuf::from); - - // Versioned cache key since copilot asset names don't include the version. - let cache_key = format!("v{version}-{}", platform.asset_name); - - if std::env::var_os("CARGO_FEATURE_BUNDLED_CLI").is_some() { - // Embed mode: we need the archive bytes to bake into the rlib, so - // always run the download (cache hit short-circuits inside - // `cached_download`). - let archive = cached_download( - &format!("{base_url}/{}", platform.asset_name), - &cache_key, - &expected_hash, - &cache_dir, - ); - verify_binary_present_in_archive(&archive, platform.binary_name, platform.asset_name); - emit_embedded(out, &archive); - println!("cargo:rustc-cfg=has_bundled_cli"); - } else { - // With `bundled-cli` off the extracted binary *is* the cache. - // Skip the upstream download entirely when it already exists at - // the expected path. No two separate caches. - // - // Runtime resolution (see `src/resolve.rs::extracted_cli_path`) - // recomputes this same path from `COPILOT_SDK_CLI_VERSION` + the - // OS-derived binary name + optional `COPILOT_CLI_EXTRACT_DIR`, - // so we don't bake an absolute path into the crate. - let install_dir = extracted_install_dir(&version); - let final_path = install_dir.join(platform.binary_name); - - // Invalidate build.rs whenever the cached binary disappears (cache GC, - // manual rm, OS reset, switching extract dir). Without this, cargo - // replays the saved `has_extracted_cli` cfg from its build-script - // output cache even when the file is gone, and runtime resolution - // fails with BinaryNotFound. - println!("cargo:rerun-if-changed={}", final_path.display()); - - if !final_path.is_file() { - let archive = cached_download( - &format!("{base_url}/{}", platform.asset_name), - &cache_key, - &expected_hash, - &cache_dir, - ); - verify_binary_present_in_archive(&archive, platform.binary_name, platform.asset_name); - extract_to_cache(&archive, &install_dir, platform); - } - - // Re-check after potential download+extract above; not an `else` - // because we need to verify the extraction actually produced the file. - if final_path.is_file() { - println!("cargo:rustc-cfg=has_extracted_cli"); - } - } -} - -/// Install directory used when `bundled-cli` is off. Mirrors the runtime -/// convention in `src/resolve.rs::extracted_cli_path`: both sides MUST -/// compute the same path from the same inputs, otherwise the runtime -/// resolver won't find what build.rs extracted. -/// -/// If `COPILOT_CLI_EXTRACT_DIR` is set the binary lives directly under -/// that directory (no per-version subdir) — useful for vendored slots and -/// for `.cargo/config.toml [env]`-style pinning that's symmetric between -/// build-time write and runtime read. Otherwise the binary lives under -/// `/github-copilot-sdk/cli//`. -fn extracted_install_dir(version: &str) -> PathBuf { - if let Some(custom) = std::env::var_os("COPILOT_CLI_EXTRACT_DIR") { - PathBuf::from(custom) - } else { - let cache = dirs::cache_dir().unwrap_or_else(std::env::temp_dir); - cache - .join("github-copilot-sdk") - .join("cli") - .join(sanitize_version(version)) - } -} - -/// Emit the `bundled_cli.rs` glue + `copilot_cli.archive` blob into `OUT_DIR` -/// for embed mode (`bundled-cli` cargo feature on). The version is exposed -/// crate-wide via the unconditional `cargo:rustc-env=COPILOT_SDK_CLI_VERSION` -/// emit; the binary name is OS-derived at runtime — so all we need to -/// generate here is the archive blob include. -fn emit_embedded(out: &Path, archive: &[u8]) { - std::fs::write(out.join("copilot_cli.archive"), archive) - .expect("failed to write copilot_cli.archive"); - - let generated = r#"// Auto-generated by github-copilot-sdk build.rs. Do not edit. -pub(super) static CLI_ARCHIVE: &[u8] = include_bytes!("copilot_cli.archive"); -"#; - - std::fs::write(out.join("bundled_cli.rs"), generated).expect("failed to write bundled_cli.rs"); -} - -/// Resolve the CLI version and the expected SHA-256 hash for the current -/// target's archive. Picks one of two sources in order. Panics with a clear -/// error if neither is available. -fn resolve_version_and_hash(asset_name: &str) -> (String, String) { - let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is set"); - - // 1. Snapshot file at the crate root (published-crate consumer, - // vendored-slot consumer). Combined version + per-asset hashes. - let snapshot = Path::new(&manifest_dir).join("cli-version.txt"); - if snapshot.is_file() { - let contents = std::fs::read_to_string(&snapshot) - .unwrap_or_else(|e| panic!("failed to read {}: {e}", snapshot.display())); - return parse_snapshot(&contents, asset_name) - .unwrap_or_else(|e| panic!("invalid {}: {e}", snapshot.display())); - } - - // 2. Package version plus release checksums (contributor build). - let package_json = Path::new(&manifest_dir) - .join("..") - .join("nodejs") - .join("package.json"); - if package_json.is_file() { - let version = read_version_from_package_json(&package_json); - let hash = fetch_release_hash(&version, asset_name); - return (version, hash); - } - - panic!( - "Could not resolve the Copilot CLI version.\n\ - Tried:\n\ - - {} (missing)\n\ - - {} (missing)\n\ - In a published crate or vendored slot, `cli-version.txt` should be present.\n\ - Inside the github/copilot-sdk repo, `../nodejs/package.json` is the version source.", - snapshot.display(), - package_json.display(), - ); -} - -/// Parse the `cli-version.txt` snapshot file. Format is one `key=value` per -/// line. The first non-comment line is `version=X.Y.Z`; subsequent lines map -/// asset filename to hex SHA-256. Blank lines and lines starting with `#` -/// are skipped. -fn parse_snapshot(contents: &str, asset_name: &str) -> Result<(String, String), String> { - let mut version: Option = None; - let mut hash: Option = None; - for (line_no, raw) in contents.lines().enumerate() { - let line = raw.trim(); - if line.is_empty() || line.starts_with('#') { - continue; - } - let Some((key, value)) = line.split_once('=') else { - return Err(format!( - "line {}: expected `key=value`, got `{raw}`", - line_no + 1 - )); - }; - match key.trim() { - "version" => version = Some(value.trim().to_string()), - k if k == asset_name => hash = Some(value.trim().to_string()), - _ => {} - } - } - let version = version.ok_or("missing `version=` line")?; - let hash = hash.ok_or_else(|| format!("missing hash for asset `{asset_name}`"))?; - Ok((version, hash)) -} - -fn read_version_from_package_json(path: &Path) -> String { - let contents = std::fs::read_to_string(path) - .unwrap_or_else(|e| panic!("failed to read {}: {e}", path.display())); - let package_json: serde_json::Value = serde_json::from_str(&contents) - .unwrap_or_else(|e| panic!("failed to parse {}: {e}", path.display())); - package_json["copilotCliVersion"] - .as_str() - .unwrap_or_else(|| panic!("copilotCliVersion is missing in {}", path.display())) - .to_string() -} - -fn fetch_release_hash(version: &str, asset_name: &str) -> String { - let url = - format!("https://github.com/github/copilot-cli/releases/download/v{version}/SHA256SUMS.txt"); - let checksums = download_with_retry(&url); - let checksums = - std::str::from_utf8(&checksums).expect("SHA256SUMS.txt is not valid UTF-8"); - find_sha256_for_asset(checksums, asset_name) -} - -fn find_sha256_for_asset(sums: &str, asset_name: &str) -> String { - sums.lines() - .find_map(|line| { - let (hash, name) = line.split_once(char::is_whitespace)?; - (name.trim_start().trim_start_matches('*') == asset_name).then(|| hash.to_string()) - }) - .unwrap_or_else(|| panic!("SHA256SUMS.txt does not contain {asset_name}")) -} - -#[derive(Clone, Copy)] -struct Platform { - asset_name: &'static str, - binary_name: &'static str, -} - -fn target_platform() -> Option { - let os = std::env::var("CARGO_CFG_TARGET_OS").ok()?; - let arch = std::env::var("CARGO_CFG_TARGET_ARCH").ok()?; - - match (os.as_str(), arch.as_str()) { - ("macos", "aarch64") => Some(Platform { - asset_name: "copilot-darwin-arm64.tar.gz", - binary_name: "copilot", - }), - ("macos", "x86_64") => Some(Platform { - asset_name: "copilot-darwin-x64.tar.gz", - binary_name: "copilot", - }), - ("linux", "x86_64") => Some(Platform { - asset_name: "copilot-linux-x64.tar.gz", - binary_name: "copilot", - }), - ("linux", "aarch64") => Some(Platform { - asset_name: "copilot-linux-arm64.tar.gz", - binary_name: "copilot", - }), - ("windows", "x86_64") => Some(Platform { - asset_name: "copilot-win32-x64.zip", - binary_name: "copilot.exe", - }), - ("windows", "aarch64") => Some(Platform { - asset_name: "copilot-win32-arm64.zip", - binary_name: "copilot.exe", - }), - _ => None, - } -} - -/// Write the single binary entry from `archive` to -/// `/` and return the resulting path. -/// Idempotent — returns the existing path if a previous build already -/// populated the target. -/// -/// Uses file-level staging + atomic rename so a concurrent reader during -/// a parallel `cargo build` race never observes a partially-written -/// binary. `fs::rename` for files is atomic on both Unix and Windows -/// (Windows uses `MoveFileExW` with `MOVEFILE_REPLACE_EXISTING`); for -/// directories it is not, which is why we stage at file granularity. -fn extract_to_cache(archive: &[u8], install_dir: &Path, platform: Platform) -> PathBuf { - let final_path = install_dir.join(platform.binary_name); - - // Caller already gated on `final_path.is_file()`; this is a safety - // net for any future caller that forgets. - if final_path.is_file() { - return final_path; - } - - std::fs::create_dir_all(install_dir).unwrap_or_else(|e| { - panic!( - "failed to create install dir {}: {e}", - install_dir.display() - ) - }); - - let bytes = extract_binary_bytes(archive, platform); - - // Staging file is a sibling of the final binary so the rename stays - // on the same filesystem (cross-fs rename is not atomic). PID + nanos - // disambiguate concurrent builds racing on the same cache. - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_nanos()) - .unwrap_or(0); - let staging_path = install_dir.join(format!( - ".{}.staging-{}-{nanos}", - platform.binary_name, - std::process::id(), - )); - - { - let mut f = std::fs::File::create(&staging_path).unwrap_or_else(|e| { - let _ = std::fs::remove_file(&staging_path); - panic!( - "failed to create staging file {}: {e}", - staging_path.display() - ); - }); - - if let Err(e) = f.write_all(&bytes) { - let _ = std::fs::remove_file(&staging_path); - panic!( - "failed to write staging file {}: {e}", - staging_path.display() - ); - } - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - if let Err(e) = f.set_permissions(std::fs::Permissions::from_mode(0o755)) { - let _ = std::fs::remove_file(&staging_path); - panic!("failed to chmod {}: {e}", staging_path.display()); - } - } - - // Backdate the staged binary to the Unix epoch before it lands. We emit - // `cargo:rerun-if-changed` on `final_path` (see caller) so a *deleted* - // cache binary forces a re-extract — but cargo stamps the build-script - // `output` reference when the script is spawned, seconds before this - // freshly-downloaded binary is written. A current mtime would therefore - // be *newer* than that reference, so the next identical `cargo` - // invocation would see the watched file as "changed" and pointlessly - // rerun build.rs + recompile the crate + relink every downstream crate. - // Pinning to the epoch keeps the file unambiguously older than any real - // build reference; `rename` preserves mtime (same inode), so it lands - // already-backdated and a no-change rebuild stays a true no-op. The - // deleted-file recovery contract is untouched: a missing file can't be - // stat'd, so cargo still treats it as stale and reruns regardless. - // - // Best-effort: a filesystem that refuses the epoch (e.g. FAT's 1980 floor - // clamps it — still older than any real reference) or rejects the call - // just reverts to the pre-fix redundant-rebuild behaviour, never a broken - // build. - if let Err(e) = f.set_modified(std::time::SystemTime::UNIX_EPOCH) { - println!( - "cargo:warning=Could not backdate {} (a redundant rebuild may occur): {e}", - staging_path.display() - ); - } - } - - // Atomic file-replace on both Unix and Windows. If a concurrent build - // already produced the same file the rename overwrites it; the bytes - // are SHA-verified-identical so replacement is safe. - if let Err(e) = std::fs::rename(&staging_path, &final_path) { - let _ = std::fs::remove_file(&staging_path); - panic!( - "failed to rename {} -> {}: {e}", - staging_path.display(), - final_path.display() - ); - } - - // Surface where the binary landed so contributors can find it. Quiet - // on the hot path: the caller's `is_file()` short-circuit (and the - // safety net at the top of this function) means this only fires on a - // true cache miss. - println!( - "cargo:warning=Extracted Copilot CLI to {}", - final_path.display() - ); - - final_path -} - -/// Replace characters outside `[a-zA-Z0-9._-]` with `_` so the version -/// string is always safe to use as a path component. Kept in sync with -/// `embeddedcli::sanitize_version` and `resolve::sanitize_version` so all -/// three resolve to the same cache directory for any given version. -fn sanitize_version(version: &str) -> String { - version - .chars() - .map(|c| match c { - 'a'..='z' | 'A'..='Z' | '0'..='9' | '.' | '-' | '_' => c, - _ => '_', - }) - .collect() -} - -/// Extract the single `binary_name` entry from the release archive. Reused -/// between embed mode's `verify_binary_present_in_archive` and the -/// `extract_to_cache` path used when `bundled-cli` is off. Panics if the -/// entry isn't found — callers have already invoked -/// `verify_binary_present_in_archive`. -fn extract_binary_bytes(archive: &[u8], platform: Platform) -> Vec { - if platform.asset_name.ends_with(".zip") { - let cursor = std::io::Cursor::new(archive); - let mut zip = zip::ZipArchive::new(cursor) - .unwrap_or_else(|e| panic!("failed to open zip archive: {e}")); - for i in 0..zip.len() { - let mut entry = zip - .by_index(i) - .unwrap_or_else(|e| panic!("failed to read zip entry {i}: {e}")); - let name = entry.name().to_string(); - if name == platform.binary_name || name.ends_with(&format!("/{}", platform.binary_name)) - { - let mut bytes = Vec::with_capacity(entry.size() as usize); - std::io::copy(&mut entry, &mut bytes) - .unwrap_or_else(|e| panic!("failed to read zip entry bytes: {e}")); - return bytes; - } - } - } else { - let gz = flate2::read::GzDecoder::new(archive); - let mut tar = tar::Archive::new(gz); - for entry in tar - .entries() - .unwrap_or_else(|e| panic!("failed to read tar entries: {e}")) - { - let mut entry = entry.unwrap_or_else(|e| panic!("failed to read tar entry: {e}")); - let path = entry - .path() - .unwrap_or_else(|e| panic!("failed to read tar entry path: {e}")); - let name = path.to_string_lossy().into_owned(); - if name == platform.binary_name || name.ends_with(&format!("/{}", platform.binary_name)) - { - let mut bytes = Vec::with_capacity(entry.size() as usize); - entry - .read_to_end(&mut bytes) - .unwrap_or_else(|e| panic!("failed to read tar entry bytes: {e}")); - return bytes; - } - } - } - panic!( - "binary `{}` not found in archive `{}`", - platform.binary_name, platform.asset_name - ); -} - -/// Read a file from the download cache, or download it (with retries) and save -/// to cache. Verifies SHA-256 on every path. Evicts stale/corrupt cache entries -/// automatically. Cache I/O failures are treated as cache misses — they never -/// break the build. -fn cached_download( - url: &str, - cache_key: &str, - expected_hash: &str, - cache_dir: &Option, -) -> Vec { - if let Some(dir) = cache_dir { - let cached_path = dir.join(cache_key); - if cached_path.is_file() { - match std::fs::read(&cached_path) { - Ok(data) if hex_sha256(&data) == expected_hash => { - // Silent cache hit — nothing to surface. - return data; - } - Ok(_) => { - println!("cargo:warning=Cached archive hash mismatch, re-downloading"); - let _ = std::fs::remove_file(&cached_path); - } - Err(e) => { - println!( - "cargo:warning=Failed to read cache {}, re-downloading: {e}", - cached_path.display() - ); - } - } - } - } - - println!("cargo:warning=Downloading {url}"); - let data = download_with_retry(url); - let actual_hash = hex_sha256(&data); - if actual_hash != expected_hash { - panic!( - "Archive integrity check failed for {url}!\n expected: {expected_hash}\n actual: {actual_hash}\n \ - This could indicate a corrupted download or a supply-chain attack." - ); - } - - if let Some(dir) = cache_dir { - if let Err(e) = std::fs::create_dir_all(dir) { - println!( - "cargo:warning=Failed to create cache directory {}: {e}", - dir.display() - ); - } else { - let cached_path = dir.join(cache_key); - println!("cargo:warning=Caching archive at {}", cached_path.display()); - if let Err(e) = std::fs::write(&cached_path, &data) { - println!( - "cargo:warning=Failed to write cache file {}: {e}", - cached_path.display() - ); - } - } - } - - data -} - -/// Maximum number of HTTP attempts (one initial + this many retries on transient errors). -const MAX_RETRIES: u32 = 3; - -/// Download `url` with bounded retries on transient network errors. Backoff is -/// exponential starting at 1s. 4xx responses fail fast; 5xx and connect/read -/// errors are retried. -fn download_with_retry(url: &str) -> Vec { - let mut attempt = 0u32; - loop { - attempt += 1; - match try_download(url) { - Ok(bytes) => return bytes, - Err(err) if err.transient && attempt <= MAX_RETRIES => { - let backoff = Duration::from_secs(1u64 << (attempt - 1)); - println!( - "cargo:warning=Transient download failure for {url} (attempt {attempt}/{}): {} — retrying in {}s", - MAX_RETRIES + 1, - err.message, - backoff.as_secs(), - ); - std::thread::sleep(backoff); - } - Err(err) => panic!("Failed to download {url}: {}", err.message), - } - } -} - -struct DownloadError { - message: String, - transient: bool, -} - -fn try_download(url: &str) -> Result, DownloadError> { - let connector = native_tls::TlsConnector::new().map_err(|e| DownloadError { - message: format!("native-tls init error: {e}"), - transient: false, - })?; - let agent = ureq::AgentBuilder::new() - .tls_connector(std::sync::Arc::new(connector)) - .timeout_connect(Duration::from_secs(30)) - .timeout_read(Duration::from_secs(120)) - .build(); - - match agent.get(url).call() { - Ok(response) => { - let mut bytes = Vec::new(); - response - .into_reader() - .read_to_end(&mut bytes) - .map_err(|e| DownloadError { - message: format!("read error: {e}"), - transient: true, - })?; - Ok(bytes) - } - // 5xx — server-side, treat as transient. - Err(ureq::Error::Status(code, response)) if (500..600).contains(&code) => { - Err(DownloadError { - message: format!("HTTP {code} {}", response.status_text()), - transient: true, - }) - } - // 4xx — client-side, fail fast. - Err(ureq::Error::Status(code, response)) => Err(DownloadError { - message: format!("HTTP {code} {}", response.status_text()), - transient: false, - }), - // Transport-layer (DNS, connect, TLS, read timeout) — treat as transient. - Err(ureq::Error::Transport(t)) => Err(DownloadError { - message: format!("transport error: {t}"), - transient: true, - }), - } -} - -fn sha256(data: &[u8]) -> [u8; 32] { - let mut hasher = sha2::Sha256::new(); - hasher.update(data); - hasher.finalize().into() -} - -/// Walks the downloaded archive at build time to confirm an entry matching -/// `binary_name` exists. Panics with a clear message if not — defends against -/// silent breakage if the upstream archive layout ever changes. -fn verify_binary_present_in_archive(archive: &[u8], binary_name: &str, asset_name: &str) { - let found = if asset_name.ends_with(".zip") { - archive_contains_zip_entry(archive, binary_name) - } else { - archive_contains_tar_entry(archive, binary_name) - }; - if !found { - panic!( - "Copilot CLI archive `{asset_name}` does not contain an entry named `{binary_name}`. \ - The upstream archive layout may have changed; runtime extraction would fail. \ - Update `verify_binary_present_in_archive` in build.rs and the matching `extract_binary` in src/embeddedcli.rs." - ); - } -} - -fn archive_contains_tar_entry(targz: &[u8], binary_name: &str) -> bool { - let gz = flate2::read::GzDecoder::new(targz); - let mut archive = tar::Archive::new(gz); - let Ok(entries) = archive.entries() else { - return false; - }; - for entry in entries.flatten() { - let Ok(path) = entry.path() else { - continue; - }; - let name = path.to_string_lossy(); - if name == binary_name || name.ends_with(&format!("/{binary_name}")) { - return true; - } - } - false -} - -fn archive_contains_zip_entry(zip_bytes: &[u8], binary_name: &str) -> bool { - let cursor = std::io::Cursor::new(zip_bytes); - let Ok(mut archive) = zip::ZipArchive::new(cursor) else { - return false; - }; - for i in 0..archive.len() { - let Ok(entry) = archive.by_index(i) else { - continue; - }; - let name = entry.name(); - if name == binary_name || name.ends_with(&format!("/{binary_name}")) { - return true; - } - } - false -} - -fn hex_sha256(data: &[u8]) -> String { - sha256(data).iter().map(|b| format!("{b:02x}")).collect() -} diff --git a/rust/build/in_process.rs b/rust/build/runtime.rs similarity index 88% rename from rust/build/in_process.rs rename to rust/build/runtime.rs index c01e7ffc4f..66eecaa670 100644 --- a/rust/build/in_process.rs +++ b/rust/build/runtime.rs @@ -11,7 +11,7 @@ pub(crate) fn main() { println!("cargo:rerun-if-env-changed=BUNDLED_CLI_CACHE_DIR"); println!("cargo::rustc-check-cfg=cfg(has_bundled_cli)"); println!("cargo::rustc-check-cfg=cfg(has_extracted_cli)"); - println!("cargo:rerun-if-changed=cli-version-in-process.txt"); + println!("cargo:rerun-if-changed=cli-version.txt"); // Only declare the package metadata rerun when it actually exists. // Cargo treats `rerun-if-changed` for a missing path as "always rerun" @@ -19,7 +19,7 @@ pub(crate) fn main() { // `nodejs/` (vendored slots, published crates) would force build.rs // to re-run on every `cargo build` even when nothing has changed. // The package file is only the source-of-truth in this repo's - // contributor builds; everywhere else `cli-version-in-process.txt` is canonical. + // contributor builds; everywhere else `cli-version.txt` is canonical. let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is set"); let package_json = Path::new(&manifest_dir) .join("..") @@ -61,11 +61,11 @@ pub(crate) fn main() { // Resolve version and, when available locally, the release SHA-256 from // one of two sources, in order: - // 1. `cli-version-in-process.txt` snapshot at the crate root (published-crate + // 1. `cli-version.txt` snapshot at the crate root (published-crate // consumer; generated by the publish workflow from SHA256SUMS.txt). // 2. Sibling `../nodejs/package.json` plus the release SHA256SUMS.txt // (contributor build inside the github/copilot-sdk repo). - let (version, local_expected_hash) = resolve_version_and_optional_hash(platform.package_name); + let (version, local_expected_hash) = resolve_version_and_optional_hash(platform.name); // Bake the version into the crate regardless of mode. This is the // single source of truth for "what CLI version did build.rs target", @@ -76,11 +76,7 @@ pub(crate) fn main() { // `target/` reuse stays cache-coherent. println!("cargo:rustc-env=COPILOT_SDK_CLI_VERSION={version}"); - let asset_platform = platform - .package_name - .strip_prefix("copilot-") - .expect("platform package names start with copilot-"); - let archive_name = format!("github-copilot-{version}-{asset_platform}.tgz"); + let archive_name = platform.asset_name(&version); let download_url = format!( "https://github.com/github/copilot-cli/releases/download/v{version}/{archive_name}" ); @@ -94,7 +90,7 @@ pub(crate) fn main() { if std::env::var_os("CARGO_FEATURE_BUNDLED_CLI").is_some() { let expected_hash = local_expected_hash .clone() - .unwrap_or_else(|| fetch_in_process_release_hash(&version, platform.package_name)); + .unwrap_or_else(|| fetch_release_hash(&version, &archive_name)); let archive = cached_download(&download_url, &cache_key, &expected_hash, &cache_dir); verify_runtime_package(&archive, platform, &archive_name); emit_embedded(out, &archive, platform, include_runtime); @@ -133,8 +129,8 @@ pub(crate) fn main() { .is_some_and(|contents| marker_matches_version(contents, &version)), }; if !cache_is_current { - let expected_hash = local_expected_hash - .unwrap_or_else(|| fetch_in_process_release_hash(&version, platform.package_name)); + let expected_hash = + local_expected_hash.unwrap_or_else(|| fetch_release_hash(&version, &archive_name)); let expected_marker = format!("{version}\n{expected_hash}\n"); if install_dir.exists() { std::fs::remove_dir_all(&install_dir).unwrap_or_else(|e| { @@ -202,12 +198,16 @@ pub(super) static CLI_ARCHIVE: &[u8] = include_bytes!("copilot_cli.archive"); std::fs::write(out.join("bundled_cli.rs"), generated).expect("failed to write bundled_cli.rs"); } -fn build_embedded_archive(package: &[u8], platform: Platform, include_runtime: bool) -> Vec { +fn build_embedded_archive( + release_archive: &[u8], + platform: Platform, + include_runtime: bool, +) -> Vec { let encoder = flate2::GzBuilder::new() .mtime(0) .write(Vec::new(), flate2::Compression::default()); let mut archive = tar::Builder::new(encoder); - let (runtime, wrapper) = append_hostless_runtime_tree(&mut archive, package, platform); + let (runtime, wrapper) = append_hostless_runtime_tree(&mut archive, release_archive, platform); append_archive_file(&mut archive, platform.binary_name, &wrapper, 0o755); if include_runtime { append_archive_file( @@ -227,24 +227,25 @@ fn build_embedded_archive(package: &[u8], platform: Platform, include_runtime: b fn append_hostless_runtime_tree( archive: &mut tar::Builder, - package: &[u8], + release_archive: &[u8], platform: Platform, ) -> (Vec, Vec) { - let decoder = flate2::read::GzDecoder::new(package); + let decoder = flate2::read::GzDecoder::new(release_archive); let mut source = tar::Archive::new(decoder); let mut runtime = None; let mut wrapper = None; for entry in source .entries() - .unwrap_or_else(|e| panic!("failed to read npm package entries: {e}")) + .unwrap_or_else(|e| panic!("failed to read release archive entries: {e}")) { - let mut entry = entry.unwrap_or_else(|e| panic!("failed to read npm package entry: {e}")); + let mut entry = + entry.unwrap_or_else(|e| panic!("failed to read release archive entry: {e}")); if !entry.header().entry_type().is_file() { continue; } let source_path = entry .path() - .unwrap_or_else(|e| panic!("failed to read npm package path: {e}")); + .unwrap_or_else(|e| panic!("failed to read release archive path: {e}")); let Some(destination) = hostless_runtime_path(&source_path.to_string_lossy(), platform) else { continue; @@ -252,7 +253,7 @@ fn append_hostless_runtime_tree( let mut bytes = Vec::with_capacity(entry.size() as usize); entry .read_to_end(&mut bytes) - .unwrap_or_else(|e| panic!("failed to read npm package entry bytes: {e}")); + .unwrap_or_else(|e| panic!("failed to read release archive entry bytes: {e}")); let mode = entry.header().mode().unwrap_or(0o644); if destination == Path::new("runtime.node") { runtime = Some(bytes.clone()); @@ -264,7 +265,7 @@ fn append_hostless_runtime_tree( archive, destination .to_str() - .expect("npm package paths are valid UTF-8"), + .expect("release archive paths are valid UTF-8"), &bytes, mode, ); @@ -272,14 +273,14 @@ fn append_hostless_runtime_tree( ( runtime.unwrap_or_else(|| { panic!( - "package `{}` does not contain prebuilds//runtime.node", - platform.package_name + "release archive for `{}` does not contain prebuilds//runtime.node", + platform.name ) }), wrapper.unwrap_or_else(|| { panic!( - "package `{}` does not contain prebuilds//{}", - platform.package_name, + "release archive for `{}` does not contain prebuilds//{}", + platform.name, platform.runtime_wrapper_name() ) }), @@ -311,6 +312,7 @@ fn hostless_runtime_path(source: &str, platform: Platform) -> Option { "webview", ]; if EXCLUDED_TOP_LEVEL.contains(&top_level) + || top_level == platform.binary_name || (top_level.starts_with("tree-sitter") && top_level.ends_with(".wasm")) || (top_level.starts_with("voice-") && top_level.ends_with(".js")) || file_name == "cli-native.node" @@ -320,11 +322,7 @@ fn hostless_runtime_path(source: &str, platform: Platform) -> Option { return None; } if top_level == "prebuilds" { - let npm_platform = platform - .package_name - .strip_prefix("copilot-") - .expect("platform package name has copilot- prefix"); - if parts.get(1) != Some(&npm_platform) || parts.len() < 3 { + if parts.get(1) != Some(&platform.name) || parts.len() < 3 { return None; } return Some(parts[2..].iter().copied().collect()); @@ -351,18 +349,18 @@ fn append_archive_file( } /// Resolve the CLI version and any locally snapshotted release hash for the -/// current target's platform package. Contributor builds defer fetching the +/// current target's release asset. Contributor builds defer fetching the /// checksum until a download is actually required. -fn resolve_version_and_optional_hash(package_name: &str) -> (String, Option) { +fn resolve_version_and_optional_hash(platform_name: &str) -> (String, Option) { let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is set"); // 1. Snapshot file at the crate root (published-crate consumer, // vendored-slot consumer). Combined version + per-asset hashes. - let snapshot = Path::new(&manifest_dir).join("cli-version-in-process.txt"); + let snapshot = Path::new(&manifest_dir).join("cli-version.txt"); if snapshot.is_file() { let contents = std::fs::read_to_string(&snapshot) .unwrap_or_else(|e| panic!("failed to read {}: {e}", snapshot.display())); - let (version, hash) = parse_snapshot(&contents, package_name) + let (version, hash) = parse_snapshot(&contents, platform_name) .unwrap_or_else(|e| panic!("invalid {}: {e}", snapshot.display())); return (version, Some(hash)); } @@ -382,21 +380,13 @@ fn resolve_version_and_optional_hash(package_name: &str) -> (String, Option String { - let platform = package_name - .strip_prefix("copilot-") - .expect("platform package names start with copilot-"); - let asset_name = format!("github-copilot-{version}-{platform}.tgz"); - fetch_release_hash(version, &asset_name) -} - fn marker_matches_version(contents: &str, version: &str) -> bool { let mut lines = contents.lines(); lines.next() == Some(version) @@ -406,13 +396,12 @@ fn marker_matches_version(contents: &str, version: &str) -> bool { && lines.next().is_none() } -/// Parse the `cli-version-in-process.txt` snapshot file. Format is one `key=value` per +/// Parse the `cli-version.txt` snapshot file. Format is one `key=value` per /// line. The first non-comment line is `version=X.Y.Z`; subsequent lines map -/// platform package name to SHA-256. Blank lines and lines starting with `#` -/// are skipped. -fn parse_snapshot(contents: &str, package_name: &str) -> Result<(String, String), String> { +/// release asset names to SHA-256. Blank lines and comments are skipped. +fn parse_snapshot(contents: &str, platform_name: &str) -> Result<(String, String), String> { let mut version: Option = None; - let mut hash: Option = None; + let mut hashes = Vec::new(); for (line_no, raw) in contents.lines().enumerate() { let line = raw.trim(); if line.is_empty() || line.starts_with('#') { @@ -426,12 +415,15 @@ fn parse_snapshot(contents: &str, package_name: &str) -> Result<(String, String) }; match key.trim() { "version" => version = Some(value.trim().to_string()), - k if k == package_name => hash = Some(value.trim().to_string()), - _ => {} + asset_name => hashes.push((asset_name, value.trim())), } } let version = version.ok_or("missing `version=` line")?; - let hash = hash.ok_or_else(|| format!("missing hash for package `{package_name}`"))?; + let asset_name = format!("github-copilot-{version}-{platform_name}.tgz"); + let hash = hashes + .into_iter() + .find_map(|(name, hash)| (name == asset_name).then(|| hash.to_string())) + .ok_or_else(|| format!("missing hash for asset `{asset_name}`"))?; Ok((version, hash)) } @@ -466,13 +458,17 @@ fn find_sha256_for_asset(sums: &str, asset_name: &str) -> String { #[derive(Clone, Copy)] struct Platform { - package_name: &'static str, + name: &'static str, binary_name: &'static str, } impl Platform { + fn asset_name(&self, version: &str) -> String { + format!("github-copilot-{version}-{}.tgz", self.name) + } + fn runtime_wrapper_name(&self) -> &'static str { - if self.package_name.contains("win32") { + if self.name.starts_with("win32") { "copilot-runtime.exe" } else { "copilot-runtime" @@ -480,9 +476,9 @@ impl Platform { } fn runtime_library_name(&self) -> &'static str { - if self.package_name.contains("win32") { + if self.name.starts_with("win32") { "copilot_runtime.dll" - } else if self.package_name.contains("darwin") { + } else if self.name.starts_with("darwin") { "libcopilot_runtime.dylib" } else { "libcopilot_runtime.so" @@ -497,35 +493,35 @@ fn target_platform() -> Option { match (os.as_str(), arch.as_str(), target_env.as_str()) { ("macos", "aarch64", _) => Some(Platform { - package_name: "copilot-darwin-arm64", + name: "darwin-arm64", binary_name: "copilot", }), ("macos", "x86_64", _) => Some(Platform { - package_name: "copilot-darwin-x64", + name: "darwin-x64", binary_name: "copilot", }), ("linux", "x86_64", "musl") => Some(Platform { - package_name: "copilot-linuxmusl-x64", + name: "linuxmusl-x64", binary_name: "copilot", }), ("linux", "aarch64", "musl") => Some(Platform { - package_name: "copilot-linuxmusl-arm64", + name: "linuxmusl-arm64", binary_name: "copilot", }), ("linux", "x86_64", _) => Some(Platform { - package_name: "copilot-linux-x64", + name: "linux-x64", binary_name: "copilot", }), ("linux", "aarch64", _) => Some(Platform { - package_name: "copilot-linux-arm64", + name: "linux-arm64", binary_name: "copilot", }), ("windows", "x86_64", _) => Some(Platform { - package_name: "copilot-win32-x64", + name: "win32-x64", binary_name: "copilot.exe", }), ("windows", "aarch64", _) => Some(Platform { - package_name: "copilot-win32-arm64", + name: "win32-arm64", binary_name: "copilot.exe", }), _ => None, @@ -561,15 +557,16 @@ fn extract_to_cache( let mut runtime = None; for entry in source .entries() - .unwrap_or_else(|e| panic!("failed to read npm package entries: {e}")) + .unwrap_or_else(|e| panic!("failed to read release archive entries: {e}")) { - let mut entry = entry.unwrap_or_else(|e| panic!("failed to read npm package entry: {e}")); + let mut entry = + entry.unwrap_or_else(|e| panic!("failed to read release archive entry: {e}")); if !entry.header().entry_type().is_file() { continue; } let source_path = entry .path() - .unwrap_or_else(|e| panic!("failed to read npm package path: {e}")); + .unwrap_or_else(|e| panic!("failed to read release archive path: {e}")); let Some(destination) = hostless_runtime_path(&source_path.to_string_lossy(), platform) else { continue; @@ -580,7 +577,7 @@ fn extract_to_cache( let mut bytes = Vec::with_capacity(entry.size() as usize); entry .read_to_end(&mut bytes) - .unwrap_or_else(|e| panic!("failed to read npm package entry bytes: {e}")); + .unwrap_or_else(|e| panic!("failed to read release archive entry bytes: {e}")); let executable = entry.header().mode().unwrap_or(0o644) & 0o111 != 0; if destination == Path::new("runtime.node") { runtime = Some(bytes.clone()); @@ -637,7 +634,7 @@ fn install_cached_file_path( | std::path::Component::ParentDir ) }), - "unsafe runtime package path: {}", + "unsafe runtime archive path: {}", relative_path.display() ); let final_path = install_dir.join(relative_path); @@ -878,13 +875,13 @@ fn try_download(url: &str) -> Result, DownloadError> { } } -fn verify_runtime_package(archive: &[u8], platform: Platform, package_name: &str) { +fn verify_runtime_package(archive: &[u8], platform: Platform, asset_name: &str) { for file_name in ["runtime.node", platform.runtime_wrapper_name()] { if archive_contains_tar_entry(archive, file_name) { continue; } panic!( - "Copilot runtime package `{package_name}` does not contain an entry named `{file_name}`" + "Copilot runtime archive `{asset_name}` does not contain an entry named `{file_name}`" ); } } diff --git a/rust/scripts/snapshot-bundled-cli-version.sh b/rust/scripts/snapshot-bundled-cli-version.sh index 08b19ebc2b..d6c9b82edb 100755 --- a/rust/scripts/snapshot-bundled-cli-version.sh +++ b/rust/scripts/snapshot-bundled-cli-version.sh @@ -34,12 +34,14 @@ CHECKSUMS_URL="https://github.com/github/copilot-cli/releases/download/v${VERSIO SHA256SUMS="$(curl --fail --silent --show-error --location --retry 3 "${CHECKSUMS_URL}")" ASSETS=( - "copilot-darwin-arm64.tar.gz" - "copilot-darwin-x64.tar.gz" - "copilot-linux-arm64.tar.gz" - "copilot-linux-x64.tar.gz" - "copilot-win32-arm64.zip" - "copilot-win32-x64.zip" + "github-copilot-${VERSION}-darwin-arm64.tgz" + "github-copilot-${VERSION}-darwin-x64.tgz" + "github-copilot-${VERSION}-linux-arm64.tgz" + "github-copilot-${VERSION}-linux-x64.tgz" + "github-copilot-${VERSION}-linuxmusl-arm64.tgz" + "github-copilot-${VERSION}-linuxmusl-x64.tgz" + "github-copilot-${VERSION}-win32-arm64.tgz" + "github-copilot-${VERSION}-win32-x64.tgz" ) TEMP_OUTPUT="${OUTPUT}.tmp.$$" diff --git a/rust/scripts/snapshot-bundled-in-process-version.sh b/rust/scripts/snapshot-bundled-in-process-version.sh deleted file mode 100755 index 9fe2298c78..0000000000 --- a/rust/scripts/snapshot-bundled-in-process-version.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash -# -# Snapshot the Copilot CLI version + per-platform release hashes for the -# rust crate's bundled-in-process build path. - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -RUST_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" -REPO_ROOT="$(cd "${RUST_DIR}/.." && pwd)" -PACKAGE_FILE="${REPO_ROOT}/nodejs/package.json" -OUTPUT="${RUST_DIR}/cli-version-in-process.txt" - -if [[ ! -f "${PACKAGE_FILE}" ]]; then - echo "error: ${PACKAGE_FILE} not found" >&2 - exit 1 -fi - -VERSION="$(node -e "console.log(require('${PACKAGE_FILE}').copilotCliVersion)")" -if [[ -z "${VERSION}" ]]; then - echo "error: could not read copilotCliVersion from ${PACKAGE_FILE}" >&2 - exit 1 -fi -CHECKSUMS_URL="https://github.com/github/copilot-cli/releases/download/v${VERSION}/SHA256SUMS.txt" -SHA256SUMS="$(curl --fail --silent --show-error --location --retry 3 "${CHECKSUMS_URL}")" - -PACKAGES=( - "copilot-darwin-arm64" - "copilot-darwin-x64" - "copilot-linux-arm64" - "copilot-linux-x64" - "copilot-linuxmusl-arm64" - "copilot-linuxmusl-x64" - "copilot-win32-arm64" - "copilot-win32-x64" -) - -TEMP_OUTPUT="${OUTPUT}.tmp.$$" -trap 'rm -f "${TEMP_OUTPUT}"' EXIT -{ - echo "# Auto-generated by rust/scripts/snapshot-bundled-in-process-version.sh" - echo "# Do not edit. Regenerated by the publish workflow on every release." - echo "version=${VERSION}" - for package in "${PACKAGES[@]}"; do - platform="${package#copilot-}" - asset="github-copilot-${VERSION}-${platform}.tgz" - hash="$(printf '%s\n' "${SHA256SUMS}" | awk -v asset="${asset}" '$2 == asset || $2 == "*" asset { print $1; exit }')" - if [[ -z "${hash}" ]]; then - echo "error: SHA256SUMS.txt does not contain ${asset}" >&2 - exit 1 - fi - echo "${package}=${hash}" - done -} > "${TEMP_OUTPUT}" -mv "${TEMP_OUTPUT}" "${OUTPUT}" -trap - EXIT - -echo "Wrote ${OUTPUT} (version=${VERSION}, ${#PACKAGES[@]} hashes)" diff --git a/rust/src/embeddedcli.rs b/rust/src/embeddedcli.rs index 3cc527a2e2..3618dca08a 100644 --- a/rust/src/embeddedcli.rs +++ b/rust/src/embeddedcli.rs @@ -2,11 +2,12 @@ //! crate (gated on the `bundled-cli` cargo feature, which is in the default //! feature set). //! -//! Normal builds embed the platform release archive from GitHub Releases. -//! Builds with `bundled-in-process` instead embed a filtered archive from the -//! platform npm package containing the CLI executable, runtime wrapper, native -//! runtime artifacts, and auxiliary runtime assets. Extraction to a real -//! on-disk path is deferred until the relevant installer is called. +//! All bundled builds embed a filtered archive derived from the platform's +//! `github-copilot--.tgz` GitHub Release asset. It contains +//! the CLI executable, runtime wrapper, and auxiliary runtime assets; enabling +//! `bundled-in-process` additionally includes the native runtime library. +//! Extraction to a real on-disk path is deferred until the relevant installer +//! is called. //! //! The embedded bytes are part of the consumer's signed binary and therefore //! trusted *as the source of truth* — but the bytes that land on disk are not. diff --git a/rust/tests/cli_resolution_test.rs b/rust/tests/cli_resolution_test.rs index 847ac7a4d0..74daeaf246 100644 --- a/rust/tests/cli_resolution_test.rs +++ b/rust/tests/cli_resolution_test.rs @@ -197,8 +197,8 @@ async fn extract_dir_runtime_override_is_honored() { let _ = fake; } -/// Build-time version pins, when present, must match the selected bundling -/// implementation's checksum format. +/// Build-time version pins, when present, must contain the release asset +/// checksum for every supported target. /// When absent, build.rs falls through to `../nodejs/package.json` and /// the release's `SHA256SUMS.txt` — /// both are accepted, this test only checks the pin file's format if it's @@ -206,19 +206,15 @@ async fn extract_dir_runtime_override_is_honored() { #[test] fn pin_file_when_present_is_well_formed() { let manifest_dir = env!("CARGO_MANIFEST_DIR"); - let (filename, expected_package_count) = if cfg!(feature = "bundled-in-process") { - ("cli-version-in-process.txt", 8) - } else { - ("cli-version.txt", 6) - }; + let filename = "cli-version.txt"; let pin = PathBuf::from(manifest_dir).join(filename); if !pin.is_file() { // Contributor build path — no assertion needed. return; } let contents = std::fs::read_to_string(&pin).expect("read CLI version snapshot"); - let mut saw_version = false; - let mut package_count = 0; + let mut version = None; + let mut assets = Vec::new(); for raw in contents.lines() { let line = raw.trim(); if line.is_empty() || line.starts_with('#') { @@ -229,7 +225,7 @@ fn pin_file_when_present_is_well_formed() { .unwrap_or_else(|| panic!("malformed line: {raw:?}")); assert!(!value.trim().is_empty(), "empty value for key {key:?}"); if key.trim() == "version" { - saw_version = true; + version = Some(value.trim().to_string()); } else { assert_eq!( value.trim().len(), @@ -240,11 +236,22 @@ fn pin_file_when_present_is_well_formed() { value.trim().bytes().all(|byte| byte.is_ascii_hexdigit()), "invalid SHA-256 hash for key {key:?}" ); - package_count += 1; + assets.push(key.trim().to_string()); } } - assert!(saw_version, "{filename} missing `version=` line"); - assert_eq!(package_count, expected_package_count); + let version = version.unwrap_or_else(|| panic!("{filename} missing `version=` line")); + let expected_assets = [ + "darwin-arm64", + "darwin-x64", + "linux-arm64", + "linux-x64", + "linuxmusl-arm64", + "linuxmusl-x64", + "win32-arm64", + "win32-x64", + ] + .map(|platform| format!("github-copilot-{version}-{platform}.tgz")); + assert_eq!(assets, expected_assets); } /// With `bundled-cli` on AND a supported target, `install_bundled_cli` From 355b9b8b15a884fafb47c99c04d2879b2e1d8090 Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Thu, 3 Sep 2026 15:31:02 +0200 Subject: [PATCH 2/6] Trim runtime migration scope Remove Java and Rust changes because those SDKs were already release-backed, simplify Python's direct runtime staging, and drop the misleading .NET npm URL alias. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: daec7b3b-799c-4396-b372-6eb638d0faf8 --- .github/workflows/publish.yml | 16 +- dotnet/src/build/GitHub.Copilot.SDK.targets | 5 +- java/README.md | 4 +- java/copilot-native/pom.xml | 11 +- java/copilot-native/scripts/fetch-native.mjs | 27 +- .../scripts/fetch-native.test.mjs | 181 +---- .../adr/adr-007-native-bundling-strategy.md | 2 +- .../copilot/ffi/NativeRuntimeLoader.java | 5 +- python/README.md | 9 +- python/copilot/_cli_download.py | 175 ++--- python/test_cli_download.py | 75 +- rust/.gitignore | 1 + rust/Cargo.lock | 82 ++- rust/Cargo.toml | 5 +- rust/README.md | 7 +- rust/build.rs | 2 +- rust/build/{runtime.rs => in_process.rs} | 137 ++-- rust/build/out_of_process.rs | 692 ++++++++++++++++++ rust/scripts/snapshot-bundled-cli-version.sh | 14 +- .../snapshot-bundled-in-process-version.sh | 58 ++ rust/src/embeddedcli.rs | 11 +- rust/tests/cli_resolution_test.rs | 33 +- 22 files changed, 1131 insertions(+), 421 deletions(-) rename rust/build/{runtime.rs => in_process.rs} (88%) create mode 100644 rust/build/out_of_process.rs create mode 100755 rust/scripts/snapshot-bundled-in-process-version.sh diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7a30ba2b19..5e1d277259 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -316,13 +316,17 @@ jobs: - name: Set version run: sed -i -E 's/^version = ".*"$/version = "${{ needs.version.outputs.version }}"/' Cargo.toml - name: Snapshot CLI version + hashes for build.rs - run: bash scripts/snapshot-bundled-cli-version.sh - - name: Verify CLI version snapshot exists run: | - if [[ ! -f cli-version.txt ]]; then - echo "::error::cli-version.txt was not generated. The Snapshot step must run before packaging." - exit 1 - fi + bash scripts/snapshot-bundled-cli-version.sh + bash scripts/snapshot-bundled-in-process-version.sh + - name: Verify CLI version snapshots exist + run: | + for snapshot in cli-version.txt cli-version-in-process.txt; do + if [[ ! -f "${snapshot}" ]]; then + echo "::error::${snapshot} was not generated. The Snapshot step must run before packaging." + exit 1 + fi + done - name: Package (dry run) run: cargo publish --dry-run --allow-dirty - name: Upload artifact diff --git a/dotnet/src/build/GitHub.Copilot.SDK.targets b/dotnet/src/build/GitHub.Copilot.SDK.targets index d5258480c6..28692ca2c9 100644 --- a/dotnet/src/build/GitHub.Copilot.SDK.targets +++ b/dotnet/src/build/GitHub.Copilot.SDK.targets @@ -55,12 +55,9 @@ https://your-mirror.example.com/copilot-cli/releases/download - COPILOT_CLI_DOWNLOAD_BASE_URL is also honored. CopilotNpmRegistryUrl remains - a compatibility alias, but its value is interpreted as a release base URL; - these targets never fall back to npm. --> + COPILOT_CLI_DOWNLOAD_BASE_URL is also honored. --> $(COPILOT_CLI_DOWNLOAD_BASE_URL) - $(CopilotNpmRegistryUrl) https://github.com/github/copilot-cli/releases/download diff --git a/java/README.md b/java/README.md index c179983ffe..bb71d7ab86 100644 --- a/java/README.md +++ b/java/README.md @@ -548,9 +548,9 @@ mvn jacoco:prepare-agent@wire-up-coverage-instrumentation antrun:run@print-test- #### Development Setup for native embedding -Run native-runtime Maven commands from the `java` directory. Native packaging requires Node.js in addition to JDK 25 and Maven because `copilot-native/scripts/fetch-native.mjs` retrieves the pinned runtime asset from the corresponding GitHub release. It downloads `github-copilot--.tgz`, verifies the asset against that release's `SHA256SUMS.txt`, and does not fall back to npm runtime packages. Set `COPILOT_CLI_DOWNLOAD_BASE_URL` to use a release mirror. +Run native-runtime Maven commands from the `java` directory. Native packaging requires Node.js in addition to JDK 25 and Maven because `copilot-native/scripts/fetch-native.mjs` retrieves the pinned runtime package from the corresponding GitHub release. -On a native Linux glibc host, Maven activates `native-linux-x64` or `native-linux-arm64` for the matching architecture when `copilot.native.libc=glibc` is set. On Windows x64, Windows ARM64, and Apple Silicon macOS, Maven activates `native-win32-x64`, `native-win32-arm64`, or `native-darwin-arm64` automatically. The matching profile validates the host, runs the native script tests, fetches the pinned platform release asset from `github/copilot-cli` during `generate-resources`, packages the classifier JAR during `package`, and verifies its native contents. +On a native Linux glibc host, Maven activates `native-linux-x64` or `native-linux-arm64` for the matching architecture when `copilot.native.libc=glibc` is set. On Windows x64, Windows ARM64, and Apple Silicon macOS, Maven activates `native-win32-x64`, `native-win32-arm64`, or `native-darwin-arm64` automatically. The matching profile validates the host, runs the native script tests, fetches the pinned platform package from the corresponding `github/copilot-cli` release during `generate-resources`, packages the classifier JAR during `package`, and verifies its native contents. Before opting in, validate that Node.js reports glibc for the build host: diff --git a/java/copilot-native/pom.xml b/java/copilot-native/pom.xml index aeaafc076b..0abe03ad18 100644 --- a/java/copilot-native/pom.xml +++ b/java/copilot-native/pom.xml @@ -59,11 +59,10 @@ org.codehaus.mojo @@ -910,7 +909,7 @@ diff --git a/java/copilot-native/scripts/fetch-native.mjs b/java/copilot-native/scripts/fetch-native.mjs index 4e355333e1..49d2180a73 100644 --- a/java/copilot-native/scripts/fetch-native.mjs +++ b/java/copilot-native/scripts/fetch-native.mjs @@ -7,10 +7,10 @@ * * Steps: * 1. Read the pinned version from `nodejs/package.json`. - * 2. Download the platform release asset and `SHA256SUMS.txt` from the matching release. + * 2. Download the platform npm tarball and `SHA256SUMS.txt` from the matching release. * 3. Verify the downloaded tarball against the release checksum. * 4. Stage the hostless runtime tree, flattening the selected prebuild directory - * beside the release archive's retained top-level runtime assets. + * beside the package's retained top-level runtime assets. * 5. Write an inventory consumed by the SDK's generic classpath extractor. * 6. Write `//native//platform.properties`. * @@ -49,18 +49,13 @@ if (!repoRoot || !stagingDir || !classifier) { } const packagePath = path.join(repoRoot, 'nodejs', 'package.json'); -let packageJson; -try { - packageJson = JSON.parse(fs.readFileSync(packagePath, 'utf8')); -} catch (error) { - throw new Error(`Could not read pinned Copilot CLI version from ${packagePath}: ${error.message}`); -} +const packageName = `@github/copilot-${classifier}`; +const packageJson = JSON.parse(fs.readFileSync(packagePath, 'utf8')); const version = packageJson.copilotCliVersion; if (!version) { console.error(`Could not find copilotCliVersion in ${packagePath}`); process.exit(1); } -const assetName = `github-copilot-${version}-${classifier}.tgz`; const outDir = path.join(stagingDir, classifier); const resourceDir = path.join(outDir, 'native', classifier); @@ -75,7 +70,7 @@ const stagingSchema = 'hostless-runtime-v3'; const stampPath = path.join(outDir, '.version'); // Idempotence: skip the download only when every required staged artifact -// matches the release identity recorded in the stamp. +// matches the package identity recorded in the stamp. if ( fs.existsSync(runtimePath) && fs.existsSync(wrapperPath) && @@ -95,7 +90,7 @@ if ( stampTreeDigest === currentTreeDigest && currentPlatformProperties === expectedPlatformProperties ) { - console.log(`${assetName} already staged at ${runtimePath}`); + console.log(`${packageName}@${version} already staged at ${runtimePath}`); process.exit(0); } } @@ -103,7 +98,8 @@ if ( fs.rmSync(outDir, { recursive: true, force: true }); fs.mkdirSync(resourceDir, { recursive: true }); -console.log(`Downloading ${assetName} ...`); +console.log(`Downloading ${packageName}@${version} ...`); +const assetName = `github-copilot-${version}-${classifier}.tgz`; const releaseBase = ( process.env.COPILOT_CLI_DOWNLOAD_BASE_URL ?? 'https://github.com/github/copilot-cli/releases/download' @@ -122,9 +118,8 @@ if (process.env.COPILOT_CLI_RELEASE_TARBALL) { if (!expectedHash || !/^[a-fA-F0-9]{64}$/.test(expectedHash)) { throw new Error(`Missing or invalid SHA-256 for ${assetName}`); } -expectedHash = expectedHash.toLowerCase(); const actual = createHash('sha256').update(archive).digest('hex'); -if (actual !== expectedHash) { +if (actual !== expectedHash.toLowerCase()) { console.error(`Integrity verification failed for ${assetName}`); console.error(` expected: ${expectedHash}`); console.error(` actual: ${actual}`); @@ -171,7 +166,7 @@ inventory.sort(); fs.writeFileSync(inventoryPath, `${inventory.join('\n')}\n`); if (!fs.existsSync(runtimePath) || !fs.existsSync(wrapperPath)) { - throw new Error(`Release asset ${assetName} is missing the runtime wrapper pair`); + throw new Error(`Package ${packageName}@${version} is missing the runtime wrapper pair`); } fs.writeFileSync(platformPropertiesPath, expectedPlatformProperties); const treeDigest = digestTree(resourceDir); @@ -236,7 +231,7 @@ async function download(url) { for (let attempt = 0; attempt < 3; attempt++) { try { // lgtm[js/file-access-to-http] The repository-pinned CLI version intentionally selects the release asset. - const response = await fetch(url, { signal: AbortSignal.timeout(60_000) }); + const response = await fetch(url); if (response.ok) { return Buffer.from(await response.arrayBuffer()); } diff --git a/java/copilot-native/scripts/fetch-native.test.mjs b/java/copilot-native/scripts/fetch-native.test.mjs index f3537d6a52..3213d416f0 100644 --- a/java/copilot-native/scripts/fetch-native.test.mjs +++ b/java/copilot-native/scripts/fetch-native.test.mjs @@ -4,12 +4,12 @@ import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; -import { execFileSync, spawn } from 'node:child_process'; import fs from 'node:fs'; -import http from 'node:http'; +import os from 'node:os'; import path from 'node:path'; +import { execFileSync, spawnSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; -import test, { after, before } from 'node:test'; +import test from 'node:test'; const version = '1.0.79'; const checksum = '0'.repeat(64); @@ -17,94 +17,65 @@ const runtimeContent = 'runtime content'; const wrapperContent = 'wrapper content'; const stagingSchema = 'hostless-runtime-v3'; const scriptPath = fileURLToPath(new URL('./fetch-native.mjs', import.meta.url)); -const testRoot = fileURLToPath(new URL('../target/fetch-native-tests/', import.meta.url)); -const releaseFiles = new Map(); -const releaseRequests = []; -let releaseBase; -let releaseServer; - -before(async () => { - releaseServer = http.createServer((request, response) => { - releaseRequests.push(request.url); - const content = releaseFiles.get(request.url); - if (content === undefined) { - response.writeHead(404).end(); - } else { - response.writeHead(200, { 'Content-Length': content.length }).end(content); - } - }); - await new Promise((resolve, reject) => { - releaseServer.once('error', reject); - releaseServer.listen(0, '127.0.0.1', resolve); - }); - const address = releaseServer.address(); - releaseBase = `http://127.0.0.1:${address.port}`; -}); - -after(async () => { - await new Promise((resolve, reject) => { - releaseServer.close((error) => (error ? reject(error) : resolve())); - }); -}); for (const classifier of ['linux-x64', 'linux-arm64', 'win32-x64', 'win32-arm64', 'darwin-arm64']) { - test(`${classifier}: complete hostless artifacts use incremental fast path without a CLI`, async (t) => { + test(`${classifier}: complete hostless artifacts use incremental fast path without a CLI`, (t) => { const fixture = createFixture(t, classifier); - const result = await runScript(fixture); + const result = runScript(fixture); assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /already staged/); }); - test(`${classifier}: missing runtime wrapper does not use incremental fast path`, async (t) => { + test(`${classifier}: missing runtime wrapper does not use incremental fast path`, (t) => { const fixture = createFixture(t, classifier); fs.rmSync(fixture.wrapperPath); - const result = await runScript(fixture); + const result = runScript(fixture); assertRestagingAttempted(fixture, result); }); - test(`${classifier}: v2 staging schema does not use incremental fast path`, async (t) => { + test(`${classifier}: v2 staging schema does not use incremental fast path`, (t) => { const fixture = createFixture(t, classifier); const stampPath = path.join(fixture.stagingDir, classifier, '.version'); fs.writeFileSync(stampPath, fs.readFileSync(stampPath, 'utf8').replace(stagingSchema, 'hostless-runtime-v2')); - const result = await runScript(fixture); + const result = runScript(fixture); assertRestagingAttempted(fixture, result); }); - test(`${classifier}: missing platform metadata does not use incremental fast path`, async (t) => { + test(`${classifier}: missing platform metadata does not use incremental fast path`, (t) => { const fixture = createFixture(t, classifier); fs.rmSync(fixture.platformPropertiesPath); - const result = await runScript(fixture); + const result = runScript(fixture); assertRestagingAttempted(fixture, result); }); - test(`${classifier}: missing retained runtime asset does not use incremental fast path`, async (t) => { + test(`${classifier}: missing retained runtime asset does not use incremental fast path`, (t) => { const fixture = createFixture(t, classifier); fs.rmSync(fixture.ripgrepPath); - const result = await runScript(fixture); + const result = runScript(fixture); assertRestagingAttempted(fixture, result); }); - test(`${classifier}: complete matching artifacts use incremental fast path`, async (t) => { + test(`${classifier}: complete matching artifacts use incremental fast path`, (t) => { const fixture = createFixture(t, classifier); - const result = await runScript(fixture); + const result = runScript(fixture); assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /already staged/); }); } -test('downloads the exact release asset, verifies its manifest checksum, and stages runtime assets', async (t) => { +test('stages retained package assets and excludes CLI-only content', (t) => { const classifier = 'linux-x64'; const fixture = createFixture(t, classifier); const packageRoot = path.join(fixture.repoRoot, 'package-root', 'package'); @@ -114,7 +85,6 @@ test('downloads the exact release asset, verifies its manifest checksum, and sta fs.writeFileSync(path.join(packageRoot, 'copilot'), 'excluded'); fs.writeFileSync(path.join(packageRoot, 'prebuilds', classifier, 'runtime.node'), runtimeContent); fs.writeFileSync(path.join(packageRoot, 'prebuilds', classifier, 'copilot-runtime'), wrapperContent); - fs.chmodSync(path.join(packageRoot, 'prebuilds', classifier, 'copilot-runtime'), 0o755); fs.writeFileSync(path.join(packageRoot, 'ripgrep', 'bin', classifier, 'rg'), 'ripgrep content'); fs.chmodSync(path.join(packageRoot, 'ripgrep', 'bin', classifier, 'rg'), 0o755); fs.writeFileSync(path.join(packageRoot, 'definitions', 'future.json'), '{}'); @@ -130,23 +100,12 @@ test('downloads the exact release asset, verifies its manifest checksum, and sta ); fs.rmSync(path.join(fixture.stagingDir, classifier), { recursive: true, force: true }); - const assetName = `github-copilot-${version}-${classifier}.tgz`; - const releasePrefix = `/v${version}`; - releaseFiles.set(`${releasePrefix}/SHA256SUMS.txt`, Buffer.from(`${packageChecksum} ${assetName}\n`)); - releaseFiles.set(`${releasePrefix}/${assetName}`, fs.readFileSync(tarball)); - const requestOffset = releaseRequests.length; - t.after(() => { - releaseFiles.delete(`${releasePrefix}/SHA256SUMS.txt`); - releaseFiles.delete(`${releasePrefix}/${assetName}`); + const result = runScript(fixture, { + COPILOT_CLI_RELEASE_TARBALL: tarball, + COPILOT_CLI_RELEASE_SHA256: packageChecksum, }); - const result = await runScript(fixture); - assert.equal(result.status, 0, result.stderr); - assert.deepEqual(releaseRequests.slice(requestOffset), [ - `${releasePrefix}/SHA256SUMS.txt`, - `${releasePrefix}/${assetName}`, - ]); const resourceDir = path.join(fixture.stagingDir, classifier, 'native', classifier); assert.equal(fs.readFileSync(path.join(resourceDir, 'ripgrep', 'bin', classifier, 'rg'), 'utf8'), 'ripgrep content'); assert.equal(fs.readFileSync(path.join(resourceDir, 'definitions', 'future.json'), 'utf8'), '{}'); @@ -154,79 +113,11 @@ test('downloads the exact release asset, verifies its manifest checksum, and sta assert.equal(fs.existsSync(path.join(resourceDir, 'copilot')), false); assert.equal(fs.existsSync(path.join(resourceDir, 'LICENSE.md')), false); assert.equal(fs.existsSync(path.join(resourceDir, 'README.md')), false); - const inventory = fs.readFileSync(path.join(resourceDir, 'runtime-assets.list'), 'utf8'); - assert.match(inventory, /^644\truntime\.node$/m); - assert.match(inventory, /^755\tcopilot-runtime$/m); - assert.match(inventory, /^755\tripgrep\/bin\/linux-x64\/rg$/m); -}); - -test('stages a pre-downloaded release asset with its supplied SHA-256 without contacting a release server', async (t) => { - const classifier = 'linux-x64'; - const fixture = createFixture(t, classifier); - const packageRoot = path.join(fixture.repoRoot, 'local-package', 'package'); - const prebuildRoot = path.join(packageRoot, 'prebuilds', classifier); - fs.mkdirSync(prebuildRoot, { recursive: true }); - fs.writeFileSync(path.join(prebuildRoot, 'runtime.node'), runtimeContent); - fs.writeFileSync(path.join(prebuildRoot, 'copilot-runtime'), wrapperContent); - fs.chmodSync(path.join(prebuildRoot, 'copilot-runtime'), 0o755); - const tarball = path.join(fixture.repoRoot, 'pre-downloaded-release.tgz'); - execFileSync('tar', ['-czf', tarball, '-C', path.dirname(packageRoot), 'package']); - const expectedHash = createHash('sha256').update(fs.readFileSync(tarball)).digest('hex'); - fs.rmSync(path.join(fixture.stagingDir, classifier), { recursive: true, force: true }); - const requestOffset = releaseRequests.length; - - const result = await runScript(fixture, { - COPILOT_CLI_RELEASE_TARBALL: tarball, - COPILOT_CLI_RELEASE_SHA256: expectedHash, - }); - - assert.equal(result.status, 0, result.stderr); - assert.deepEqual(releaseRequests.slice(requestOffset), []); - const resourceDir = path.join(fixture.stagingDir, classifier, 'native', classifier); - assert.equal(fs.readFileSync(path.join(resourceDir, 'runtime.node'), 'utf8'), runtimeContent); - assert.equal(fs.readFileSync(path.join(resourceDir, 'copilot-runtime'), 'utf8'), wrapperContent); -}); - -test('rejects a release asset that does not match SHA256SUMS.txt without npm fallback', async (t) => { - const classifier = 'linux-x64'; - const fixture = createFixture(t, classifier); - const assetName = `github-copilot-${version}-${classifier}.tgz`; - const releasePrefix = `/v${version}`; - releaseFiles.set(`${releasePrefix}/SHA256SUMS.txt`, Buffer.from(`${checksum} ${assetName}\n`)); - releaseFiles.set(`${releasePrefix}/${assetName}`, Buffer.from('not the release archive')); - t.after(() => { - releaseFiles.delete(`${releasePrefix}/SHA256SUMS.txt`); - releaseFiles.delete(`${releasePrefix}/${assetName}`); - }); - fs.rmSync(path.join(fixture.stagingDir, classifier), { recursive: true, force: true }); - - const result = await runScript(fixture); - - assert.notEqual(result.status, 0); - assert.match(result.stderr, /Integrity verification failed/); - assert.doesNotMatch(result.stderr, /npm pack/); -}); - -test('fails when SHA256SUMS.txt does not list the exact release asset', async (t) => { - const classifier = 'linux-x64'; - const fixture = createFixture(t, classifier); - const releasePrefix = `/v${version}`; - releaseFiles.set(`${releasePrefix}/SHA256SUMS.txt`, Buffer.from(`${checksum} another-asset.tgz\n`)); - t.after(() => releaseFiles.delete(`${releasePrefix}/SHA256SUMS.txt`)); - fs.rmSync(path.join(fixture.stagingDir, classifier), { recursive: true, force: true }); - - const result = await runScript(fixture); - - assert.notEqual(result.status, 0); - assert.match( - result.stderr, - new RegExp(`SHA256SUMS\\.txt does not contain github-copilot-${version}-${classifier}\\.tgz`), - ); + assert.match(fs.readFileSync(path.join(resourceDir, 'runtime-assets.list'), 'utf8'), /ripgrep\/bin\/linux-x64\/rg/); }); function createFixture(t, classifier) { - fs.mkdirSync(testRoot, { recursive: true }); - const root = fs.mkdtempSync(path.join(testRoot, 'fixture-')); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'fetch-native-test-')); t.after(() => fs.rmSync(root, { recursive: true, force: true })); const repoRoot = path.join(root, 'repo'); @@ -275,28 +166,14 @@ function createFixture(t, classifier) { } function runScript(fixture, extraEnv = {}) { - return new Promise((resolve, reject) => { - const child = spawn(process.execPath, [scriptPath, fixture.repoRoot, fixture.stagingDir, fixture.classifier], { - env: { - ...process.env, - COPILOT_CLI_DOWNLOAD_BASE_URL: releaseBase, - ...extraEnv, - }, - }); - let stdout = ''; - let stderr = ''; - child.stdout.setEncoding('utf8'); - child.stderr.setEncoding('utf8'); - child.stdout.on('data', (chunk) => { - stdout += chunk; - }); - child.stderr.on('data', (chunk) => { - stderr += chunk; - }); - child.once('error', reject); - child.once('close', (status, signal) => { - resolve({ status, signal, stdout, stderr }); - }); + return spawnSync(process.execPath, [scriptPath, fixture.repoRoot, fixture.stagingDir, fixture.classifier], { + encoding: 'utf8', + env: { + ...process.env, + COPILOT_CLI_RELEASE_TARBALL: path.join(fixture.root, 'missing.tgz'), + COPILOT_CLI_RELEASE_SHA256: checksum, + ...extraEnv, + }, }); } diff --git a/java/docs/adr/adr-007-native-bundling-strategy.md b/java/docs/adr/adr-007-native-bundling-strategy.md index e5d5c85836..3c13d451cf 100644 --- a/java/docs/adr/adr-007-native-bundling-strategy.md +++ b/java/docs/adr/adr-007-native-bundling-strategy.md @@ -361,7 +361,7 @@ The pattern follows DJL's `LibUtils.loadLibrary()` approach: detect the platform 2. Locates the matching `runtime.node` binary on the classpath (via `getResourceAsStream` from the classifier JAR). 3. Extracts `runtime.node` and the transitional CLI entrypoint into `~/.copilot/runtime-cache/` if valid cached files are not already present. 4. Loads it via [JNA](#references) using the C ABI entry points, per the [binding technology decision](#binding-technology-jna-over-panama-ffm) above. The JNA-specific code is confined behind an internal binding interface to preserve a future FFM migration path. -* A validated supported-host profile fetches `github-copilot--.tgz` from the pinned `github/copilot-cli` release, verifies it against the release's `SHA256SUMS.txt`, and packages the version-matched runtime files without an npm runtime-package fallback. +* A validated supported-host profile fetches the matching platform tarball from the pinned `github/copilot-cli` release, verifies its release SHA-256, and packages the version-matched runtime files. * The current release work publishes the `linux-x64`, `linux-arm64`, `win32-x64`, `win32-arm64`, and `darwin-arm64` classifiers. The planned classifier set expands to the other detected platforms. * Adding an implemented platform requires validated host activation, a profile that supplies the classifier and platform CLI filename, and lifecycle bindings for the shared host validation, fetch, script test, package, and verification executions. * `cli-native.node` is not bundled. It provides terminal UI features that are irrelevant to the Java SDK's programmatic API surface. diff --git a/java/sdk/src/main/java/com/github/copilot/ffi/NativeRuntimeLoader.java b/java/sdk/src/main/java/com/github/copilot/ffi/NativeRuntimeLoader.java index dd5f91cf28..bd4b185a07 100644 --- a/java/sdk/src/main/java/com/github/copilot/ffi/NativeRuntimeLoader.java +++ b/java/sdk/src/main/java/com/github/copilot/ffi/NativeRuntimeLoader.java @@ -318,8 +318,9 @@ static Path resolve(String cliPathEnv, Path cacheBase, ClassLoader loader, Strin * *

* Checks, in order, the flat bundled layout ({@code runtime.node} directly next - * to the CLI) and the release-package layout - * ({@code prebuilds//runtime.node} next to the CLI). + * to the CLI) and the npm package layout + * ({@code prebuilds//runtime.node} next to the CLI), matching the + * two layouts the {@code @github/copilot-} packages may ship. */ static Path resolveFromCliPath(String cliPathStr) throws IOException { if (cliPathStr == null || cliPathStr.isBlank()) { diff --git a/python/README.md b/python/README.md index c857145b96..c45847806f 100644 --- a/python/README.md +++ b/python/README.md @@ -30,9 +30,10 @@ python -m copilot download-runtime ``` This downloads the platform release package, verifies it against the release's -`SHA256SUMS.txt`, and caches `copilot-runtime`, its adjacent `runtime.node`, and -the hostless runtime assets locally. If you skip this step, the SDK downloads the -same package automatically on first managed stdio/TCP use. +`SHA256SUMS.txt`, and directly stages `copilot-runtime`, its adjacent `runtime.node`, +and the filtered hostless runtime assets locally without retaining the downloaded +archive. If you skip this step, the SDK performs the same staging automatically on +first managed stdio/TCP use. To pre-provision the native library required by the in-process (FFI) transport (see [In-process (FFI) transport](#in-process-ffi-transport)), pass `--in-process`: @@ -43,7 +44,7 @@ python -m copilot download-runtime --in-process This also creates a `copilot` compatibility entrypoint from `copilot-runtime` inside the complete materialized bundle. Its adjacent `runtime.node` can then be -used for in-process hosting. The cached release package is reused, so this does +used for in-process hosting. That canonical staged library is reused, so this does not download a second runtime artifact. | Platform | Cache path | diff --git a/python/copilot/_cli_download.py b/python/copilot/_cli_download.py index e485a923c5..235667aa56 100644 --- a/python/copilot/_cli_download.py +++ b/python/copilot/_cli_download.py @@ -2,10 +2,10 @@ The platform-specific GitHub release package contains the out-of-process runtime wrapper, native runtime library, and runtime assets, but omits the legacy SEA -``copilot[.exe]``. It is downloaded and verified once, then materialized into the -SDK's existing cache layout. ``download_cli`` preserves the historical CLI filename -by creating a compatibility alias from the runtime wrapper inside the complete -materialized bundle: +``copilot[.exe]``. Its bytes are downloaded and verified, then the filtered hostless +bundle is materialized directly into the SDK's existing cache layout. ``download_cli`` +preserves the historical CLI filename by creating a compatibility alias from the +runtime wrapper inside the complete materialized bundle: - Linux: ~/.cache/github-copilot-sdk/cli/{version}/prebuilds/{platform}/copilot - macOS: ~/Library/Caches/github-copilot-sdk/cli/{version}/prebuilds/{platform}/copilot @@ -161,60 +161,8 @@ def _verify_checksum(data: bytes, expected_hash: str, filename: str) -> None: ) -def _validate_file(path: Path, label: str) -> None: - if not path.is_file() or path.stat().st_size == 0: - raise RuntimeError(f"{label} not found or empty at {path}.") - - -def _extract_release_package(data: bytes, destination: Path) -> None: - """Safely extract the npm-style ``package/`` tree from a release tarball.""" - with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as archive: - for member in archive: - parts = PurePosixPath(member.name).parts - if len(parts) < 2 or parts[0] != "package": - continue - relative = Path(*parts[1:]) - if relative.is_absolute() or ".." in relative.parts: - raise RuntimeError(f"Unsafe release package path: {member.name}") - target = destination / relative - if member.isdir(): - target.mkdir(parents=True, exist_ok=True) - continue - if not member.isfile(): - raise RuntimeError(f"Unsupported release package entry: {member.name}") - extracted = archive.extractfile(member) - if extracted is None: - raise RuntimeError(f"Failed to read release package entry: {member.name}") - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(extracted.read()) - if sys.platform != "win32": - target.chmod(member.mode & 0o777) - - -def _release_package_dir(version: str, runtime_platform: str) -> Path: - return get_cache_dir(version) / "packages" / runtime_platform - - -def _validate_release_package(package_dir: Path, runtime_platform: str) -> None: - prebuilds = package_dir / "prebuilds" / runtime_platform - wrapper_name = "copilot-runtime.exe" if sys.platform == "win32" else "copilot-runtime" - _validate_file(prebuilds / wrapper_name, "Copilot runtime wrapper") - _validate_file(prebuilds / "runtime.node", "Copilot runtime.node") - - -def _ensure_release_package(version: str, *, force: bool = False) -> Path: - """Download, verify, and cache the unified platform release package.""" - runtime_platform = get_runtime_platform() - package_dir = _release_package_dir(version, runtime_platform) - if package_dir.exists() and not force: - _validate_release_package(package_dir, runtime_platform) - return package_dir - if _should_skip_download(): - raise RuntimeError( - f"Copilot runtime release package is not cached in {package_dir} " - "and automatic downloads are disabled." - ) - +def _fetch_verified_release_package(version: str, runtime_platform: str) -> bytes: + """Download and verify the unified platform release package.""" asset_name = get_release_asset_name(version, runtime_platform) expected_hash = _fetch_checksums(version).get(asset_name) if not expected_hash: @@ -222,27 +170,16 @@ def _ensure_release_package(version: str, *, force: bool = False) -> Path: url = get_download_url(version, asset_name) data = _fetch_url_bytes(url, timeout=600) _verify_checksum(data, expected_hash, asset_name) + return data - import shutil - package_dir.parent.mkdir(parents=True, exist_ok=True) - staging_dir = Path(tempfile.mkdtemp(dir=package_dir.parent, prefix=".release-package-")) - staged_package = staging_dir / "package" - try: - staged_package.mkdir() - _extract_release_package(data, staged_package) - _validate_release_package(staged_package, runtime_platform) - if force and package_dir.exists(): - shutil.rmtree(package_dir) - try: - staged_package.replace(package_dir) - except OSError: - if not package_dir.exists(): - raise - _validate_release_package(package_dir, runtime_platform) - finally: - shutil.rmtree(staging_dir, ignore_errors=True) - return package_dir +def _runtime_bundle_is_complete(pair_dir: Path, wrapper_name: str) -> bool: + required = ( + pair_dir / wrapper_name, + pair_dir / "runtime.node", + pair_dir / _HOSTLESS_ASSETS_MARKER, + ) + return all(path.is_file() and path.stat().st_size > 0 for path in required) def download_cli(version: str | None = None, *, force: bool = False) -> str: @@ -360,24 +297,23 @@ def _hostless_runtime_path(member_name: str, runtime_platform: str) -> Path | No return destination -def _materialize_runtime_bundle( - package_dir: Path, runtime_platform: str, destination: Path -) -> None: - """Copy the hostless runtime tree, retaining unknown package assets by default.""" - import shutil - - for source in package_dir.rglob("*"): - if source.is_dir(): - continue - member_name = PurePosixPath("package", *source.relative_to(package_dir).parts).as_posix() - relative = _hostless_runtime_path(member_name, runtime_platform) - if relative is None: - continue - if not source.is_file(): - raise RuntimeError(f"Unsupported runtime package entry: {member_name}") - target = destination / relative - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(source, target) +def _materialize_runtime_bundle(data: bytes, runtime_platform: str, destination: Path) -> None: + """Extract the hostless runtime tree, retaining unknown package assets by default.""" + with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as archive: + for member in archive: + relative = _hostless_runtime_path(member.name, runtime_platform) + if relative is None or member.isdir(): + continue + if not member.isfile(): + raise RuntimeError(f"Unsupported runtime package entry: {member.name}") + extracted = archive.extractfile(member) + if extracted is None: + raise RuntimeError(f"Failed to read runtime package entry: {member.name}") + target = destination / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(extracted.read()) + if sys.platform != "win32": + target.chmod(member.mode & 0o777) def ensure_runtime_wrapper(version: str | None = None, force: bool = False) -> str: @@ -394,20 +330,26 @@ def ensure_runtime_wrapper(version: str | None = None, force: bool = False) -> s wrapper_exists = wrapper_path.is_file() and wrapper_path.stat().st_size > 0 runtime_exists = runtime_path.is_file() and runtime_path.stat().st_size > 0 - if wrapper_exists and runtime_exists and assets_marker.is_file() and not force: + if _runtime_bundle_is_complete(pair_dir, wrapper_name) and not force: return str(wrapper_path) if not force and wrapper_exists != runtime_exists: raise RuntimeError( f"Incomplete Copilot runtime bundle in {pair_dir}: " f"{wrapper_name} and runtime.node are required." ) - package_dir = _ensure_release_package(ver, force=force) + if _should_skip_download(): + raise RuntimeError( + f"Copilot runtime bundle is not cached in {pair_dir} " + "and automatic downloads are disabled." + ) + + data = _fetch_verified_release_package(ver, runtime_platform) import shutil pair_dir.parent.mkdir(parents=True, exist_ok=True) staging_dir = Path(tempfile.mkdtemp(dir=pair_dir.parent, prefix=".runtime-bundle-")) try: - _materialize_runtime_bundle(package_dir, runtime_platform, staging_dir) + _materialize_runtime_bundle(data, runtime_platform, staging_dir) staged_wrapper = staging_dir / wrapper_name staged_runtime = staging_dir / "runtime.node" if ( @@ -446,14 +388,15 @@ def ensure_runtime_wrapper(version: str | None = None, force: bool = False) -> s def ensure_runtime_library(cli_path: str, version: str | None = None) -> str | None: """Ensure the native in-process (FFI) runtime library sits next to ``cli_path``. - The unified platform release package contains ``runtime.node`` under - ``package/prebuilds/``. This helper copies that verified library next - to the CLI binary under its natural platform name (``libcopilot_runtime.so`` / - ``.dylib`` / ``copilot_runtime.dll``). + The canonical staged bundle contains ``prebuilds//runtime.node``. + This helper reuses that verified library and copies it next to the CLI binary + under its natural platform name (``libcopilot_runtime.so`` / ``.dylib`` / + ``copilot_runtime.dll``). - This is opt-in — only invoked when the in-process transport is actually selected - (lazy) or via ``python -m copilot download-runtime --in-process`` (explicit). The - default stdio download path never fetches these extra bytes. + Copying the library next to an external CLI is opt-in — this is only invoked when + the in-process transport is selected (lazy) or via + ``python -m copilot download-runtime --in-process`` (explicit). The default stdio + path leaves the library in the canonical staged bundle. Returns the absolute path to the library, or None if it could not be provisioned (e.g. download disabled or unsupported platform). Raises RuntimeError on @@ -481,19 +424,22 @@ def ensure_runtime_library(cli_path: str, version: str | None = None) -> str | N if lib_path.exists(): return str(lib_path) - package_dir = _release_package_dir(ver, runtime_platform) - if _should_skip_download() and not package_dir.exists(): + pair_dir = get_cache_dir(ver) / "prebuilds" / runtime_platform + wrapper_name = "copilot-runtime.exe" if sys.platform == "win32" else "copilot-runtime" + if _should_skip_download() and not _runtime_bundle_is_complete(pair_dir, wrapper_name): return None - package_dir = _ensure_release_package(ver) - lib_bytes = (package_dir / "prebuilds" / runtime_platform / "runtime.node").read_bytes() + wrapper_path = Path(ensure_runtime_wrapper(ver)) + canonical_runtime = wrapper_path.with_name("runtime.node") # Write atomically next to the CLI so concurrent starts don't observe a partial # library. A rename within the same directory is atomic on POSIX and Windows. + import shutil + cli_dir.mkdir(parents=True, exist_ok=True) fd, tmp_name = tempfile.mkstemp(dir=cli_dir, prefix=".runtime-lib-") try: - with os.fdopen(fd, "wb") as out: - out.write(lib_bytes) + with os.fdopen(fd, "wb") as out, canonical_runtime.open("rb") as source: + shutil.copyfileobj(source, out) os.replace(tmp_name, lib_path) except OSError: try: @@ -534,8 +480,11 @@ def get_or_download_cli(version: str | None = None) -> str | None: except RuntimeError: return None - if _should_skip_download() and not _release_package_dir(ver, runtime_platform).exists(): - return None + if _should_skip_download(): + pair_dir = get_cache_dir(ver) / "prebuilds" / runtime_platform + wrapper_name = "copilot-runtime.exe" if sys.platform == "win32" else "copilot-runtime" + if not _runtime_bundle_is_complete(pair_dir, wrapper_name): + return None # Download return download_cli(ver) diff --git a/python/test_cli_download.py b/python/test_cli_download.py index 24433d862a..7a86659e62 100644 --- a/python/test_cli_download.py +++ b/python/test_cli_download.py @@ -6,7 +6,9 @@ import io import os import tarfile +from concurrent.futures import ThreadPoolExecutor from http.client import IncompleteRead +from threading import Barrier from unittest.mock import MagicMock, patch import pytest @@ -117,7 +119,7 @@ def fetch(url: str, *, timeout: int) -> bytes: _cli_download.ensure_runtime_wrapper(version="1.2.3") -def test_cli_and_runtime_share_one_cached_release_package(tmp_path, monkeypatch): +def test_cli_and_runtime_share_one_staged_bundle(tmp_path, monkeypatch): version = "1.2.3" runtime_platform = "linux-x64" cli_name = "copilot.exe" if os.name == "nt" else "copilot" @@ -125,7 +127,6 @@ def test_cli_and_runtime_share_one_cached_release_package(tmp_path, monkeypatch) data = _release_package(runtime_platform) cache_dir = tmp_path / "cache" install_dir = cache_dir / "prebuilds" / runtime_platform - package_dir = cache_dir / "packages" / runtime_platform fetch = _release_fetches(version, runtime_platform, data) with ( @@ -142,7 +143,7 @@ def test_cli_and_runtime_share_one_cached_release_package(tmp_path, monkeypatch) assert wrapper == str(install_dir / wrapper_name) assert (install_dir / cli_name).read_bytes() == b"wrapper" assert not (cache_dir / cli_name).exists() - assert not (package_dir / cli_name).exists() + assert not (cache_dir / "packages").exists() assert (install_dir / wrapper_name).read_bytes() == b"wrapper" assert (install_dir / "runtime.node").read_bytes() == b"runtime" assert (install_dir / "ripgrep" / "bin" / runtime_platform / "rg").read_bytes() == b"ripgrep" @@ -155,7 +156,69 @@ def test_cli_and_runtime_share_one_cached_release_package(tmp_path, monkeypatch) assert (install_dir / wrapper_name).stat().st_mode & 0o111 -def test_skip_download_returns_none_without_cached_package(tmp_path, monkeypatch): +def test_concurrent_staging_materializes_one_complete_bundle(tmp_path): + version = "1.2.3" + runtime_platform = "linux-x64" + wrapper_name = "copilot-runtime.exe" if os.name == "nt" else "copilot-runtime" + data = _release_package(runtime_platform) + cache_dir = tmp_path / "cache" + fetch = _release_fetches(version, runtime_platform, data) + fetch_barrier = Barrier(2) + + def concurrent_fetch(url: str, *, timeout: int) -> bytes: + fetch_barrier.wait(timeout=10) + return fetch(url, timeout=timeout) + + with ( + patch.object(_cli_download, "get_cache_dir", return_value=cache_dir), + patch.object(_cli_download, "get_runtime_platform", return_value=runtime_platform), + patch.object(_cli_download, "_fetch_url_bytes", side_effect=concurrent_fetch) as fetch_mock, + ThreadPoolExecutor(max_workers=2) as executor, + ): + futures = [executor.submit(_cli_download.ensure_runtime_wrapper, version) for _ in range(2)] + wrappers = [future.result() for future in futures] + + install_dir = cache_dir / "prebuilds" / runtime_platform + expected_wrapper = str(install_dir / wrapper_name) + assert wrappers == [expected_wrapper, expected_wrapper] + assert (install_dir / wrapper_name).read_bytes() == b"wrapper" + assert (install_dir / "runtime.node").read_bytes() == b"runtime" + assert (install_dir / ".hostless-runtime-assets-v2").is_file() + assert not list((cache_dir / "prebuilds").glob(".runtime-bundle-*")) + assert not (cache_dir / "packages").exists() + assert fetch_mock.call_count == 4 + + +def test_force_restages_complete_bundle_and_compatibility_alias(tmp_path): + version = "1.2.3" + runtime_platform = "linux-x64" + cli_name = "copilot.exe" if os.name == "nt" else "copilot" + wrapper_name = "copilot-runtime.exe" if os.name == "nt" else "copilot-runtime" + data = _release_package(runtime_platform) + cache_dir = tmp_path / "cache" + install_dir = cache_dir / "prebuilds" / runtime_platform + install_dir.mkdir(parents=True) + (install_dir / cli_name).write_bytes(b"old-alias") + (install_dir / wrapper_name).write_bytes(b"old-wrapper") + (install_dir / "runtime.node").write_bytes(b"old-runtime") + (install_dir / ".hostless-runtime-assets-v2").write_text("1\n", encoding="ascii") + fetch = _release_fetches(version, runtime_platform, data) + + with ( + patch.object(_cli_download, "get_cache_dir", return_value=cache_dir), + patch.object(_cli_download, "get_runtime_platform", return_value=runtime_platform), + patch.object(_cli_download, "_fetch_url_bytes", side_effect=fetch) as fetch_mock, + ): + cli = _cli_download.download_cli(version, force=True) + + assert cli == str(install_dir / cli_name) + assert (install_dir / cli_name).read_bytes() == b"wrapper" + assert (install_dir / wrapper_name).read_bytes() == b"wrapper" + assert (install_dir / "runtime.node").read_bytes() == b"runtime" + assert fetch_mock.call_count == 2 + + +def test_skip_download_returns_none_without_cached_bundle(tmp_path, monkeypatch): monkeypatch.setenv("COPILOT_SKIP_CLI_DOWNLOAD", "true") runtime_platform = "linux-x64" @@ -189,7 +252,7 @@ def test_cached_cli_rejects_alias_from_incomplete_bundle(tmp_path): _cli_download.download_cli(version) -def test_explicit_cli_gets_library_from_cached_release_package(tmp_path): +def test_explicit_cli_reuses_library_from_canonical_staged_bundle(tmp_path): version = "1.2.3" runtime_platform = "linux-x64" data = _release_package(runtime_platform) @@ -211,6 +274,8 @@ def test_explicit_cli_gets_library_from_cached_release_package(tmp_path): assert library == str(cli_dir / _ffi_runtime_host._natural_library_name()) assert (cli_dir / _ffi_runtime_host._natural_library_name()).read_bytes() == b"runtime" + assert (cache_dir / "prebuilds" / runtime_platform / "runtime.node").read_bytes() == b"runtime" + assert not (cache_dir / "packages").exists() assert wrapper.endswith( f"prebuilds/{runtime_platform}/" f"{'copilot-runtime.exe' if os.name == 'nt' else 'copilot-runtime'}" diff --git a/rust/.gitignore b/rust/.gitignore index c4095ffc0f..c149fa3946 100644 --- a/rust/.gitignore +++ b/rust/.gitignore @@ -1,3 +1,4 @@ /target Cargo.lock.bak cli-version.txt +cli-version-in-process.txt diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 8a9188832d..b91eebd06c 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -23,6 +23,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -129,6 +138,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crypto-common" version = "0.1.7" @@ -145,6 +160,17 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "digest" version = "0.10.7" @@ -429,6 +455,7 @@ dependencies = [ "ureq", "uuid", "windows-sys 0.61.2", + "zip", ] [[package]] @@ -1064,7 +1091,7 @@ checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror", + "thiserror 1.0.69", ] [[package]] @@ -1558,7 +1585,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", ] [[package]] @@ -1572,6 +1608,17 @@ dependencies = [ "syn", ] +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -1765,7 +1812,7 @@ dependencies = [ "native-tls", "rand", "sha1", - "thiserror", + "thiserror 1.0.69", "utf-8", ] @@ -2363,8 +2410,37 @@ dependencies = [ "syn", ] +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror 2.0.18", + "zopfli", +] + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index af6f7e28ff..c66480d511 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -21,6 +21,7 @@ include = [ "README.md", "LICENSE", "cli-version.txt", + "cli-version-in-process.txt", ] [lib] @@ -28,7 +29,7 @@ name = "github_copilot_sdk" [features] default = ["bundled-cli"] -bundled-cli = ["dep:tar", "dep:flate2"] +bundled-cli = ["dep:tar", "dep:flate2", "dep:zip"] bundled-in-process = ["bundled-cli", "dep:libloading"] derive = ["dep:schemars"] test-support = [] @@ -68,6 +69,7 @@ reqwest = { version = "0.12", default-features = false, features = ["stream", "h tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "native-tls"] } [target.'cfg(windows)'.dependencies] +zip = { version = "2", default-features = false, features = ["deflate"], optional = true } windows-sys = { version = "0.61", default-features = false, features = [ "Win32_Foundation", "Win32_System_Diagnostics_ToolHelp", @@ -124,3 +126,4 @@ sha2 = "0.10" tar = "0.4" ureq = { version = "2", default-features = false, features = ["native-tls"] } native-tls = "0.2" +zip = { version = "2", default-features = false, features = ["deflate"] } diff --git a/rust/README.md b/rust/README.md index 02035193a0..a561e6da09 100644 --- a/rust/README.md +++ b/rust/README.md @@ -965,9 +965,8 @@ github-copilot-sdk = { version = "0.1", default-features = false } The resolved version is baked into the crate via `cargo:rustc-env=COPILOT_SDK_CLI_VERSION` regardless of mode. The runtime resolver consumes it to recompute the on-disk path by convention, so no absolute paths leak into the rlib. -2. **Build time:** `build.rs` downloads - `github-copilot--.tgz` and verifies its SHA-256 against - the release's `SHA256SUMS.txt` or the publish snapshot. +2. **Build time:** `build.rs` downloads the platform-specific release archive and + verifies its SHA-256 against the release's `SHA256SUMS.txt` or the publish snapshot. Then: - **`bundled-cli` on (default):** creates and embeds a minimal archive containing the CLI executable, `copilot-runtime[.exe]`, and `runtime.node`. - **`bundled-in-process` on:** the archive additionally contains the platform-native runtime library (`.dll`, `.so`, or `.dylib`). @@ -1066,7 +1065,7 @@ In embed mode `build.rs` re-downloads on every clean build by default. Set `BUND ### Platforms -Supported: `darwin-arm64`, `darwin-x64`, `linux-x64`, `linux-arm64`, `linuxmusl-x64`, `linuxmusl-arm64`, `win32-x64`, `win32-arm64`. The target platform is auto-detected from Cargo's target OS, architecture, and environment (cross-compilation works). +Supported: `darwin-arm64`, `darwin-x64`, `linux-x64`, `linux-arm64`, `win32-x64`, `win32-arm64`. The target platform is auto-detected from `CARGO_CFG_TARGET_OS` and `CARGO_CFG_TARGET_ARCH` (cross-compilation works). ## Features diff --git a/rust/build.rs b/rust/build.rs index 2d8eb9992d..c01464bb4a 100644 --- a/rust/build.rs +++ b/rust/build.rs @@ -1,4 +1,4 @@ -#[path = "build/runtime.rs"] +#[path = "build/in_process.rs"] mod implementation; fn main() { diff --git a/rust/build/runtime.rs b/rust/build/in_process.rs similarity index 88% rename from rust/build/runtime.rs rename to rust/build/in_process.rs index 66eecaa670..c01e7ffc4f 100644 --- a/rust/build/runtime.rs +++ b/rust/build/in_process.rs @@ -11,7 +11,7 @@ pub(crate) fn main() { println!("cargo:rerun-if-env-changed=BUNDLED_CLI_CACHE_DIR"); println!("cargo::rustc-check-cfg=cfg(has_bundled_cli)"); println!("cargo::rustc-check-cfg=cfg(has_extracted_cli)"); - println!("cargo:rerun-if-changed=cli-version.txt"); + println!("cargo:rerun-if-changed=cli-version-in-process.txt"); // Only declare the package metadata rerun when it actually exists. // Cargo treats `rerun-if-changed` for a missing path as "always rerun" @@ -19,7 +19,7 @@ pub(crate) fn main() { // `nodejs/` (vendored slots, published crates) would force build.rs // to re-run on every `cargo build` even when nothing has changed. // The package file is only the source-of-truth in this repo's - // contributor builds; everywhere else `cli-version.txt` is canonical. + // contributor builds; everywhere else `cli-version-in-process.txt` is canonical. let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is set"); let package_json = Path::new(&manifest_dir) .join("..") @@ -61,11 +61,11 @@ pub(crate) fn main() { // Resolve version and, when available locally, the release SHA-256 from // one of two sources, in order: - // 1. `cli-version.txt` snapshot at the crate root (published-crate + // 1. `cli-version-in-process.txt` snapshot at the crate root (published-crate // consumer; generated by the publish workflow from SHA256SUMS.txt). // 2. Sibling `../nodejs/package.json` plus the release SHA256SUMS.txt // (contributor build inside the github/copilot-sdk repo). - let (version, local_expected_hash) = resolve_version_and_optional_hash(platform.name); + let (version, local_expected_hash) = resolve_version_and_optional_hash(platform.package_name); // Bake the version into the crate regardless of mode. This is the // single source of truth for "what CLI version did build.rs target", @@ -76,7 +76,11 @@ pub(crate) fn main() { // `target/` reuse stays cache-coherent. println!("cargo:rustc-env=COPILOT_SDK_CLI_VERSION={version}"); - let archive_name = platform.asset_name(&version); + let asset_platform = platform + .package_name + .strip_prefix("copilot-") + .expect("platform package names start with copilot-"); + let archive_name = format!("github-copilot-{version}-{asset_platform}.tgz"); let download_url = format!( "https://github.com/github/copilot-cli/releases/download/v{version}/{archive_name}" ); @@ -90,7 +94,7 @@ pub(crate) fn main() { if std::env::var_os("CARGO_FEATURE_BUNDLED_CLI").is_some() { let expected_hash = local_expected_hash .clone() - .unwrap_or_else(|| fetch_release_hash(&version, &archive_name)); + .unwrap_or_else(|| fetch_in_process_release_hash(&version, platform.package_name)); let archive = cached_download(&download_url, &cache_key, &expected_hash, &cache_dir); verify_runtime_package(&archive, platform, &archive_name); emit_embedded(out, &archive, platform, include_runtime); @@ -129,8 +133,8 @@ pub(crate) fn main() { .is_some_and(|contents| marker_matches_version(contents, &version)), }; if !cache_is_current { - let expected_hash = - local_expected_hash.unwrap_or_else(|| fetch_release_hash(&version, &archive_name)); + let expected_hash = local_expected_hash + .unwrap_or_else(|| fetch_in_process_release_hash(&version, platform.package_name)); let expected_marker = format!("{version}\n{expected_hash}\n"); if install_dir.exists() { std::fs::remove_dir_all(&install_dir).unwrap_or_else(|e| { @@ -198,16 +202,12 @@ pub(super) static CLI_ARCHIVE: &[u8] = include_bytes!("copilot_cli.archive"); std::fs::write(out.join("bundled_cli.rs"), generated).expect("failed to write bundled_cli.rs"); } -fn build_embedded_archive( - release_archive: &[u8], - platform: Platform, - include_runtime: bool, -) -> Vec { +fn build_embedded_archive(package: &[u8], platform: Platform, include_runtime: bool) -> Vec { let encoder = flate2::GzBuilder::new() .mtime(0) .write(Vec::new(), flate2::Compression::default()); let mut archive = tar::Builder::new(encoder); - let (runtime, wrapper) = append_hostless_runtime_tree(&mut archive, release_archive, platform); + let (runtime, wrapper) = append_hostless_runtime_tree(&mut archive, package, platform); append_archive_file(&mut archive, platform.binary_name, &wrapper, 0o755); if include_runtime { append_archive_file( @@ -227,25 +227,24 @@ fn build_embedded_archive( fn append_hostless_runtime_tree( archive: &mut tar::Builder, - release_archive: &[u8], + package: &[u8], platform: Platform, ) -> (Vec, Vec) { - let decoder = flate2::read::GzDecoder::new(release_archive); + let decoder = flate2::read::GzDecoder::new(package); let mut source = tar::Archive::new(decoder); let mut runtime = None; let mut wrapper = None; for entry in source .entries() - .unwrap_or_else(|e| panic!("failed to read release archive entries: {e}")) + .unwrap_or_else(|e| panic!("failed to read npm package entries: {e}")) { - let mut entry = - entry.unwrap_or_else(|e| panic!("failed to read release archive entry: {e}")); + let mut entry = entry.unwrap_or_else(|e| panic!("failed to read npm package entry: {e}")); if !entry.header().entry_type().is_file() { continue; } let source_path = entry .path() - .unwrap_or_else(|e| panic!("failed to read release archive path: {e}")); + .unwrap_or_else(|e| panic!("failed to read npm package path: {e}")); let Some(destination) = hostless_runtime_path(&source_path.to_string_lossy(), platform) else { continue; @@ -253,7 +252,7 @@ fn append_hostless_runtime_tree( let mut bytes = Vec::with_capacity(entry.size() as usize); entry .read_to_end(&mut bytes) - .unwrap_or_else(|e| panic!("failed to read release archive entry bytes: {e}")); + .unwrap_or_else(|e| panic!("failed to read npm package entry bytes: {e}")); let mode = entry.header().mode().unwrap_or(0o644); if destination == Path::new("runtime.node") { runtime = Some(bytes.clone()); @@ -265,7 +264,7 @@ fn append_hostless_runtime_tree( archive, destination .to_str() - .expect("release archive paths are valid UTF-8"), + .expect("npm package paths are valid UTF-8"), &bytes, mode, ); @@ -273,14 +272,14 @@ fn append_hostless_runtime_tree( ( runtime.unwrap_or_else(|| { panic!( - "release archive for `{}` does not contain prebuilds//runtime.node", - platform.name + "package `{}` does not contain prebuilds//runtime.node", + platform.package_name ) }), wrapper.unwrap_or_else(|| { panic!( - "release archive for `{}` does not contain prebuilds//{}", - platform.name, + "package `{}` does not contain prebuilds//{}", + platform.package_name, platform.runtime_wrapper_name() ) }), @@ -312,7 +311,6 @@ fn hostless_runtime_path(source: &str, platform: Platform) -> Option { "webview", ]; if EXCLUDED_TOP_LEVEL.contains(&top_level) - || top_level == platform.binary_name || (top_level.starts_with("tree-sitter") && top_level.ends_with(".wasm")) || (top_level.starts_with("voice-") && top_level.ends_with(".js")) || file_name == "cli-native.node" @@ -322,7 +320,11 @@ fn hostless_runtime_path(source: &str, platform: Platform) -> Option { return None; } if top_level == "prebuilds" { - if parts.get(1) != Some(&platform.name) || parts.len() < 3 { + let npm_platform = platform + .package_name + .strip_prefix("copilot-") + .expect("platform package name has copilot- prefix"); + if parts.get(1) != Some(&npm_platform) || parts.len() < 3 { return None; } return Some(parts[2..].iter().copied().collect()); @@ -349,18 +351,18 @@ fn append_archive_file( } /// Resolve the CLI version and any locally snapshotted release hash for the -/// current target's release asset. Contributor builds defer fetching the +/// current target's platform package. Contributor builds defer fetching the /// checksum until a download is actually required. -fn resolve_version_and_optional_hash(platform_name: &str) -> (String, Option) { +fn resolve_version_and_optional_hash(package_name: &str) -> (String, Option) { let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is set"); // 1. Snapshot file at the crate root (published-crate consumer, // vendored-slot consumer). Combined version + per-asset hashes. - let snapshot = Path::new(&manifest_dir).join("cli-version.txt"); + let snapshot = Path::new(&manifest_dir).join("cli-version-in-process.txt"); if snapshot.is_file() { let contents = std::fs::read_to_string(&snapshot) .unwrap_or_else(|e| panic!("failed to read {}: {e}", snapshot.display())); - let (version, hash) = parse_snapshot(&contents, platform_name) + let (version, hash) = parse_snapshot(&contents, package_name) .unwrap_or_else(|e| panic!("invalid {}: {e}", snapshot.display())); return (version, Some(hash)); } @@ -380,13 +382,21 @@ fn resolve_version_and_optional_hash(platform_name: &str) -> (String, Option String { + let platform = package_name + .strip_prefix("copilot-") + .expect("platform package names start with copilot-"); + let asset_name = format!("github-copilot-{version}-{platform}.tgz"); + fetch_release_hash(version, &asset_name) +} + fn marker_matches_version(contents: &str, version: &str) -> bool { let mut lines = contents.lines(); lines.next() == Some(version) @@ -396,12 +406,13 @@ fn marker_matches_version(contents: &str, version: &str) -> bool { && lines.next().is_none() } -/// Parse the `cli-version.txt` snapshot file. Format is one `key=value` per +/// Parse the `cli-version-in-process.txt` snapshot file. Format is one `key=value` per /// line. The first non-comment line is `version=X.Y.Z`; subsequent lines map -/// release asset names to SHA-256. Blank lines and comments are skipped. -fn parse_snapshot(contents: &str, platform_name: &str) -> Result<(String, String), String> { +/// platform package name to SHA-256. Blank lines and lines starting with `#` +/// are skipped. +fn parse_snapshot(contents: &str, package_name: &str) -> Result<(String, String), String> { let mut version: Option = None; - let mut hashes = Vec::new(); + let mut hash: Option = None; for (line_no, raw) in contents.lines().enumerate() { let line = raw.trim(); if line.is_empty() || line.starts_with('#') { @@ -415,15 +426,12 @@ fn parse_snapshot(contents: &str, platform_name: &str) -> Result<(String, String }; match key.trim() { "version" => version = Some(value.trim().to_string()), - asset_name => hashes.push((asset_name, value.trim())), + k if k == package_name => hash = Some(value.trim().to_string()), + _ => {} } } let version = version.ok_or("missing `version=` line")?; - let asset_name = format!("github-copilot-{version}-{platform_name}.tgz"); - let hash = hashes - .into_iter() - .find_map(|(name, hash)| (name == asset_name).then(|| hash.to_string())) - .ok_or_else(|| format!("missing hash for asset `{asset_name}`"))?; + let hash = hash.ok_or_else(|| format!("missing hash for package `{package_name}`"))?; Ok((version, hash)) } @@ -458,17 +466,13 @@ fn find_sha256_for_asset(sums: &str, asset_name: &str) -> String { #[derive(Clone, Copy)] struct Platform { - name: &'static str, + package_name: &'static str, binary_name: &'static str, } impl Platform { - fn asset_name(&self, version: &str) -> String { - format!("github-copilot-{version}-{}.tgz", self.name) - } - fn runtime_wrapper_name(&self) -> &'static str { - if self.name.starts_with("win32") { + if self.package_name.contains("win32") { "copilot-runtime.exe" } else { "copilot-runtime" @@ -476,9 +480,9 @@ impl Platform { } fn runtime_library_name(&self) -> &'static str { - if self.name.starts_with("win32") { + if self.package_name.contains("win32") { "copilot_runtime.dll" - } else if self.name.starts_with("darwin") { + } else if self.package_name.contains("darwin") { "libcopilot_runtime.dylib" } else { "libcopilot_runtime.so" @@ -493,35 +497,35 @@ fn target_platform() -> Option { match (os.as_str(), arch.as_str(), target_env.as_str()) { ("macos", "aarch64", _) => Some(Platform { - name: "darwin-arm64", + package_name: "copilot-darwin-arm64", binary_name: "copilot", }), ("macos", "x86_64", _) => Some(Platform { - name: "darwin-x64", + package_name: "copilot-darwin-x64", binary_name: "copilot", }), ("linux", "x86_64", "musl") => Some(Platform { - name: "linuxmusl-x64", + package_name: "copilot-linuxmusl-x64", binary_name: "copilot", }), ("linux", "aarch64", "musl") => Some(Platform { - name: "linuxmusl-arm64", + package_name: "copilot-linuxmusl-arm64", binary_name: "copilot", }), ("linux", "x86_64", _) => Some(Platform { - name: "linux-x64", + package_name: "copilot-linux-x64", binary_name: "copilot", }), ("linux", "aarch64", _) => Some(Platform { - name: "linux-arm64", + package_name: "copilot-linux-arm64", binary_name: "copilot", }), ("windows", "x86_64", _) => Some(Platform { - name: "win32-x64", + package_name: "copilot-win32-x64", binary_name: "copilot.exe", }), ("windows", "aarch64", _) => Some(Platform { - name: "win32-arm64", + package_name: "copilot-win32-arm64", binary_name: "copilot.exe", }), _ => None, @@ -557,16 +561,15 @@ fn extract_to_cache( let mut runtime = None; for entry in source .entries() - .unwrap_or_else(|e| panic!("failed to read release archive entries: {e}")) + .unwrap_or_else(|e| panic!("failed to read npm package entries: {e}")) { - let mut entry = - entry.unwrap_or_else(|e| panic!("failed to read release archive entry: {e}")); + let mut entry = entry.unwrap_or_else(|e| panic!("failed to read npm package entry: {e}")); if !entry.header().entry_type().is_file() { continue; } let source_path = entry .path() - .unwrap_or_else(|e| panic!("failed to read release archive path: {e}")); + .unwrap_or_else(|e| panic!("failed to read npm package path: {e}")); let Some(destination) = hostless_runtime_path(&source_path.to_string_lossy(), platform) else { continue; @@ -577,7 +580,7 @@ fn extract_to_cache( let mut bytes = Vec::with_capacity(entry.size() as usize); entry .read_to_end(&mut bytes) - .unwrap_or_else(|e| panic!("failed to read release archive entry bytes: {e}")); + .unwrap_or_else(|e| panic!("failed to read npm package entry bytes: {e}")); let executable = entry.header().mode().unwrap_or(0o644) & 0o111 != 0; if destination == Path::new("runtime.node") { runtime = Some(bytes.clone()); @@ -634,7 +637,7 @@ fn install_cached_file_path( | std::path::Component::ParentDir ) }), - "unsafe runtime archive path: {}", + "unsafe runtime package path: {}", relative_path.display() ); let final_path = install_dir.join(relative_path); @@ -875,13 +878,13 @@ fn try_download(url: &str) -> Result, DownloadError> { } } -fn verify_runtime_package(archive: &[u8], platform: Platform, asset_name: &str) { +fn verify_runtime_package(archive: &[u8], platform: Platform, package_name: &str) { for file_name in ["runtime.node", platform.runtime_wrapper_name()] { if archive_contains_tar_entry(archive, file_name) { continue; } panic!( - "Copilot runtime archive `{asset_name}` does not contain an entry named `{file_name}`" + "Copilot runtime package `{package_name}` does not contain an entry named `{file_name}`" ); } } diff --git a/rust/build/out_of_process.rs b/rust/build/out_of_process.rs new file mode 100644 index 0000000000..c0a9dd3050 --- /dev/null +++ b/rust/build/out_of_process.rs @@ -0,0 +1,692 @@ +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use sha2::Digest; + +pub(crate) fn main() { + println!("cargo:rerun-if-env-changed=DOCS_RS"); + println!("cargo:rerun-if-env-changed=COPILOT_SKIP_CLI_DOWNLOAD"); + println!("cargo:rerun-if-env-changed=COPILOT_CLI_EXTRACT_DIR"); + println!("cargo:rerun-if-env-changed=BUNDLED_CLI_CACHE_DIR"); + println!("cargo::rustc-check-cfg=cfg(has_bundled_cli)"); + println!("cargo::rustc-check-cfg=cfg(has_extracted_cli)"); + println!("cargo:rerun-if-changed=cli-version.txt"); + + // Only declare the package metadata rerun when it actually exists. + // Cargo treats `rerun-if-changed` for a missing path as "always rerun" + // — so unconditionally declaring this on consumers without a sibling + // `nodejs/` (vendored slots, published crates) would force build.rs + // to re-run on every `cargo build` even when nothing has changed. + // The package file is only the source-of-truth in this repo's + // contributor builds; everywhere else `cli-version.txt` is canonical. + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is set"); + let package_json = Path::new(&manifest_dir) + .join("..") + .join("nodejs") + .join("package.json"); + if package_json.is_file() { + println!("cargo:rerun-if-changed={}", package_json.display()); + } + + // Hard opt-out: disable the entire download / bundle / cache mechanism + // in one step. For consumers who always supply the CLI via + // `CliProgram::Path` or `COPILOT_CLI_PATH` and don't want build.rs to + // touch the network (offline builds, locked-down CI, etc.). Works + // regardless of the `bundled-cli` cargo feature state — with neither + // `has_bundled_cli` nor `has_extracted_cli` emitted, runtime resolution + // falls straight through to `Error::BinaryNotFound` unless an explicit + // path source resolves first. + if std::env::var_os("COPILOT_SKIP_CLI_DOWNLOAD").is_some() { + println!( + "cargo:warning=COPILOT_SKIP_CLI_DOWNLOAD is set — skipping CLI download/bundle/cache" + ); + return; + } + + // docs.rs builds in a sandboxed environment without network access. + // Skip the CLI download so documentation can be generated successfully. + if std::env::var_os("DOCS_RS").is_some() { + println!("cargo:warning=DOCS_RS is set — skipping CLI download/bundle/cache"); + return; + } + + let Some(platform) = target_platform() else { + println!("cargo:warning=Unsupported target platform for Copilot CLI bundling — skipping"); + return; + }; + + let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR is always set by cargo"); + let out = Path::new(&out_dir); + + // Resolve version + per-asset SHA-256 from one of two sources, in order: + // 1. `cli-version.txt` snapshot at the crate root (published-crate + // consumer; generated by the publish workflow from SHA256SUMS.txt). + // 2. Sibling `../nodejs/package.json` plus the release SHA256SUMS.txt + // (contributor build inside the github/copilot-sdk repo). + let (version, expected_hash) = resolve_version_and_hash(platform.asset_name); + + // Bake the version into the crate regardless of mode. This is the + // single source of truth for "what CLI version did build.rs target", + // consumed by both the embed-mode path computation in embeddedcli.rs + // and the runtime path computation in resolve.rs (when `bundled-cli` + // is off). It's a small, machine-independent datum: no absolute + // paths, no username/home leakage, so sccache / cross-machine + // `target/` reuse stays cache-coherent. + println!("cargo:rustc-env=COPILOT_SDK_CLI_VERSION={version}"); + + let base_url = format!("https://github.com/github/copilot-cli/releases/download/v{version}"); + let cache_dir = std::env::var("BUNDLED_CLI_CACHE_DIR") + .ok() + .map(std::path::PathBuf::from); + + // Versioned cache key since copilot asset names don't include the version. + let cache_key = format!("v{version}-{}", platform.asset_name); + + if std::env::var_os("CARGO_FEATURE_BUNDLED_CLI").is_some() { + // Embed mode: we need the archive bytes to bake into the rlib, so + // always run the download (cache hit short-circuits inside + // `cached_download`). + let archive = cached_download( + &format!("{base_url}/{}", platform.asset_name), + &cache_key, + &expected_hash, + &cache_dir, + ); + verify_binary_present_in_archive(&archive, platform.binary_name, platform.asset_name); + emit_embedded(out, &archive); + println!("cargo:rustc-cfg=has_bundled_cli"); + } else { + // With `bundled-cli` off the extracted binary *is* the cache. + // Skip the upstream download entirely when it already exists at + // the expected path. No two separate caches. + // + // Runtime resolution (see `src/resolve.rs::extracted_cli_path`) + // recomputes this same path from `COPILOT_SDK_CLI_VERSION` + the + // OS-derived binary name + optional `COPILOT_CLI_EXTRACT_DIR`, + // so we don't bake an absolute path into the crate. + let install_dir = extracted_install_dir(&version); + let final_path = install_dir.join(platform.binary_name); + + // Invalidate build.rs whenever the cached binary disappears (cache GC, + // manual rm, OS reset, switching extract dir). Without this, cargo + // replays the saved `has_extracted_cli` cfg from its build-script + // output cache even when the file is gone, and runtime resolution + // fails with BinaryNotFound. + println!("cargo:rerun-if-changed={}", final_path.display()); + + if !final_path.is_file() { + let archive = cached_download( + &format!("{base_url}/{}", platform.asset_name), + &cache_key, + &expected_hash, + &cache_dir, + ); + verify_binary_present_in_archive(&archive, platform.binary_name, platform.asset_name); + extract_to_cache(&archive, &install_dir, platform); + } + + // Re-check after potential download+extract above; not an `else` + // because we need to verify the extraction actually produced the file. + if final_path.is_file() { + println!("cargo:rustc-cfg=has_extracted_cli"); + } + } +} + +/// Install directory used when `bundled-cli` is off. Mirrors the runtime +/// convention in `src/resolve.rs::extracted_cli_path`: both sides MUST +/// compute the same path from the same inputs, otherwise the runtime +/// resolver won't find what build.rs extracted. +/// +/// If `COPILOT_CLI_EXTRACT_DIR` is set the binary lives directly under +/// that directory (no per-version subdir) — useful for vendored slots and +/// for `.cargo/config.toml [env]`-style pinning that's symmetric between +/// build-time write and runtime read. Otherwise the binary lives under +/// `/github-copilot-sdk/cli//`. +fn extracted_install_dir(version: &str) -> PathBuf { + if let Some(custom) = std::env::var_os("COPILOT_CLI_EXTRACT_DIR") { + PathBuf::from(custom) + } else { + let cache = dirs::cache_dir().unwrap_or_else(std::env::temp_dir); + cache + .join("github-copilot-sdk") + .join("cli") + .join(sanitize_version(version)) + } +} + +/// Emit the `bundled_cli.rs` glue + `copilot_cli.archive` blob into `OUT_DIR` +/// for embed mode (`bundled-cli` cargo feature on). The version is exposed +/// crate-wide via the unconditional `cargo:rustc-env=COPILOT_SDK_CLI_VERSION` +/// emit; the binary name is OS-derived at runtime — so all we need to +/// generate here is the archive blob include. +fn emit_embedded(out: &Path, archive: &[u8]) { + std::fs::write(out.join("copilot_cli.archive"), archive) + .expect("failed to write copilot_cli.archive"); + + let generated = r#"// Auto-generated by github-copilot-sdk build.rs. Do not edit. +pub(super) static CLI_ARCHIVE: &[u8] = include_bytes!("copilot_cli.archive"); +"#; + + std::fs::write(out.join("bundled_cli.rs"), generated).expect("failed to write bundled_cli.rs"); +} + +/// Resolve the CLI version and the expected SHA-256 hash for the current +/// target's archive. Picks one of two sources in order. Panics with a clear +/// error if neither is available. +fn resolve_version_and_hash(asset_name: &str) -> (String, String) { + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is set"); + + // 1. Snapshot file at the crate root (published-crate consumer, + // vendored-slot consumer). Combined version + per-asset hashes. + let snapshot = Path::new(&manifest_dir).join("cli-version.txt"); + if snapshot.is_file() { + let contents = std::fs::read_to_string(&snapshot) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", snapshot.display())); + return parse_snapshot(&contents, asset_name) + .unwrap_or_else(|e| panic!("invalid {}: {e}", snapshot.display())); + } + + // 2. Package version plus release checksums (contributor build). + let package_json = Path::new(&manifest_dir) + .join("..") + .join("nodejs") + .join("package.json"); + if package_json.is_file() { + let version = read_version_from_package_json(&package_json); + let hash = fetch_release_hash(&version, asset_name); + return (version, hash); + } + + panic!( + "Could not resolve the Copilot CLI version.\n\ + Tried:\n\ + - {} (missing)\n\ + - {} (missing)\n\ + In a published crate or vendored slot, `cli-version.txt` should be present.\n\ + Inside the github/copilot-sdk repo, `../nodejs/package.json` is the version source.", + snapshot.display(), + package_json.display(), + ); +} + +/// Parse the `cli-version.txt` snapshot file. Format is one `key=value` per +/// line. The first non-comment line is `version=X.Y.Z`; subsequent lines map +/// asset filename to hex SHA-256. Blank lines and lines starting with `#` +/// are skipped. +fn parse_snapshot(contents: &str, asset_name: &str) -> Result<(String, String), String> { + let mut version: Option = None; + let mut hash: Option = None; + for (line_no, raw) in contents.lines().enumerate() { + let line = raw.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let Some((key, value)) = line.split_once('=') else { + return Err(format!( + "line {}: expected `key=value`, got `{raw}`", + line_no + 1 + )); + }; + match key.trim() { + "version" => version = Some(value.trim().to_string()), + k if k == asset_name => hash = Some(value.trim().to_string()), + _ => {} + } + } + let version = version.ok_or("missing `version=` line")?; + let hash = hash.ok_or_else(|| format!("missing hash for asset `{asset_name}`"))?; + Ok((version, hash)) +} + +fn read_version_from_package_json(path: &Path) -> String { + let contents = std::fs::read_to_string(path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", path.display())); + let package_json: serde_json::Value = serde_json::from_str(&contents) + .unwrap_or_else(|e| panic!("failed to parse {}: {e}", path.display())); + package_json["copilotCliVersion"] + .as_str() + .unwrap_or_else(|| panic!("copilotCliVersion is missing in {}", path.display())) + .to_string() +} + +fn fetch_release_hash(version: &str, asset_name: &str) -> String { + let url = + format!("https://github.com/github/copilot-cli/releases/download/v{version}/SHA256SUMS.txt"); + let checksums = download_with_retry(&url); + let checksums = + std::str::from_utf8(&checksums).expect("SHA256SUMS.txt is not valid UTF-8"); + find_sha256_for_asset(checksums, asset_name) +} + +fn find_sha256_for_asset(sums: &str, asset_name: &str) -> String { + sums.lines() + .find_map(|line| { + let (hash, name) = line.split_once(char::is_whitespace)?; + (name.trim_start().trim_start_matches('*') == asset_name).then(|| hash.to_string()) + }) + .unwrap_or_else(|| panic!("SHA256SUMS.txt does not contain {asset_name}")) +} + +#[derive(Clone, Copy)] +struct Platform { + asset_name: &'static str, + binary_name: &'static str, +} + +fn target_platform() -> Option { + let os = std::env::var("CARGO_CFG_TARGET_OS").ok()?; + let arch = std::env::var("CARGO_CFG_TARGET_ARCH").ok()?; + + match (os.as_str(), arch.as_str()) { + ("macos", "aarch64") => Some(Platform { + asset_name: "copilot-darwin-arm64.tar.gz", + binary_name: "copilot", + }), + ("macos", "x86_64") => Some(Platform { + asset_name: "copilot-darwin-x64.tar.gz", + binary_name: "copilot", + }), + ("linux", "x86_64") => Some(Platform { + asset_name: "copilot-linux-x64.tar.gz", + binary_name: "copilot", + }), + ("linux", "aarch64") => Some(Platform { + asset_name: "copilot-linux-arm64.tar.gz", + binary_name: "copilot", + }), + ("windows", "x86_64") => Some(Platform { + asset_name: "copilot-win32-x64.zip", + binary_name: "copilot.exe", + }), + ("windows", "aarch64") => Some(Platform { + asset_name: "copilot-win32-arm64.zip", + binary_name: "copilot.exe", + }), + _ => None, + } +} + +/// Write the single binary entry from `archive` to +/// `/` and return the resulting path. +/// Idempotent — returns the existing path if a previous build already +/// populated the target. +/// +/// Uses file-level staging + atomic rename so a concurrent reader during +/// a parallel `cargo build` race never observes a partially-written +/// binary. `fs::rename` for files is atomic on both Unix and Windows +/// (Windows uses `MoveFileExW` with `MOVEFILE_REPLACE_EXISTING`); for +/// directories it is not, which is why we stage at file granularity. +fn extract_to_cache(archive: &[u8], install_dir: &Path, platform: Platform) -> PathBuf { + let final_path = install_dir.join(platform.binary_name); + + // Caller already gated on `final_path.is_file()`; this is a safety + // net for any future caller that forgets. + if final_path.is_file() { + return final_path; + } + + std::fs::create_dir_all(install_dir).unwrap_or_else(|e| { + panic!( + "failed to create install dir {}: {e}", + install_dir.display() + ) + }); + + let bytes = extract_binary_bytes(archive, platform); + + // Staging file is a sibling of the final binary so the rename stays + // on the same filesystem (cross-fs rename is not atomic). PID + nanos + // disambiguate concurrent builds racing on the same cache. + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + let staging_path = install_dir.join(format!( + ".{}.staging-{}-{nanos}", + platform.binary_name, + std::process::id(), + )); + + { + let mut f = std::fs::File::create(&staging_path).unwrap_or_else(|e| { + let _ = std::fs::remove_file(&staging_path); + panic!( + "failed to create staging file {}: {e}", + staging_path.display() + ); + }); + + if let Err(e) = f.write_all(&bytes) { + let _ = std::fs::remove_file(&staging_path); + panic!( + "failed to write staging file {}: {e}", + staging_path.display() + ); + } + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if let Err(e) = f.set_permissions(std::fs::Permissions::from_mode(0o755)) { + let _ = std::fs::remove_file(&staging_path); + panic!("failed to chmod {}: {e}", staging_path.display()); + } + } + + // Backdate the staged binary to the Unix epoch before it lands. We emit + // `cargo:rerun-if-changed` on `final_path` (see caller) so a *deleted* + // cache binary forces a re-extract — but cargo stamps the build-script + // `output` reference when the script is spawned, seconds before this + // freshly-downloaded binary is written. A current mtime would therefore + // be *newer* than that reference, so the next identical `cargo` + // invocation would see the watched file as "changed" and pointlessly + // rerun build.rs + recompile the crate + relink every downstream crate. + // Pinning to the epoch keeps the file unambiguously older than any real + // build reference; `rename` preserves mtime (same inode), so it lands + // already-backdated and a no-change rebuild stays a true no-op. The + // deleted-file recovery contract is untouched: a missing file can't be + // stat'd, so cargo still treats it as stale and reruns regardless. + // + // Best-effort: a filesystem that refuses the epoch (e.g. FAT's 1980 floor + // clamps it — still older than any real reference) or rejects the call + // just reverts to the pre-fix redundant-rebuild behaviour, never a broken + // build. + if let Err(e) = f.set_modified(std::time::SystemTime::UNIX_EPOCH) { + println!( + "cargo:warning=Could not backdate {} (a redundant rebuild may occur): {e}", + staging_path.display() + ); + } + } + + // Atomic file-replace on both Unix and Windows. If a concurrent build + // already produced the same file the rename overwrites it; the bytes + // are SHA-verified-identical so replacement is safe. + if let Err(e) = std::fs::rename(&staging_path, &final_path) { + let _ = std::fs::remove_file(&staging_path); + panic!( + "failed to rename {} -> {}: {e}", + staging_path.display(), + final_path.display() + ); + } + + // Surface where the binary landed so contributors can find it. Quiet + // on the hot path: the caller's `is_file()` short-circuit (and the + // safety net at the top of this function) means this only fires on a + // true cache miss. + println!( + "cargo:warning=Extracted Copilot CLI to {}", + final_path.display() + ); + + final_path +} + +/// Replace characters outside `[a-zA-Z0-9._-]` with `_` so the version +/// string is always safe to use as a path component. Kept in sync with +/// `embeddedcli::sanitize_version` and `resolve::sanitize_version` so all +/// three resolve to the same cache directory for any given version. +fn sanitize_version(version: &str) -> String { + version + .chars() + .map(|c| match c { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '.' | '-' | '_' => c, + _ => '_', + }) + .collect() +} + +/// Extract the single `binary_name` entry from the release archive. Reused +/// between embed mode's `verify_binary_present_in_archive` and the +/// `extract_to_cache` path used when `bundled-cli` is off. Panics if the +/// entry isn't found — callers have already invoked +/// `verify_binary_present_in_archive`. +fn extract_binary_bytes(archive: &[u8], platform: Platform) -> Vec { + if platform.asset_name.ends_with(".zip") { + let cursor = std::io::Cursor::new(archive); + let mut zip = zip::ZipArchive::new(cursor) + .unwrap_or_else(|e| panic!("failed to open zip archive: {e}")); + for i in 0..zip.len() { + let mut entry = zip + .by_index(i) + .unwrap_or_else(|e| panic!("failed to read zip entry {i}: {e}")); + let name = entry.name().to_string(); + if name == platform.binary_name || name.ends_with(&format!("/{}", platform.binary_name)) + { + let mut bytes = Vec::with_capacity(entry.size() as usize); + std::io::copy(&mut entry, &mut bytes) + .unwrap_or_else(|e| panic!("failed to read zip entry bytes: {e}")); + return bytes; + } + } + } else { + let gz = flate2::read::GzDecoder::new(archive); + let mut tar = tar::Archive::new(gz); + for entry in tar + .entries() + .unwrap_or_else(|e| panic!("failed to read tar entries: {e}")) + { + let mut entry = entry.unwrap_or_else(|e| panic!("failed to read tar entry: {e}")); + let path = entry + .path() + .unwrap_or_else(|e| panic!("failed to read tar entry path: {e}")); + let name = path.to_string_lossy().into_owned(); + if name == platform.binary_name || name.ends_with(&format!("/{}", platform.binary_name)) + { + let mut bytes = Vec::with_capacity(entry.size() as usize); + entry + .read_to_end(&mut bytes) + .unwrap_or_else(|e| panic!("failed to read tar entry bytes: {e}")); + return bytes; + } + } + } + panic!( + "binary `{}` not found in archive `{}`", + platform.binary_name, platform.asset_name + ); +} + +/// Read a file from the download cache, or download it (with retries) and save +/// to cache. Verifies SHA-256 on every path. Evicts stale/corrupt cache entries +/// automatically. Cache I/O failures are treated as cache misses — they never +/// break the build. +fn cached_download( + url: &str, + cache_key: &str, + expected_hash: &str, + cache_dir: &Option, +) -> Vec { + if let Some(dir) = cache_dir { + let cached_path = dir.join(cache_key); + if cached_path.is_file() { + match std::fs::read(&cached_path) { + Ok(data) if hex_sha256(&data) == expected_hash => { + // Silent cache hit — nothing to surface. + return data; + } + Ok(_) => { + println!("cargo:warning=Cached archive hash mismatch, re-downloading"); + let _ = std::fs::remove_file(&cached_path); + } + Err(e) => { + println!( + "cargo:warning=Failed to read cache {}, re-downloading: {e}", + cached_path.display() + ); + } + } + } + } + + println!("cargo:warning=Downloading {url}"); + let data = download_with_retry(url); + let actual_hash = hex_sha256(&data); + if actual_hash != expected_hash { + panic!( + "Archive integrity check failed for {url}!\n expected: {expected_hash}\n actual: {actual_hash}\n \ + This could indicate a corrupted download or a supply-chain attack." + ); + } + + if let Some(dir) = cache_dir { + if let Err(e) = std::fs::create_dir_all(dir) { + println!( + "cargo:warning=Failed to create cache directory {}: {e}", + dir.display() + ); + } else { + let cached_path = dir.join(cache_key); + println!("cargo:warning=Caching archive at {}", cached_path.display()); + if let Err(e) = std::fs::write(&cached_path, &data) { + println!( + "cargo:warning=Failed to write cache file {}: {e}", + cached_path.display() + ); + } + } + } + + data +} + +/// Maximum number of HTTP attempts (one initial + this many retries on transient errors). +const MAX_RETRIES: u32 = 3; + +/// Download `url` with bounded retries on transient network errors. Backoff is +/// exponential starting at 1s. 4xx responses fail fast; 5xx and connect/read +/// errors are retried. +fn download_with_retry(url: &str) -> Vec { + let mut attempt = 0u32; + loop { + attempt += 1; + match try_download(url) { + Ok(bytes) => return bytes, + Err(err) if err.transient && attempt <= MAX_RETRIES => { + let backoff = Duration::from_secs(1u64 << (attempt - 1)); + println!( + "cargo:warning=Transient download failure for {url} (attempt {attempt}/{}): {} — retrying in {}s", + MAX_RETRIES + 1, + err.message, + backoff.as_secs(), + ); + std::thread::sleep(backoff); + } + Err(err) => panic!("Failed to download {url}: {}", err.message), + } + } +} + +struct DownloadError { + message: String, + transient: bool, +} + +fn try_download(url: &str) -> Result, DownloadError> { + let connector = native_tls::TlsConnector::new().map_err(|e| DownloadError { + message: format!("native-tls init error: {e}"), + transient: false, + })?; + let agent = ureq::AgentBuilder::new() + .tls_connector(std::sync::Arc::new(connector)) + .timeout_connect(Duration::from_secs(30)) + .timeout_read(Duration::from_secs(120)) + .build(); + + match agent.get(url).call() { + Ok(response) => { + let mut bytes = Vec::new(); + response + .into_reader() + .read_to_end(&mut bytes) + .map_err(|e| DownloadError { + message: format!("read error: {e}"), + transient: true, + })?; + Ok(bytes) + } + // 5xx — server-side, treat as transient. + Err(ureq::Error::Status(code, response)) if (500..600).contains(&code) => { + Err(DownloadError { + message: format!("HTTP {code} {}", response.status_text()), + transient: true, + }) + } + // 4xx — client-side, fail fast. + Err(ureq::Error::Status(code, response)) => Err(DownloadError { + message: format!("HTTP {code} {}", response.status_text()), + transient: false, + }), + // Transport-layer (DNS, connect, TLS, read timeout) — treat as transient. + Err(ureq::Error::Transport(t)) => Err(DownloadError { + message: format!("transport error: {t}"), + transient: true, + }), + } +} + +fn sha256(data: &[u8]) -> [u8; 32] { + let mut hasher = sha2::Sha256::new(); + hasher.update(data); + hasher.finalize().into() +} + +/// Walks the downloaded archive at build time to confirm an entry matching +/// `binary_name` exists. Panics with a clear message if not — defends against +/// silent breakage if the upstream archive layout ever changes. +fn verify_binary_present_in_archive(archive: &[u8], binary_name: &str, asset_name: &str) { + let found = if asset_name.ends_with(".zip") { + archive_contains_zip_entry(archive, binary_name) + } else { + archive_contains_tar_entry(archive, binary_name) + }; + if !found { + panic!( + "Copilot CLI archive `{asset_name}` does not contain an entry named `{binary_name}`. \ + The upstream archive layout may have changed; runtime extraction would fail. \ + Update `verify_binary_present_in_archive` in build.rs and the matching `extract_binary` in src/embeddedcli.rs." + ); + } +} + +fn archive_contains_tar_entry(targz: &[u8], binary_name: &str) -> bool { + let gz = flate2::read::GzDecoder::new(targz); + let mut archive = tar::Archive::new(gz); + let Ok(entries) = archive.entries() else { + return false; + }; + for entry in entries.flatten() { + let Ok(path) = entry.path() else { + continue; + }; + let name = path.to_string_lossy(); + if name == binary_name || name.ends_with(&format!("/{binary_name}")) { + return true; + } + } + false +} + +fn archive_contains_zip_entry(zip_bytes: &[u8], binary_name: &str) -> bool { + let cursor = std::io::Cursor::new(zip_bytes); + let Ok(mut archive) = zip::ZipArchive::new(cursor) else { + return false; + }; + for i in 0..archive.len() { + let Ok(entry) = archive.by_index(i) else { + continue; + }; + let name = entry.name(); + if name == binary_name || name.ends_with(&format!("/{binary_name}")) { + return true; + } + } + false +} + +fn hex_sha256(data: &[u8]) -> String { + sha256(data).iter().map(|b| format!("{b:02x}")).collect() +} diff --git a/rust/scripts/snapshot-bundled-cli-version.sh b/rust/scripts/snapshot-bundled-cli-version.sh index d6c9b82edb..08b19ebc2b 100755 --- a/rust/scripts/snapshot-bundled-cli-version.sh +++ b/rust/scripts/snapshot-bundled-cli-version.sh @@ -34,14 +34,12 @@ CHECKSUMS_URL="https://github.com/github/copilot-cli/releases/download/v${VERSIO SHA256SUMS="$(curl --fail --silent --show-error --location --retry 3 "${CHECKSUMS_URL}")" ASSETS=( - "github-copilot-${VERSION}-darwin-arm64.tgz" - "github-copilot-${VERSION}-darwin-x64.tgz" - "github-copilot-${VERSION}-linux-arm64.tgz" - "github-copilot-${VERSION}-linux-x64.tgz" - "github-copilot-${VERSION}-linuxmusl-arm64.tgz" - "github-copilot-${VERSION}-linuxmusl-x64.tgz" - "github-copilot-${VERSION}-win32-arm64.tgz" - "github-copilot-${VERSION}-win32-x64.tgz" + "copilot-darwin-arm64.tar.gz" + "copilot-darwin-x64.tar.gz" + "copilot-linux-arm64.tar.gz" + "copilot-linux-x64.tar.gz" + "copilot-win32-arm64.zip" + "copilot-win32-x64.zip" ) TEMP_OUTPUT="${OUTPUT}.tmp.$$" diff --git a/rust/scripts/snapshot-bundled-in-process-version.sh b/rust/scripts/snapshot-bundled-in-process-version.sh new file mode 100755 index 0000000000..9fe2298c78 --- /dev/null +++ b/rust/scripts/snapshot-bundled-in-process-version.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# +# Snapshot the Copilot CLI version + per-platform release hashes for the +# rust crate's bundled-in-process build path. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RUST_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +REPO_ROOT="$(cd "${RUST_DIR}/.." && pwd)" +PACKAGE_FILE="${REPO_ROOT}/nodejs/package.json" +OUTPUT="${RUST_DIR}/cli-version-in-process.txt" + +if [[ ! -f "${PACKAGE_FILE}" ]]; then + echo "error: ${PACKAGE_FILE} not found" >&2 + exit 1 +fi + +VERSION="$(node -e "console.log(require('${PACKAGE_FILE}').copilotCliVersion)")" +if [[ -z "${VERSION}" ]]; then + echo "error: could not read copilotCliVersion from ${PACKAGE_FILE}" >&2 + exit 1 +fi +CHECKSUMS_URL="https://github.com/github/copilot-cli/releases/download/v${VERSION}/SHA256SUMS.txt" +SHA256SUMS="$(curl --fail --silent --show-error --location --retry 3 "${CHECKSUMS_URL}")" + +PACKAGES=( + "copilot-darwin-arm64" + "copilot-darwin-x64" + "copilot-linux-arm64" + "copilot-linux-x64" + "copilot-linuxmusl-arm64" + "copilot-linuxmusl-x64" + "copilot-win32-arm64" + "copilot-win32-x64" +) + +TEMP_OUTPUT="${OUTPUT}.tmp.$$" +trap 'rm -f "${TEMP_OUTPUT}"' EXIT +{ + echo "# Auto-generated by rust/scripts/snapshot-bundled-in-process-version.sh" + echo "# Do not edit. Regenerated by the publish workflow on every release." + echo "version=${VERSION}" + for package in "${PACKAGES[@]}"; do + platform="${package#copilot-}" + asset="github-copilot-${VERSION}-${platform}.tgz" + hash="$(printf '%s\n' "${SHA256SUMS}" | awk -v asset="${asset}" '$2 == asset || $2 == "*" asset { print $1; exit }')" + if [[ -z "${hash}" ]]; then + echo "error: SHA256SUMS.txt does not contain ${asset}" >&2 + exit 1 + fi + echo "${package}=${hash}" + done +} > "${TEMP_OUTPUT}" +mv "${TEMP_OUTPUT}" "${OUTPUT}" +trap - EXIT + +echo "Wrote ${OUTPUT} (version=${VERSION}, ${#PACKAGES[@]} hashes)" diff --git a/rust/src/embeddedcli.rs b/rust/src/embeddedcli.rs index 3618dca08a..3cc527a2e2 100644 --- a/rust/src/embeddedcli.rs +++ b/rust/src/embeddedcli.rs @@ -2,12 +2,11 @@ //! crate (gated on the `bundled-cli` cargo feature, which is in the default //! feature set). //! -//! All bundled builds embed a filtered archive derived from the platform's -//! `github-copilot--.tgz` GitHub Release asset. It contains -//! the CLI executable, runtime wrapper, and auxiliary runtime assets; enabling -//! `bundled-in-process` additionally includes the native runtime library. -//! Extraction to a real on-disk path is deferred until the relevant installer -//! is called. +//! Normal builds embed the platform release archive from GitHub Releases. +//! Builds with `bundled-in-process` instead embed a filtered archive from the +//! platform npm package containing the CLI executable, runtime wrapper, native +//! runtime artifacts, and auxiliary runtime assets. Extraction to a real +//! on-disk path is deferred until the relevant installer is called. //! //! The embedded bytes are part of the consumer's signed binary and therefore //! trusted *as the source of truth* — but the bytes that land on disk are not. diff --git a/rust/tests/cli_resolution_test.rs b/rust/tests/cli_resolution_test.rs index 74daeaf246..847ac7a4d0 100644 --- a/rust/tests/cli_resolution_test.rs +++ b/rust/tests/cli_resolution_test.rs @@ -197,8 +197,8 @@ async fn extract_dir_runtime_override_is_honored() { let _ = fake; } -/// Build-time version pins, when present, must contain the release asset -/// checksum for every supported target. +/// Build-time version pins, when present, must match the selected bundling +/// implementation's checksum format. /// When absent, build.rs falls through to `../nodejs/package.json` and /// the release's `SHA256SUMS.txt` — /// both are accepted, this test only checks the pin file's format if it's @@ -206,15 +206,19 @@ async fn extract_dir_runtime_override_is_honored() { #[test] fn pin_file_when_present_is_well_formed() { let manifest_dir = env!("CARGO_MANIFEST_DIR"); - let filename = "cli-version.txt"; + let (filename, expected_package_count) = if cfg!(feature = "bundled-in-process") { + ("cli-version-in-process.txt", 8) + } else { + ("cli-version.txt", 6) + }; let pin = PathBuf::from(manifest_dir).join(filename); if !pin.is_file() { // Contributor build path — no assertion needed. return; } let contents = std::fs::read_to_string(&pin).expect("read CLI version snapshot"); - let mut version = None; - let mut assets = Vec::new(); + let mut saw_version = false; + let mut package_count = 0; for raw in contents.lines() { let line = raw.trim(); if line.is_empty() || line.starts_with('#') { @@ -225,7 +229,7 @@ fn pin_file_when_present_is_well_formed() { .unwrap_or_else(|| panic!("malformed line: {raw:?}")); assert!(!value.trim().is_empty(), "empty value for key {key:?}"); if key.trim() == "version" { - version = Some(value.trim().to_string()); + saw_version = true; } else { assert_eq!( value.trim().len(), @@ -236,22 +240,11 @@ fn pin_file_when_present_is_well_formed() { value.trim().bytes().all(|byte| byte.is_ascii_hexdigit()), "invalid SHA-256 hash for key {key:?}" ); - assets.push(key.trim().to_string()); + package_count += 1; } } - let version = version.unwrap_or_else(|| panic!("{filename} missing `version=` line")); - let expected_assets = [ - "darwin-arm64", - "darwin-x64", - "linux-arm64", - "linux-x64", - "linuxmusl-arm64", - "linuxmusl-x64", - "win32-arm64", - "win32-x64", - ] - .map(|platform| format!("github-copilot-{version}-{platform}.tgz")); - assert_eq!(assets, expected_assets); + assert!(saw_version, "{filename} missing `version=` line"); + assert_eq!(package_count, expected_package_count); } /// With `bundled-cli` on AND a supported target, `install_bundled_cli` From 7885c5fdbec9f8b1b9da8132ef30bd91879dddf8 Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Thu, 3 Sep 2026 17:10:41 +0200 Subject: [PATCH 3/6] Remove Java Copilot npm consumption Fetch Java codegen schemas from the checksum-verified CLI release artifact and use the shared release-backed CLI setup for smoke tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: daec7b3b-799c-4396-b372-6eb638d0faf8 --- .github/actions/setup-copilot/action.yml | 13 +- ...en-code-to-accept-upgrade-changes.lock.yml | 8 +- ...dwritten-code-to-accept-upgrade-changes.md | 9 +- .github/workflows/java-codegen-check.yml | 12 +- .github/workflows/java-codegen-fix.lock.yml | 2 +- .github/workflows/java-codegen-fix.md | 22 +-- .github/workflows/java-smoke-test.yml | 52 +----- .../workflows/update-copilot-dependency.yml | 21 +-- java/copilot-native/pom.xml | 2 +- java/copilot-native/scripts/fetch-native.mjs | 11 +- java/pom.xml | 9 - java/scripts/codegen/fetch-schemas.mjs | 127 ++++++++++++++ java/scripts/codegen/fetch-schemas.test.mjs | 77 +++++++++ java/scripts/codegen/java.ts | 52 +----- java/scripts/codegen/package-lock.json | 160 ------------------ java/scripts/codegen/package.json | 7 +- java/sdk/pom.xml | 51 ------ .../copilot/ffi/NativeRuntimeLoader.java | 5 +- 18 files changed, 271 insertions(+), 369 deletions(-) create mode 100644 java/scripts/codegen/fetch-schemas.mjs create mode 100644 java/scripts/codegen/fetch-schemas.test.mjs diff --git a/.github/actions/setup-copilot/action.yml b/.github/actions/setup-copilot/action.yml index 506f472ca2..a9c39a2a0a 100644 --- a/.github/actions/setup-copilot/action.yml +++ b/.github/actions/setup-copilot/action.yml @@ -4,6 +4,9 @@ outputs: cli-path: description: "Path to the Copilot CLI" value: ${{ steps.cli-path.outputs.path }} + javascript-cli-path: + description: "Path to the JavaScript Copilot CLI entrypoint" + value: ${{ steps.cli-path.outputs.javascript-path }} runs: using: "composite" steps: @@ -28,10 +31,14 @@ runs: echo "Could not prepare the Copilot CLI runtime" >&2 exit 1 fi + javascript_cli_path=$(npm --prefix "$(pwd)/nodejs" run --silent prepare:runtime -- --print-legacy-path) + if [ -z "$javascript_cli_path" ]; then + echo "Could not prepare the Copilot CLI JavaScript entrypoint" >&2 + exit 1 + fi echo "path=$cli_path" >> $GITHUB_OUTPUT + echo "javascript-path=$javascript_cli_path" >> $GITHUB_OUTPUT shell: bash - name: Verify CLI works - run: | - legacy_cli=$(npm --prefix "$(pwd)/nodejs" run --silent prepare:runtime -- --print-legacy-path) - node "$legacy_cli" --version + run: node "${{ steps.cli-path.outputs.javascript-path }}" --version shell: bash diff --git a/.github/workflows/java-adapt-handwritten-code-to-accept-upgrade-changes.lock.yml b/.github/workflows/java-adapt-handwritten-code-to-accept-upgrade-changes.lock.yml index 7b7d524fe7..7554b0bb76 100644 --- a/.github/workflows/java-adapt-handwritten-code-to-accept-upgrade-changes.lock.yml +++ b/.github/workflows/java-adapt-handwritten-code-to-accept-upgrade-changes.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"a5f19a89f89b0693f86ca89ea90e3a633fe19c17bf4d27214fa9124429cdc156","body_hash":"8db09798070cbcba22c42c50a316ae45c8e8c650eeb23c556b44fde8d519550a","compiler_version":"v0.88.2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"03beaef805d9bdf4b878b9fd1dd47c62793f30f179373406e081b88a9817b182","body_hash":"f535cb24328c6e9b3963e72de09404b08a4f4580a8174bc6ad580c0f005837ae","compiler_version":"v0.88.2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_CI_TRIGGER_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"9271a1804551c0dc4fb0085a97979950aa2f8489","version":"v0.88.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12","digest":"sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12@sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12","digest":"sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12@sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12","digest":"sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12@sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.15","digest":"sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.15@sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e","pinned_image":"ghcr.io/github/gh-aw-node@sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_release_by_tag","get_tag","list_branches","list_commits","list_releases","list_starred_repositories","list_tags","search_code","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","missing_data","missing_tool","noop","push_to_pull_request_branch"]}]} # This file was automatically generated by gh-aw (v0.88.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -24,7 +24,7 @@ # For more information: https://github.github.com/gh-aw/introduction/overview/ # # Adapt handwritten Java SDK code to work with regenerated types after a -# @github/copilot version bump. Assumes codegen succeeded and generated code +# Copilot CLI release update. Assumes codegen succeeded and generated code # compiles. Fixes handwritten source and tests only. # # Secrets used: @@ -1516,7 +1516,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: WORKFLOW_NAME: "Java Handwritten Code Adaptation After CLI Upgrade" - WORKFLOW_DESCRIPTION: "Adapt handwritten Java SDK code to work with regenerated types after a\n@github/copilot version bump. Assumes codegen succeeded and generated code\ncompiles. Fixes handwritten source and tests only." + WORKFLOW_DESCRIPTION: "Adapt handwritten Java SDK code to work with regenerated types after a\nCopilot CLI release update. Assumes codegen succeeded and generated code\ncompiles. Fixes handwritten source and tests only." HAS_PATCH: ${{ needs.agent.outputs.has_patch }} GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" GH_AW_DETECTION_SKIP_PROMPT_SUMMARY: "true" @@ -1580,7 +1580,7 @@ jobs: S2STOKENS: true TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} WORKFLOW_NAME: "Java Handwritten Code Adaptation After CLI Upgrade" - WORKFLOW_DESCRIPTION: "Adapt handwritten Java SDK code to work with regenerated types after a\n@github/copilot version bump. Assumes codegen succeeded and generated code\ncompiles. Fixes handwritten source and tests only." + WORKFLOW_DESCRIPTION: "Adapt handwritten Java SDK code to work with regenerated types after a\nCopilot CLI release update. Assumes codegen succeeded and generated code\ncompiles. Fixes handwritten source and tests only." HAS_PATCH: ${{ needs.agent.outputs.has_patch }} GH_AW_DETECTION_CONTINUE_ON_ERROR: "true" run: | diff --git a/.github/workflows/java-adapt-handwritten-code-to-accept-upgrade-changes.md b/.github/workflows/java-adapt-handwritten-code-to-accept-upgrade-changes.md index dd1bfe2bbc..91f11d1f86 100644 --- a/.github/workflows/java-adapt-handwritten-code-to-accept-upgrade-changes.md +++ b/.github/workflows/java-adapt-handwritten-code-to-accept-upgrade-changes.md @@ -1,7 +1,7 @@ --- description: | Adapt handwritten Java SDK code to work with regenerated types after a - @github/copilot version bump. Assumes codegen succeeded and generated code + Copilot CLI release update. Assumes codegen succeeded and generated code compiles. Fixes handwritten source and tests only. on: @@ -45,14 +45,13 @@ safe-outputs: # Java Handwritten Code Adaptation After CLI Upgrade -You are an automation agent that fixes handwritten Java SDK source and test code after a `@github/copilot` version bump has regenerated the typed schemas. +You are an automation agent that fixes handwritten Java SDK source and test code after a Copilot CLI release update has regenerated the typed schemas. ## Assumptions - The branch `${{ inputs.branch }}` already has: - - Updated `java/scripts/codegen/package.json` with the new version + - Updated the shared CLI release pin in `nodejs/package.json` - Regenerated `java/sdk/src/generated/java/` code that compiles successfully - - Updated the Java POM CLI/version pin property - Your job is ONLY to fix **handwritten** code, NOT generated code. ## Boundaries @@ -147,7 +146,7 @@ If this passes, commit and push: ```bash git add java/sdk/src/main/java java/sdk/src/test/java -git commit -m "Fix handwritten Java code for @github/copilot schema changes +git commit -m "Fix handwritten Java code for CLI schema changes Adapt constructor calls, enum references, and test assertions to match regenerated types after CLI version bump." diff --git a/.github/workflows/java-codegen-check.yml b/.github/workflows/java-codegen-check.yml index f2f4527966..e490a5cf4e 100644 --- a/.github/workflows/java-codegen-check.yml +++ b/.github/workflows/java-codegen-check.yml @@ -5,11 +5,13 @@ on: branches: - main paths: + - 'nodejs/package.json' - 'java/scripts/codegen/**' - 'java/sdk/src/generated/**' - '.github/workflows/java-codegen-check.yml' pull_request: paths: + - 'nodejs/package.json' - 'java/scripts/codegen/**' - 'java/sdk/src/generated/**' - '.github/workflows/java-codegen-check.yml' @@ -48,9 +50,13 @@ jobs: working-directory: ./java/scripts/codegen run: npm ci + - name: Test schema fetcher + working-directory: ./java/scripts/codegen + run: npm test + - name: Run codegen working-directory: ./java/scripts/codegen - run: npx tsx java.ts + run: npm run generate - name: Check for uncommitted changes id: check-changes @@ -68,7 +74,7 @@ jobs: - name: Fail on stale generated files (push to main) if: steps.check-changes.outputs.changed == 'true' && github.event_name != 'pull_request' run: | - echo "::error::Generated files are out of date. Run 'cd java/scripts/codegen && npx tsx java.ts' and commit the changes." + echo "::error::Generated files are out of date. Run 'cd java/scripts/codegen && npm run generate' and commit the changes." git diff exit 1 @@ -93,7 +99,7 @@ jobs: if: steps.push-regen.outcome == 'failure' run: | echo "::error::Could not push regenerated files to the PR branch. This is expected for Dependabot PRs (read-only token) and fork PRs." - echo "To fix: check out this PR branch locally, run 'cd java/scripts/codegen && npx tsx java.ts', commit, and push." + echo "To fix: check out this PR branch locally, run 'cd java/scripts/codegen && npm run generate', commit, and push." exit 1 - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 diff --git a/.github/workflows/java-codegen-fix.lock.yml b/.github/workflows/java-codegen-fix.lock.yml index 91c4ea1816..fa64a0fe88 100644 --- a/.github/workflows/java-codegen-fix.lock.yml +++ b/.github/workflows/java-codegen-fix.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"b0390c9ab9beb0d7e106314299e89e486269ab7f64d8489d5021132d79aa6b9b","body_hash":"63d6ce13a5131b158ddffb10a469aa59e0fdc2278eec4d8de7f6763e0b6f2ea2","compiler_version":"v0.88.2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"b0390c9ab9beb0d7e106314299e89e486269ab7f64d8489d5021132d79aa6b9b","body_hash":"c7cc7984b1d512e871371a7fd99bc4f7e4615d4192348170eca763cf584f4855","compiler_version":"v0.88.2","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.80"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_CI_TRIGGER_TOKEN","GH_AW_DEFAULT_OTLP_HEADERS","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"9271a1804551c0dc4fb0085a97979950aa2f8489","version":"v0.88.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12","digest":"sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.28.12@sha256:390051be4ed1847f774fd8980b61d3a3523574c0175d00c3fc7cdf2002a88202"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12","digest":"sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.28.12@sha256:d7d533d87c80d87ff91ac0e21e9299055c3beedff1536262b97ed700fb065a32"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12","digest":"sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.28.12@sha256:52c34aca98d2a6833c329f1505912a6949c4fda16618c010c979bd59ea99254f"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.15","digest":"sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.15@sha256:60cd97533e93d8e7be36b979c0f08a70846189bda6190f28bbd6d427bc0d9b6e"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e","pinned_image":"ghcr.io/github/gh-aw-node@sha256:bac2192f6374d6262116399b34fc5e143d576f82719e90a18261cae7480f4d4e"},{"image":"ghcr.io/github/github-mcp-server:v1.11.0","digest":"sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699","pinned_image":"ghcr.io/github/github-mcp-server:v1.11.0@sha256:fbec75de11c255213fa08d80fb166abe73d851fff631c51c0079872967720699"}],"mcp_servers":[{"name":"github","tools":["get_commit","get_file_contents","get_latest_release","get_me","get_release_by_tag","get_tag","list_branches","list_commits","list_releases","list_starred_repositories","list_tags","search_code","search_repositories"]},{"name":"safeoutputs","tools":["add_comment","missing_data","missing_tool","noop","push_to_pull_request_branch"]}]} # This file was automatically generated by gh-aw (v0.88.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # diff --git a/.github/workflows/java-codegen-fix.md b/.github/workflows/java-codegen-fix.md index b1dcb1f636..8f6a845433 100644 --- a/.github/workflows/java-codegen-fix.md +++ b/.github/workflows/java-codegen-fix.md @@ -52,7 +52,7 @@ You are an automation agent that fixes Java compilation and test failures caused ## Context -A Dependabot PR bumped the `@github/copilot` npm dependency in `java/scripts/codegen/package.json`. The `java-codegen-check` workflow ran the code generator (`java/scripts/codegen/java.ts`) against the new schemas and `mvn verify` subsequently failed. Your job is to fix **both** the code generator script (if needed) and the handwritten SDK/test source code so the build passes. +A Copilot CLI release pin update fetched new schemas from GitHub Releases. The `java-codegen-check` workflow ran the code generator (`java/scripts/codegen/java.ts`) against those schemas and `mvn verify` subsequently failed. Your job is to fix **both** the code generator script (if needed) and the handwritten SDK/test source code so the build passes. **❌❌❌ YOU MUST NEVER EDIT any of the java source code in `java/sdk/src/generated/` directly.** ✅✅Rather, the way to affect changes in these files is to change the code generator script and re-generate the classes in `java/sdk/src/generated`. @@ -66,9 +66,9 @@ ${{ inputs.error_summary }} ## Architecture overview -The code generator (`java/scripts/codegen/java.ts`) reads JSON schemas from `node_modules/@github/copilot/schemas/` and produces Java source files under `java/sdk/src/generated/java/`. These generated types are consumed by handwritten code in `java/sdk/src/main/java/` (primarily `CopilotSession.java`) and tested by handwritten tests in `java/sdk/src/test/java/`. +The code generator (`java/scripts/codegen/java.ts`) reads JSON schemas from `java/scripts/codegen/target/schemas/`. The schemas are extracted from the pinned `github-copilot--linux-x64.tgz` GitHub Release asset by `fetch-schemas.mjs`. The generator produces Java source files under `java/sdk/src/generated/java/`. These generated types are consumed by handwritten code in `java/sdk/src/main/java/` (primarily `CopilotSession.java`) and tested by handwritten tests in `java/sdk/src/test/java/`. -When `@github/copilot` is bumped, the schemas may change in ways the code generator does not yet handle. Common schema changes include: +When the Copilot CLI release pin is bumped, the schemas may change in ways the code generator does not yet handle. Common schema changes include: - **`$ref` references**: Inline nested type definitions replaced with `$ref` pointers to `#/definitions/` entries. The code generator must resolve these references and emit standalone Java types instead of nested records. - **Field type changes**: Numeric fields changing between `double`, `Long`, `int`, etc. @@ -97,10 +97,10 @@ mvn --version node --version ``` -Install codegen dependencies: +Install codegen dependencies and fetch the pinned release schemas: ```bash -cd java/scripts/codegen && npm ci && cd ../../.. +cd java/scripts/codegen && npm ci && npm run fetch:schemas && cd ../../.. ``` ### Step 1: Reproduce the failure @@ -135,13 +135,13 @@ To diagnose, compare the current schemas with the generated output: ```bash # List available schemas -ls java/scripts/codegen/node_modules/@github/copilot/schemas/ +ls java/scripts/codegen/target/schemas/ # Check for $ref usage in schemas (indicates the codegen may need $ref resolution) -grep -r '"$ref"' java/scripts/codegen/node_modules/@github/copilot/schemas/ | head -20 +grep -r '"$ref"' java/scripts/codegen/target/schemas/ | head -20 # Look at a specific schema that relates to failing types -cat java/scripts/codegen/node_modules/@github/copilot/schemas/.json | head -80 +head -80 java/scripts/codegen/target/schemas/.json ``` ### Step 3: Fix the code generator (if needed) @@ -157,7 +157,7 @@ If the diagnosis shows the code generator does not handle the new schema format: 3. **Re-run code generation** to produce updated generated files: ```bash - cd java/scripts/codegen && npx tsx java.ts && cd ../../.. + cd java/scripts/codegen && npm run generate && cd ../../.. ``` 4. **Verify the generated output** looks reasonable: @@ -213,7 +213,7 @@ After `mvn verify` passes, commit all changes and use the `push-to-pull-request- ```bash git add -A -git commit -m "Fix Java codegen and build failures after @github/copilot update +git commit -m "Fix Java codegen and build failures after CLI update Automated fix applied by java-codegen-fix workflow." ``` @@ -236,7 +236,7 @@ Do **NOT** push broken code. ## Important constraints -- **NEVER** hand-edit files under `java/sdk/src/generated/java/` — these are auto-generated. They are updated by running `cd java/scripts/codegen && npx tsx java.ts`. +- **NEVER** hand-edit files under `java/sdk/src/generated/java/` — these are auto-generated. They are updated by running `cd java/scripts/codegen && npm run generate`. - **NEVER** modify `java/sdk/pom.xml` — build config is not in scope - **NEVER** modify `java/scripts/codegen/package.json` or `java/scripts/codegen/package-lock.json` — dependency versions are not in scope - **NEVER** modify files under `.github/` — workflow files are not in scope diff --git a/.github/workflows/java-smoke-test.yml b/.github/workflows/java-smoke-test.yml index e7e9a417d2..5f808f8d0d 100644 --- a/.github/workflows/java-smoke-test.yml +++ b/.github/workflows/java-smoke-test.yml @@ -29,27 +29,8 @@ jobs: distribution: "microsoft" cache: "maven" - - uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v6 - with: - node-version: 22 - - - name: Read pinned @github/copilot version from pom.xml - id: cli-version - run: | - PROP="readonly-copilot-sdk-ref-impl-version-from-lastmerge-file-updated-by-reference-impl-sync" - VERSION=$(sed -n "s|.*<${PROP}>\(.*\).*|\1|p" pom.xml | head -n 1 | tr -d '[:space:]') - if [[ -z "$VERSION" || "$VERSION" == "PRIMER_TO_REPLACE" ]]; then - echo "::error::Could not read pinned @github/copilot version from pom.xml property <${PROP}>" >&2 - exit 1 - fi - echo "Pinned @github/copilot version: $VERSION" - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - - - name: Install Copilot CLI globally (pinned to pom.xml version) - run: npm install -g "@github/copilot@${{ steps.cli-version.outputs.version }}" - - - name: Verify CLI works - run: copilot --version + - uses: ./.github/actions/setup-copilot + id: setup-copilot - name: Build SDK and install to local repo run: mvn -DskipTests -Pskip-test-harness clean install @@ -57,6 +38,7 @@ jobs: - name: Create and run smoke test via Copilot CLI env: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_CLI_JS: ${{ steps.setup-copilot.outputs.javascript-cli-path }} run: | cat > /tmp/smoke-test-prompt.txt << 'PROMPT_EOF' You are running inside the copilot-sdk monorepo, in the java/ subdirectory. @@ -74,7 +56,7 @@ jobs: If any step fails, exit with a non-zero exit code. Do not silently fix errors. PROMPT_EOF - copilot --yolo --prompt "$(cat /tmp/smoke-test-prompt.txt)" + node "$COPILOT_CLI_JS" --yolo --prompt "$(cat /tmp/smoke-test-prompt.txt)" - name: Run smoke test jar env: @@ -102,27 +84,8 @@ jobs: distribution: "microsoft" cache: "maven" - - uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v6 - with: - node-version: 22 - - - name: Read pinned @github/copilot version from pom.xml - id: cli-version - run: | - PROP="readonly-copilot-sdk-ref-impl-version-from-lastmerge-file-updated-by-reference-impl-sync" - VERSION=$(sed -n "s|.*<${PROP}>\(.*\).*|\1|p" pom.xml | head -n 1 | tr -d '[:space:]') - if [[ -z "$VERSION" || "$VERSION" == "PRIMER_TO_REPLACE" ]]; then - echo "::error::Could not read pinned @github/copilot version from pom.xml property <${PROP}>" >&2 - exit 1 - fi - echo "Pinned @github/copilot version: $VERSION" - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - - - name: Install Copilot CLI globally (pinned to pom.xml version) - run: npm install -g "@github/copilot@${{ steps.cli-version.outputs.version }}" - - - name: Verify CLI works - run: copilot --version + - uses: ./.github/actions/setup-copilot + id: setup-copilot - name: Build SDK and install to local repo run: mvn -DskipTests -Pskip-test-harness clean install @@ -130,6 +93,7 @@ jobs: - name: Create and run smoke test via Copilot CLI env: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_CLI_JS: ${{ steps.setup-copilot.outputs.javascript-cli-path }} run: | cat > /tmp/smoke-test-prompt.txt << 'PROMPT_EOF' You are running inside the copilot-sdk monorepo, in the java/ subdirectory. @@ -150,7 +114,7 @@ jobs: If any step fails, exit with a non-zero exit code. Do not silently fix errors. PROMPT_EOF - copilot --yolo --prompt "$(cat /tmp/smoke-test-prompt.txt)" + node "$COPILOT_CLI_JS" --yolo --prompt "$(cat /tmp/smoke-test-prompt.txt)" - name: Run smoke test jar env: diff --git a/.github/workflows/update-copilot-dependency.yml b/.github/workflows/update-copilot-dependency.yml index fb4a51f43f..f1e4d9fca9 100644 --- a/.github/workflows/update-copilot-dependency.yml +++ b/.github/workflows/update-copilot-dependency.yml @@ -87,25 +87,6 @@ jobs: java-version: "25" distribution: "microsoft" - - name: Update @github/copilot in Java codegen - env: - VERSION: ${{ inputs.version }} - working-directory: ./java/scripts/codegen - run: npm install "@github/copilot@$VERSION" - - - name: Update Java POM CLI version property - env: - VERSION: ${{ inputs.version }} - working-directory: ./java - run: | - PROP="readonly-copilot-sdk-ref-impl-version-from-lastmerge-file-updated-by-reference-impl-sync" - sed -i -E "s|(<${PROP}>)[^<]*()|\1^${VERSION}\2|" pom.xml - # Use fixed-string matching (-F) because npm versions contain regex - # metacharacters: '^' (caret ranges) and '.' (dots in semver) would - # otherwise be interpreted as start-of-line and any-char respectively, - # causing false negatives or spurious matches. - grep -qF "<${PROP}>^${VERSION}" pom.xml - - name: Run Java codegen working-directory: ./java run: mvn generate-sources -Pcodegen @@ -177,7 +158,7 @@ jobs: - Validated the release assets listed in `SHA256SUMS.txt` - Re-ran all code generators (`scripts/codegen`) - Formatted generated output - - Updated Java codegen dependency, POM property, and regenerated Java types + - Regenerated Java types from the pinned CLI release schemas ### Java Handwritten Code Adaptation Plan diff --git a/java/copilot-native/pom.xml b/java/copilot-native/pom.xml index 0abe03ad18..4270d238d4 100644 --- a/java/copilot-native/pom.xml +++ b/java/copilot-native/pom.xml @@ -59,7 +59,7 @@ ${project.basedir}/.. - - ^1.0.83-5 true diff --git a/java/scripts/codegen/fetch-schemas.mjs b/java/scripts/codegen/fetch-schemas.mjs new file mode 100644 index 0000000000..d1bcb4c0b7 --- /dev/null +++ b/java/scripts/codegen/fetch-schemas.mjs @@ -0,0 +1,127 @@ +#!/usr/bin/env node + +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import { createHash } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const scriptDir = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(scriptDir, '../../..'); +const packagePath = path.join(repoRoot, 'nodejs', 'package.json'); +const outputDir = path.resolve( + process.env.COPILOT_CLI_SCHEMA_OUTPUT ?? path.join(scriptDir, 'target', 'schemas'), +); +// Schemas are platform-independent; use one asset consistently on every codegen host. +const platform = process.env.COPILOT_CLI_SCHEMA_PLATFORM ?? 'linux-x64'; +const version = + process.env.COPILOT_CLI_VERSION ?? + JSON.parse(fs.readFileSync(packagePath, 'utf8')).copilotCliVersion; + +if (!version) { + throw new Error(`Could not find copilotCliVersion in ${packagePath}`); +} + +const assetName = `github-copilot-${version}-${platform}.tgz`; +const releaseBase = ( + process.env.COPILOT_CLI_DOWNLOAD_BASE_URL ?? + 'https://github.com/github/copilot-cli/releases/download' +).replace(/\/+$/, ''); + +let archive; +let expectedHash; +if (process.env.COPILOT_CLI_RELEASE_TARBALL) { + archive = fs.readFileSync(process.env.COPILOT_CLI_RELEASE_TARBALL); + expectedHash = process.env.COPILOT_CLI_RELEASE_SHA256; +} else { + const releaseUrl = `${releaseBase}/v${version}`; + const checksums = (await download(`${releaseUrl}/SHA256SUMS.txt`)).toString('utf8'); + expectedHash = findChecksum(checksums, assetName); + archive = await download(`${releaseUrl}/${assetName}`); +} + +if (!expectedHash || !/^[a-fA-F0-9]{64}$/.test(expectedHash)) { + throw new Error(`Missing or invalid SHA-256 for ${assetName}`); +} +const actualHash = createHash('sha256').update(archive).digest('hex'); +if (actualHash !== expectedHash.toLowerCase()) { + throw new Error( + `Integrity verification failed for ${assetName}: expected ${expectedHash}, got ${actualHash}`, + ); +} + +const schemaNames = ['api.schema.json', 'session-events.schema.json']; +const members = execFileSync('tar', ['-tzf', '-'], { + encoding: 'utf8', + input: archive, + maxBuffer: 512 * 1024 * 1024, +}) + .split(/\r?\n/) + .filter(Boolean); +const outputParent = path.dirname(outputDir); +fs.mkdirSync(outputParent, { recursive: true }); +const stagingDir = fs.mkdtempSync(path.join(outputParent, '.schemas-')); + +try { + for (const schemaName of schemaNames) { + const member = `package/schemas/${schemaName}`; + if (members.filter((candidate) => candidate === member).length !== 1) { + throw new Error(`${assetName} must contain exactly one ${member}`); + } + const contents = execFileSync('tar', ['-xOzf', '-', member], { + encoding: null, + input: archive, + maxBuffer: 512 * 1024 * 1024, + }); + JSON.parse(contents.toString('utf8')); + fs.writeFileSync(path.join(stagingDir, schemaName), contents); + } + + fs.rmSync(outputDir, { recursive: true, force: true }); + fs.renameSync(stagingDir, outputDir); +} finally { + fs.rmSync(stagingDir, { recursive: true, force: true }); +} + +console.log(`Staged Copilot CLI ${version} schemas at ${outputDir}`); + +async function download(url) { + let lastError; + for (let attempt = 0; attempt < 3; attempt++) { + try { + // lgtm[js/file-access-to-http] The repository-pinned CLI version selects the release asset. + const response = await fetch(url, { signal: AbortSignal.timeout(600_000) }); + if (response.ok) { + return Buffer.from(await response.arrayBuffer()); + } + await response.body?.cancel(); + lastError = new Error(`${response.status} ${response.statusText}`); + if (response.status >= 400 && response.status < 500 && response.status !== 408 && response.status !== 429) { + break; + } + } catch (error) { + lastError = error; + } + if (attempt < 2) { + await new Promise((resolve) => setTimeout(resolve, 2 ** attempt * 1000)); + } + } + throw new Error(`Failed to download ${url}: ${lastError}`); +} + +function findChecksum(checksums, expectedAssetName) { + for (const line of checksums.split(/\r?\n/)) { + const [hash, name] = line.trim().split(/\s+/, 2); + if ( + name?.replace(/^\*/, '') === expectedAssetName && + /^[a-fA-F0-9]{64}$/.test(hash) + ) { + return hash.toLowerCase(); + } + } + throw new Error(`SHA256SUMS.txt does not contain ${expectedAssetName}`); +} diff --git a/java/scripts/codegen/fetch-schemas.test.mjs b/java/scripts/codegen/fetch-schemas.test.mjs new file mode 100644 index 0000000000..19d7ba6715 --- /dev/null +++ b/java/scripts/codegen/fetch-schemas.test.mjs @@ -0,0 +1,77 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { execFileSync, spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; + +const scriptPath = path.join(path.dirname(fileURLToPath(import.meta.url)), 'fetch-schemas.mjs'); + +test('extracts schemas from a verified release archive', (t) => { + const fixture = createFixture(t); + const outputDir = path.join(fixture.root, 'output'); + const result = runFetch(fixture, outputDir); + + assert.equal(result.status, 0, result.stderr); + assert.deepEqual( + JSON.parse(fs.readFileSync(path.join(outputDir, 'api.schema.json'), 'utf8')), + { title: 'API' }, + ); + assert.deepEqual( + JSON.parse(fs.readFileSync(path.join(outputDir, 'session-events.schema.json'), 'utf8')), + { title: 'Events' }, + ); +}); + +test('rejects an archive with the wrong checksum', (t) => { + const fixture = createFixture(t); + const result = runFetch( + { ...fixture, hash: '0'.repeat(64) }, + path.join(fixture.root, 'output'), + ); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Integrity verification failed/); +}); + +test('requires both schema files', (t) => { + const fixture = createFixture(t, { includeEvents: false }); + const result = runFetch(fixture, path.join(fixture.root, 'output')); + + assert.notEqual(result.status, 0); + assert.match(result.stderr, /must contain exactly one package\/schemas\/session-events\.schema\.json/); +}); + +function createFixture(t, { includeEvents = true } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'copilot-java-schemas-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const packageDir = path.join(root, 'package'); + const schemasDir = path.join(packageDir, 'schemas'); + fs.mkdirSync(schemasDir, { recursive: true }); + fs.writeFileSync(path.join(schemasDir, 'api.schema.json'), '{"title":"API"}\n'); + if (includeEvents) { + fs.writeFileSync(path.join(schemasDir, 'session-events.schema.json'), '{"title":"Events"}\n'); + } + const archivePath = path.join(root, 'release.tgz'); + execFileSync('tar', ['-czf', archivePath, '-C', root, 'package']); + const hash = createHash('sha256').update(fs.readFileSync(archivePath)).digest('hex'); + return { root, archivePath, hash }; +} + +function runFetch(fixture, outputDir) { + return spawnSync(process.execPath, [scriptPath], { + encoding: 'utf8', + env: { + ...process.env, + COPILOT_CLI_RELEASE_TARBALL: fixture.archivePath, + COPILOT_CLI_RELEASE_SHA256: fixture.hash, + COPILOT_CLI_SCHEMA_OUTPUT: outputDir, + }, + }); +} diff --git a/java/scripts/codegen/java.ts b/java/scripts/codegen/java.ts index 785049afa1..abcd555338 100644 --- a/java/scripts/codegen/java.ts +++ b/java/scripts/codegen/java.ts @@ -174,53 +174,15 @@ function toEnumConstant(value: string): string { // ── Schema path resolution ─────────────────────────────────────────────────── -/** - * Resolve a JSON schema shipped by the `@github/copilot` CLI package. - * - * The CLI package layout changed in 1.0.64-1: the umbrella `@github/copilot` - * package became a thin loader and its bundled assets (including the JSON - * schemas) moved into the platform-specific packages installed as optional - * dependencies, e.g. `@github/copilot-linux-x64` or `@github/copilot-win32-x64`. - * - * We search both the Java codegen install (`scripts/codegen/node_modules`) and - * the Node SDK install (`nodejs/node_modules`), checking the umbrella package - * first (older versions) and then whichever platform package is present. - */ +/** Resolve a JSON schema staged from the pinned GitHub Release artifact. */ async function resolveCopilotSchemaPath(fileName: string): Promise { - const nodeModulesDirs = [ - path.join(REPO_ROOT, "scripts/codegen/node_modules"), - path.join(REPO_ROOT, "nodejs/node_modules"), - ]; - - const candidates: string[] = []; - for (const nodeModulesDir of nodeModulesDirs) { - candidates.push(path.join(nodeModulesDir, "@github/copilot/schemas", fileName)); - const githubScopeDir = path.join(nodeModulesDir, "@github"); - try { - for (const entry of await fs.readdir(githubScopeDir)) { - if (entry.startsWith("copilot-")) { - candidates.push(path.join(githubScopeDir, entry, "schemas", fileName)); - } - } - } catch (err) { - const code = (err as NodeJS.ErrnoException).code; - if (code !== "ENOENT" && code !== "ENOTDIR") { - throw err; - } - // @github scope directory may not exist; try the next location. - } - } - - for (const candidate of candidates) { - try { - await fs.access(candidate); - return candidate; - } catch { - // Try the next candidate. - } + const schemaPath = path.join(REPO_ROOT, "scripts/codegen/target/schemas", fileName); + try { + await fs.access(schemaPath); + return schemaPath; + } catch { + throw new Error(`${fileName} not found. Run 'npm run fetch:schemas' in java/scripts/codegen.`); } - - throw new Error(`${fileName} not found. Run 'npm ci' in java/scripts/codegen or java/nodejs first.`); } async function getSessionEventsSchemaPath(): Promise { diff --git a/java/scripts/codegen/package-lock.json b/java/scripts/codegen/package-lock.json index 5e10cd839b..a92322d340 100644 --- a/java/scripts/codegen/package-lock.json +++ b/java/scripts/codegen/package-lock.json @@ -6,7 +6,6 @@ "": { "name": "copilot-sdk-java-codegen", "dependencies": { - "@github/copilot": "^1.0.83-5", "json-schema": "^0.4.0", "tsx": "^4.23.13" } @@ -427,165 +426,6 @@ "node": ">=18" } }, - "node_modules/@github/copilot": { - "version": "1.0.83-5", - "resolved": "https://registry.npmjs.org/@github/copilot/-/copilot-1.0.83-5.tgz", - "integrity": "sha512-WtyV+Uom2EHR9agUeshRH9aUgvp4zEvAgY0k+DcD9qJKJ8nzPfGXpMrcOJF37CsvxQ3p8Djcvg4aY2ys3XM9CA==", - "license": "SEE LICENSE IN LICENSE.md", - "dependencies": { - "detect-libc": "^2.1.2" - }, - "bin": { - "copilot": "npm-loader.js" - }, - "optionalDependencies": { - "@github/copilot-darwin-arm64": "1.0.83-5", - "@github/copilot-darwin-x64": "1.0.83-5", - "@github/copilot-linux-arm64": "1.0.83-5", - "@github/copilot-linux-x64": "1.0.83-5", - "@github/copilot-linuxmusl-arm64": "1.0.83-5", - "@github/copilot-linuxmusl-x64": "1.0.83-5", - "@github/copilot-win32-arm64": "1.0.83-5", - "@github/copilot-win32-x64": "1.0.83-5" - } - }, - "node_modules/@github/copilot-darwin-arm64": { - "version": "1.0.83-5", - "resolved": "https://registry.npmjs.org/@github/copilot-darwin-arm64/-/copilot-darwin-arm64-1.0.83-5.tgz", - "integrity": "sha512-pj4yCrsVs7Nj16ogROYSKVyMDsmNMt4DRaiQEKUVhR9/Vs5n5DG+N55M2B1chT5wSk5MKtB9bov5Ep+x3jMU7g==", - "cpu": [ - "arm64" - ], - "license": "SEE LICENSE IN LICENSE.md", - "optional": true, - "os": [ - "darwin" - ], - "bin": { - "copilot-darwin-arm64": "copilot" - } - }, - "node_modules/@github/copilot-darwin-x64": { - "version": "1.0.83-5", - "resolved": "https://registry.npmjs.org/@github/copilot-darwin-x64/-/copilot-darwin-x64-1.0.83-5.tgz", - "integrity": "sha512-mU2TBR7zW9kIfM6+r2W+empF0O3J+M/EOKja04IWx4EMc/XQVRnRRdeDaQAPfVF5prEBLxxqe90Ctq6a14IHEA==", - "cpu": [ - "x64" - ], - "license": "SEE LICENSE IN LICENSE.md", - "optional": true, - "os": [ - "darwin" - ], - "bin": { - "copilot-darwin-x64": "copilot" - } - }, - "node_modules/@github/copilot-linux-arm64": { - "version": "1.0.83-5", - "resolved": "https://registry.npmjs.org/@github/copilot-linux-arm64/-/copilot-linux-arm64-1.0.83-5.tgz", - "integrity": "sha512-UG5X20iRyV6A/7tKiVjh/CcqoXDEKvvKd3zcH0l8LEkXhHNGmSY6WiF3FOvp3R5tFhwr2L97UIq8JoKJwUG9TA==", - "cpu": [ - "arm64" - ], - "license": "SEE LICENSE IN LICENSE.md", - "optional": true, - "os": [ - "linux" - ], - "bin": { - "copilot-linux-arm64": "copilot" - } - }, - "node_modules/@github/copilot-linux-x64": { - "version": "1.0.83-5", - "resolved": "https://registry.npmjs.org/@github/copilot-linux-x64/-/copilot-linux-x64-1.0.83-5.tgz", - "integrity": "sha512-sv0CwXtP1gyjDo2JbXC0OCIdMhKgK1ngaVc6HNIdIJpNZebMNTbe8QRxuV7DjXu9tSGP+tL9Kq3nPp/RipZq1Q==", - "cpu": [ - "x64" - ], - "license": "SEE LICENSE IN LICENSE.md", - "optional": true, - "os": [ - "linux" - ], - "bin": { - "copilot-linux-x64": "copilot" - } - }, - "node_modules/@github/copilot-linuxmusl-arm64": { - "version": "1.0.83-5", - "resolved": "https://registry.npmjs.org/@github/copilot-linuxmusl-arm64/-/copilot-linuxmusl-arm64-1.0.83-5.tgz", - "integrity": "sha512-orid/rhuGwypXypYCYDvbffY9CI0JPHntMUzuTP8lUQg45UsSUv3AtJOQPH3Rh3IGr7WJxG5lSxv/oKVv/+H7A==", - "cpu": [ - "arm64" - ], - "license": "SEE LICENSE IN LICENSE.md", - "optional": true, - "os": [ - "linux" - ], - "bin": { - "copilot-linuxmusl-arm64": "copilot" - } - }, - "node_modules/@github/copilot-linuxmusl-x64": { - "version": "1.0.83-5", - "resolved": "https://registry.npmjs.org/@github/copilot-linuxmusl-x64/-/copilot-linuxmusl-x64-1.0.83-5.tgz", - "integrity": "sha512-bcc5seLXLitC3+bH6QTzgFYBZPOmhiKHfkehPGwSdJ5S0l+D5GfuwVfNp7x+W0DDSF/2wXWKF2j6iTCruA45Zg==", - "cpu": [ - "x64" - ], - "license": "SEE LICENSE IN LICENSE.md", - "optional": true, - "os": [ - "linux" - ], - "bin": { - "copilot-linuxmusl-x64": "copilot" - } - }, - "node_modules/@github/copilot-win32-arm64": { - "version": "1.0.83-5", - "resolved": "https://registry.npmjs.org/@github/copilot-win32-arm64/-/copilot-win32-arm64-1.0.83-5.tgz", - "integrity": "sha512-nBFaP31NfWh787eLra3QqgSoEmd7wJGSKg7ANvsY8XDOC5lns8yQBm5y4WVvWNBCVwb196NEwInwGUHDXbj2cw==", - "cpu": [ - "arm64" - ], - "license": "SEE LICENSE IN LICENSE.md", - "optional": true, - "os": [ - "win32" - ], - "bin": { - "copilot-win32-arm64": "copilot.exe" - } - }, - "node_modules/@github/copilot-win32-x64": { - "version": "1.0.83-5", - "resolved": "https://registry.npmjs.org/@github/copilot-win32-x64/-/copilot-win32-x64-1.0.83-5.tgz", - "integrity": "sha512-DlxPHBRal+5oHyQ7f/ChB8zrc71eIpqxd7CzXFU5kcU0YzskitnIp9CLsymqWTEdSoWSNq9+7bhKemIUHWTkfA==", - "cpu": [ - "x64" - ], - "license": "SEE LICENSE IN LICENSE.md", - "optional": true, - "os": [ - "win32" - ], - "bin": { - "copilot-win32-x64": "copilot.exe" - } - }, - "node_modules/detect-libc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", - "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=8" - } - }, "node_modules/esbuild": { "version": "0.28.1", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", diff --git a/java/scripts/codegen/package.json b/java/scripts/codegen/package.json index 5ef4484fb7..a5de56c29f 100644 --- a/java/scripts/codegen/package.json +++ b/java/scripts/codegen/package.json @@ -3,11 +3,12 @@ "private": true, "type": "module", "scripts": { - "generate": "tsx java.ts", - "generate:java": "tsx java.ts" + "fetch:schemas": "node fetch-schemas.mjs", + "generate": "npm run fetch:schemas && tsx java.ts", + "generate:java": "npm run generate", + "test": "node --test fetch-schemas.test.mjs" }, "dependencies": { - "@github/copilot": "^1.0.83-5", "json-schema": "^0.4.0", "tsx": "^4.23.13" } diff --git a/java/sdk/pom.xml b/java/sdk/pom.xml index aa40b25189..693df17eb6 100644 --- a/java/sdk/pom.xml +++ b/java/sdk/pom.xml @@ -775,57 +775,6 @@ did not produce the multi-release output. Re-build on JDK 25+ and verify the - - - update-schemas-from-npm-artifact - - - - org.codehaus.mojo - exec-maven-plugin - - - update-copilot-schema-version - generate-sources - - exec - - - npm - ${project.parent.basedir}/scripts/codegen - - install - @github/copilot@${copilot.schema.version} - - - - - - - org.apache.maven.plugins - maven-enforcer-plugin - - - require-schema-version - validate - - enforce - - - - - copilot.schema.version - You must specify -Dcopilot.schema.version=VERSION (e.g. 1.0.25) - - - - - - - - - diff --git a/java/sdk/src/main/java/com/github/copilot/ffi/NativeRuntimeLoader.java b/java/sdk/src/main/java/com/github/copilot/ffi/NativeRuntimeLoader.java index bd4b185a07..3a1c6b35bc 100644 --- a/java/sdk/src/main/java/com/github/copilot/ffi/NativeRuntimeLoader.java +++ b/java/sdk/src/main/java/com/github/copilot/ffi/NativeRuntimeLoader.java @@ -318,9 +318,8 @@ static Path resolve(String cliPathEnv, Path cacheBase, ClassLoader loader, Strin * *

* Checks, in order, the flat bundled layout ({@code runtime.node} directly next - * to the CLI) and the npm package layout - * ({@code prebuilds//runtime.node} next to the CLI), matching the - * two layouts the {@code @github/copilot-} packages may ship. + * to the CLI) and the release package layout + * ({@code prebuilds//runtime.node} next to the CLI). */ static Path resolveFromCliPath(String cliPathStr) throws IOException { if (cliPathStr == null || cliPathStr.isBlank()) { From 29128c629c6bb09c43bb21cff6f9c6c3d6c5c0ab Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Fri, 4 Sep 2026 10:51:31 +0200 Subject: [PATCH 4/6] Address runtime migration review feedback Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: daec7b3b-799c-4396-b372-6eb638d0faf8 --- dotnet/src/build/GitHub.Copilot.SDK.targets | 4 ++-- dotnet/test/Unit/MSBuildTargetsTests.cs | 3 +++ go/internal/embeddedcli/embeddedcli.go | 6 +++--- go/internal/embeddedcli/embeddedcli_test.go | 18 +++++++++++++++--- python/test_cli_download.py | 7 +++++-- 5 files changed, 28 insertions(+), 10 deletions(-) diff --git a/dotnet/src/build/GitHub.Copilot.SDK.targets b/dotnet/src/build/GitHub.Copilot.SDK.targets index 28692ca2c9..a5aba612ab 100644 --- a/dotnet/src/build/GitHub.Copilot.SDK.targets +++ b/dotnet/src/build/GitHub.Copilot.SDK.targets @@ -112,8 +112,8 @@ <_CopilotCliDownloadTimeoutMs>$([System.Convert]::ToInt32($([MSBuild]::Multiply($(CopilotCliDownloadTimeout), 1000)))) - - + + diff --git a/dotnet/test/Unit/MSBuildTargetsTests.cs b/dotnet/test/Unit/MSBuildTargetsTests.cs index fa610006b1..a6e3bb5660 100644 --- a/dotnet/test/Unit/MSBuildTargetsTests.cs +++ b/dotnet/test/Unit/MSBuildTargetsTests.cs @@ -152,6 +152,7 @@ public async Task IncompleteCache_WithRuntimePairButNoMarker_IsReacquired() using var sandbox = MSBuildSandbox.Create(); sandbox.WriteRuntimeCacheAsset("prebuilds", GetReleasePlatform(), RuntimeWrapperName, "partial-wrapper"); sandbox.WriteRuntimeCacheAsset("prebuilds", GetReleasePlatform(), "runtime.node", "partial-runtime"); + sandbox.WriteRuntimeCacheAsset("definitions", "stale.json", "stale"); var archive = sandbox.CreateReleaseArchive("complete-wrapper"); var assetName = $"github-copilot-0.0.0-test-{GetReleasePlatform()}.tgz"; var assetPath = $"/v0.0.0-test/{assetName}"; @@ -173,6 +174,8 @@ public async Task IncompleteCache_WithRuntimePairButNoMarker_IsReacquired() Assert.Equal("complete-wrapper", File.ReadAllText(sandbox.ExpectedRuntimeAsset(RuntimeWrapperName))); Assert.Equal("runtime", File.ReadAllText(sandbox.ExpectedRuntimeAsset("runtime.node"))); Assert.True(File.Exists(sandbox.ExpectedCacheAsset(".copilot-runtime-complete"))); + Assert.False(File.Exists(sandbox.ExpectedCacheAsset("definitions", "stale.json"))); + Assert.False(File.Exists(sandbox.ExpectedRuntimeAsset("definitions", "stale.json"))); } [Fact] diff --git a/go/internal/embeddedcli/embeddedcli.go b/go/internal/embeddedcli/embeddedcli.go index 5d9eea39de..2e3b8add16 100644 --- a/go/internal/embeddedcli/embeddedcli.go +++ b/go/internal/embeddedcli/embeddedcli.go @@ -320,11 +320,11 @@ func installAt(installDir string) (string, error) { if err != nil { return "", err } - if err := installRuntimeAssets(installDir); err != nil { - return "", err - } runtimeLibPath = libPath } + if err := installRuntimeAssets(installDir); err != nil { + return "", err + } return finalPath, nil } diff --git a/go/internal/embeddedcli/embeddedcli_test.go b/go/internal/embeddedcli/embeddedcli_test.go index 50b6a29085..a56dd8106e 100644 --- a/go/internal/embeddedcli/embeddedcli_test.go +++ b/go/internal/embeddedcli/embeddedcli_test.go @@ -259,13 +259,17 @@ func TestInstallAtWritesBinaryAndLicense(t *testing.T) { } } -func TestPathInstallsAdjacentRuntimePair(t *testing.T) { +func TestInstallAtInstallsRuntimePairAndAssetsWithoutRuntimeLib(t *testing.T) { resetGlobals() tempDir := t.TempDir() wrapper := []byte("wrapper") node := []byte("runtime") + assets := runtimeAssetsArchive(t, map[string]assetFixture{ + "definitions/future.json": {content: []byte("{}"), mode: 0644}, + }) wrapperHash := sha256.Sum256(wrapper) nodeHash := sha256.Sum256(node) + assetsHash := sha256.Sum256(assets) Setup(Config{ Cli: bytes.NewReader(wrapper), CliHash: wrapperHash[:], @@ -273,18 +277,26 @@ func TestPathInstallsAdjacentRuntimePair(t *testing.T) { RuntimeExecutableHash: wrapperHash[:], RuntimeNode: bytes.NewReader(node), RuntimeNodeHash: nodeHash[:], + RuntimeAssets: bytes.NewReader(assets), + RuntimeAssetsHash: assetsHash[:], Version: "1.2.3", Dir: tempDir, }) - runtimePath := RuntimePath() - installDir := filepath.Dir(runtimePath) + path, err := installAt(tempDir) + if err != nil { + t.Fatal(err) + } + installDir := filepath.Dir(path) if got, err := os.ReadFile(filepath.Join(installDir, runtimeExecutableName())); err != nil || !bytes.Equal(got, wrapper) { t.Fatalf("runtime wrapper content=%q err=%v", got, err) } if got, err := os.ReadFile(filepath.Join(installDir, "runtime.node")); err != nil || !bytes.Equal(got, node) { t.Fatalf("runtime.node content=%q err=%v", got, err) } + if got, err := os.ReadFile(filepath.Join(installDir, "definitions", "future.json")); err != nil || string(got) != "{}" { + t.Fatalf("definition content=%q err=%v", got, err) + } } func TestInstallAtExistingBinaryHashMismatch(t *testing.T) { diff --git a/python/test_cli_download.py b/python/test_cli_download.py index 7a86659e62..04b4e7f07c 100644 --- a/python/test_cli_download.py +++ b/python/test_cli_download.py @@ -277,8 +277,11 @@ def test_explicit_cli_reuses_library_from_canonical_staged_bundle(tmp_path): assert (cache_dir / "prebuilds" / runtime_platform / "runtime.node").read_bytes() == b"runtime" assert not (cache_dir / "packages").exists() assert wrapper.endswith( - f"prebuilds/{runtime_platform}/" - f"{'copilot-runtime.exe' if os.name == 'nt' else 'copilot-runtime'}" + os.path.join( + "prebuilds", + runtime_platform, + "copilot-runtime.exe" if os.name == "nt" else "copilot-runtime", + ) ) assert fetch_mock.call_count == 2 From 62fbefda88146a5ee9a1b5d10b89a51813193b1c Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Fri, 4 Sep 2026 13:47:16 +0200 Subject: [PATCH 5/6] Reuse prepared runtime in Python E2E tests Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: daec7b3b-799c-4396-b372-6eb638d0faf8 --- python/e2e/testharness/context.py | 1 + python/test_e2e_harness_cli_path.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/python/e2e/testharness/context.py b/python/e2e/testharness/context.py index 616fa9bb0a..12eb9466fa 100644 --- a/python/e2e/testharness/context.py +++ b/python/e2e/testharness/context.py @@ -147,6 +147,7 @@ def _apply_inprocess_environment(self) -> None: { "GH_TOKEN": DEFAULT_GITHUB_TOKEN, "GITHUB_TOKEN": DEFAULT_GITHUB_TOKEN, + "COPILOT_CLI_PATH": self.cli_path, "COPILOT_HMAC_KEY": "", "CAPI_HMAC_KEY": "", } diff --git a/python/test_e2e_harness_cli_path.py b/python/test_e2e_harness_cli_path.py index 83167ffd33..c81fb48d05 100644 --- a/python/test_e2e_harness_cli_path.py +++ b/python/test_e2e_harness_cli_path.py @@ -37,3 +37,20 @@ class Result: with pytest.raises(RuntimeError) as excinfo: context._prepare_pinned_cli(tmp_path) assert "download failed" in str(excinfo.value) + + +def test_inprocess_environment_reuses_prepared_runtime(tmp_path, monkeypatch): + cli = tmp_path / "copilot-runtime" + cli.write_text("runtime\n") + + test_context = context.E2ETestContext() + test_context.cli_path = str(cli) + test_context.work_dir = str(tmp_path) + monkeypatch.setattr(test_context, "get_env", lambda: {"HTTPS_PROXY": "https://proxy"}) + monkeypatch.delenv("COPILOT_CLI_PATH", raising=False) + + try: + test_context._apply_inprocess_environment() + assert context.os.environ["COPILOT_CLI_PATH"] == str(cli) + finally: + test_context._restore_inprocess_environment() From 08206c9b6aea84a4f23dfae4a8a3a0ec8ec99db6 Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Fri, 4 Sep 2026 13:49:38 +0200 Subject: [PATCH 6/6] Address incremental review feedback Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: daec7b3b-799c-4396-b372-6eb638d0faf8 --- go/cmd/bundler/main.go | 10 +++++++--- python/copilot/_cli_download.py | 2 ++ python/test_cli_download.py | 9 +++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/go/cmd/bundler/main.go b/go/cmd/bundler/main.go index 0d79243715..41b5863347 100644 --- a/go/cmd/bundler/main.go +++ b/go/cmd/bundler/main.go @@ -30,6 +30,7 @@ import ( "regexp" "runtime" "strings" + "time" "github.com/klauspost/compress/zstd" ) @@ -923,7 +924,10 @@ func mustDecodeBase64(s string) []byte { `, buildConstraint, pkgName, binaryName, licenseName, runtimeEmbed, muslEmbed, cliVersion, hashBase64, runtimeConfig, muslConfig, runtimeReader, muslReaders) } -var releaseChecksumCache = map[string]map[string]string{} +var ( + releaseChecksumCache = map[string]map[string]string{} + releaseHTTPClient = &http.Client{Timeout: 10 * time.Minute} +) func cliDownloadBaseURL() string { if override := strings.TrimRight(os.Getenv(cliDownloadBaseURLEnvironment), "/"); override != "" { @@ -960,7 +964,7 @@ func getReleaseChecksum(version, assetName string) (string, error) { if !ok { checksumsURL := fmt.Sprintf("%s/v%s/SHA256SUMS.txt", baseURL, version) fmt.Printf("Downloading checksums from %s...\n", checksumsURL) - resp, err := http.Get(checksumsURL) + resp, err := releaseHTTPClient.Get(checksumsURL) if err != nil { return "", fmt.Errorf("failed to download checksums: %w", err) } @@ -995,7 +999,7 @@ func downloadCLIBinary(runtimePlatform, binaryName, cliVersion, destDir string) fmt.Printf("Downloading from %s...\n", tarballURL) - resp, err := http.Get(tarballURL) + resp, err := releaseHTTPClient.Get(tarballURL) if err != nil { return "", "", fmt.Errorf("failed to download: %w", err) } diff --git a/python/copilot/_cli_download.py b/python/copilot/_cli_download.py index 235667aa56..72424cb650 100644 --- a/python/copilot/_cli_download.py +++ b/python/copilot/_cli_download.py @@ -272,6 +272,8 @@ def _fetch_url_bytes(url: str, *, timeout: int) -> bytes: def _hostless_runtime_path(member_name: str, runtime_platform: str) -> Path | None: + if "\\" in member_name: + raise RuntimeError(f"Unsafe runtime package path: {member_name}") parts = PurePosixPath(member_name).parts if not parts or parts[0] != "package" or len(parts) < 2: return None diff --git a/python/test_cli_download.py b/python/test_cli_download.py index 04b4e7f07c..fcf65df41d 100644 --- a/python/test_cli_download.py +++ b/python/test_cli_download.py @@ -82,6 +82,15 @@ def test_release_asset_uses_platform_package_name(monkeypatch): ) +@pytest.mark.parametrize( + "member_name", + ["package/../outside", r"package\..\outside"], +) +def test_hostless_runtime_path_rejects_traversal(member_name): + with pytest.raises(RuntimeError, match="Unsafe runtime package path"): + _cli_download._hostless_runtime_path(member_name, "linux-x64") + + def test_rejects_release_package_checksum_mismatch(tmp_path): runtime_platform = "linux-x64" data = _release_package(runtime_platform)