fix(package): scan semantic content for internal references

sdk/typescript/scripts/check-package.mjs

@@ -1,10 +1,21 @@
 import { spawnSync } from "node:child_process";
 import { createHash } from "node:crypto";
-import { readFileSync } from "node:fs";
+import {
+  chmodSync,
+  lstatSync,
+  mkdtempSync,
+  readFileSync,
+  readdirSync,
+  rmSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
-import { brotliDecompressSync, gunzipSync } from "node:zlib";
+import { gunzipSync } from "node:zlib";
+import { assertNoInternalReferences } from "./package-internal-references.mjs";
 import { assertExpectedGitHead } from "./package-provenance.mjs";
 import { packageSmokeTimeouts } from "./package-smoke-timeouts.mjs";
+import { plainTarEntries } from "./package-tar-entries.mjs";
 import { regularTarListingLines } from "./package-tar-listing.mjs";
 
 const PACKAGE_SMOKE_PROCESS_TIMEOUT_MS =
@@ -22,13 +33,22 @@ if (archive === undefined || args.length > 2) {
   );
 }
 
+const archivePath = resolve(archive);
 const MAX_EXPANDED_ASSET_BYTES = 32 * 1024 * 1024;
-const archiveBytes = gunzipSync(readFileSync(archive), {
+const compressedArchive = readFileSync(archivePath);
+const archiveBytes = gunzipSync(compressedArchive, {
   maxOutputLength: MAX_EXPANDED_ASSET_BYTES,
 });
+const rawEntries = plainTarEntries(archiveBytes);
 const PUBLIC_LOGO_SHA256 =
   "9b9c2b09b2fa064611fb62307d321d5c2ea70cf0789f7ce34cdb0fc0d9190b3a";
-const tarOptions = { maxBuffer: archiveBytes.byteLength + 1024 };
+const processEnvironment = { ...process.env };
+delete processEnvironment.TAR_OPTIONS;
+const tarOptions = {
+  env: { ...processEnvironment, LC_ALL: "C" },
+  input: compressedArchive,
+  maxBuffer: archiveBytes.byteLength + 1024,
+};
 function tar(args, encoding = "buffer") {
   const result = spawnSync("tar", ["--ignore-zeros", ...args], {
     ...tarOptions,
@@ -44,59 +64,21 @@ function tar(args, encoding = "buffer") {
   return result.stdout;
 }
 
-let offset = 0;
-const archiveFiles = new Map();
-for (; offset + 512 <= archiveBytes.byteLength; ) {
-  const header = archiveBytes.subarray(offset, offset + 512);
-  if (header.every((byte) => byte === 0)) {
-    offset += 512;
-    continue;
-  }
-  const name = header.subarray(0, 100).toString("utf8").split("\0", 1)[0];
-  const prefix = header.subarray(345, 500).toString("utf8").split("\0", 1)[0];
-  const path = prefix === "" ? name : `${prefix}/${name}`;
-  const sizeField = header
-    .subarray(124, 136)
-    .toString("ascii")
-    .split("\0", 1)[0]
-    .trim();
-  if (!/^[0-7]*$/u.test(sizeField)) {
-    throw new Error("npm tarball contains an invalid tar entry.");
-  }
-  if (path.endsWith("/") && header[156] !== 0x35) {
-    throw new Error("npm tarball contains an invalid tar entry.");
-  }
-  const size = Number.parseInt(sizeField || "0", 8);
-  const contentsStart = offset + 512;
-  const nextOffset = contentsStart + Math.ceil(size / 512) * 512;
-  if (nextOffset > archiveBytes.byteLength) {
-    throw new Error("npm tarball contains an invalid tar entry.");
-  }
-  if (header[156] === 0 || header[156] === 0x30) {
-    archiveFiles.set(
-      path,
-      archiveBytes.subarray(contentsStart, contentsStart + size),
-    );
-  }
-  offset = nextOffset;
-}
-if (archiveBytes.subarray(offset).some((byte) => byte !== 0)) {
-  throw new Error("npm tarball contains trailing tar data.");
-}
-
-function archiveFile(path) {
-  const contents = archiveFiles.get(path);
-  if (contents === undefined) {
-    throw new Error("npm tarball contains an invalid tar entry: " + path + ".");
-  }
-  return contents;
+function invalidTarEntry() {
+  throw new Error("npm tarball contains an invalid tar entry.");
 }
 
-const entries = tar(["-tzf", archive], "utf8").split(/\r?\n/u).filter(Boolean);
+const entries = tar(["-tzf", "-"], "utf8").split(/\r?\n/u).filter(Boolean);
 const files = new Set(entries);
 if (files.size !== entries.length) {
   throw new Error("npm tarball contains duplicate paths.");
 }
+if (
+  rawEntries.length !== entries.length ||
+  rawEntries.some(({ path }, index) => path !== entries[index])
+) {
+  invalidTarEntry();
+}
 const required = [
   "package/package.json",
   "package/README.md",
@@ -138,18 +120,11 @@ if (pluginFiles.size !== pluginPaths.length) {
 }
 
 const pluginEntries = new Set();
-const pluginDirectories = new Set(["package/_bundled_plugin"]);
 for (const file of pluginFiles) {
-  const archivePath = `package/_bundled_plugin/${file}`;
-  pluginEntries.add(archivePath);
-  if (!files.has(archivePath)) {
-    throw new Error(`npm tarball is missing ${archivePath}.`);
-  }
-  const parts = file.split("/");
-  for (let index = 1; index < parts.length; index++) {
-    pluginDirectories.add(
-      `package/_bundled_plugin/${parts.slice(0, index).join("/")}`,
-    );
+  const pluginArchivePath = `package/_bundled_plugin/${file}`;
+  pluginEntries.add(pluginArchivePath);
+  if (!files.has(pluginArchivePath)) {
+    throw new Error(`npm tarball is missing ${pluginArchivePath}.`);
   }
 }
 
@@ -209,29 +184,20 @@ for (const file of distFiles) {
 }
 const unsafePath = /(?:^|\/)\.{1,2}(?:\/|$)/u;
 for (const file of files) {
-  const normalized = file.endsWith("/") ? file.slice(0, -1) : file;
-  const allowed = file.endsWith("/")
-    ? normalized === "package" ||
-      normalized === "package/bin" ||
-      normalized === "package/dist" ||
-      pluginDirectories.has(normalized)
-    : allowedRoot.has(normalized) ||
-      distFiles.has(normalized) ||
-      pluginEntries.has(normalized);
+  const allowed =
+    allowedRoot.has(file) || distFiles.has(file) || pluginEntries.has(file);
   if (!allowed || unsafePath.test(file) || file.includes("\\")) {
     throw new Error(`npm tarball contains an unexpected file: ${file}.`);
   }
 }
 
-const listing = tar(["-tvzf", archive], "utf8");
+const listing = tar(["-tvzf", "-"], "utf8");
 const listingLines = regularTarListingLines(listing);
 if (
   listingLines.length !== entries.length ||
-  listingLines.some(
-    (line, index) => line.startsWith("d") !== entries[index].endsWith("/"),
-  )
+  listingLines.some((line) => !line.startsWith("-"))
 ) {
-  throw new Error("npm tarball contains an invalid tar entry.");
+  invalidTarEntry();
 }
 const launcherPermissions =
   listingLines[entries.indexOf("package/bin/codex-security.mjs")]?.split(
@@ -241,6 +207,86 @@ const launcherPermissions =
 if ([3, 6, 9].some((index) => launcherPermissions[index] !== "x")) {
   throw new Error("npm package CLI launcher is not executable.");
 }
+
+function extractedArchiveFiles() {
+  const rawSizes = new Map();
+  const expectedPaths = new Map();
+  for (const { path, size } of rawEntries) {
+    if (expectedPaths.get(path) === "directory") invalidTarEntry();
+    rawSizes.set(path, size);
+    expectedPaths.set(path, "file");
+    const parts = path.split("/");
+    for (let index = 1; index < parts.length; index++) {
+      const directory = parts.slice(0, index).join("/");
+      if (rawSizes.has(directory)) invalidTarEntry();
+      expectedPaths.set(directory, "directory");
+    }
+  }
+
+  const extractionRoot = mkdtempSync(
+    join(tmpdir(), "codex-security-package-check-"),
+  );
+  try {
+    chmodSync(extractionRoot, 0o700);
+    tar([
+      "--keep-old-files",
+      "--no-same-owner",
+      "--no-same-permissions",
+      "--no-acls",
+      "--no-xattrs",
+      "-xzf",
+      "-",
+      "-C",
+      extractionRoot,
+    ]);
+
+    const archiveFiles = new Map();
+    let expandedBytes = 0;
+    function visit(directory, relative = "") {
+      for (const name of readdirSync(directory)) {
+        const path = relative === "" ? name : `${relative}/${name}`;
+        const expectedType = expectedPaths.get(path);
+        if (expectedType === undefined) invalidTarEntry();
+        const extractedPath = join(extractionRoot, path);
+        const stats = lstatSync(extractedPath);
+        expectedPaths.delete(path);
+
+        if (expectedType === "directory") {
+          if (!stats.isDirectory()) invalidTarEntry();
+          visit(extractedPath, path);
+          continue;
+        }
+
+        if (
+          !stats.isFile() ||
+          stats.nlink !== 1 ||
+          stats.size !== rawSizes.get(path) ||
+          stats.size > MAX_EXPANDED_ASSET_BYTES ||
+          expandedBytes > MAX_EXPANDED_ASSET_BYTES - stats.size
+        ) {
+          invalidTarEntry();
+        }
+        expandedBytes += stats.size;
+        archiveFiles.set(path, readFileSync(extractedPath));
+      }
+    }
+    visit(extractionRoot);
+
+    if (expectedPaths.size !== 0 || archiveFiles.size !== rawEntries.length) {
+      invalidTarEntry();
+    }
+    return archiveFiles;
+  } finally {
+    rmSync(extractionRoot, { force: true, recursive: true });
+  }
+}
+
+const archiveFiles = extractedArchiveFiles();
+function archiveFile(path) {
+  const contents = archiveFiles.get(path);
+  if (contents === undefined) invalidTarEntry();
+  return contents;
+}
 const packageJson = JSON.parse(
   archiveFile("package/package.json").toString("utf8"),
 );
@@ -255,40 +301,6 @@ assertExpectedGitHead(
   process.env.CODEX_SECURITY_EXPECTED_GIT_HEAD,
 );
 
-const internalMarker =
-  /(?:internal\.api\.openai\.org|gateway\.[a-z0-9.-]*internal|\.openai\.org|openai\.firewall\.socket\.dev|socket\x2dfirewall\x2dregistry|openai\.(?:enterprise\.)?slack\.com|app\.slack\.com\/client|(?:app\.notion\.com\/p|notion\.so)\/openai|linear\.app\/openai|(?:github\.com[:/]|api\.github\.com\/repos\/|raw\.githubusercontent\.com\/)openai\/openai(?:\.git)?(?:[^a-z0-9_-]|$)|LicenseRef\x2dProprietary|\/Users\/|\/home\/dev-user|flow\.apps\.openai\.org|(?:^|[^a-z0-9_-])go\/[a-z0-9_-]+)/iu;
-
-const payloads = [archiveBytes.toString("utf8")];
-const compressedFiles = [...files].filter((file) => /\.br$/iu.test(file));
-const compressedParts = new Map();
-for (const file of files) {
-  const match = /^(.*\.br)\.part-([0-9]+)$/iu.exec(file);
-  if (match === null) continue;
-  const [, name, part] = match;
-  const parts = compressedParts.get(name) ?? [];
-  parts.push({ file, part: Number(part) });
-  compressedParts.set(name, parts);
-}
-
-function brotliPayload(bytes, file) {
-  const result = brotliDecompressSync(bytes, {
-    info: true,
-    maxOutputLength: MAX_EXPANDED_ASSET_BYTES,
-  });
-  if (result.engine.bytesWritten !== bytes.length) {
-    throw new Error(`npm tarball contains trailing Brotli data: ${file}.`);
-  }
-  return result.buffer;
-}
-
-for (const file of compressedFiles) {
-  payloads.push(brotliPayload(archiveFile(file), file).toString("utf8"));
-}
-for (const parts of compressedParts.values()) {
-  parts.sort((left, right) => left.part - right.part);
-  const bytes = Buffer.concat(parts.map(({ file }) => archiveFile(file)));
-  payloads.push(brotliPayload(bytes, parts[0].file).toString("utf8"));
-}
 for (const file of files) {
   if (/\.png$/iu.test(file)) {
     const digest = createHash("sha256").update(archiveFile(file)).digest("hex");
@@ -298,16 +310,15 @@ for (const file of files) {
   }
 }
 
-for (const contents of payloads) {
-  if (internalMarker.test(contents)) {
-    throw new Error("npm tarball contains an internal reference.");
-  }
-}
+assertNoInternalReferences(archiveFiles, MAX_EXPANDED_ASSET_BYTES);
 
 if (args.length === 1) {
   const smoke = spawnSync(
     process.execPath,
-    [fileURLToPath(new URL("./smoke-package.mjs", import.meta.url)), archive],
+    [
+      fileURLToPath(new URL("./smoke-package.mjs", import.meta.url)),
+      archivePath,
+    ],
     {
       stdio: "inherit",
       timeout: PACKAGE_SMOKE_PROCESS_TIMEOUT_MS,

sdk/typescript/scripts/package-internal-references.mjs

@@ -0,0 +1,53 @@
+import { brotliDecompressSync } from "node:zlib";
+
+const internalMarker =
+  /(?:internal\.api\.openai\.org|gateway\.[a-z0-9.-]*internal|\.openai\.org|openai\.firewall\.socket\.dev|socket\x2dfirewall\x2dregistry|openai\.(?:enterprise\.)?slack\.com|app\.slack\.com\/client|(?:app\.notion\.com\/p|notion\.so)\/openai|linear\.app\/openai|(?:github\.com[:/]|api\.github\.com\/repos\/|raw\.githubusercontent\.com\/)openai\/openai(?:\.git)?(?:[^a-z0-9_-]|$)|LicenseRef\x2dProprietary|\/Users\/|\/home\/dev-user|flow\.apps\.openai\.org|(?:^|[^a-z0-9_-])go\/[a-z0-9_-]+)/iu;
+
+function brotliPayload(bytes, file, maxExpandedAssetBytes) {
+  const result = brotliDecompressSync(bytes, {
+    info: true,
+    maxOutputLength: maxExpandedAssetBytes,
+  });
+  if (result.engine.bytesWritten !== bytes.length) {
+    throw new Error(`npm tarball contains trailing Brotli data: ${file}.`);
+  }
+  return result.buffer;
+}
+
+export function assertNoInternalReference(contents) {
+  if (internalMarker.test(contents.toString("utf8"))) {
+    throw new Error("npm tarball contains an internal reference.");
+  }
+}
+
+export function assertNoInternalReferences(
+  archiveFiles,
+  maxExpandedAssetBytes,
+) {
+  const compressedParts = new Map();
+
+  for (const [file, contents] of archiveFiles) {
+    assertNoInternalReference(Buffer.from(file));
+    const match = /^(.*\.br)\.part-([0-9]+)$/iu.exec(file);
+    if (match !== null) {
+      const [, name, part] = match;
+      const parts = compressedParts.get(name) ?? [];
+      parts.push({ file, part: Number(part), contents });
+      compressedParts.set(name, parts);
+    } else if (/\.br$/iu.test(file)) {
+      assertNoInternalReference(
+        brotliPayload(contents, file, maxExpandedAssetBytes),
+      );
+    } else if (!/\.png$/iu.test(file)) {
+      assertNoInternalReference(contents);
+    }
+  }
+
+  for (const parts of compressedParts.values()) {
+    parts.sort((left, right) => left.part - right.part);
+    const bytes = Buffer.concat(parts.map(({ contents }) => contents));
+    assertNoInternalReference(
+      brotliPayload(bytes, parts[0].file, maxExpandedAssetBytes),
+    );
+  }
+}