From d454775061d702419829a47037f48160977bf1ae Mon Sep 17 00:00:00 2001 From: Jack Zhuang <277994282+os-zhuang@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:33:21 +0800 Subject: [PATCH 1/2] fix(create-objectstack): scope skill installs to the curated skills/ catalog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scaffolder (and docs) ran `npx skills add objectstack-ai/framework --all`, and the skills CLI's --all implies --skill '*', which includes metadata.internal skills — so the repo-internal dogfood-verification skill (.claude/skills/) landed in every scaffolded project's .agents/skills/ (15.1 third-party eval, P2). - scaffolder + docs now install from the /skills subpath, which scopes skills-CLI discovery to the published catalog - mark .claude/skills/dogfood-verification metadata.internal: true so interactive repo-root discovery hides it too - template-consistency ratchets: every SKILL.md outside skills/ must be metadata.internal, curated entries must not be, no customer-facing surface may advertise a repo-root install - scaffold-e2e: install from this checkout's skills/ subpath and assert set-equality with the curated catalog; assert repo-root listing surfaces exactly the curated set Co-Authored-By: Claude Fable 5 --- .changeset/scaffold-skill-catalog-boundary.md | 5 + .claude/skills/dogfood-verification/SKILL.md | 8 ++ .github/workflows/scaffold-e2e.yml | 39 ++++++++ content/docs/ai/skills-reference.mdx | 4 +- content/docs/ai/skills.mdx | 2 +- .../build-with-claude-code.mdx | 2 +- .../getting-started/your-first-project.mdx | 2 +- packages/create-objectstack/README.md | 3 +- packages/create-objectstack/src/index.ts | 11 ++- .../src/template-consistency.test.ts | 92 +++++++++++++++++++ .../src/templates/AGENTS.md | 2 +- 11 files changed, 159 insertions(+), 11 deletions(-) create mode 100644 .changeset/scaffold-skill-catalog-boundary.md diff --git a/.changeset/scaffold-skill-catalog-boundary.md b/.changeset/scaffold-skill-catalog-boundary.md new file mode 100644 index 0000000000..5d4168b7d6 --- /dev/null +++ b/.changeset/scaffold-skill-catalog-boundary.md @@ -0,0 +1,5 @@ +--- +"create-objectstack": patch +--- + +Stop leaking repo-internal skills into scaffolded projects. The scaffolder (and the docs) advertised `npx skills add objectstack-ai/framework --all`, and the skills CLI's `--all` implies `--skill '*'` — which includes even `metadata.internal` skills — so repo-internal tooling like `.claude/skills/dogfood-verification` landed in every new project's `.agents/skills/`. All install commands are now scoped to the published catalog via the `/skills` subpath (`npx skills add objectstack-ai/framework/skills --all`), the internal skill is additionally marked `metadata.internal: true` to hide it from interactive discovery, and a template-consistency ratchet plus a scaffold-e2e assertion keep the boundary from regressing. diff --git a/.claude/skills/dogfood-verification/SKILL.md b/.claude/skills/dogfood-verification/SKILL.md index eb7fb85cfd..1e7159e2df 100644 --- a/.claude/skills/dogfood-verification/SKILL.md +++ b/.claude/skills/dogfood-verification/SKILL.md @@ -8,6 +8,14 @@ description: > "act as a real admin/user", "dogfood the Setup/Studio app", or browser-verify a change in the running app. NOT a customer-published skill — this is internal agent tooling (lives in .claude/, never in the published `skills/` dir). +metadata: + # Hides this skill from interactive `npx skills add objectstack-ai/framework` + # discovery. NOTE: the skills CLI's `--all` implies `--skill '*'`, which + # includes internal skills — so the hard boundary against leaking into + # customer projects is the `/skills` subpath in every advertised install + # command (scaffolder, docs). Every SKILL.md outside `skills/` must carry + # this marker — template-consistency.test.ts enforces both layers. + internal: true --- # Dogfood verification diff --git a/.github/workflows/scaffold-e2e.yml b/.github/workflows/scaffold-e2e.yml index 1b476310e0..065782dc70 100644 --- a/.github/workflows/scaffold-e2e.yml +++ b/.github/workflows/scaffold-e2e.yml @@ -115,6 +115,45 @@ jobs: curl -fsS http://localhost:8080/api/v1/ready kill "$SERVER_PID" + - name: 'Skills boundary: only the curated skills/ catalog installs' + # 15.1 third-party eval: the repo-internal dogfood-verification skill + # (.claude/skills/) leaked into scaffolded projects. Two leak paths, + # two assertions against THIS checkout: + # 1. The scaffolder/docs command is `skills add …/framework/skills + # --all` — the /skills subpath is the hard boundary, because the + # skills CLI's --all implies --skill '*' which INCLUDES + # metadata.internal skills. Install from the subpath and assert + # set-equality with the curated catalog. + # 2. Interactive `skills add …/framework` (no --all) relies on + # metadata.internal hiding — assert repo-root discovery surfaces + # exactly the curated set and nothing internal. + run: | + cd "$RUNNER_TEMP" + node "$GITHUB_WORKSPACE/packages/create-objectstack/bin/create-objectstack.js" skills-probe --skip-install --skip-skills + cd skills-probe + expected=$(cd "$GITHUB_WORKSPACE/skills" && ls -d objectstack-*/ | tr -d '/' | sort) + expected_count=$(echo "$expected" | wc -l | tr -d ' ') + + npx -y skills add "$GITHUB_WORKSPACE/skills" --all --copy + installed=$(find . -name SKILL.md -not -path '*/node_modules/*' -exec dirname {} \; | xargs -rn1 basename | sort -u) + echo "installed skills:"; echo "$installed" + if [ "$installed" != "$expected" ]; then + echo "::error::scaffolded skill set != curated skills/ catalog (internal-skill leak or missing skill)" + printf 'expected:\n%s\n' "$expected" + exit 1 + fi + + listing=$(npx -y skills add "$GITHUB_WORKSPACE" --list 2>&1) + echo "$listing" + if echo "$listing" | grep -q 'dogfood-verification'; then + echo "::error::repo-root discovery surfaced an internal skill (metadata.internal marker missing?)" + exit 1 + fi + if ! echo "$listing" | grep -q "Found $expected_count skill"; then + echo "::error::repo-root discovery does not match the curated catalog (expected $expected_count skills) — a SKILL.md outside skills/ is missing metadata.internal" + exit 1 + fi + - name: Build official runtime image from this checkout # examples/docker builds FROM ghcr.io/objectstack-ai/objectstack. # Build that base HERE from docker/Dockerfile instead of pulling, so diff --git a/content/docs/ai/skills-reference.mdx b/content/docs/ai/skills-reference.mdx index 39be9bab6d..f5dfbf4ae2 100644 --- a/content/docs/ai/skills-reference.mdx +++ b/content/docs/ai/skills-reference.mdx @@ -26,10 +26,10 @@ npm create objectstack@latest my-app **Existing projects — install the whole bundle:** ```bash -npx skills add objectstack-ai/framework --all +npx skills add objectstack-ai/framework/skills --all ``` -Run the same command again at any time to pull the latest versions — `--all` is idempotent, so updating is one command regardless of how many skills the bundle contains. To choose which skills to add instead of the full set, run `npx skills add objectstack-ai/framework` without `--all`. +Run the same command again at any time to pull the latest versions — `--all` is idempotent, so updating is one command regardless of how many skills the bundle contains. To choose which skills to add instead of the full set, run `npx skills add objectstack-ai/framework/skills` without `--all`. (The `/skills` subpath scopes discovery to the published catalog — always keep it in the command.) Skills are versioned as **one bundle**. You do not track or update each skill individually — `--all` always installs the matching set for the `@objectstack/spec` version in your project. diff --git a/content/docs/ai/skills.mdx b/content/docs/ai/skills.mdx index 081d944120..d647eda205 100644 --- a/content/docs/ai/skills.mdx +++ b/content/docs/ai/skills.mdx @@ -34,7 +34,7 @@ Skills are distributed through the [skills.sh](https://skills.sh) registry from npm create objectstack@latest my-app # Existing project — add the whole bundle (re-run to update) -npx skills add objectstack-ai/framework --all +npx skills add objectstack-ai/framework/skills --all ``` The bundle is versioned and updated as **one unit** — `--all` is idempotent, so you never track skills individually. For the full per-skill catalog and install options, see the [AI Skills Reference](/docs/ai/skills-reference#installing-skills). diff --git a/content/docs/getting-started/build-with-claude-code.mdx b/content/docs/getting-started/build-with-claude-code.mdx index 0b7510cd0c..fb88fd8400 100644 --- a/content/docs/getting-started/build-with-claude-code.mdx +++ b/content/docs/getting-started/build-with-claude-code.mdx @@ -56,7 +56,7 @@ The scaffolder does more than copy files. It: - derives a **namespace** from the name (`support-desk` → `support_desk`), so every object you create is named `support_desk_*`; - installs dependencies; -- runs `npx skills add objectstack-ai/framework --all` to install the **AI skills +- runs `npx skills add objectstack-ai/framework/skills --all` to install the **AI skills bundle**; - writes an **`AGENTS.md`** (and `.github/copilot-instructions.md`) that teach your coding agent the project layout, the naming rules, and — critically — *to run diff --git a/content/docs/getting-started/your-first-project.mdx b/content/docs/getting-started/your-first-project.mdx index 9acac917e2..328bd4bc21 100644 --- a/content/docs/getting-started/your-first-project.mdx +++ b/content/docs/getting-started/your-first-project.mdx @@ -44,7 +44,7 @@ The scaffolder (`create-objectstack`) does four things: renamed to carry that prefix (`my_app_note`), which is what `os validate` later enforces. 2. **Installs dependencies** (pnpm if available, otherwise npm). -3. **Installs the AI skills bundle** (`npx skills add objectstack-ai/framework --all`) +3. **Installs the AI skills bundle** (`npx skills add objectstack-ai/framework/skills --all`) so a coding agent is productive in the project from the first prompt. 4. **Writes `AGENTS.md`** (and `.github/copilot-instructions.md`) with the project conventions — including the rule to run `npm run validate` after diff --git a/packages/create-objectstack/README.md b/packages/create-objectstack/README.md index 11898bbda2..ec450fdea6 100644 --- a/packages/create-objectstack/README.md +++ b/packages/create-objectstack/README.md @@ -59,7 +59,8 @@ offline. (`blank_note` → `my_app_note`). 2. Installs dependencies (pnpm if available, otherwise npm). 3. Installs the ObjectStack AI skills bundle for coding agents - (`npx skills add objectstack-ai/framework --all`). + (`npx skills add objectstack-ai/framework/skills --all` — scoped to the + curated `skills/` catalog). 4. Writes `AGENTS.md` and `.github/copilot-instructions.md` with the project conventions — unless the template ships its own. diff --git a/packages/create-objectstack/src/index.ts b/packages/create-objectstack/src/index.ts index 53989302bf..3532f07c40 100644 --- a/packages/create-objectstack/src/index.ts +++ b/packages/create-objectstack/src/index.ts @@ -24,7 +24,10 @@ * - README.md first H1 * * Finally we run ` install` and (best-effort) install the ObjectStack - * skills bundle via `npx skills add objectstack-ai/framework --all`. + * skills bundle via `npx skills add objectstack-ai/framework/skills --all`. + * The `/skills` subpath scopes discovery to the curated, customer-published + * catalog — repo-internal skills (e.g. under `.claude/skills/`) must never + * reach scaffolded projects. */ import { Command } from 'commander'; @@ -477,7 +480,7 @@ const program = new Command() if (!options.skipInstall && !options.skipSkills) { printStep('Installing AI skills for your coding agent...'); try { - execSync('npx -y skills add objectstack-ai/framework --all', { + execSync('npx -y skills add objectstack-ai/framework/skills --all', { stdio: 'inherit', cwd: targetDir, }); @@ -485,7 +488,7 @@ const program = new Command() } catch { printWarning( 'Skills installation skipped. Run manually:\n' + - ' npx skills add objectstack-ai/framework', + ' npx skills add objectstack-ai/framework/skills', ); console.log(''); } @@ -507,7 +510,7 @@ const program = new Command() if (options.skipInstall || options.skipSkills) { console.log(''); console.log(chalk.bold(' AI Skills (recommended):')); - console.log(chalk.dim(' npx skills add objectstack-ai/framework')); + console.log(chalk.dim(' npx skills add objectstack-ai/framework/skills')); } console.log(''); } catch (error) { diff --git a/packages/create-objectstack/src/template-consistency.test.ts b/packages/create-objectstack/src/template-consistency.test.ts index 22db434966..f748d22a31 100644 --- a/packages/create-objectstack/src/template-consistency.test.ts +++ b/packages/create-objectstack/src/template-consistency.test.ts @@ -8,10 +8,12 @@ import { describe, it, expect } from 'vitest'; import fs from 'node:fs'; import path from 'node:path'; +import { execFileSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; import { syncObjectStackDeps } from './pkg-utils.js'; const pkgRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const repoRoot = path.resolve(pkgRoot, '..', '..'); const ownPkg = JSON.parse(fs.readFileSync(path.join(pkgRoot, 'package.json'), 'utf8')); const ownMajor = Number(ownPkg.version.split('.')[0]); @@ -82,6 +84,96 @@ describe('README template table', () => { }); }); +// Skills catalog boundary (15.1 third-party eval): scaffolded projects once +// received the repo-internal `dogfood-verification` skill because the +// scaffolder installed with a repo-wide `skills add … --all`, whose discovery +// also walks `.claude/skills/`. The published catalog is exactly the root +// `skills/` directory; everything else must stay repo-internal. +describe('skills catalog boundary', () => { + const frontmatterOf = (file: string): string => + /^---\n([\s\S]*?)\n---/.exec(fs.readFileSync(file, 'utf8'))?.[1] ?? ''; + const isMarkedInternal = (fm: string): boolean => + /^metadata:\s*$/m.test(fm) && /^ +internal:\s*true\s*(#.*)?$/m.test(fm); + + const trackedSkillFiles = execFileSync('git', ['ls-files', '*SKILL.md'], { + cwd: repoRoot, + encoding: 'utf8', + }) + .split('\n') + .filter(Boolean); + + it('finds the curated skills/ catalog (sanity)', () => { + expect( + trackedSkillFiles.filter((f) => f.startsWith('skills/')).length, + ).toBeGreaterThan(0); + }); + + it('marks every SKILL.md outside skills/ as metadata.internal', () => { + for (const rel of trackedSkillFiles) { + if (rel.startsWith('skills/')) continue; + expect( + isMarkedInternal(frontmatterOf(path.join(repoRoot, rel))), + `${rel} is outside the published skills/ catalog but is not hidden from ` + + 'the skills CLI. Add to its frontmatter:\n' + + 'metadata:\n internal: true\n' + + 'or move it into skills/ if it is meant for customers.', + ).toBe(true); + } + }); + + it('never marks a curated skills/ entry internal', () => { + for (const rel of trackedSkillFiles) { + if (!rel.startsWith('skills/')) continue; + expect( + isMarkedInternal(frontmatterOf(path.join(repoRoot, rel))), + `${rel} is in the published catalog but marked metadata.internal — ` + + 'customers would silently stop receiving it.', + ).toBe(false); + } + }); + + it('scaffolder installs from the curated skills/ subpath, not the repo root', () => { + expect(REGISTRY_SOURCE).toContain( + 'skills add objectstack-ai/framework/skills --all', + ); + expect(REGISTRY_SOURCE).not.toMatch( + /skills add objectstack-ai\/framework(?!\/skills)/, + ); + }); + + // The /skills subpath is the hard boundary: the skills CLI's `--all` + // implies `--skill '*'`, which INCLUDES metadata.internal skills — so any + // customer-facing surface advertising a repo-root install would leak + // internal skills again. + it('no customer-facing surface advertises a repo-root skills install', () => { + const surfaces = [ + 'content/docs', + 'skills', + 'packages/create-objectstack', + // this file mentions the bare form on purpose (needle + error message) + ':(exclude)packages/create-objectstack/src/template-consistency.test.ts', + ]; + let candidates = ''; + try { + candidates = execFileSync( + 'git', + ['grep', '-nF', 'skills add objectstack-ai/framework', '--', ...surfaces], + { cwd: repoRoot, encoding: 'utf8' }, + ); + } catch { + // git grep exits 1 on no matches — nothing to check then. + } + const rootInstalls = candidates + .split('\n') + .filter((line) => /skills add objectstack-ai\/framework(?!\/skills)/.test(line)); + expect( + rootInstalls, + 'these lines advertise `skills add objectstack-ai/framework` without ' + + 'the /skills subpath — repo-root + --all installs internal skills', + ).toEqual([]); + }); +}); + describe('syncObjectStackDeps', () => { it('rewrites @objectstack/* ranges in deps and devDeps', () => { const pkg = { diff --git a/packages/create-objectstack/src/templates/AGENTS.md b/packages/create-objectstack/src/templates/AGENTS.md index c8fad71059..790f606b05 100644 --- a/packages/create-objectstack/src/templates/AGENTS.md +++ b/packages/create-objectstack/src/templates/AGENTS.md @@ -77,7 +77,7 @@ This project uses ObjectStack skills from `objectstack-ai/framework`. Install or update skills with the standard [skills CLI](https://skills.sh/): ```bash -npx skills add objectstack-ai/framework +npx skills add objectstack-ai/framework/skills ``` Skills are triggered automatically based on task context: From ae59ac8c97fb74ce5547910d99cf10e5d4ca658f Mon Sep 17 00:00:00 2001 From: Jack Zhuang <277994282+os-zhuang@users.noreply.github.com> Date: Fri, 17 Jul 2026 17:57:35 +0800 Subject: [PATCH 2/2] fix(ci): strip ANSI codes before asserting the skills --list output The skills CLI colors the count even when piped (Found \e[32m9\e[39m skills), so the repo-root listing greps in the new skills-boundary step never matched in CI. The subpath-install set-equality assertion had already passed. Co-Authored-By: Claude Fable 5 --- .github/workflows/scaffold-e2e.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scaffold-e2e.yml b/.github/workflows/scaffold-e2e.yml index 065782dc70..79eb81f8e1 100644 --- a/.github/workflows/scaffold-e2e.yml +++ b/.github/workflows/scaffold-e2e.yml @@ -143,7 +143,9 @@ jobs: exit 1 fi - listing=$(npx -y skills add "$GITHUB_WORKSPACE" --list 2>&1) + # Strip ANSI color codes — the skills CLI colors the count even + # when piped ("Found \e[32m9\e[39m skills"), which broke the greps. + listing=$(npx -y skills add "$GITHUB_WORKSPACE" --list 2>&1 | sed 's/\x1b\[[0-9;]*m//g') echo "$listing" if echo "$listing" | grep -q 'dogfood-verification'; then echo "::error::repo-root discovery surfaced an internal skill (metadata.internal marker missing?)"