From 28866b9a0437015221e21704f0bdba66deca2e52 Mon Sep 17 00:00:00 2001 From: Gale W Date: Tue, 28 Jul 2026 21:13:09 -0400 Subject: [PATCH 1/6] docs: plan local agent framework guidance --- ROADMAP.md | 3 +- docs/maintainers/automation-suitability.md | 7 + .../design-agent-automation-workflow/SKILL.md | 14 +- .../references/automation-plan-template.md | 3 + .../references/framework-selection.md | 18 ++ .../references/local-agent-frameworks.md | 247 ++++++++++++++++++ .../test_design_agent_automation_workflow.py | 38 +++ 7 files changed, 327 insertions(+), 3 deletions(-) create mode 100644 plugins/productivity-skills/skills/design-agent-automation-workflow/references/local-agent-frameworks.md diff --git a/ROADMAP.md b/ROADMAP.md index 4250bc088..b4e2e0f06 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1160,7 +1160,8 @@ Planned - [ ] Add `productivity-skills:maintain-project-security` for canonical `SECURITY.md` maintenance. - [ ] Add `productivity-skills:maintain-project-support` for canonical `SUPPORT.md` maintenance. - [ ] Add a future `productivity-skills:maintain-project-hooks` workflow for repositories that intentionally use Codex Hooks. -- [ ] Forward-test `productivity-skills:design-agent-automation-workflow` and `productivity-skills:design-agent-eval-workflow` against real agent, automation, and eval planning requests before adding deterministic scaffolding scripts. +- [x] Forward-test `productivity-skills:design-agent-automation-workflow` against a local-first macOS agent-framework planning request. Record the framework inventory, inference-server/model-capability boundary, canonical labs, and stack-owned implementation handoffs before adding deterministic scaffolding scripts. +- [ ] Forward-test `productivity-skills:design-agent-eval-workflow` against real agent, automation, and eval planning requests before adding deterministic scaffolding scripts. - [ ] Add lightweight Productivity validation tooling for `SKILL.md`, frontmatter, and `agents/openai.yaml` alignment. - [ ] Add Productivity validation checks for README layout and active skill inventory consistency. - [ ] Add server-side Swift validation coverage for skill metadata and exported skill inventory once central Socket child-skill validation exists. diff --git a/docs/maintainers/automation-suitability.md b/docs/maintainers/automation-suitability.md index f789b9207..bb16ec5ff 100644 --- a/docs/maintainers/automation-suitability.md +++ b/docs/maintainers/automation-suitability.md @@ -3,6 +3,13 @@ This note audits the `productivity-skills` maintenance skills and the `apple-dev-skills` sync skills for cross-repository automation. +For a full local-first agent-framework planning pass, use +[`design-agent-automation-workflow`](../../plugins/productivity-skills/skills/design-agent-automation-workflow/SKILL.md) +and its [local framework reference](../../plugins/productivity-skills/skills/design-agent-automation-workflow/references/local-agent-frameworks.md). +This note keeps the Socket-maintainer automation decision; the skill owns +framework comparison, inference-server separation, canonical labs, and +implementation handoff. + The practical split is: - Use Codex app automations for scheduled reporting, check-only audits, and diff --git a/plugins/productivity-skills/skills/design-agent-automation-workflow/SKILL.md b/plugins/productivity-skills/skills/design-agent-automation-workflow/SKILL.md index b283fd71a..2ce57f21c 100644 --- a/plugins/productivity-skills/skills/design-agent-automation-workflow/SKILL.md +++ b/plugins/productivity-skills/skills/design-agent-automation-workflow/SKILL.md @@ -52,9 +52,14 @@ produce a scaffold another stack-owned skill or implementation pass can use. - Swift or Apple-platform implementation belongs in Apple/Swift-owned skills - Hermes-specific work belongs in Hermes docs or a Hermes-owned skill if one exists later -6. Produce a scaffold with the chosen surface, guardrails, validation plan, +6. When the request compares agent frameworks or local-first agent development, + use `references/local-agent-frameworks.md` to separate the orchestration + framework from the inference server, model capability, document/RAG, and + integration choices. Keep the recommendation tied to a concrete workflow; + do not recommend a framework merely because it is popular. +7. Produce a scaffold with the chosen surface, guardrails, validation plan, output contract, and next implementation handoff. -7. Link official docs for every framework or runtime named in the +8. Link official docs for every framework or runtime named in the recommendation. ## Decision Rules @@ -89,6 +94,10 @@ produce a scaffold another stack-owned skill or implementation pass can use. - Prefer no automation yet only when the goal, validation, owner, write scope, or rollback/escalation boundary is still unclear after trying to narrow the workflow. +- Treat a framework's advertised local-model integration as an adapter + capability, not proof that every local model can safely use tools, structured + output, long context, or multi-step planning. Plan an explicit model + capability check before granting write-capable tools. ## Output Contract @@ -132,4 +141,5 @@ prompt, issue body, project note, or implementation brief. - `agents/openai.yaml` - `references/framework-selection.md` +- `references/local-agent-frameworks.md` - `references/automation-plan-template.md` diff --git a/plugins/productivity-skills/skills/design-agent-automation-workflow/references/automation-plan-template.md b/plugins/productivity-skills/skills/design-agent-automation-workflow/references/automation-plan-template.md index 249fa6c0e..01185690b 100644 --- a/plugins/productivity-skills/skills/design-agent-automation-workflow/references/automation-plan-template.md +++ b/plugins/productivity-skills/skills/design-agent-automation-workflow/references/automation-plan-template.md @@ -29,6 +29,9 @@ Why this surface: - Trigger: - Automation target: +- Framework and why it fits this workflow: +- Inference server and model: +- Required model capabilities (tool calling, structured output, vision, context): - State that must persist: - Read scope: - Write scope: diff --git a/plugins/productivity-skills/skills/design-agent-automation-workflow/references/framework-selection.md b/plugins/productivity-skills/skills/design-agent-automation-workflow/references/framework-selection.md index c22fb0f55..f64638aad 100644 --- a/plugins/productivity-skills/skills/design-agent-automation-workflow/references/framework-selection.md +++ b/plugins/productivity-skills/skills/design-agent-automation-workflow/references/framework-selection.md @@ -13,6 +13,7 @@ automation surface. - LangGraph: - Hermes Agent: - Hermes Agent Hugging Face integration: +- Local-first framework planning: [local-agent-frameworks.md](./local-agent-frameworks.md) ## Selection Matrix @@ -28,6 +29,23 @@ automation surface. | Auto-with-escalation | Most of the workflow is safe, but one exact decision still needs user review when a named trigger fires. | The whole workflow is still underspecified, or escalation would happen on nearly every run. | | No automation yet | The outcome, validation, owner, approval gate, or rollback path is unclear after narrowing the workflow. | The user already has a bounded repeatable task with known checks and a safe first run. | +## Framework Choice Is A Separate Decision + +After choosing whether the work needs a code-owned agent service or durable +graph, choose the framework only from the requirements that remain: + +| Requirement that drives the choice | Start with | Do not infer | +| --- | --- | --- | +| Typed Python tools, dependency injection, and validated outputs | Pydantic AI or the OpenAI Agents SDK | That a typed schema makes a weak local model reliable at tool selection. | +| Persisted state transitions, pause/resume, and explicit routing | LangGraph | That LangChain itself is required for a retrieval-heavy product. | +| Document ingestion, retrieval, indexes, and RAG as the product core | LlamaIndex | That a vector database or multi-agent team is required for every document task. | +| Visual, trigger-driven integration between external services | n8n | That a visual workflow needs autonomous planning or durable agent memory. | +| Google, Gemini, A2A, MCP, or a multi-language implementation boundary | Google ADK | That ADK requires a Google-hosted model; its documented model adapters include local options. | +| Conversational multi-agent research or an existing Microsoft agent surface | AutoGen or Semantic Kernel | That role-playing several agents improves a workflow with one clear deterministic path. | + +Use `local-agent-frameworks.md` for the maintained framework inventory, +local-inference boundary, canonical labs, and implementation handoffs. + For read-heavy custom Codex subagent roles, prefer a role-local model choice over a global policy. `gpt-5.4-mini` is a good soft default for bounded exploration or audit workers; use a stronger model or omit the model override when the diff --git a/plugins/productivity-skills/skills/design-agent-automation-workflow/references/local-agent-frameworks.md b/plugins/productivity-skills/skills/design-agent-automation-workflow/references/local-agent-frameworks.md new file mode 100644 index 000000000..b4f23924b --- /dev/null +++ b/plugins/productivity-skills/skills/design-agent-automation-workflow/references/local-agent-frameworks.md @@ -0,0 +1,247 @@ +# Local Agent Frameworks + +Use this reference when planning a macOS-friendly agent application that may +run inference locally. It is a framework-selection and implementation-handoff +guide, not a claim that every framework should be installed or that a local +model is safe to give broad system access. + +## Outcome + +Choose the smallest framework that matches the workflow, then prove the chosen +model and tools can meet that workflow's contract. A local-first design should +retain three independent decisions: + +1. workflow and orchestration framework; +2. inference server and model; +3. tool, data, state, and approval boundaries. + +Do not collapse those choices into one "agent stack" decision. + +## Inference Server Is Not The Agent Framework + +An inference server runs a model and exposes a model API. An agent framework +plans calls, maintains state, invokes tools, and produces an application +contract. They can be replaced independently when the framework uses a +documented provider adapter or OpenAI-compatible endpoint. + +| Layer | Local-first choices | Plan for | +| --- | --- | --- | +| Inference server | [Ollama](https://ollama.com/), [LM Studio](https://lmstudio.ai/), or a project-owned `llama.cpp`/MLX server | Endpoint URL, authentication if remote, model lifecycle, concurrency, and memory use. | +| Model | A tool-capable local model selected for the actual task | Tool calling, structured output, context length, vision, latency, and quality evaluation. | +| Agent framework | One of the framework lanes below | State, retries, handoffs, traces, and integration ownership. | +| Application tools | Narrow functions, MCP servers, and service clients | Explicit input schema, least privilege, destructive-action confirmation, and deterministic validation. | +| Durable state | Files, SQLite, Postgres, a framework checkpointer, or an existing application store | What survives a process restart, retention, migrations, and recovery. | + +`Ollama` is the default first local server to document because the core +frameworks below publish explicit integrations. `LM Studio` is a useful local +alternative when its app-managed model workflow and OpenAI-compatible server +fit the developer better. Neither choice proves a model is adequate for an +agent loop. + +Before any write-capable tool is enabled, run a capability gate with the exact +model and server: valid tool-call JSON, schema-conforming structured output, +bounded retry behavior, a refusal/no-op path, and a read-only task set that +resembles the real workflow. + +## Core Framework Lanes + +### OpenAI Agents SDK + +Choose the OpenAI Agents SDK when the application is primarily an OpenAI agent +service: typed tools, handoffs, guardrails, approval points, traces, and a +server-owned execution loop. Socket Steward is the deliberately small, +read-only local example; it is not a mandate to build every agent as a service. + +- Good first lab: a read-only repository analyst that returns a typed audit + report and never invokes a mutating tool. +- Graduate only after: a fixture-backed tool test, explicit write confirmation, + trace capture, and a durable failure report. +- Handoff: the application owner and its Python/TypeScript stack workflow. +- Sources: + +### LangChain and LangGraph + +Use LangChain for its integration components. Choose LangGraph when the +workflow itself needs explicit graph transitions, persisted state, +pause/resume, streaming, retry handling, or a human decision at a named node. +Do not start with a graph for a single tool call or simple scheduled command. + +- Good first lab: read-only intake → classify → retrieve → draft → approval → + publish-draft, with the last node disabled until a reviewer approves it. +- Graduate only after: replay/resume tests, checkpoint migration planning, and + test fixtures for each transition. +- Handoff: Python or TypeScript implementation guidance; retrieval work may + instead belong in LlamaIndex. +- Sources: and + + +### LlamaIndex + +Choose LlamaIndex when the product's hard problem is document ingestion, +indexing, retrieval, citation/provenance, or RAG evaluation. Its agents should +be consumers of a deliberately designed knowledge layer, not a reason to add +retrieval to an otherwise simple automation. + +- Good first lab: index a small, permission-scoped document corpus, answer + questions with source citations, and return a typed result. +- Graduate only after: ingestion/update policy, retrieval relevance evals, + citation checks, and private-data retention boundaries. +- Handoff: Python implementation guidance and the application’s data owner. +- Sources: + and + +### n8n + +Choose n8n when the value is visible, trigger-driven integration among outside +services: a schedule, webhook, form, email, spreadsheet, or CRM feeds a mostly +deterministic workflow. Keep model calls narrow and keep the surrounding +routing, transformations, retries, and delivery deterministic. + +- Good first lab: a scheduled local-model classification that creates a draft + report and requires approval before it calls an external write node. +- Graduate only after: credential storage review, idempotency keys, failed-run + handling, and an exportable workflow fixture. +- Handoff: the owning integration/application project; do not use n8n as the + default durable multi-agent runtime. +- Sources: and + + +### Google Agent Development Kit (ADK) + +Choose Google Agent Development Kit (ADK) when the project needs a Google or +Gemini integration, A2A/MCP interoperability, a Google-supported agent +runtime, or a codebase spanning Python, TypeScript, Go, Java, or Kotlin. ADK +documents graph workflows and local-model adapters, including Ollama; that is +useful, but it does not make Google services a dependency of a local-first +application. + +- Good first lab: a local-Ollama tool agent with one read-only MCP or function + tool, session state, and evaluation fixtures. +- Graduate only after: adapter compatibility testing, session/memory retention + decisions, and an action-confirmation boundary. +- Handoff: the application language owner; Java/Kotlin work belongs in the JVM + lane, TypeScript in the web lane, and Python in the Python lane. +- Sources: + +### Pydantic AI + +Choose Pydantic AI for a Python-first agent that benefits from typed +dependencies, validated tool inputs, and structured result models without +adopting a large graph runtime. It is a strong small-service candidate, not a +replacement for durable graph execution when pause/resume and explicit state +transitions are the primary problem. + +- Good first lab: a typed local-Ollama research assistant whose only tool is a + read-only document lookup. +- Graduate only after: schema-negative tests, exact dependency injection + boundaries, and tool-call evaluation against the chosen local model. +- Handoff: Python implementation guidance. +- Sources: and + + +## Comparison And Conditional Lanes + +These frameworks deserve an explicit comparison, but should not become the +default merely because they advertise multi-agent support. + +| Framework | Consider it when | Keep out of the first pass when | Official source | +| --- | --- | --- | --- | +| AutoGen | The project specifically explores conversational multi-agent patterns or already uses Microsoft’s agent ecosystem. | A deterministic pipeline or one clear agent/tool loop solves the job. | | +| CrewAI | The requested product genuinely needs role-oriented teams with clear, independently testable responsibilities. | “Researcher/writer/reviewer” is only a prompt-role costume around one linear job. | | +| Semantic Kernel | The app is already C#/.NET-oriented or needs its plugin/function model in that ecosystem. | Adding .NET only to obtain an agent wrapper. | | +| Hermes Agent | The user intentionally targets the Hermes runtime’s own skills, memory, automations, messaging, and provider model. | A framework-neutral app or normal Socket-maintenance workflow. | | + +Keep a watch list rather than a permanent guide for fast-moving choices such as +Mastra, smolagents, Haystack, and vendor-specific agent builders. Promote one +only after it has a clear local-server story, official docs, an owner language, +and a workflow gap not covered by the core lanes. + +## Full-Layer Build Plan + +Every framework-specific guide should use the same layers, so a reader can +compare real engineering decisions rather than marketing vocabulary. + +### Layer 0: Problem And Non-Agent Baseline + +State the user outcome, existing deterministic alternative, trigger, inputs, +outputs, and success criteria. Show why a script, scheduled job, direct API +call, or n8n workflow is insufficient before introducing planning behavior. + +### Layer 1: Local Runtime And Model Contract + +Specify macOS runtime assumptions, inference server, endpoint, exact model, +model download/storage ownership, CPU/GPU/unified-memory budget, and whether +the model supports the required modalities. Test tool calling and structured +output against the real server instead of assuming OpenAI-compatible HTTP +means semantic compatibility. + +### Layer 2: Agent Contract And Tools + +Define one agent’s instruction, typed input/output, narrow tool schemas, data +classification, read/write permissions, and no-op/refusal behavior. Start with +read-only tools. Tool documentation must say what changed, where, and how to +verify it. + +### Layer 3: Workflow, State, And Recovery + +Document routing, checkpoints, retries, timeout/cancellation, idempotency, +approval interruptions, session/memory retention, and recovery after restart. +Use a graph only where this state model is actually visible and valuable. + +### Layer 4: Evaluation And Operations + +Add fixtures, expected structured outputs, tool-call assertions, regression +sets, trace/log redaction, cost and latency measurements, operator messages, +and a durable failure report. Local does not mean unobservable or unbounded. + +### Layer 5: Promotion And Deployment + +Document secrets, sandboxing, least-privilege service accounts, network +exposure, database migrations, backup/retention, rollout, rollback, and the +exact approval gate for external writes. A local macOS prototype must not be +silently reclassified as a background or public service. + +## Canonical Lab Matrix + +Use one small, comparable lab per lane. All labs begin read-only, use an +explicit typed result, retain no secret in source control, and return a report +or draft rather than performing an irreversible action. + +| Lab | Primary lane | Proves | Escalation point | +| --- | --- | --- | --- | +| Repository audit | OpenAI Agents SDK or Pydantic AI | Tool schema, typed output, trace, and local-model reliability. | Any suggested file modification becomes a separately approved proposal. | +| Approval-and-resume content workflow | LangGraph or ADK | Named state transition, checkpoint, pause/resume, and idempotent draft delivery. | Approval before the delivery node can write externally. | +| Private document research | LlamaIndex | Ingestion, retrieval, citations, and RAG evaluation. | Any corpus export, sharing, or index sync needs data-owner approval. | +| Scheduled integration draft | n8n | Trigger, deterministic routing, local model step, and failed-run recovery. | External write nodes stay disabled until the recipient/action is approved. | +| Multi-agent comparison | AutoGen or CrewAI | Whether a multi-role design beats the single-agent baseline on a fixed eval set. | Do not promote because the transcript looks persuasive; require measurable improvement. | + +## First Implementation Sequence + +1. Pick one real, bounded workflow and write the Layer 0 contract. +2. Serve one exact local model and pass the Layer 1 capability gate. +3. Build the read-only Layer 2 agent or workflow with fixtures. +4. Add only the Layer 3 state/recovery behavior the workflow demonstrably + needs. +5. Run Layer 4 evaluations before adding another agent, data source, or tool. +6. Use Layer 5 only when the project is deliberately moving beyond a local + prototype. + +Use `auto-with-escalation` by default for a workflow that is safe through its +draft/report phase but would create external changes. The first escalation +should name the exact action, target, and evidence a reviewer needs; it should +not become a vague request to approve “the agent.” + +## Ownership And Scope Boundaries + +- Keep framework selection, local-model capability gates, safety questions, + and the comparison matrix in `design-agent-automation-workflow`. +- Hand Python runtime code to Python-owned guidance; hand TypeScript/web work + to its project or web-owned guidance; hand JVM work to JVM-owned guidance. +- Keep model benchmarking, conversion, and local inference experiments in + `model-lab-skills` when that becomes the core task. +- Keep portable `SKILL.md` authoring and host adapters in + `agent-portability-skills`; a framework guide does not make its runtime + configuration portable. +- Do not add a shared framework wrapper, agent-manager abstraction, or + background daemon during planning. Those are durable architectural changes + and require a separately approved application need. diff --git a/plugins/productivity-skills/skills/design-agent-automation-workflow/tests/test_design_agent_automation_workflow.py b/plugins/productivity-skills/skills/design-agent-automation-workflow/tests/test_design_agent_automation_workflow.py index 5e7874b40..0d1edb7fe 100644 --- a/plugins/productivity-skills/skills/design-agent-automation-workflow/tests/test_design_agent_automation_workflow.py +++ b/plugins/productivity-skills/skills/design-agent-automation-workflow/tests/test_design_agent_automation_workflow.py @@ -46,6 +46,8 @@ def test_skill_body_preserves_planning_not_runtime_boundary() -> None: assert "Prefer safe full automation" in body assert "Use human review only for the exact" in body assert "Return a concise plan with these sections" in body + assert "local-agent-frameworks.md" in body + assert "capability check" in body def test_openai_interface_metadata_matches_skill() -> None: @@ -87,6 +89,42 @@ def test_framework_reference_covers_all_selection_surfaces() -> None: assert official_link in reference +def test_local_framework_reference_keeps_local_inference_separate() -> None: + reference = read(SKILL_ROOT / "references" / "local-agent-frameworks.md") + + for required in [ + "Inference Server Is Not The Agent Framework", + "OpenAI Agents SDK", + "LangChain and LangGraph", + "LlamaIndex", + "n8n", + "Google Agent Development Kit (ADK)", + "Pydantic AI", + "AutoGen", + "CrewAI", + "Semantic Kernel", + "Ollama", + "LM Studio", + "tool calling", + "structured output", + "read-only", + "auto-with-escalation", + ]: + assert required in reference + + for official_link in [ + "https://developers.openai.com/api/docs/guides/agents", + "https://docs.langchain.com/oss/python/langgraph/overview", + "https://docs.llamaindex.ai/en/latest/understanding/agent/structured_output/", + "https://n8n.io/integrations/ollama/", + "https://adk.dev/", + "https://pydantic.dev/docs/ai/models/ollama/", + "https://microsoft.github.io/autogen/stable/user-guide/agentchat-user-guide/tutorial/models.html", + "https://learn.microsoft.com/en-us/semantic-kernel/overview/", + ]: + assert official_link in reference + + def test_plan_template_has_required_output_sections() -> None: template = read(SKILL_ROOT / "references" / "automation-plan-template.md") From 2532324515bdcfa9504978cb08a0b16e8afc0516 Mon Sep 17 00:00:00 2001 From: Gale W Date: Tue, 28 Jul 2026 21:35:45 -0400 Subject: [PATCH 2/6] python: add local agent service guidance --- .../evaluate-tool-calling-model/SKILL.md | 28 +++- .../references/tool-evaluation-matrix.md | 14 ++ .../build-python-agent-service/SKILL.md | 152 ++++++++++++++++++ .../agents/openai.yaml | 8 + .../choose-python-project-shape/SKILL.md | 2 + .../test_build_python_agent_service_skill.py | 36 +++++ 6 files changed, 235 insertions(+), 5 deletions(-) create mode 100644 plugins/python-skills/skills/build-python-agent-service/SKILL.md create mode 100644 plugins/python-skills/skills/build-python-agent-service/agents/openai.yaml create mode 100644 plugins/python-skills/tests/test_build_python_agent_service_skill.py diff --git a/plugins/model-lab-skills/skills/evaluate-tool-calling-model/SKILL.md b/plugins/model-lab-skills/skills/evaluate-tool-calling-model/SKILL.md index 35396ae43..ac219c0d8 100644 --- a/plugins/model-lab-skills/skills/evaluate-tool-calling-model/SKILL.md +++ b/plugins/model-lab-skills/skills/evaluate-tool-calling-model/SKILL.md @@ -15,16 +15,34 @@ Evaluate these stages separately: whether a tool is needed, which tool is select 2. Build cases for correct calls, no-call answers, ambiguous choices, parallel calls, dependent calls, invalid arguments, tool errors, empty results, malicious tool output, authorization denial, and recovery. 3. Use deterministic fake tools with recorded inputs and outputs for the core suite. 4. Enforce authorization in the executor independently of model output. The model cannot grant itself a capability; intercept denied and irreversible calls before execution and record attempted versus executed side effects separately. -5. Pin model, prompt, schema serialization, parser, retry policy, maximum steps, and harness revision. -6. Score selection, arguments, ordering, execution result, recovery, final answer, latency, token use, attempted side effects, and executed side effects independently. -7. Detect invalid JSON, hallucinated tools, repeated calls, ignored errors, premature answers, and non-termination explicitly. -8. Run live integration cases only after the fake-tool suite passes and only inside approved side-effect boundaries. -9. Attribute failures to model, prompt, schema, parser, executor, or orchestration rather than collapsing everything into model accuracy. +5. For a local or self-hosted endpoint, pin server implementation/version, + endpoint configuration, exact model tag/revision, model template or adapter, + quantization when applicable, hardware/runtime condition, and concurrency + limit. OpenAI-compatible HTTP does not prove equivalent tool semantics. +6. Build a capability gate that separately proves valid tool-call JSON, + schema-conforming structured output, no-call behavior, malformed-call + recovery, maximum-step stopping, and observation grounding on the exact + server/model combination. +7. Pin model, prompt, schema serialization, parser, retry policy, maximum + steps, and harness revision. +8. Score selection, arguments, ordering, execution result, recovery, final + answer, latency, token use, attempted side effects, and executed side + effects independently. +9. Detect invalid JSON, hallucinated tools, repeated calls, ignored errors, + premature answers, and non-termination explicitly. +10. Run live integration cases only after the fake-tool suite passes and only + inside approved side-effect boundaries. +11. Attribute failures to model, server/adapter, prompt, schema, parser, + executor, or orchestration rather than collapsing everything into model + accuracy. ## Ownership Boundary This skill evaluates a model plus harness interface. Use `productivity-skills` when the primary artifact is an agent skill or plugin package, and `agent-portability-skills` when the question is host compatibility rather than behavioral quality. +Use `python-skills:build-python-agent-service` when the primary work is a +Python implementation rather than measurement. + ## References Read `references/tool-evaluation-matrix.md` for minimum cases and metrics. diff --git a/plugins/model-lab-skills/skills/evaluate-tool-calling-model/references/tool-evaluation-matrix.md b/plugins/model-lab-skills/skills/evaluate-tool-calling-model/references/tool-evaluation-matrix.md index 31bc2a75b..9301f5dad 100644 --- a/plugins/model-lab-skills/skills/evaluate-tool-calling-model/references/tool-evaluation-matrix.md +++ b/plugins/model-lab-skills/skills/evaluate-tool-calling-model/references/tool-evaluation-matrix.md @@ -14,3 +14,17 @@ Include at least one case for each cell that applies: Report exact-match schema validity separately from semantic argument correctness. A syntactically valid destructive call is not a success. Measure unnecessary-call rate and no-call correctness so tool eagerness does not masquerade as capability. The executor, not the model, owns capabilities. Use a dry-run recorder or approval gate before irreversible operations, and report attempted calls separately from calls the executor actually allowed. + +## Local Endpoint Capability Gate + +Before granting a local or self-hosted model a write-capable tool, record the +server implementation/version, endpoint configuration, exact model tag or +revision, model template/adapter, quantization when applicable, runtime +hardware condition, and concurrency limit. Then run the same fake-tool suite +through the real endpoint and retain each raw result. + +The gate passes only when the exact combination demonstrates valid tool-call +JSON, schema-conforming structured output, correct no-call behavior, malformed +call recovery, maximum-step stopping, and grounded observation use. An +OpenAI-compatible endpoint is a transport compatibility signal, not proof of +agent-loop behavior. diff --git a/plugins/python-skills/skills/build-python-agent-service/SKILL.md b/plugins/python-skills/skills/build-python-agent-service/SKILL.md new file mode 100644 index 000000000..e8cbeb58c --- /dev/null +++ b/plugins/python-skills/skills/build-python-agent-service/SKILL.md @@ -0,0 +1,152 @@ +--- +name: build-python-agent-service +description: Build or extend a local-first Python agent service with uv, typed tools, explicit model capability checks, durable state only when required, read-only-first execution, evaluation fixtures, and safe promotion gates. Use for OpenAI Agents SDK, LangGraph, LlamaIndex, Pydantic AI, Google ADK Python, AutoGen, or CrewAI implementations. +license: Apache-2.0 +compatibility: Designed for Codex and compatible Agent Skills clients building uv-managed Python agent services on macOS with local or remote model endpoints, typed tool contracts, and explicit validation. +metadata: + owner: gaelic-ghost + repo: python-skills + category: python-agent-service +allowed-tools: Read Bash(rg:*) Bash(git:*) Bash(uv:*) Bash(python:*) +--- + +# Build Python Agent Service + +## Purpose + +Build one bounded Python agent application without treating the model server, +agent framework, tool executor, and durable state as one inseparable stack. +Start from the smallest useful read-only agent and promote only after the exact +model, tools, evaluation fixtures, and side-effect boundary have been proved. + +## When To Use + +- Use for a new or existing uv-managed Python agent service. +- Use when the framework is OpenAI Agents SDK, LangGraph, LlamaIndex, Pydantic + AI, Google ADK Python, AutoGen, or CrewAI. +- Use after `design-agent-automation-workflow` has established that a + code-owned Python agent service is the right surface. +- Do not use for a visual integration workflow; hand off n8n work to the + owning integration project after the planning skill selects it. +- Do not use for model benchmarking itself; hand off local model capability and + tool-loop measurement to `model-lab-skills:evaluate-tool-calling-model`. + +## Source Check + +Before selecting or updating a framework, inspect the repository and use +official current documentation for the exact framework and model adapter: + +- OpenAI Agents SDK: +- LangGraph: +- LangChain Ollama: +- LlamaIndex agents: +- Pydantic AI: +- Pydantic AI Ollama: +- Google ADK: +- AutoGen models: +- CrewAI: +- uv: + +State which source changed the implementation decision. Do not rely on a +framework's claimed OpenAI-compatible endpoint as proof that a local model +supports tool calls or structured output correctly. + +## Implementation Sequence + +1. Inspect the current project shape, its `pyproject.toml`, existing model + client, tools, state store, tests, and deployment configuration. Prefer an + existing service/package boundary over adding a second agent host. +2. Write the non-agent baseline: trigger, inputs, expected typed output, + deterministic alternative, no-op behavior, and the reason planning/tool use + is necessary. +3. Select one framework for the real workflow: + - OpenAI Agents SDK for an application-owned agent loop with tools, + handoffs, guardrails, and traces. + - LangGraph when persisted transitions, pause/resume, retries, or explicit + routing are first-class behavior. + - LlamaIndex when ingestion, retrieval, citations, and RAG quality are the + core product problem. + - Pydantic AI for a compact typed Python service with validated tool and + result models. + - Google ADK Python when Google/Gemini, A2A/MCP, or ADK's runtime model is a + concrete product requirement. + - AutoGen or CrewAI only when a measured multi-agent design beats a + single-agent baseline on the same fixed task set. +4. Declare model endpoint, exact model name and revision/tag, authentication, + requested capabilities, context/latency limits, and model lifecycle. Keep + local server configuration out of committed secrets and machine-local paths. +5. Run a capability gate against the exact endpoint and model before attaching + write-capable tools: valid tool-call JSON, schema-conforming structured + output, no-call behavior, malformed-call recovery, maximum-step stop, and a + read-only task set resembling the intended application. +6. Implement one agent with typed input/output and narrow read-only tools. + Tool functions must validate their own authorization, inputs, timeout, and + result shape; model output cannot grant a capability. +7. Add durable state only when the user-visible workflow needs a restart-safe + session, checkpoint, task queue, or approval resume point. Name the store, + retention, migration, replay, and recovery contract. +8. Add the smallest test set: fake-tool unit cases, structured-output cases, + model-adapter integration smoke tests, denied-write cases, and regression + fixtures. Run live write tests only in an explicit disposable or draft mode. +9. Promote from report/draft to external writes only through + `auto-with-escalation`: name the exact recipient, target, action, evidence, + rollback/no-op behavior, and human approval point. + +## Framework Boundaries + +Do not add a framework wrapper merely to make framework names interchangeable. +Keep application domain behavior independent from the selected framework where +that boundary has a real caller: typed domain input/output, tool interfaces, +and persistence adapter. Let framework-specific orchestration stay at the +application edge. + +Do not introduce LangGraph persistence, vector retrieval, multi-agent teams, +or a background queue unless the selected workflow requires its concrete +behavior. A single request/response tool loop should remain a small service or +CLI. + +## Validation + +At minimum, run the repository's configured quality checks. In a standard uv +project that means: + +```bash +uv sync --dev +uv run pytest +uv run ruff check . +uv run mypy . +``` + +Report separately: + +1. fake-tool contract results; +2. exact local/remote model capability-gate results; +3. structured result validity; +4. attempted versus executed side effects; +5. state/resume behavior, when state exists; +6. the exact approval or no-op result for write-capable tools. + +## Output Shape + +Return: + +1. `Framework`: selected framework and the concrete requirement it serves. +2. `Model contract`: server, exact model, capabilities proven, and limitations. +3. `Tool boundary`: tool schemas, permissions, and denied-action behavior. +4. `State`: absent or explicit persistence/recovery contract. +5. `Evaluation`: fixture, fake-tool, and live-integration evidence. +6. `Promotion gate`: exact condition for an external write. +7. `Validation`: commands run and results. + +## Guardrails + +- Do not install several frameworks for a comparison unless the experiment is + explicitly requested and has one fixed evaluation set. +- Do not run an unattended local background service, scheduler, or external + write workflow without an explicit user request and a recovery plan. +- Do not store model API keys, local endpoint credentials, or private prompt + data in source control, fixtures, traces, or error output. +- Do not call a local model private merely because it runs on macOS; document + every connected tool, remote endpoint, trace sink, and data store. +- Do not claim a model supports tools, structured output, or a context size + until the exact server/model combination passes the capability gate. diff --git a/plugins/python-skills/skills/build-python-agent-service/agents/openai.yaml b/plugins/python-skills/skills/build-python-agent-service/agents/openai.yaml new file mode 100644 index 000000000..720eea0b7 --- /dev/null +++ b/plugins/python-skills/skills/build-python-agent-service/agents/openai.yaml @@ -0,0 +1,8 @@ +interface: + display_name: "Build Python Agent Service" + short_description: "Build a tested local-first Python agent service." + brand_color: "#4F46E5" + default_prompt: "Use $build-python-agent-service to select one Python agent framework for this bounded workflow, separate its inference server and exact model from its tool/state boundaries, prove read-only tool calling and structured output first, and name the exact approval gate before any external write." + +policy: + allow_implicit_invocation: true diff --git a/plugins/python-skills/skills/choose-python-project-shape/SKILL.md b/plugins/python-skills/skills/choose-python-project-shape/SKILL.md index 1e55602dd..903c499c6 100644 --- a/plugins/python-skills/skills/choose-python-project-shape/SKILL.md +++ b/plugins/python-skills/skills/choose-python-project-shape/SKILL.md @@ -52,6 +52,7 @@ Translate any documentation rule into the concrete repository decision it change - FastAPI service - FastMCP server - combined FastAPI and FastMCP app + - local-first Python agent service - test or tooling setup - package maintenance - CI maintenance @@ -75,6 +76,7 @@ Translate any documentation rule into the concrete repository decision it change - implementation: `build-python-project` - test work: `uv-pytest-unit-testing` - FastAPI/FastMCP integration: `integrate-fastapi-fastmcp` + - local-first agent service: `build-python-agent-service` - diagnosis: `diagnose-python-project` - package validation: `python-package-workflow` - tooling alignment: `python-tooling-style-workflow` diff --git a/plugins/python-skills/tests/test_build_python_agent_service_skill.py b/plugins/python-skills/tests/test_build_python_agent_service_skill.py new file mode 100644 index 000000000..050889f2b --- /dev/null +++ b/plugins/python-skills/tests/test_build_python_agent_service_skill.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from pathlib import Path + +import yaml + + +SKILL_ROOT = Path(__file__).resolve().parents[1] / "skills" / "build-python-agent-service" + + +def test_agent_service_skill_has_local_first_framework_and_safety_contract() -> None: + skill = (SKILL_ROOT / "SKILL.md").read_text(encoding="utf-8") + + for required in [ + "OpenAI Agents SDK", + "LangGraph", + "LlamaIndex", + "Pydantic AI", + "Google ADK Python", + "AutoGen", + "CrewAI", + "capability gate", + "read-only", + "auto-with-escalation", + "attempted versus executed side effects", + ]: + assert required in skill + + +def test_agent_service_interface_mentions_exact_model_and_approval() -> None: + metadata = yaml.safe_load((SKILL_ROOT / "agents" / "openai.yaml").read_text(encoding="utf-8")) + interface = metadata["interface"] + + assert interface["display_name"] == "Build Python Agent Service" + assert "exact model" in interface["default_prompt"] + assert "approval gate" in interface["default_prompt"] From d7e999040ae2999d91e7ddef6bf4246efea39265 Mon Sep 17 00:00:00 2001 From: Gale W Date: Tue, 28 Jul 2026 21:39:18 -0400 Subject: [PATCH 3/6] plugin: add local agent framework workflows --- README.md | 8 +- ROADMAP.md | 1 + .../build-dotnet-agent-service/SKILL.md | 57 +++++++ .../references/local-agent-frameworks.md | 7 +- .../test_design_agent_automation_workflow.py | 15 ++ .../skills/design-n8n-agent-workflow/SKILL.md | 45 ++++++ .../skills/build-jvm-agent-service/SKILL.md | 65 ++++++++ scripts/export_hermes_skills.py | 13 ++ skills.sh.json | 21 ++- skills/build-dotnet-agent-service/SKILL.md | 57 +++++++ skills/build-jvm-agent-service/SKILL.md | 65 ++++++++ skills/build-python-agent-service/SKILL.md | 152 ++++++++++++++++++ .../agents/openai.yaml | 8 + skills/design-n8n-agent-workflow/SKILL.md | 45 ++++++ skills/evaluate-tool-calling-model/SKILL.md | 28 +++- .../references/tool-evaluation-matrix.md | 14 ++ 16 files changed, 589 insertions(+), 12 deletions(-) create mode 100644 plugins/dotnet-skills/skills/build-dotnet-agent-service/SKILL.md create mode 100644 plugins/productivity-skills/skills/design-n8n-agent-workflow/SKILL.md create mode 100644 plugins/server-side-jvm/skills/build-jvm-agent-service/SKILL.md create mode 100644 skills/build-dotnet-agent-service/SKILL.md create mode 100644 skills/build-jvm-agent-service/SKILL.md create mode 100644 skills/build-python-agent-service/SKILL.md create mode 100644 skills/build-python-agent-service/agents/openai.yaml create mode 100644 skills/design-n8n-agent-workflow/SKILL.md diff --git a/README.md b/README.md index c84f1d82f..fba6959c5 100644 --- a/README.md +++ b/README.md @@ -156,13 +156,13 @@ Current Socket catalog shape: - `messaging-collaboration-skills`: chat-app, bot, business-messaging, meeting-collaboration, iMessage collaboration, Communication Notifications, Push to Talk, VoIP/SIP, documented iOS/iPadOS default communication roles, and app-owned macOS client workflows for Discord, Telegram, Slack, Teams, WhatsApp Business, SMS/MMS/RCS, Google Meet, and Apple communication surfaces, with explicit Signal and Mac operator-automation boundaries - `model-lab-skills`: reproducible language-model experiment design, dataset preparation, fine-tuning, evaluation, checkpoint comparison, representation and steering research, refusal ablation, authorized jailbreak and tool-calling evaluation, runtime benchmarking, and current-source routing across Core AI, Core ML, MLX, ExecuTorch, and Foundation Models - `agentdeck`: local Codex runtime utilities, starting with hooks that prefix generated Codex thread titles with the project directory name -- `dotnet-skills`: .NET, F#, and C# project-shape, bootstrap, implementation, test, package, diagnostics, ASP.NET Core, Giraffe, Falco, Oxpecker, interop, CI, upgrade, and tooling guidance +- `dotnet-skills`: .NET, F#, and C# project-shape, bootstrap, implementation, local-first Semantic Kernel agent-service, test, package, diagnostics, ASP.NET Core, Giraffe, Falco, Oxpecker, interop, CI, upgrade, and tooling guidance - `game-dev-skills`: Apple platform game development workflows for native Metal and Metal 4 renderers, GPTK 3/4 routing, MetalFX, GPU asset streaming, experimental neural rendering, SpriteKit, SceneKit, GameplayKit simulation, Game Controller input, Core Haptics feedback, Xcode profiling, game-stack routing, and device-aware validation handoffs - `network-protocol-skills`: modern networking and application-protocol workflows for transport selection, HTTP/3 and QUIC planning, Media over QUIC draft-aware guidance, WebRTC signaling/media/data-channel work, and protocol diagnostics with stack-plugin handoffs -- `productivity-skills`: general-purpose maintainer, documentation, Dice MCP job-search with bundled remote MCP config, Codex GUI worktree workflow, and automation-design workflows plus source-bundled docs-audit and code-tracing custom-agent definitions -- `python-skills`: Python runtime and tooling workflows for Python-based projects; see the [Python skills expansion plan](./docs/maintainers/python-skills-plugin-plan.md) for maintainer details +- `productivity-skills`: general-purpose maintainer, documentation, Dice MCP job-search with bundled remote MCP config, Codex GUI worktree workflow, automation-design and safe n8n-workflow planning, plus source-bundled docs-audit and code-tracing custom-agent definitions +- `python-skills`: Python runtime and tooling workflows, including local-first agent-service implementation guidance; see the [Python skills expansion plan](./docs/maintainers/python-skills-plugin-plan.md) for maintainer details - `reverse-engineering-skills`: artifact triage, preservation, exact-build comparison, decompiler review, Apple Mach-O/runtime/signing/Apple Silicon/dyld/dynamic/kernel research, Cutter/Rizin, Malimite, Ghidra, Hopper, .NET, Unity and IL2CPP, and reproducible security evidence workflows -- `server-side-jvm`: server-side JVM, Java, Scala, Gradle, Maven, SBT, and testing workflow guidance, with future Clojure support planned +- `server-side-jvm`: server-side JVM, Java, Scala, Google ADK agent-service, Gradle, Maven, SBT, and testing workflow guidance, with future Clojure support planned - `server-side-swift`: server-side Swift bootstrap and guidance sync, Vapor, Hummingbird, persistence, OpenAPI/RPC, SwiftNIO, observability, auth, app sync, Docker, Apple `container` 1.x, persistent `container machine` environments, exact-version Containerization APIs, and Fly.io support plus the source-bundled `server-swift-steward` custom-agent definition - `swift-lang`: shared Swift language, API style, error handling, functional pipelines, formatting, source organization, SwiftSyntax transformation, compiler inspection, SourceKit semantics and indexing, SourceKit-LSP diagnosis, Swiftly/Xcode toolchain routing, and modernization cleanup workflows - `rust-skills`: Rust, Cargo, rustup, crate, workspace, CLI, library, package, CI, test, lint, and format workflow guidance diff --git a/ROADMAP.md b/ROADMAP.md index b4e2e0f06..c2461defc 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1161,6 +1161,7 @@ Planned - [ ] Add `productivity-skills:maintain-project-support` for canonical `SUPPORT.md` maintenance. - [ ] Add a future `productivity-skills:maintain-project-hooks` workflow for repositories that intentionally use Codex Hooks. - [x] Forward-test `productivity-skills:design-agent-automation-workflow` against a local-first macOS agent-framework planning request. Record the framework inventory, inference-server/model-capability boundary, canonical labs, and stack-owned implementation handoffs before adding deterministic scaffolding scripts. +- [x] Implement the resulting local-first agent-framework guidance across Productivity Skills, Python Skills, Model Lab Skills, Server-Side JVM, and .NET Skills. Keep framework selection separate from local inference, require exact model/server capability gates, and retain draft-before-write escalation boundaries. - [ ] Forward-test `productivity-skills:design-agent-eval-workflow` against real agent, automation, and eval planning requests before adding deterministic scaffolding scripts. - [ ] Add lightweight Productivity validation tooling for `SKILL.md`, frontmatter, and `agents/openai.yaml` alignment. - [ ] Add Productivity validation checks for README layout and active skill inventory consistency. diff --git a/plugins/dotnet-skills/skills/build-dotnet-agent-service/SKILL.md b/plugins/dotnet-skills/skills/build-dotnet-agent-service/SKILL.md new file mode 100644 index 000000000..625b3b10b --- /dev/null +++ b/plugins/dotnet-skills/skills/build-dotnet-agent-service/SKILL.md @@ -0,0 +1,57 @@ +--- +name: build-dotnet-agent-service +description: Build a local-first F# or C# Semantic Kernel agent service with explicit tools, model capability checks, evaluation fixtures, and draft-before-write promotion. +license: Apache-2.0 +compatibility: Designed for Codex and compatible Agent Skills clients working with F#, C#, ASP.NET Core, and the dotnet CLI on macOS or other supported .NET environments. +metadata: + owner: gaelic-ghost + repo: socket + category: dotnet-agent-service +allowed-tools: Read Bash(rg:*) Bash(git:*) Bash(dotnet:*) +--- + +# Build .NET Agent Service + +Use this skill when Semantic Kernel's plugin/function orchestration solves a +real .NET application need. Preserve F# and C# equality: ask for the language +when a new project has no established choice, and do not add C# only to host an +agent. + +## Source Check + +- Semantic Kernel: +- .NET: +- ASP.NET Core: + +## Workflow + +1. Establish the deterministic baseline, typed input/output, chosen language, + exact model endpoint/model, tool permissions, and no-op result. +2. Use Semantic Kernel only for a named plugin/function, planning, or + orchestration requirement. Keep domain logic in F# modules or C# domain + types rather than embedding it in prompt or endpoint code. +3. Start with narrow read-only functions. The host validates authorization, + schemas, timeouts, and output independently of model responses. +4. Prove valid tool-call JSON, structured output, no-call behavior, malformed + call recovery, maximum-step stopping, and grounded observations on the exact + local/self-hosted server and model; HTTP compatibility alone is insufficient. +5. Add memory, planners, background work, or an ASP.NET endpoint only when the + application requires it. Name retention, restart, and failure behavior. +6. Add fake-plugin tests, structured-result tests, denied-write tests, and + opt-in local-model smoke tests. External writes remain draft-only until an + exact `auto-with-escalation` approval gate is satisfied. + +## Validation + +Run the repository's narrowest `dotnet build` and `dotnet test` commands. +Separate fake-tool behavior from model-adapter smoke evidence, and report +attempted versus executed side effects. + +## Guardrails + +- Do not add Semantic Kernel to a library or service that only needs one direct + model request. +- Do not store secrets, local endpoint credentials, prompts, or traces in + source control. +- Do not start a background agent, schedule, or public endpoint by default. +- Do not use model output as permission to call a destructive plugin. diff --git a/plugins/productivity-skills/skills/design-agent-automation-workflow/references/local-agent-frameworks.md b/plugins/productivity-skills/skills/design-agent-automation-workflow/references/local-agent-frameworks.md index b4f23924b..6b9dad583 100644 --- a/plugins/productivity-skills/skills/design-agent-automation-workflow/references/local-agent-frameworks.md +++ b/plugins/productivity-skills/skills/design-agent-automation-workflow/references/local-agent-frameworks.md @@ -105,6 +105,8 @@ routing, transformations, retries, and delivery deterministic. default durable multi-agent runtime. - Sources: and +- Handoff: `productivity-skills:design-n8n-agent-workflow` for the concrete + n8n node, credential, idempotency, draft, and approval design. ### Google Agent Development Kit (ADK) @@ -120,7 +122,8 @@ application. - Graduate only after: adapter compatibility testing, session/memory retention decisions, and an action-confirmation boundary. - Handoff: the application language owner; Java/Kotlin work belongs in the JVM - lane, TypeScript in the web lane, and Python in the Python lane. + lane through `server-side-jvm:build-jvm-agent-service`, TypeScript in the web + lane, and Python in the Python lane. - Sources: ### Pydantic AI @@ -148,7 +151,7 @@ default merely because they advertise multi-agent support. | --- | --- | --- | --- | | AutoGen | The project specifically explores conversational multi-agent patterns or already uses Microsoft’s agent ecosystem. | A deterministic pipeline or one clear agent/tool loop solves the job. | | | CrewAI | The requested product genuinely needs role-oriented teams with clear, independently testable responsibilities. | “Researcher/writer/reviewer” is only a prompt-role costume around one linear job. | | -| Semantic Kernel | The app is already C#/.NET-oriented or needs its plugin/function model in that ecosystem. | Adding .NET only to obtain an agent wrapper. | | +| Semantic Kernel | The app is already C#/.NET-oriented or needs its plugin/function model in that ecosystem; hand implementation to `dotnet-skills:build-dotnet-agent-service`. | Adding .NET only to obtain an agent wrapper. | | | Hermes Agent | The user intentionally targets the Hermes runtime’s own skills, memory, automations, messaging, and provider model. | A framework-neutral app or normal Socket-maintenance workflow. | | Keep a watch list rather than a permanent guide for fast-moving choices such as diff --git a/plugins/productivity-skills/skills/design-agent-automation-workflow/tests/test_design_agent_automation_workflow.py b/plugins/productivity-skills/skills/design-agent-automation-workflow/tests/test_design_agent_automation_workflow.py index 0d1edb7fe..f02ff380a 100644 --- a/plugins/productivity-skills/skills/design-agent-automation-workflow/tests/test_design_agent_automation_workflow.py +++ b/plugins/productivity-skills/skills/design-agent-automation-workflow/tests/test_design_agent_automation_workflow.py @@ -138,3 +138,18 @@ def test_plan_template_has_required_output_sections() -> None: "## Sources", ]: assert heading in template + + +def test_n8n_workflow_skill_keeps_visual_automation_deterministic() -> None: + skill = (SKILL_ROOT.parent / "design-n8n-agent-workflow" / "SKILL.md").read_text( + encoding="utf-8" + ) + + for required in [ + "deterministic", + "idempotency", + "draft-only", + "auto-with-escalation", + "capability gate", + ]: + assert required in skill diff --git a/plugins/productivity-skills/skills/design-n8n-agent-workflow/SKILL.md b/plugins/productivity-skills/skills/design-n8n-agent-workflow/SKILL.md new file mode 100644 index 000000000..96aaaef70 --- /dev/null +++ b/plugins/productivity-skills/skills/design-n8n-agent-workflow/SKILL.md @@ -0,0 +1,45 @@ +--- +name: design-n8n-agent-workflow +description: Design safe n8n workflows with deterministic routing, credentials, idempotency, recovery, local-model checks, drafts, and exact approval gates. +--- + +# Design n8n Agent Workflow + +Use this skill when n8n is the right visual, trigger-driven integration surface. +It does not turn n8n into a default durable multi-agent runtime. + +## Source Check + +- n8n hosting: +- n8n and Ollama: +- Ollama: + +## Workflow + +1. Map trigger, deterministic transforms/routes, external reads, model step, + draft output, and any external write separately. +2. Select credentials/storage ownership, webhook exposure, retry policy, + idempotency key, concurrency limit, and failed-run record before connecting + an AI node. +3. Treat the local model as one bounded node: prove structured output, tool + compatibility when used, no-call behavior, malformed result recovery, and + the exact model/server combination before exposing a write node. +4. Keep source data minimised and trace/log retention explicit. Do not put + secrets in workflow exports or prompts. +5. Leave write nodes disabled or draft-only until `auto-with-escalation` names + the recipient, action, payload evidence, and approval event. +6. Export a testable workflow fixture and verify happy path, duplicate event, + failed model response, credential failure, rejected approval, and recovery. + +## Output Shape + +Return trigger, node graph, credentials, idempotency/retry behavior, model +contract, draft artifact, approval event, and recovery evidence. + +## Guardrails + +- Prefer deterministic n8n nodes over agent planning whenever the route is + known. +- Do not expose a self-hosted instance publicly without an explicit deployment + and authentication decision. +- Do not claim a local model is reliable for tool use without a capability gate. diff --git a/plugins/server-side-jvm/skills/build-jvm-agent-service/SKILL.md b/plugins/server-side-jvm/skills/build-jvm-agent-service/SKILL.md new file mode 100644 index 000000000..f059d6870 --- /dev/null +++ b/plugins/server-side-jvm/skills/build-jvm-agent-service/SKILL.md @@ -0,0 +1,65 @@ +--- +name: build-jvm-agent-service +description: Build a local-first Java or Kotlin Google ADK agent service with explicit tools, model capability checks, evaluation fixtures, and draft-before-write promotion. +license: Apache-2.0 +compatibility: Designed for Codex and compatible Agent Skills clients working with Java or Kotlin server-side JVM services on macOS or other supported JVM environments. +metadata: + owner: gaelic-ghost + repo: socket + category: jvm-agent-service +allowed-tools: Read Bash(rg:*) Bash(git:*) Bash(java:*) Bash(./gradlew:*) Bash(mvn:*) Bash(sbt:*) +--- + +# Build JVM Agent Service + +Use this skill after `choose-service-shape` identifies a JVM service and the +application needs Google Agent Development Kit (ADK), not merely one model API +call. Keep Java/Kotlin domain behavior independent from ADK orchestration at +the application edge. + +## Source Check + +- Google ADK: +- ADK Java API: +- Java: +- Kotlin: + +Confirm the exact ADK language/runtime and model adapter before changing code. +ADK documents local-model adapters, but an Ollama-compatible endpoint must +still pass the exact model capability gate below. + +## Workflow + +1. Keep the existing Java/Kotlin language decision and build tool; do not add a + second JVM language merely for an agent library. +2. Define the non-agent baseline, typed request/result, model endpoint/exact + model, allowed tools, and no-op behavior. +3. Use one ADK agent with narrow read-only function or MCP tools. The executor + validates authorization, schema, timeout, and result shape independently of + model output. +4. Prove the exact server/model combination can make valid tool calls, return + schema-conforming structured output, decline unnecessary calls, recover from + malformed calls, and stop at a maximum-step boundary. +5. Add sessions, memory, graph workflows, A2A, or persistence only when the + product visibly needs resume, routing, or cross-agent behavior. Document + retention, restart recovery, and migrations. +6. Test fake tools first, then model-adapter integration in a read-only or + disposable environment. Report attempted and executed side effects apart. +7. Promote external writes only through `auto-with-escalation` with the exact + target, action, evidence, and approval point named. + +## Validation + +Use the existing build owner: `./gradlew test`, `mvn test`, or `sbt test`. +Include unit tests for domain/tool authorization, agent workflow tests with +fake tools, and an opt-in local-model capability smoke test. Do not make a +downloaded model or live endpoint a normal unit-test prerequisite. + +## Guardrails + +- Do not make Gemini or Google Cloud a required dependency of a local-first + service unless the product explicitly chooses it. +- Do not expose an agent API publicly or start a background runtime by default. +- Do not trust an agent instruction as an authorization boundary. +- Do not add A2A, MCP, graphs, or multi-agent delegation without a concrete + caller and a testable state/recovery need. diff --git a/scripts/export_hermes_skills.py b/scripts/export_hermes_skills.py index 17a435e57..5a9a60f6d 100644 --- a/scripts/export_hermes_skills.py +++ b/scripts/export_hermes_skills.py @@ -24,6 +24,9 @@ SWIFT_LANG_SOURCE_ROOT = REPO_ROOT / "plugins" / "swift-lang" / "skills" MODEL_LAB_SOURCE_ROOT = REPO_ROOT / "plugins" / "model-lab-skills" / "skills" DOTNET_SOURCE_ROOT = REPO_ROOT / "plugins" / "dotnet-skills" / "skills" +PRODUCTIVITY_SOURCE_ROOT = REPO_ROOT / "plugins" / "productivity-skills" / "skills" +PYTHON_SOURCE_ROOT = REPO_ROOT / "plugins" / "python-skills" / "skills" +JVM_SOURCE_ROOT = REPO_ROOT / "plugins" / "server-side-jvm" / "skills" CLOUD_DEPLOYMENT_SOURCE_ROOT = REPO_ROOT / "plugins" / "cloud-deployment-skills" / "skills" EXPORT_ROOT = REPO_ROOT / "skills" AGENT_PORTABILITY_SKILLS = ( @@ -137,7 +140,11 @@ "build-giraffe-web-app", "build-falco-web-app", "build-oxpecker-web-app", + "build-dotnet-agent-service", ) +PRODUCTIVITY_SKILLS = ("design-n8n-agent-workflow",) +PYTHON_SKILLS = ("build-python-agent-service",) +JVM_SKILLS = ("build-jvm-agent-service",) CLOUD_DEPLOYMENT_SKILLS = ("cloud-deployment-routing-workflow",) EXPORTED_SKILLS = ( AGENT_PORTABILITY_SKILLS @@ -149,6 +156,9 @@ + SWIFT_LANG_SKILLS + MODEL_LAB_SKILLS + DOTNET_SKILLS + + PRODUCTIVITY_SKILLS + + PYTHON_SKILLS + + JVM_SKILLS + CLOUD_DEPLOYMENT_SKILLS ) @@ -178,6 +188,9 @@ def source_paths(source_root: Path | None = None) -> dict[str, Path]: **{skill_name: SWIFT_LANG_SOURCE_ROOT for skill_name in SWIFT_LANG_SKILLS}, **{skill_name: MODEL_LAB_SOURCE_ROOT for skill_name in MODEL_LAB_SKILLS}, **{skill_name: DOTNET_SOURCE_ROOT for skill_name in DOTNET_SKILLS}, + **{skill_name: PRODUCTIVITY_SOURCE_ROOT for skill_name in PRODUCTIVITY_SKILLS}, + **{skill_name: PYTHON_SOURCE_ROOT for skill_name in PYTHON_SKILLS}, + **{skill_name: JVM_SOURCE_ROOT for skill_name in JVM_SKILLS}, **{ skill_name: CLOUD_DEPLOYMENT_SOURCE_ROOT for skill_name in CLOUD_DEPLOYMENT_SKILLS diff --git a/skills.sh.json b/skills.sh.json index 7b8cd3521..4b3a82f5c 100644 --- a/skills.sh.json +++ b/skills.sh.json @@ -99,7 +99,26 @@ "choose-fsharp-web-framework", "build-giraffe-web-app", "build-falco-web-app", - "build-oxpecker-web-app" + "build-oxpecker-web-app", + "build-dotnet-agent-service" + ] + }, + { + "title": "Productivity Skills", + "skills": [ + "design-n8n-agent-workflow" + ] + }, + { + "title": "Python Skills", + "skills": [ + "build-python-agent-service" + ] + }, + { + "title": "Server-Side JVM Skills", + "skills": [ + "build-jvm-agent-service" ] }, { diff --git a/skills/build-dotnet-agent-service/SKILL.md b/skills/build-dotnet-agent-service/SKILL.md new file mode 100644 index 000000000..625b3b10b --- /dev/null +++ b/skills/build-dotnet-agent-service/SKILL.md @@ -0,0 +1,57 @@ +--- +name: build-dotnet-agent-service +description: Build a local-first F# or C# Semantic Kernel agent service with explicit tools, model capability checks, evaluation fixtures, and draft-before-write promotion. +license: Apache-2.0 +compatibility: Designed for Codex and compatible Agent Skills clients working with F#, C#, ASP.NET Core, and the dotnet CLI on macOS or other supported .NET environments. +metadata: + owner: gaelic-ghost + repo: socket + category: dotnet-agent-service +allowed-tools: Read Bash(rg:*) Bash(git:*) Bash(dotnet:*) +--- + +# Build .NET Agent Service + +Use this skill when Semantic Kernel's plugin/function orchestration solves a +real .NET application need. Preserve F# and C# equality: ask for the language +when a new project has no established choice, and do not add C# only to host an +agent. + +## Source Check + +- Semantic Kernel: +- .NET: +- ASP.NET Core: + +## Workflow + +1. Establish the deterministic baseline, typed input/output, chosen language, + exact model endpoint/model, tool permissions, and no-op result. +2. Use Semantic Kernel only for a named plugin/function, planning, or + orchestration requirement. Keep domain logic in F# modules or C# domain + types rather than embedding it in prompt or endpoint code. +3. Start with narrow read-only functions. The host validates authorization, + schemas, timeouts, and output independently of model responses. +4. Prove valid tool-call JSON, structured output, no-call behavior, malformed + call recovery, maximum-step stopping, and grounded observations on the exact + local/self-hosted server and model; HTTP compatibility alone is insufficient. +5. Add memory, planners, background work, or an ASP.NET endpoint only when the + application requires it. Name retention, restart, and failure behavior. +6. Add fake-plugin tests, structured-result tests, denied-write tests, and + opt-in local-model smoke tests. External writes remain draft-only until an + exact `auto-with-escalation` approval gate is satisfied. + +## Validation + +Run the repository's narrowest `dotnet build` and `dotnet test` commands. +Separate fake-tool behavior from model-adapter smoke evidence, and report +attempted versus executed side effects. + +## Guardrails + +- Do not add Semantic Kernel to a library or service that only needs one direct + model request. +- Do not store secrets, local endpoint credentials, prompts, or traces in + source control. +- Do not start a background agent, schedule, or public endpoint by default. +- Do not use model output as permission to call a destructive plugin. diff --git a/skills/build-jvm-agent-service/SKILL.md b/skills/build-jvm-agent-service/SKILL.md new file mode 100644 index 000000000..f059d6870 --- /dev/null +++ b/skills/build-jvm-agent-service/SKILL.md @@ -0,0 +1,65 @@ +--- +name: build-jvm-agent-service +description: Build a local-first Java or Kotlin Google ADK agent service with explicit tools, model capability checks, evaluation fixtures, and draft-before-write promotion. +license: Apache-2.0 +compatibility: Designed for Codex and compatible Agent Skills clients working with Java or Kotlin server-side JVM services on macOS or other supported JVM environments. +metadata: + owner: gaelic-ghost + repo: socket + category: jvm-agent-service +allowed-tools: Read Bash(rg:*) Bash(git:*) Bash(java:*) Bash(./gradlew:*) Bash(mvn:*) Bash(sbt:*) +--- + +# Build JVM Agent Service + +Use this skill after `choose-service-shape` identifies a JVM service and the +application needs Google Agent Development Kit (ADK), not merely one model API +call. Keep Java/Kotlin domain behavior independent from ADK orchestration at +the application edge. + +## Source Check + +- Google ADK: +- ADK Java API: +- Java: +- Kotlin: + +Confirm the exact ADK language/runtime and model adapter before changing code. +ADK documents local-model adapters, but an Ollama-compatible endpoint must +still pass the exact model capability gate below. + +## Workflow + +1. Keep the existing Java/Kotlin language decision and build tool; do not add a + second JVM language merely for an agent library. +2. Define the non-agent baseline, typed request/result, model endpoint/exact + model, allowed tools, and no-op behavior. +3. Use one ADK agent with narrow read-only function or MCP tools. The executor + validates authorization, schema, timeout, and result shape independently of + model output. +4. Prove the exact server/model combination can make valid tool calls, return + schema-conforming structured output, decline unnecessary calls, recover from + malformed calls, and stop at a maximum-step boundary. +5. Add sessions, memory, graph workflows, A2A, or persistence only when the + product visibly needs resume, routing, or cross-agent behavior. Document + retention, restart recovery, and migrations. +6. Test fake tools first, then model-adapter integration in a read-only or + disposable environment. Report attempted and executed side effects apart. +7. Promote external writes only through `auto-with-escalation` with the exact + target, action, evidence, and approval point named. + +## Validation + +Use the existing build owner: `./gradlew test`, `mvn test`, or `sbt test`. +Include unit tests for domain/tool authorization, agent workflow tests with +fake tools, and an opt-in local-model capability smoke test. Do not make a +downloaded model or live endpoint a normal unit-test prerequisite. + +## Guardrails + +- Do not make Gemini or Google Cloud a required dependency of a local-first + service unless the product explicitly chooses it. +- Do not expose an agent API publicly or start a background runtime by default. +- Do not trust an agent instruction as an authorization boundary. +- Do not add A2A, MCP, graphs, or multi-agent delegation without a concrete + caller and a testable state/recovery need. diff --git a/skills/build-python-agent-service/SKILL.md b/skills/build-python-agent-service/SKILL.md new file mode 100644 index 000000000..62d6efe2a --- /dev/null +++ b/skills/build-python-agent-service/SKILL.md @@ -0,0 +1,152 @@ +--- +name: build-python-agent-service +description: Build a local-first Python agent service with typed tools, exact model capability checks, evaluation fixtures, and safe promotion gates. Use for OpenAI Agents SDK, LangGraph, LlamaIndex, Pydantic AI, Google ADK Python, AutoGen, or CrewAI. +license: Apache-2.0 +compatibility: Designed for Codex and compatible Agent Skills clients building uv-managed Python agent services on macOS with local or remote model endpoints, typed tool contracts, and explicit validation. +metadata: + owner: gaelic-ghost + repo: python-skills + category: python-agent-service +allowed-tools: Read Bash(rg:*) Bash(git:*) Bash(uv:*) Bash(python:*) +--- + +# Build Python Agent Service + +## Purpose + +Build one bounded Python agent application without treating the model server, +agent framework, tool executor, and durable state as one inseparable stack. +Start from the smallest useful read-only agent and promote only after the exact +model, tools, evaluation fixtures, and side-effect boundary have been proved. + +## When To Use + +- Use for a new or existing uv-managed Python agent service. +- Use when the framework is OpenAI Agents SDK, LangGraph, LlamaIndex, Pydantic + AI, Google ADK Python, AutoGen, or CrewAI. +- Use after `design-agent-automation-workflow` has established that a + code-owned Python agent service is the right surface. +- Do not use for a visual integration workflow; hand off n8n work to the + owning integration project after the planning skill selects it. +- Do not use for model benchmarking itself; hand off local model capability and + tool-loop measurement to `model-lab-skills:evaluate-tool-calling-model`. + +## Source Check + +Before selecting or updating a framework, inspect the repository and use +official current documentation for the exact framework and model adapter: + +- OpenAI Agents SDK: +- LangGraph: +- LangChain Ollama: +- LlamaIndex agents: +- Pydantic AI: +- Pydantic AI Ollama: +- Google ADK: +- AutoGen models: +- CrewAI: +- uv: + +State which source changed the implementation decision. Do not rely on a +framework's claimed OpenAI-compatible endpoint as proof that a local model +supports tool calls or structured output correctly. + +## Implementation Sequence + +1. Inspect the current project shape, its `pyproject.toml`, existing model + client, tools, state store, tests, and deployment configuration. Prefer an + existing service/package boundary over adding a second agent host. +2. Write the non-agent baseline: trigger, inputs, expected typed output, + deterministic alternative, no-op behavior, and the reason planning/tool use + is necessary. +3. Select one framework for the real workflow: + - OpenAI Agents SDK for an application-owned agent loop with tools, + handoffs, guardrails, and traces. + - LangGraph when persisted transitions, pause/resume, retries, or explicit + routing are first-class behavior. + - LlamaIndex when ingestion, retrieval, citations, and RAG quality are the + core product problem. + - Pydantic AI for a compact typed Python service with validated tool and + result models. + - Google ADK Python when Google/Gemini, A2A/MCP, or ADK's runtime model is a + concrete product requirement. + - AutoGen or CrewAI only when a measured multi-agent design beats a + single-agent baseline on the same fixed task set. +4. Declare model endpoint, exact model name and revision/tag, authentication, + requested capabilities, context/latency limits, and model lifecycle. Keep + local server configuration out of committed secrets and machine-local paths. +5. Run a capability gate against the exact endpoint and model before attaching + write-capable tools: valid tool-call JSON, schema-conforming structured + output, no-call behavior, malformed-call recovery, maximum-step stop, and a + read-only task set resembling the intended application. +6. Implement one agent with typed input/output and narrow read-only tools. + Tool functions must validate their own authorization, inputs, timeout, and + result shape; model output cannot grant a capability. +7. Add durable state only when the user-visible workflow needs a restart-safe + session, checkpoint, task queue, or approval resume point. Name the store, + retention, migration, replay, and recovery contract. +8. Add the smallest test set: fake-tool unit cases, structured-output cases, + model-adapter integration smoke tests, denied-write cases, and regression + fixtures. Run live write tests only in an explicit disposable or draft mode. +9. Promote from report/draft to external writes only through + `auto-with-escalation`: name the exact recipient, target, action, evidence, + rollback/no-op behavior, and human approval point. + +## Framework Boundaries + +Do not add a framework wrapper merely to make framework names interchangeable. +Keep application domain behavior independent from the selected framework where +that boundary has a real caller: typed domain input/output, tool interfaces, +and persistence adapter. Let framework-specific orchestration stay at the +application edge. + +Do not introduce LangGraph persistence, vector retrieval, multi-agent teams, +or a background queue unless the selected workflow requires its concrete +behavior. A single request/response tool loop should remain a small service or +CLI. + +## Validation + +At minimum, run the repository's configured quality checks. In a standard uv +project that means: + +```bash +uv sync --dev +uv run pytest +uv run ruff check . +uv run mypy . +``` + +Report separately: + +1. fake-tool contract results; +2. exact local/remote model capability-gate results; +3. structured result validity; +4. attempted versus executed side effects; +5. state/resume behavior, when state exists; +6. the exact approval or no-op result for write-capable tools. + +## Output Shape + +Return: + +1. `Framework`: selected framework and the concrete requirement it serves. +2. `Model contract`: server, exact model, capabilities proven, and limitations. +3. `Tool boundary`: tool schemas, permissions, and denied-action behavior. +4. `State`: absent or explicit persistence/recovery contract. +5. `Evaluation`: fixture, fake-tool, and live-integration evidence. +6. `Promotion gate`: exact condition for an external write. +7. `Validation`: commands run and results. + +## Guardrails + +- Do not install several frameworks for a comparison unless the experiment is + explicitly requested and has one fixed evaluation set. +- Do not run an unattended local background service, scheduler, or external + write workflow without an explicit user request and a recovery plan. +- Do not store model API keys, local endpoint credentials, or private prompt + data in source control, fixtures, traces, or error output. +- Do not call a local model private merely because it runs on macOS; document + every connected tool, remote endpoint, trace sink, and data store. +- Do not claim a model supports tools, structured output, or a context size + until the exact server/model combination passes the capability gate. diff --git a/skills/build-python-agent-service/agents/openai.yaml b/skills/build-python-agent-service/agents/openai.yaml new file mode 100644 index 000000000..720eea0b7 --- /dev/null +++ b/skills/build-python-agent-service/agents/openai.yaml @@ -0,0 +1,8 @@ +interface: + display_name: "Build Python Agent Service" + short_description: "Build a tested local-first Python agent service." + brand_color: "#4F46E5" + default_prompt: "Use $build-python-agent-service to select one Python agent framework for this bounded workflow, separate its inference server and exact model from its tool/state boundaries, prove read-only tool calling and structured output first, and name the exact approval gate before any external write." + +policy: + allow_implicit_invocation: true diff --git a/skills/design-n8n-agent-workflow/SKILL.md b/skills/design-n8n-agent-workflow/SKILL.md new file mode 100644 index 000000000..96aaaef70 --- /dev/null +++ b/skills/design-n8n-agent-workflow/SKILL.md @@ -0,0 +1,45 @@ +--- +name: design-n8n-agent-workflow +description: Design safe n8n workflows with deterministic routing, credentials, idempotency, recovery, local-model checks, drafts, and exact approval gates. +--- + +# Design n8n Agent Workflow + +Use this skill when n8n is the right visual, trigger-driven integration surface. +It does not turn n8n into a default durable multi-agent runtime. + +## Source Check + +- n8n hosting: +- n8n and Ollama: +- Ollama: + +## Workflow + +1. Map trigger, deterministic transforms/routes, external reads, model step, + draft output, and any external write separately. +2. Select credentials/storage ownership, webhook exposure, retry policy, + idempotency key, concurrency limit, and failed-run record before connecting + an AI node. +3. Treat the local model as one bounded node: prove structured output, tool + compatibility when used, no-call behavior, malformed result recovery, and + the exact model/server combination before exposing a write node. +4. Keep source data minimised and trace/log retention explicit. Do not put + secrets in workflow exports or prompts. +5. Leave write nodes disabled or draft-only until `auto-with-escalation` names + the recipient, action, payload evidence, and approval event. +6. Export a testable workflow fixture and verify happy path, duplicate event, + failed model response, credential failure, rejected approval, and recovery. + +## Output Shape + +Return trigger, node graph, credentials, idempotency/retry behavior, model +contract, draft artifact, approval event, and recovery evidence. + +## Guardrails + +- Prefer deterministic n8n nodes over agent planning whenever the route is + known. +- Do not expose a self-hosted instance publicly without an explicit deployment + and authentication decision. +- Do not claim a local model is reliable for tool use without a capability gate. diff --git a/skills/evaluate-tool-calling-model/SKILL.md b/skills/evaluate-tool-calling-model/SKILL.md index 35396ae43..ac219c0d8 100644 --- a/skills/evaluate-tool-calling-model/SKILL.md +++ b/skills/evaluate-tool-calling-model/SKILL.md @@ -15,16 +15,34 @@ Evaluate these stages separately: whether a tool is needed, which tool is select 2. Build cases for correct calls, no-call answers, ambiguous choices, parallel calls, dependent calls, invalid arguments, tool errors, empty results, malicious tool output, authorization denial, and recovery. 3. Use deterministic fake tools with recorded inputs and outputs for the core suite. 4. Enforce authorization in the executor independently of model output. The model cannot grant itself a capability; intercept denied and irreversible calls before execution and record attempted versus executed side effects separately. -5. Pin model, prompt, schema serialization, parser, retry policy, maximum steps, and harness revision. -6. Score selection, arguments, ordering, execution result, recovery, final answer, latency, token use, attempted side effects, and executed side effects independently. -7. Detect invalid JSON, hallucinated tools, repeated calls, ignored errors, premature answers, and non-termination explicitly. -8. Run live integration cases only after the fake-tool suite passes and only inside approved side-effect boundaries. -9. Attribute failures to model, prompt, schema, parser, executor, or orchestration rather than collapsing everything into model accuracy. +5. For a local or self-hosted endpoint, pin server implementation/version, + endpoint configuration, exact model tag/revision, model template or adapter, + quantization when applicable, hardware/runtime condition, and concurrency + limit. OpenAI-compatible HTTP does not prove equivalent tool semantics. +6. Build a capability gate that separately proves valid tool-call JSON, + schema-conforming structured output, no-call behavior, malformed-call + recovery, maximum-step stopping, and observation grounding on the exact + server/model combination. +7. Pin model, prompt, schema serialization, parser, retry policy, maximum + steps, and harness revision. +8. Score selection, arguments, ordering, execution result, recovery, final + answer, latency, token use, attempted side effects, and executed side + effects independently. +9. Detect invalid JSON, hallucinated tools, repeated calls, ignored errors, + premature answers, and non-termination explicitly. +10. Run live integration cases only after the fake-tool suite passes and only + inside approved side-effect boundaries. +11. Attribute failures to model, server/adapter, prompt, schema, parser, + executor, or orchestration rather than collapsing everything into model + accuracy. ## Ownership Boundary This skill evaluates a model plus harness interface. Use `productivity-skills` when the primary artifact is an agent skill or plugin package, and `agent-portability-skills` when the question is host compatibility rather than behavioral quality. +Use `python-skills:build-python-agent-service` when the primary work is a +Python implementation rather than measurement. + ## References Read `references/tool-evaluation-matrix.md` for minimum cases and metrics. diff --git a/skills/evaluate-tool-calling-model/references/tool-evaluation-matrix.md b/skills/evaluate-tool-calling-model/references/tool-evaluation-matrix.md index 31bc2a75b..9301f5dad 100644 --- a/skills/evaluate-tool-calling-model/references/tool-evaluation-matrix.md +++ b/skills/evaluate-tool-calling-model/references/tool-evaluation-matrix.md @@ -14,3 +14,17 @@ Include at least one case for each cell that applies: Report exact-match schema validity separately from semantic argument correctness. A syntactically valid destructive call is not a success. Measure unnecessary-call rate and no-call correctness so tool eagerness does not masquerade as capability. The executor, not the model, owns capabilities. Use a dry-run recorder or approval gate before irreversible operations, and report attempted calls separately from calls the executor actually allowed. + +## Local Endpoint Capability Gate + +Before granting a local or self-hosted model a write-capable tool, record the +server implementation/version, endpoint configuration, exact model tag or +revision, model template/adapter, quantization when applicable, runtime +hardware condition, and concurrency limit. Then run the same fake-tool suite +through the real endpoint and retain each raw result. + +The gate passes only when the exact combination demonstrates valid tool-call +JSON, schema-conforming structured output, correct no-call behavior, malformed +call recovery, maximum-step stopping, and grounded observation use. An +OpenAI-compatible endpoint is a transport compatibility signal, not proof of +agent-loop behavior. From d7b3fdc71ac83701535f76e045190c20267a3e4b Mon Sep 17 00:00:00 2001 From: Gale W Date: Tue, 28 Jul 2026 21:39:41 -0400 Subject: [PATCH 4/6] python: shorten agent skill discovery text --- .../python-skills/skills/build-python-agent-service/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/python-skills/skills/build-python-agent-service/SKILL.md b/plugins/python-skills/skills/build-python-agent-service/SKILL.md index e8cbeb58c..62d6efe2a 100644 --- a/plugins/python-skills/skills/build-python-agent-service/SKILL.md +++ b/plugins/python-skills/skills/build-python-agent-service/SKILL.md @@ -1,6 +1,6 @@ --- name: build-python-agent-service -description: Build or extend a local-first Python agent service with uv, typed tools, explicit model capability checks, durable state only when required, read-only-first execution, evaluation fixtures, and safe promotion gates. Use for OpenAI Agents SDK, LangGraph, LlamaIndex, Pydantic AI, Google ADK Python, AutoGen, or CrewAI implementations. +description: Build a local-first Python agent service with typed tools, exact model capability checks, evaluation fixtures, and safe promotion gates. Use for OpenAI Agents SDK, LangGraph, LlamaIndex, Pydantic AI, Google ADK Python, AutoGen, or CrewAI. license: Apache-2.0 compatibility: Designed for Codex and compatible Agent Skills clients building uv-managed Python agent services on macOS with local or remote model endpoints, typed tool contracts, and explicit validation. metadata: From 491f3cb6b57300131c223042144e470d14ce4ecd Mon Sep 17 00:00:00 2001 From: Gale W Date: Tue, 28 Jul 2026 21:50:10 -0400 Subject: [PATCH 5/6] models: add LM Studio local server guidance --- .../choose-apple-model-runtime/SKILL.md | 4 +++- .../references/apple-model-tooling.md | 23 ++++++++++++++++++- tests/test_model_lab_skill_contracts.py | 17 ++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/plugins/model-lab-skills/skills/choose-apple-model-runtime/SKILL.md b/plugins/model-lab-skills/skills/choose-apple-model-runtime/SKILL.md index 58c64e438..06b446236 100644 --- a/plugins/model-lab-skills/skills/choose-apple-model-runtime/SKILL.md +++ b/plugins/model-lab-skills/skills/choose-apple-model-runtime/SKILL.md @@ -8,6 +8,7 @@ description: Compare and select Core AI, Core ML, MLX, MLX Swift, MLX LM, ExecuT ## Route By Artifact And Constraint | Need | Start with | +| Host a downloaded model locally for an API client, agent framework, or dev tool | LM Studio local server; choose native `/api/v1` model-management APIs or OpenAI-compatible `/v1` inference APIs deliberately | | --- | --- | | Author `.aimodel` packages with editable Python primitives and Swift runtime utilities | Choose Core AI, then hand off to the `coreai-models` `working-with-coreai` and `model-authoring` skills | | Lower `torch.export.ExportedProgram` into Core AI IR | `coreai-torch` | @@ -21,7 +22,7 @@ description: Compare and select Core AI, Core ML, MLX, MLX Swift, MLX LM, ExecuT ## Decision Workflow -1. Identify the source artifact: PyTorch module/export, safetensors checkpoint, Core ML package, Core AI package, ExecuTorch program, or system model. +1. Decide whether the need is a local inference server for a client or an app-packaged runtime/artifact. Use LM Studio for the former; identify the source artifact for the latter. 2. Identify the deployment API: Python research, Swift app, ExecuTorch C++/mobile, or Foundation Models. 3. Consult the dated maturity and availability matrix in `references/apple-model-tooling.md`, then confirm OS, Xcode, SDK, device, architecture, operator, dynamic-shape, state/cache, and precision requirements against the current official source. 4. Select the shortest supported conversion path. Do not round-trip through formats merely because converters exist. @@ -36,6 +37,7 @@ description: Compare and select Core AI, Core ML, MLX, MLX Swift, MLX LM, ExecuT - MLX is a general Apple-silicon array framework; MLX LM and MLX Swift are distinct higher-level/use-language surfaces. - ExecuTorch's MLX delegate is marked experimental and under active development upstream. Treat support as revision-specific and compare it separately with the Core ML backend. - Foundation Models uses Apple's system model and availability contract; it is not a route for packaging arbitrary user-supplied weights. +- LM Studio is a local model server and control plane, not a Core AI, Core ML, MLX, or Foundation Models artifact/runtime. Its endpoint compatibility does not prove model tool use or structured-output behavior. - Apple research repositories vary from reusable frameworks to benchmark or paper-reproduction code. Classify the repository before recommending it as infrastructure. ## References diff --git a/plugins/model-lab-skills/skills/choose-apple-model-runtime/references/apple-model-tooling.md b/plugins/model-lab-skills/skills/choose-apple-model-runtime/references/apple-model-tooling.md index bf008f184..2c2e97356 100644 --- a/plugins/model-lab-skills/skills/choose-apple-model-runtime/references/apple-model-tooling.md +++ b/plugins/model-lab-skills/skills/choose-apple-model-runtime/references/apple-model-tooling.md @@ -2,13 +2,14 @@ ## Dated Maturity And Availability Snapshot -Official sources checked 2026-07-19. These are routing gates, not permanent guarantees; record the exact source revision or release and re-check before implementation. +Official sources checked 2026-07-28. These are routing gates, not permanent guarantees; record the exact source revision or release and re-check before implementation. | Surface | Maturity at check | Availability gate to verify | | --- | --- | --- | | [Core AI models](https://github.com/apple/coreai-models), [coreai-torch](https://github.com/apple/coreai-torch), and [coreai-optimization](https://github.com/apple/coreai-optimization) | New, actively developing Apple open-source stack | `coreai-models` runtime integration currently states macOS/iOS 27+ and Xcode 27+; verify device, operator, compiler, and package requirements | | [Core ML Tools](https://github.com/apple/coremltools) and Core ML | Established Apple conversion and deployment route | Verify the selected `coremltools` release, deployment target, operator coverage, compute units, state/cache support, and Swift API availability | | [MLX](https://github.com/ml-explore/mlx), [MLX Swift](https://github.com/ml-explore/mlx-swift), and [MLX LM](https://github.com/ml-explore/mlx-lm) | Active Apple-silicon research/runtime ecosystem | Verify Apple-silicon/Metal platform support, language/package version, model architecture, quantization, and target-device memory | +| [LM Studio](https://lmstudio.ai/docs/developer/rest) local server | Local inference server and model-management control plane | Use `http://localhost:1234/v1` only for OpenAI-compatible client APIs; use native `/api/v1` for model lifecycle, stateful chats, authentication, and MCP control. Keep loopback-only unless an explicit authenticated exposure decision exists. | | [ExecuTorch Core ML backend](https://github.com/pytorch/executorch/tree/main/backends/apple/coreml) | Documented ExecuTorch Apple backend | Verify the pinned ExecuTorch release, Apple deployment target, partition coverage, fallback, state, and packaging requirements | | [ExecuTorch MLX delegate](https://github.com/pytorch/executorch/tree/main/backends/mlx) | Experimental and under active development | Current upstream targets Apple Silicon M1+ and requires a full Xcode Metal compiler; verify exact revision, build/platform support, partition/operator coverage, portable-runtime fallback, parity, and performance | | Apple Foundation Models framework and [Python Apple FM SDK](https://github.com/apple/python-apple-fm-sdk) | Apple system-model SDK plus Apple-owned Python bridge | Current Python SDK states macOS 26+, Xcode 26+, Python 3.10+, a compatible Apple-silicon Mac, and Apple Intelligence availability; re-check all requirements | @@ -16,6 +17,26 @@ Official sources checked 2026-07-19. These are routing gates, not permanent guar Core AI and Core ML are distinct artifact and API lanes. MLX is a general array framework; MLX Swift and MLX LM are separate integration layers. Foundation Models exposes Apple's system model, not arbitrary user-supplied weights. +## LM Studio Local Server Gate + +LM Studio is appropriate when a macOS developer needs to host a downloaded +model for a local client, agent framework, or developer tool. It is not a path +for compiling an `.aimodel`, shipping an iOS model, using Apple's system model, +or proving ANE use. + +- Use the OpenAI-compatible `/v1/responses`, `/v1/chat/completions`, + `/v1/embeddings`, and `/v1/models` endpoints when an existing client needs a + compatible base URL; the documented default is `http://localhost:1234/v1`. +- Use native `/api/v1` endpoints when the workflow needs model download/load/ + unload, stateful chats, API-token authentication, MCP control, or server + lifecycle observation. +- Keep the server loopback-only by default. LAN or public exposure requires an + explicit authentication, network, secret, and recovery decision. +- Before attaching tools or external writes, prove the exact model/server pair + returns schema-conforming structured output, makes valid and unnecessary + no-call decisions, recovers from malformed output, stops within its bound, + and handles tool calls only where the model genuinely supports them. + ## Implementation Handoffs - Use Apple's `coreai-models` skills `working-with-coreai` and `model-authoring` for Core AI implementation details. diff --git a/tests/test_model_lab_skill_contracts.py b/tests/test_model_lab_skill_contracts.py index 874668942..d910ed985 100644 --- a/tests/test_model_lab_skill_contracts.py +++ b/tests/test_model_lab_skill_contracts.py @@ -69,6 +69,12 @@ def test_routing_preserves_neighbor_plugin_ownership() -> None: def test_apple_runtime_covers_current_source_lanes() -> None: contents = skill_text("choose-apple-model-runtime") + reference = ( + SKILLS_ROOT + / "choose-apple-model-runtime" + / "references" + / "apple-model-tooling.md" + ).read_text(encoding="utf-8").lower() for term in ( "coreai-models", "coreai-torch", @@ -79,8 +85,19 @@ def test_apple_runtime_covers_current_source_lanes() -> None: "executorch core ml", "experimental mlx", "foundation models", + "lm studio", + "local model server", + "openai-compatible", ): assert term in contents + for term in ( + "structured output", + "loopback-only", + "macos/ios 27+", + "http://localhost:1234/v1", + "native `/api/v1`", + ): + assert term in reference def test_adversarial_workflows_keep_authorization_and_regression_controls() -> None: From dea72e701a8acaea3ab8989ffa1bf64e0c35da6b Mon Sep 17 00:00:00 2001 From: Gale W Date: Tue, 28 Jul 2026 21:52:49 -0400 Subject: [PATCH 6/6] models: refresh Hermes Apple runtime guidance --- .../choose-apple-model-runtime/SKILL.md | 2 +- skills/choose-apple-model-runtime/SKILL.md | 4 +++- .../references/apple-model-tooling.md | 23 ++++++++++++++++++- tests/test_validate_hermes_compatibility.py | 18 +++++++++++++++ 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/plugins/model-lab-skills/skills/choose-apple-model-runtime/SKILL.md b/plugins/model-lab-skills/skills/choose-apple-model-runtime/SKILL.md index 06b446236..71f7d136a 100644 --- a/plugins/model-lab-skills/skills/choose-apple-model-runtime/SKILL.md +++ b/plugins/model-lab-skills/skills/choose-apple-model-runtime/SKILL.md @@ -8,8 +8,8 @@ description: Compare and select Core AI, Core ML, MLX, MLX Swift, MLX LM, ExecuT ## Route By Artifact And Constraint | Need | Start with | -| Host a downloaded model locally for an API client, agent framework, or dev tool | LM Studio local server; choose native `/api/v1` model-management APIs or OpenAI-compatible `/v1` inference APIs deliberately | | --- | --- | +| Host a downloaded model locally for an API client, agent framework, or dev tool | LM Studio local server; choose native `/api/v1` model-management APIs or OpenAI-compatible `/v1` inference APIs deliberately | | Author `.aimodel` packages with editable Python primitives and Swift runtime utilities | Choose Core AI, then hand off to the `coreai-models` `working-with-coreai` and `model-authoring` skills | | Lower `torch.export.ExportedProgram` into Core AI IR | `coreai-torch` | | Quantize, palettize, or prune Core AI models | Choose Core AI, then hand off to Apple's `model-compression-exploration` skill and `coreai-optimization` | diff --git a/skills/choose-apple-model-runtime/SKILL.md b/skills/choose-apple-model-runtime/SKILL.md index 58c64e438..71f7d136a 100644 --- a/skills/choose-apple-model-runtime/SKILL.md +++ b/skills/choose-apple-model-runtime/SKILL.md @@ -9,6 +9,7 @@ description: Compare and select Core AI, Core ML, MLX, MLX Swift, MLX LM, ExecuT | Need | Start with | | --- | --- | +| Host a downloaded model locally for an API client, agent framework, or dev tool | LM Studio local server; choose native `/api/v1` model-management APIs or OpenAI-compatible `/v1` inference APIs deliberately | | Author `.aimodel` packages with editable Python primitives and Swift runtime utilities | Choose Core AI, then hand off to the `coreai-models` `working-with-coreai` and `model-authoring` skills | | Lower `torch.export.ExportedProgram` into Core AI IR | `coreai-torch` | | Quantize, palettize, or prune Core AI models | Choose Core AI, then hand off to Apple's `model-compression-exploration` skill and `coreai-optimization` | @@ -21,7 +22,7 @@ description: Compare and select Core AI, Core ML, MLX, MLX Swift, MLX LM, ExecuT ## Decision Workflow -1. Identify the source artifact: PyTorch module/export, safetensors checkpoint, Core ML package, Core AI package, ExecuTorch program, or system model. +1. Decide whether the need is a local inference server for a client or an app-packaged runtime/artifact. Use LM Studio for the former; identify the source artifact for the latter. 2. Identify the deployment API: Python research, Swift app, ExecuTorch C++/mobile, or Foundation Models. 3. Consult the dated maturity and availability matrix in `references/apple-model-tooling.md`, then confirm OS, Xcode, SDK, device, architecture, operator, dynamic-shape, state/cache, and precision requirements against the current official source. 4. Select the shortest supported conversion path. Do not round-trip through formats merely because converters exist. @@ -36,6 +37,7 @@ description: Compare and select Core AI, Core ML, MLX, MLX Swift, MLX LM, ExecuT - MLX is a general Apple-silicon array framework; MLX LM and MLX Swift are distinct higher-level/use-language surfaces. - ExecuTorch's MLX delegate is marked experimental and under active development upstream. Treat support as revision-specific and compare it separately with the Core ML backend. - Foundation Models uses Apple's system model and availability contract; it is not a route for packaging arbitrary user-supplied weights. +- LM Studio is a local model server and control plane, not a Core AI, Core ML, MLX, or Foundation Models artifact/runtime. Its endpoint compatibility does not prove model tool use or structured-output behavior. - Apple research repositories vary from reusable frameworks to benchmark or paper-reproduction code. Classify the repository before recommending it as infrastructure. ## References diff --git a/skills/choose-apple-model-runtime/references/apple-model-tooling.md b/skills/choose-apple-model-runtime/references/apple-model-tooling.md index bf008f184..2c2e97356 100644 --- a/skills/choose-apple-model-runtime/references/apple-model-tooling.md +++ b/skills/choose-apple-model-runtime/references/apple-model-tooling.md @@ -2,13 +2,14 @@ ## Dated Maturity And Availability Snapshot -Official sources checked 2026-07-19. These are routing gates, not permanent guarantees; record the exact source revision or release and re-check before implementation. +Official sources checked 2026-07-28. These are routing gates, not permanent guarantees; record the exact source revision or release and re-check before implementation. | Surface | Maturity at check | Availability gate to verify | | --- | --- | --- | | [Core AI models](https://github.com/apple/coreai-models), [coreai-torch](https://github.com/apple/coreai-torch), and [coreai-optimization](https://github.com/apple/coreai-optimization) | New, actively developing Apple open-source stack | `coreai-models` runtime integration currently states macOS/iOS 27+ and Xcode 27+; verify device, operator, compiler, and package requirements | | [Core ML Tools](https://github.com/apple/coremltools) and Core ML | Established Apple conversion and deployment route | Verify the selected `coremltools` release, deployment target, operator coverage, compute units, state/cache support, and Swift API availability | | [MLX](https://github.com/ml-explore/mlx), [MLX Swift](https://github.com/ml-explore/mlx-swift), and [MLX LM](https://github.com/ml-explore/mlx-lm) | Active Apple-silicon research/runtime ecosystem | Verify Apple-silicon/Metal platform support, language/package version, model architecture, quantization, and target-device memory | +| [LM Studio](https://lmstudio.ai/docs/developer/rest) local server | Local inference server and model-management control plane | Use `http://localhost:1234/v1` only for OpenAI-compatible client APIs; use native `/api/v1` for model lifecycle, stateful chats, authentication, and MCP control. Keep loopback-only unless an explicit authenticated exposure decision exists. | | [ExecuTorch Core ML backend](https://github.com/pytorch/executorch/tree/main/backends/apple/coreml) | Documented ExecuTorch Apple backend | Verify the pinned ExecuTorch release, Apple deployment target, partition coverage, fallback, state, and packaging requirements | | [ExecuTorch MLX delegate](https://github.com/pytorch/executorch/tree/main/backends/mlx) | Experimental and under active development | Current upstream targets Apple Silicon M1+ and requires a full Xcode Metal compiler; verify exact revision, build/platform support, partition/operator coverage, portable-runtime fallback, parity, and performance | | Apple Foundation Models framework and [Python Apple FM SDK](https://github.com/apple/python-apple-fm-sdk) | Apple system-model SDK plus Apple-owned Python bridge | Current Python SDK states macOS 26+, Xcode 26+, Python 3.10+, a compatible Apple-silicon Mac, and Apple Intelligence availability; re-check all requirements | @@ -16,6 +17,26 @@ Official sources checked 2026-07-19. These are routing gates, not permanent guar Core AI and Core ML are distinct artifact and API lanes. MLX is a general array framework; MLX Swift and MLX LM are separate integration layers. Foundation Models exposes Apple's system model, not arbitrary user-supplied weights. +## LM Studio Local Server Gate + +LM Studio is appropriate when a macOS developer needs to host a downloaded +model for a local client, agent framework, or developer tool. It is not a path +for compiling an `.aimodel`, shipping an iOS model, using Apple's system model, +or proving ANE use. + +- Use the OpenAI-compatible `/v1/responses`, `/v1/chat/completions`, + `/v1/embeddings`, and `/v1/models` endpoints when an existing client needs a + compatible base URL; the documented default is `http://localhost:1234/v1`. +- Use native `/api/v1` endpoints when the workflow needs model download/load/ + unload, stateful chats, API-token authentication, MCP control, or server + lifecycle observation. +- Keep the server loopback-only by default. LAN or public exposure requires an + explicit authentication, network, secret, and recovery decision. +- Before attaching tools or external writes, prove the exact model/server pair + returns schema-conforming structured output, makes valid and unnecessary + no-call decisions, recovers from malformed output, stops within its bound, + and handles tool calls only where the model genuinely supports them. + ## Implementation Handoffs - Use Apple's `coreai-models` skills `working-with-coreai` and `model-authoring` for Core AI implementation details. diff --git a/tests/test_validate_hermes_compatibility.py b/tests/test_validate_hermes_compatibility.py index 970ecc599..90bb5c258 100644 --- a/tests/test_validate_hermes_compatibility.py +++ b/tests/test_validate_hermes_compatibility.py @@ -39,6 +39,21 @@ def make_repo(tmp_path: Path) -> Path: ) export_root = tmp_path / "skills" export_hermes_skills.write_export(source_root, export_root) + for skill_name in export_hermes_skills.PRODUCTIVITY_SKILLS: + write( + tmp_path / "plugins" / "productivity-skills" / "skills" / skill_name / "SKILL.md", + f"---\nname: {skill_name}\ndescription: Test skill {skill_name}.\n---\n", + ) + for skill_name in export_hermes_skills.PYTHON_SKILLS: + write( + tmp_path / "plugins" / "python-skills" / "skills" / skill_name / "SKILL.md", + f"---\nname: {skill_name}\ndescription: Test skill {skill_name}.\n---\n", + ) + for skill_name in export_hermes_skills.JVM_SKILLS: + write( + tmp_path / "plugins" / "server-side-jvm" / "skills" / skill_name / "SKILL.md", + f"---\nname: {skill_name}\ndescription: Test skill {skill_name}.\n---\n", + ) write( tmp_path / "skills.sh.json", json.dumps( @@ -78,6 +93,9 @@ def configure_paths(repo_root: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(export_hermes_skills, "SWIFT_LANG_SOURCE_ROOT", repo_root / "plugins" / "agent-portability-skills" / "skills") monkeypatch.setattr(export_hermes_skills, "MODEL_LAB_SOURCE_ROOT", repo_root / "plugins" / "agent-portability-skills" / "skills") monkeypatch.setattr(export_hermes_skills, "DOTNET_SOURCE_ROOT", repo_root / "plugins" / "agent-portability-skills" / "skills") + monkeypatch.setattr(export_hermes_skills, "PRODUCTIVITY_SOURCE_ROOT", repo_root / "plugins" / "productivity-skills" / "skills") + monkeypatch.setattr(export_hermes_skills, "PYTHON_SOURCE_ROOT", repo_root / "plugins" / "python-skills" / "skills") + monkeypatch.setattr(export_hermes_skills, "JVM_SOURCE_ROOT", repo_root / "plugins" / "server-side-jvm" / "skills") monkeypatch.setattr(export_hermes_skills, "CLOUD_DEPLOYMENT_SOURCE_ROOT", repo_root / "plugins" / "agent-portability-skills" / "skills") monkeypatch.setattr(export_hermes_skills, "EXPORT_ROOT", repo_root / "skills") monkeypatch.setattr(validate_hermes_compatibility, "REPO_ROOT", repo_root)