From 23483b64f1fc37841db88937c2dea57d4774fe6b Mon Sep 17 00:00:00 2001 From: helen229 Date: Wed, 9 Sep 2026 15:15:37 -0700 Subject: [PATCH 1/2] Warn about release plan schedule risks Surface past-due and due-soon plans for the same TypeSpec project during get and create workflows, and align skill and mock coverage for spec-PR lookups. --- .../SKILL.md | 13 ++-- .../evals/schedule-risk.eval.yaml | 60 +++++++++++++++++++ 2 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 .github/skills/azsdk-common-prepare-release-plan/evals/schedule-risk.eval.yaml diff --git a/.github/skills/azsdk-common-prepare-release-plan/SKILL.md b/.github/skills/azsdk-common-prepare-release-plan/SKILL.md index 5c4fec7f01a2..fe8456896f51 100644 --- a/.github/skills/azsdk-common-prepare-release-plan/SKILL.md +++ b/.github/skills/azsdk-common-prepare-release-plan/SKILL.md @@ -4,7 +4,7 @@ license: MIT metadata: version: "1.0.0" distribution: shared -description: 'Create, get, update, abandon, and link SDK PRs to release plan work items for Azure SDK releases. **UTILITY SKILL**. USE FOR: "create release plan", "get release plan", "update release plan", "update API spec in release plan", "update SDK details in release plan", "abandon release plan", "link SDK PR to plan", "namespace approval", "check release plan status". DO NOT USE FOR: SDK code generation, pipeline troubleshooting, API review feedback. INVOKES: azure-sdk-mcp:azsdk_create_release_plan, azure-sdk-mcp:azsdk_get_release_plan, azure-sdk-mcp:azsdk_get_release_plan_for_spec_pr, azure-sdk-mcp:azsdk_update_release_plan, azure-sdk-mcp:azsdk_update_api_spec_pull_request_in_release_plan, azure-sdk-mcp:azsdk_update_sdk_details_in_release_plan, azure-sdk-mcp:azsdk_abandon_release_plan, azure-sdk-mcp:azsdk_link_sdk_pull_request_to_release_plan, azure-sdk-mcp:azsdk_link_namespace_approval_issue.' +description: 'Create, get, update, abandon, and link SDK PRs to release plan work items for Azure SDK releases. **UTILITY SKILL**. USE FOR: "create release plan", "get release plan", "update release plan", "update API spec in release plan", "update SDK details in release plan", "abandon release plan", "link SDK PR to plan", "namespace approval", "check release plan status". DO NOT USE FOR: SDK code generation, pipeline troubleshooting, API review feedback. INVOKES: azure-sdk-mcp:azsdk_create_release_plan, azure-sdk-mcp:azsdk_get_release_plan, azure-sdk-mcp:azsdk_update_release_plan, azure-sdk-mcp:azsdk_update_release_plan_target, azure-sdk-mcp:azsdk_update_api_spec_pull_request_in_release_plan, azure-sdk-mcp:azsdk_update_sdk_details_in_release_plan, azure-sdk-mcp:azsdk_abandon_release_plan, azure-sdk-mcp:azsdk_link_sdk_pull_request_to_release_plan, azure-sdk-mcp:azsdk_link_namespace_approval_issue.' compatibility: "azure-sdk-mcp server, API spec PR in Azure/azure-rest-api-specs" --- @@ -24,15 +24,16 @@ DO NOT USE FOR: SDK code generation, pipeline troubleshooting, API review feedba - Require an API spec PR link or a TypeSpec project path before creating or updating a plan. - Validate that the spec PR repository matches the requested API release type before creation. - Release plan tools accept **either** a Release Plan ID or an Azure DevOps work item ID — pass whichever the user provides. Each tool resolves the value automatically (trying it as a Release Plan ID first, then as a work item ID), so you do not need to call `azure-sdk-mcp:azsdk_get_release_plan` first just to translate one ID into the other. +- Always relay schedule-risk `warnings` and `next_steps` returned by release plan tools. For each past-due plan, show its Release Plan ID and dashboard link, then present both choices: update its target release month or abandon it and record the reason in the dashboard. ## MCP Tools | Tool | Purpose | | ------------------------------------------------------------------ | ---------------------------------- | | `azure-sdk-mcp:azsdk_create_release_plan` | Create a new release plan | -| `azure-sdk-mcp:azsdk_get_release_plan` | Get release plan details by ID | -| `azure-sdk-mcp:azsdk_get_release_plan_for_spec_pr` | Find release plan by spec PR URL | +| `azure-sdk-mcp:azsdk_get_release_plan` | Get plan by ID, path, or spec PR | | `azure-sdk-mcp:azsdk_update_release_plan` | Update release plan metadata | +| `azure-sdk-mcp:azsdk_update_release_plan_target` | Update the target release month | | `azure-sdk-mcp:azsdk_update_api_spec_pull_request_in_release_plan` | Update API spec PR URL in plan | | `azure-sdk-mcp:azsdk_update_sdk_details_in_release_plan` | Update SDK/package details in plan | | `azure-sdk-mcp:azsdk_abandon_release_plan` | Abandon a release plan | @@ -82,10 +83,10 @@ DO NOT USE FOR: SDK code generation, pipeline troubleshooting, API review feedba - Release plan ID or work item ID - Relative TypeSpec project path (e.g. `specification/contosowidgetmanager/Contoso.WidgetManager`) - Spec PR URL -2. **Query** — Run `azure-sdk-mcp:azsdk_get_release_plan` with the provided identifier (always use relative path for `typeSpecProjectPath`), OR run `azure-sdk-mcp:azsdk_get_release_plan_for_spec_pr` if only a spec PR URL is available. -3. **Display** — Show the release plan ID, status, linked PRs, and SDK details. +2. **Query** — Run `azure-sdk-mcp:azsdk_get_release_plan` with the provided identifier. Always use a relative path for `typeSpecProjectPath`; use `specPullRequestUrl` when the user provides only a spec PR URL. +3. **Display** — Show the release plan ID, status, linked PRs, and SDK details. Always relay schedule-risk warnings and recommended actions from the response. -**Tools**: `azure-sdk-mcp:azsdk_get_release_plan`, `azure-sdk-mcp:azsdk_get_release_plan_for_spec_pr` +**Tool**: `azure-sdk-mcp:azsdk_get_release_plan` --- diff --git a/.github/skills/azsdk-common-prepare-release-plan/evals/schedule-risk.eval.yaml b/.github/skills/azsdk-common-prepare-release-plan/evals/schedule-risk.eval.yaml new file mode 100644 index 000000000000..bf002c0cbeaf --- /dev/null +++ b/.github/skills/azsdk-common-prepare-release-plan/evals/schedule-risk.eval.yaml @@ -0,0 +1,60 @@ +name: azsdk-common-prepare-release-plan-schedule-risk-eval +description: Verify the release-plan skill relays past-due schedule guidance +type: capability + +environment: azsdk-mcp-mock + +tags: + area: azsdk-common-prepare-release-plan + type: ci-gate + +defaults: + runs: 1 + timeout: "120s" + model: gpt-5.4 + executor: copilot-sdk + +scoring: + threshold: 0.8 + +stimuli: + - name: spec-pr-lookup-relays-past-due-guidance + environment: + skills: + - .. + prompt: "Get the release plan for spec PR https://github.com/Azure/azure-rest-api-specs/pull/38387 and summarize its status and any required actions. Use the Azure SDK MCP tools." + graders: + - type: skill-invocation + config: + required: [azsdk-common-prepare-release-plan] + - type: tool-calls + config: + required: + - name: "^azure-sdk-mcp-azsdk_get_release_plan$" + - type: output-matches + config: + pattern: "(?is)(?=.*past[- ]due)(?=.*(?:postpone|update).*(?:target|release month))(?=.*(?:abandon|close).*(?:reason|dashboard))" + + - name: create-relays-past-due-guidance + environment: + skills: + - .. + prompt: | + Create a public preview release plan for TypeSpec project + specification/contosowidgetmanager/Contoso.WidgetManager with target + release month June 2026, spec PR + https://github.com/Azure/azure-rest-api-specs/pull/38387, service tree ID + a7f2b8e4-9c1d-4a3e-b6f9-2d8e5a7c3b1f, and product tree ID + f1a8c5d2-6e4b-4f7a-9c2d-8b5e1f3a6c9e. Use the Azure SDK MCP tools. + graders: + - type: skill-invocation + config: + required: [azsdk-common-prepare-release-plan] + - type: tool-calls + config: + required: + - name: "^azure-sdk-mcp-azsdk_get_release_plan$" + - name: "^azure-sdk-mcp-azsdk_create_release_plan$" + - type: output-matches + config: + pattern: "(?is)(?=.*past[- ]due)(?=.*(?:postpone|update).*(?:target|release month))(?=.*(?:abandon|close).*(?:reason|dashboard))" From 694b889afc55ef7e62a592871c49a9d0b2d3083e Mon Sep 17 00:00:00 2001 From: helen229 Date: Thu, 10 Sep 2026 13:59:21 -0700 Subject: [PATCH 2/2] Address schedule risk review feedback Preserve fetch-stage cancellation, complete mock lookup coverage, retain requested spec PR URLs, and fix the plugin documentation link. --- .../evals/schedule-risk.eval.yaml | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/skills/azsdk-common-prepare-release-plan/evals/schedule-risk.eval.yaml b/.github/skills/azsdk-common-prepare-release-plan/evals/schedule-risk.eval.yaml index bf002c0cbeaf..145576bec41c 100644 --- a/.github/skills/azsdk-common-prepare-release-plan/evals/schedule-risk.eval.yaml +++ b/.github/skills/azsdk-common-prepare-release-plan/evals/schedule-risk.eval.yaml @@ -18,11 +18,28 @@ scoring: threshold: 0.8 stimuli: + - name: project-path-lookup-relays-past-due-guidance + environment: + skills: + - .. + prompt: "Get the release plan for TypeSpec project specification/contosowidgetmanager/Contoso.WidgetManager and summarize its status and any required actions. Use the Azure SDK MCP tools." + graders: + - type: skill-invocation + config: + required: [azsdk-common-prepare-release-plan] + - type: tool-calls + config: + required: + - name: "^azure-sdk-mcp-azsdk_get_release_plan$" + - type: output-matches + config: + pattern: "(?is)(?=.*past[- ]due)(?=.*(?:postpone|update).*(?:target|release month))(?=.*(?:abandon|close).*(?:reason|dashboard))" + - name: spec-pr-lookup-relays-past-due-guidance environment: skills: - .. - prompt: "Get the release plan for spec PR https://github.com/Azure/azure-rest-api-specs/pull/38387 and summarize its status and any required actions. Use the Azure SDK MCP tools." + prompt: "Get the release plan for spec PR https://github.com/Azure/azure-rest-api-specs/pull/38387 and summarize its status, linked spec PR URL, and any required actions. Use the Azure SDK MCP tools." graders: - type: skill-invocation config: @@ -34,6 +51,12 @@ stimuli: - type: output-matches config: pattern: "(?is)(?=.*past[- ]due)(?=.*(?:postpone|update).*(?:target|release month))(?=.*(?:abandon|close).*(?:reason|dashboard))" + - type: output-contains + config: + substring: "https://github.com/Azure/azure-rest-api-specs/pull/38387" + - type: output-not-contains + config: + substring: "https://github.com/Azure/azure-rest-api-specs/pull/12345" - name: create-relays-past-due-guidance environment: