feat(eval): grouped help and command examples for the eval commands

package.json

@@ -64,8 +64,8 @@
   "dependencies": {
     "@aws-cdk/toolkit-lib": "1.38.2",
     "@aws-sdk/client-bedrock-agent": "^3.1092.0",
-    "@aws-sdk/client-bedrock-agentcore": "^3.1092.0",
-    "@aws-sdk/client-bedrock-agentcore-control": "^3.1102.0",
+    "@aws-sdk/client-bedrock-agentcore": "^3.1129.0",
+    "@aws-sdk/client-bedrock-agentcore-control": "^3.1129.0",
     "@aws-sdk/client-cloudformation": "^3.1092.0",
     "@aws-sdk/client-cloudwatch-logs": "^3.1092.0",
     "@aws-sdk/client-iam": "^3.1080.0",

src/components/CliOnlyScreen.test.tsx

@@ -141,3 +141,40 @@ describe("paths without a screen of their own", () => {
     r.unmount();
   });
 });
+
+describe("option help groups", () => {
+  // A heading is its own line, so match it that way: "evaluation" also appears
+  // inside the "batch-evaluation" breadcrumb, and "configuration" inside flags
+  // like --protocol-configuration.
+  const headingLine = (title: string) => `\n ${title}\n`;
+
+  test("a grouped command renders one section per heading, in --help order", async () => {
+    const r = renderScreen("/agentcore/eval/batch-evaluation/evaluate");
+
+    await waitForText(r.lastFrame, "this command runs from the command line");
+    const frame = r.lastFrame()!;
+    const positions = [
+      "configuration",
+      "session source (choose exactly one)",
+      "source filters",
+      "evaluation",
+    ].map((title) => frame.indexOf(headingLine(title)));
+
+    expect(positions.every((position) => position >= 0)).toBe(true);
+    expect(positions).toEqual([...positions].sort((a, b) => a - b));
+    expect(frame).not.toContain(headingLine("other options"));
+    expect(frame).not.toContain(headingLine("options"));
+    r.unmount();
+  });
+
+  test("a command whose flags carry no group keeps a single options section", async () => {
+    const r = renderScreen("/agentcore/gateway/create");
+
+    await waitForText(r.lastFrame, "this command runs from the command line");
+    const frame = r.lastFrame()!;
+    expect(frame).toContain(headingLine("options"));
+    expect(frame).not.toContain(headingLine("configuration"));
+    expect(frame).not.toContain(headingLine("source filters"));
+    r.unmount();
+  });
+});

src/components/CliOnlyScreen.tsx

@@ -57,12 +57,15 @@ export function CliOnlyScreen({ ctx, path }: CliOnlyScreenProps) {
 
   const table = (rows: [string, string][]) => Object.fromEntries(rows);
   // --help is Commander's own and means nothing on a screen that is the help.
-  const options = table(
-    help
-      .visibleOptions(command)
-      .filter((option) => option.long !== "--help")
-      .map((option) => [help.optionTerm(option), help.optionDescription(option)]),
-  );
+  const optionGroups: [string, [string, string][]][] = [];
+  for (const option of help.visibleOptions(command)) {
+    if (option.long === "--help") continue;
+    const title = sectionTitle(option.helpGroupHeading);
+    const row: [string, string] = [help.optionTerm(option), help.optionDescription(option)];
+    const group = optionGroups.find(([existing]) => existing === title);
+    if (group) group[1].push(row);
+    else optionGroups.push([title, [row]]);
+  }
   const args = table(
     help
       .visibleArguments(command)
@@ -96,11 +99,11 @@ export function CliOnlyScreen({ ctx, path }: CliOnlyScreenProps) {
               <KeyValueTable items={args} />
             </Section>
           )}
-          {Object.keys(options).length > 0 && (
-            <Section title="options">
-              <KeyValueTable items={options} />
+          {optionGroups.map(([title, rows]) => (
+            <Section key={title} title={title}>
+              <KeyValueTable items={table(rows)} />
             </Section>
-          )}
+          ))}
           {details !== undefined && (
             // formatParameterDetails already carries its own heading and layout.
             <Text color={theme.colors.muted}>{details.trim()}</Text>
@@ -111,6 +114,9 @@ export function CliOnlyScreen({ ctx, path }: CliOnlyScreenProps) {
   );
 }
 
+const sectionTitle = (group: string | undefined) =>
+  group ? group.replace(/:$/, "").toLowerCase() : "options";
+
 function Section({ title, children }: { title: string; children: React.ReactNode }) {
   return (
     <Box flexDirection="column" marginTop={1}>

src/handlers/eval/batch-evaluation/evaluate/index.tsx

@@ -8,21 +8,73 @@ import type { SessionMetadataShape } from "@aws-sdk/client-bedrock-agentcore";
 import { coreOptsFromCtx, parseJsonFlag } from "../../../utils";
 import { SessionSource } from "../../sessionSource";
 
+const CONFIGURATION = "Configuration:";
+const EVALUATION = "Evaluation:";
+
+const groundTruthHelp = `(JSON: list of objects)
+Expected answers for the sessions being evaluated, so an evaluator can score a
+response against a reference instead of judging it on its own. Each entry names
+one session; omit an entry for a session that has no reference answer.
+
+Accepts inline JSON, file://<path>, or - to read stdin.
+
+JSON syntax:
+  [
+    {
+      "sessionId": "string",              // [required] the session this applies to
+      "testScenarioId": "string",         // groups sessions replaying one scenario
+      "groundTruth": {                    // exactly one key; only inline today
+        "inline": {
+          "turns": [                      // the expected exchange, in order
+            {
+              "input": { "prompt": "string" },
+              "expectedResponse": { "text": "string" }
+            },
+            ...
+          ],
+          "assertions": [                 // statements the response must satisfy
+            { "text": "string" },
+            ...
+          ],
+          "expectedTrajectory": {
+            "toolNames": ["string", ...]  // tools the agent should have called
+          }
+        }
+      },
+      "metadata": { "string": "string", ... }
+    },
+    ...
+  ]
+
+Example:
+  --ground-truth '[{"sessionId":"session-123","groundTruth":{"inline":{"turns":[{"input":{"prompt":"Where is my order?"},"expectedResponse":{"text":"It shipped on Tuesday."}}]}}}]'
+
+  --ground-truth file://ground-truth.json`;
+
 export const createEvaluateBatchEvaluationHandler = (core: Core, io: AppIO) =>
   createHandler({
     name: "evaluate",
     description: "evaluate existing sessions service-side (async; returns a job ID)",
     flags: [
+      flag("name", "batch evaluation name (must be unique in the account)", z.string().optional(), {
+        group: CONFIGURATION,
+      }),
+      flag("description", "optional description", z.string().optional(), {
+        group: CONFIGURATION,
+      }),
+      flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional(), {
+        group: CONFIGURATION,
+      }),
       ...SessionSource.flags,
-      flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional()),
+      flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional(), {
+        group: EVALUATION,
+      }),
       flag(
         "ground-truth",
-        "session ground truth (JSON SessionMetadataShape[]; inline, file://<path>, or -)",
+        "expected answers for the sessions (JSON SessionMetadataShape[])",
         z.string().optional(),
+        { group: EVALUATION, help: groundTruthHelp },
       ),
-      flag("name", "batch evaluation name (must be unique in the account)", z.string().optional()),
-      flag("description", "optional description", z.string().optional()),
-      flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional()),
     ],
     handle: async (ctx, flags) => {
       if (!flags["name"]) {

src/handlers/eval/batch-evaluation/simulate/index.tsx

@@ -7,41 +7,76 @@ import type { Core } from "../../../types";
 import { coreOptsFromCtx } from "../../../utils";
 import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request";
 
+const RUNTIME_INVOCATION = "Runtime invocation:";
+const DATASET = "Dataset:";
+const CONFIGURATION = "Configuration:";
+
+const payloadTemplateHelp = `(JSON object)
+The request body sent to the Runtime for each dataset example. Every occurrence
+of {input} is replaced with that example's input, so the template describes the
+shape your agent expects and {input} marks where the prompt goes.
+
+Example:
+  --payload-template '{"prompt":"{input}"}'
+
+  --payload-template '{"messages":[{"role":"user","content":"{input}"}],"stream":false}'`;
+
 // Composes invokeDataset (replay) → startBatchEvaluation (grade). Invoke flags mirror
 // `runtime invoke`.
 export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) =>
   createHandler({
     name: "simulate",
     description: "replay a dataset against a Runtime, then batch-evaluate the resulting sessions",
     flags: [
-      flag("runtime-id", "Runtime ID to invoke per scenario", z.string().optional()),
-      flag("qualifier", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional()),
+      flag("runtime-id", "Runtime ID to invoke per scenario", z.string().optional(), {
+        group: RUNTIME_INVOCATION,
+      }),
+      flag("qualifier", "Runtime endpoint qualifier (default DEFAULT)", z.string().optional(), {
+        group: RUNTIME_INVOCATION,
+      }),
       flag(
         "payload-template",
-        'JSON payload template; {input} is the scenario input, e.g. {"prompt":"{input}"}',
+        "request body per example (JSON object); {input} is replaced with the input",
         z.string().optional(),
+        { group: RUNTIME_INVOCATION, help: payloadTemplateHelp },
       ),
       flag("header", "an ordered application header (repeatable)", z.array(z.string()).optional(), {
+        group: RUNTIME_INVOCATION,
         sensitive: true,
       }),
       flag(
         "bearer-token",
         "CUSTOM_JWT bearer token (for JWT-auth Runtimes)",
         z.string().optional(),
-        { sensitive: true },
+        { group: RUNTIME_INVOCATION, sensitive: true },
       ),
-      flag("user-id", "Runtime user ID", z.string().optional()),
-      flag("dataset", "dataset source: local JSONL path or a dataset ID", z.string().optional()),
-      flag("dataset-version", "dataset version (with a dataset ID)", z.string().optional()),
-      flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional()),
-      flag("name", "batch evaluation name (unique in the account)", z.string().optional()),
-      flag("description", "description for the batch evaluation", z.string().optional()),
-      flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional()),
+      flag("user-id", "Runtime user ID", z.string().optional(), {
+        group: RUNTIME_INVOCATION,
+      }),
+      flag("dataset", "dataset source: local JSONL path or a dataset ID", z.string().optional(), {
+        group: DATASET,
+      }),
+      flag("dataset-version", "dataset version (with a dataset ID)", z.string().optional(), {
+        group: DATASET,
+      }),
       flag(
         "ingestion-wait-ms",
         "ms to wait for span ingestion before grading (default 180000; 0 to skip)",
         z.coerce.number().int().nonnegative().optional(),
+        { group: DATASET },
       ),
+      flag("name", "batch evaluation name (unique in the account)", z.string().optional(), {
+        group: CONFIGURATION,
+      }),
+      flag("description", "description for the batch evaluation", z.string().optional(), {
+        group: CONFIGURATION,
+      }),
+      flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional(), {
+        group: CONFIGURATION,
+      }),
+      flag("evaluators", "evaluator ID(s) to apply", z.array(z.string()).optional(), {
+        group: "Evaluation:",
+      }),
     ],
     handle: async (ctx, flags) => {
       if (!flags["runtime-id"])

src/handlers/eval/batch-insights/run/index.tsx

@@ -7,23 +7,35 @@ import type { Core } from "../../../types";
 import { coreOptsFromCtx } from "../../../utils";
 import { SessionSource } from "../../sessionSource";
 
+const CONFIGURATION = "Configuration:";
+const ANALYSIS = "Analysis:";
+
 const DEFAULT_INSIGHT = "Builtin.Insight.FailureAnalysis";
 
 export const createRunBatchInsightsHandler = (core: Core, io: AppIO) =>
   createHandler({
     name: "run",
     description: "start an asynchronous batch insights run over existing sessions",
     flags: [
+      flag("name", "batch insights name (must be unique in the account)", z.string().optional(), {
+        group: CONFIGURATION,
+      }),
+      flag("description", "optional description", z.string().optional(), {
+        group: CONFIGURATION,
+      }),
+      flag("kms-key-arn", "KMS key to encrypt insights data at rest", z.string().optional(), {
+        group: CONFIGURATION,
+      }),
       ...SessionSource.flags,
-      flag("insight", "insight ID(s) to run", z.array(z.string()).default([DEFAULT_INSIGHT])),
+      flag("insight", "insight ID(s) to run", z.array(z.string()).default([DEFAULT_INSIGHT]), {
+        group: ANALYSIS,
+      }),
       flag(
         "evaluators",
         "optional evaluator ID(s) to run alongside the insights",
         z.array(z.string()).optional(),
+        { group: ANALYSIS },
       ),
-      flag("name", "batch insights name (must be unique in the account)", z.string().optional()),
-      flag("description", "optional description", z.string().optional()),
-      flag("kms-key-arn", "KMS key to encrypt insights data at rest", z.string().optional()),
     ],
     handle: async (ctx, flags) => {
       if (!flags["name"]) {