From ee2487dd4f0c8961ae5196347a0ac651800ff0ed Mon Sep 17 00:00:00 2001 From: bussyjd Date: Tue, 14 Jul 2026 10:05:41 +0400 Subject: [PATCH 1/9] feat(agent): expose spec.maxConcurrentRuns for the Hermes run cap Renders gateway.api_server.max_concurrent_runs into the agent's config.yaml. Nil omits the gateway block (Hermes internal default, 10); 0 disables the in-process cap so edge concurrency middleware governs instead. Existing agents render byte-identical. Part of #742. Claude-Session: https://claude.ai/code/session_014YjPMViNrZ7zBVgUQzwEKk --- .../base/templates/agent-crd.yaml | 8 +++++++ internal/monetizeapi/types.go | 6 ++++++ internal/monetizeapi/zz_generated.deepcopy.go | 5 +++++ .../serviceoffercontroller/agent_render.go | 9 ++++++++ .../agent_render_test.go | 21 +++++++++++++++++++ 5 files changed, 49 insertions(+) diff --git a/internal/embed/infrastructure/base/templates/agent-crd.yaml b/internal/embed/infrastructure/base/templates/agent-crd.yaml index ad7b2217..e3960df6 100644 --- a/internal/embed/infrastructure/base/templates/agent-crd.yaml +++ b/internal/embed/infrastructure/base/templates/agent-crd.yaml @@ -73,6 +73,14 @@ spec: type: string maxItems: 32 type: array + maxConcurrentRuns: + description: |- + Hermes gateway.api_server.max_concurrent_runs. Nil = omit the + gateway block (Hermes internal default, 10). 0 disables the + in-process cap — edge concurrency middleware governs instead. + maximum: 10000 + minimum: 0 + type: integer maxTurns: description: Hermes agent.max_turns. Nil = 30 (historical sub-agent default). diff --git a/internal/monetizeapi/types.go b/internal/monetizeapi/types.go index 1386e0ea..d9bdd874 100644 --- a/internal/monetizeapi/types.go +++ b/internal/monetizeapi/types.go @@ -980,6 +980,12 @@ type AgentSpec struct { // +kubebuilder:validation:Minimum=1 // +kubebuilder:validation:Maximum=10000 MaxTurns *int `json:"maxTurns,omitempty"` + // Hermes gateway.api_server.max_concurrent_runs. Nil = omit the + // gateway block (Hermes internal default, 10). 0 disables the + // in-process cap — edge concurrency middleware governs instead. + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:validation:Maximum=10000 + MaxConcurrentRuns *int `json:"maxConcurrentRuns,omitempty"` // Hermes agent.disabled_toolsets. Nil = ["memory","web"] (historical // default). Explicit empty list disables none. // +kubebuilder:validation:MaxItems=32 diff --git a/internal/monetizeapi/zz_generated.deepcopy.go b/internal/monetizeapi/zz_generated.deepcopy.go index b5d25e31..4ca20281 100644 --- a/internal/monetizeapi/zz_generated.deepcopy.go +++ b/internal/monetizeapi/zz_generated.deepcopy.go @@ -199,6 +199,11 @@ func (in *AgentSpec) DeepCopyInto(out *AgentSpec) { *out = new(int) **out = **in } + if in.MaxConcurrentRuns != nil { + in, out := &in.MaxConcurrentRuns, &out.MaxConcurrentRuns + *out = new(int) + **out = **in + } if in.DisabledToolsets != nil { in, out := &in.DisabledToolsets, &out.DisabledToolsets *out = make([]string, len(*in)) diff --git a/internal/serviceoffercontroller/agent_render.go b/internal/serviceoffercontroller/agent_render.go index 38c8f6fa..6a14341a 100644 --- a/internal/serviceoffercontroller/agent_render.go +++ b/internal/serviceoffercontroller/agent_render.go @@ -175,6 +175,15 @@ skills: - /data/.hermes/obol-skills `) + // gateway block only when the operator set maxConcurrentRuns; nil => + // omit entirely so existing agents stay byte-identical. + if agent.Spec.MaxConcurrentRuns != nil { + fmt.Fprintf(&b, `gateway: + api_server: + max_concurrent_runs: %d +`, *agent.Spec.MaxConcurrentRuns) + } + // mcp_servers only when the operator listed servers; empty => omit entirely // so existing agents stay byte-identical to the pre-field template. if len(agent.Spec.MCPServers) > 0 { diff --git a/internal/serviceoffercontroller/agent_render_test.go b/internal/serviceoffercontroller/agent_render_test.go index aa639a50..564bceae 100644 --- a/internal/serviceoffercontroller/agent_render_test.go +++ b/internal/serviceoffercontroller/agent_render_test.go @@ -446,6 +446,7 @@ func TestRenderHermesConfig_UnsetFieldsByteIdentical(t *testing.T) { agent.Spec.ModelProvider = "" agent.Spec.MCPServers = nil agent.Spec.MaxTurns = nil + agent.Spec.MaxConcurrentRuns = nil agent.Spec.DisabledToolsets = nil got := renderHermesConfig(agent, "lit-key") @@ -454,6 +455,26 @@ func TestRenderHermesConfig_UnsetFieldsByteIdentical(t *testing.T) { } } +func TestRenderHermesConfig_MaxConcurrentRunsZero(t *testing.T) { + agent := testAgentForHermesConfig("qwen3.5:9b") + agent.Spec.MaxConcurrentRuns = ptrInt(0) + + got := renderHermesConfig(agent, "lit-key") + want := "gateway:\n api_server:\n max_concurrent_runs: 0\n" + if !strings.Contains(got, want) { + t.Errorf("config missing max_concurrent_runs: 0 block\n---\n%s", got) + } +} + +func TestRenderHermesConfig_MaxConcurrentRunsNilOmitsGateway(t *testing.T) { + agent := testAgentForHermesConfig("qwen3.5:9b") + // MaxConcurrentRuns left nil (default/unset). + got := renderHermesConfig(agent, "lit-key") + if strings.Contains(got, "gateway:") { + t.Errorf("nil MaxConcurrentRuns must omit gateway block entirely; got:\n%s", got) + } +} + // TestRenderHermesConfig_OptionalSpecOverrides covers non-default // ModelProvider, MaxTurns, and MCPServers (incl. unexpanded ${VAR} env). func TestRenderHermesConfig_OptionalSpecOverrides(t *testing.T) { From 78041caedc85e6e1250ae1c28d3a0c66a687dc04 Mon Sep 17 00:00:00 2001 From: bussyjd Date: Tue, 14 Jul 2026 10:05:44 +0400 Subject: [PATCH 2/9] fix(x402): skip settlement when the buyer disconnected MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the buyer's request context is already canceled when the upstream handler succeeds, do not settle — the buyer would be debited for a response nobody receives (proven on-chain against hyperliquid-analyst, see #742). Reports failure reason client_disconnected; the interceptor hijacks the write path so nothing reaches the dead socket. Fixes the propagated-cancel leg of #742. Claude-Session: https://claude.ai/code/session_014YjPMViNrZ7zBVgUQzwEKk --- internal/x402/forwardauth.go | 8 +++++++ internal/x402/forwardauth_test.go | 37 +++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/internal/x402/forwardauth.go b/internal/x402/forwardauth.go index f4c8e30f..e477a4b4 100644 --- a/internal/x402/forwardauth.go +++ b/internal/x402/forwardauth.go @@ -316,6 +316,14 @@ func NewForwardAuthMiddleware(cfg ForwardAuthConfig, requirements []x402types.Pa return true } + // Buyer already gone (client disconnect propagated) — + // don't take their money for a response nobody receives. + if err := r.Context().Err(); err != nil { + log.Printf("x402: buyer disconnected before settlement, skipping settlement: %v", err) + reportFailure("client_disconnected") + return false + } + settleResp, err := facilitatorSettle(r.Context(), settleClient, cfg.FacilitatorURL, payloadBytes, matchedReq) if err != nil { log.Printf("x402: settlement failed: %v", err) diff --git a/internal/x402/forwardauth_test.go b/internal/x402/forwardauth_test.go index 0fc7c3f8..be68d887 100644 --- a/internal/x402/forwardauth_test.go +++ b/internal/x402/forwardauth_test.go @@ -2,6 +2,7 @@ package x402 import ( "bytes" + "context" "encoding/base64" "encoding/json" "io" @@ -571,6 +572,42 @@ func TestForwardAuth_NoSettleOnHandlerError(t *testing.T) { } } +func TestForwardAuth_NoSettleOnClientDisconnect(t *testing.T) { + var verifyCalled, settleCalled atomic.Int32 + fac := mockFacilitatorV1(true, true, &verifyCalled, &settleCalled) + defer fac.Close() + + mw := NewForwardAuthMiddleware(ForwardAuthConfig{ + FacilitatorURL: fac.URL, + VerifyOnly: false, + }, testRequirements()) + + // Cancel after verify succeeds but before WriteHeader triggers + // settlement. Cancelling before ServeHTTP aborts facilitator verify + // (same r.Context()) so settleFunc never runs and verifyCalled stays 0. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + inner := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + cancel() // buyer disconnects after verify, before response commits + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"result":"ok"}`)) + }) + + req := httptest.NewRequest("POST", "/v1/chat/completions", nil) + req = req.WithContext(ctx) + req.Header.Set("X-PAYMENT", validPaymentHeader()) + rec := httptest.NewRecorder() + mw(inner).ServeHTTP(rec, req) + + if verifyCalled.Load() != 1 { + t.Errorf("verify called %d times, want 1", verifyCalled.Load()) + } + if settleCalled.Load() != 0 { + t.Errorf("settle called %d times, want 0 (client disconnected)", settleCalled.Load()) + } +} + func TestForwardAuth_UpstreamAuthPropagation(t *testing.T) { var verifyCalled, settleCalled atomic.Int32 fac := mockFacilitatorV1(true, true, &verifyCalled, &settleCalled) From 397190ed936ada9604fde88d2e25b7f5f9874876 Mon Sep 17 00:00:00 2001 From: bussyjd Date: Tue, 14 Jul 2026 10:15:59 +0400 Subject: [PATCH 3/9] fix(flows): de-flake flow-03 first LiteLLM completion The first chat completion after stack-up intermittently fails with empty output (curl -f hides the response; observed across three release-smoke runs while steps 3-7 through the same port-forward and key always pass). Retry up to 3x and surface the HTTP status so a real failure is diagnosable instead of an empty string. Claude-Session: https://claude.ai/code/session_014YjPMViNrZ7zBVgUQzwEKk --- flows/flow-03-inference.sh | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/flows/flow-03-inference.sh b/flows/flow-03-inference.sh index ada2600d..7061a632 100755 --- a/flows/flow-03-inference.sh +++ b/flows/flow-03-inference.sh @@ -48,15 +48,28 @@ for i in $(seq 1 15); do sleep 2 done -out=$(curl -sf --max-time 120 -X POST http://localhost:8001/v1/chat/completions \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer $LITELLM_KEY" \ - -d "{\"model\":\"$LITELLM_MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2? Reply with the number only.\"}],\"max_tokens\":10,\"stream\":false}" 2>&1) || true +# First completion after stack-up is flaky (port-forward/router warm-up race: +# observed failing with empty output under curl -f while the very next request +# succeeds). Retry a few times and keep the HTTP status so a real failure is +# diagnosable instead of an empty string. +out="" code="" +for attempt in 1 2 3; do + out=$(curl -s --max-time 120 -w "\n%{http_code}" -X POST http://localhost:8001/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_KEY" \ + -d "{\"model\":\"$LITELLM_MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"What is 2+2? Reply with the number only.\"}],\"max_tokens\":10,\"stream\":false}" 2>&1) || true + code="${out##*$'\n'}" + out="${out%$'\n'*}" + if echo "$out" | grep -q "choices"; then + break + fi + sleep 10 +done if echo "$out" | grep -q "choices"; then - pass "LiteLLM inference returned choices" + pass "LiteLLM inference returned choices (attempt $attempt)" else - fail "LiteLLM inference failed — ${out:0:300}" + fail "LiteLLM inference failed — HTTP ${code:-000}: ${out:0:300}" fi # §3d: Tool-call passthrough From 171731b403280dfaf5e1a9316230eb23ad4d618f Mon Sep 17 00:00:00 2001 From: bussyjd Date: Tue, 14 Jul 2026 10:50:00 +0400 Subject: [PATCH 4/9] fix(model): register discovered endpoints with /v1 api_base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Local-server auto-discovery registered OpenAI-compatible endpoints (vLLM on :8000) with a bare api_base. LiteLLM's OpenAI provider does not append /v1 (CLAUDE.md pitfall 6), so every request through the discovered entry hit POST /chat/completions and 404'd. When an explicit 'obol model setup custom --endpoint .../v1' entry shared the same model group, requests coin-flipped between the good and bad deployment — root cause of the intermittent release-smoke flow-11 step[43] 404 and the flow-03 step[2] flake (latent since v0.10.0, 69f25bf5). Claude-Session: https://claude.ai/code/session_014YjPMViNrZ7zBVgUQzwEKk --- internal/model/discover.go | 6 +++++- internal/model/discover_test.go | 5 +++-- internal/model/model.go | 20 ++++++-------------- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/internal/model/discover.go b/internal/model/discover.go index 115867c2..e1a7d3ea 100644 --- a/internal/model/discover.go +++ b/internal/model/discover.go @@ -81,7 +81,11 @@ func buildDiscoveredProvider(ep inference.EndpointInfo) DiscoveredProvider { if m.ID == "" { continue } - entries = append(entries, buildCustomEndpointEntry(m.ID, cluster, "")) + // The /v1 suffix is required — LiteLLM's OpenAI provider does not + // append it (CLAUDE.md pitfall 6). Without it, discovered vLLM + // endpoints 404 on /chat/completions and poison the model group + // alongside any correct `model setup custom` entry. + entries = append(entries, buildCustomEndpointEntry(m.ID, cluster+"/v1", "")) } return DiscoveredProvider{ diff --git a/internal/model/discover_test.go b/internal/model/discover_test.go index 7e1b6eff..a4e0500e 100644 --- a/internal/model/discover_test.go +++ b/internal/model/discover_test.go @@ -99,8 +99,9 @@ func TestBuildDiscoveredProvider_TranslatesHost(t *testing.T) { if first.LiteLLMParams.Model != "openai/meta-llama/Llama-3.1-8B-Instruct" { t.Errorf("LiteLLMParams.Model = %q, want openai/meta-llama/Llama-3.1-8B-Instruct", first.LiteLLMParams.Model) } - if first.LiteLLMParams.APIBase != "http://host.k3d.internal:8000" { - t.Errorf("APIBase = %q, want http://host.k3d.internal:8000", first.LiteLLMParams.APIBase) + // /v1 suffix required — LiteLLM's OpenAI provider does not append it. + if first.LiteLLMParams.APIBase != "http://host.k3d.internal:8000/v1" { + t.Errorf("APIBase = %q, want http://host.k3d.internal:8000/v1", first.LiteLLMParams.APIBase) } // buildCustomEndpointEntry sets api_key to "none" when no key is given. if first.LiteLLMParams.APIKey != "none" { diff --git a/internal/model/model.go b/internal/model/model.go index 553c5347..6e972081 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -1574,6 +1574,12 @@ func buildCustomEndpointEntry(modelName, clusterEndpoint, apiKey string) ModelEn return buildCustomEndpointEntryWithOptions(modelName, clusterEndpoint, apiKey, CustomEndpointOptions{}) } +// INVARIANT: clusterEndpoint must be the full OpenAI-compatible base +// INCLUDING the /v1 suffix — LiteLLM's `openai/` provider sends requests +// to /chat/completions verbatim and never appends /v1 +// (CLAUDE.md pitfall 6). Both callers uphold this: AddCustomEndpoint +// validates by probing /chat/completions, and discovery only +// registers endpoints that answered /v1/models (and appends /v1). func buildCustomEndpointEntryWithOptions(modelName, clusterEndpoint, apiKey string, options CustomEndpointOptions) ModelEntry { entry := ModelEntry{ ModelName: modelName, @@ -1696,20 +1702,6 @@ func decodeBase64(s string) (string, error) { return string(decoded[:n]), nil } -// WarnAndStripV1Suffix checks if an endpoint URL has a trailing /v1 suffix, -// warns the user, and returns the stripped URL. For OpenAI-compatible providers, -// LiteLLM auto-appends /v1, causing double /v1/v1 if the user includes it. -func WarnAndStripV1Suffix(endpoint string) string { - trimmed := strings.TrimRight(endpoint, "/") - if strings.HasSuffix(trimmed, "/v1") { - fmt.Printf(" Warning: stripping trailing /v1 from endpoint URL (LiteLLM adds it automatically)\n") - fmt.Printf(" %s → %s\n", trimmed, strings.TrimSuffix(trimmed, "/v1")) - - return strings.TrimSuffix(trimmed, "/v1") - } - - return endpoint -} // localhostToClusterEndpoint translates localhost URLs to k3d-internal URLs // so that services running on the host are reachable from inside the k3d cluster. From aa37271fe25ec4f0c6385e5e41e4e369522ad0ea Mon Sep 17 00:00:00 2001 From: bussyjd Date: Tue, 14 Jul 2026 11:19:41 +0400 Subject: [PATCH 5/9] chore(model): drop dead HasProviderConfigured MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Zero callers anywhere (including tests) — flagged by the /v1 pipeline audit's dead-code sweep alongside WarnAndStripV1Suffix (removed in #745). Other deadcode-tool hits in internal/{buy,inference} are test-covered exported API and were left alone. Claude-Session: https://claude.ai/code/session_014YjPMViNrZ7zBVgUQzwEKk --- internal/model/model.go | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/internal/model/model.go b/internal/model/model.go index 553c5347..9e109b8c 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -308,40 +308,6 @@ func HasConfiguredModels(cfg *config.Config) bool { return false } -// HasProviderConfigured returns true if LiteLLM already has at least one -// model entry for the given provider (e.g., "anthropic", "openai"). -func HasProviderConfigured(cfg *config.Config, provider string) bool { - kubectlBinary := filepath.Join(cfg.BinDir, "kubectl") - kubeconfigPath := filepath.Join(cfg.ConfigDir, "kubeconfig.yaml") - - raw, err := kubectl.Output(kubectlBinary, kubeconfigPath, - "get", "configmap", configMapName, "-n", namespace, "-o", "jsonpath={.data.config\\.yaml}") - if err != nil { - return false - } - - var litellmConfig LiteLLMConfig - if err := yaml.Unmarshal([]byte(raw), &litellmConfig); err != nil { - return false - } - - for _, entry := range litellmConfig.ModelList { - // Check wildcard entries like "anthropic/*" - if entry.ModelName == provider+"/*" { - return true - } - // Check if the model's litellm_params.model starts with "provider/" - if strings.HasPrefix(entry.LiteLLMParams.Model, provider+"/") { - return true - } - // Check via model name inference - if ProviderFromModelName(entry.ModelName) == provider { - return true - } - } - - return false -} // LoadDotEnv reads KEY=value pairs from a .env file. // Returns an empty map if the file doesn't exist or is unreadable. From 85e88d6c83813ed16d4fb80a1982fdf722b732c2 Mon Sep 17 00:00:00 2001 From: bussyjd Date: Tue, 14 Jul 2026 12:11:37 +0400 Subject: [PATCH 6/9] chore: delete committee-verified dead code (-595 LOC) Two-vendor (GPT + Grok) audited deletions, intersection-only: - inference: dead ProbeEndpoint/ScanLocalEndpoints/DetectServerType/ ParseModelsResponse wrappers (live path is the *Context variants used by model discovery); tests retargeted to live functions - erc8004: superseded NewClient/Register/RegisterDetailed/SetAgentURI (+ETH/RegisterWithOpts/SubmitRegister); tests retargeted to the WithOpts variants - x402: ResolveAssetInfo/BuildV1Requirement/TokenSupportedOnChain (production uses ResolveAssetInfoForPayment/BuildV2Requirement/ ResolveToken); test coverage migrated, not dropped - hermes: hermesExecArgs legacy test adapter + dead List/Skills/ ResolveInstance; agentruntime/dns/stack/schemas/serviceoffercontroller zero-reference symbols; dead flow-lib shell functions - flow-11: fix stale diag capture (deploy/litellm -c x402-buyer -> deploy/x402-buyer, stale since the b68fc346 split) Held back by committee split or product decision: RouteRulesForOffer (golden-test seam), erc8004 SetMetadata, all TEE/CoCo + enclave + OpenClaw capabilities (kept intentionally, see #748). Claude-Session: https://claude.ai/code/session_014YjPMViNrZ7zBVgUQzwEKk --- cmd/obol/buy.go | 12 ---- cmd/obol/sell.go | 24 -------- flows/flow-07-sell-verify.sh | 21 ------- flows/flow-08-buy.sh | 10 ---- flows/flow-11-dual-stack.sh | 28 +--------- flows/lib.sh | 16 ------ internal/agentruntime/runtime.go | 51 ----------------- internal/dns/resolver.go | 13 ----- internal/erc8004/client.go | 64 --------------------- internal/erc8004/client_test.go | 59 +++++++++++++------- internal/erc8004/signer.go | 2 +- internal/hermes/hermes.go | 65 ---------------------- internal/hermes/hermes_test.go | 18 ------ internal/inference/detect.go | 27 --------- internal/inference/detect_test.go | 50 +---------------- internal/model/model.go | 9 +-- internal/schemas/payment.go | 13 ----- internal/schemas/payment_test.go | 36 ------------ internal/serviceoffercontroller/catalog.go | 11 ---- internal/serviceoffercontroller/render.go | 8 --- internal/stack/stack.go | 13 ----- internal/x402/chains.go | 31 ----------- internal/x402/chains_test.go | 27 +-------- internal/x402/paymentrequired.go | 12 ---- internal/x402/tokens.go | 6 -- internal/x402/tokens_test.go | 18 ------ 26 files changed, 51 insertions(+), 593 deletions(-) diff --git a/cmd/obol/buy.go b/cmd/obol/buy.go index 9fe0efca..6534b639 100644 --- a/cmd/obol/buy.go +++ b/cmd/obol/buy.go @@ -806,18 +806,6 @@ func authMultiplier(priceUnit string) int { return 1 } -// perRequestEstimate divides a per-MTok atomic price by the temporary -// tokens-per-request constant the controller uses today. Until the -// facilitator implements usage-based settlement, this is the actual -// flat per-call charge a buyer pays — surface it so users can compare -// against perRequest offers without converting in their head. -func perRequestEstimate(perMTokAtomic *big.Int) *big.Int { - if perMTokAtomic == nil { - return nil - } - return new(big.Int).Quo(perMTokAtomic, big.NewInt(int64(schemas.ApproxTokensPerRequest))) -} - func pricePerAuth(price *big.Int, multiplier int) (*big.Int, error) { if price == nil || price.Sign() <= 0 { return nil, errors.New("internal: price is non-positive") diff --git a/cmd/obol/sell.go b/cmd/obol/sell.go index 946e1854..7e278377 100644 --- a/cmd/obol/sell.go +++ b/cmd/obol/sell.go @@ -3932,30 +3932,6 @@ func envOrDefault(key, fallback string) string { return fallback } -func formatPriceTableSummary(priceTable schemas.PriceTable, symbol string) string { - if symbol == "" { - symbol = "USDC" - } - switch { - case priceTable.PerRequest != "": - return fmt.Sprintf("%s %s/request", priceTable.PerRequest, symbol) - case priceTable.PerMTok != "": - return fmt.Sprintf("%s %s/request (approx from %s %s/MTok @ %d tok/request)", - priceTable.EffectiveRequestPrice(), symbol, - priceTable.PerMTok, symbol, - schemas.ApproxTokensPerRequest, - ) - case priceTable.PerHour != "": - return fmt.Sprintf("%s %s/request (approx from %s %s/hour @ %d min/request)", - priceTable.EffectiveRequestPrice(), symbol, - priceTable.PerHour, symbol, - schemas.ApproxMinutesPerRequest, - ) - default: - return fmt.Sprintf("0 %s/request", symbol) - } -} - func formatRoutePriceSummary(route x402verifier.RouteRule) string { symbol := route.AssetSymbol if symbol == "" { diff --git a/flows/flow-07-sell-verify.sh b/flows/flow-07-sell-verify.sh index 37ce54f5..35378ce6 100755 --- a/flows/flow-07-sell-verify.sh +++ b/flows/flow-07-sell-verify.sh @@ -61,27 +61,6 @@ if [ -n "$TUNNEL_URL" ]; then TUNNEL_IP=$(resolve_public_ipv4 "$TUNNEL_HOST" || true) fi -tunnel_get_code() { - local url="$1" - local code - if [ -n "$TUNNEL_HOST" ] && [ -n "$TUNNEL_IP" ]; then - if code=$(curl -sS --max-time 15 -o /dev/null -w '%{http_code}' \ - --resolve "$TUNNEL_HOST:443:$TUNNEL_IP" \ - "$url" 2>/dev/null); then - printf '%s\n' "$code" - else - printf '000\n' - fi - else - if code=$(curl -sS --max-time 15 -o /dev/null -w '%{http_code}' \ - "$url" 2>/dev/null); then - printf '%s\n' "$code" - else - printf '000\n' - fi - fi -} - tunnel_get_file_code() { local url="$1" local outfile="$2" diff --git a/flows/flow-08-buy.sh b/flows/flow-08-buy.sh index 1de2a63f..8c12de08 100755 --- a/flows/flow-08-buy.sh +++ b/flows/flow-08-buy.sh @@ -59,10 +59,6 @@ purchase_request_ready() { -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>&1 || true } -purchase_request_absent() { - ! "$OBOL" kubectl get purchaserequests.obol.org "$PURCHASE_NAME" -n "$AGENT_NS" >/dev/null 2>&1 -} - buyer_sidecar_status() { "$OBOL" kubectl exec -n llm deployment/litellm -c litellm -- \ python3 -c " @@ -86,12 +82,6 @@ print('ready') " 2>&1 || true } -agent_buy_skill_balance() { - "$OBOL" kubectl exec \ - -n "$AGENT_NS" "deploy/$AGENT_DEPLOY" -c "$AGENT_CONTAINER" -- \ - python3 "$AGENT_BUY_PY" balance --chain base-sepolia 2>&1 || true -} - agent_wallet_anvil_balance() { env -u CHAIN cast call "$USDC_ADDRESS" "balanceOf(address)(uint256)" "$AGENT_WALLET" \ --rpc-url "$ANVIL_RPC" 2>&1 || true diff --git a/flows/flow-11-dual-stack.sh b/flows/flow-11-dual-stack.sh index 51b08a62..64957e52 100755 --- a/flows/flow-11-dual-stack.sh +++ b/flows/flow-11-dual-stack.sh @@ -411,30 +411,6 @@ PY exit 1 } -run_tail_or_fail() { - local desc="$1" - local success="$2" - local success_lines="${3:-3}" - shift 3 - - step "$desc" - local out rc - set +e - out=$("$@" 2>&1) - rc=$? - set -e - - if [ "$rc" -ne 0 ]; then - printf '%s\n' "$out" | tail -120 - fail "$desc failed (exit $rc)" - emit_metrics - exit "$rc" - fi - - printf '%s\n' "$out" | tail -"$success_lines" - pass "$success" -} - refresh_alice_ports() { ALICE_HTTP_PORT="${FLOW11_ALICE_HTTP_PORT:-$(pick_free_port)}" ALICE_HTTP_ALT_PORT="${FLOW11_ALICE_HTTP_ALT_PORT:-$(pick_free_port)}" @@ -1456,9 +1432,9 @@ else mkdir -p "$diag_dir" echo " [diag] capturing cluster state to $diag_dir" >&2 alice kubectl logs -n x402 deploy/x402-verifier --tail=200 > "$diag_dir/alice-verifier.log" 2>&1 || true - alice kubectl logs -n llm deploy/litellm -c x402-buyer --tail=200 > "$diag_dir/alice-buyer.log" 2>&1 || true + alice kubectl logs -n llm deploy/x402-buyer --tail=200 > "$diag_dir/alice-buyer.log" 2>&1 || true bob kubectl logs -n x402 deploy/x402-verifier --tail=200 > "$diag_dir/bob-verifier.log" 2>&1 || true - bob kubectl logs -n llm deploy/litellm -c x402-buyer --tail=200 > "$diag_dir/bob-buyer.log" 2>&1 || true + bob kubectl logs -n llm deploy/x402-buyer --tail=200 > "$diag_dir/bob-buyer.log" 2>&1 || true alice kubectl get serviceoffer -A -o yaml > "$diag_dir/alice-serviceoffers.yaml" 2>&1 || true bob kubectl get purchaserequest -A -o yaml > "$diag_dir/bob-purchaserequests.yaml" 2>&1 || true cleanup_pid "$PF_AGENT" diff --git a/flows/lib.sh b/flows/lib.sh index 2cbd90bb..afe69822 100755 --- a/flows/lib.sh +++ b/flows/lib.sh @@ -1376,20 +1376,4 @@ PY } fi -ensure_image_in_k3d() { - local img="$1" - local cluster="$2" - local node="k3d-${cluster}-server-0" - if ! docker exec "$node" crictl images 2>/dev/null | grep -q "$(echo "$img" | cut -d: -f1)\b"; then - docker_pull_public_image "$img" "${K3D_IMAGE_PULL_TIMEOUT:-300}" || return 1 - local tar - tar=$(mktemp -t k3d-img-XXXXXX.tar) - docker save "$img" -o "$tar" - docker cp "$tar" "$node:/tmp/$(basename "$tar")" - docker exec "$node" ctr -n k8s.io images import "/tmp/$(basename "$tar")" - docker exec "$node" rm -f "/tmp/$(basename "$tar")" - rm -f "$tar" - fi -} - init_obol_ingress_env_static diff --git a/internal/agentruntime/runtime.go b/internal/agentruntime/runtime.go index f8f2b403..3cbb7e7f 100644 --- a/internal/agentruntime/runtime.go +++ b/internal/agentruntime/runtime.go @@ -1,7 +1,6 @@ package agentruntime import ( - "errors" "fmt" "os" "path/filepath" @@ -144,10 +143,6 @@ func WorkspacePath(cfg *config.Config, runtime Runtime, id string) string { return filepath.Join(HomePath(cfg, runtime, id), "workspace") } -func SkillsPath(cfg *config.Config, runtime Runtime, id string) string { - return filepath.Join(HomePath(cfg, runtime, id), "skills") -} - func KeystoreVolumePath(cfg *config.Config, runtime Runtime, id string) string { return filepath.Join(cfg.DataDir, Namespace(runtime, id), "remote-signer-keystores") } @@ -174,49 +169,3 @@ func ListInstanceIDs(cfg *config.Config, runtime Runtime) ([]string, error) { return ids, nil } - -func ResolveInstance(cfg *config.Config, runtime Runtime, args []string) (id string, remaining []string, err error) { - instances, err := ListInstanceIDs(cfg, runtime) - if err != nil { - return "", nil, err - } - - desc := Describe(runtime) - - switch len(instances) { - case 0: - return "", nil, fmt.Errorf("no %s instances found — run 'obol agent new --runtime %s' to create one", desc.DisplayName, runtime) - case 1: - return instances[0], args, nil - default: - if len(args) > 0 { - for _, inst := range instances { - if args[0] == inst { - return inst, args[1:], nil - } - } - } - - return "", nil, fmt.Errorf("multiple %s instances found, specify one: %s", desc.DisplayName, strings.Join(instances, ", ")) - } -} - -func MustDefaultDeploymentPath(cfg *config.Config) string { - return DeploymentPath(cfg, Hermes, DefaultInstanceID) -} - -func ResolveSingleDefaultNamespace(cfg *config.Config, runtime Runtime) (string, error) { - ids, err := ListInstanceIDs(cfg, runtime) - if err != nil { - return "", err - } - - switch len(ids) { - case 0: - return "", errors.New("no instances found") - case 1: - return Namespace(runtime, ids[0]), nil - default: - return "", fmt.Errorf("multiple %s instances found (%s), specify an instance", Describe(runtime).DisplayName, strings.Join(ids, ", ")) - } -} diff --git a/internal/dns/resolver.go b/internal/dns/resolver.go index 74dc3f23..e13a3aa6 100644 --- a/internal/dns/resolver.go +++ b/internal/dns/resolver.go @@ -108,19 +108,6 @@ func RemoveSystemResolver() { RemoveHostsEntries() } -// IsResolverConfigured checks whether the system resolver is already set up. -func IsResolverConfigured() bool { - switch runtime.GOOS { - case osDarwin: - _, err := os.Stat(filepath.Join(macResolverDir, macResolverFile)) - return err == nil - case osLinux: - return hasNMDnsmasqConfig() && isNMResolvConfActive() - default: - return false - } -} - // --- /etc/hosts management --- // // macOS Sequoia (15.x) has a known issue where /etc/resolver/ files don't diff --git a/internal/erc8004/client.go b/internal/erc8004/client.go index 6ce32efb..5c99c55e 100644 --- a/internal/erc8004/client.go +++ b/internal/erc8004/client.go @@ -27,12 +27,6 @@ type Client struct { chainID *big.Int } -// NewClient connects to rpcURL and binds to the Identity Registry on Base Sepolia. -// For multi-network support, use NewClientForNetwork instead. -func NewClient(ctx context.Context, rpcURL string) (*Client, error) { - return newClient(ctx, rpcURL, IdentityRegistryBaseSepolia) -} - // NewClientForNetwork connects to the eRPC base URL and binds to the Identity // Registry on the given network. The RPC URL is constructed as // rpcBaseURL + "/" + net.ERPCNetwork. @@ -81,36 +75,6 @@ func (c *Client) ChainID() *big.Int { return new(big.Int).Set(c.chainID) } -// ETH returns the underlying ethclient for direct RPC calls. -func (c *Client) ETH() *ethclient.Client { - return c.eth -} - -// Register mints a new agent NFT with the given agentURI using a raw private key. -// Returns the minted agentId (token ID). -func (c *Client) Register(ctx context.Context, key *ecdsa.PrivateKey, agentURI string) (*big.Int, error) { - agentID, _, err := c.RegisterDetailed(ctx, key, agentURI) - return agentID, err -} - -// RegisterDetailed mints a new agent NFT with the given agentURI and returns -// both the minted agentId and transaction hash. -func (c *Client) RegisterDetailed(ctx context.Context, key *ecdsa.PrivateKey, agentURI string) (*big.Int, string, error) { - opts, err := bind.NewKeyedTransactorWithChainID(key, c.chainID) - if err != nil { - return nil, "", fmt.Errorf("erc8004: transactor: %w", err) - } - opts.Context = ctx - return c.RegisterWithOptsDetailed(ctx, opts, agentURI) -} - -// RegisterWithOpts mints a new agent NFT using the provided TransactOpts. -// This allows callers to supply a custom Signer (e.g. remote-signer). -func (c *Client) RegisterWithOpts(ctx context.Context, opts *bind.TransactOpts, agentURI string) (*big.Int, error) { - agentID, _, err := c.RegisterWithOptsDetailed(ctx, opts, agentURI) - return agentID, err -} - // RegisterWithOptsDetailed mints a new agent NFT using the provided // TransactOpts and returns both the minted agentId and transaction hash. func (c *Client) RegisterWithOptsDetailed(ctx context.Context, opts *bind.TransactOpts, agentURI string) (*big.Int, string, error) { @@ -127,23 +91,6 @@ func (c *Client) RegisterWithOptsDetailed(ctx context.Context, opts *bind.Transa return c.parseRegisteredEvent(receipt, tx.Hash()) } -// SubmitRegister submits a registration transaction and returns its hash -// without waiting for the receipt. -func (c *Client) SubmitRegister(ctx context.Context, key *ecdsa.PrivateKey, agentURI string) (string, error) { - opts, err := bind.NewKeyedTransactorWithChainID(key, c.chainID) - if err != nil { - return "", fmt.Errorf("erc8004: transactor: %w", err) - } - opts.Context = ctx - - tx, err := c.contract.Transact(opts, "register", agentURI) - if err != nil { - return "", fmt.Errorf("erc8004: register tx: %w", err) - } - - return tx.Hash().Hex(), nil -} - // CurrentBlockNumber returns the current tip height for the connected chain. func (c *Client) CurrentBlockNumber(ctx context.Context) (uint64, error) { height, err := c.eth.BlockNumber(ctx) @@ -270,17 +217,6 @@ func (c *Client) SetMetadataWithOpts(ctx context.Context, opts *bind.TransactOpt return nil } -// SetAgentURI updates the agentURI for an existing agent NFT. -func (c *Client) SetAgentURI(ctx context.Context, key *ecdsa.PrivateKey, agentID *big.Int, uri string) error { - opts, err := bind.NewKeyedTransactorWithChainID(key, c.chainID) - if err != nil { - return fmt.Errorf("erc8004: transactor: %w", err) - } - opts.Context = ctx - _, err = c.SetAgentURIWithOpts(ctx, opts, agentID, uri) - return err -} - // SetAgentURIWithOpts updates the agentURI using a caller-supplied // TransactOpts. Used by remote-signer flows where the CLI never sees raw // key material; the opts.Signer delegates to an HTTP signer. Returns the diff --git a/internal/erc8004/client_test.go b/internal/erc8004/client_test.go index 611e0517..44908bdf 100644 --- a/internal/erc8004/client_test.go +++ b/internal/erc8004/client_test.go @@ -15,6 +15,7 @@ import ( "time" "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" ) @@ -117,7 +118,7 @@ func TestNewClient(t *testing.T) { ctx := context.Background() - client, err := NewClient(ctx, srv.URL) + client, err := newClient(ctx, srv.URL, IdentityRegistryBaseSepolia) if err != nil { t.Fatalf("NewClient: %v", err) } @@ -268,13 +269,18 @@ func TestRegister(t *testing.T) { ctx := context.Background() - client, err := NewClient(ctx, srv.URL) + client, err := newClient(ctx, srv.URL, IdentityRegistryBaseSepolia) if err != nil { t.Fatalf("NewClient: %v", err) } defer client.Close() - agentID, err := client.Register(ctx, key, "https://example.com/.well-known/agent-registration.json") + opts, err := bind.NewKeyedTransactorWithChainID(key, client.chainID) + if err != nil { + t.Fatalf("transactor: %v", err) + } + opts.Context = ctx + agentID, _, err := client.RegisterWithOptsDetailed(ctx, opts, "https://example.com/.well-known/agent-registration.json") if err != nil { t.Fatalf("Register: %v", err) } @@ -312,7 +318,7 @@ func TestGetMetadata(t *testing.T) { ctx := context.Background() - client, err := NewClient(ctx, srv.URL) + client, err := newClient(ctx, srv.URL, IdentityRegistryBaseSepolia) if err != nil { t.Fatalf("NewClient: %v", err) } @@ -355,7 +361,7 @@ func TestTokenURI(t *testing.T) { ctx := context.Background() - client, err := NewClient(ctx, srv.URL) + client, err := newClient(ctx, srv.URL, IdentityRegistryBaseSepolia) if err != nil { t.Fatalf("NewClient: %v", err) } @@ -470,13 +476,18 @@ func TestSetAgentURI(t *testing.T) { ctx := context.Background() - client, err := NewClient(ctx, srv.URL) + client, err := newClient(ctx, srv.URL, IdentityRegistryBaseSepolia) if err != nil { t.Fatalf("NewClient: %v", err) } defer client.Close() - err = client.SetAgentURI(ctx, key, big.NewInt(42), "https://example.com/updated") + opts, err := bind.NewKeyedTransactorWithChainID(key, client.chainID) + if err != nil { + t.Fatalf("transactor: %v", err) + } + opts.Context = ctx + _, err = client.SetAgentURIWithOpts(ctx, opts, big.NewInt(42), "https://example.com/updated") if err != nil { t.Fatalf("SetAgentURI: %v", err) } @@ -496,7 +507,7 @@ func TestSetMetadata(t *testing.T) { ctx := context.Background() - client, err := NewClient(ctx, srv.URL) + client, err := newClient(ctx, srv.URL, IdentityRegistryBaseSepolia) if err != nil { t.Fatalf("NewClient: %v", err) } @@ -524,7 +535,7 @@ func TestSetMetadata_TransactRevert(t *testing.T) { ctx := context.Background() - client, err := NewClient(ctx, srv.URL) + client, err := newClient(ctx, srv.URL, IdentityRegistryBaseSepolia) if err != nil { t.Fatalf("NewClient: %v", err) } @@ -567,7 +578,7 @@ func TestSetMetadata_RevertSurfacesErrorString(t *testing.T) { defer srv.Close() ctx := context.Background() - client, err := NewClient(ctx, srv.URL) + client, err := newClient(ctx, srv.URL, IdentityRegistryBaseSepolia) if err != nil { t.Fatalf("NewClient: %v", err) } @@ -606,7 +617,7 @@ func TestSetMetadata_RevertSurfacesCustomErrorSelector(t *testing.T) { defer srv.Close() ctx := context.Background() - client, err := NewClient(ctx, srv.URL) + client, err := newClient(ctx, srv.URL, IdentityRegistryBaseSepolia) if err != nil { t.Fatalf("NewClient: %v", err) } @@ -624,7 +635,7 @@ func TestSetMetadata_RevertSurfacesCustomErrorSelector(t *testing.T) { func TestNewClient_DialError(t *testing.T) { ctx := context.Background() // Use an unreachable address to trigger a dial/chain-id error. - _, err := NewClient(ctx, "http://127.0.0.1:1") + _, err := newClient(ctx, "http://127.0.0.1:1", IdentityRegistryBaseSepolia) if err == nil { t.Fatal("expected error from unreachable RPC URL, got nil") } @@ -645,13 +656,18 @@ func TestRegister_NoRegisteredEvent(t *testing.T) { ctx := context.Background() - client, err := NewClient(ctx, srv.URL) + client, err := newClient(ctx, srv.URL, IdentityRegistryBaseSepolia) if err != nil { t.Fatalf("NewClient: %v", err) } defer client.Close() - _, err = client.Register(ctx, key, "https://example.com/agent") + opts, err := bind.NewKeyedTransactorWithChainID(key, client.chainID) + if err != nil { + t.Fatalf("transactor: %v", err) + } + opts.Context = ctx + _, _, err = client.RegisterWithOptsDetailed(ctx, opts, "https://example.com/agent") if err == nil { t.Fatal("expected error when Registered event not found, got nil") } @@ -679,13 +695,18 @@ func TestRegister_TxError(t *testing.T) { ctx := context.Background() - client, err := NewClient(ctx, srv.URL) + client, err := newClient(ctx, srv.URL, IdentityRegistryBaseSepolia) if err != nil { t.Fatalf("NewClient: %v", err) } defer client.Close() - _, err = client.Register(ctx, key, "https://example.com/agent") + opts, err := bind.NewKeyedTransactorWithChainID(key, client.chainID) + if err != nil { + t.Fatalf("transactor: %v", err) + } + opts.Context = ctx + _, _, err = client.RegisterWithOptsDetailed(ctx, opts, "https://example.com/agent") if err == nil { t.Fatal("expected error from sendRawTransaction failure, got nil") } @@ -717,7 +738,7 @@ func TestGetMetadata_EmptyResult(t *testing.T) { ctx := context.Background() - client, err := NewClient(ctx, srv.URL) + client, err := newClient(ctx, srv.URL, IdentityRegistryBaseSepolia) if err != nil { t.Fatalf("NewClient: %v", err) } @@ -777,7 +798,7 @@ func TestWaitForAgent_RetriesUntilOwnerVisible(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - client, err := NewClient(ctx, srv.URL) + client, err := newClient(ctx, srv.URL, IdentityRegistryBaseSepolia) if err != nil { t.Fatalf("NewClient: %v", err) } @@ -813,7 +834,7 @@ func TestWaitForAgent_TimeoutReturnsError(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - client, err := NewClient(ctx, srv.URL) + client, err := newClient(ctx, srv.URL, IdentityRegistryBaseSepolia) if err != nil { t.Fatalf("NewClient: %v", err) } diff --git a/internal/erc8004/signer.go b/internal/erc8004/signer.go index 6201733a..ee5c6ff9 100644 --- a/internal/erc8004/signer.go +++ b/internal/erc8004/signer.go @@ -220,7 +220,7 @@ func (s *RemoteSigner) SignTypedData(ctx context.Context, addr common.Address, d } // RemoteTransactOpts creates a bind.TransactOpts that delegates signing to the -// remote-signer. The returned opts can be used with Client.RegisterWithOpts and +// remote-signer. The returned opts can be used with Client.RegisterWithOptsDetailed and // Client.SetMetadataWithOpts. func (s *RemoteSigner) RemoteTransactOpts(ctx context.Context, addr common.Address, chainID *big.Int) *bind.TransactOpts { return &bind.TransactOpts{ diff --git a/internal/hermes/hermes.go b/internal/hermes/hermes.go index fc0ce1aa..310c1cda 100644 --- a/internal/hermes/hermes.go +++ b/internal/hermes/hermes.go @@ -61,12 +61,6 @@ type DashboardOptions struct { NoBrowser bool } -type instance struct { - ID string `json:"id"` - Namespace string `json:"namespace"` - URL string `json:"url"` -} - func DeploymentPath(cfg *config.Config, id string) string { return agentruntime.DeploymentPath(cfg, agentruntime.Hermes, id) } @@ -292,43 +286,6 @@ func Setup(cfg *config.Config, id string, _ SetupOptions, u *ui.UI) error { return Sync(cfg, id, u) } -func List(cfg *config.Config, u *ui.UI) error { - ids, err := agentruntime.ListInstanceIDs(cfg, agentruntime.Hermes) - if err != nil { - return err - } - - var instances []instance - for _, id := range ids { - instances = append(instances, instance{ - ID: id, - Namespace: agentruntime.Namespace(agentruntime.Hermes, id), - URL: "http://" + agentruntime.Hostname(agentruntime.Hermes, id), - }) - } - - if u.IsJSON() { - return u.JSON(instances) - } - - if len(instances) == 0 { - u.Print("No Hermes instances installed") - u.Print("\nTo create one: obol agent new --runtime hermes") - return nil - } - - u.Info("Hermes instances:") - u.Blank() - for _, inst := range instances { - u.Bold(" " + inst.ID) - u.Detail(" Namespace", inst.Namespace) - u.Detail(" URL", inst.URL) - u.Blank() - } - u.Printf("Total: %d instance(s)", len(instances)) - return nil -} - func Delete(cfg *config.Config, id string, force bool, u *ui.UI) error { namespace := agentruntime.Namespace(agentruntime.Hermes, id) deploymentDir := DeploymentPath(cfg, id) @@ -568,18 +525,10 @@ func hermesDeploymentInstalled(cfg *config.Config, id string) (bool, error) { return true, nil } -func Skills(cfg *config.Config, id string, args []string) error { - return cliViaKubectlExec(cfg, id, append([]string{"skills"}, args...)) -} - func CLI(cfg *config.Config, id string, args []string) error { return cliViaKubectlExec(cfg, id, args) } -func ResolveInstance(cfg *config.Config, args []string) (string, []string, error) { - return agentruntime.ResolveInstance(cfg, agentruntime.Hermes, args) -} - func ResolveCLIInvocation(cfg *config.Config, args []string) (string, []string, error) { selectedID, hermesArgs, err := splitCLISelection(args) if err != nil { @@ -664,20 +613,6 @@ func cliViaKubectlExec(cfg *config.Config, id string, args []string) error { return agentruntime.ExecInPod(cfg, agentruntime.Hermes, id, append([]string{hermesBinary}, args...)) } -// hermesExecArgs preserves the legacy argv-builder signature (namespace, -// in-pod hermes args, TTY flag) so existing tests stay valid. It composes -// the runtime-agnostic agentruntime.BuildExecArgs with the hermes binary -// path, deriving the instance id from the namespace suffix. -func hermesExecArgs(namespace string, args []string, withTTY bool) []string { - id := strings.TrimPrefix(namespace, string(agentruntime.Hermes)+"-") - return agentruntime.BuildExecArgs( - agentruntime.Hermes, - id, - append([]string{hermesBinary}, args...), - withTTY, - ) -} - func getToken(cfg *config.Config, id string) (string, error) { namespace := agentruntime.Namespace(agentruntime.Hermes, id) kubeconfigPath := filepath.Join(cfg.ConfigDir, "kubeconfig.yaml") diff --git a/internal/hermes/hermes_test.go b/internal/hermes/hermes_test.go index 25a98b37..a9638c81 100644 --- a/internal/hermes/hermes_test.go +++ b/internal/hermes/hermes_test.go @@ -368,24 +368,6 @@ func extractChecksumAnnotation(t *testing.T, values, key string) string { return "" } -func TestHermesExecArgs_UsesNativeHermesBinary(t *testing.T) { - got := hermesExecArgs("hermes-obol-agent", []string{"skills", "audit"}, false) - want := []string{ - "exec", "-i", - "-c", "hermes", - "-n", "hermes-obol-agent", - "deploy/hermes", - "--", - "/opt/hermes/.venv/bin/hermes", - "skills", - "audit", - } - - if !reflect.DeepEqual(got, want) { - t.Fatalf("hermesExecArgs() = %#v, want %#v", got, want) - } -} - func TestResolveCLIInvocation_DefaultsToObolAgent(t *testing.T) { cfg := testConfig(t) mkdirInstance(t, cfg, agentruntime.DefaultInstanceID) diff --git a/internal/inference/detect.go b/internal/inference/detect.go index d2e061a2..c626b09b 100644 --- a/internal/inference/detect.go +++ b/internal/inference/detect.go @@ -142,11 +142,6 @@ func resolvedProbePorts() []portProbe { // probeTimeout is the per-endpoint HTTP timeout. const probeTimeout = 2 * time.Second -// ProbeEndpoint hits host:port/v1/models and returns discovered info. -func ProbeEndpoint(host string, port int) (*EndpointInfo, error) { - return ProbeEndpointContext(context.Background(), host, port) -} - // ProbeEndpointContext is the context-aware version of ProbeEndpoint. // It creates a shared HTTP client used for both server type detection // and model fetching to avoid redundant connections. @@ -172,11 +167,6 @@ func ProbeEndpointContext(ctx context.Context, host string, port int) (*Endpoint }, nil } -// ScanLocalEndpoints probes all common local ports and returns any that respond. -func ScanLocalEndpoints() ([]EndpointInfo, error) { - return ScanLocalEndpointsContext(context.Background()) -} - // ScanLocalEndpointsContext probes common ports concurrently with context support. // All ports are probed in parallel using goroutines; results are collected // and returned in stable port order. The probed port list is the union of @@ -233,13 +223,6 @@ func ScanLocalEndpointsContext(ctx context.Context) ([]EndpointInfo, error) { return found, nil } -// DetectServerType probes baseURL to determine the server software. -// Returns "ollama", "llama-server", "openai-compat", or "". -func DetectServerType(ctx context.Context, baseURL string) string { - client := &http.Client{Timeout: probeTimeout} - return detectServerTypeWithClient(ctx, client, baseURL) -} - // detectServerTypeWithClient probes baseURL using the provided client. // // Detection order priority: @@ -306,16 +289,6 @@ func fetchModels(ctx context.Context, client *http.Client, baseURL string) ([]Mo return mr.Data, nil } -// ParseModelsResponse parses raw JSON bytes into a slice of ModelInfo. -// Exported for testing. -func ParseModelsResponse(data []byte) ([]ModelInfo, error) { - var mr modelsResponse - if err := json.Unmarshal(data, &mr); err != nil { - return nil, fmt.Errorf("parsing models JSON: %w", err) - } - return mr.Data, nil -} - // FormatEndpointDisplay pretty-prints a list of discovered endpoints. func FormatEndpointDisplay(endpoints []EndpointInfo) string { var b strings.Builder diff --git a/internal/inference/detect_test.go b/internal/inference/detect_test.go index ae65f092..9667e332 100644 --- a/internal/inference/detect_test.go +++ b/internal/inference/detect_test.go @@ -20,7 +20,7 @@ func TestProbeEndpoint_NotRunning(t *testing.T) { port := ln.Addr().(*net.TCPAddr).Port ln.Close() - _, err = ProbeEndpoint("127.0.0.1", port) + _, err = ProbeEndpointContext(context.Background(), "127.0.0.1", port) if err == nil { t.Fatal("expected error probing non-running port, got nil") } @@ -32,7 +32,7 @@ func TestProbeEndpoint_NotRunning(t *testing.T) { func TestScanLocalEndpoints_NoneFound(t *testing.T) { // When nothing is running on any common port, should return empty list. // This test is safe because CI/test environments rarely run inference servers. - endpoints, err := ScanLocalEndpoints() + endpoints, err := ScanLocalEndpointsContext(context.Background()) if err != nil { t.Fatalf("ScanLocalEndpoints returned error: %v", err) } @@ -79,7 +79,7 @@ func TestDetectServerType(t *testing.T) { })) defer srv.Close() - got := DetectServerType(context.Background(), srv.URL) + got := detectServerTypeWithClient(context.Background(), &http.Client{Timeout: probeTimeout}, srv.URL) if got != tt.want { t.Errorf("DetectServerType() = %q, want %q", got, tt.want) } @@ -87,50 +87,6 @@ func TestDetectServerType(t *testing.T) { } } -func TestParseModelsResponse(t *testing.T) { - payload := modelsResponse{ - Data: []ModelInfo{ - {ID: "llama-3.2-3b", OwnedBy: "meta", Created: 1700000000}, - {ID: "qwen-2.5-coder", OwnedBy: "alibaba", Created: 1700000001}, - }, - } - raw, err := json.Marshal(payload) - if err != nil { - t.Fatalf("failed to marshal test payload: %v", err) - } - - models, err := ParseModelsResponse(raw) - if err != nil { - t.Fatalf("ParseModelsResponse returned error: %v", err) - } - if len(models) != 2 { - t.Fatalf("expected 2 models, got %d", len(models)) - } - if models[0].ID != "llama-3.2-3b" { - t.Errorf("models[0].ID = %q, want %q", models[0].ID, "llama-3.2-3b") - } - if models[1].OwnedBy != "alibaba" { - t.Errorf("models[1].OwnedBy = %q, want %q", models[1].OwnedBy, "alibaba") - } -} - -func TestParseModelsResponse_Invalid(t *testing.T) { - _, err := ParseModelsResponse([]byte(`{invalid json`)) - if err == nil { - t.Fatal("expected error for invalid JSON, got nil") - } -} - -func TestParseModelsResponse_Empty(t *testing.T) { - models, err := ParseModelsResponse([]byte(`{"data":[]}`)) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(models) != 0 { - t.Fatalf("expected 0 models, got %d", len(models)) - } -} - // TestProbeEndpointContext_HappyPath uses httptest to serve /v1/models // and verifies ProbeEndpointContext returns correct EndpointInfo. func TestProbeEndpointContext_HappyPath(t *testing.T) { diff --git a/internal/model/model.go b/internal/model/model.go index 9e109b8c..a7172a3b 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -308,7 +308,6 @@ func HasConfiguredModels(cfg *config.Config) bool { return false } - // LoadDotEnv reads KEY=value pairs from a .env file. // Returns an empty map if the file doesn't exist or is unreadable. // Skips comments (#) and blank lines. Does not call os.Setenv. @@ -942,8 +941,8 @@ func RemoveModel(cfg *config.Config, u *ui.UI, modelName string) error { return nil } -// AddCustomEndpoint adds a custom OpenAI-compatible endpoint to LiteLLM -// after validating it works. +// AddCustomEndpointWithOptions adds a custom OpenAI-compatible endpoint to +// LiteLLM after validating it works. // // LiteLLM `model_name` contract — the canonical identifier is the bare // `modelName`. Same convention every other code path in this stack uses: @@ -958,10 +957,6 @@ func RemoveModel(cfg *config.Config, u *ui.UI, modelName string) error { // each other in the LiteLLM ConfigMap; that is the natural "repoint my // model" behavior an operator running `obol model setup custom` wants when // they re-run the command. -func AddCustomEndpoint(cfg *config.Config, u *ui.UI, endpoint, modelName, apiKey string) error { - return AddCustomEndpointWithOptions(cfg, u, endpoint, modelName, apiKey, CustomEndpointOptions{}) -} - func AddCustomEndpointWithOptions(cfg *config.Config, u *ui.UI, endpoint, modelName, apiKey string, options CustomEndpointOptions) error { kubectlBinary := filepath.Join(cfg.BinDir, "kubectl") kubeconfigPath := filepath.Join(cfg.ConfigDir, "kubeconfig.yaml") diff --git a/internal/schemas/payment.go b/internal/schemas/payment.go index 740d1eee..73de8e21 100644 --- a/internal/schemas/payment.go +++ b/internal/schemas/payment.go @@ -109,19 +109,6 @@ type PriceTable struct { PerEpoch string `json:"perEpoch,omitempty" yaml:"perEpoch,omitempty"` } -// EffectiveRequestPrice returns the per-request price to use for x402 gating. -// If PerRequest is set, it is returned directly. If only PerMTok is set, it is -// temporarily approximated using ApproxTokensPerRequest until exact metering -// exists. Invalid decimal inputs fall back to "0". -func (p PriceTable) EffectiveRequestPrice() string { - price, err := p.EffectiveRequestPriceE() - if err != nil { - return "0" - } - - return price -} - // EffectiveRequestPriceE returns the per-request price to use for x402 gating. // It performs the same conversion as EffectiveRequestPrice but returns parsing // errors to callers that need to validate input. diff --git a/internal/schemas/payment_test.go b/internal/schemas/payment_test.go index 109966ab..608712d3 100644 --- a/internal/schemas/payment_test.go +++ b/internal/schemas/payment_test.go @@ -7,28 +7,6 @@ import ( "gopkg.in/yaml.v3" ) -func TestEffectiveRequestPrice_PerRequest(t *testing.T) { - p := PriceTable{PerRequest: "0.001"} - if got := p.EffectiveRequestPrice(); got != "0.001" { - t.Errorf("EffectiveRequestPrice() = %q, want %q", got, "0.001") - } -} - -func TestEffectiveRequestPrice_PerMTok(t *testing.T) { - p := PriceTable{PerMTok: "0.50"} - if got := p.EffectiveRequestPrice(); got != "0.0005" { - t.Errorf("EffectiveRequestPrice() = %q, want %q", got, "0.0005") - } -} - -func TestEffectiveRequestPrice_PerHour(t *testing.T) { - // 6.00 USDC/hour * (5 min / 60 min) = 0.50 USDC/request - p := PriceTable{PerHour: "6.00"} - if got := p.EffectiveRequestPrice(); got != "0.5" { - t.Errorf("EffectiveRequestPrice() = %q, want %q", got, "0.5") - } -} - func TestApproximateRequestPriceFromPerHour(t *testing.T) { // 0.50 USDC/hour * (5/60) = 0.04166... got, err := ApproximateRequestPriceFromPerHour("0.50") @@ -56,20 +34,6 @@ func TestApproximateRequestPriceFromPerHour_Invalid(t *testing.T) { } } -func TestEffectiveRequestPrice_Empty(t *testing.T) { - p := PriceTable{} - if got := p.EffectiveRequestPrice(); got != "0" { - t.Errorf("EffectiveRequestPrice() = %q, want %q", got, "0") - } -} - -func TestEffectiveRequestPrice_PerRequestPrecedence(t *testing.T) { - p := PriceTable{PerRequest: "0.001", PerMTok: "0.50"} - if got := p.EffectiveRequestPrice(); got != "0.001" { - t.Errorf("EffectiveRequestPrice() = %q, want %q (PerRequest should take precedence)", got, "0.001") - } -} - func TestApproximateRequestPriceFromPerMTok(t *testing.T) { got, err := ApproximateRequestPriceFromPerMTok("1.25") if err != nil { diff --git a/internal/serviceoffercontroller/catalog.go b/internal/serviceoffercontroller/catalog.go index 1a75c294..a33cfad6 100644 --- a/internal/serviceoffercontroller/catalog.go +++ b/internal/serviceoffercontroller/catalog.go @@ -97,14 +97,3 @@ func skillCatalogDeployedContentHash(deployment *unstructured.Unstructured) stri hash, _, _ := unstructured.NestedString(deployment.Object, "spec", "template", "metadata", "annotations", "obol.org/content-hash") return hash } - -func (c *Controller) skillCatalogDeployedContentHash(ctx context.Context) (string, error) { - deployment, err := c.deployments.Namespace(skillCatalogNamespace).Get(ctx, skillCatalogConfigMapName, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - return "", nil - } - if err != nil { - return "", err - } - return skillCatalogDeployedContentHash(deployment), nil -} diff --git a/internal/serviceoffercontroller/render.go b/internal/serviceoffercontroller/render.go index 5621e9dd..c7340b09 100644 --- a/internal/serviceoffercontroller/render.go +++ b/internal/serviceoffercontroller/render.go @@ -1057,14 +1057,6 @@ func buildActiveRegistrationDocument(owner *monetizeapi.ServiceOffer, offers []* return registration } -func buildTombstoneRegistrationDocument(offer *monetizeapi.ServiceOffer, baseURL, agentID string) erc8004.AgentRegistration { - registration := buildActiveRegistrationDocument(offer, []*monetizeapi.ServiceOffer{offer}, baseURL, agentID) - registration.Active = false - registration.X402Support = false - registration.Description = fmt.Sprintf("%s (deactivated)", registration.Description) - return registration -} - func buildRegistrationServices(owner *monetizeapi.ServiceOffer, offers []*monetizeapi.ServiceOffer, baseURL string) []erc8004.ServiceDef { baseURL = strings.TrimRight(baseURL, "/") type offerKey struct { diff --git a/internal/stack/stack.go b/internal/stack/stack.go index ad57ace9..9135aceb 100644 --- a/internal/stack/stack.go +++ b/internal/stack/stack.go @@ -175,18 +175,10 @@ func ollamaHostIPForBackend(backendName string) (string, error) { return stackdefaults.OllamaHostIPForBackend(backendName) } -func dockerDesktopGatewayIP() string { - return stackdefaults.DockerDesktopGatewayIP() -} - func dockerBridgeGatewayIP() (string, error) { return stackdefaults.DockerBridgeGatewayIP() } -func bridgeInterfaceIP(name string) (string, error) { - return stackdefaults.BridgeInterfaceIP(name) -} - // Up starts the cluster using the configured backend func Up(cfg *config.Config, u *ui.UI, wildcardDNS bool) error { stackID := getStackID(cfg) @@ -395,11 +387,6 @@ func getStackID(cfg *config.Config) string { return strings.TrimSpace(string(data)) } -// GetStackID reads the stored stack ID (exported for use in main) -func GetStackID(cfg *config.Config) string { - return getStackID(cfg) -} - // syncDefaults deploys the default infrastructure using helmfile. // // On helmfile failure we deliberately leave the cluster running. Historically diff --git a/internal/x402/chains.go b/internal/x402/chains.go index 2a6efd34..f12b1bc7 100644 --- a/internal/x402/chains.go +++ b/internal/x402/chains.go @@ -243,23 +243,6 @@ func (c ChainInfo) DefaultAsset() AssetInfo { } } -// ResolveAssetInfo applies any route-level asset overrides on top of the -// chain's default settlement asset. -func ResolveAssetInfo(chain ChainInfo, rule *RouteRule) AssetInfo { - if rule == nil { - return chain.DefaultAsset() - } - // The inline asset fields describe the primary payment option. - return ResolveAssetInfoForPayment(chain, RoutePayment{ - AssetAddress: rule.AssetAddress, - AssetSymbol: rule.AssetSymbol, - AssetDecimals: rule.AssetDecimals, - AssetTransferMethod: rule.AssetTransferMethod, - EIP712Name: rule.EIP712Name, - EIP712Version: rule.EIP712Version, - }) -} - // ResolveAssetInfoForPayment applies a single payment option's asset // overrides on top of the chain default. Same precedence and registry-flag // re-derivation as ResolveAssetInfo, but scoped to one accepted payment so a @@ -383,20 +366,6 @@ func ClampMaxTimeoutSeconds(n int64) int64 { return n } -// BuildV1Requirement creates a v1 PaymentRequirementsV1 for USDC payment on -// the given chain. amount is the decimal USDC amount (e.g., "0.001" = $0.001). -func BuildV1Requirement(chain ChainInfo, amount, recipientAddress string, maxTimeoutSeconds int64) x402types.PaymentRequirementsV1 { - asset := chain.DefaultAsset() - return x402types.PaymentRequirementsV1{ - Scheme: "exact", - Network: chain.NetworkID, - MaxAmountRequired: decimalToAtomic(amount, asset.Decimals), - Asset: asset.Address, - PayTo: recipientAddress, - MaxTimeoutSeconds: int(ClampMaxTimeoutSeconds(maxTimeoutSeconds)), - } -} - // BuildV2Requirement creates a v2 PaymentRequirements for USDC payment on the // given chain. amount is the decimal USDC amount (e.g. "0.001" = $0.001). func BuildV2Requirement(chain ChainInfo, amount, recipientAddress string, maxTimeoutSeconds int64) x402types.PaymentRequirements { diff --git a/internal/x402/chains_test.go b/internal/x402/chains_test.go index 6c35dff7..65835e2a 100644 --- a/internal/x402/chains_test.go +++ b/internal/x402/chains_test.go @@ -60,27 +60,6 @@ func TestChainUSDCAddresses(t *testing.T) { } } -func TestBuildV1Requirement(t *testing.T) { - req := BuildV1Requirement(ChainBaseSepolia, "0.001", "0xRecipient", 0) - - if req.Scheme != "exact" { - t.Errorf("Scheme = %q, want %q", req.Scheme, "exact") - } - if req.Network != "base-sepolia" { - t.Errorf("Network = %q, want %q", req.Network, "base-sepolia") - } - // 0.001 USDC = 1000 atomic units (6 decimals) - if req.MaxAmountRequired != "1000" { - t.Errorf("MaxAmountRequired = %q, want %q", req.MaxAmountRequired, "1000") - } - if req.Asset != ChainBaseSepolia.USDCAddress { - t.Errorf("Asset = %q, want %q", req.Asset, ChainBaseSepolia.USDCAddress) - } - if req.PayTo != "0xRecipient" { - t.Errorf("PayTo = %q, want %q", req.PayTo, "0xRecipient") - } -} - func TestResolveAssetInfo_Override(t *testing.T) { tests := []struct { name string @@ -101,7 +80,7 @@ func TestResolveAssetInfo_Override(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - asset := ResolveAssetInfo(tc.chain, &RouteRule{ + asset := ResolveAssetInfoForPayment(tc.chain, RoutePayment{ AssetAddress: tc.assetAddress, AssetSymbol: "OBOL", AssetDecimals: 18, @@ -133,7 +112,7 @@ func TestResolveAssetInfo_Override(t *testing.T) { // NOT inherit the gasless-approve flag from the registry, otherwise a buyer // would skip the on-chain approve and the payment would fail at settlement. func TestResolveAssetInfo_RejectAddressMismatch(t *testing.T) { - asset := ResolveAssetInfo(ChainEthereumMainnet, &RouteRule{ + asset := ResolveAssetInfoForPayment(ChainEthereumMainnet, RoutePayment{ AssetAddress: "0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", AssetSymbol: "OBOL", AssetDecimals: 18, @@ -175,7 +154,7 @@ func TestBuildExtensionsForAsset(t *testing.T) { } } - baseSepoliaOBOL := ResolveAssetInfo(ChainBaseSepolia, &RouteRule{ + baseSepoliaOBOL := ResolveAssetInfoForPayment(ChainBaseSepolia, RoutePayment{ AssetAddress: "0x0a09371a8b011d5110656ceBCc70603e53FD2c78", AssetSymbol: "OBOL", AssetDecimals: 18, diff --git a/internal/x402/paymentrequired.go b/internal/x402/paymentrequired.go index f45ba22a..2461a8c7 100644 --- a/internal/x402/paymentrequired.go +++ b/internal/x402/paymentrequired.go @@ -392,18 +392,6 @@ func buildTypeCopy(siteURL, endpoint string, d PaymentDisplay) typeCopy { } } -// x402GuideRef returns the self-contained "how to pay" pointer interpolated -// into the "other AI agent" copy prompts. Rather than send a foreign agent -// to the broad obol.org/llms.txt, we point it at THIS operator's own -// catalog (`/skill.md`, human + agent readable, with the full x402 v2 loop) -// and OpenAPI document (`/openapi.json`, exact request shapes) — both served -// over the same tunnel as the paid endpoint, so a single fetch is enough to -// learn how to pay. siteURL is the public origin (scheme://host); when empty -// the prompt degrades to a generic x402 mention. -func x402GuideRef(siteURL string) string { - return buyprompts.GuideRef(siteURL) -} - // normalizeOfferType collapses the spec.type values into the three render // branches. Empty falls back to "inference" historically (the original // default), but the storefront defaults new offers to "http" — match that diff --git a/internal/x402/tokens.go b/internal/x402/tokens.go index fc437ad8..111d75dd 100644 --- a/internal/x402/tokens.go +++ b/internal/x402/tokens.go @@ -103,12 +103,6 @@ func SupportedTokens() []string { return tokens } -// TokenSupportedOnChain reports whether a named token is registered for a chain. -func TokenSupportedOnChain(tokenName, chainName string) bool { - _, ok := ResolveToken(tokenName, chainName) - return ok -} - // TokensOnChain returns a sorted slice of token symbols registered for the // given chain. Returns an empty slice when no tokens are registered for that // chain (or the chain is unknown). diff --git a/internal/x402/tokens_test.go b/internal/x402/tokens_test.go index f5030678..eec8db6e 100644 --- a/internal/x402/tokens_test.go +++ b/internal/x402/tokens_test.go @@ -238,21 +238,3 @@ func TestChainsForToken(t *testing.T) { t.Errorf("ChainsForToken(WETH) = %v, want nil", got) } } - -func TestTokenSupportedOnChain(t *testing.T) { - if !TokenSupportedOnChain("USDC", "base") { - t.Error("USDC should be supported on base") - } - if !TokenSupportedOnChain("OBOL", "ethereum") { - t.Error("OBOL should be supported on ethereum") - } - if !TokenSupportedOnChain("OBOL", "base-sepolia") { - t.Error("OBOL should be supported on base-sepolia") - } - if TokenSupportedOnChain("OBOL", "base") { - t.Error("OBOL should not be supported on base mainnet") - } - if TokenSupportedOnChain("OBOL", "polygon") { - t.Error("OBOL should not be supported on polygon") - } -} From 49be12f1975e8a3828c319bf550e3d046e9facda Mon Sep 17 00:00:00 2001 From: bussyjd Date: Tue, 14 Jul 2026 12:11:52 +0400 Subject: [PATCH 7/9] docs: refresh CLAUDE.md pitfalls + obol-stack-dev skill (v3.2.0) - pitfall 6 generalized: EVERY LiteLLM openai/ api_base must include /v1 (LiteLLM posts /chat/completions verbatim); buyer-side tooling uses the opposite convention; discovery fixed in #745 - new pitfall 22: poisoned model group (two deployments, one bare api_base -> intermittent 404, no retry; GET /model/info vs CM is the decisive check) - new pitfall 23: buyer disconnect != no charge (hermes non-streaming path completes + settles after abort; keep runs short via spec.maxTurns; maxConcurrentRuns semantics; 4xx never settled) - skill: lessons table updated (poisoned model group, parserless QA endpoint -> relaunch with publisher-exact launcher), version 3.2.0 Claude-Session: https://claude.ai/code/session_014YjPMViNrZ7zBVgUQzwEKk --- .agents/skills/obol-stack-dev/SKILL.md | 8 +++++--- CLAUDE.md | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.agents/skills/obol-stack-dev/SKILL.md b/.agents/skills/obol-stack-dev/SKILL.md index 92cc01d4..e2e419e6 100644 --- a/.agents/skills/obol-stack-dev/SKILL.md +++ b/.agents/skills/obol-stack-dev/SKILL.md @@ -2,7 +2,7 @@ name: obol-stack-dev description: CLI-first Obol Stack development and QA runbook. Use when working on obol-stack lifecycle, obol CLI surfaces, x402 seller/buyer tests, live Base Sepolia OBOL smoke, Anvil fork regressions, ERC-8004 registration, LiteLLM paid routing, release-smoke, cloudflared, Renovate image bumps, or remote QA worktrees. metadata: - version: "3.1.0" + version: "3.2.0" domain: infrastructure role: specialist scope: development-and-testing @@ -67,7 +67,7 @@ OBOL_TOKEN_BASE_SEPOLIA=0x0a09371a8b011d5110656ceBCc70603e53FD2c78 **Release notes**: start from `.github/release-template.md`. Keep generated `What's Changed` / `New Contributors` / `Full Changelog` at the bottom. v0.9.0 is the style reference. No private keys, seed phrases, hostnames, personal paths, or raw bearer tokens. -## Hard-Won Lessons (from release-smoke 2026-05-13) +## Hard-Won Lessons (release-smoke 2026-05-13 + 2026-07-14) When the smoke gate goes red, check these first — each was a multi-hour debug: @@ -84,7 +84,9 @@ When the smoke gate goes red, check these first — each was a multi-hour debug: | facilitator arm64 image runs amd64 binary | Was an `ObolNetwork/x402-rs` prom-overlay arm64 manifest packaging bug. | **Fixed upstream**: `ObolNetwork/x402-rs#3` (merged 2026-05-13, `668b7bb`) dropped the redundant `--platform=$BUILDPLATFORM` pin from the prom-overlay builder stage. Registry image republished; arm64 manifest now ships an aarch64 ELF (digest `sha256:b209345c…`). The `X402_FACILITATOR_SKIP_PULL` knob has been removed from `flows/lib.sh`. | | flow-13 catalog assertion crashes with `'str' object has no attribute 'get'` | The smoke helper parsed `/api/services.json` as the old bare array and iterated object keys from the current catalog envelope. | `flows/lib-dual-stack.sh::assert_bob_service_catalog_contains` accepts both the current `{"services":[...]}` envelope and the legacy list form. | | Hermes install times out fetching `bedag/raw-2.0.2.tgz` | Hermes deploys generated Kubernetes resources through the third-party `bedag/raw` Helm chart. A chart repo/network miss blocks agent install even when the product logic is fine. | Classify as dependency/bootstrap drift. Longer-term debt reduction: replace the external raw chart with a small first-party/local raw-manifest chart or render/apply an embedded first-party Hermes chart. | -| flow-02 stalls on `monitoring-kube-state-metrics` `ContainerCreating` | The kubelet is stuck pulling `registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.18.0`. | Treat as registry/bootstrap drift. Preflight or pre-cache the exact image before full release-smoke; do not change flow assertions to hide it. | +| flow-02 stalls on `monitoring-kube-state-metrics` `ContainerCreating` | The kubelet is stuck pulling `registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.18.0`. | Treat as registry/bootstrap drift. Preflight or pre-cache the exact image before full release-smoke; do not change flow assertions to hide it. Cleared on plain rerun 2026-07-14 — a one-off is not a regression. | +| flow-11 step 43 / flow-03 step 2: intermittent 404 `{'detail':'Not Found'}` on paid or first inference | Poisoned LiteLLM model group: auto-discovery registered the host vLLM with a bare `api_base` (no `/v1`) alongside the correct `setup custom` entry; LiteLLM shuffles ~50/50 and never retries a 404. Latent since v0.10.0, NOT flow- or rc-specific. | Fixed in #745 (`internal/model/discover.go` appends `/v1`). Decisive evidence: vLLM access log alternating `POST /chat/completions 404` / `POST /v1/chat/completions 200`. Drift checker can't see it (names only) — #746. | +| flow-03 step 3 / flow-04: tool-call not returned, agent HTTP 000 | The QA vLLM endpoint was relaunched without its tool-call/reasoning parser flags (`--enable-auto-tool-choice --tool-call-parser ... --reasoning-parser ...`). flow-03 asserts tool-call passthrough; a parserless endpoint fails it every run. | Always relaunch the QA endpoint with its publisher-exact launcher script, never a hand-rolled unit. Preflight before a smoke run: one direct `curl` chat-completion with `tools[]` against the endpoint must return `tool_calls`. | **Diagnosis pattern**: a 503 from the verifier or 404 from a paid route almost never means the verifier is bad — it usually means the deployed image isn't what you think it is, the chain id form mismatched, or the upstream wasn't reachable. Confirm the running image first (`obol kubectl get deploy -n x402 x402-verifier -o jsonpath='{.spec.template.spec.containers[*].image}'`) before diving into x402 logic. diff --git a/CLAUDE.md b/CLAUDE.md index c3348de1..a1621c6a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -439,7 +439,7 @@ A registry digest pin instead of `:latest` on the verifier means your dev rewrit 3. **ConfigMap propagation** — ~60-120s for k3d file watcher; force restart for immediate effect 4. **ExternalName services** — do not work with Traefik Gateway API; use ClusterIP + Endpoints 5. **eRPC `eth_call` cache** — default TTL is 10s for unfinalized reads; `buy.py balance` can lag behind an already-settled paid request for a few seconds -6. **`/v1` required in `api_base` for `paid/*` route** — LiteLLM's OpenAI provider does NOT append `/v1` to a bare `api_base`. The buyer route must be `http://x402-buyer.llm.svc.cluster.local:8402/v1`, not `...:8402`. Without `/v1`, LiteLLM calls `/chat/completions` on the buyer; the buyer's mux returns `404 page not found` (Go default), which LiteLLM surfaces as `OpenAIException - 404 page not found`. +6. **`/v1` required in EVERY `openai/` `api_base`** — LiteLLM's OpenAI provider does NOT append `/v1` to a bare `api_base`; it POSTs `/chat/completions` verbatim. Applies to the `paid/*` buyer route (`...:8402/v1`), `obol model setup custom` endpoints (validation enforces it by probing `/chat/completions`), AND local-server auto-discovery (fixed in #745 — discovery previously registered bare bases). The invariant lives on `buildCustomEndpointEntryWithOptions` (internal/model/model.go); buyer-side tooling uses the OPPOSITE convention (base without `/v1`, appends the full path — `buyprompts.ChatCompletionsURL`). Full producer/consumer audit: #745. Type-level enforcement tracked in #746. 7. **LiteLLM restart is fallback, not the default buy path** — the validated happy path is `buy.py buy`/`process --all`/same-name top-up without a manual LiteLLM restart. Controller hot-add/hot-delete + buyer reload is expected to make `paid/` appear and disappear in place. If a paid alias still fails after controller reconciliation and buyer sidecar reports the upstream, restart LiteLLM as a fallback investigation step. Reloader watches ONLY `litellm-secrets` (key rotation needs a pod replacement; it rolls gaplessly via RollingUpdate maxUnavailable:0). It must NOT watch `litellm-config` — model_list changes are hot-applied via /model/new//model/delete and a CM-triggered rollout would gap inference (issue #321); `obol model status` surfaces CM-vs-router drift if a hot call silently failed. Buyer CM writes (top-ups/refills) hot-reload via `/admin/reload` with no restart — do not add the buyer CMs to any Reloader annotation. 8. **x402-verifier CA bundle missing → TLS failure** — The `x402-verifier` image is distroless (no CA store). The `ca-certificates` ConfigMap in `x402` namespace must be populated from the host CA bundle or the verifier cannot TLS-verify calls to the facilitator (`https://x402.gcp.obol.tech`), causing `x509: certificate signed by unknown authority` on every payment. **Fixed**: `obol stack up` calls `x402verifier.PopulateCABundle` after infrastructure deployment; `obol sell http` calls it before creating the ServiceOffer. If `Payment verification failed` errors still occur, check verifier logs for the x509 error and repopulate manually: `kubectl create configmap ca-certificates -n x402 --from-file=ca-certificates.crt=/etc/ssl/cert.pem --dry-run=client -o yaml | kubectl replace -f -` 9. **`EnsureVerifier` overwrites helmfile's image pin under `OBOL_DEVELOPMENT=true`** — `internal/x402/setup.go` reads embedded `x402.yaml` (hard-coded image pin) and `kubectl apply`s it. Without an in-memory rewrite this overwrites the helmfile-managed `:latest` deployment with the embedded pin → every source change to the verifier silently bypassed. Fix shipped in `5a10fb8` (rewrites pins in-memory before apply); structural regression test: `internal/x402/setup_structure_test.go` (`TestEnsureVerifier_NoInlineRegex`). **If you add a new component installed via `kubectl apply` of an embedded manifest**, give it the same dev-rewrite treatment. @@ -456,6 +456,8 @@ A registry digest pin instead of `:latest` on the verifier means your dev rewrit 19. **`obol sell info set` times out but profile CM updated** — the dev CLI wrote `x402/obol-storefront-profile` but the in-cluster `serviceoffer-controller` is still on an image that publishes `/api/services.json` as a bare `services[]` array (no `displayName` envelope). Symptom: `configmap/obol-storefront-profile configured` then `timed out waiting for controller to publish /api/services.json`; `kubectl get cm -n x402 obol-skill-md -o jsonpath='{.data.services\.json}'` starts with `[` not `{`. Fix: rebuild + import + restart `serviceoffer-controller` (see `sell info` bullet above). `obol stack up` with a warm cache (`built == 0`) does not pick up controller source changes unless `OBOL_FORCE_REBUILD_LOCAL_DEV_IMAGES=serviceoffer-controller`. 20. **402 page silently falls back to JSON when the template errors** — `sendPaymentRequiredHTML` swallows template-exec errors and re-sends the JSON body, so referencing a field in `payment_required.html` that isn't in the render's data struct doesn't crash anything: browsers just start getting JSON. Symptom: `Content-Type: application/json` on an `Accept: text/html` request. Any template-field addition needs the struct field added in `paymentrequired.go` AND a test asserting the HTML branch still renders (the existing branding tests check Content-Type/markup). 21. **`html/template` rejects `data:` URIs in URL contexts (`#ZgotmplZ`)** — inline logos/favicons from `sell info set --logo-file` are `data:image/...;base64` URIs; interpolating them into `src=`/`href=` via a plain string yields the literal `#ZgotmplZ` (broken image). Branding asset URLs must go through `storefront.SafeAssetURL` (validates http(s)/`data:image` then returns `template.URL`); regression test `TestPaymentRequiredHTML_InlineDataURILogo`. +22. **Poisoned LiteLLM model group — intermittent 404 `{'detail': 'Not Found'}` on ~50% of requests** — two `model_list` deployments share a `model_name` but disagree on `api_base` (one with `/v1`, one without); LiteLLM shuffles between them and does not retry a 404. Historic cause: auto-discovery registering the same host endpoint that `obol model setup custom` later added correctly (#745). Diagnosis: compare the router's live view (`GET /model/info`) against the `litellm-config` CM — the drift checker compares model NAMES only and cannot see divergent `api_base` (#746). The vLLM access log is decisive: alternating `POST /chat/completions 404` / `POST /v1/chat/completions 200`. +23. **Buyer disconnect ≠ no charge (zombie settlement)** — a client abort routinely does NOT propagate past cloudflared; the upstream agent finishes the turn, the handler returns 2xx, and settlement fires → buyer debited for a response nobody received (proven on-chain 2026-07-14, the Bankr incident). The verifier skips settlement when the request context is already canceled (#743), but that only covers propagated cancels — the real protection is keeping paid agent runs SHORT (`Agent.spec.maxTurns`; runs must finish inside the ~100s tunnel window and typical client timeouts). Concurrency: Hermes caps simultaneous runs in-process (`gateway.api_server.max_concurrent_runs`, internal default 10, 429 "Too many concurrent runs"; exposed as `Agent.spec.maxConcurrentRuns`, 0 = disabled) — keep it coherent with the edge `spec.limits.maxInFlight` Traefik gate; 4xx responses are never settled, so both gates are financially safe. For a fuller debug catalog with symptom->fix mapping, see `.agents/skills/obol-stack-dev/references/release-smoke-debugging.md`. From 70f5b7cc744f9481292090629d4e8e6271e4d08d Mon Sep 17 00:00:00 2001 From: bussyjd Date: Tue, 14 Jul 2026 13:29:53 +0400 Subject: [PATCH 8/9] fix(hermes): pass explicit model context_length/max_tokens to deployed agents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hermes agents cannot discover the real context window through LiteLLM (its /models response strips max_model_len) and fall back to assuming 65536 tokens — then request max_tokens=65536, which any backend with a smaller window rejects. Hermes misclassifies that as context exhaustion, compresses a near-empty conversation, and dies with "Context length exceeded (21 tokens). Cannot compress further." — the exact failure of flow-13/14 step 45 (agent readiness preflight), reproduced and fixed standalone against hermes-agent v2026.7.1 + vLLM. Hermes also hard-requires >=64K context, so the QA endpoint itself must serve >=65536 (spark1 vLLM relaunched with --max-model-len 65536). - generateConfig: honor OBOL_LLM_CONTEXT_LENGTH / OBOL_LLM_MAX_TOKENS env knobs -> model.context_length / model.max_tokens (hermes config resolution step 0, overrides its probing) - release-smoke: default OBOL_LLM_CONTEXT_LENGTH=65536, OBOL_LLM_MAX_TOKENS=8192 whenever the OBOL flows are enabled Claude-Session: https://claude.ai/code/session_014YjPMViNrZ7zBVgUQzwEKk --- flows/release-smoke.sh | 9 +++++++ internal/hermes/hermes.go | 25 +++++++++++++----- internal/hermes/hermes_test.go | 47 ++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 6 deletions(-) diff --git a/flows/release-smoke.sh b/flows/release-smoke.sh index 1ae8606d..177c404c 100755 --- a/flows/release-smoke.sh +++ b/flows/release-smoke.sh @@ -165,6 +165,15 @@ EOF exit 2 fi + # Hermes agents require a >=64K context model and cannot see the real + # window through LiteLLM (its /models strips max_model_len), so the + # deployment config must state it explicitly. Without these, hermes + # assumes 65536 and requests max_tokens=65536, which any endpoint with a + # smaller max_model_len rejects — flow-13/14 then fail their agent + # readiness preflight with "Context length exceeded ... Cannot compress". + export OBOL_LLM_CONTEXT_LENGTH="${OBOL_LLM_CONTEXT_LENGTH:-65536}" + export OBOL_LLM_MAX_TOKENS="${OBOL_LLM_MAX_TOKENS:-8192}" + if ! preflight_openai_llm_endpoint; then cat >&2 <=65536). + if n, err := strconv.Atoi(os.Getenv("OBOL_LLM_CONTEXT_LENGTH")); err == nil && n > 0 { + model["context_length"] = n + } + if n, err := strconv.Atoi(os.Getenv("OBOL_LLM_MAX_TOKENS")); err == nil && n > 0 { + model["max_tokens"] = n + } payload := map[string]any{ - "model": map[string]any{ - "default": primary, - "provider": "custom", - "base_url": "http://litellm.llm.svc.cluster.local:4000/v1", - "api_key": litellmMasterKey(cfg), - }, + "model": model, "terminal": map[string]any{ "backend": "local", "cwd": "/data/.hermes/workspace", diff --git a/internal/hermes/hermes_test.go b/internal/hermes/hermes_test.go index a9638c81..60d1e469 100644 --- a/internal/hermes/hermes_test.go +++ b/internal/hermes/hermes_test.go @@ -162,6 +162,53 @@ func TestGenerateConfig_PrimaryIsRoundTrippable(t *testing.T) { } } +func TestGenerateConfig_ContextEnvKnobs(t *testing.T) { + t.Setenv("OBOL_LLM_CONTEXT_LENGTH", "65536") + t.Setenv("OBOL_LLM_MAX_TOKENS", "8192") + + raw, err := generateConfig(testConfig(t), "qwen36-nvfp4") + if err != nil { + t.Fatalf("generateConfig() error = %v", err) + } + + var cfg map[string]any + if err := yaml.Unmarshal(raw, &cfg); err != nil { + t.Fatalf("yaml.Unmarshal() error = %v", err) + } + modelCfg, ok := cfg["model"].(map[string]any) + if !ok { + t.Fatalf("model config missing or wrong type: %#v", cfg["model"]) + } + if got := modelCfg["context_length"]; got != 65536 { + t.Fatalf("model.context_length = %#v, want 65536", got) + } + if got := modelCfg["max_tokens"]; got != 8192 { + t.Fatalf("model.max_tokens = %#v, want 8192", got) + } +} + +func TestGenerateConfig_ContextEnvKnobsAbsent(t *testing.T) { + t.Setenv("OBOL_LLM_CONTEXT_LENGTH", "") + t.Setenv("OBOL_LLM_MAX_TOKENS", "not-a-number") + + raw, err := generateConfig(testConfig(t), "qwen36-nvfp4") + if err != nil { + t.Fatalf("generateConfig() error = %v", err) + } + + var cfg map[string]any + if err := yaml.Unmarshal(raw, &cfg); err != nil { + t.Fatalf("yaml.Unmarshal() error = %v", err) + } + modelCfg := cfg["model"].(map[string]any) + if _, present := modelCfg["context_length"]; present { + t.Fatalf("model.context_length should be absent when env unset, got %#v", modelCfg["context_length"]) + } + if _, present := modelCfg["max_tokens"]; present { + t.Fatalf("model.max_tokens should be absent for invalid env, got %#v", modelCfg["max_tokens"]) + } +} + func TestGenerateConfig_UsesLiteLLMCustomProvider(t *testing.T) { raw, err := generateConfig(testConfig(t), "gpt-5.2") if err != nil { From 8f9b58e58df56de5c886ac680415866858c40447 Mon Sep 17 00:00:00 2001 From: bussyjd Date: Tue, 14 Jul 2026 13:34:41 +0400 Subject: [PATCH 9/9] fix(flows): give flow-03 tool-call step reasoning headroom (max_tokens 100->2048) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reasoning models (qwen3 family behind the vLLM reasoning parser) emit reasoning_content before the tool call; at max_tokens=100 the budget is sometimes exhausted mid-reasoning and the response comes back finish_reason=length with no tool_calls — an intermittent FAIL on an otherwise healthy endpoint (passed run5, failed run6, temperature=0 notwithstanding). 2048 verified live: finish_reason=stop, tool call returned. Claude-Session: https://claude.ai/code/session_014YjPMViNrZ7zBVgUQzwEKk --- flows/flow-03-inference.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flows/flow-03-inference.sh b/flows/flow-03-inference.sh index 7061a632..ec80dd6c 100755 --- a/flows/flow-03-inference.sh +++ b/flows/flow-03-inference.sh @@ -107,7 +107,7 @@ tool_out=$(curl -sf --max-time 120 -X POST http://localhost:8001/v1/chat/complet "messages":[{"role":"user","content":"Call the get_weather tool for London. Do not answer in text."}], "tools":[{"type":"function","function":{"name":"get_weather","description":"Get current weather","parameters":{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}}}], "tool_choice":{"type":"function","function":{"name":"get_weather"}}, - "temperature":0,"max_tokens":100,"stream":false + "temperature":0,"max_tokens":2048,"stream":false }' 2>&1) || true if echo "$tool_out" | tool_call_name >/dev/null 2>&1; then @@ -121,7 +121,7 @@ else "model":"'"$LITELLM_MODEL"'", "messages":[{"role":"user","content":"Call the get_weather tool with location London. Do not answer in text."}], "tools":[{"type":"function","function":{"name":"get_weather","description":"Get current weather","parameters":{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}}}], - "temperature":0,"max_tokens":100,"stream":false + "temperature":0,"max_tokens":2048,"stream":false }' 2>&1) || true if echo "$tool_out" | tool_call_name >/dev/null 2>&1; then