From a7f3ba9a2f000571e21b042939f7f643ba8e85d5 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 23 Sep 2026 11:44:55 -0500 Subject: [PATCH 1/2] fix(core): make web search limits non-fatal and report real errors Oversized provider responses failed the whole search: WebSearchMcp.call rejected bodies over 256 KB, and a single 424 KB Firecrawl result was enough. The tool then replaced the cause with a generic message. All providers now read at most 1 MiB, parse a cut-off body as partial JSON, and keep the results that arrived complete. WebSearch.query caps each result's content at 4,000 characters so one page cannot crowd out the rest of the bounded tool output. Web search errors now carry readable messages, including the provider's own explanation for non-2xx responses, MCP isError results, and JSON-RPC errors, which previously surfaced as no results. --- packages/core/src/plugin/websearch/exa.ts | 12 +- .../core/src/plugin/websearch/firecrawl.ts | 47 ++--- packages/core/src/plugin/websearch/mcp.ts | 41 ++-- .../core/src/plugin/websearch/parallel.ts | 71 +++---- .../core/src/plugin/websearch/response.ts | 82 ++++++++ packages/core/src/plugin/websearch/tavily.ts | 40 ++-- .../core/src/plugin/websearch/tinyfish.ts | 44 ++-- packages/core/src/tool/plugin/websearch.ts | 31 ++- packages/core/src/websearch.ts | 51 ++++- packages/core/test/plugin/websearch.test.ts | 197 ++++++++++++++++++ packages/core/test/tool-websearch.test.ts | 68 +++++- packages/core/test/websearch.test.ts | 28 +++ 12 files changed, 543 insertions(+), 169 deletions(-) create mode 100644 packages/core/src/plugin/websearch/response.ts diff --git a/packages/core/src/plugin/websearch/exa.ts b/packages/core/src/plugin/websearch/exa.ts index 1dc3a26108b1..c948fc38ee53 100644 --- a/packages/core/src/plugin/websearch/exa.ts +++ b/packages/core/src/plugin/websearch/exa.ts @@ -47,23 +47,25 @@ export const Plugin = define({ const credential = connection ? yield* ctx.integration.connection.resolve(connection) : undefined const url = new URL(endpoint) if (credential?.type === "key") url.searchParams.set("exaApiKey", credential.key) - const result = yield* WebSearchMcp.call( + const response = yield* WebSearchMcp.call( http, url.toString(), "web_search_exa", { input: McpInput, output: McpOutput }, { query: input.query, numResults: 8 }, ) - const content = result?.content.find((item) => item.text) - return content ? parseResults(content.text) : [] + const content = response.result?.content.find((item) => item.text) + return content ? parseResults(content.text, response.truncated) : [] }), }) }) }), }) -function parseResults(text: string) { - return text.split(/\n\n---\n\n/).flatMap((block) => { +function parseResults(text: string, truncated: boolean) { + const blocks = text.split(/\n\n---\n\n/) + // The final block of a truncated response may end mid-line, including inside its URL. + return (truncated ? blocks.slice(0, -1) : blocks).flatMap((block) => { const url = block.match(/^URL:\s*(.+)$/m)?.[1]?.trim() if (!url) return [] const title = block.match(/^Title:\s*(.+)$/m)?.[1]?.trim() diff --git a/packages/core/src/plugin/websearch/firecrawl.ts b/packages/core/src/plugin/websearch/firecrawl.ts index 1ea260e0c5da..3fcdd2786274 100644 --- a/packages/core/src/plugin/websearch/firecrawl.ts +++ b/packages/core/src/plugin/websearch/firecrawl.ts @@ -5,6 +5,7 @@ import { Effect, Option, Schema, Scope } from "effect" import { HttpClient } from "effect/unstable/http" import { App } from "../../app.js" import { WebSearchMcp } from "./mcp.js" +import { WebSearchResponse } from "./response.js" export const endpoint = "https://mcp.firecrawl.dev/v2/mcp" @@ -17,21 +18,15 @@ const McpOutput = Schema.Struct({ content: Schema.Array(Schema.Struct({ type: Schema.Literal("text"), text: Schema.String })), }) -const SearchResponse = Schema.fromJsonString( - Schema.Struct({ - success: Schema.Boolean, - data: Schema.Struct({ - web: Schema.Array( - Schema.Struct({ - url: Schema.String, - title: Schema.NullOr(Schema.String).pipe(Schema.optional), - description: Schema.NullOr(Schema.String).pipe(Schema.optional), - }), - ), - }), - }), +const decodeSearchResponse = Schema.decodeUnknownOption( + Schema.Struct({ data: Schema.Struct({ web: Schema.Array(Schema.Unknown) }) }), ) -const decodeSearchResponse = Schema.decodeUnknownOption(SearchResponse) + +const SearchResult = Schema.Struct({ + url: Schema.String, + title: Schema.NullOr(Schema.String).pipe(Schema.optional), + description: Schema.NullOr(Schema.String).pipe(Schema.optional), +}) export const Plugin = define({ id: "opencode.websearch.firecrawl", @@ -56,7 +51,7 @@ export const Plugin = define({ Effect.gen(function* () { const connection = yield* ctx.integration.connection.active("firecrawl") const credential = connection ? yield* ctx.integration.connection.resolve(connection) : undefined - const result = yield* WebSearchMcp.call( + const response = yield* WebSearchMcp.call( http, endpoint, "firecrawl_search", @@ -67,16 +62,18 @@ export const Plugin = define({ ...(credential?.type === "key" ? { Authorization: `Bearer ${credential.key}` } : {}), }, ) - const content = result?.content.find((item) => item.text) - const response = content ? Option.getOrUndefined(decodeSearchResponse(content.text)) : undefined - return ( - response?.data.web.map((item) => ({ - url: item.url, - ...(item.title ? { title: item.title } : {}), - ...(item.description ? { content: item.description } : {}), - time: {}, - })) ?? [] - ) + const content = response.result?.content.find((item) => item.text) + const search = content + ? Option.getOrUndefined( + WebSearchResponse.json(content.text, response.truncated).pipe(Option.flatMap(decodeSearchResponse)), + ) + : undefined + return WebSearchResponse.items(SearchResult, search?.data.web ?? [], response.truncated).map((item) => ({ + url: item.url, + ...(item.title ? { title: item.title } : {}), + ...(item.description ? { content: item.description } : {}), + time: {}, + })) }), }) }) diff --git a/packages/core/src/plugin/websearch/mcp.ts b/packages/core/src/plugin/websearch/mcp.ts index 478d4feb5f8d..7fbf9a0677e5 100644 --- a/packages/core/src/plugin/websearch/mcp.ts +++ b/packages/core/src/plugin/websearch/mcp.ts @@ -1,26 +1,22 @@ export * as WebSearchMcp from "./mcp.js" -import { Duration, Effect, Schema } from "effect" +import { Duration, Effect, Option, Schema } from "effect" import { HttpClient, HttpClientRequest } from "effect/unstable/http" -import { collectBoundedResponseBody } from "../../tool/http-body.js" +import { WebSearchResponse } from "./response.js" -export const MAX_RESPONSE_BYTES = 256 * 1024 - -export const parseResponse = (body: string, result: Schema.Struct) => { - const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Struct({ result }))) - const parse = (payload: string) => { - const trimmed = payload.trim() - if (!trimmed.startsWith("{")) return Effect.undefined - return decode(trimmed).pipe(Effect.map((response) => response.result)) - } +export const parseResponse = ( + body: { readonly text: string; readonly truncated: boolean }, + tool: string, + result: Schema.Struct, +) => { + const decode = Schema.decodeUnknownEffect(Schema.Struct({ result })) return Effect.gen(function* () { - const trimmed = body.trim() - const direct = trimmed ? yield* parse(trimmed) : undefined - if (direct) return direct - for (const line of body.split("\n")) { - if (!line.startsWith("data: ")) continue - const data = yield* parse(line.substring(6)) - if (data) return data + for (const payload of WebSearchResponse.payloads(body.text)) { + const message = WebSearchResponse.json(payload, body.truncated) + if (Option.isNone(message)) return yield* Effect.fail(new Error(`${tool} returned invalid JSON`)) + const failure = WebSearchResponse.failure(message.value) + if (failure !== undefined) return yield* Effect.fail(new Error(failure)) + return (yield* decode(message.value)).result } }) } @@ -52,13 +48,8 @@ export const call = new Error(`${tool} response exceeded ${MAX_RESPONSE_BYTES} bytes`), - ) - return yield* parseResponse(body.toString("utf8"), schema.output) + const body = yield* WebSearchResponse.execute(http, request) + return { result: yield* parseResponse(body, tool, schema.output), truncated: body.truncated } }).pipe( Effect.scoped, Effect.timeoutOrElse({ diff --git a/packages/core/src/plugin/websearch/parallel.ts b/packages/core/src/plugin/websearch/parallel.ts index 5184155482bd..3371f3d210f2 100644 --- a/packages/core/src/plugin/websearch/parallel.ts +++ b/packages/core/src/plugin/websearch/parallel.ts @@ -1,10 +1,11 @@ export * as WebSearchParallel from "./parallel.js" import { define } from "@opencode/plugin/effect/plugin" -import { Effect, Schema, Scope } from "effect" +import { Effect, Option, Schema, Scope } from "effect" import { HttpClient } from "effect/unstable/http" import { App } from "../../app.js" import { WebSearchMcp } from "./mcp.js" +import { WebSearchResponse } from "./response.js" export const endpoint = "https://search.parallel.ai/mcp" @@ -14,38 +15,19 @@ const McpInput = Schema.Struct({ model_name: Schema.String.check(Schema.isMaxLength(100)).pipe(Schema.optional), }) -const SearchResponse = Schema.Struct({ - search_id: Schema.String, - results: Schema.Array( - Schema.Struct({ - url: Schema.String, - title: Schema.NullOr(Schema.String).pipe(Schema.optional), - publish_date: Schema.NullOr(Schema.String).pipe(Schema.optional), - excerpts: Schema.Array(Schema.String), - }), - ), - warnings: Schema.NullOr( - Schema.Array( - Schema.Struct({ - type: Schema.Literals(["spec_validation_warning", "input_validation_warning", "warning"]), - message: Schema.String, - detail: Schema.NullOr(Schema.Record(Schema.String, Schema.Json)).pipe(Schema.optional), - }), - ), - ).pipe(Schema.optional), - usage: Schema.NullOr( - Schema.Array( - Schema.Struct({ - name: Schema.String, - count: Schema.Int, - }), - ), - ).pipe(Schema.optional), - session_id: Schema.String, -}) const McpOutput = Schema.Struct({ content: Schema.Array(Schema.Struct({ type: Schema.Literal("text"), text: Schema.String })), - structuredContent: SearchResponse, + // Parallel sends this after a text copy of the same JSON, so a truncated response can lose it. + structuredContent: Schema.Unknown.pipe(Schema.optional), +}) + +const decodeSearchResponse = Schema.decodeUnknownOption(Schema.Struct({ results: Schema.Array(Schema.Unknown) })) + +const SearchResult = Schema.Struct({ + url: Schema.String, + title: Schema.NullOr(Schema.String).pipe(Schema.optional), + publish_date: Schema.NullOr(Schema.String).pipe(Schema.optional), + excerpts: Schema.Array(Schema.String), }) export const Plugin = define({ @@ -71,7 +53,7 @@ export const Plugin = define({ Effect.gen(function* () { const connection = yield* ctx.integration.connection.active("parallel") const credential = connection ? yield* ctx.integration.connection.resolve(connection) : undefined - const result = yield* WebSearchMcp.call( + const response = yield* WebSearchMcp.call( http, endpoint, "web_search", @@ -85,17 +67,22 @@ export const Plugin = define({ ...(credential?.type === "key" ? { Authorization: `Bearer ${credential.key}` } : {}), }, ) - return ( - result?.structuredContent.results.map((item) => { - const published = item.publish_date ? Date.parse(item.publish_date) : undefined - return { - url: item.url, - ...(item.title ? { title: item.title } : {}), - ...(item.excerpts.length ? { content: item.excerpts.join("\n\n") } : {}), - time: { ...(published !== undefined && Number.isFinite(published) ? { published } : {}) }, - } - }) ?? [] + const content = response.result?.content.find((item) => item.text) + const search = Option.getOrUndefined( + decodeSearchResponse( + response.result?.structuredContent ?? + (content ? Option.getOrUndefined(WebSearchResponse.json(content.text, response.truncated)) : undefined), + ), ) + return WebSearchResponse.items(SearchResult, search?.results ?? [], response.truncated).map((item) => { + const published = item.publish_date ? Date.parse(item.publish_date) : undefined + return { + url: item.url, + ...(item.title ? { title: item.title } : {}), + ...(item.excerpts.length ? { content: item.excerpts.join("\n\n") } : {}), + time: { ...(published !== undefined && Number.isFinite(published) ? { published } : {}) }, + } + }) }), }) }) diff --git a/packages/core/src/plugin/websearch/response.ts b/packages/core/src/plugin/websearch/response.ts new file mode 100644 index 000000000000..29553f7ce1d9 --- /dev/null +++ b/packages/core/src/plugin/websearch/response.ts @@ -0,0 +1,82 @@ +export * as WebSearchResponse from "./response.js" + +import { parseJSON } from "@opencode/ai/protocols/utils/partial-json" +import { Effect, Option, Schema, Stream } from "effect" +import { HttpClient, HttpClientError } from "effect/unstable/http" +import type { HttpClientRequest, HttpClientResponse } from "effect/unstable/http" + +export const MAX_BYTES = 1024 * 1024 + +// Keeps the status for rate-limit failover and adds the provider's own explanation of the failure. +export const execute = (http: HttpClient.HttpClient, request: HttpClientRequest.HttpClientRequest) => + Effect.gen(function* () { + const response = yield* HttpClient.withScope(http).execute(request) + if (response.status >= 200 && response.status < 300) return yield* read(response) + const description = yield* read(response).pipe( + Effect.map((body) => + payloads(body.text) + .flatMap((payload) => Option.toArray(json(payload, body.truncated))) + .map(failure) + .find((message) => message !== undefined), + ), + Effect.orElseSucceed(() => undefined), + ) + return yield* new HttpClientError.HttpClientError({ + reason: new HttpClientError.StatusCodeError({ request, response, description }), + }) + }) + +// Stops reading at MAX_BYTES instead of failing; parsers keep the results that arrived complete. +export const read = (response: HttpClientResponse.HttpClientResponse) => + Effect.gen(function* () { + let size = 0 + const chunks = yield* response.stream.pipe( + Stream.takeUntil((chunk) => (size += chunk.byteLength) > MAX_BYTES), + Stream.runCollect, + ) + return { text: Buffer.concat(chunks).subarray(0, MAX_BYTES).toString("utf8"), truncated: size > MAX_BYTES } + }) + +// A JSON body, or the data lines of a server-sent event stream. +export const payloads = (text: string) => + [text, ...text.split("\n").flatMap((line) => (line.startsWith("data: ") ? [line.slice(6)] : []))] + .map((payload) => payload.trim()) + .filter((payload) => payload.startsWith("{")) + +const decodeJson = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Unknown)) +const decodePartialJson = Option.liftThrowable(parseJSON) + +export const json = (text: string, truncated: boolean) => (truncated ? decodePartialJson(text) : decodeJson(text)) + +const decodeFailure = Schema.decodeUnknownOption( + Schema.Union([ + Schema.Struct({ error: Schema.Struct({ message: Schema.String }) }), + Schema.Struct({ + result: Schema.Struct({ + isError: Schema.Literal(true), + content: Schema.Array(Schema.Struct({ text: Schema.String })), + }), + }), + Schema.Struct({ detail: Schema.Struct({ error: Schema.String }) }), + ]), +) + +// JSON-RPC errors, MCP tool errors, and Tavily's error detail. +export const failure = (value: unknown) => + Option.getOrUndefined( + Option.map(decodeFailure(value), (decoded) => { + if ("error" in decoded) return decoded.error.message + if ("result" in decoded) return decoded.result.content.map((item) => item.text).join("\n") + return decoded.detail.error + }), + ) + +// The last item of a truncated response may be cut mid-value, so it is dropped rather than trusted. +export const items = >( + schema: S, + values: readonly unknown[], + truncated: boolean, +) => { + const decode = Schema.decodeUnknownOption(schema) + return (truncated ? values.slice(0, -1) : values).flatMap((value) => Option.toArray(decode(value))) +} diff --git a/packages/core/src/plugin/websearch/tavily.ts b/packages/core/src/plugin/websearch/tavily.ts index 4b568a1fb0d2..ab27355dff02 100644 --- a/packages/core/src/plugin/websearch/tavily.ts +++ b/packages/core/src/plugin/websearch/tavily.ts @@ -1,9 +1,10 @@ export * as WebSearchTavily from "./tavily.js" import { define } from "@opencode/plugin/effect/plugin" -import { Duration, Effect, Schema, Scope } from "effect" -import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http" +import { Duration, Effect, Option, Schema, Scope } from "effect" +import { HttpClient, HttpClientRequest } from "effect/unstable/http" import { App } from "../../app.js" +import { WebSearchResponse } from "./response.js" export const endpoint = "https://api.tavily.com/search" @@ -14,14 +15,12 @@ const SearchRequest = Schema.Struct({ max_results: Schema.Number, }) -const SearchResponse = Schema.Struct({ - results: Schema.Array( - Schema.Struct({ - title: Schema.String, - url: Schema.String, - content: Schema.String, - }), - ), +const decodeSearchResponse = Schema.decodeUnknownOption(Schema.Struct({ results: Schema.Array(Schema.Unknown) })) + +const SearchResult = Schema.Struct({ + title: Schema.String, + url: Schema.String, + content: Schema.String, }) export const Plugin = define({ @@ -63,17 +62,16 @@ export const Plugin = define({ max_results: 8, }), ) - const response = yield* HttpClient.withScope(HttpClient.filterStatusOk(http)) - .execute(request) - .pipe( - Effect.flatMap(HttpClientResponse.schemaBodyJson(SearchResponse)), - Effect.scoped, - Effect.timeoutOrElse({ - duration: Duration.seconds(25), - orElse: () => Effect.fail(new Error("Tavily web search request timed out")), - }), - ) - return response.results.map((item) => ({ + const body = yield* WebSearchResponse.execute(http, request).pipe( + Effect.scoped, + Effect.timeoutOrElse({ + duration: Duration.seconds(25), + orElse: () => Effect.fail(new Error("Tavily web search request timed out")), + }), + ) + const search = WebSearchResponse.json(body.text, body.truncated).pipe(Option.flatMap(decodeSearchResponse)) + if (Option.isNone(search)) return yield* Effect.fail(new Error("Tavily returned an invalid response")) + return WebSearchResponse.items(SearchResult, search.value.results, body.truncated).map((item) => ({ url: item.url, title: item.title, ...(item.content ? { content: item.content } : {}), diff --git a/packages/core/src/plugin/websearch/tinyfish.ts b/packages/core/src/plugin/websearch/tinyfish.ts index 77d86acddc1d..d88ed4317d48 100644 --- a/packages/core/src/plugin/websearch/tinyfish.ts +++ b/packages/core/src/plugin/websearch/tinyfish.ts @@ -5,6 +5,7 @@ import { Effect, Option, Schema, Scope } from "effect" import { HttpClient } from "effect/unstable/http" import { App } from "../../app.js" import { WebSearchMcp } from "./mcp.js" +import { WebSearchResponse } from "./response.js" export const endpoint = "https://agent.tinyfish.ai/mcp" @@ -16,18 +17,13 @@ const McpOutput = Schema.Struct({ content: Schema.Array(Schema.Struct({ type: Schema.Literal("text"), text: Schema.String })), }) -const SearchResponse = Schema.fromJsonString( - Schema.Struct({ - results: Schema.Array( - Schema.Struct({ - url: Schema.String, - title: Schema.String, - snippet: Schema.String, - }), - ), - }), -) -const decodeSearchResponse = Schema.decodeUnknownOption(SearchResponse) +const decodeSearchResponse = Schema.decodeUnknownOption(Schema.Struct({ results: Schema.Array(Schema.Unknown) })) + +const SearchResult = Schema.Struct({ + url: Schema.String, + title: Schema.String, + snippet: Schema.String, +}) export const Plugin = define({ id: "opencode.websearch.tinyfish", @@ -52,7 +48,7 @@ export const Plugin = define({ Effect.gen(function* () { const connection = yield* ctx.integration.connection.active("tinyfish") const credential = connection ? yield* ctx.integration.connection.resolve(connection) : undefined - const result = yield* WebSearchMcp.call( + const response = yield* WebSearchMcp.call( http, endpoint, "search", @@ -65,16 +61,18 @@ export const Plugin = define({ : { "X-TinyFish-Access-Mode": "keyless" }), }, ) - const content = result?.content.find((item) => item.text) - const response = content ? Option.getOrUndefined(decodeSearchResponse(content.text)) : undefined - return ( - response?.results.map((item) => ({ - url: item.url, - title: item.title, - ...(item.snippet ? { content: item.snippet } : {}), - time: {}, - })) ?? [] - ) + const content = response.result?.content.find((item) => item.text) + const search = content + ? Option.getOrUndefined( + WebSearchResponse.json(content.text, response.truncated).pipe(Option.flatMap(decodeSearchResponse)), + ) + : undefined + return WebSearchResponse.items(SearchResult, search?.results ?? [], response.truncated).map((item) => ({ + url: item.url, + title: item.title, + ...(item.snippet ? { content: item.snippet } : {}), + time: {}, + })) }), }) }) diff --git a/packages/core/src/tool/plugin/websearch.ts b/packages/core/src/tool/plugin/websearch.ts index 0338dfdb601e..84606ffec199 100644 --- a/packages/core/src/tool/plugin/websearch.ts +++ b/packages/core/src/tool/plugin/websearch.ts @@ -4,7 +4,6 @@ import type { Context } from "@opencode/plugin/effect/plugin" import type { SessionHooks } from "@opencode/plugin/effect/session" import { ToolFailure } from "@opencode/ai" import { Effect, Schema, Semaphore } from "effect" -import { HttpClientError } from "effect/unstable/http" import { Form } from "../../form.js" import { Permission } from "../../permission.js" import { WebSearch } from "../../websearch.js" @@ -12,10 +11,6 @@ import { WebSearch } from "../../websearch.js" export const name = "websearch" export const NO_RESULTS = "No search results found. Please try a different query." const providerSelectionLock = Semaphore.makeUnsafe(1) -const httpErrors = new Map([ - [429, "Web search rate limited (HTTP 429)"], - [401, "Web search authentication failed (HTTP 401)"], -]) export const description = `Search the web using the user's selected search integration. Use this for current information beyond knowledge cutoff. @@ -164,17 +159,21 @@ export const Plugin = { return { output, content, metadata: { provider: output.provider } } }).pipe( Effect.mapError((error) => { - const fallback = `Unable to search the web for ${input.query}` - if (!Schema.is(WebSearch.RequestError)(error)) return new ToolFailure({ message: fallback, error }) - const status = HttpClientError.isHttpClientError(error.cause) ? error.cause.response?.status : undefined - return new ToolFailure({ - message: - status === undefined - ? fallback - : (httpErrors.get(status) ?? `Web search request failed (HTTP ${status})`), - error, - metadata: { provider: error.providerID }, - }) + const message = `Unable to search the web for ${input.query}` + // A cause becomes the model-visible error, so web search failures are described here instead; + // other causes, such as permission errors, keep their own session error. + if (Schema.is(WebSearch.RequestError)(error)) + return new ToolFailure({ + message: `${message} (${error.providerID}): ${error.message}`, + metadata: { provider: error.providerID }, + }) + if ( + error instanceof WebSearch.ProviderRequiredError || + error instanceof WebSearch.ProviderNotFoundError || + error instanceof WebSearch.DisabledError + ) + return new ToolFailure({ message: `${message}: ${error.message}` }) + return new ToolFailure({ message, error }) }), ), }), diff --git a/packages/core/src/websearch.ts b/packages/core/src/websearch.ts index 7ec63bb7b970..89fcb3ebc77e 100644 --- a/packages/core/src/websearch.ts +++ b/packages/core/src/websearch.ts @@ -29,6 +29,8 @@ export const Response = WebSearch.Response export type Response = WebSearch.Response export const ProviderKey = "websearch:provider" +// Keeps one oversized page from crowding the other results out of the bounded tool output. +export const MAX_RESULT_CONTENT_LENGTH = 4_000 export const Selection = Schema.Union([ID, Schema.Literal("random"), Schema.Literal(false)]) export type Selection = typeof Selection.Type @@ -39,18 +41,43 @@ export interface ProviderImplementation extends Provider { export class ProviderRequiredError extends Schema.TaggedError()( "WebSearch.ProviderRequired", {}, -) {} +) { + override get message() { + return "No web search provider is selected" + } +} export class ProviderNotFoundError extends Schema.TaggedError()("WebSearch.ProviderNotFound", { providerID: ID, -}) {} +}) { + override get message() { + return `Web search provider not found: ${this.providerID}` + } +} -export class DisabledError extends Schema.TaggedError()("WebSearch.Disabled", {}) {} +export class DisabledError extends Schema.TaggedError()("WebSearch.Disabled", {}) { + override get message() { + return "Web search is disabled" + } +} export class RequestError extends Schema.TaggedError()("WebSearch.Request", { providerID: ID, cause: Schema.Defect(), -}) {} +}) { + // HTTP client messages include the request URL, which can carry a provider credential. + override get message() { + const cause = this.cause + if (!HttpClientError.isHttpClientError(cause)) + return cause instanceof Error && cause.message ? cause.message : "Request failed" + const status = cause.response?.status + if (status !== undefined && cause.reason.description) return `HTTP ${status}: ${cause.reason.description}` + if (status === 429) return "Rate limited (HTTP 429)" + if (status === 401) return "Authentication failed (HTTP 401)" + if (status !== undefined) return `Request failed (HTTP ${status})` + return cause.cause instanceof Error && cause.cause.message ? `Request failed: ${cause.cause.message}` : "Request failed" + } +} export type Error = ProviderRequiredError | ProviderNotFoundError | DisabledError | RequestError @@ -195,7 +222,13 @@ const layer = Layer.effect( const result = yield* provider .execute({ query: input.query }) .pipe(Effect.flatMap(decodeResults), Effect.result) - if (result._tag === "Success") return new Response({ providerID: provider.id, results: result.success }) + if (result._tag === "Success") + return new Response({ + providerID: provider.id, + results: result.success.map((item) => + item.content === undefined ? item : { ...item, content: limitContent(item.content) }, + ), + }) const cause = result.failure const error = new RequestError({ providerID: provider.id, cause }) if (choice !== "random" || !HttpClientError.isHttpClientError(cause) || cause.response?.status !== 429) @@ -212,6 +245,14 @@ const layer = Layer.effect( }), ) +function limitContent(content: string) { + if (content.length <= MAX_RESULT_CONTENT_LENGTH) return content + const code = content.charCodeAt(MAX_RESULT_CONTENT_LENGTH - 1) + // Do not split a surrogate pair; a lone surrogate is invalid text for model requests. + const end = code >= 0xd800 && code <= 0xdbff ? MAX_RESULT_CONTENT_LENGTH - 1 : MAX_RESULT_CONTENT_LENGTH + return `${content.slice(0, end)}\n[truncated]` +} + function cooldownMillis(value: string | undefined, now: number) { if (!value?.trim()) return 60_000 const seconds = Number(value) diff --git a/packages/core/test/plugin/websearch.test.ts b/packages/core/test/plugin/websearch.test.ts index 775f576df5c5..4ff2e64ad118 100644 --- a/packages/core/test/plugin/websearch.test.ts +++ b/packages/core/test/plugin/websearch.test.ts @@ -5,6 +5,7 @@ import { WebSearch } from "@opencode/core/websearch" import { WebSearchExa } from "@opencode/core/plugin/websearch/exa" import { WebSearchFirecrawl } from "@opencode/core/plugin/websearch/firecrawl" import { WebSearchParallel } from "@opencode/core/plugin/websearch/parallel" +import { WebSearchResponse } from "@opencode/core/plugin/websearch/response" import { WebSearchTavily } from "@opencode/core/plugin/websearch/tavily" import { WebSearchTinyFish } from "@opencode/core/plugin/websearch/tinyfish" import { host, integrationHost, webSearchHost } from "./host" @@ -29,6 +30,7 @@ beforeEach(() => { }) const it = webSearchIntegrationTest +const sseMessage = (message: object) => `event: message\ndata: ${JSON.stringify({ jsonrpc: "2.0", id: 1, ...message })}\n\n` describe("built-in web search providers", () => { ;[ @@ -93,6 +95,201 @@ describe("built-in web search providers", () => { }), ) + describe("responses larger than the size cap", () => { + const huge = "x".repeat(2 * WebSearchResponse.MAX_BYTES) + const sse = (result: object) => sseMessage({ result }) + const text = (value: string) => ({ content: [{ type: "text", text: value }] }) + const exaBlock = (url: string, title: string, content: string) => + `Title: ${title}\nURL: ${url}\nPublished: N/A\nAuthor: N/A\nHighlights:\n${content}` + const parallelSearch = { + search_id: "search_1", + results: [ + { url: "https://effect.website", title: "Effect", publish_date: null, excerpts: ["Effect documentation"] }, + { url: "https://huge.example.com", title: "Huge", publish_date: null, excerpts: [huge] }, + { url: "https://after.example.com", title: "After", publish_date: null, excerpts: ["after"] }, + ], + session_id: "ses_parallel", + } + ;[ + { + plugin: WebSearchExa.Plugin, + body: sse( + text( + [ + exaBlock("https://effect.website", "Effect", "Effect documentation"), + exaBlock("https://huge.example.com", "Huge", huge), + exaBlock("https://after.example.com", "After", "after"), + ].join("\n\n---\n\n"), + ), + ), + }, + { + plugin: WebSearchFirecrawl.Plugin, + body: sse( + text( + JSON.stringify({ + success: true, + data: { + web: [ + { url: "https://effect.website", title: "Effect", description: "Effect documentation" }, + { url: "https://huge.example.com", title: "Huge", description: huge }, + { url: "https://after.example.com", title: "After", description: "after" }, + ], + }, + }), + ), + ), + }, + { + plugin: WebSearchParallel.Plugin, + body: JSON.stringify({ + jsonrpc: "2.0", + id: 1, + result: { ...text(JSON.stringify(parallelSearch)), structuredContent: parallelSearch }, + }), + }, + { + plugin: WebSearchTavily.Plugin, + body: JSON.stringify({ + results: [ + { title: "Effect", url: "https://effect.website", content: "Effect documentation" }, + { title: "Huge", url: "https://huge.example.com", content: huge }, + { title: "After", url: "https://after.example.com", content: "after" }, + ], + }), + }, + { + plugin: WebSearchTinyFish.Plugin, + body: sse( + text( + JSON.stringify({ + results: [ + { title: "Effect", url: "https://effect.website", snippet: "Effect documentation" }, + { title: "Huge", url: "https://huge.example.com", snippet: huge }, + { title: "After", url: "https://after.example.com", snippet: "after" }, + ], + }), + ), + ), + }, + ].forEach((provider) => { + it.effect(`keeps the complete results from ${provider.plugin.id}`, () => + Effect.gen(function* () { + resetWebSearchFixture(provider.body) + const integrations = yield* Integration.Service + const websearch = yield* WebSearch.Service + yield* provider.plugin.effect( + host({ integration: integrationHost(integrations), websearch: webSearchHost(websearch) }), + ) + const providerID = (yield* websearch.providers())[0]!.id + + expect(yield* websearch.query({ query: "effect", providerID })).toEqual( + new WebSearch.Response({ + providerID, + results: [{ url: "https://effect.website", title: "Effect", content: "Effect documentation", time: {} }], + }), + ) + }), + ) + }) + }) + + it.effect("reports MCP tool errors instead of returning no results", () => + Effect.gen(function* () { + const integrations = yield* Integration.Service + const websearch = yield* WebSearch.Service + yield* WebSearchFirecrawl.Plugin.effect( + host({ integration: integrationHost(integrations), websearch: webSearchHost(websearch) }), + ) + const search = websearch.query({ query: "effect", providerID: WebSearch.ID.make("firecrawl") }).pipe(Effect.flip) + + resetWebSearchFixture( + sseMessage({ result: { isError: true, content: [{ type: "text", text: "Rate limit exceeded" }] } }), + ) + expect((yield* search).message).toBe("Rate limit exceeded") + resetWebSearchFixture(sseMessage({ error: { code: -32602, message: "Invalid arguments" } })) + expect((yield* search).message).toBe("Invalid arguments") + }), + ) + + it.effect("reports the provider's explanation for HTTP failures", () => + Effect.gen(function* () { + const integrations = yield* Integration.Service + const websearch = yield* WebSearch.Service + yield* WebSearchTinyFish.Plugin.effect( + host({ integration: integrationHost(integrations), websearch: webSearchHost(websearch) }), + ) + yield* WebSearchTavily.Plugin.effect( + host({ integration: integrationHost(integrations), websearch: webSearchHost(websearch) }), + ) + const quota = + "Free daily Search quota used (50/50). Sign up for continued access: https://agent.tinyfish.ai/sign-up" + + resetWebSearchFixture( + JSON.stringify({ jsonrpc: "2.0", error: { code: -31001, message: quota }, id: 1 }), + 401, + ) + const tinyfish = yield* websearch + .query({ query: "effect", providerID: WebSearch.ID.make("tinyfish") }) + .pipe(Effect.flip) + expect(tinyfish.message).toBe(`HTTP 401: ${quota}`) + + resetWebSearchFixture(JSON.stringify({ detail: { error: "Unauthorized: missing or invalid API key." } }), 401) + const tavily = yield* websearch + .query({ query: "effect", providerID: WebSearch.ID.make("tavily") }) + .pipe(Effect.flip) + expect(tavily.message).toBe("HTTP 401: Unauthorized: missing or invalid API key.") + }), + ) + + it.effect("limits oversized Firecrawl results instead of failing the search", () => + Effect.gen(function* () { + const thread = "comment ".repeat(64 * 1024) + resetWebSearchFixture( + `event: message\ndata: ${JSON.stringify({ + jsonrpc: "2.0", + id: 1, + result: { + content: [ + { + type: "text", + text: JSON.stringify({ + success: true, + data: { + web: [ + { url: "https://news.ycombinator.com/item?id=1", title: "Thread", description: thread }, + { url: "https://effect.website", title: "Effect", description: "Effect documentation" }, + ], + }, + }), + }, + ], + }, + })}\n\n`, + ) + const integrations = yield* Integration.Service + const websearch = yield* WebSearch.Service + yield* WebSearchFirecrawl.Plugin.effect( + host({ integration: integrationHost(integrations), websearch: webSearchHost(websearch) }), + ) + + expect(yield* websearch.query({ query: "opencode", providerID: WebSearch.ID.make("firecrawl") })).toEqual( + new WebSearch.Response({ + providerID: WebSearch.ID.make("firecrawl"), + results: [ + { + url: "https://news.ycombinator.com/item?id=1", + title: "Thread", + content: `${thread.slice(0, WebSearch.MAX_RESULT_CONTENT_LENGTH)}\n[truncated]`, + time: {}, + }, + { url: "https://effect.website", title: "Effect", content: "Effect documentation", time: {} }, + ], + }), + ) + }), + ) + it.effect("registers Exa with its MCP schema", () => Effect.gen(function* () { const integrations = yield* Integration.Service diff --git a/packages/core/test/tool-websearch.test.ts b/packages/core/test/tool-websearch.test.ts index 7e3f8e8dd723..94dd5a58733e 100644 --- a/packages/core/test/tool-websearch.test.ts +++ b/packages/core/test/tool-websearch.test.ts @@ -1,6 +1,6 @@ import { describe, expect } from "bun:test" import { Context, Effect, Layer } from "effect" -import type { HttpClientError } from "effect/unstable/http" +import { HttpClientError, HttpClientRequest } from "effect/unstable/http" import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" import { LayerNode } from "@opencode/util/effect/layer-node" import { Permission } from "@opencode/core/permission" @@ -44,7 +44,7 @@ class Fixture { formResponse: Form.TerminalState = { status: "cancelled" } formResponses: Form.TerminalState[] = [] formWait = Effect.void - error: HttpClientError.HttpClientError | undefined + error: HttpClientError.HttpClientError | Error | undefined results: readonly WebSearch.Result[] = [ { url: "https://example.com", title: "Search results", content: "search results", time: {} }, ] @@ -62,7 +62,7 @@ const setup = Effect.gen(function* () { execute: () => Effect.gen(function* () { fixture.events.push("query") - if (fixture.error) return yield* fixture.error + if (fixture.error) return yield* Effect.fail(fixture.error) return fixture.results }), }), @@ -418,7 +418,7 @@ describe("WebSearchTool registration", () => { .pipe(Effect.flip) expect(toSessionError(error)).toEqual({ type: "tool.execution", - message: "Web search rate limited (HTTP 429)", + message: `Unable to search the web for ${query} (${error.metadata?.provider}): Rate limited (HTTP 429)`, }) expect(error.metadata).toMatchObject({ provider: expect.stringMatching(/^(exa|parallel)$/) }) }), @@ -437,9 +437,9 @@ describe("WebSearchTool registration", () => { yield* Effect.forEach( [ - { status: 403, message: "Web search request failed (HTTP 403)" }, - { status: 429, message: "Web search rate limited (HTTP 429)" }, - { status: 401, message: "Web search authentication failed (HTTP 401)" }, + { status: 403, message: "Unable to search the web for effect (exa): Request failed (HTTP 403)" }, + { status: 429, message: "Unable to search the web for effect (exa): Rate limited (HTTP 429)" }, + { status: 401, message: "Unable to search the web for effect (exa): Authentication failed (HTTP 401)" }, ], ({ status, message }, index) => Effect.gen(function* () { @@ -469,4 +469,58 @@ describe("WebSearchTool registration", () => { ) }), ) + + it.effect("reports the underlying reason for provider failures", () => + Effect.gen(function* () { + const fixture = yield* setup + const tools = yield* fixture.registry.snapshot() + yield* fixture.websearch.select(WebSearch.ID.make("exa")) + + yield* Effect.forEach( + [ + { + error: new Error("web_search_exa request timed out"), + message: "Unable to search the web for effect (exa): web_search_exa request timed out", + }, + { + error: new HttpClientError.HttpClientError({ + reason: new HttpClientError.TransportError({ + request: HttpClientRequest.post("https://mcp.exa.ai/mcp?exaApiKey=secret"), + cause: new TypeError("Unable to connect. Is the computer able to access the url?"), + }), + }), + message: + "Unable to search the web for effect (exa): Request failed: Unable to connect. Is the computer able to access the url?", + }, + ], + (item, index) => + Effect.gen(function* () { + fixture.error = item.error + const error = yield* tools + .execute({ + sessionID, + ...toolIdentity, + call: { type: "tool-call", id: `call-reason-${index}`, name: "websearch", input: { query: "effect" } }, + }) + .pipe(Effect.flip) + expect(toSessionError(error)).toEqual({ type: "tool.execution", message: item.message }) + expect(error.metadata).toEqual({ provider: "exa" }) + }), + { discard: true }, + ) + + yield* fixture.websearch.select(false) + const disabled = yield* tools + .execute({ + sessionID, + ...toolIdentity, + call: { type: "tool-call", id: "call-disabled", name: "websearch", input: { query: "effect" } }, + }) + .pipe(Effect.flip) + expect(toSessionError(disabled)).toEqual({ + type: "tool.execution", + message: "Unable to search the web for effect: Web search is disabled", + }) + }), + ) }) diff --git a/packages/core/test/websearch.test.ts b/packages/core/test/websearch.test.ts index 59f0c32d1055..162c6ee9b61e 100644 --- a/packages/core/test/websearch.test.ts +++ b/packages/core/test/websearch.test.ts @@ -86,6 +86,34 @@ describe("WebSearch", () => { }), ) + it.effect("limits each result's content for every provider", () => + Effect.gen(function* () { + const websearch = yield* WebSearch.Service + const providerID = WebSearch.ID.make("large") + const limit = WebSearch.MAX_RESULT_CONTENT_LENGTH + yield* websearch.transform((editor) => { + editor.add({ + id: providerID, + name: "Large", + execute: () => + Effect.succeed([ + { url: "https://large.example.com", content: "b".repeat(limit + 1), time: {} }, + { url: "https://emoji.example.com", content: `${"a".repeat(limit - 1)}😀 after the limit`, time: {} }, + { url: "https://exact.example.com", content: "c".repeat(limit), time: {} }, + { url: "https://empty.example.com", time: {} }, + ]), + }) + }) + + expect((yield* websearch.query({ query: "large", providerID })).results).toEqual([ + { url: "https://large.example.com", content: `${"b".repeat(limit)}\n[truncated]`, time: {} }, + { url: "https://emoji.example.com", content: `${"a".repeat(limit - 1)}\n[truncated]`, time: {} }, + { url: "https://exact.example.com", content: "c".repeat(limit), time: {} }, + { url: "https://empty.example.com", time: {} }, + ]) + }), + ) + it.effect("requires a provider when no default is set", () => Effect.gen(function* () { yield* register("exa") From 1c339219c18faa3fa9f32684e7f4125a8da71ca2 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Wed, 23 Sep 2026 12:57:33 -0500 Subject: [PATCH 2/2] fix(core): share web search limits with OpenCode Web Search OpenCode Web Search, the default search provider for Console-connected users, made its own request with no size cap and discarded the body of failed responses. It now uses the shared response handling, and OpenCode API error bodies ({ _tag, message }) supply the failure explanation. Reading a failed response for its explanation is bounded to one second so a stalled error body still fails promptly. Also formats the earlier changes with Prettier and tidies the tests. --- packages/core/src/plugin/provider/opencode.ts | 36 ++++---- .../core/src/plugin/websearch/parallel.ts | 4 +- .../core/src/plugin/websearch/response.ts | 38 +++++---- packages/core/src/tool/plugin/websearch.ts | 2 +- packages/core/src/websearch.ts | 4 +- .../test/plugin/provider-opencode.test.ts | 36 +++++++- packages/core/test/plugin/websearch.test.ts | 82 +++++++++---------- 7 files changed, 125 insertions(+), 77 deletions(-) diff --git a/packages/core/src/plugin/provider/opencode.ts b/packages/core/src/plugin/provider/opencode.ts index 331e802068a3..758f3110009a 100644 --- a/packages/core/src/plugin/provider/opencode.ts +++ b/packages/core/src/plugin/provider/opencode.ts @@ -1,4 +1,4 @@ -import { Duration, Effect, Equal, Schema, Semaphore, Stream } from "effect" +import { Duration, Effect, Equal, Option, Schema, Semaphore, Stream } from "effect" import type { Scope } from "effect" import type { IntegrationOAuthMethodRegistration } from "@opencode/plugin/effect/integration" import { define } from "@opencode/plugin/effect/plugin" @@ -11,6 +11,7 @@ import { IntegrationConnection } from "../../integration/connection.js" import { ManagedPolicy } from "../../managed-policy.js" import { Provider } from "../../provider.js" import { WebSearch } from "../../websearch.js" +import { WebSearchResponse } from "../websearch/response.js" import { ConfigPolicy } from "@opencode/schema/config/policy" import { ConfigProvider } from "@opencode/schema/config/provider" import { Money } from "@opencode/schema/money" @@ -45,6 +46,9 @@ const TokenPending = Schema.Struct({ error: Schema.String }) const DeviceToken = Schema.Union([Token, TokenPending]) const User = Schema.Struct({ id: Schema.String, email: Schema.String }) const Org = Schema.Struct({ id: Schema.String, name: Schema.String }) +const decodeWebSearchResponse = Schema.decodeUnknownOption( + Schema.Struct({ providerID: WebSearch.ID, results: Schema.Array(Schema.Unknown) }), +) function oauth(http: HttpClient.HttpClient) { return { @@ -313,25 +317,27 @@ export const OpencodePlugin = define Effect.fail(new Error("OpenCode web search request timed out")), - }), - ) - if (response.providerID !== descriptor.providerID) { + const body = yield* WebSearchResponse.execute(http, request).pipe( + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), + Effect.scoped, + Effect.timeoutOrElse({ + duration: Duration.seconds(25), + orElse: () => Effect.fail(new Error("OpenCode web search request timed out")), + }), + ) + const response = WebSearchResponse.json(body.text, body.truncated).pipe( + Option.flatMap(decodeWebSearchResponse), + ) + if (Option.isNone(response)) + return yield* Effect.fail(new Error("OpenCode web search returned an invalid response")) + if (response.value.providerID !== descriptor.providerID) { return yield* Effect.fail( new Error( - `OpenCode web search returned provider ${response.providerID} instead of ${descriptor.providerID}`, + `OpenCode web search returned provider ${response.value.providerID} instead of ${descriptor.providerID}`, ), ) } - return response.results + return WebSearchResponse.items(WebSearch.Result, response.value.results, body.truncated) }), }) editor.default.set(descriptor.providerID) diff --git a/packages/core/src/plugin/websearch/parallel.ts b/packages/core/src/plugin/websearch/parallel.ts index 3371f3d210f2..0f7407c6072a 100644 --- a/packages/core/src/plugin/websearch/parallel.ts +++ b/packages/core/src/plugin/websearch/parallel.ts @@ -71,7 +71,9 @@ export const Plugin = define({ const search = Option.getOrUndefined( decodeSearchResponse( response.result?.structuredContent ?? - (content ? Option.getOrUndefined(WebSearchResponse.json(content.text, response.truncated)) : undefined), + (content + ? Option.getOrUndefined(WebSearchResponse.json(content.text, response.truncated)) + : undefined), ), ) return WebSearchResponse.items(SearchResult, search?.results ?? [], response.truncated).map((item) => { diff --git a/packages/core/src/plugin/websearch/response.ts b/packages/core/src/plugin/websearch/response.ts index 29553f7ce1d9..eca274912273 100644 --- a/packages/core/src/plugin/websearch/response.ts +++ b/packages/core/src/plugin/websearch/response.ts @@ -1,9 +1,8 @@ export * as WebSearchResponse from "./response.js" import { parseJSON } from "@opencode/ai/protocols/utils/partial-json" -import { Effect, Option, Schema, Stream } from "effect" -import { HttpClient, HttpClientError } from "effect/unstable/http" -import type { HttpClientRequest, HttpClientResponse } from "effect/unstable/http" +import { Duration, Effect, Option, Schema, Stream } from "effect" +import { HttpClient, HttpClientError, type HttpClientRequest, type HttpClientResponse } from "effect/unstable/http" export const MAX_BYTES = 1024 * 1024 @@ -12,22 +11,27 @@ export const execute = (http: HttpClient.HttpClient, request: HttpClientRequest. Effect.gen(function* () { const response = yield* HttpClient.withScope(http).execute(request) if (response.status >= 200 && response.status < 300) return yield* read(response) - const description = yield* read(response).pipe( - Effect.map((body) => - payloads(body.text) - .flatMap((payload) => Option.toArray(json(payload, body.truncated))) - .map(failure) - .find((message) => message !== undefined), - ), - Effect.orElseSucceed(() => undefined), - ) return yield* new HttpClientError.HttpClientError({ - reason: new HttpClientError.StatusCodeError({ request, response, description }), + reason: new HttpClientError.StatusCodeError({ + request, + response, + description: yield* read(response).pipe( + Effect.map((body) => + payloads(body.text) + .flatMap((payload) => Option.toArray(json(payload, body.truncated))) + .map(failure) + .find((message) => message !== undefined), + ), + // The explanation is optional; a slow or broken body must not delay the failure. + Effect.timeoutOrElse({ duration: Duration.seconds(1), orElse: () => Effect.undefined }), + Effect.orElseSucceed(() => undefined), + ), + }), }) }) // Stops reading at MAX_BYTES instead of failing; parsers keep the results that arrived complete. -export const read = (response: HttpClientResponse.HttpClientResponse) => +const read = (response: HttpClientResponse.HttpClientResponse) => Effect.gen(function* () { let size = 0 const chunks = yield* response.stream.pipe( @@ -58,16 +62,18 @@ const decodeFailure = Schema.decodeUnknownOption( }), }), Schema.Struct({ detail: Schema.Struct({ error: Schema.String }) }), + Schema.Struct({ _tag: Schema.String, message: Schema.String }), ]), ) -// JSON-RPC errors, MCP tool errors, and Tavily's error detail. +// JSON-RPC errors, MCP tool errors, Tavily's error detail, and OpenCode API errors. export const failure = (value: unknown) => Option.getOrUndefined( Option.map(decodeFailure(value), (decoded) => { if ("error" in decoded) return decoded.error.message if ("result" in decoded) return decoded.result.content.map((item) => item.text).join("\n") - return decoded.detail.error + if ("detail" in decoded) return decoded.detail.error + return decoded.message }), ) diff --git a/packages/core/src/tool/plugin/websearch.ts b/packages/core/src/tool/plugin/websearch.ts index 84606ffec199..68096ce05ac5 100644 --- a/packages/core/src/tool/plugin/websearch.ts +++ b/packages/core/src/tool/plugin/websearch.ts @@ -162,7 +162,7 @@ export const Plugin = { const message = `Unable to search the web for ${input.query}` // A cause becomes the model-visible error, so web search failures are described here instead; // other causes, such as permission errors, keep their own session error. - if (Schema.is(WebSearch.RequestError)(error)) + if (error instanceof WebSearch.RequestError) return new ToolFailure({ message: `${message} (${error.providerID}): ${error.message}`, metadata: { provider: error.providerID }, diff --git a/packages/core/src/websearch.ts b/packages/core/src/websearch.ts index 89fcb3ebc77e..86667082580a 100644 --- a/packages/core/src/websearch.ts +++ b/packages/core/src/websearch.ts @@ -75,7 +75,9 @@ export class RequestError extends Schema.TaggedError()("WebSearch. if (status === 429) return "Rate limited (HTTP 429)" if (status === 401) return "Authentication failed (HTTP 401)" if (status !== undefined) return `Request failed (HTTP ${status})` - return cause.cause instanceof Error && cause.cause.message ? `Request failed: ${cause.cause.message}` : "Request failed" + return cause.cause instanceof Error && cause.cause.message + ? `Request failed: ${cause.cause.message}` + : "Request failed" } } diff --git a/packages/core/test/plugin/provider-opencode.test.ts b/packages/core/test/plugin/provider-opencode.test.ts index 180390bfb4b5..a84697d97290 100644 --- a/packages/core/test/plugin/provider-opencode.test.ts +++ b/packages/core/test/plugin/provider-opencode.test.ts @@ -17,6 +17,7 @@ import { ModelResolver } from "@opencode/core/model-resolver" import { Plugin } from "@opencode/core/plugin" import { PluginHost } from "@opencode/core/plugin/host" import { OpencodePlugin } from "@opencode/core/plugin/provider/opencode" +import { WebSearchResponse } from "@opencode/core/plugin/websearch/response" import { Provider } from "@opencode/core/provider" import { WebSearch } from "@opencode/core/websearch" import { withEnv } from "../fixture/env" @@ -805,7 +806,13 @@ describe("OpencodePlugin", () => { body?: unknown }> = [] const gate = Promise.withResolvers() - const state = { advertised: true, providerID: "opencode", waitForConfig: false } + const state = { + advertised: true, + providerID: "opencode", + waitForConfig: false, + unavailable: false, + oversized: false, + } const server = Bun.serve({ port: 0, fetch: async (request) => { @@ -832,6 +839,11 @@ describe("OpencodePlugin", () => { }) } if (path === "/api/websearch" || path === "/other/api/websearch") { + if (state.unavailable) + return Response.json( + { _tag: "ServiceUnavailableError", message: "Web search request failed: firecrawl" }, + { status: 503 }, + ) return Response.json({ providerID: state.providerID, results: [ @@ -841,6 +853,16 @@ describe("OpencodePlugin", () => { content: "Open source AI coding agent.", time: { published: 1_700_000_000_000 }, }, + ...(state.oversized + ? [ + { + url: "https://huge.example.com", + content: "x".repeat(2 * WebSearchResponse.MAX_BYTES), + time: {}, + }, + { url: "https://after.example.com", content: "after", time: {} }, + ] + : []), ], }) } @@ -927,6 +949,18 @@ describe("OpencodePlugin", () => { value: account("replacement"), }) + state.oversized = true + expect((yield* websearch.query({ query: "oversized" })).results.map((result) => result.url)).toEqual([ + "https://github.com/anomalyco/opencode", + ]) + state.oversized = false + + state.unavailable = true + expect((yield* websearch.query({ query: "unavailable" }).pipe(Effect.flip)).message).toBe( + "HTTP 503: Web search request failed: firecrawl", + ) + state.unavailable = false + state.providerID = "unexpected" expect((yield* websearch.query({ query: "wrong provider" }).pipe(Effect.flip))._tag).toBe("WebSearch.Request") diff --git a/packages/core/test/plugin/websearch.test.ts b/packages/core/test/plugin/websearch.test.ts index 4ff2e64ad118..815b4fc00581 100644 --- a/packages/core/test/plugin/websearch.test.ts +++ b/packages/core/test/plugin/websearch.test.ts @@ -30,7 +30,8 @@ beforeEach(() => { }) const it = webSearchIntegrationTest -const sseMessage = (message: object) => `event: message\ndata: ${JSON.stringify({ jsonrpc: "2.0", id: 1, ...message })}\n\n` +const sseMessage = (message: object) => + `event: message\ndata: ${JSON.stringify({ jsonrpc: "2.0", id: 1, ...message })}\n\n` describe("built-in web search providers", () => { ;[ @@ -97,8 +98,7 @@ describe("built-in web search providers", () => { describe("responses larger than the size cap", () => { const huge = "x".repeat(2 * WebSearchResponse.MAX_BYTES) - const sse = (result: object) => sseMessage({ result }) - const text = (value: string) => ({ content: [{ type: "text", text: value }] }) + const mcpText = (text: string) => sseMessage({ result: { content: [{ type: "text", text }] } }) const exaBlock = (url: string, title: string, content: string) => `Title: ${title}\nURL: ${url}\nPublished: N/A\nAuthor: N/A\nHighlights:\n${content}` const parallelSearch = { @@ -113,43 +113,46 @@ describe("built-in web search providers", () => { ;[ { plugin: WebSearchExa.Plugin, - body: sse( - text( - [ - exaBlock("https://effect.website", "Effect", "Effect documentation"), - exaBlock("https://huge.example.com", "Huge", huge), - exaBlock("https://after.example.com", "After", "after"), - ].join("\n\n---\n\n"), - ), + providerID: WebSearch.ID.make("exa"), + body: mcpText( + [ + exaBlock("https://effect.website", "Effect", "Effect documentation"), + exaBlock("https://huge.example.com", "Huge", huge), + exaBlock("https://after.example.com", "After", "after"), + ].join("\n\n---\n\n"), ), }, { plugin: WebSearchFirecrawl.Plugin, - body: sse( - text( - JSON.stringify({ - success: true, - data: { - web: [ - { url: "https://effect.website", title: "Effect", description: "Effect documentation" }, - { url: "https://huge.example.com", title: "Huge", description: huge }, - { url: "https://after.example.com", title: "After", description: "after" }, - ], - }, - }), - ), + providerID: WebSearch.ID.make("firecrawl"), + body: mcpText( + JSON.stringify({ + success: true, + data: { + web: [ + { url: "https://effect.website", title: "Effect", description: "Effect documentation" }, + { url: "https://huge.example.com", title: "Huge", description: huge }, + { url: "https://after.example.com", title: "After", description: "after" }, + ], + }, + }), ), }, { plugin: WebSearchParallel.Plugin, + providerID: WebSearch.ID.make("parallel"), body: JSON.stringify({ jsonrpc: "2.0", id: 1, - result: { ...text(JSON.stringify(parallelSearch)), structuredContent: parallelSearch }, + result: { + content: [{ type: "text", text: JSON.stringify(parallelSearch) }], + structuredContent: parallelSearch, + }, }), }, { plugin: WebSearchTavily.Plugin, + providerID: WebSearch.ID.make("tavily"), body: JSON.stringify({ results: [ { title: "Effect", url: "https://effect.website", content: "Effect documentation" }, @@ -160,16 +163,15 @@ describe("built-in web search providers", () => { }, { plugin: WebSearchTinyFish.Plugin, - body: sse( - text( - JSON.stringify({ - results: [ - { title: "Effect", url: "https://effect.website", snippet: "Effect documentation" }, - { title: "Huge", url: "https://huge.example.com", snippet: huge }, - { title: "After", url: "https://after.example.com", snippet: "after" }, - ], - }), - ), + providerID: WebSearch.ID.make("tinyfish"), + body: mcpText( + JSON.stringify({ + results: [ + { title: "Effect", url: "https://effect.website", snippet: "Effect documentation" }, + { title: "Huge", url: "https://huge.example.com", snippet: huge }, + { title: "After", url: "https://after.example.com", snippet: "after" }, + ], + }), ), }, ].forEach((provider) => { @@ -181,11 +183,10 @@ describe("built-in web search providers", () => { yield* provider.plugin.effect( host({ integration: integrationHost(integrations), websearch: webSearchHost(websearch) }), ) - const providerID = (yield* websearch.providers())[0]!.id - expect(yield* websearch.query({ query: "effect", providerID })).toEqual( + expect(yield* websearch.query({ query: "effect", providerID: provider.providerID })).toEqual( new WebSearch.Response({ - providerID, + providerID: provider.providerID, results: [{ url: "https://effect.website", title: "Effect", content: "Effect documentation", time: {} }], }), ) @@ -225,10 +226,7 @@ describe("built-in web search providers", () => { const quota = "Free daily Search quota used (50/50). Sign up for continued access: https://agent.tinyfish.ai/sign-up" - resetWebSearchFixture( - JSON.stringify({ jsonrpc: "2.0", error: { code: -31001, message: quota }, id: 1 }), - 401, - ) + resetWebSearchFixture(JSON.stringify({ jsonrpc: "2.0", error: { code: -31001, message: quota }, id: 1 }), 401) const tinyfish = yield* websearch .query({ query: "effect", providerID: WebSearch.ID.make("tinyfish") }) .pipe(Effect.flip)