diff --git a/packages/core/src/plugin/provider/opencode.ts b/packages/core/src/plugin/provider/opencode.ts index 331e802068a3..758f3110009a 100644 --- a/packages/core/src/plugin/provider/opencode.ts +++ b/packages/core/src/plugin/provider/opencode.ts @@ -1,4 +1,4 @@ -import { Duration, Effect, Equal, Schema, Semaphore, Stream } from "effect" +import { Duration, Effect, Equal, Option, Schema, Semaphore, Stream } from "effect" import type { Scope } from "effect" import type { IntegrationOAuthMethodRegistration } from "@opencode/plugin/effect/integration" import { define } from "@opencode/plugin/effect/plugin" @@ -11,6 +11,7 @@ import { IntegrationConnection } from "../../integration/connection.js" import { ManagedPolicy } from "../../managed-policy.js" import { Provider } from "../../provider.js" import { WebSearch } from "../../websearch.js" +import { WebSearchResponse } from "../websearch/response.js" import { ConfigPolicy } from "@opencode/schema/config/policy" import { ConfigProvider } from "@opencode/schema/config/provider" import { Money } from "@opencode/schema/money" @@ -45,6 +46,9 @@ const TokenPending = Schema.Struct({ error: Schema.String }) const DeviceToken = Schema.Union([Token, TokenPending]) const User = Schema.Struct({ id: Schema.String, email: Schema.String }) const Org = Schema.Struct({ id: Schema.String, name: Schema.String }) +const decodeWebSearchResponse = Schema.decodeUnknownOption( + Schema.Struct({ providerID: WebSearch.ID, results: Schema.Array(Schema.Unknown) }), +) function oauth(http: HttpClient.HttpClient) { return { @@ -313,25 +317,27 @@ export const OpencodePlugin = define Effect.fail(new Error("OpenCode web search request timed out")), - }), - ) - if (response.providerID !== descriptor.providerID) { + const body = yield* WebSearchResponse.execute(http, request).pipe( + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }), + Effect.scoped, + Effect.timeoutOrElse({ + duration: Duration.seconds(25), + orElse: () => Effect.fail(new Error("OpenCode web search request timed out")), + }), + ) + const response = WebSearchResponse.json(body.text, body.truncated).pipe( + Option.flatMap(decodeWebSearchResponse), + ) + if (Option.isNone(response)) + return yield* Effect.fail(new Error("OpenCode web search returned an invalid response")) + if (response.value.providerID !== descriptor.providerID) { return yield* Effect.fail( new Error( - `OpenCode web search returned provider ${response.providerID} instead of ${descriptor.providerID}`, + `OpenCode web search returned provider ${response.value.providerID} instead of ${descriptor.providerID}`, ), ) } - return response.results + return WebSearchResponse.items(WebSearch.Result, response.value.results, body.truncated) }), }) editor.default.set(descriptor.providerID) diff --git a/packages/core/src/plugin/websearch/exa.ts b/packages/core/src/plugin/websearch/exa.ts index 1dc3a26108b1..c948fc38ee53 100644 --- a/packages/core/src/plugin/websearch/exa.ts +++ b/packages/core/src/plugin/websearch/exa.ts @@ -47,23 +47,25 @@ export const Plugin = define({ const credential = connection ? yield* ctx.integration.connection.resolve(connection) : undefined const url = new URL(endpoint) if (credential?.type === "key") url.searchParams.set("exaApiKey", credential.key) - const result = yield* WebSearchMcp.call( + const response = yield* WebSearchMcp.call( http, url.toString(), "web_search_exa", { input: McpInput, output: McpOutput }, { query: input.query, numResults: 8 }, ) - const content = result?.content.find((item) => item.text) - return content ? parseResults(content.text) : [] + const content = response.result?.content.find((item) => item.text) + return content ? parseResults(content.text, response.truncated) : [] }), }) }) }), }) -function parseResults(text: string) { - return text.split(/\n\n---\n\n/).flatMap((block) => { +function parseResults(text: string, truncated: boolean) { + const blocks = text.split(/\n\n---\n\n/) + // The final block of a truncated response may end mid-line, including inside its URL. + return (truncated ? blocks.slice(0, -1) : blocks).flatMap((block) => { const url = block.match(/^URL:\s*(.+)$/m)?.[1]?.trim() if (!url) return [] const title = block.match(/^Title:\s*(.+)$/m)?.[1]?.trim() diff --git a/packages/core/src/plugin/websearch/firecrawl.ts b/packages/core/src/plugin/websearch/firecrawl.ts index 1ea260e0c5da..3fcdd2786274 100644 --- a/packages/core/src/plugin/websearch/firecrawl.ts +++ b/packages/core/src/plugin/websearch/firecrawl.ts @@ -5,6 +5,7 @@ import { Effect, Option, Schema, Scope } from "effect" import { HttpClient } from "effect/unstable/http" import { App } from "../../app.js" import { WebSearchMcp } from "./mcp.js" +import { WebSearchResponse } from "./response.js" export const endpoint = "https://mcp.firecrawl.dev/v2/mcp" @@ -17,21 +18,15 @@ const McpOutput = Schema.Struct({ content: Schema.Array(Schema.Struct({ type: Schema.Literal("text"), text: Schema.String })), }) -const SearchResponse = Schema.fromJsonString( - Schema.Struct({ - success: Schema.Boolean, - data: Schema.Struct({ - web: Schema.Array( - Schema.Struct({ - url: Schema.String, - title: Schema.NullOr(Schema.String).pipe(Schema.optional), - description: Schema.NullOr(Schema.String).pipe(Schema.optional), - }), - ), - }), - }), +const decodeSearchResponse = Schema.decodeUnknownOption( + Schema.Struct({ data: Schema.Struct({ web: Schema.Array(Schema.Unknown) }) }), ) -const decodeSearchResponse = Schema.decodeUnknownOption(SearchResponse) + +const SearchResult = Schema.Struct({ + url: Schema.String, + title: Schema.NullOr(Schema.String).pipe(Schema.optional), + description: Schema.NullOr(Schema.String).pipe(Schema.optional), +}) export const Plugin = define({ id: "opencode.websearch.firecrawl", @@ -56,7 +51,7 @@ export const Plugin = define({ Effect.gen(function* () { const connection = yield* ctx.integration.connection.active("firecrawl") const credential = connection ? yield* ctx.integration.connection.resolve(connection) : undefined - const result = yield* WebSearchMcp.call( + const response = yield* WebSearchMcp.call( http, endpoint, "firecrawl_search", @@ -67,16 +62,18 @@ export const Plugin = define({ ...(credential?.type === "key" ? { Authorization: `Bearer ${credential.key}` } : {}), }, ) - const content = result?.content.find((item) => item.text) - const response = content ? Option.getOrUndefined(decodeSearchResponse(content.text)) : undefined - return ( - response?.data.web.map((item) => ({ - url: item.url, - ...(item.title ? { title: item.title } : {}), - ...(item.description ? { content: item.description } : {}), - time: {}, - })) ?? [] - ) + const content = response.result?.content.find((item) => item.text) + const search = content + ? Option.getOrUndefined( + WebSearchResponse.json(content.text, response.truncated).pipe(Option.flatMap(decodeSearchResponse)), + ) + : undefined + return WebSearchResponse.items(SearchResult, search?.data.web ?? [], response.truncated).map((item) => ({ + url: item.url, + ...(item.title ? { title: item.title } : {}), + ...(item.description ? { content: item.description } : {}), + time: {}, + })) }), }) }) diff --git a/packages/core/src/plugin/websearch/mcp.ts b/packages/core/src/plugin/websearch/mcp.ts index 478d4feb5f8d..7fbf9a0677e5 100644 --- a/packages/core/src/plugin/websearch/mcp.ts +++ b/packages/core/src/plugin/websearch/mcp.ts @@ -1,26 +1,22 @@ export * as WebSearchMcp from "./mcp.js" -import { Duration, Effect, Schema } from "effect" +import { Duration, Effect, Option, Schema } from "effect" import { HttpClient, HttpClientRequest } from "effect/unstable/http" -import { collectBoundedResponseBody } from "../../tool/http-body.js" +import { WebSearchResponse } from "./response.js" -export const MAX_RESPONSE_BYTES = 256 * 1024 - -export const parseResponse = (body: string, result: Schema.Struct) => { - const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Struct({ result }))) - const parse = (payload: string) => { - const trimmed = payload.trim() - if (!trimmed.startsWith("{")) return Effect.undefined - return decode(trimmed).pipe(Effect.map((response) => response.result)) - } +export const parseResponse = ( + body: { readonly text: string; readonly truncated: boolean }, + tool: string, + result: Schema.Struct, +) => { + const decode = Schema.decodeUnknownEffect(Schema.Struct({ result })) return Effect.gen(function* () { - const trimmed = body.trim() - const direct = trimmed ? yield* parse(trimmed) : undefined - if (direct) return direct - for (const line of body.split("\n")) { - if (!line.startsWith("data: ")) continue - const data = yield* parse(line.substring(6)) - if (data) return data + for (const payload of WebSearchResponse.payloads(body.text)) { + const message = WebSearchResponse.json(payload, body.truncated) + if (Option.isNone(message)) return yield* Effect.fail(new Error(`${tool} returned invalid JSON`)) + const failure = WebSearchResponse.failure(message.value) + if (failure !== undefined) return yield* Effect.fail(new Error(failure)) + return (yield* decode(message.value)).result } }) } @@ -52,13 +48,8 @@ export const call = new Error(`${tool} response exceeded ${MAX_RESPONSE_BYTES} bytes`), - ) - return yield* parseResponse(body.toString("utf8"), schema.output) + const body = yield* WebSearchResponse.execute(http, request) + return { result: yield* parseResponse(body, tool, schema.output), truncated: body.truncated } }).pipe( Effect.scoped, Effect.timeoutOrElse({ diff --git a/packages/core/src/plugin/websearch/parallel.ts b/packages/core/src/plugin/websearch/parallel.ts index 5184155482bd..0f7407c6072a 100644 --- a/packages/core/src/plugin/websearch/parallel.ts +++ b/packages/core/src/plugin/websearch/parallel.ts @@ -1,10 +1,11 @@ export * as WebSearchParallel from "./parallel.js" import { define } from "@opencode/plugin/effect/plugin" -import { Effect, Schema, Scope } from "effect" +import { Effect, Option, Schema, Scope } from "effect" import { HttpClient } from "effect/unstable/http" import { App } from "../../app.js" import { WebSearchMcp } from "./mcp.js" +import { WebSearchResponse } from "./response.js" export const endpoint = "https://search.parallel.ai/mcp" @@ -14,38 +15,19 @@ const McpInput = Schema.Struct({ model_name: Schema.String.check(Schema.isMaxLength(100)).pipe(Schema.optional), }) -const SearchResponse = Schema.Struct({ - search_id: Schema.String, - results: Schema.Array( - Schema.Struct({ - url: Schema.String, - title: Schema.NullOr(Schema.String).pipe(Schema.optional), - publish_date: Schema.NullOr(Schema.String).pipe(Schema.optional), - excerpts: Schema.Array(Schema.String), - }), - ), - warnings: Schema.NullOr( - Schema.Array( - Schema.Struct({ - type: Schema.Literals(["spec_validation_warning", "input_validation_warning", "warning"]), - message: Schema.String, - detail: Schema.NullOr(Schema.Record(Schema.String, Schema.Json)).pipe(Schema.optional), - }), - ), - ).pipe(Schema.optional), - usage: Schema.NullOr( - Schema.Array( - Schema.Struct({ - name: Schema.String, - count: Schema.Int, - }), - ), - ).pipe(Schema.optional), - session_id: Schema.String, -}) const McpOutput = Schema.Struct({ content: Schema.Array(Schema.Struct({ type: Schema.Literal("text"), text: Schema.String })), - structuredContent: SearchResponse, + // Parallel sends this after a text copy of the same JSON, so a truncated response can lose it. + structuredContent: Schema.Unknown.pipe(Schema.optional), +}) + +const decodeSearchResponse = Schema.decodeUnknownOption(Schema.Struct({ results: Schema.Array(Schema.Unknown) })) + +const SearchResult = Schema.Struct({ + url: Schema.String, + title: Schema.NullOr(Schema.String).pipe(Schema.optional), + publish_date: Schema.NullOr(Schema.String).pipe(Schema.optional), + excerpts: Schema.Array(Schema.String), }) export const Plugin = define({ @@ -71,7 +53,7 @@ export const Plugin = define({ Effect.gen(function* () { const connection = yield* ctx.integration.connection.active("parallel") const credential = connection ? yield* ctx.integration.connection.resolve(connection) : undefined - const result = yield* WebSearchMcp.call( + const response = yield* WebSearchMcp.call( http, endpoint, "web_search", @@ -85,17 +67,24 @@ export const Plugin = define({ ...(credential?.type === "key" ? { Authorization: `Bearer ${credential.key}` } : {}), }, ) - return ( - result?.structuredContent.results.map((item) => { - const published = item.publish_date ? Date.parse(item.publish_date) : undefined - return { - url: item.url, - ...(item.title ? { title: item.title } : {}), - ...(item.excerpts.length ? { content: item.excerpts.join("\n\n") } : {}), - time: { ...(published !== undefined && Number.isFinite(published) ? { published } : {}) }, - } - }) ?? [] + const content = response.result?.content.find((item) => item.text) + const search = Option.getOrUndefined( + decodeSearchResponse( + response.result?.structuredContent ?? + (content + ? Option.getOrUndefined(WebSearchResponse.json(content.text, response.truncated)) + : undefined), + ), ) + return WebSearchResponse.items(SearchResult, search?.results ?? [], response.truncated).map((item) => { + const published = item.publish_date ? Date.parse(item.publish_date) : undefined + return { + url: item.url, + ...(item.title ? { title: item.title } : {}), + ...(item.excerpts.length ? { content: item.excerpts.join("\n\n") } : {}), + time: { ...(published !== undefined && Number.isFinite(published) ? { published } : {}) }, + } + }) }), }) }) diff --git a/packages/core/src/plugin/websearch/response.ts b/packages/core/src/plugin/websearch/response.ts new file mode 100644 index 000000000000..eca274912273 --- /dev/null +++ b/packages/core/src/plugin/websearch/response.ts @@ -0,0 +1,88 @@ +export * as WebSearchResponse from "./response.js" + +import { parseJSON } from "@opencode/ai/protocols/utils/partial-json" +import { Duration, Effect, Option, Schema, Stream } from "effect" +import { HttpClient, HttpClientError, type HttpClientRequest, type HttpClientResponse } from "effect/unstable/http" + +export const MAX_BYTES = 1024 * 1024 + +// Keeps the status for rate-limit failover and adds the provider's own explanation of the failure. +export const execute = (http: HttpClient.HttpClient, request: HttpClientRequest.HttpClientRequest) => + Effect.gen(function* () { + const response = yield* HttpClient.withScope(http).execute(request) + if (response.status >= 200 && response.status < 300) return yield* read(response) + return yield* new HttpClientError.HttpClientError({ + reason: new HttpClientError.StatusCodeError({ + request, + response, + description: yield* read(response).pipe( + Effect.map((body) => + payloads(body.text) + .flatMap((payload) => Option.toArray(json(payload, body.truncated))) + .map(failure) + .find((message) => message !== undefined), + ), + // The explanation is optional; a slow or broken body must not delay the failure. + Effect.timeoutOrElse({ duration: Duration.seconds(1), orElse: () => Effect.undefined }), + Effect.orElseSucceed(() => undefined), + ), + }), + }) + }) + +// Stops reading at MAX_BYTES instead of failing; parsers keep the results that arrived complete. +const read = (response: HttpClientResponse.HttpClientResponse) => + Effect.gen(function* () { + let size = 0 + const chunks = yield* response.stream.pipe( + Stream.takeUntil((chunk) => (size += chunk.byteLength) > MAX_BYTES), + Stream.runCollect, + ) + return { text: Buffer.concat(chunks).subarray(0, MAX_BYTES).toString("utf8"), truncated: size > MAX_BYTES } + }) + +// A JSON body, or the data lines of a server-sent event stream. +export const payloads = (text: string) => + [text, ...text.split("\n").flatMap((line) => (line.startsWith("data: ") ? [line.slice(6)] : []))] + .map((payload) => payload.trim()) + .filter((payload) => payload.startsWith("{")) + +const decodeJson = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Unknown)) +const decodePartialJson = Option.liftThrowable(parseJSON) + +export const json = (text: string, truncated: boolean) => (truncated ? decodePartialJson(text) : decodeJson(text)) + +const decodeFailure = Schema.decodeUnknownOption( + Schema.Union([ + Schema.Struct({ error: Schema.Struct({ message: Schema.String }) }), + Schema.Struct({ + result: Schema.Struct({ + isError: Schema.Literal(true), + content: Schema.Array(Schema.Struct({ text: Schema.String })), + }), + }), + Schema.Struct({ detail: Schema.Struct({ error: Schema.String }) }), + Schema.Struct({ _tag: Schema.String, message: Schema.String }), + ]), +) + +// JSON-RPC errors, MCP tool errors, Tavily's error detail, and OpenCode API errors. +export const failure = (value: unknown) => + Option.getOrUndefined( + Option.map(decodeFailure(value), (decoded) => { + if ("error" in decoded) return decoded.error.message + if ("result" in decoded) return decoded.result.content.map((item) => item.text).join("\n") + if ("detail" in decoded) return decoded.detail.error + return decoded.message + }), + ) + +// The last item of a truncated response may be cut mid-value, so it is dropped rather than trusted. +export const items = >( + schema: S, + values: readonly unknown[], + truncated: boolean, +) => { + const decode = Schema.decodeUnknownOption(schema) + return (truncated ? values.slice(0, -1) : values).flatMap((value) => Option.toArray(decode(value))) +} diff --git a/packages/core/src/plugin/websearch/tavily.ts b/packages/core/src/plugin/websearch/tavily.ts index 4b568a1fb0d2..ab27355dff02 100644 --- a/packages/core/src/plugin/websearch/tavily.ts +++ b/packages/core/src/plugin/websearch/tavily.ts @@ -1,9 +1,10 @@ export * as WebSearchTavily from "./tavily.js" import { define } from "@opencode/plugin/effect/plugin" -import { Duration, Effect, Schema, Scope } from "effect" -import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http" +import { Duration, Effect, Option, Schema, Scope } from "effect" +import { HttpClient, HttpClientRequest } from "effect/unstable/http" import { App } from "../../app.js" +import { WebSearchResponse } from "./response.js" export const endpoint = "https://api.tavily.com/search" @@ -14,14 +15,12 @@ const SearchRequest = Schema.Struct({ max_results: Schema.Number, }) -const SearchResponse = Schema.Struct({ - results: Schema.Array( - Schema.Struct({ - title: Schema.String, - url: Schema.String, - content: Schema.String, - }), - ), +const decodeSearchResponse = Schema.decodeUnknownOption(Schema.Struct({ results: Schema.Array(Schema.Unknown) })) + +const SearchResult = Schema.Struct({ + title: Schema.String, + url: Schema.String, + content: Schema.String, }) export const Plugin = define({ @@ -63,17 +62,16 @@ export const Plugin = define({ max_results: 8, }), ) - const response = yield* HttpClient.withScope(HttpClient.filterStatusOk(http)) - .execute(request) - .pipe( - Effect.flatMap(HttpClientResponse.schemaBodyJson(SearchResponse)), - Effect.scoped, - Effect.timeoutOrElse({ - duration: Duration.seconds(25), - orElse: () => Effect.fail(new Error("Tavily web search request timed out")), - }), - ) - return response.results.map((item) => ({ + const body = yield* WebSearchResponse.execute(http, request).pipe( + Effect.scoped, + Effect.timeoutOrElse({ + duration: Duration.seconds(25), + orElse: () => Effect.fail(new Error("Tavily web search request timed out")), + }), + ) + const search = WebSearchResponse.json(body.text, body.truncated).pipe(Option.flatMap(decodeSearchResponse)) + if (Option.isNone(search)) return yield* Effect.fail(new Error("Tavily returned an invalid response")) + return WebSearchResponse.items(SearchResult, search.value.results, body.truncated).map((item) => ({ url: item.url, title: item.title, ...(item.content ? { content: item.content } : {}), diff --git a/packages/core/src/plugin/websearch/tinyfish.ts b/packages/core/src/plugin/websearch/tinyfish.ts index 77d86acddc1d..d88ed4317d48 100644 --- a/packages/core/src/plugin/websearch/tinyfish.ts +++ b/packages/core/src/plugin/websearch/tinyfish.ts @@ -5,6 +5,7 @@ import { Effect, Option, Schema, Scope } from "effect" import { HttpClient } from "effect/unstable/http" import { App } from "../../app.js" import { WebSearchMcp } from "./mcp.js" +import { WebSearchResponse } from "./response.js" export const endpoint = "https://agent.tinyfish.ai/mcp" @@ -16,18 +17,13 @@ const McpOutput = Schema.Struct({ content: Schema.Array(Schema.Struct({ type: Schema.Literal("text"), text: Schema.String })), }) -const SearchResponse = Schema.fromJsonString( - Schema.Struct({ - results: Schema.Array( - Schema.Struct({ - url: Schema.String, - title: Schema.String, - snippet: Schema.String, - }), - ), - }), -) -const decodeSearchResponse = Schema.decodeUnknownOption(SearchResponse) +const decodeSearchResponse = Schema.decodeUnknownOption(Schema.Struct({ results: Schema.Array(Schema.Unknown) })) + +const SearchResult = Schema.Struct({ + url: Schema.String, + title: Schema.String, + snippet: Schema.String, +}) export const Plugin = define({ id: "opencode.websearch.tinyfish", @@ -52,7 +48,7 @@ export const Plugin = define({ Effect.gen(function* () { const connection = yield* ctx.integration.connection.active("tinyfish") const credential = connection ? yield* ctx.integration.connection.resolve(connection) : undefined - const result = yield* WebSearchMcp.call( + const response = yield* WebSearchMcp.call( http, endpoint, "search", @@ -65,16 +61,18 @@ export const Plugin = define({ : { "X-TinyFish-Access-Mode": "keyless" }), }, ) - const content = result?.content.find((item) => item.text) - const response = content ? Option.getOrUndefined(decodeSearchResponse(content.text)) : undefined - return ( - response?.results.map((item) => ({ - url: item.url, - title: item.title, - ...(item.snippet ? { content: item.snippet } : {}), - time: {}, - })) ?? [] - ) + const content = response.result?.content.find((item) => item.text) + const search = content + ? Option.getOrUndefined( + WebSearchResponse.json(content.text, response.truncated).pipe(Option.flatMap(decodeSearchResponse)), + ) + : undefined + return WebSearchResponse.items(SearchResult, search?.results ?? [], response.truncated).map((item) => ({ + url: item.url, + title: item.title, + ...(item.snippet ? { content: item.snippet } : {}), + time: {}, + })) }), }) }) diff --git a/packages/core/src/tool/plugin/websearch.ts b/packages/core/src/tool/plugin/websearch.ts index 0338dfdb601e..68096ce05ac5 100644 --- a/packages/core/src/tool/plugin/websearch.ts +++ b/packages/core/src/tool/plugin/websearch.ts @@ -4,7 +4,6 @@ import type { Context } from "@opencode/plugin/effect/plugin" import type { SessionHooks } from "@opencode/plugin/effect/session" import { ToolFailure } from "@opencode/ai" import { Effect, Schema, Semaphore } from "effect" -import { HttpClientError } from "effect/unstable/http" import { Form } from "../../form.js" import { Permission } from "../../permission.js" import { WebSearch } from "../../websearch.js" @@ -12,10 +11,6 @@ import { WebSearch } from "../../websearch.js" export const name = "websearch" export const NO_RESULTS = "No search results found. Please try a different query." const providerSelectionLock = Semaphore.makeUnsafe(1) -const httpErrors = new Map([ - [429, "Web search rate limited (HTTP 429)"], - [401, "Web search authentication failed (HTTP 401)"], -]) export const description = `Search the web using the user's selected search integration. Use this for current information beyond knowledge cutoff. @@ -164,17 +159,21 @@ export const Plugin = { return { output, content, metadata: { provider: output.provider } } }).pipe( Effect.mapError((error) => { - const fallback = `Unable to search the web for ${input.query}` - if (!Schema.is(WebSearch.RequestError)(error)) return new ToolFailure({ message: fallback, error }) - const status = HttpClientError.isHttpClientError(error.cause) ? error.cause.response?.status : undefined - return new ToolFailure({ - message: - status === undefined - ? fallback - : (httpErrors.get(status) ?? `Web search request failed (HTTP ${status})`), - error, - metadata: { provider: error.providerID }, - }) + const message = `Unable to search the web for ${input.query}` + // A cause becomes the model-visible error, so web search failures are described here instead; + // other causes, such as permission errors, keep their own session error. + if (error instanceof WebSearch.RequestError) + return new ToolFailure({ + message: `${message} (${error.providerID}): ${error.message}`, + metadata: { provider: error.providerID }, + }) + if ( + error instanceof WebSearch.ProviderRequiredError || + error instanceof WebSearch.ProviderNotFoundError || + error instanceof WebSearch.DisabledError + ) + return new ToolFailure({ message: `${message}: ${error.message}` }) + return new ToolFailure({ message, error }) }), ), }), diff --git a/packages/core/src/websearch.ts b/packages/core/src/websearch.ts index 7ec63bb7b970..86667082580a 100644 --- a/packages/core/src/websearch.ts +++ b/packages/core/src/websearch.ts @@ -29,6 +29,8 @@ export const Response = WebSearch.Response export type Response = WebSearch.Response export const ProviderKey = "websearch:provider" +// Keeps one oversized page from crowding the other results out of the bounded tool output. +export const MAX_RESULT_CONTENT_LENGTH = 4_000 export const Selection = Schema.Union([ID, Schema.Literal("random"), Schema.Literal(false)]) export type Selection = typeof Selection.Type @@ -39,18 +41,45 @@ export interface ProviderImplementation extends Provider { export class ProviderRequiredError extends Schema.TaggedError()( "WebSearch.ProviderRequired", {}, -) {} +) { + override get message() { + return "No web search provider is selected" + } +} export class ProviderNotFoundError extends Schema.TaggedError()("WebSearch.ProviderNotFound", { providerID: ID, -}) {} +}) { + override get message() { + return `Web search provider not found: ${this.providerID}` + } +} -export class DisabledError extends Schema.TaggedError()("WebSearch.Disabled", {}) {} +export class DisabledError extends Schema.TaggedError()("WebSearch.Disabled", {}) { + override get message() { + return "Web search is disabled" + } +} export class RequestError extends Schema.TaggedError()("WebSearch.Request", { providerID: ID, cause: Schema.Defect(), -}) {} +}) { + // HTTP client messages include the request URL, which can carry a provider credential. + override get message() { + const cause = this.cause + if (!HttpClientError.isHttpClientError(cause)) + return cause instanceof Error && cause.message ? cause.message : "Request failed" + const status = cause.response?.status + if (status !== undefined && cause.reason.description) return `HTTP ${status}: ${cause.reason.description}` + if (status === 429) return "Rate limited (HTTP 429)" + if (status === 401) return "Authentication failed (HTTP 401)" + if (status !== undefined) return `Request failed (HTTP ${status})` + return cause.cause instanceof Error && cause.cause.message + ? `Request failed: ${cause.cause.message}` + : "Request failed" + } +} export type Error = ProviderRequiredError | ProviderNotFoundError | DisabledError | RequestError @@ -195,7 +224,13 @@ const layer = Layer.effect( const result = yield* provider .execute({ query: input.query }) .pipe(Effect.flatMap(decodeResults), Effect.result) - if (result._tag === "Success") return new Response({ providerID: provider.id, results: result.success }) + if (result._tag === "Success") + return new Response({ + providerID: provider.id, + results: result.success.map((item) => + item.content === undefined ? item : { ...item, content: limitContent(item.content) }, + ), + }) const cause = result.failure const error = new RequestError({ providerID: provider.id, cause }) if (choice !== "random" || !HttpClientError.isHttpClientError(cause) || cause.response?.status !== 429) @@ -212,6 +247,14 @@ const layer = Layer.effect( }), ) +function limitContent(content: string) { + if (content.length <= MAX_RESULT_CONTENT_LENGTH) return content + const code = content.charCodeAt(MAX_RESULT_CONTENT_LENGTH - 1) + // Do not split a surrogate pair; a lone surrogate is invalid text for model requests. + const end = code >= 0xd800 && code <= 0xdbff ? MAX_RESULT_CONTENT_LENGTH - 1 : MAX_RESULT_CONTENT_LENGTH + return `${content.slice(0, end)}\n[truncated]` +} + function cooldownMillis(value: string | undefined, now: number) { if (!value?.trim()) return 60_000 const seconds = Number(value) diff --git a/packages/core/test/plugin/provider-opencode.test.ts b/packages/core/test/plugin/provider-opencode.test.ts index 180390bfb4b5..a84697d97290 100644 --- a/packages/core/test/plugin/provider-opencode.test.ts +++ b/packages/core/test/plugin/provider-opencode.test.ts @@ -17,6 +17,7 @@ import { ModelResolver } from "@opencode/core/model-resolver" import { Plugin } from "@opencode/core/plugin" import { PluginHost } from "@opencode/core/plugin/host" import { OpencodePlugin } from "@opencode/core/plugin/provider/opencode" +import { WebSearchResponse } from "@opencode/core/plugin/websearch/response" import { Provider } from "@opencode/core/provider" import { WebSearch } from "@opencode/core/websearch" import { withEnv } from "../fixture/env" @@ -805,7 +806,13 @@ describe("OpencodePlugin", () => { body?: unknown }> = [] const gate = Promise.withResolvers() - const state = { advertised: true, providerID: "opencode", waitForConfig: false } + const state = { + advertised: true, + providerID: "opencode", + waitForConfig: false, + unavailable: false, + oversized: false, + } const server = Bun.serve({ port: 0, fetch: async (request) => { @@ -832,6 +839,11 @@ describe("OpencodePlugin", () => { }) } if (path === "/api/websearch" || path === "/other/api/websearch") { + if (state.unavailable) + return Response.json( + { _tag: "ServiceUnavailableError", message: "Web search request failed: firecrawl" }, + { status: 503 }, + ) return Response.json({ providerID: state.providerID, results: [ @@ -841,6 +853,16 @@ describe("OpencodePlugin", () => { content: "Open source AI coding agent.", time: { published: 1_700_000_000_000 }, }, + ...(state.oversized + ? [ + { + url: "https://huge.example.com", + content: "x".repeat(2 * WebSearchResponse.MAX_BYTES), + time: {}, + }, + { url: "https://after.example.com", content: "after", time: {} }, + ] + : []), ], }) } @@ -927,6 +949,18 @@ describe("OpencodePlugin", () => { value: account("replacement"), }) + state.oversized = true + expect((yield* websearch.query({ query: "oversized" })).results.map((result) => result.url)).toEqual([ + "https://github.com/anomalyco/opencode", + ]) + state.oversized = false + + state.unavailable = true + expect((yield* websearch.query({ query: "unavailable" }).pipe(Effect.flip)).message).toBe( + "HTTP 503: Web search request failed: firecrawl", + ) + state.unavailable = false + state.providerID = "unexpected" expect((yield* websearch.query({ query: "wrong provider" }).pipe(Effect.flip))._tag).toBe("WebSearch.Request") diff --git a/packages/core/test/plugin/websearch.test.ts b/packages/core/test/plugin/websearch.test.ts index 775f576df5c5..815b4fc00581 100644 --- a/packages/core/test/plugin/websearch.test.ts +++ b/packages/core/test/plugin/websearch.test.ts @@ -5,6 +5,7 @@ import { WebSearch } from "@opencode/core/websearch" import { WebSearchExa } from "@opencode/core/plugin/websearch/exa" import { WebSearchFirecrawl } from "@opencode/core/plugin/websearch/firecrawl" import { WebSearchParallel } from "@opencode/core/plugin/websearch/parallel" +import { WebSearchResponse } from "@opencode/core/plugin/websearch/response" import { WebSearchTavily } from "@opencode/core/plugin/websearch/tavily" import { WebSearchTinyFish } from "@opencode/core/plugin/websearch/tinyfish" import { host, integrationHost, webSearchHost } from "./host" @@ -29,6 +30,8 @@ beforeEach(() => { }) const it = webSearchIntegrationTest +const sseMessage = (message: object) => + `event: message\ndata: ${JSON.stringify({ jsonrpc: "2.0", id: 1, ...message })}\n\n` describe("built-in web search providers", () => { ;[ @@ -93,6 +96,198 @@ describe("built-in web search providers", () => { }), ) + describe("responses larger than the size cap", () => { + const huge = "x".repeat(2 * WebSearchResponse.MAX_BYTES) + const mcpText = (text: string) => sseMessage({ result: { content: [{ type: "text", text }] } }) + const exaBlock = (url: string, title: string, content: string) => + `Title: ${title}\nURL: ${url}\nPublished: N/A\nAuthor: N/A\nHighlights:\n${content}` + const parallelSearch = { + search_id: "search_1", + results: [ + { url: "https://effect.website", title: "Effect", publish_date: null, excerpts: ["Effect documentation"] }, + { url: "https://huge.example.com", title: "Huge", publish_date: null, excerpts: [huge] }, + { url: "https://after.example.com", title: "After", publish_date: null, excerpts: ["after"] }, + ], + session_id: "ses_parallel", + } + ;[ + { + plugin: WebSearchExa.Plugin, + providerID: WebSearch.ID.make("exa"), + body: mcpText( + [ + exaBlock("https://effect.website", "Effect", "Effect documentation"), + exaBlock("https://huge.example.com", "Huge", huge), + exaBlock("https://after.example.com", "After", "after"), + ].join("\n\n---\n\n"), + ), + }, + { + plugin: WebSearchFirecrawl.Plugin, + providerID: WebSearch.ID.make("firecrawl"), + body: mcpText( + JSON.stringify({ + success: true, + data: { + web: [ + { url: "https://effect.website", title: "Effect", description: "Effect documentation" }, + { url: "https://huge.example.com", title: "Huge", description: huge }, + { url: "https://after.example.com", title: "After", description: "after" }, + ], + }, + }), + ), + }, + { + plugin: WebSearchParallel.Plugin, + providerID: WebSearch.ID.make("parallel"), + body: JSON.stringify({ + jsonrpc: "2.0", + id: 1, + result: { + content: [{ type: "text", text: JSON.stringify(parallelSearch) }], + structuredContent: parallelSearch, + }, + }), + }, + { + plugin: WebSearchTavily.Plugin, + providerID: WebSearch.ID.make("tavily"), + body: JSON.stringify({ + results: [ + { title: "Effect", url: "https://effect.website", content: "Effect documentation" }, + { title: "Huge", url: "https://huge.example.com", content: huge }, + { title: "After", url: "https://after.example.com", content: "after" }, + ], + }), + }, + { + plugin: WebSearchTinyFish.Plugin, + providerID: WebSearch.ID.make("tinyfish"), + body: mcpText( + JSON.stringify({ + results: [ + { title: "Effect", url: "https://effect.website", snippet: "Effect documentation" }, + { title: "Huge", url: "https://huge.example.com", snippet: huge }, + { title: "After", url: "https://after.example.com", snippet: "after" }, + ], + }), + ), + }, + ].forEach((provider) => { + it.effect(`keeps the complete results from ${provider.plugin.id}`, () => + Effect.gen(function* () { + resetWebSearchFixture(provider.body) + const integrations = yield* Integration.Service + const websearch = yield* WebSearch.Service + yield* provider.plugin.effect( + host({ integration: integrationHost(integrations), websearch: webSearchHost(websearch) }), + ) + + expect(yield* websearch.query({ query: "effect", providerID: provider.providerID })).toEqual( + new WebSearch.Response({ + providerID: provider.providerID, + results: [{ url: "https://effect.website", title: "Effect", content: "Effect documentation", time: {} }], + }), + ) + }), + ) + }) + }) + + it.effect("reports MCP tool errors instead of returning no results", () => + Effect.gen(function* () { + const integrations = yield* Integration.Service + const websearch = yield* WebSearch.Service + yield* WebSearchFirecrawl.Plugin.effect( + host({ integration: integrationHost(integrations), websearch: webSearchHost(websearch) }), + ) + const search = websearch.query({ query: "effect", providerID: WebSearch.ID.make("firecrawl") }).pipe(Effect.flip) + + resetWebSearchFixture( + sseMessage({ result: { isError: true, content: [{ type: "text", text: "Rate limit exceeded" }] } }), + ) + expect((yield* search).message).toBe("Rate limit exceeded") + resetWebSearchFixture(sseMessage({ error: { code: -32602, message: "Invalid arguments" } })) + expect((yield* search).message).toBe("Invalid arguments") + }), + ) + + it.effect("reports the provider's explanation for HTTP failures", () => + Effect.gen(function* () { + const integrations = yield* Integration.Service + const websearch = yield* WebSearch.Service + yield* WebSearchTinyFish.Plugin.effect( + host({ integration: integrationHost(integrations), websearch: webSearchHost(websearch) }), + ) + yield* WebSearchTavily.Plugin.effect( + host({ integration: integrationHost(integrations), websearch: webSearchHost(websearch) }), + ) + const quota = + "Free daily Search quota used (50/50). Sign up for continued access: https://agent.tinyfish.ai/sign-up" + + resetWebSearchFixture(JSON.stringify({ jsonrpc: "2.0", error: { code: -31001, message: quota }, id: 1 }), 401) + const tinyfish = yield* websearch + .query({ query: "effect", providerID: WebSearch.ID.make("tinyfish") }) + .pipe(Effect.flip) + expect(tinyfish.message).toBe(`HTTP 401: ${quota}`) + + resetWebSearchFixture(JSON.stringify({ detail: { error: "Unauthorized: missing or invalid API key." } }), 401) + const tavily = yield* websearch + .query({ query: "effect", providerID: WebSearch.ID.make("tavily") }) + .pipe(Effect.flip) + expect(tavily.message).toBe("HTTP 401: Unauthorized: missing or invalid API key.") + }), + ) + + it.effect("limits oversized Firecrawl results instead of failing the search", () => + Effect.gen(function* () { + const thread = "comment ".repeat(64 * 1024) + resetWebSearchFixture( + `event: message\ndata: ${JSON.stringify({ + jsonrpc: "2.0", + id: 1, + result: { + content: [ + { + type: "text", + text: JSON.stringify({ + success: true, + data: { + web: [ + { url: "https://news.ycombinator.com/item?id=1", title: "Thread", description: thread }, + { url: "https://effect.website", title: "Effect", description: "Effect documentation" }, + ], + }, + }), + }, + ], + }, + })}\n\n`, + ) + const integrations = yield* Integration.Service + const websearch = yield* WebSearch.Service + yield* WebSearchFirecrawl.Plugin.effect( + host({ integration: integrationHost(integrations), websearch: webSearchHost(websearch) }), + ) + + expect(yield* websearch.query({ query: "opencode", providerID: WebSearch.ID.make("firecrawl") })).toEqual( + new WebSearch.Response({ + providerID: WebSearch.ID.make("firecrawl"), + results: [ + { + url: "https://news.ycombinator.com/item?id=1", + title: "Thread", + content: `${thread.slice(0, WebSearch.MAX_RESULT_CONTENT_LENGTH)}\n[truncated]`, + time: {}, + }, + { url: "https://effect.website", title: "Effect", content: "Effect documentation", time: {} }, + ], + }), + ) + }), + ) + it.effect("registers Exa with its MCP schema", () => Effect.gen(function* () { const integrations = yield* Integration.Service diff --git a/packages/core/test/tool-websearch.test.ts b/packages/core/test/tool-websearch.test.ts index 7e3f8e8dd723..94dd5a58733e 100644 --- a/packages/core/test/tool-websearch.test.ts +++ b/packages/core/test/tool-websearch.test.ts @@ -1,6 +1,6 @@ import { describe, expect } from "bun:test" import { Context, Effect, Layer } from "effect" -import type { HttpClientError } from "effect/unstable/http" +import { HttpClientError, HttpClientRequest } from "effect/unstable/http" import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder" import { LayerNode } from "@opencode/util/effect/layer-node" import { Permission } from "@opencode/core/permission" @@ -44,7 +44,7 @@ class Fixture { formResponse: Form.TerminalState = { status: "cancelled" } formResponses: Form.TerminalState[] = [] formWait = Effect.void - error: HttpClientError.HttpClientError | undefined + error: HttpClientError.HttpClientError | Error | undefined results: readonly WebSearch.Result[] = [ { url: "https://example.com", title: "Search results", content: "search results", time: {} }, ] @@ -62,7 +62,7 @@ const setup = Effect.gen(function* () { execute: () => Effect.gen(function* () { fixture.events.push("query") - if (fixture.error) return yield* fixture.error + if (fixture.error) return yield* Effect.fail(fixture.error) return fixture.results }), }), @@ -418,7 +418,7 @@ describe("WebSearchTool registration", () => { .pipe(Effect.flip) expect(toSessionError(error)).toEqual({ type: "tool.execution", - message: "Web search rate limited (HTTP 429)", + message: `Unable to search the web for ${query} (${error.metadata?.provider}): Rate limited (HTTP 429)`, }) expect(error.metadata).toMatchObject({ provider: expect.stringMatching(/^(exa|parallel)$/) }) }), @@ -437,9 +437,9 @@ describe("WebSearchTool registration", () => { yield* Effect.forEach( [ - { status: 403, message: "Web search request failed (HTTP 403)" }, - { status: 429, message: "Web search rate limited (HTTP 429)" }, - { status: 401, message: "Web search authentication failed (HTTP 401)" }, + { status: 403, message: "Unable to search the web for effect (exa): Request failed (HTTP 403)" }, + { status: 429, message: "Unable to search the web for effect (exa): Rate limited (HTTP 429)" }, + { status: 401, message: "Unable to search the web for effect (exa): Authentication failed (HTTP 401)" }, ], ({ status, message }, index) => Effect.gen(function* () { @@ -469,4 +469,58 @@ describe("WebSearchTool registration", () => { ) }), ) + + it.effect("reports the underlying reason for provider failures", () => + Effect.gen(function* () { + const fixture = yield* setup + const tools = yield* fixture.registry.snapshot() + yield* fixture.websearch.select(WebSearch.ID.make("exa")) + + yield* Effect.forEach( + [ + { + error: new Error("web_search_exa request timed out"), + message: "Unable to search the web for effect (exa): web_search_exa request timed out", + }, + { + error: new HttpClientError.HttpClientError({ + reason: new HttpClientError.TransportError({ + request: HttpClientRequest.post("https://mcp.exa.ai/mcp?exaApiKey=secret"), + cause: new TypeError("Unable to connect. Is the computer able to access the url?"), + }), + }), + message: + "Unable to search the web for effect (exa): Request failed: Unable to connect. Is the computer able to access the url?", + }, + ], + (item, index) => + Effect.gen(function* () { + fixture.error = item.error + const error = yield* tools + .execute({ + sessionID, + ...toolIdentity, + call: { type: "tool-call", id: `call-reason-${index}`, name: "websearch", input: { query: "effect" } }, + }) + .pipe(Effect.flip) + expect(toSessionError(error)).toEqual({ type: "tool.execution", message: item.message }) + expect(error.metadata).toEqual({ provider: "exa" }) + }), + { discard: true }, + ) + + yield* fixture.websearch.select(false) + const disabled = yield* tools + .execute({ + sessionID, + ...toolIdentity, + call: { type: "tool-call", id: "call-disabled", name: "websearch", input: { query: "effect" } }, + }) + .pipe(Effect.flip) + expect(toSessionError(disabled)).toEqual({ + type: "tool.execution", + message: "Unable to search the web for effect: Web search is disabled", + }) + }), + ) }) diff --git a/packages/core/test/websearch.test.ts b/packages/core/test/websearch.test.ts index 59f0c32d1055..162c6ee9b61e 100644 --- a/packages/core/test/websearch.test.ts +++ b/packages/core/test/websearch.test.ts @@ -86,6 +86,34 @@ describe("WebSearch", () => { }), ) + it.effect("limits each result's content for every provider", () => + Effect.gen(function* () { + const websearch = yield* WebSearch.Service + const providerID = WebSearch.ID.make("large") + const limit = WebSearch.MAX_RESULT_CONTENT_LENGTH + yield* websearch.transform((editor) => { + editor.add({ + id: providerID, + name: "Large", + execute: () => + Effect.succeed([ + { url: "https://large.example.com", content: "b".repeat(limit + 1), time: {} }, + { url: "https://emoji.example.com", content: `${"a".repeat(limit - 1)}😀 after the limit`, time: {} }, + { url: "https://exact.example.com", content: "c".repeat(limit), time: {} }, + { url: "https://empty.example.com", time: {} }, + ]), + }) + }) + + expect((yield* websearch.query({ query: "large", providerID })).results).toEqual([ + { url: "https://large.example.com", content: `${"b".repeat(limit)}\n[truncated]`, time: {} }, + { url: "https://emoji.example.com", content: `${"a".repeat(limit - 1)}\n[truncated]`, time: {} }, + { url: "https://exact.example.com", content: "c".repeat(limit), time: {} }, + { url: "https://empty.example.com", time: {} }, + ]) + }), + ) + it.effect("requires a provider when no default is set", () => Effect.gen(function* () { yield* register("exa")