fix(core): make web search limits non-fatal and report real errors

packages/core/src/plugin/provider/opencode.ts

@@ -1,4 +1,4 @@
-import { Duration, Effect, Equal, Schema, Semaphore, Stream } from "effect"
+import { Duration, Effect, Equal, Option, Schema, Semaphore, Stream } from "effect"
 import type { Scope } from "effect"
 import type { IntegrationOAuthMethodRegistration } from "@opencode/plugin/effect/integration"
 import { define } from "@opencode/plugin/effect/plugin"
@@ -11,6 +11,7 @@ import { IntegrationConnection } from "../../integration/connection.js"
 import { ManagedPolicy } from "../../managed-policy.js"
 import { Provider } from "../../provider.js"
 import { WebSearch } from "../../websearch.js"
+import { WebSearchResponse } from "../websearch/response.js"
 import { ConfigPolicy } from "@opencode/schema/config/policy"
 import { ConfigProvider } from "@opencode/schema/config/provider"
 import { Money } from "@opencode/schema/money"
@@ -45,6 +46,9 @@ const TokenPending = Schema.Struct({ error: Schema.String })
 const DeviceToken = Schema.Union([Token, TokenPending])
 const User = Schema.Struct({ id: Schema.String, email: Schema.String })
 const Org = Schema.Struct({ id: Schema.String, name: Schema.String })
+const decodeWebSearchResponse = Schema.decodeUnknownOption(
+  Schema.Struct({ providerID: WebSearch.ID, results: Schema.Array(Schema.Unknown) }),
+)
 
 function oauth(http: HttpClient.HttpClient) {
   return {
@@ -313,25 +317,27 @@ export const OpencodePlugin = define<HttpClient.HttpClient | Bus.Service | Manag
                 providerID: descriptor.providerID,
               }),
             )
-            const response = yield* HttpClient.withScope(HttpClient.filterStatusOk(http))
-              .execute(request)
-              .pipe(
-                Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }),
-                Effect.flatMap(HttpClientResponse.schemaBodyJson(WebSearch.Response)),
-                Effect.scoped,
-                Effect.timeoutOrElse({
-                  duration: Duration.seconds(25),
-                  orElse: () => Effect.fail(new Error("OpenCode web search request timed out")),
-                }),
-              )
-            if (response.providerID !== descriptor.providerID) {
+            const body = yield* WebSearchResponse.execute(http, request).pipe(
+              Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }),
+              Effect.scoped,
+              Effect.timeoutOrElse({
+                duration: Duration.seconds(25),
+                orElse: () => Effect.fail(new Error("OpenCode web search request timed out")),
+              }),
+            )
+            const response = WebSearchResponse.json(body.text, body.truncated).pipe(
+              Option.flatMap(decodeWebSearchResponse),
+            )
+            if (Option.isNone(response))
+              return yield* Effect.fail(new Error("OpenCode web search returned an invalid response"))
+            if (response.value.providerID !== descriptor.providerID) {
               return yield* Effect.fail(
                 new Error(
-                  `OpenCode web search returned provider ${response.providerID} instead of ${descriptor.providerID}`,
+                  `OpenCode web search returned provider ${response.value.providerID} instead of ${descriptor.providerID}`,
                 ),
               )
             }
-            return response.results
+            return WebSearchResponse.items(WebSearch.Result, response.value.results, body.truncated)
           }),
       })
       editor.default.set(descriptor.providerID)

packages/core/src/plugin/websearch/exa.ts

@@ -47,23 +47,25 @@ export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
             const credential = connection ? yield* ctx.integration.connection.resolve(connection) : undefined
             const url = new URL(endpoint)
             if (credential?.type === "key") url.searchParams.set("exaApiKey", credential.key)
-            const result = yield* WebSearchMcp.call(
+            const response = yield* WebSearchMcp.call(
               http,
               url.toString(),
               "web_search_exa",
               { input: McpInput, output: McpOutput },
               { query: input.query, numResults: 8 },
             )
-            const content = result?.content.find((item) => item.text)
-            return content ? parseResults(content.text) : []
+            const content = response.result?.content.find((item) => item.text)
+            return content ? parseResults(content.text, response.truncated) : []
           }),
       })
     })
   }),
 })
 
-function parseResults(text: string) {
-  return text.split(/\n\n---\n\n/).flatMap((block) => {
+function parseResults(text: string, truncated: boolean) {
+  const blocks = text.split(/\n\n---\n\n/)
+  // The final block of a truncated response may end mid-line, including inside its URL.
+  return (truncated ? blocks.slice(0, -1) : blocks).flatMap((block) => {
     const url = block.match(/^URL:\s*(.+)$/m)?.[1]?.trim()
     if (!url) return []
     const title = block.match(/^Title:\s*(.+)$/m)?.[1]?.trim()

packages/core/src/plugin/websearch/firecrawl.ts

@@ -5,6 +5,7 @@ import { Effect, Option, Schema, Scope } from "effect"
 import { HttpClient } from "effect/unstable/http"
 import { App } from "../../app.js"
 import { WebSearchMcp } from "./mcp.js"
+import { WebSearchResponse } from "./response.js"
 
 export const endpoint = "https://mcp.firecrawl.dev/v2/mcp"
 
@@ -17,21 +18,15 @@ const McpOutput = Schema.Struct({
   content: Schema.Array(Schema.Struct({ type: Schema.Literal("text"), text: Schema.String })),
 })
 
-const SearchResponse = Schema.fromJsonString(
-  Schema.Struct({
-    success: Schema.Boolean,
-    data: Schema.Struct({
-      web: Schema.Array(
-        Schema.Struct({
-          url: Schema.String,
-          title: Schema.NullOr(Schema.String).pipe(Schema.optional),
-          description: Schema.NullOr(Schema.String).pipe(Schema.optional),
-        }),
-      ),
-    }),
-  }),
+const decodeSearchResponse = Schema.decodeUnknownOption(
+  Schema.Struct({ data: Schema.Struct({ web: Schema.Array(Schema.Unknown) }) }),
 )
-const decodeSearchResponse = Schema.decodeUnknownOption(SearchResponse)
+
+const SearchResult = Schema.Struct({
+  url: Schema.String,
+  title: Schema.NullOr(Schema.String).pipe(Schema.optional),
+  description: Schema.NullOr(Schema.String).pipe(Schema.optional),
+})
 
 export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
   id: "opencode.websearch.firecrawl",
@@ -56,7 +51,7 @@ export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
           Effect.gen(function* () {
             const connection = yield* ctx.integration.connection.active("firecrawl")
             const credential = connection ? yield* ctx.integration.connection.resolve(connection) : undefined
-            const result = yield* WebSearchMcp.call(
+            const response = yield* WebSearchMcp.call(
               http,
               endpoint,
               "firecrawl_search",
@@ -67,16 +62,18 @@ export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
                 ...(credential?.type === "key" ? { Authorization: `Bearer ${credential.key}` } : {}),
               },
             )
-            const content = result?.content.find((item) => item.text)
-            const response = content ? Option.getOrUndefined(decodeSearchResponse(content.text)) : undefined
-            return (
-              response?.data.web.map((item) => ({
-                url: item.url,
-                ...(item.title ? { title: item.title } : {}),
-                ...(item.description ? { content: item.description } : {}),
-                time: {},
-              })) ?? []
-            )
+            const content = response.result?.content.find((item) => item.text)
+            const search = content
+              ? Option.getOrUndefined(
+                  WebSearchResponse.json(content.text, response.truncated).pipe(Option.flatMap(decodeSearchResponse)),
+                )
+              : undefined
+            return WebSearchResponse.items(SearchResult, search?.data.web ?? [], response.truncated).map((item) => ({
+              url: item.url,
+              ...(item.title ? { title: item.title } : {}),
+              ...(item.description ? { content: item.description } : {}),
+              time: {},
+            }))
           }),
       })
     })

packages/core/src/plugin/websearch/mcp.ts

@@ -1,26 +1,22 @@
 export * as WebSearchMcp from "./mcp.js"
 
-import { Duration, Effect, Schema } from "effect"
+import { Duration, Effect, Option, Schema } from "effect"
 import { HttpClient, HttpClientRequest } from "effect/unstable/http"
-import { collectBoundedResponseBody } from "../../tool/http-body.js"
+import { WebSearchResponse } from "./response.js"
 
-export const MAX_RESPONSE_BYTES = 256 * 1024
-
-export const parseResponse = <F extends Schema.Struct.Fields>(body: string, result: Schema.Struct<F>) => {
-  const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Struct({ result })))
-  const parse = (payload: string) => {
-    const trimmed = payload.trim()
-    if (!trimmed.startsWith("{")) return Effect.undefined
-    return decode(trimmed).pipe(Effect.map((response) => response.result))
-  }
+export const parseResponse = <F extends Schema.Struct.Fields>(
+  body: { readonly text: string; readonly truncated: boolean },
+  tool: string,
+  result: Schema.Struct<F>,
+) => {
+  const decode = Schema.decodeUnknownEffect(Schema.Struct({ result }))
   return Effect.gen(function* () {
-    const trimmed = body.trim()
-    const direct = trimmed ? yield* parse(trimmed) : undefined
-    if (direct) return direct
-    for (const line of body.split("\n")) {
-      if (!line.startsWith("data: ")) continue
-      const data = yield* parse(line.substring(6))
-      if (data) return data
+    for (const payload of WebSearchResponse.payloads(body.text)) {
+      const message = WebSearchResponse.json(payload, body.truncated)
+      if (Option.isNone(message)) return yield* Effect.fail(new Error(`${tool} returned invalid JSON`))
+      const failure = WebSearchResponse.failure(message.value)
+      if (failure !== undefined) return yield* Effect.fail(new Error(failure))
+      return (yield* decode(message.value)).result
     }
   })
 }
@@ -52,13 +48,8 @@ export const call = <F extends Schema.Struct.Fields, R extends Schema.Struct.Fie
       }),
     )
     return yield* Effect.gen(function* () {
-      const response = yield* HttpClient.withScope(HttpClient.filterStatusOk(http)).execute(request)
-      const body = yield* collectBoundedResponseBody(
-        response,
-        MAX_RESPONSE_BYTES,
-        () => new Error(`${tool} response exceeded ${MAX_RESPONSE_BYTES} bytes`),
-      )
-      return yield* parseResponse(body.toString("utf8"), schema.output)
+      const body = yield* WebSearchResponse.execute(http, request)
+      return { result: yield* parseResponse(body, tool, schema.output), truncated: body.truncated }
     }).pipe(
       Effect.scoped,
       Effect.timeoutOrElse({

packages/core/src/plugin/websearch/parallel.ts

@@ -1,10 +1,11 @@
 export * as WebSearchParallel from "./parallel.js"
 
 import { define } from "@opencode/plugin/effect/plugin"
-import { Effect, Schema, Scope } from "effect"
+import { Effect, Option, Schema, Scope } from "effect"
 import { HttpClient } from "effect/unstable/http"
 import { App } from "../../app.js"
 import { WebSearchMcp } from "./mcp.js"
+import { WebSearchResponse } from "./response.js"
 
 export const endpoint = "https://search.parallel.ai/mcp"
 
@@ -14,38 +15,19 @@ const McpInput = Schema.Struct({
   model_name: Schema.String.check(Schema.isMaxLength(100)).pipe(Schema.optional),
 })
 
-const SearchResponse = Schema.Struct({
-  search_id: Schema.String,
-  results: Schema.Array(
-    Schema.Struct({
-      url: Schema.String,
-      title: Schema.NullOr(Schema.String).pipe(Schema.optional),
-      publish_date: Schema.NullOr(Schema.String).pipe(Schema.optional),
-      excerpts: Schema.Array(Schema.String),
-    }),
-  ),
-  warnings: Schema.NullOr(
-    Schema.Array(
-      Schema.Struct({
-        type: Schema.Literals(["spec_validation_warning", "input_validation_warning", "warning"]),
-        message: Schema.String,
-        detail: Schema.NullOr(Schema.Record(Schema.String, Schema.Json)).pipe(Schema.optional),
-      }),
-    ),
-  ).pipe(Schema.optional),
-  usage: Schema.NullOr(
-    Schema.Array(
-      Schema.Struct({
-        name: Schema.String,
-        count: Schema.Int,
-      }),
-    ),
-  ).pipe(Schema.optional),
-  session_id: Schema.String,
-})
 const McpOutput = Schema.Struct({
   content: Schema.Array(Schema.Struct({ type: Schema.Literal("text"), text: Schema.String })),
-  structuredContent: SearchResponse,
+  // Parallel sends this after a text copy of the same JSON, so a truncated response can lose it.
+  structuredContent: Schema.Unknown.pipe(Schema.optional),
+})
+
+const decodeSearchResponse = Schema.decodeUnknownOption(Schema.Struct({ results: Schema.Array(Schema.Unknown) }))
+
+const SearchResult = Schema.Struct({
+  url: Schema.String,
+  title: Schema.NullOr(Schema.String).pipe(Schema.optional),
+  publish_date: Schema.NullOr(Schema.String).pipe(Schema.optional),
+  excerpts: Schema.Array(Schema.String),
 })
 
 export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
@@ -71,7 +53,7 @@ export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
           Effect.gen(function* () {
             const connection = yield* ctx.integration.connection.active("parallel")
             const credential = connection ? yield* ctx.integration.connection.resolve(connection) : undefined
-            const result = yield* WebSearchMcp.call(
+            const response = yield* WebSearchMcp.call(
               http,
               endpoint,
               "web_search",
@@ -85,17 +67,24 @@ export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
                 ...(credential?.type === "key" ? { Authorization: `Bearer ${credential.key}` } : {}),
               },
             )
-            return (
-              result?.structuredContent.results.map((item) => {
-                const published = item.publish_date ? Date.parse(item.publish_date) : undefined
-                return {
-                  url: item.url,
-                  ...(item.title ? { title: item.title } : {}),
-                  ...(item.excerpts.length ? { content: item.excerpts.join("\n\n") } : {}),
-                  time: { ...(published !== undefined && Number.isFinite(published) ? { published } : {}) },
-                }
-              }) ?? []
+            const content = response.result?.content.find((item) => item.text)
+            const search = Option.getOrUndefined(
+              decodeSearchResponse(
+                response.result?.structuredContent ??
+                  (content
+                    ? Option.getOrUndefined(WebSearchResponse.json(content.text, response.truncated))
+                    : undefined),
+              ),
             )
+            return WebSearchResponse.items(SearchResult, search?.results ?? [], response.truncated).map((item) => {
+              const published = item.publish_date ? Date.parse(item.publish_date) : undefined
+              return {
+                url: item.url,
+                ...(item.title ? { title: item.title } : {}),
+                ...(item.excerpts.length ? { content: item.excerpts.join("\n\n") } : {}),
+                time: { ...(published !== undefined && Number.isFinite(published) ? { published } : {}) },
+              }
+            })
           }),
       })
     })