diff --git a/src/providers/registry-transport.ts b/src/providers/registry-transport.ts new file mode 100644 index 00000000000..6897a63291f --- /dev/null +++ b/src/providers/registry-transport.ts @@ -0,0 +1,112 @@ +import type { OcxProviderConfig } from "../types"; +import { + PROVIDER_REGISTRY, + getProviderRegistryEntry, + normalizedProviderEndpoint, + providerMatchesRegistryTransport, +} from "./registry"; +import type { ProviderRegistryEntry } from "./registry/types"; + +/** + * `providerMatchesRegistryTransport` for a configured name that may be a generated-metadata + * ALIAS rather than a registry id. + * + * A registry row claims extra names through `extraMetadataAliases` (`gemini` for `google`, + * `anthropic-key` for `anthropic-apikey`, ...), and `resolveMetadataProvider` resolves those + * names — case-folded, the way saved provider keys arrive — to the row's metadata bundle. A + * provider saved under an alias is owned by the declaring entry, so its transport must be + * validated against that entry; an id-only lookup finds no `gemini` row and would drop a + * verdict the registry still owns. + * + * Routing binds a name to a registry transport by exact id only — `routedProviderConfig` does + * a case-sensitive `entry.id === providerName` lookup — so an alias- or case-named row keeps + * its configured destination, and its configured adapter, auth mode, and normalized endpoint + * must literally equal one of the entry's declared destinations: its fixed transport, a + * documented `baseUrlChoices` endpoint, or a `destinationAliases` former endpoint that still + * answers for the row. An exact id is canonicalized instead: routing overwrites the adapter + * and derives the auth mode, and an `allowBaseUrlOverride` preset keeps only its configured + * URL, so ownership there is proven by the endpoint alone. A `preserveCustomDestination` + * preset's stored row is not canonicalized, so it again needs a literal match. Reusing the + * owner's pinning rule or an arbitrary URL would apply vendor verdicts to destinations + * routing still serves as custom. + */ +export function providerMatchesRegistryTransportOrAlias( + name: string, + provider: Pick & Partial>, +): boolean { + const exact = getProviderRegistryEntry(name); + if (exact !== undefined) { + // Routing discards the configured URL for a pinned name, so its mismatch under the + // pinned rule cannot move the wire. + if (exact.allowBaseUrlOverride !== true && exact.preserveCustomDestination !== true) { + return providerMatchesRegistryTransport(name, provider); + } + // A stored row on a preserved preset is not canonicalized — the configured adapter, + // auth mode, and endpoint all reach the wire, so each must equal a declared destination. + if (exact.preserveCustomDestination === true) { + return configuredTransportMatchesDeclaredDestinations(exact, provider); + } + // Routing canonicalizes the adapter to `entry.adapter` and derives the auth mode for a + // transport-matched row, preserving only the configured URL on an overridable preset; + // the destination the wire reaches is therefore the entry's own whenever the configured + // endpoint is one the entry declares. + return configuredEndpointIsDeclaredDestination(exact, provider); + } + const lower = name.toLowerCase(); + const owner = PROVIDER_REGISTRY.find(row => + row.id.toLowerCase() === lower + || (row.extraMetadataAliases ?? []).some(alias => alias.toLowerCase() === lower)); + return owner !== undefined && configuredTransportMatchesDeclaredDestinations(owner, provider); +} + +/** + * The destinations a registry row declares as its own: its fixed transport (skipped when + * the URL is a template, which no saved row can equal), documented `baseUrlChoices` + * endpoints (a "custom" choice declares no URL and cannot match), and `destinationAliases` + * former endpoints on their own adapters. + */ +function declaredDestinations(entry: ProviderRegistryEntry): { adapter: string; baseUrl: string }[] { + const declared = [ + ...(entry.destinationAliases ?? []), + ...(entry.baseUrlChoices ?? []).flatMap(choice => + choice.baseUrl === undefined ? [] : [{ adapter: entry.adapter, baseUrl: choice.baseUrl }]), + ]; + if (!/\{[^}]*\}/.test(entry.baseUrl)) { + declared.push({ adapter: entry.adapter, baseUrl: entry.baseUrl }); + } + return declared; +} + +/** + * Whether a row's configured transport literally equals one of `entry`'s declared + * destinations on the destination's own adapter. Used for names routing does not pin and + * for preserved presets whose stored row is the wire: the destination the request actually + * reaches must be one the registry row owns for generated vendor verdicts to apply. + */ +function configuredTransportMatchesDeclaredDestinations( + entry: ProviderRegistryEntry, + provider: Pick & Partial>, +): boolean { + if (typeof provider.baseUrl !== "string") return false; + // An unset authMode is the legacy key default, so it can only satisfy a key-auth owner. + if ((provider.authMode ?? "key") !== entry.authKind) return false; + const endpoint = normalizedProviderEndpoint(provider.baseUrl); + return declaredDestinations(entry).some(target => + target.adapter === provider.adapter && normalizedProviderEndpoint(target.baseUrl) === endpoint); +} + +/** + * Whether a row's configured endpoint is one of `entry`'s declared destinations. Used for + * transport-matched exact ids on overridable presets: routing overwrites the adapter with + * `entry.adapter` and derives the auth mode, so only the URL distinguishes a canonicalized + * row from a retargeted one. + */ +function configuredEndpointIsDeclaredDestination( + entry: ProviderRegistryEntry, + provider: Pick & Partial>, +): boolean { + if (typeof provider.baseUrl !== "string") return false; + const endpoint = normalizedProviderEndpoint(provider.baseUrl); + return declaredDestinations(entry).some(target => + normalizedProviderEndpoint(target.baseUrl) === endpoint); +} diff --git a/src/providers/registry.ts b/src/providers/registry.ts index 61c08eef7bc..97ef18e58ad 100644 --- a/src/providers/registry.ts +++ b/src/providers/registry.ts @@ -81,7 +81,7 @@ export function registryModelServiceTierCapabilityApplies( return guard === undefined || guard(provider.baseUrl); } -function normalizedProviderEndpoint(value: string): string { +export function normalizedProviderEndpoint(value: string): string { const trimmed = value.trim(); try { const parsed = new URL(trimmed); diff --git a/src/vision/eligibility.ts b/src/vision/eligibility.ts index d772c8f9769..8ee3e8c75b7 100644 --- a/src/vision/eligibility.ts +++ b/src/vision/eligibility.ts @@ -26,6 +26,7 @@ import { getModelMetadataCaseInsensitive, resolveMetadataProvider } from "../gen import { nativeInputModalities } from "../codex/catalog/metadata"; import { SUPPORTED_NATIVE_OPENAI_SLUGS } from "../codex/catalog/native-models"; import { enrichProviderFromRegistry } from "../providers/derive"; +import { providerMatchesRegistryTransportOrAlias } from "../providers/registry-transport"; import { isCanonicalOpenAiForwardProvider } from "../providers/openai-tiers-destination"; /** @@ -152,7 +153,10 @@ function advertisesImageInput(modalities: readonly string[] | undefined): boolea /** Vendor-table modalities for a routed row, or undefined when the table has no opinion. */ function metadataImageInput(provider: string, modelId: string): boolean | undefined { - const resolved = resolveMetadataProvider(provider) ?? provider; + // Bundle keys are lowercase, so a case-varied configured name (e.g. `ZAI`) folds the same + // way resolveMetadataProvider folds its aliases; the transport guard above decides whether + // that bundle is allowed to speak for the destination at all. + const resolved = resolveMetadataProvider(provider) ?? provider.toLowerCase(); const meta = getModelMetadataCaseInsensitive(resolved, modelId); return advertisesImageInput(meta?.input); } @@ -280,6 +284,19 @@ function modelAcceptsImageInputWithCache( } const fromRow = advertisesImageInput(candidate.inputModalities); if (fromRow !== undefined) return fromRow; + // A preset name is not transport identity. Routing binds a name to a registry transport by + // exact id only, so vendor metadata is authoritative only while the configured adapter and + // endpoint still belong to the registry row that owns that name — where "owns" includes + // canonical metadata aliases like `gemini` or `anthropic-key`, resolved to the entry that + // declares them, and where "belong" means the configured adapter/auth/endpoint literally + // equals a declared destination (the fixed transport, a `baseUrlChoices` endpoint, or a + // `destinationAliases` former endpoint) rather than reusing the owner's pinning rule. An + // `allowBaseUrlOverride` exact id is bound to the same declared set but only on the + // endpoint, because routing canonicalizes its adapter and auth and preserves just the + // configured URL; a `preserveCustomDestination` row is compared literally, because + // routing serves the stored row unchanged. Otherwise the capability is unknown and + // request dispatch must preserve the custom destination's image boundary. + if (provider !== undefined && !providerMatchesRegistryTransportOrAlias(candidate.provider, provider)) return undefined; return metadataImageInput(candidate.provider, candidate.id); } diff --git a/structure/runtime.md b/structure/runtime.md index 438e95547c6..f3781cd9ecd 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -458,7 +458,7 @@ declare `modelInputModalities: ["text", "image"]` per model for the nine Claude explicit operator overrides; unknown models receive no new declaration. Client eligibility filters and Anthropic image wire handling remain unchanged. -`src/vision/plan.ts` prevents raw image bytes from reaching any target whose effective capability is positively known to exclude image input. Evidence is consulted highest-first: `modelCapabilities`, an explicit custom row for the same routed identity, `noVisionModels`, an explicit per-model modality list without `image`, then backend-specific/registry/vendor metadata. A proven text-only target is preprocessed through the configured Vision Sidecar; a positively image-capable target receives the image directly. Genuinely unknown custom models retain the existing compatibility path rather than being guessed text-only. +`src/vision/plan.ts` prevents raw image bytes from reaching any target whose effective capability is positively known to exclude image input. Evidence is consulted highest-first: `modelCapabilities`, an explicit custom row for the same routed identity, `noVisionModels`, an explicit per-model modality list without `image`, then backend-specific/registry/vendor metadata. Registry/vendor metadata applies to a configured provider only while its adapter and destination still match the transport of the registry row owning its name — canonical metadata aliases such as `gemini` resolve to the entry that declares them and must then equal one of the entry's declared destinations (fixed transport, documented `baseUrlChoices`, or `destinationAliases` former endpoints) on that destination's own adapter, because routing binds transports by exact registry id only; preset rows are validated against the same declared set — `allowBaseUrlOverride` ids on the endpoint only (routing canonicalizes their adapter and auth), `preserveCustomDestination` rows literally (routing serves the stored row unchanged) — and a custom endpoint remains unknown. A proven text-only target is preprocessed through the configured Vision Sidecar; a positively image-capable target receives the image directly. Genuinely unknown custom models retain the existing compatibility path rather than being guessed text-only. Canonical ChatGPT Codex forwarding uses the generated `openai-codex` capability bundle rather than the public `openai` bundle. This matters when the two backends differ: for example, the vendored metadata records `gpt-5.3-codex-spark` as text-only on `openai-codex` while the public OpenAI row lists image input. The native Chat fast path and web-search image verbalization consume the same effective-capability decision. diff --git a/tests/vision/vision-eligibility.test.ts b/tests/vision/vision-eligibility.test.ts index 28baa98b9ab..a4c56be4a25 100644 --- a/tests/vision/vision-eligibility.test.ts +++ b/tests/vision/vision-eligibility.test.ts @@ -10,6 +10,7 @@ import { visionEligibleModelOptions, type VisionCandidateModel, } from "../../src/vision/eligibility"; +import { requiresVisionPreprocessing } from "../../src/vision/plan"; const emptyConfig: Pick = { providers: {} }; @@ -189,6 +190,169 @@ describe("vision eligibility core", () => { expect(modelAcceptsImageInput(config, { provider: "runtime", id: "vision" })).toBe(true); }); + test("11d. vendor metadata does not cross a preserved custom destination boundary", () => { + const provider = { + adapter: "openai-responses", + authMode: "key", + baseUrl: "https://operator-gateway.example/v1", + } as const; + const config = configWithProviders({ "zhipu-bigmodel-responses": provider }); + const candidate = { provider: "zhipu-bigmodel-responses", id: "glm-5.3" }; + + // The generated Z.AI bundle calls glm-5.3 text-only, but this preset explicitly permits a + // same-named custom endpoint. Its images must neither leave for a sidecar nor be stripped. + expect(modelAcceptsImageInput(config, candidate)).toBeUndefined(); + expect(requiresVisionPreprocessing(config, provider, candidate.id, candidate.provider)).toBe(false); + }); + + test("11e. a canonical metadata alias keeps the owning registry row's verdict", () => { + // `gemini` is an extraMetadataAlias of the `google` registry row — a provider saved under + // that name resolves to the google bundle but owns no registry id of its own. An id-only + // transport check dropped the verdict entirely, and a generated text-only model silently + // degraded to "unknown" (eligible, no sidecar). + const provider = { + adapter: "google", + authMode: "key", + baseUrl: "https://generativelanguage.googleapis.com", + } as const; + const config = configWithProviders({ gemini: provider }); + const candidate = { provider: "gemini", id: "gemini-live-2.5-flash-preview-native-audio" }; + + expect(modelAcceptsImageInput(config, candidate)).toBe(false); + expect(requiresVisionPreprocessing(config, provider, candidate.id, candidate.provider)).toBe(true); + + // Saved provider keys are case-folded by resolveMetadataProvider; the transport binding + // must fold the same way or a title-cased alias would lose the same verdict. + const folded = configWithProviders({ Gemini: provider }); + expect(modelAcceptsImageInput(folded, { provider: "Gemini", id: candidate.id })).toBe(false); + }); + + test("11f. an alias-named custom destination is not bound by the owning entry's pinning", () => { + // `google` is a name-pinned preset: routing discards a custom baseUrl saved under that + // exact id, so the vendor verdict applies regardless. `gemini` only ALIASES the google + // bundle — routing preserves its configured destination, so at a custom gateway the + // capability is unknown and the image boundary is preserved, while the pinned `google` + // row keeps the text-only verdict. + const provider = { + adapter: "google", + authMode: "key", + baseUrl: "https://operator-gateway.example/google", + } as const; + const config = configWithProviders({ gemini: provider, google: provider }); + const id = "gemini-live-2.5-flash-preview-native-audio"; + + expect(modelAcceptsImageInput(config, { provider: "google", id })).toBe(false); + expect(modelAcceptsImageInput(config, { provider: "gemini", id })).toBeUndefined(); + expect(isVisionEligibleModel(config, { provider: "gemini", id })).toBe(true); + expect(requiresVisionPreprocessing(config, provider, id, "gemini")).toBe(false); + }); + + test("11g. an alias-named row on a different wire or auth mode is not the owner's transport", () => { + // A custom `gemini` fronting an OpenAI-shaped gateway keeps its own transport end to end: + // the google bundle cannot strip or redirect its images even though the name resolves. + const openaiShaped = { + adapter: "openai-chat", + authMode: "key", + baseUrl: "https://operator.example/v1", + } as const; + const config = configWithProviders({ gemini: openaiShaped }); + const id = "gemini-live-2.5-flash-preview-native-audio"; + + expect(modelAcceptsImageInput(config, { provider: "gemini", id })).toBeUndefined(); + expect(requiresVisionPreprocessing(config, openaiShaped, id, "gemini")).toBe(false); + + // Same name and endpoint as 11e but forward auth: not the key-auth google transport, so + // the verdict cannot speak for this destination either. + const forwardAuth = { + adapter: "google", + authMode: "forward", + baseUrl: "https://generativelanguage.googleapis.com", + } as const; + const forwarded = configWithProviders({ gemini: forwardAuth }); + expect(modelAcceptsImageInput(forwarded, { provider: "gemini", id })).toBeUndefined(); + expect(requiresVisionPreprocessing(forwarded, forwardAuth, id, "gemini")).toBe(false); + }); + + test("11h. a case-varied name is bound by the same transport rule as an alias", () => { + // Routing is case-sensitive, so `Gemini` is custom too: at a custom endpoint it keeps its + // image boundary (mirroring 11f), and only the canonical endpoint keeps the verdict (11e). + const custom = { + adapter: "google", + authMode: "key", + baseUrl: "https://operator-gateway.example/google", + } as const; + const config = configWithProviders({ Gemini: custom }); + const id = "gemini-live-2.5-flash-preview-native-audio"; + + expect(modelAcceptsImageInput(config, { provider: "Gemini", id })).toBeUndefined(); + expect(requiresVisionPreprocessing(config, custom, id, "Gemini")).toBe(false); + }); + + test("11i. an override preset keeps vendor verdicts only at declared endpoints", () => { + // `moonshot` sets allowBaseUrlOverride + baseUrlChoices, so routing honors a configured + // URL — including the "custom" choice. The generated moonshot bundle can therefore only + // speak while the configured endpoint is a declared one (the preset default or a listed + // choice); an arbitrary gateway falls back to unknown. + const customGateway = { + adapter: "openai-chat", + authMode: "key", + baseUrl: "https://gateway.example/v1", + } as const; + const config = configWithProviders({ moonshot: customGateway }); + const candidate = { provider: "moonshot", id: "kimi-k2.5" }; + + expect(modelAcceptsImageInput(config, candidate)).toBeUndefined(); + expect(requiresVisionPreprocessing(config, customGateway, candidate.id, candidate.provider)).toBe(false); + + const declaredChina = { + adapter: "openai-chat", + authMode: "key", + baseUrl: "https://api.moonshot.cn/v1", + } as const; + const china = configWithProviders({ moonshot: declaredChina }); + expect(modelAcceptsImageInput(china, candidate)).toBe(true); + expect(requiresVisionPreprocessing(china, declaredChina, candidate.id, candidate.provider)).toBe(false); + + const canonical = configWithProviders({ moonshot: { + adapter: "openai-chat", + authMode: "key", + baseUrl: "https://api.moonshot.ai/v1", + } as const }); + expect(modelAcceptsImageInput(canonical, candidate)).toBe(true); + }); + + test("11j. a case-varied name on a declared former endpoint keeps the verdict", () => { + // `zai` declares its pre-move Chat endpoint through destinationAliases: a saved `ZAI` + // row — case-varied, so routing serves it as custom — still points at a vendor-owned + // destination, so the generated text-only verdict for glm-5.3 applies. + const provider = { + adapter: "openai-chat", + authMode: "key", + baseUrl: "https://api.z.ai/api/coding/paas/v4", + } as const; + const config = configWithProviders({ ZAI: provider }); + const candidate = { provider: "ZAI", id: "glm-5.3" }; + + expect(modelAcceptsImageInput(config, candidate)).toBe(false); + expect(requiresVisionPreprocessing(config, provider, candidate.id, candidate.provider)).toBe(true); + }); + + test("11k. a canonical override preset missing authMode keeps the verdict", () => { + // Routing canonicalizes an exact override id to the entry's adapter and derived auth — + // only the configured URL reaches the wire — so a legacy `google-antigravity` row that + // predates authMode still lands on the declared Google transport, and the generated + // text-only verdict for gemini-live-2.5-flash-preview-native-audio applies. + const provider = { + adapter: "google", + baseUrl: "https://daily-cloudcode-pa.googleapis.com", + } as const; + const config = configWithProviders({ "google-antigravity": provider }); + const candidate = { provider: "google-antigravity", id: "gemini-live-2.5-flash-preview-native-audio" }; + + expect(modelAcceptsImageInput(config, candidate)).toBe(false); + expect(requiresVisionPreprocessing(config, provider, candidate.id, candidate.provider)).toBe(true); + }); + test("12. only the selected Anthropic OAuth provider contributes Anthropic options", () => { const config = configWithProviders({ anthropic: {