From 799c99b04216633ebe88a61a14c42a9a14890cf1 Mon Sep 17 00:00:00 2001 From: Kaushik Kampli Date: Tue, 21 Jul 2026 19:42:04 +0530 Subject: [PATCH 1/5] embeddings: default check_embedding_ctx_length=False for OpenAI-compatible endpoints langchain OpenAIEmbeddings defaults to check_embedding_ctx_length=True, which tokenizes client-side with tiktoken and sends OpenAI token IDs. Non-OpenAI models behind an OpenAI-compatible endpoint (e.g. Qwen) can't interpret those IDs and return nonsensical embeddings. Default the flag to False so raw text is sent and the server tokenizes correctly. No-op for genuine OpenAI models; callers can opt back in with check_embedding_ctx_length=True. Scoped to the OpenAI path, so the Bedrock/Amazon branch is unaffected. Co-authored-by: Cursor --- singlestoredb/ai/embeddings.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/singlestoredb/ai/embeddings.py b/singlestoredb/ai/embeddings.py index aba8e1c47..814642592 100644 --- a/singlestoredb/ai/embeddings.py +++ b/singlestoredb/ai/embeddings.py @@ -152,6 +152,13 @@ def _inject_headers(request: Any, **_ignored: Any) -> None: ) if http_client is not None: openai_kwargs['http_client'] = http_client + # Non-OpenAI models behind an OpenAI-compatible endpoint (e.g. Qwen) get wrong + # embeddings under langchain's default client-side tiktoken tokenization, which + # sends OpenAI token IDs instead of raw text. Default to raw text so the server + # tokenizes correctly; this is a no-op for genuine OpenAI models. Callers can + # opt back in with check_embedding_ctx_length=True. Scoped to this OpenAI path; + # the Bedrock branch above never receives this kwarg. + kwargs.setdefault('check_embedding_ctx_length', False) return OpenAIEmbeddings( **openai_kwargs, **kwargs, From c22e2b6b8d9eae93bf084c866eec47cebe48f89e Mon Sep 17 00:00:00 2001 From: Kaushik Kampli Date: Tue, 21 Jul 2026 21:10:08 +0530 Subject: [PATCH 2/5] embeddings: gate the raw-text default to non-OpenAI platforms Only default check_embedding_ctx_length=False for self-hosted / non-OpenAI models (e.g. Qwen on the 'Nova' platform). Genuine OpenAI/Azure models ('Azure'/'OpenAI') keep langchain's default (True), preserving correct tiktoken tokenization and client-side long-input chunking for them. Co-authored-by: Cursor --- singlestoredb/ai/embeddings.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/singlestoredb/ai/embeddings.py b/singlestoredb/ai/embeddings.py index 814642592..4a6b43573 100644 --- a/singlestoredb/ai/embeddings.py +++ b/singlestoredb/ai/embeddings.py @@ -152,13 +152,13 @@ def _inject_headers(request: Any, **_ignored: Any) -> None: ) if http_client is not None: openai_kwargs['http_client'] = http_client - # Non-OpenAI models behind an OpenAI-compatible endpoint (e.g. Qwen) get wrong - # embeddings under langchain's default client-side tiktoken tokenization, which - # sends OpenAI token IDs instead of raw text. Default to raw text so the server - # tokenizes correctly; this is a no-op for genuine OpenAI models. Callers can - # opt back in with check_embedding_ctx_length=True. Scoped to this OpenAI path; - # the Bedrock branch above never receives this kwarg. - kwargs.setdefault('check_embedding_ctx_length', False) + # Genuine OpenAI/Azure models tokenize with tiktoken correctly, so keep langchain's + # default (client-side tokenization + long-input chunking). For every other platform + # (e.g. 'Nova' self-hosted models like Qwen), tiktoken sends OpenAI token IDs instead + # of the raw text, which the model can't interpret -> nonsensical embeddings. For + # those, send raw text and let the server tokenize. Callers can override explicitly. + if info.hosting_platform not in ('Azure', 'OpenAI'): + kwargs.setdefault('check_embedding_ctx_length', False) return OpenAIEmbeddings( **openai_kwargs, **kwargs, From 0b2233c7b544411b42b412365ef9efde5caea9ac Mon Sep 17 00:00:00 2001 From: Kaushik Kampli Date: Wed, 22 Jul 2026 18:29:42 +0530 Subject: [PATCH 3/5] embeddings: per-platform tokenization + self-chunking for non-OpenAI models - Azure (OpenAI) models: keep check_embedding_ctx_length=True so langchain uses tiktoken (correct for these models) with the passed model name, incl. its long- input chunking. - Non-OpenAI models (e.g. Qwen on 'Nova'): send raw text (check_embedding_ctx_length =False) so the server's own tokenizer is used, and split long inputs into character-bounded chunks client-side, embedding each and length-weighted-averaging them into one vector per input -- irrespective of the flag -- so long texts never hit the server's hard context limit (which otherwise errors or silently truncates). Output stays 1:1 with input rows; short inputs pass through unchanged. Co-authored-by: Cursor --- singlestoredb/ai/embeddings.py | 109 ++++++++++++++++++++++++++++++--- 1 file changed, 101 insertions(+), 8 deletions(-) diff --git a/singlestoredb/ai/embeddings.py b/singlestoredb/ai/embeddings.py index 4a6b43573..d5256466f 100644 --- a/singlestoredb/ai/embeddings.py +++ b/singlestoredb/ai/embeddings.py @@ -1,7 +1,9 @@ import os from typing import Any from typing import Callable +from typing import List from typing import Optional +from typing import Tuple from typing import Union import httpx @@ -30,6 +32,88 @@ from botocore.config import Config +class _ChunkedOpenAIEmbeddings(OpenAIEmbeddings): + """OpenAIEmbeddings for non-OpenAI models behind an OpenAI-compatible endpoint. + + These models (e.g. Qwen served on the 'Nova' platform) tokenize server-side with + their own tokenizer, so inputs are sent as raw text (``check_embedding_ctx_length`` + should be False). Because the server rejects (or silently truncates) inputs longer + than its context window, this class splits long inputs into character-bounded chunks + itself, embeds each chunk, and length-weighted-averages them back into a single + vector per input -- irrespective of the flag -- so long texts never hit the server's + hard limit. + """ + + max_chunk_chars: int = 24000 + """Maximum characters per chunk. Conservative (~6-8k tokens for typical text) so a + chunk fits even deployments capped at 8192 tokens. Override per model if the + deployment's context window is known to be larger.""" + + def _chunks(self, text: str) -> List[str]: + n = max(1, self.max_chunk_chars) + if len(text) <= n: + return [text] + return [text[i:i + n] for i in range(0, len(text), n)] + + @staticmethod + def _average(vectors: List[List[float]], weights: List[int]) -> List[float]: + total = float(sum(weights)) or 1.0 + dim = len(vectors[0]) + avg = [0.0] * dim + for vec, w in zip(vectors, weights): + for k in range(dim): + avg[k] += vec[k] * w + avg = [x / total for x in avg] + norm = sum(x * x for x in avg) ** 0.5 + if norm > 0: + avg = [x / norm for x in avg] + return avg + + def _plan(self, texts: List[str]) -> Tuple[List[str], List[int]]: + flat: List[str] = [] + owner: List[int] = [] + for i, text in enumerate(texts): + for chunk in self._chunks(text): + flat.append(chunk) + owner.append(i) + return flat, owner + + def _reduce( + self, + num_texts: int, + owner: List[int], + flat: List[str], + embeddings: List[List[float]], + ) -> List[List[float]]: + out: List[List[float]] = [] + for i in range(num_texts): + idxs = [j for j, o in enumerate(owner) if o == i] + if len(idxs) == 1: + out.append(embeddings[idxs[0]]) + else: + out.append( + self._average( + [embeddings[j] for j in idxs], + [max(1, len(flat[j])) for j in idxs], + ), + ) + return out + + def embed_documents( + self, texts: List[str], chunk_size: Optional[int] = None, **kwargs: Any, + ) -> List[List[float]]: + flat, owner = self._plan(texts) + embeddings = super().embed_documents(flat, chunk_size=chunk_size, **kwargs) + return self._reduce(len(texts), owner, flat, embeddings) + + async def aembed_documents( + self, texts: List[str], chunk_size: Optional[int] = None, **kwargs: Any, + ) -> List[List[float]]: + flat, owner = self._plan(texts) + embeddings = await super().aembed_documents(flat, chunk_size=chunk_size, **kwargs) + return self._reduce(len(texts), owner, flat, embeddings) + + def SingleStoreEmbeddingsFactory( model_name: str, api_key: Optional[str] = None, @@ -152,14 +236,23 @@ def _inject_headers(request: Any, **_ignored: Any) -> None: ) if http_client is not None: openai_kwargs['http_client'] = http_client - # Genuine OpenAI/Azure models tokenize with tiktoken correctly, so keep langchain's - # default (client-side tokenization + long-input chunking). For every other platform - # (e.g. 'Nova' self-hosted models like Qwen), tiktoken sends OpenAI token IDs instead - # of the raw text, which the model can't interpret -> nonsensical embeddings. For - # those, send raw text and let the server tokenize. Callers can override explicitly. - if info.hosting_platform not in ('Azure', 'OpenAI'): - kwargs.setdefault('check_embedding_ctx_length', False) - return OpenAIEmbeddings( + + if info.hosting_platform == 'Azure': + # Genuine OpenAI (Azure) models: tiktoken is the correct tokenizer, and the + # model name is passed above so it selects the right encoding. Keep langchain's + # client-side tokenization + long-input chunking (all correct for these models). + kwargs.setdefault('check_embedding_ctx_length', True) + return OpenAIEmbeddings( + **openai_kwargs, + **kwargs, + ) + + # Non-OpenAI models (e.g. Qwen on 'Nova'): tiktoken would send OpenAI token IDs the + # model can't interpret -> nonsensical embeddings. Send raw text so the server + # tokenizes with the model's own tokenizer, and chunk long inputs ourselves (the + # server otherwise rejects or silently truncates over-context input). + kwargs.setdefault('check_embedding_ctx_length', False) + return _ChunkedOpenAIEmbeddings( **openai_kwargs, **kwargs, ) From f57be7ddcb068eadba69624ad6578125c811be66 Mon Sep 17 00:00:00 2001 From: Kaushik Kampli Date: Wed, 22 Jul 2026 18:32:25 +0530 Subject: [PATCH 4/5] embeddings: minimal fix - check_embedding_ctx_length=True for Azure, False otherwise Non-OpenAI models (e.g. Qwen on 'Nova') get wrong embeddings from langchain's default tiktoken tokenization (it sends OpenAI token IDs). Send raw text for them so the server tokenizes with the model's own tokenizer; keep tiktoken (True) for Azure/OpenAI models, where it's correct. Long-input chunking deferred. Co-authored-by: Cursor --- singlestoredb/ai/embeddings.py | 105 +++------------------------------ 1 file changed, 7 insertions(+), 98 deletions(-) diff --git a/singlestoredb/ai/embeddings.py b/singlestoredb/ai/embeddings.py index d5256466f..212d0f6cf 100644 --- a/singlestoredb/ai/embeddings.py +++ b/singlestoredb/ai/embeddings.py @@ -1,9 +1,7 @@ import os from typing import Any from typing import Callable -from typing import List from typing import Optional -from typing import Tuple from typing import Union import httpx @@ -32,88 +30,6 @@ from botocore.config import Config -class _ChunkedOpenAIEmbeddings(OpenAIEmbeddings): - """OpenAIEmbeddings for non-OpenAI models behind an OpenAI-compatible endpoint. - - These models (e.g. Qwen served on the 'Nova' platform) tokenize server-side with - their own tokenizer, so inputs are sent as raw text (``check_embedding_ctx_length`` - should be False). Because the server rejects (or silently truncates) inputs longer - than its context window, this class splits long inputs into character-bounded chunks - itself, embeds each chunk, and length-weighted-averages them back into a single - vector per input -- irrespective of the flag -- so long texts never hit the server's - hard limit. - """ - - max_chunk_chars: int = 24000 - """Maximum characters per chunk. Conservative (~6-8k tokens for typical text) so a - chunk fits even deployments capped at 8192 tokens. Override per model if the - deployment's context window is known to be larger.""" - - def _chunks(self, text: str) -> List[str]: - n = max(1, self.max_chunk_chars) - if len(text) <= n: - return [text] - return [text[i:i + n] for i in range(0, len(text), n)] - - @staticmethod - def _average(vectors: List[List[float]], weights: List[int]) -> List[float]: - total = float(sum(weights)) or 1.0 - dim = len(vectors[0]) - avg = [0.0] * dim - for vec, w in zip(vectors, weights): - for k in range(dim): - avg[k] += vec[k] * w - avg = [x / total for x in avg] - norm = sum(x * x for x in avg) ** 0.5 - if norm > 0: - avg = [x / norm for x in avg] - return avg - - def _plan(self, texts: List[str]) -> Tuple[List[str], List[int]]: - flat: List[str] = [] - owner: List[int] = [] - for i, text in enumerate(texts): - for chunk in self._chunks(text): - flat.append(chunk) - owner.append(i) - return flat, owner - - def _reduce( - self, - num_texts: int, - owner: List[int], - flat: List[str], - embeddings: List[List[float]], - ) -> List[List[float]]: - out: List[List[float]] = [] - for i in range(num_texts): - idxs = [j for j, o in enumerate(owner) if o == i] - if len(idxs) == 1: - out.append(embeddings[idxs[0]]) - else: - out.append( - self._average( - [embeddings[j] for j in idxs], - [max(1, len(flat[j])) for j in idxs], - ), - ) - return out - - def embed_documents( - self, texts: List[str], chunk_size: Optional[int] = None, **kwargs: Any, - ) -> List[List[float]]: - flat, owner = self._plan(texts) - embeddings = super().embed_documents(flat, chunk_size=chunk_size, **kwargs) - return self._reduce(len(texts), owner, flat, embeddings) - - async def aembed_documents( - self, texts: List[str], chunk_size: Optional[int] = None, **kwargs: Any, - ) -> List[List[float]]: - flat, owner = self._plan(texts) - embeddings = await super().aembed_documents(flat, chunk_size=chunk_size, **kwargs) - return self._reduce(len(texts), owner, flat, embeddings) - - def SingleStoreEmbeddingsFactory( model_name: str, api_key: Optional[str] = None, @@ -237,22 +153,15 @@ def _inject_headers(request: Any, **_ignored: Any) -> None: if http_client is not None: openai_kwargs['http_client'] = http_client + # Azure (OpenAI) models tokenize correctly with tiktoken using the model name passed + # above, so keep langchain's default client-side tokenization. Every other platform + # (e.g. Qwen on 'Nova') would get wrong tiktoken token IDs, so send raw text and let + # the server tokenize with the model's own tokenizer. if info.hosting_platform == 'Azure': - # Genuine OpenAI (Azure) models: tiktoken is the correct tokenizer, and the - # model name is passed above so it selects the right encoding. Keep langchain's - # client-side tokenization + long-input chunking (all correct for these models). kwargs.setdefault('check_embedding_ctx_length', True) - return OpenAIEmbeddings( - **openai_kwargs, - **kwargs, - ) - - # Non-OpenAI models (e.g. Qwen on 'Nova'): tiktoken would send OpenAI token IDs the - # model can't interpret -> nonsensical embeddings. Send raw text so the server - # tokenizes with the model's own tokenizer, and chunk long inputs ourselves (the - # server otherwise rejects or silently truncates over-context input). - kwargs.setdefault('check_embedding_ctx_length', False) - return _ChunkedOpenAIEmbeddings( + else: + kwargs.setdefault('check_embedding_ctx_length', False) + return OpenAIEmbeddings( **openai_kwargs, **kwargs, ) From 6def72b5abdd96fbca8cc8de4f5cd9bf5de753ca Mon Sep 17 00:00:00 2001 From: Kaushik Kampli Date: Wed, 22 Jul 2026 18:34:05 +0530 Subject: [PATCH 5/5] embeddings: restore self-chunking for non-OpenAI models Re-add _ChunkedOpenAIEmbeddings: for non-Azure models (e.g. Qwen on 'Nova'), send raw text (check_embedding_ctx_length=False) and split long inputs into character-bounded chunks, embedding each and length-weighted-averaging into one vector per input, so long texts never hit the server's context limit. Azure keeps tiktoken (True). Co-authored-by: Cursor --- singlestoredb/ai/embeddings.py | 105 ++++++++++++++++++++++++++++++--- 1 file changed, 98 insertions(+), 7 deletions(-) diff --git a/singlestoredb/ai/embeddings.py b/singlestoredb/ai/embeddings.py index 212d0f6cf..d5256466f 100644 --- a/singlestoredb/ai/embeddings.py +++ b/singlestoredb/ai/embeddings.py @@ -1,7 +1,9 @@ import os from typing import Any from typing import Callable +from typing import List from typing import Optional +from typing import Tuple from typing import Union import httpx @@ -30,6 +32,88 @@ from botocore.config import Config +class _ChunkedOpenAIEmbeddings(OpenAIEmbeddings): + """OpenAIEmbeddings for non-OpenAI models behind an OpenAI-compatible endpoint. + + These models (e.g. Qwen served on the 'Nova' platform) tokenize server-side with + their own tokenizer, so inputs are sent as raw text (``check_embedding_ctx_length`` + should be False). Because the server rejects (or silently truncates) inputs longer + than its context window, this class splits long inputs into character-bounded chunks + itself, embeds each chunk, and length-weighted-averages them back into a single + vector per input -- irrespective of the flag -- so long texts never hit the server's + hard limit. + """ + + max_chunk_chars: int = 24000 + """Maximum characters per chunk. Conservative (~6-8k tokens for typical text) so a + chunk fits even deployments capped at 8192 tokens. Override per model if the + deployment's context window is known to be larger.""" + + def _chunks(self, text: str) -> List[str]: + n = max(1, self.max_chunk_chars) + if len(text) <= n: + return [text] + return [text[i:i + n] for i in range(0, len(text), n)] + + @staticmethod + def _average(vectors: List[List[float]], weights: List[int]) -> List[float]: + total = float(sum(weights)) or 1.0 + dim = len(vectors[0]) + avg = [0.0] * dim + for vec, w in zip(vectors, weights): + for k in range(dim): + avg[k] += vec[k] * w + avg = [x / total for x in avg] + norm = sum(x * x for x in avg) ** 0.5 + if norm > 0: + avg = [x / norm for x in avg] + return avg + + def _plan(self, texts: List[str]) -> Tuple[List[str], List[int]]: + flat: List[str] = [] + owner: List[int] = [] + for i, text in enumerate(texts): + for chunk in self._chunks(text): + flat.append(chunk) + owner.append(i) + return flat, owner + + def _reduce( + self, + num_texts: int, + owner: List[int], + flat: List[str], + embeddings: List[List[float]], + ) -> List[List[float]]: + out: List[List[float]] = [] + for i in range(num_texts): + idxs = [j for j, o in enumerate(owner) if o == i] + if len(idxs) == 1: + out.append(embeddings[idxs[0]]) + else: + out.append( + self._average( + [embeddings[j] for j in idxs], + [max(1, len(flat[j])) for j in idxs], + ), + ) + return out + + def embed_documents( + self, texts: List[str], chunk_size: Optional[int] = None, **kwargs: Any, + ) -> List[List[float]]: + flat, owner = self._plan(texts) + embeddings = super().embed_documents(flat, chunk_size=chunk_size, **kwargs) + return self._reduce(len(texts), owner, flat, embeddings) + + async def aembed_documents( + self, texts: List[str], chunk_size: Optional[int] = None, **kwargs: Any, + ) -> List[List[float]]: + flat, owner = self._plan(texts) + embeddings = await super().aembed_documents(flat, chunk_size=chunk_size, **kwargs) + return self._reduce(len(texts), owner, flat, embeddings) + + def SingleStoreEmbeddingsFactory( model_name: str, api_key: Optional[str] = None, @@ -153,15 +237,22 @@ def _inject_headers(request: Any, **_ignored: Any) -> None: if http_client is not None: openai_kwargs['http_client'] = http_client - # Azure (OpenAI) models tokenize correctly with tiktoken using the model name passed - # above, so keep langchain's default client-side tokenization. Every other platform - # (e.g. Qwen on 'Nova') would get wrong tiktoken token IDs, so send raw text and let - # the server tokenize with the model's own tokenizer. if info.hosting_platform == 'Azure': + # Genuine OpenAI (Azure) models: tiktoken is the correct tokenizer, and the + # model name is passed above so it selects the right encoding. Keep langchain's + # client-side tokenization + long-input chunking (all correct for these models). kwargs.setdefault('check_embedding_ctx_length', True) - else: - kwargs.setdefault('check_embedding_ctx_length', False) - return OpenAIEmbeddings( + return OpenAIEmbeddings( + **openai_kwargs, + **kwargs, + ) + + # Non-OpenAI models (e.g. Qwen on 'Nova'): tiktoken would send OpenAI token IDs the + # model can't interpret -> nonsensical embeddings. Send raw text so the server + # tokenizes with the model's own tokenizer, and chunk long inputs ourselves (the + # server otherwise rejects or silently truncates over-context input). + kwargs.setdefault('check_embedding_ctx_length', False) + return _ChunkedOpenAIEmbeddings( **openai_kwargs, **kwargs, )