"""Knowledge-scoped embedding options, shared by indexing and retrieval.""" import math import unicodedata from langchain_core.embeddings import Embeddings def validate_params(params): if (not isinstance(params, dict) or len(params) > 52 or sum(key not in {'batch_size', 'normalize'} for key in params) > 50): raise ValueError('可选参数最多50项') result, names = {}, set() for key, value in params.items(): if not isinstance(key, str) or not key.strip() or len(key.strip()) > 64 or any(ord(c) < 32 for c in key): raise ValueError('参数名称须为1~64个字符') key = key.strip() normalized = unicodedata.normalize('NFKC', key).casefold() if normalized in names: raise ValueError('参数名称不能重复') names.add(normalized) if value is not None and (not isinstance(value, (str, int, float, bool)) or isinstance(value, str) and len(value) > 2000 or isinstance(value, float) and not math.isfinite(value)): raise ValueError('参数值须为2000字以内的文本或数值') result[key] = value batch = result.get('batch_size') if batch is not None and (type(batch) is not int or not 1 <= batch <= 256): raise ValueError('批处理数量须为1~256的整数') if result.get('normalize') is not None and type(result['normalize']) is not bool: raise ValueError('向量归一化参数无效') for key in ('query_prefix', 'document_prefix'): if key in result and (not isinstance(result[key], str) or len(result[key]) > 2000): raise ValueError('向量指令须为2000字以内的文本') return {key: value for key, value in result.items() if value is not None and (value != '' or key not in {'query_prefix', 'document_prefix'})} def knowledge_params(knowledge): params = validate_params((knowledge.meta or {}).get('embedding_params') or {}) params.setdefault('normalize', True) return {key: value for key, value in params.items() if key in {'batch_size', 'normalize', 'query_prefix', 'document_prefix'}} class ConfiguredEmbedding(Embeddings): def __init__(self, model, params): self.model = model self.params = params self.batch_size = params.get('batch_size') or getattr(model, 'batch_size', 10) or 10 def _vector(self, vector): if not self.params.get('normalize', True): return vector norm = math.sqrt(sum(float(v) ** 2 for v in vector)) return [float(v) / norm for v in vector] if norm else vector def embed_query(self, text): return self._vector(self.model.embed_query(self.params.get('query_prefix', '') + text)) def embed_documents(self, texts): prefix = self.params.get('document_prefix', '') result = [] for start in range(0, len(texts), self.batch_size): result.extend(self._vector(v) for v in self.model.embed_documents( [prefix + text for text in texts[start:start + self.batch_size]])) return result