UnisKB/apps/knowledge/embedding_params.py

67 lines
3.1 KiB
Python
Raw Normal View History

"""Knowledge-scoped embedding options, shared by indexing and retrieval."""
import math
import unicodedata
from langchain_core.embeddings import Embeddings
def validate_params(params):
if (not isinstance(params, dict) or len(params) > 52 or
sum(key not in {'batch_size', 'normalize'} for key in params) > 50):
raise ValueError('可选参数最多50项')
result, names = {}, set()
for key, value in params.items():
if not isinstance(key, str) or not key.strip() or len(key.strip()) > 64 or any(ord(c) < 32 for c in key):
raise ValueError('参数名称须为1~64个字符')
key = key.strip()
normalized = unicodedata.normalize('NFKC', key).casefold()
if normalized in names:
raise ValueError('参数名称不能重复')
names.add(normalized)
if value is not None and (not isinstance(value, (str, int, float, bool)) or
isinstance(value, str) and len(value) > 2000 or
isinstance(value, float) and not math.isfinite(value)):
raise ValueError('参数值须为2000字以内的文本或数值')
result[key] = value
batch = result.get('batch_size')
if batch is not None and (type(batch) is not int or not 1 <= batch <= 256):
raise ValueError('批处理数量须为1~256的整数')
if result.get('normalize') is not None and type(result['normalize']) is not bool:
raise ValueError('向量归一化参数无效')
for key in ('query_prefix', 'document_prefix'):
if key in result and (not isinstance(result[key], str) or len(result[key]) > 2000):
raise ValueError('向量指令须为2000字以内的文本')
return {key: value for key, value in result.items() if value is not None and
(value != '' or key not in {'query_prefix', 'document_prefix'})}
def knowledge_params(knowledge):
params = validate_params((knowledge.meta or {}).get('embedding_params') or {})
2026-11-05 10:16:48 +00:00
params.setdefault('normalize', True)
return {key: value for key, value in params.items()
if key in {'batch_size', 'normalize', 'query_prefix', 'document_prefix'}}
class ConfiguredEmbedding(Embeddings):
def __init__(self, model, params):
self.model = model
self.params = params
self.batch_size = params.get('batch_size') or getattr(model, 'batch_size', 10) or 10
def _vector(self, vector):
2026-11-05 10:16:48 +00:00
if not self.params.get('normalize', True):
return vector
norm = math.sqrt(sum(float(v) ** 2 for v in vector))
return [float(v) / norm for v in vector] if norm else vector
def embed_query(self, text):
return self._vector(self.model.embed_query(self.params.get('query_prefix', '') + text))
def embed_documents(self, texts):
prefix = self.params.get('document_prefix', '')
result = []
for start in range(0, len(texts), self.batch_size):
result.extend(self._vector(v) for v in self.model.embed_documents(
[prefix + text for text in texts[start:start + self.batch_size]]))
return result