351 lines
22 KiB
Python
351 lines
22 KiB
Python
import uuid
|
||
import json
|
||
from datetime import timedelta
|
||
from types import SimpleNamespace
|
||
from unittest import TestCase
|
||
from unittest.mock import patch
|
||
from django.db import transaction
|
||
from django.test import RequestFactory
|
||
from django.utils import timezone
|
||
from rest_framework.exceptions import ValidationError, NotFound
|
||
from common.exception.app_exception import AppUnauthorizedFailed, NotFound404
|
||
from knowledge import versions as v
|
||
from knowledge.models import (KnowledgeFolder, Knowledge, Document, Paragraph, Problem, ProblemParagraphMapping,
|
||
Tag, DocumentTag, Embedding, File, KnowledgeVersion, KnowledgeVersionLease)
|
||
from users.models import User
|
||
from models_provider.models import Model
|
||
from application.models import Application, ApplicationFolder, ApplicationVersion, ApplicationKnowledgeMapping
|
||
|
||
|
||
class VersionTests(TestCase):
|
||
def setUp(self):
|
||
self.atomic = transaction.atomic(); self.atomic.__enter__()
|
||
from django.db import connection
|
||
if connection.vendor == 'postgresql':
|
||
with connection.cursor() as cursor:
|
||
cursor.execute('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ')
|
||
self.user = User.objects.create(username='CODEX_VERSION_TEST', nick_name='版本测试', role='ADMIN')
|
||
self.folder = KnowledgeFolder.objects.create(id='default', name='默认')
|
||
ApplicationFolder.objects.create(id='default', name='默认')
|
||
self.model = Model.objects.create(name='测试向量', model_name='test-embedding', provider='test',
|
||
model_type='EMBEDDING', credential='encrypted-test', user=self.user)
|
||
self.root = Knowledge.objects.create(name='CODEX知识库', desc='原始', folder=self.folder, user=self.user,
|
||
embedding_model=self.model, meta={'embedding_params': {'batch_size': 8}, 'segmentation': {'chunk_size': 500}})
|
||
self.doc = Document.objects.create(knowledge=self.root, name='样例', char_length=8, status='nn2')
|
||
self.para = Paragraph.objects.create(knowledge=self.root, document=self.doc, content='旧答案:100元', position=1)
|
||
self.problem = Problem.objects.create(knowledge=self.root, content='价格多少')
|
||
ProblemParagraphMapping.objects.create(knowledge=self.root, document=self.doc, paragraph=self.para, problem=self.problem)
|
||
tag = Tag.objects.create(knowledge=self.root, key='部门', value='测试')
|
||
DocumentTag.objects.create(document=self.doc, tag=tag)
|
||
Embedding.objects.create(id=str(uuid.uuid4()), knowledge=self.root, document=self.doc, paragraph=self.para,
|
||
source_id=str(self.para.id), source_type='1', embedding='[1,0,0]', search_vector='price')
|
||
|
||
def tearDown(self):
|
||
transaction.set_rollback(True); self.atomic.__exit__(None, None, None)
|
||
|
||
def create(self, name='初版'):
|
||
return v.create_version('default', self.root.id, self.user, {'name': name})
|
||
|
||
def edit_embedding(self, values):
|
||
from knowledge.serializers.knowledge import KnowledgeSerializer
|
||
return KnowledgeSerializer.Operate(data={'user_id': str(self.user.id), 'workspace_id': 'default',
|
||
'knowledge_id': str(self.root.id)}).edit(values, select_one=False)
|
||
|
||
def test_embedding_guard_blocks_version_but_allows_unchanged_and_batch(self):
|
||
from common.exception.app_exception import AppApiException
|
||
self.create()
|
||
self.edit_embedding({'embedding_model_id': str(self.model.id), 'embedding_params': {'batch_size': 12}})
|
||
with self.assertRaises(AppApiException):
|
||
self.edit_embedding({'embedding_model_id': str(uuid.uuid4())})
|
||
with self.assertRaises(AppApiException):
|
||
self.edit_embedding({'embedding_params': {'batch_size': 12, 'normalize': False}})
|
||
|
||
def test_embedding_guard_detects_current_and_published_assistants(self):
|
||
from common.exception.app_exception import AppApiException
|
||
other = Knowledge.objects.create(name='另一个知识库', folder=self.folder, user=self.user, embedding_model=self.model)
|
||
app = Application.objects.create(name='助手', user=self.user)
|
||
ApplicationKnowledgeMapping.objects.create(application=app, knowledge=self.root)
|
||
self.edit_embedding({'embedding_model_id': str(self.model.id)})
|
||
ApplicationKnowledgeMapping.objects.create(application=app, knowledge=other)
|
||
with self.assertRaises(AppApiException):
|
||
self.edit_embedding({'embedding_model_id': str(uuid.uuid4())})
|
||
ApplicationVersion.objects.create(application=app, knowledge_setting={
|
||
'_knowledge_ids': [str(self.root.id), str(other.id)]})
|
||
ApplicationKnowledgeMapping.objects.filter(application=app, knowledge=other).delete()
|
||
with self.assertRaises(AppApiException):
|
||
self.edit_embedding({'embedding_model_id': str(uuid.uuid4())})
|
||
ApplicationVersion.objects.create(application=app, knowledge_setting={'_knowledge_ids': [str(self.root.id)]})
|
||
self.edit_embedding({'embedding_model_id': str(self.model.id)})
|
||
|
||
def test_embedding_guard_detects_workflow_and_allows_single_reference(self):
|
||
from knowledge.embedding_guard import embedding_change_reason, ASSISTANT_REASON
|
||
other = Knowledge.objects.create(name='另一个知识库', folder=self.folder, user=self.user, embedding_model=self.model)
|
||
node = {'type': 'search-knowledge-node', 'properties': {'node_data': {
|
||
'knowledge_id_list': [str(self.root.id)]}}}
|
||
app = Application.objects.create(name='工作流', type='WORK_FLOW', user=self.user,
|
||
work_flow={'nodes': [node]})
|
||
self.assertEqual(embedding_change_reason(self.root), '')
|
||
node['properties']['node_data']['knowledge_id_list'].append(str(other.id))
|
||
app.work_flow = {'nodes': [node]}; app.save()
|
||
self.assertEqual(embedding_change_reason(self.root), ASSISTANT_REASON)
|
||
|
||
def test_embedding_guard_allows_model_change_with_single_assistant(self):
|
||
other_model = Model.objects.create(name='备用向量', model_name='test-embedding-2', provider='test',
|
||
model_type='EMBEDDING', credential='encrypted-test', user=self.user)
|
||
app = Application.objects.create(name='助手', user=self.user)
|
||
ApplicationKnowledgeMapping.objects.create(application=app, knowledge=self.root)
|
||
self.edit_embedding({'embedding_model_id': str(other_model.id)})
|
||
self.root.refresh_from_db()
|
||
self.assertEqual(self.root.embedding_model_id, other_model.id)
|
||
|
||
def test_full_snapshot_and_id_remapping(self):
|
||
version = self.create()
|
||
self.assertTrue(version.search_ready)
|
||
self.assertEqual(version.document_count, 1)
|
||
paragraph = Paragraph.objects.get(knowledge=version.snapshot)
|
||
self.assertNotEqual(paragraph.id, self.para.id)
|
||
self.assertEqual(paragraph.content, self.para.content)
|
||
mapping = ProblemParagraphMapping.objects.get(knowledge=version.snapshot)
|
||
self.assertEqual(mapping.paragraph_id, paragraph.id)
|
||
self.assertEqual(mapping.document_id, paragraph.document_id)
|
||
self.assertEqual(DocumentTag.objects.filter(document__knowledge=version.snapshot).count(), 1)
|
||
vector = Embedding.objects.get(knowledge=version.snapshot)
|
||
self.assertEqual(vector.paragraph_id, paragraph.id)
|
||
self.assertEqual(vector.source_id, str(paragraph.id))
|
||
|
||
def test_live_edits_do_not_change_snapshot(self):
|
||
version = self.create()
|
||
self.para.content = '新答案:200元'; self.para.save()
|
||
self.assertEqual(Paragraph.objects.get(knowledge=version.snapshot).content, '旧答案:100元')
|
||
second = self.create('第二版')
|
||
self.assertEqual(Paragraph.objects.get(knowledge=second.snapshot).content, '新答案:200元')
|
||
self.assertNotEqual(version.content_hash, second.content_hash)
|
||
|
||
def test_text_uuid_is_preserved(self):
|
||
self.para.content = '业务标识:' + str(self.doc.id); self.para.save()
|
||
version = self.create()
|
||
self.assertEqual(Paragraph.objects.get(knowledge=version.snapshot).content, self.para.content)
|
||
|
||
def test_binary_files_shared_and_markdown_rewritten(self):
|
||
file = File(id=uuid.uuid4(), file_name='测试.png', loid=12345, source_id=str(self.root.id),
|
||
source_type='KNOWLEDGE', sha256_hash='test', file_size=10, meta={'knowledge_id': str(self.root.id)})
|
||
File.objects.bulk_create([file])
|
||
self.para.content = ' + ')'; self.para.save()
|
||
version = self.create()
|
||
cloned = File.objects.get(source_id=str(version.snapshot_id))
|
||
self.assertEqual(cloned.loid, file.loid)
|
||
self.assertNotEqual(cloned.id, file.id)
|
||
self.assertIn(str(cloned.id), Paragraph.objects.get(knowledge=version.snapshot).content)
|
||
with patch('knowledge.models.knowledge.select_one') as unlink:
|
||
v.delete_version('default', self.root.id, version.id)
|
||
unlink.assert_not_called()
|
||
self.assertTrue(File.objects.filter(pk=file.id).exists())
|
||
|
||
def test_version_delete_cleans_only_snapshot(self):
|
||
version = self.create(); snapshot_id = version.snapshot_id
|
||
v.delete_version('default', self.root.id, version.id)
|
||
for model in (Document, Paragraph, Problem, Embedding, Tag, ProblemParagraphMapping):
|
||
self.assertFalse(model.objects.filter(knowledge_id=snapshot_id).exists())
|
||
self.assertFalse(Knowledge.objects.filter(pk=snapshot_id).exists())
|
||
self.assertTrue(Paragraph.objects.filter(pk=self.para.id).exists())
|
||
|
||
def test_version_number_not_reused(self):
|
||
first = self.create(); v.delete_version('default', self.root.id, first.id)
|
||
self.assertEqual(self.create().number, 2)
|
||
|
||
def test_create_transaction_rolls_back_on_index_failure(self):
|
||
with patch.object(v, '_build_index', side_effect=RuntimeError('index failed')), self.assertRaises(RuntimeError): self.create()
|
||
self.assertEqual(Knowledge.objects.count(), 1)
|
||
self.assertEqual(KnowledgeVersion.objects.count(), 0)
|
||
|
||
def test_pending_processing_rejected(self):
|
||
self.doc.status = 'nn0'; self.doc.save()
|
||
with self.assertRaises(ValidationError): self.create()
|
||
|
||
def test_missing_vectors_are_browsable_but_not_selectable(self):
|
||
Embedding.objects.all().delete(); version = self.create()
|
||
self.assertFalse(version.search_ready)
|
||
with self.assertRaises(ValidationError): v.validate_selection({str(self.root.id): str(version.id)}, 'default', [self.root.id])
|
||
|
||
def test_cross_knowledge_selection_rejected(self):
|
||
version = self.create()
|
||
with self.assertRaises(ValidationError): v.validate_selection({str(self.root.id): str(version.id)}, 'default', [])
|
||
|
||
def test_invalid_version_rejected(self):
|
||
with self.assertRaises(ValidationError): v.validate_selection({str(self.root.id): 'invalid'}, 'default', [self.root.id])
|
||
|
||
def test_cross_workspace_rejected(self):
|
||
version = self.create()
|
||
with self.assertRaises(NotFound404): v.get_version('other', self.root.id, version.id)
|
||
|
||
def test_fixed_selection_and_live_lookup(self):
|
||
version = self.create()
|
||
self.assertEqual(v.resolve_search([self.root.id], {}, 'default'), [str(self.root.id)])
|
||
self.assertEqual(v.resolve_search([self.root.id], {str(self.root.id): str(version.id)}, 'default'), [str(version.snapshot_id)])
|
||
|
||
def test_document_scope_maps_to_frozen_documents(self):
|
||
version = self.create(); document = Document.objects.get(knowledge=version.snapshot)
|
||
self.assertEqual(v.resolve_document_scope([self.doc.id], {str(self.root.id): str(version.id)}, 'default'), [str(document.id)])
|
||
self.assertEqual(v.resolve_document_scope([self.doc.id], {}, 'default'), [str(self.doc.id)])
|
||
data = v.version_data(version)
|
||
self.assertNotIn('_document_ids', data['config'])
|
||
self.assertNotIn('model_identity', data['config'])
|
||
|
||
def test_same_chat_keeps_latest_choice(self):
|
||
first = self.create()
|
||
selection = {str(self.root.id): 'latest'}
|
||
pinned = v.pin_selection(selection, 'default', 'chat-test')
|
||
self.create('新版')
|
||
self.assertEqual(v.pin_selection(selection, 'default', 'chat-test'), pinned)
|
||
self.assertEqual(pinned[str(self.root.id)], str(first.id))
|
||
self.assertNotEqual(v.pin_selection(selection, 'default', 'new-chat'), pinned)
|
||
|
||
def test_active_conversation_protects_delete(self):
|
||
version = self.create(); v.pin_selection({str(self.root.id): str(version.id)}, 'default', 'chat-test')
|
||
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id)
|
||
KnowledgeVersionLease.objects.update(expires_at=timezone.now()-timedelta(seconds=1))
|
||
v.delete_version('default', self.root.id, version.id)
|
||
|
||
def test_assistant_pin_protects_delete(self):
|
||
version = self.create()
|
||
app = Application.objects.create(name='测试助手', user=self.user,
|
||
knowledge_setting={'version_selection': {str(self.root.id): str(version.id)}})
|
||
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id)
|
||
app.knowledge_setting = {}; app.save(); v.delete_version('default', self.root.id, version.id)
|
||
|
||
def test_latest_assistant_protects_only_latest_ready_version(self):
|
||
first = self.create(); second = self.create('新版')
|
||
Application.objects.create(name='测试助手', user=self.user,
|
||
knowledge_setting={'version_selection': {str(self.root.id): 'latest'}})
|
||
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, second.id)
|
||
v.delete_version('default', self.root.id, first.id)
|
||
|
||
def test_workflow_draft_and_published_version_protect_delete(self):
|
||
version = self.create()
|
||
flow = {'nodes': [{'type': 'search-knowledge-node', 'properties': {'node_data': {
|
||
'knowledge_id_list': [str(self.root.id)],
|
||
'knowledge_setting': {'version_selection': {str(self.root.id): str(version.id)}}}}}]}
|
||
app = Application.objects.create(name='工作流', type='WORK_FLOW', user=self.user, work_flow=flow)
|
||
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id)
|
||
app.work_flow = {}; app.save()
|
||
published = ApplicationVersion.objects.create(application=app, type='WORK_FLOW', work_flow=flow)
|
||
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id)
|
||
published.delete(); v.delete_version('default', self.root.id, version.id)
|
||
|
||
def test_workflow_conflicting_versions_rejected(self):
|
||
flow = {'nodes': [{'type': 'search-knowledge-node', 'properties': {'node_data': {
|
||
'knowledge_setting': {'version_selection': {str(self.root.id): 'latest'}}}}}]}
|
||
with self.assertRaises(ValidationError):
|
||
v.application_selection({'version_selection': {str(self.root.id): 'different'}}, flow)
|
||
|
||
def test_republication_releases_superseded_version(self):
|
||
version = self.create(); app = Application.objects.create(name='重新发布', user=self.user)
|
||
ApplicationVersion.objects.create(application=app, knowledge_setting={
|
||
'version_selection': {str(self.root.id): str(version.id)}})
|
||
with self.assertRaises(ValidationError): v.delete_version('default', self.root.id, version.id)
|
||
ApplicationVersion.objects.create(application=app, knowledge_setting={})
|
||
v.delete_version('default', self.root.id, version.id)
|
||
|
||
def test_migration_target_cannot_be_snapshot(self):
|
||
version = self.create(); request = RequestFactory().post('/document/migrate')
|
||
request.resolver_match = SimpleNamespace(kwargs={'knowledge_id': str(self.root.id),
|
||
'target_knowledge_id': str(version.snapshot_id)})
|
||
with self.assertRaises(AppUnauthorizedFailed): v.guard_snapshot_request(request)
|
||
|
||
def test_published_chat_keeps_published_association(self):
|
||
from application.serializers.common import ChatInfo
|
||
version = self.create()
|
||
app = Application.objects.create(name='发布测试', user=self.user)
|
||
ApplicationVersion.objects.create(application=app, knowledge_setting={
|
||
'_knowledge_ids': [str(self.root.id)], 'version_selection': {str(self.root.id): str(version.id)}})
|
||
self.assertFalse(ApplicationKnowledgeMapping.objects.filter(application=app).exists())
|
||
chat = ChatInfo(str(uuid.uuid4()), str(self.user.id), 'PLATFORM_USER', [], [], str(app.id))
|
||
chat.get_application()
|
||
self.assertEqual(chat.knowledge_id_list, [str(self.root.id)])
|
||
self.assertEqual(chat.knowledge_version_map, {str(self.root.id): str(version.id)})
|
||
|
||
def test_feedback_body_cannot_write_snapshot(self):
|
||
from application.serializers.application_chat_record import ApplicationChatRecordAddKnowledgeSerializer
|
||
version = self.create(); doc = Document.objects.get(knowledge=version.snapshot)
|
||
app = Application.objects.create(name='改进测试', user=self.user)
|
||
serializer = ApplicationChatRecordAddKnowledgeSerializer(data={'workspace_id': 'default',
|
||
'application_id': str(app.id), 'knowledge_id': str(version.snapshot_id),
|
||
'document_id': str(doc.id), 'chat_ids': []})
|
||
with self.assertRaises(NotFound404): serializer.post_improve({}, with_valid=False)
|
||
|
||
def test_model_changes_fail_explicitly(self):
|
||
version = self.create(); self.model.model_name = 'different'; self.model.save()
|
||
with self.assertRaises(ValidationError): v.resolve_search([self.root.id], {str(self.root.id): str(version.id)}, 'default')
|
||
|
||
def test_internal_snapshot_api_guard(self):
|
||
version = self.create(); request = RequestFactory().get('/admin/api/knowledge')
|
||
request.resolver_match = SimpleNamespace(kwargs={'knowledge_id': str(version.snapshot_id)})
|
||
with self.assertRaises(AppUnauthorizedFailed): v.guard_snapshot_request(request)
|
||
with self.assertRaises(NotFound404): v.root_knowledge('default', version.snapshot_id)
|
||
|
||
def test_api_read_permission_and_workspace(self):
|
||
from rest_framework.test import APIRequestFactory, force_authenticate
|
||
from knowledge.views.version import KnowledgeVersionView
|
||
self.create(); factory = APIRequestFactory()
|
||
for roles, workspace, expected in [([], 'default', 403), (['ADMIN'], 'default', 200), (['ADMIN'], 'other', 404)]:
|
||
req = factory.get('/versions')
|
||
force_authenticate(req, user=self.user, token=SimpleNamespace(role_list=roles, permission_list=[]))
|
||
response = KnowledgeVersionView.as_view()(req, workspace_id=workspace, knowledge_id=self.root.id)
|
||
self.assertEqual(response.status_code, expected)
|
||
|
||
def test_empty_name_rejected(self):
|
||
with self.assertRaises(ValidationError): self.create(' ')
|
||
|
||
def test_real_postgres_retrieval_reads_frozen_content(self):
|
||
from django.db import connection
|
||
if connection.vendor != 'postgresql':
|
||
self.skipTest('Real pgvector retrieval requires isolated PostgreSQL schema')
|
||
from application.chat_pipeline.step.search_dataset_step.impl import base_search_dataset_step as search
|
||
version = self.create()
|
||
self.para.content = '新答案:200元'; self.para.save()
|
||
manage = SimpleNamespace(context={'knowledge_version_map': {str(self.root.id): str(version.id)}, 'chat_user_type': 'PLATFORM_USER'})
|
||
model = SimpleNamespace(model_type='EMBEDDING', name='测试向量')
|
||
embedding = SimpleNamespace(embed_query=lambda text: [1.0, 0.0, 0.0])
|
||
with patch.object(search, 'get_model_by_id', return_value=model), patch.object(search, 'get_embedding_model_by_knowledge_id_list', return_value=embedding):
|
||
result = search.BaseSearchDatasetStep().execute('价格', [str(self.root.id)], [], [], 3, 0.1,
|
||
search_mode='embedding', workspace_id='default', manage=manage)
|
||
self.assertEqual(len(result), 1)
|
||
self.assertEqual(result[0].content, '旧答案:100元')
|
||
self.assertIn('v1', result[0].knowledge_name)
|
||
|
||
def test_real_postgres_workflow_retrieves_frozen_document(self):
|
||
from django.db import connection
|
||
if connection.vendor != 'postgresql': self.skipTest('Real workflow pgvector retrieval requires PostgreSQL')
|
||
from application.flow.step_node.search_knowledge_node.impl import base_search_knowledge_node as search
|
||
version = self.create(); self.para.content = '新答案:200元'; self.para.save()
|
||
body = {'knowledge_version_map': {str(self.root.id): str(version.id)},
|
||
'chat_user_type': 'PLATFORM_USER', 'workspace_id': 'default'}
|
||
node = search.BaseSearchKnowledgeNode.__new__(search.BaseSearchKnowledgeNode)
|
||
node.context = {}; node.workflow_manage = SimpleNamespace(get_body=lambda: body,
|
||
get_reference_field=lambda *_: [str(self.doc.id)])
|
||
embedding = SimpleNamespace(embed_query=lambda _: [1.0, 0.0, 0.0])
|
||
with patch.object(search, 'get_model_instance_by_model_workspace_id'), \
|
||
patch.object(search, 'get_embedding_model_by_knowledge_id_list', return_value=embedding), \
|
||
patch.object(search.connection, 'close'):
|
||
result = node.execute([str(self.root.id)], {'top_n':3, 'similarity':0.1, 'search_mode':'embedding'},
|
||
'价格', True, 'referencing', 'document', ['previous', 'documents'])
|
||
self.assertEqual(result.node_variable['paragraph_list'][0]['content'], '旧答案:100元')
|
||
|
||
def test_api_create_browse_and_delete(self):
|
||
from rest_framework.test import APIRequestFactory, force_authenticate
|
||
from knowledge.views.version import KnowledgeVersionView
|
||
factory = APIRequestFactory()
|
||
def call(method, data=None, version_id=None):
|
||
request = getattr(factory, method)('/versions', data=data or {}, format='json')
|
||
force_authenticate(request, user=self.user, token=SimpleNamespace(role_list=['ADMIN'], permission_list=[]))
|
||
kw = {'workspace_id': 'default', 'knowledge_id': self.root.id}
|
||
if version_id: kw['version_id'] = version_id
|
||
return KnowledgeVersionView.as_view()(request, **kw)
|
||
created = call('post', {'name': '接口创建'})
|
||
self.assertEqual(json.loads(created.content)['code'], 200)
|
||
version_id = json.loads(created.content)['data']['id']
|
||
detail = call('get', version_id=version_id)
|
||
self.assertEqual(json.loads(detail.content)['data']['documents'][0]['name'], self.doc.name)
|
||
removed = call('delete', version_id=version_id)
|
||
self.assertEqual(json.loads(removed.content)['code'], 200)
|