panyy
/
UnisKB
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf: 优化历史数据分词处理兜底操作 (#313)

v3.2
shaohuzhang1 2024-04-29 14:09:58 +08:00 committed by GitHub
parent a788d8f3b8
commit 62c959f905
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 24 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
apps/embedding/migrations/0002_embedding_search_vector.py
View File

@ -18,27 +18,30 @@ def update_embedding_search_vector(embedding, paragraph_list):
def save_keywords(apps, schema_editor): def save_keywords(apps, schema_editor):
document = apps.get_model("dataset", "Document") try:
embedding = apps.get_model("embedding", "Embedding") document = apps.get_model("dataset", "Document")
paragraph = apps.get_model('dataset', 'Paragraph') embedding = apps.get_model("embedding", "Embedding")
db_alias = schema_editor.connection.alias paragraph = apps.get_model('dataset', 'Paragraph')
document_list = document.objects.using(db_alias).all() db_alias = schema_editor.connection.alias
for document in document_list: document_list = document.objects.using(db_alias).all()
document.status = Status.embedding for document in document_list:
document.save() document.status = Status.embedding
paragraph_list = paragraph.objects.using(db_alias).filter(document=document).all() document.save()
embedding_list = embedding.objects.using(db_alias).filter(document=document).values('id', 'search_vector', paragraph_list = paragraph.objects.using(db_alias).filter(document=document).all()
'paragraph') embedding_list = embedding.objects.using(db_alias).filter(document=document).values('id', 'search_vector',
embedding_update_list = [update_embedding_search_vector(embedding, paragraph_list) for embedding 'paragraph')
in embedding_list] embedding_update_list = [update_embedding_search_vector(embedding, paragraph_list) for embedding
child_array = sub_array(embedding_update_list, 50) in embedding_list]
for c in child_array: child_array = sub_array(embedding_update_list, 50)
try: for c in child_array:
embedding.objects.using(db_alias).bulk_update(c, ['search_vector']) try:
except Exception as e: embedding.objects.using(db_alias).bulk_update(c, ['search_vector'])
print(e) except Exception as e:
document.status = Status.success print(e)
document.save() document.status = Status.success
document.save()
except Exception as e:
print(e)
class Migration(migrations.Migration): class Migration(migrations.Migration):