2026-04-20 02:18:51 +00:00
|
|
|
package repository
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"testing"
|
|
|
|
|
|
|
|
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
|
|
|
"github.com/google/uuid"
|
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
|
"gorm.io/driver/sqlite"
|
|
|
|
|
"gorm.io/gorm"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
// knowledgeBasesTestDDL mirrors the `knowledge_bases` section of
|
|
|
|
|
// migrations/sqlite/000000_init.up.sql. We inline the DDL here instead of
|
|
|
|
|
// using GORM AutoMigrate because KnowledgeBase carries fields tagged with
|
|
|
|
|
// `type:jsonb`, which AutoMigrate does not map cleanly onto SQLite.
|
|
|
|
|
const knowledgeBasesTestDDL = `
|
|
|
|
|
CREATE TABLE IF NOT EXISTS knowledge_bases (
|
feat(knowledge): derive knowledge base descriptions from document profiles (#3370)
* feat(knowledge): derive knowledge base descriptions from document profiles
Knowledge bases only had a hand-written description, which most users
leave empty, so agents bound to several knowledge bases had nothing to
route on and searched all of them. This adds a generated description
that is derived from per-document data instead of maintained by a model.
Document profile: the summary call now returns JSON with a 2-3 sentence
summary plus a gist, 3-5 topic keywords, a document type and one
typical question (knowledges.profile). Input is capped at 8k chars and
output at 1024 tokens; plain-text replies from custom templates still
work and simply carry no profile. The summary chunk embeds the gist and
short summary.
Knowledge base aggregate: a database-only fold over those rows (counts
per file type, tag, normalized topic and doc type; stratified samples of
titles and questions; a hash of the inputs). Deleting, moving or
re-parsing a document changes nothing but the rows, so the next rebuild
is exact without any model judgement.
Description text: one small model call over the aggregate yields a
gist, merged topics and typical questions (knowledge_bases.
generated_profile). The call is skipped when the aggregate hash is
unchanged; failures keep the previous text and record the error; an
empty knowledge base clears it. The manual description is never
touched, and the agent runtime context renders both so the model can
pass knowledge_base_ids instead of fanning out.
Refreshes are requested from summary task exit, post-process completion,
deletion, cross-KB moves and enabling the option, debounced per
knowledge base in 30 s windows via kb:profile on the summary queue.
POST /knowledge-bases/:id/profile/generate regenerates synchronously.
profile_config (enabled, model_id, custom_instructions) opts a document
knowledge base in; the settings dialog shows the generated card with
regenerate and adopt-as-description actions in five locales. Search
results now log hits per chunk type so the value of indexing summary
chunks can be measured.
Migrations 000101 (PostgreSQL) and 000021 (SQLite) add the columns; the
SQLite schema test expects the new version and columns.
* fix(knowledge): keep generated descriptions out of duplicates and refresh clones
DuplicateKnowledgeBase copies the source row through JSON, which carried
generated_profile onto a knowledge base that has no documents yet, so
an empty duplicate advertised the source's topics. The duplicate now
starts without a generated description and keeps profile_config so it
derives its own once documents arrive.
CopyKnowledgeBase builds the target field by field and skipped
profile_config, and the clone task never asked for a description after
copying the documents, so a clone stayed undescribed until its next
upload. The target now carries a copy of profile_config and both clone
paths request a refresh on completion.
* fix(knowledge): refresh the loaded clone target and carry document profiles
ProcessKBClone requested the description refresh with the reservation
stub it builds before CopyKnowledgeBase runs. For the main create path
that stub has neither a type nor profile_config, so the refresh helper
rejected it and nothing was enqueued. The refresh now uses the loaded
target returned by CopyKnowledgeBase.
cloneKnowledge copied the summary text but not the structured profile,
so even a scheduled refresh would have aggregated titles without topics
until every clone was re-summarized. Cloned documents now carry an
independent copy of the profile.
2026-09-17 14:21:55 +00:00
|
|
|
profile_config TEXT,
|
|
|
|
|
generated_profile TEXT,
|
2026-04-20 02:18:51 +00:00
|
|
|
id VARCHAR(36) PRIMARY KEY,
|
|
|
|
|
name VARCHAR(255) NOT NULL,
|
|
|
|
|
description TEXT,
|
|
|
|
|
tenant_id INTEGER NOT NULL,
|
|
|
|
|
type VARCHAR(32) NOT NULL DEFAULT 'document',
|
|
|
|
|
chunking_config TEXT NOT NULL DEFAULT '{}',
|
|
|
|
|
image_processing_config TEXT NOT NULL DEFAULT '{}',
|
|
|
|
|
embedding_model_id VARCHAR(64) NOT NULL,
|
|
|
|
|
summary_model_id VARCHAR(64) NOT NULL,
|
|
|
|
|
cos_config TEXT NOT NULL DEFAULT '{}',
|
|
|
|
|
storage_provider_config TEXT DEFAULT NULL,
|
|
|
|
|
vlm_config TEXT NOT NULL DEFAULT '{}',
|
|
|
|
|
extract_config TEXT NULL DEFAULT NULL,
|
|
|
|
|
faq_config TEXT,
|
|
|
|
|
question_generation_config TEXT NULL,
|
2026-08-05 13:56:58 +00:00
|
|
|
auto_tag_config TEXT NULL,
|
2026-04-20 02:18:51 +00:00
|
|
|
is_temporary BOOLEAN NOT NULL DEFAULT 0,
|
|
|
|
|
is_pinned INTEGER NOT NULL DEFAULT 0,
|
|
|
|
|
pinned_at DATETIME NULL,
|
|
|
|
|
asr_config TEXT,
|
|
|
|
|
vector_store_id VARCHAR(36),
|
2026-07-15 05:26:29 +00:00
|
|
|
storage_backend_id VARCHAR(36),
|
2026-05-17 03:36:33 +00:00
|
|
|
wiki_config TEXT,
|
|
|
|
|
indexing_strategy TEXT,
|
2026-05-13 11:47:48 +00:00
|
|
|
creator_id VARCHAR(36),
|
2026-04-20 02:18:51 +00:00
|
|
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
|
|
|
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
|
|
|
deleted_at DATETIME
|
|
|
|
|
);
|
|
|
|
|
`
|
|
|
|
|
|
|
|
|
|
// setupKBTestDB creates an in-memory SQLite database containing the
|
|
|
|
|
// knowledge_bases table with the vector_store_id column included, so that
|
|
|
|
|
// tests can exercise the GORM behavior of the VectorStoreID field.
|
|
|
|
|
func setupKBTestDB(t *testing.T) *gorm.DB {
|
|
|
|
|
t.Helper()
|
|
|
|
|
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
require.NoError(t, db.Exec(knowledgeBasesTestDDL).Error)
|
|
|
|
|
return db
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// makeKB builds a minimal KnowledgeBase suitable for insert. Only the columns
|
|
|
|
|
// required by the schema above and any field we explicitly care about are set;
|
|
|
|
|
// the rest are left at their Go zero values and serialize to "{}" via their
|
|
|
|
|
// custom JSON valuers.
|
|
|
|
|
func makeKB(vectorStoreID *string) *types.KnowledgeBase {
|
|
|
|
|
return &types.KnowledgeBase{
|
|
|
|
|
ID: uuid.New().String(),
|
|
|
|
|
Name: "test-kb",
|
|
|
|
|
TenantID: 1,
|
|
|
|
|
EmbeddingModelID: "test-embed",
|
|
|
|
|
SummaryModelID: "test-summary",
|
|
|
|
|
VectorStoreID: vectorStoreID,
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func kbStrPtr(s string) *string { return &s }
|
|
|
|
|
|
|
|
|
|
// reloadKB fetches the KnowledgeBase back from the database using its ID.
|
|
|
|
|
func reloadKB(t *testing.T, db *gorm.DB, id string) types.KnowledgeBase {
|
|
|
|
|
t.Helper()
|
|
|
|
|
var got types.KnowledgeBase
|
|
|
|
|
require.NoError(t, db.First(&got, "id = ?", id).Error)
|
|
|
|
|
return got
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// TestKnowledgeBase_VectorStoreID_Save_Immutable verifies that db.Save(kb)
|
|
|
|
|
// — which WeKnora's UpdateKnowledgeBase repository path uses today — does
|
|
|
|
|
// NOT overwrite the vector_store_id column, thanks to the GORM
|
|
|
|
|
// `<-:create` tag. This is the primary automated defense for the H1 review
|
|
|
|
|
// finding (silent overwrite of the binding via full-struct UPDATE).
|
|
|
|
|
func TestKnowledgeBase_VectorStoreID_Save_Immutable(t *testing.T) {
|
|
|
|
|
db := setupKBTestDB(t)
|
|
|
|
|
|
|
|
|
|
original := makeKB(kbStrPtr("store-A"))
|
|
|
|
|
require.NoError(t, db.Create(original).Error)
|
|
|
|
|
|
|
|
|
|
// Attempt to overwrite through the canonical Save() path.
|
|
|
|
|
mutated := *original
|
|
|
|
|
mutated.Name = "updated-name"
|
|
|
|
|
mutated.VectorStoreID = kbStrPtr("store-B")
|
|
|
|
|
require.NoError(t, db.Save(&mutated).Error)
|
|
|
|
|
|
|
|
|
|
reloaded := reloadKB(t, db, original.ID)
|
|
|
|
|
|
|
|
|
|
// Non-immutable fields must update.
|
|
|
|
|
assert.Equal(t, "updated-name", reloaded.Name, "Name must update through Save")
|
|
|
|
|
|
|
|
|
|
// Immutable field must NOT update.
|
|
|
|
|
require.NotNil(t, reloaded.VectorStoreID, "VectorStoreID must remain set to the original value")
|
|
|
|
|
assert.Equal(t, "store-A", *reloaded.VectorStoreID,
|
|
|
|
|
"db.Save must not overwrite vector_store_id (enforced by GORM `<-:create` tag)")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// TestKnowledgeBase_VectorStoreID_Updates_Immutable verifies that
|
|
|
|
|
// db.Updates(struct{...}) also honors `<-:create` and silently ignores the
|
|
|
|
|
// vector_store_id column.
|
|
|
|
|
func TestKnowledgeBase_VectorStoreID_Updates_Immutable(t *testing.T) {
|
|
|
|
|
db := setupKBTestDB(t)
|
|
|
|
|
|
|
|
|
|
original := makeKB(kbStrPtr("store-A"))
|
|
|
|
|
require.NoError(t, db.Create(original).Error)
|
|
|
|
|
|
|
|
|
|
// Partial struct update that includes vector_store_id must not take effect.
|
|
|
|
|
err := db.Model(&types.KnowledgeBase{}).
|
|
|
|
|
Where("id = ?", original.ID).
|
|
|
|
|
Updates(types.KnowledgeBase{
|
|
|
|
|
Name: "updated-name",
|
|
|
|
|
VectorStoreID: kbStrPtr("store-B"),
|
|
|
|
|
}).Error
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
|
|
reloaded := reloadKB(t, db, original.ID)
|
|
|
|
|
|
|
|
|
|
assert.Equal(t, "updated-name", reloaded.Name)
|
|
|
|
|
require.NotNil(t, reloaded.VectorStoreID)
|
|
|
|
|
assert.Equal(t, "store-A", *reloaded.VectorStoreID,
|
|
|
|
|
"db.Updates must not overwrite vector_store_id (enforced by GORM `<-:create` tag)")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// TestKnowledgeBase_VectorStoreID_Select_StillImmutable verifies that even
|
|
|
|
|
// an explicit Select("vector_store_id").Updates(...) does NOT bypass the
|
|
|
|
|
// `<-:create` tag. GORM's `<-:create` is strict: it withholds write
|
|
|
|
|
// permission from every UPDATE path, including selective updates. This is
|
|
|
|
|
// a stronger guarantee than a typical "column is protected unless you
|
|
|
|
|
// opt in" scheme.
|
|
|
|
|
//
|
|
|
|
|
// The implication for Phase 4 (cross-store migration) is that ORM-level
|
|
|
|
|
// updates cannot move bindings. Migration code must use raw SQL (db.Exec)
|
|
|
|
|
// — see TestKnowledgeBase_VectorStoreID_RawSQL_EscapeHatch below.
|
|
|
|
|
func TestKnowledgeBase_VectorStoreID_Select_StillImmutable(t *testing.T) {
|
|
|
|
|
db := setupKBTestDB(t)
|
|
|
|
|
|
|
|
|
|
original := makeKB(kbStrPtr("store-A"))
|
|
|
|
|
require.NoError(t, db.Create(original).Error)
|
|
|
|
|
|
|
|
|
|
err := db.Model(&types.KnowledgeBase{}).
|
|
|
|
|
Where("id = ?", original.ID).
|
|
|
|
|
Select("vector_store_id").
|
|
|
|
|
Updates(map[string]interface{}{"vector_store_id": "store-B"}).Error
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
|
|
reloaded := reloadKB(t, db, original.ID)
|
|
|
|
|
require.NotNil(t, reloaded.VectorStoreID)
|
|
|
|
|
assert.Equal(t, "store-A", *reloaded.VectorStoreID,
|
|
|
|
|
"Select(\"vector_store_id\").Updates must NOT bypass <-:create; ORM is fully locked")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// TestKnowledgeBase_VectorStoreID_RawSQL_EscapeHatch documents that raw
|
|
|
|
|
// SQL UPDATE bypasses the GORM tag system entirely, as expected. Phase 4
|
|
|
|
|
// cross-store migration is the only sanctioned caller of this path, and
|
|
|
|
|
// it must do so through a dedicated, narrowly-scoped repository method
|
|
|
|
|
// rather than through general Save/Updates/Select flows.
|
|
|
|
|
func TestKnowledgeBase_VectorStoreID_RawSQL_EscapeHatch(t *testing.T) {
|
|
|
|
|
db := setupKBTestDB(t)
|
|
|
|
|
|
|
|
|
|
original := makeKB(kbStrPtr("store-A"))
|
|
|
|
|
require.NoError(t, db.Create(original).Error)
|
|
|
|
|
|
|
|
|
|
require.NoError(t,
|
|
|
|
|
db.Exec("UPDATE knowledge_bases SET vector_store_id = ? WHERE id = ?",
|
|
|
|
|
"store-B", original.ID).Error)
|
|
|
|
|
|
|
|
|
|
reloaded := reloadKB(t, db, original.ID)
|
|
|
|
|
require.NotNil(t, reloaded.VectorStoreID)
|
|
|
|
|
assert.Equal(t, "store-B", *reloaded.VectorStoreID,
|
|
|
|
|
"raw SQL UPDATE bypasses GORM tags; Phase 4 migration must use this path through a dedicated method")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// TestKnowledgeBase_VectorStoreID_Roundtrip_Nil verifies that a nil
|
|
|
|
|
// VectorStoreID is persisted as SQL NULL and loaded back as nil.
|
|
|
|
|
func TestKnowledgeBase_VectorStoreID_Roundtrip_Nil(t *testing.T) {
|
|
|
|
|
db := setupKBTestDB(t)
|
|
|
|
|
|
|
|
|
|
original := makeKB(nil)
|
|
|
|
|
require.NoError(t, db.Create(original).Error)
|
|
|
|
|
|
|
|
|
|
reloaded := reloadKB(t, db, original.ID)
|
|
|
|
|
assert.Nil(t, reloaded.VectorStoreID,
|
|
|
|
|
"nil VectorStoreID must round-trip through the database as nil")
|
|
|
|
|
|
|
|
|
|
// Extra assurance: column is actually NULL, not an empty string.
|
|
|
|
|
var nullCount int64
|
|
|
|
|
require.NoError(t,
|
|
|
|
|
db.Table("knowledge_bases").
|
|
|
|
|
Where("id = ? AND vector_store_id IS NULL", original.ID).
|
|
|
|
|
Count(&nullCount).Error)
|
|
|
|
|
assert.Equal(t, int64(1), nullCount, "nil must be stored as SQL NULL")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// TestKnowledgeBase_VectorStoreID_Roundtrip_Value verifies that a non-nil
|
|
|
|
|
// VectorStoreID is persisted and loaded back with the same value.
|
|
|
|
|
func TestKnowledgeBase_VectorStoreID_Roundtrip_Value(t *testing.T) {
|
|
|
|
|
db := setupKBTestDB(t)
|
|
|
|
|
|
|
|
|
|
original := makeKB(kbStrPtr("store-uuid-42"))
|
|
|
|
|
require.NoError(t, db.Create(original).Error)
|
|
|
|
|
|
|
|
|
|
reloaded := reloadKB(t, db, original.ID)
|
|
|
|
|
require.NotNil(t, reloaded.VectorStoreID)
|
|
|
|
|
assert.Equal(t, "store-uuid-42", *reloaded.VectorStoreID)
|
|
|
|
|
}
|
2026-05-18 01:10:01 +00:00
|
|
|
|
|
|
|
|
// TestCountByVectorStoreID covers the binding-count helper used by the
|
|
|
|
|
// VectorStore delete guard. Verifies tenant scoping, the GORM auto soft-delete
|
|
|
|
|
// filter (no explicit `deleted_at IS NULL` literal in the query), and the
|
|
|
|
|
// edge case where a row stores an empty-string vector_store_id (SQLite
|
|
|
|
|
// treats "" and NULL differently — we want both excluded from a non-empty
|
|
|
|
|
// storeID lookup).
|
|
|
|
|
func TestCountByVectorStoreID(t *testing.T) {
|
|
|
|
|
db := setupKBTestDB(t)
|
|
|
|
|
repo := &knowledgeBaseRepository{db: db}
|
|
|
|
|
ctx := t.Context()
|
|
|
|
|
|
|
|
|
|
// Tenant 1: two active rows bound to store-A, one row bound to store-B.
|
|
|
|
|
kbA1 := makeKB(kbStrPtr("store-A"))
|
|
|
|
|
kbA2 := makeKB(kbStrPtr("store-A"))
|
|
|
|
|
kbB := makeKB(kbStrPtr("store-B"))
|
|
|
|
|
require.NoError(t, db.Create(kbA1).Error)
|
|
|
|
|
require.NoError(t, db.Create(kbA2).Error)
|
|
|
|
|
require.NoError(t, db.Create(kbB).Error)
|
|
|
|
|
|
|
|
|
|
// Tenant 2: one row bound to store-A (must not count for tenant 1).
|
|
|
|
|
kbCross := makeKB(kbStrPtr("store-A"))
|
|
|
|
|
kbCross.TenantID = 2
|
|
|
|
|
require.NoError(t, db.Create(kbCross).Error)
|
|
|
|
|
|
|
|
|
|
// Soft-deleted row bound to store-A (must not count under auto-scope).
|
|
|
|
|
kbDeleted := makeKB(kbStrPtr("store-A"))
|
|
|
|
|
require.NoError(t, db.Create(kbDeleted).Error)
|
|
|
|
|
require.NoError(t, db.Delete(kbDeleted).Error)
|
|
|
|
|
|
|
|
|
|
// Row with empty-string vector_store_id (regression — sqlite quirk).
|
|
|
|
|
kbEmpty := makeKB(kbStrPtr(""))
|
|
|
|
|
require.NoError(t, db.Create(kbEmpty).Error)
|
|
|
|
|
|
|
|
|
|
t.Run("nil db handle uses default", func(t *testing.T) {
|
|
|
|
|
count, err := repo.CountByVectorStoreID(ctx, nil, 1, "store-A")
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
assert.Equal(t, int64(2), count, "tenant 1 has 2 active KBs on store-A")
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("explicit db handle works the same", func(t *testing.T) {
|
|
|
|
|
count, err := repo.CountByVectorStoreID(ctx, db, 1, "store-A")
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
assert.Equal(t, int64(2), count)
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("different store id", func(t *testing.T) {
|
|
|
|
|
count, err := repo.CountByVectorStoreID(ctx, nil, 1, "store-B")
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
assert.Equal(t, int64(1), count)
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("unknown store id returns zero", func(t *testing.T) {
|
|
|
|
|
count, err := repo.CountByVectorStoreID(ctx, nil, 1, "store-XYZ")
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
assert.Equal(t, int64(0), count)
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("different tenant", func(t *testing.T) {
|
|
|
|
|
count, err := repo.CountByVectorStoreID(ctx, nil, 2, "store-A")
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
assert.Equal(t, int64(1), count)
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("non-empty lookup does not match empty-string rows", func(t *testing.T) {
|
|
|
|
|
count, err := repo.CountByVectorStoreID(ctx, nil, 1, "")
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
// Only the empty-string-vsid row matches "" exactly (SQLite quirk —
|
|
|
|
|
// "" is not NULL). The non-empty rows do not.
|
|
|
|
|
assert.Equal(t, int64(1), count, "empty-string lookup matches only the empty row")
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
t.Run("shared tx handle returns same result twice", func(t *testing.T) {
|
|
|
|
|
err := db.Transaction(func(tx *gorm.DB) error {
|
|
|
|
|
c1, err := repo.CountByVectorStoreID(ctx, tx, 1, "store-A")
|
|
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
c2, err := repo.CountByVectorStoreID(ctx, tx, 1, "store-A")
|
|
|
|
|
if err != nil {
|
|
|
|
|
return err
|
|
|
|
|
}
|
|
|
|
|
assert.Equal(t, c1, c2, "two reads within the same tx must agree")
|
|
|
|
|
assert.Equal(t, int64(2), c1)
|
|
|
|
|
return nil
|
|
|
|
|
})
|
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
})
|
|
|
|
|
}
|