WeKnora/internal/agent/tools/data_analysis_query_test.go

216 lines
6.8 KiB
Go
Raw Normal View History

refactor(modelcontext): consolidate source_refs parsing, fixed data_analysis table, and identifier leak report (#3372) * refactor(wiki): consolidate source_refs parsing and resolution source_refs entries ("uuid" or "uuid|title") were parsed by ten independent implementations that disagreed on whitespace handling, on whether a model-supplied title could be trusted, and on how a leading pipe was treated. The scope-enforced and legacy agent resolvers also duplicated the lookup and the routing step looked every document up a second time. - types.ParseWikiSourceRef / FormatWikiSourceRef / WikiSourceRefMatchesKnowledge are now the only readers and writer of the format. - resolveWikiSourceDocuments replaces resolveAuthorizedSourceRefs, resolveSourceRefs and wikiKnowledgeBasesForSourceRefs: one lookup per document, titles always rebuilt from server data, dedupe by canonical ID, unknown documents fail closed in every mode. - removeSourceRef now matches padded or titled refs of a deleted document. * refactor(data_analysis): expose the selected document as a fixed table name The tool required the document ID inside the SQL text and blindly string-replaced it with the physical DuckDB table name, which forced the model-context codec to rewrite handles inside quoted SQL and let the UUID-derived k_<uuid> table name reach the model through stored schema summaries. The document is now selected only by knowledge_id. Every query runs on a dedicated connection where "dataset" is a temporary view over the physical table (DuckDB temporary objects are connection-local, so concurrent sessions stay isolated). The physical name remains accepted for internal callers; any other table fails validation with an error naming the table the model should use. Schema summaries, the schema tool, the analyst prompt and the tool schema all name the same table, so data_analysis.sql no longer carries source handles and its text codec policy is removed. * feat(modelcontext): report raw identifiers that survive encoding Text-level compaction exists because raw UUIDs reach model context through producers the registry never sees: error strings, slugs, results of unknown shape. Retiring it requires knowing where those producers are. Registry.LeakedIdentifiers scans an encoded message list for UUID-shaped values in every application-composed field (user messages are skipped), classifies each as registered (codec missed a rewrite) or unregistered (producer emitted a raw ID), and SummarizeLeaks renders one line per field. The agent engine and the three RAG pipeline call sites log the report as a warning after EncodeMessages. * fix(chat_pipeline): align RAG data analysis with the fixed table contract The RAG data-analysis plugin still asked the planning model to fill in knowledge_id and to write SQL from a prompt that exposed the raw knowledge UUID, then passed the model's JSON straight to Execute. Under the new data_analysis contract a query that names anything but the dataset table is rejected, and the plugin logged that rejection at error level and moved on, so the analysis vanished without any signal. A "no analysis needed" plan took the same error path. The pipeline now owns document selection: the model returns only needs_analysis and sql, the prompt names the dataset table and never shows the knowledge ID or physical table, and the plugin builds the tool input itself. A rejected query is fed back to the model once for repair; a query that still fails is reported through pipelineWarn with the SQL and the tool error, and "not needed" is logged as such instead of as a failure. Covered by unit tests with a scripted model and executor: not-needed, pipeline-owned knowledge_id, one-shot repair, persistent failure surfaced, unparseable plan rejected.
2026-09-17 14:08:16 +00:00
package tools
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"testing"
_ "github.com/duckdb/duckdb-go/v2"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
)
// newPlainDuckDB opens an in-memory DuckDB without optional extensions; CSV
// loading needs none of them.
func newPlainDuckDB(t *testing.T) *sql.DB {
t.Helper()
db, err := sql.Open("duckdb", ":memory:")
if err != nil {
t.Fatalf("open duckdb: %v", err)
}
t.Cleanup(func() { _ = db.Close() })
return db
}
func writeCSV(t *testing.T, name string, rows ...string) string {
t.Helper()
path := filepath.Join(t.TempDir(), name)
if err := os.WriteFile(path, []byte(strings.Join(rows, "\n")+"\n"), 0o600); err != nil {
t.Fatalf("write csv: %v", err)
}
return path
}
type csvFileService struct {
interfaces.FileService
files map[string]string // file path -> CSV content
}
func (s *csvFileService) GetFile(_ context.Context, filePath string) (io.ReadCloser, error) {
content, ok := s.files[filePath]
if !ok {
return nil, fmt.Errorf("no such file %q", filePath)
}
return io.NopCloser(strings.NewReader(content)), nil
}
func TestValidateDataAnalysisSQLAcceptsLogicalAndPhysicalTables(t *testing.T) {
schema := &TableSchema{TableName: "k_doc_a"}
for _, query := range []string{
`SELECT COUNT(*) FROM dataset`,
`SELECT * FROM "dataset" WHERE "amount" > 1`,
`SELECT * FROM "k_doc_a" LIMIT 10`,
} {
if err := validateDataAnalysisSQL(query, schema); err != nil {
t.Errorf("%s: unexpected error %v", query, err)
}
}
}
func TestValidateDataAnalysisSQLRejectsOtherTablesWithGuidance(t *testing.T) {
schema := &TableSchema{TableName: "k_doc_a"}
for _, query := range []string{
`SELECT * FROM d1`,
`SELECT * FROM "d1"`,
`SELECT * FROM k_doc_b`,
`SELECT * FROM "3f1c2a8e-0000-4000-8000-000000000001"`,
} {
err := validateDataAnalysisSQL(query, schema)
if err == nil {
t.Errorf("%s: must be rejected", query)
continue
}
if !strings.Contains(err.Error(), `"dataset"`) {
t.Errorf("%s: error must tell the model to use the dataset table, got %v", query, err)
}
}
if err := validateDataAnalysisSQL(`DROP TABLE dataset`, schema); err == nil {
t.Error("non-SELECT statements must be rejected")
}
if err := validateDataAnalysisSQL(`SELECT * FROM 'd1'`, schema); err == nil {
t.Error("a quoted literal in FROM must be rejected")
}
}
func TestExecuteSingleQueryExposesSelectedDocumentAsDataset(t *testing.T) {
db := newPlainDuckDB(t)
tool := &DataAnalysisTool{BaseTool: dataAnalysisTool, db: db, sessionID: "test-dataset"}
ctx := context.Background()
t.Cleanup(func() { tool.Cleanup(ctx) })
if _, err := tool.LoadFromCSV(ctx, writeCSV(t, "a.csv", "id,amount", "1,10", "2,20"), "k_doc_a"); err != nil {
t.Fatalf("load a: %v", err)
}
if _, err := tool.LoadFromCSV(ctx, writeCSV(t, "b.csv", "id,amount", "1,5", "2,6", "3,7"), "k_doc_b"); err != nil {
t.Fatalf("load b: %v", err)
}
rowsA, err := tool.executeSingleQuery(ctx, `SELECT COUNT(*) AS n FROM dataset`, "k_doc_a")
if err != nil || len(rowsA) != 1 || rowsA[0]["n"] != "2" {
t.Fatalf("doc a via dataset: rows=%v err=%v", rowsA, err)
}
rowsB, err := tool.executeSingleQuery(ctx, `SELECT COUNT(*) AS n FROM "dataset"`, "k_doc_b")
if err != nil || len(rowsB) != 1 || rowsB[0]["n"] != "3" {
t.Fatalf("doc b via dataset: rows=%v err=%v", rowsB, err)
}
// The physical name keeps working for internal callers.
rowsPhys, err := tool.executeSingleQuery(ctx, `SELECT COUNT(*) AS n FROM "k_doc_a"`, "k_doc_a")
if err != nil || len(rowsPhys) != 1 || rowsPhys[0]["n"] != "2" {
t.Fatalf("doc a via physical table: rows=%v err=%v", rowsPhys, err)
}
// The view never outlives the query: nothing named dataset is visible on
// the shared pool afterwards.
if _, err := db.ExecContext(ctx, `SELECT COUNT(*) FROM dataset`); err == nil {
t.Fatal("temporary dataset view leaked past the query")
}
}
func TestExecuteSingleQueryIsolatesConcurrentDocuments(t *testing.T) {
db := newPlainDuckDB(t)
tool := &DataAnalysisTool{BaseTool: dataAnalysisTool, db: db, sessionID: "test-concurrent"}
ctx := context.Background()
t.Cleanup(func() { tool.Cleanup(ctx) })
const docs = 4
for i := 0; i < docs; i++ {
rows := []string{"id"}
for j := 0; j <= i; j++ {
rows = append(rows, fmt.Sprint(j))
}
name := fmt.Sprintf("k_doc_%d", i)
if _, err := tool.LoadFromCSV(ctx, writeCSV(t, name+".csv", rows...), name); err != nil {
t.Fatalf("load %s: %v", name, err)
}
}
var wg sync.WaitGroup
errs := make(chan error, docs*20)
for round := 0; round < 20; round++ {
for i := 0; i < docs; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
rows, err := tool.executeSingleQuery(
ctx, `SELECT COUNT(*) AS n FROM dataset`, fmt.Sprintf("k_doc_%d", i),
)
if err != nil {
errs <- err
return
}
if want := fmt.Sprint(i + 1); len(rows) != 1 || rows[0]["n"] != want {
errs <- fmt.Errorf("doc %d saw rows=%v, want n=%s", i, rows, want)
}
}(i)
}
}
wg.Wait()
close(errs)
for err := range errs {
t.Error(err)
}
}
func TestDataAnalysisExecuteEndToEndUsesDatasetTable(t *testing.T) {
db := newPlainDuckDB(t)
knowledge := &types.Knowledge{ID: "3f1c2a8e-0000-4000-8000-000000000001", FileType: "csv", FilePath: "doc-a.csv"}
tool := NewDataAnalysisTool(
nil,
&mapKnowledgeService{docs: map[string]*types.Knowledge{knowledge.ID: knowledge}},
nil,
&csvFileService{files: map[string]string{"doc-a.csv": "id,amount\n1,10\n2,20\n3,30\n"}},
db,
"test-e2e",
)
ctx := context.Background()
t.Cleanup(func() { tool.Cleanup(ctx) })
args, _ := json.Marshal(DataAnalysisInput{
KnowledgeID: knowledge.ID,
SQL: `SELECT COUNT(*) AS total FROM dataset WHERE "amount" <> '20'`,
})
result, err := tool.Execute(ctx, args)
if err != nil || result == nil || !result.Success {
t.Fatalf("execute failed: result=%+v err=%v", result, err)
}
if !strings.Contains(result.Output, `"total":"2"`) {
t.Fatalf("unexpected output:\n%s", result.Output)
}
// Neither the physical table name nor the knowledge ID is echoed back to
// the model through the query text.
if strings.Contains(result.Output, "k_3f1c2a8e") || strings.Contains(result.Output, knowledge.ID) {
t.Fatalf("output leaks the physical table or knowledge ID:\n%s", result.Output)
}
// The legacy shape (document ID substituted into SQL) now fails with
// guidance instead of being blindly rewritten.
args, _ = json.Marshal(DataAnalysisInput{
KnowledgeID: knowledge.ID,
SQL: fmt.Sprintf(`SELECT COUNT(*) FROM "%s"`, knowledge.ID),
})
result, err = tool.Execute(ctx, args)
if err == nil || result == nil || result.Success {
t.Fatalf("knowledge ID inside SQL must be rejected: result=%+v err=%v", result, err)
}
if !strings.Contains(result.Error, `"dataset"`) {
t.Fatalf("rejection must name the dataset table: %s", result.Error)
}
}