WeKnora/internal/agent/tools/search_knowledge_rerank_tes...

293 lines
9.3 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters!

This file contains ambiguous Unicode characters that may be confused with others in your current locale. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to highlight these characters.

package tools
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/config"
"github.com/Tencent/WeKnora/internal/models/rerank"
"github.com/Tencent/WeKnora/internal/types"
)
// stubReranker returns canned scores (or an error) without any network call.
type stubReranker struct {
scores []float64
err error
calls int
documents []string
}
func (s *stubReranker) Rerank(
_ context.Context, _ string, documents []string,
) ([]rerank.RankResult, error) {
s.calls++
s.documents = documents
if s.err != nil {
return nil, s.err
}
out := make([]rerank.RankResult, 0, len(documents))
for i := range documents {
score := 0.0
if i < len(s.scores) {
score = s.scores[i]
}
out = append(out, rerank.RankResult{Index: i, RelevanceScore: score})
}
return out, nil
}
func (s *stubReranker) GetModelName() string { return "stub-rerank" }
func (s *stubReranker) GetModelID() string { return "stub-rerank-id" }
func newRerankTestTool(model rerank.Reranker) *SearchKnowledgeTool {
return &SearchKnowledgeTool{
rerankModel: model,
config: &config.Config{
Conversation: &config.ConversationConfig{RerankThreshold: 0.3},
},
}
}
func newRerankTestResults() []*searchResultWithMeta {
return []*searchResultWithMeta{
{SearchResult: &types.SearchResult{ID: "c1", Content: "alpha", Score: 0.02}},
{SearchResult: &types.SearchResult{ID: "c2", Content: "beta", Score: 0.01}},
}
}
func TestFilterRerankRankResults_thresholdAndFallback(t *testing.T) {
t.Parallel()
rankResults := []rerank.RankResult{
{Index: 0, RelevanceScore: 0.05},
{Index: 1, RelevanceScore: 0.02},
}
filtered := filterRerankRankResults(rankResults, 0.3, false)
if len(filtered) != 0 {
t.Fatalf("expected empty filter, got %#v", filtered)
}
rankResults = []rerank.RankResult{
{Index: 0, RelevanceScore: 0.05},
{Index: 1, RelevanceScore: 0.20},
}
filtered = filterRerankRankResults(rankResults, 0.3, false)
if len(filtered) != 1 || filtered[0].Index != 1 {
t.Fatalf("expected fallback top score, got %#v", filtered)
}
rankResults = []rerank.RankResult{
{Index: 0, RelevanceScore: 0.05},
{Index: 1, RelevanceScore: 0.02},
}
filtered = filterRerankRankResults(rankResults, 0.3, true)
if len(filtered) != 1 || filtered[0].Index != 0 {
t.Fatalf("expected explicit scope to preserve top result, got %#v", filtered)
}
rankResults = []rerank.RankResult{
{Index: 0, RelevanceScore: 0.8},
{Index: 1, RelevanceScore: 0.4},
{Index: 2, RelevanceScore: 0.1},
}
filtered = filterRerankRankResults(rankResults, 0.3, false)
if len(filtered) != 2 {
t.Fatalf("expected 2 passing scores, got %#v", filtered)
}
}
func TestApplyModelRerankScores_faqUsesCompositeScale(t *testing.T) {
t.Parallel()
tool := &SearchKnowledgeTool{
config: &config.Config{
Conversation: &config.ConversationConfig{RerankThreshold: 0.3},
},
}
originals := []*searchResultWithMeta{
{
SearchResult: &types.SearchResult{ID: "faq-1", Content: "Q: WeKnora", Score: 0.011},
KnowledgeBaseType: types.KnowledgeBaseTypeFAQ,
},
{
SearchResult: &types.SearchResult{ID: "doc-1", Content: "swimming club", Score: 0.02},
},
}
rankResults := []rerank.RankResult{
{Index: 0, RelevanceScore: 0.05},
{Index: 1, RelevanceScore: 0.9},
}
out := tool.applyModelRerankScores(originals, rankResults, 0.3, false)
if len(out) != 1 || out[0].ID != "doc-1" {
t.Fatalf("weak FAQ should be filtered out, got %#v", out)
}
if out[0].Score <= 0.011 {
t.Fatalf("composite score should exceed raw retrieval score, got %.4f", out[0].Score)
}
}
// A rerank API failure must degrade to the raw retrieval order rather than
// dropping the recall set or re-scoring it with a chat model.
func TestRerankResults_modelErrorKeepsRawResults(t *testing.T) {
t.Parallel()
model := &stubReranker{err: errors.New("upstream 500")}
tool := newRerankTestTool(model)
results := newRerankTestResults()
out, err := tool.rerankResults(context.Background(), "query", results)
if err != nil {
t.Fatalf("rerankResults returned error: %v", err)
}
if len(out) != len(results) {
t.Fatalf("expected raw results to be preserved, got %d", len(out))
}
for i := range out {
if out[i] != results[i] {
t.Fatalf("result %d was replaced: %#v", i, out[i])
}
}
if model.calls != 1 {
t.Fatalf("expected exactly one rerank call, got %d", model.calls)
}
}
// Scores below agentRerankFallbackMinScore mean nothing is relevant; the tool
// must return empty instead of resurrecting the candidates.
func TestRerankResults_allBelowFallbackFloorReturnsEmpty(t *testing.T) {
t.Parallel()
tool := newRerankTestTool(&stubReranker{scores: []float64{0.10, 0.04}})
out, err := tool.rerankResults(context.Background(), "query", newRerankTestResults())
if err != nil {
t.Fatalf("rerankResults returned error: %v", err)
}
if len(out) != 0 {
t.Fatalf("expected no results, got %#v", out)
}
}
func TestRerankResults_keepsCandidatesAboveThreshold(t *testing.T) {
t.Parallel()
tool := newRerankTestTool(&stubReranker{scores: []float64{0.9, 0.05}})
out, err := tool.rerankResults(context.Background(), "query", newRerankTestResults())
if err != nil {
t.Fatalf("rerankResults returned error: %v", err)
}
if len(out) != 1 || out[0].ID != "c1" {
t.Fatalf("expected only the strong candidate, got %#v", out)
}
}
func TestRerankResults_withoutModelIsPassthrough(t *testing.T) {
t.Parallel()
tool := newRerankTestTool(nil)
results := newRerankTestResults()
out, err := tool.rerankResults(context.Background(), "query", results)
if err != nil {
t.Fatalf("rerankResults returned error: %v", err)
}
if len(out) != len(results) {
t.Fatalf("expected passthrough, got %d results", len(out))
}
}
// A chunk rarely names its document's subject, so the rerank model must see
// the document title; FAQ entries are scored on their own question.
func TestRerankScores_prefixDocumentTitle(t *testing.T) {
t.Parallel()
model := &stubReranker{scores: []float64{0.5, 0.5, 0.5}}
tool := newRerankTestTool(model)
results := []*searchResultWithMeta{
{SearchResult: &types.SearchResult{
ID: "c1", Content: "allocate inference across open-weight models",
KnowledgeTitle: " Show HN: Echo ", ChunkType: string(types.ChunkTypeText),
}},
{SearchResult: &types.SearchResult{
ID: "c2", Content: "What is Echo?", KnowledgeTitle: "FAQ set", ChunkType: string(types.ChunkTypeFAQ),
}},
{SearchResult: &types.SearchResult{ID: "c3", Content: "untitled"}},
}
if _, err := tool.rerankScores(context.Background(), "query", results); err != nil {
t.Fatalf("rerankScores returned error: %v", err)
}
want := []string{"Show HN: Echo\n\nallocate inference across open-weight models", "What is Echo?", "untitled"}
for i, w := range want {
if model.documents[i] != w {
t.Fatalf("passage %d = %q, want %q", i, model.documents[i], w)
}
}
}
// Title is shared by every chunk of a document, so a passage whose body never
// names the query subject still goes to the reranker as title + chunk.
func TestRerankPassage_prefixesTitleWhenChunkOmitsSubject(t *testing.T) {
t.Parallel()
got := (&SearchKnowledgeTool{}).rerankPassage(context.Background(), &types.SearchResult{
Content: "installation prerequisites and docker compose flags",
KnowledgeTitle: "Show HN: Echo – Fable-level results at 1/3 the cost using open-weight models",
ChunkType: string(types.ChunkTypeText),
})
if !strings.HasPrefix(got, "Show HN: Echo") {
t.Fatalf("expected document title prefix, got %q", got)
}
if !strings.Contains(got, "installation prerequisites") {
t.Fatalf("chunk body must still be present: %q", got)
}
}
// Execute must surface rerank_rejected through Data and the empty statement
// when the model scores every retrieved candidate below the fallback floor.
func TestExecuteReportsRerankRejection(t *testing.T) {
t.Parallel()
model := &stubReranker{scores: []float64{0.10, 0.04}}
svc := &stubKnowledgeBaseService{
results: []*types.SearchResult{
{
ID: "c1", Content: "comments section", KnowledgeID: "d1", KnowledgeBaseID: "kb-1",
KnowledgeTitle: "Show HN: Echo", Score: 0.8,
},
{
ID: "c2", Content: "unrelated notes", KnowledgeID: "d2", KnowledgeBaseID: "kb-1",
KnowledgeTitle: "Other", Score: 0.7,
},
},
}
tool := NewSearchKnowledgeTool(
svc, nil, nil,
types.SearchTargets{{
Type: types.SearchTargetTypeKnowledgeBase, KnowledgeBaseID: "kb-1", TenantID: 1,
}},
model,
&config.Config{Conversation: &config.ConversationConfig{RerankThreshold: 0.3}},
)
res, err := tool.Execute(context.Background(), json.RawMessage(
`{"query":"Why does Echo use open-weight models to cut cost?"}`,
))
if err != nil || res == nil || !res.Success {
t.Fatalf("Execute: res=%+v err=%v", res, err)
}
if got, _ := res.Data["rerank_rejected"].(int); got != 2 {
t.Fatalf("rerank_rejected = %v, want 2; data=%+v", res.Data["rerank_rejected"], res.Data)
}
if !strings.Contains(res.Output, "found 2 candidate chunks") ||
!strings.Contains(res.Output, "Repeating the same query") ||
!strings.Contains(res.Output, "mode=keyword") {
t.Fatalf("empty statement = %q", res.Output)
}
if strings.Contains(res.Output, "switching mode will not help") {
t.Fatalf("statement must not forbid a keyword retry: %q", res.Output)
}
}
func TestRerankThreshold_default(t *testing.T) {
t.Parallel()
tool := &SearchKnowledgeTool{}
if got := tool.rerankThreshold(); got != 0.3 {
t.Fatalf("default threshold = %v, want 0.3", got)
}
}