1096 lines
36 KiB
Go
1096 lines
36 KiB
Go
package tools
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"math"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
|
|
"github.com/Tencent/WeKnora/internal/config"
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/models/rerank"
|
|
"github.com/Tencent/WeKnora/internal/searchutil"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"github.com/Tencent/WeKnora/internal/types/interfaces"
|
|
)
|
|
|
|
// Retrieval modes accepted by search_knowledge.
|
|
const (
|
|
SearchModeHybrid = "hybrid"
|
|
SearchModeSemantic = "semantic"
|
|
SearchModeKeyword = "keyword"
|
|
)
|
|
|
|
const (
|
|
searchKnowledgeDefaultLimit = 10
|
|
searchKnowledgeMaxLimit = 30
|
|
)
|
|
|
|
var searchKnowledgeTool = BaseTool{
|
|
name: ToolSearchKnowledge,
|
|
description: "Search the knowledge bases in scope and return the most relevant chunks with their content.\n" +
|
|
"mode \"hybrid\" (default) combines semantic and keyword retrieval. \"semantic\" ranks by meaning and " +
|
|
"tolerates paraphrase. \"keyword\" matches the literal terms and is the right choice for identifiers, " +
|
|
"error codes, product names and exact phrases. A base without the requested index is searched with the " +
|
|
"index it has; the result reports that as a mode fallback.\n" +
|
|
"In hybrid and semantic mode write query as one complete natural-language question or statement that " +
|
|
"names the subject and what you want to know (\"Why does Echo use open-weight models to cut cost?\"), " +
|
|
"not a keyword list (\"Echo open-weight cost\"): results are ranked by a relevance model that scores " +
|
|
"meaning. In keyword mode write the exact terms. Pass knowledge_base_ids to focus on the bases whose " +
|
|
"profile fits the question. When nothing passes the relevance check, rephrase with terms the documents " +
|
|
"would use; repeating the same query in another mode usually does not help. For identifiers, error " +
|
|
"codes or exact names retry with mode=keyword.\n" +
|
|
"Every chunk carries a cN handle and belongs to a dN document. Use read_document(id=dN) to read the " +
|
|
"surrounding context or the whole document. Retrieval matches chunk text; the relevance model also " +
|
|
"sees the document title. To find a document by its title or file name use list_documents(keyword=...).",
|
|
schema: json.RawMessage(`{
|
|
"type": "object",
|
|
"properties": {
|
|
"query": {
|
|
"type": "string",
|
|
"description": "A complete natural-language question, not a keyword list; exact terms when mode is keyword",
|
|
"minLength": 1
|
|
},
|
|
"mode": {
|
|
"type": "string",
|
|
"enum": ["hybrid", "semantic", "keyword"],
|
|
"description": "Retrieval mode (default hybrid)"
|
|
},
|
|
"knowledge_base_ids": {
|
|
"type": "array",
|
|
"description": "Optional bN knowledge-base handles from the runtime context to restrict the search",
|
|
"items": { "type": "string" },
|
|
"maxItems": 10
|
|
},
|
|
"limit": {
|
|
"type": "integer",
|
|
"description": "Maximum chunks to return (default 10, max 30)",
|
|
"minimum": 1,
|
|
"maximum": 30
|
|
}
|
|
},
|
|
"required": ["query"]
|
|
}`),
|
|
}
|
|
|
|
// SearchKnowledgeInput defines the input parameters for search_knowledge.
|
|
type SearchKnowledgeInput struct {
|
|
Query string `json:"query"`
|
|
Mode string `json:"mode,omitempty"`
|
|
KnowledgeBaseIDs []string `json:"knowledge_base_ids,omitempty"`
|
|
Limit int `json:"limit,omitempty"`
|
|
}
|
|
|
|
// searchResultWithMeta wraps search result with metadata about which query matched it
|
|
type searchResultWithMeta struct {
|
|
*types.SearchResult
|
|
SourceQuery string
|
|
QueryType string // retrieval path that produced the row
|
|
KnowledgeBaseID string // ID of the knowledge base this result came from
|
|
KnowledgeBaseType string // Type of the knowledge base (document, faq, etc.)
|
|
}
|
|
|
|
// SearchKnowledgeTool searches chunk-indexed knowledge bases in semantic,
|
|
// keyword or hybrid mode and returns reranked, de-duplicated chunks.
|
|
type SearchKnowledgeTool struct {
|
|
BaseTool
|
|
knowledgeBaseService interfaces.KnowledgeBaseService
|
|
knowledgeService interfaces.KnowledgeService
|
|
chunkService interfaces.ChunkService
|
|
searchTargets types.SearchTargets // Pre-computed unified search targets
|
|
rerankModel rerank.Reranker
|
|
config *config.Config // Global config for fallback values
|
|
// patternFilter, when set, keeps only results whose text matches it.
|
|
// It is not exposed to the model; the MCP grep_chunks endpoint uses it
|
|
// to keep grep semantics on top of index-backed retrieval.
|
|
patternFilter *regexp.Regexp
|
|
}
|
|
|
|
// WithPatternFilter restricts results to chunks whose text matches re. The
|
|
// candidate pool is widened to the maximum limit before filtering.
|
|
func (t *SearchKnowledgeTool) WithPatternFilter(re *regexp.Regexp) *SearchKnowledgeTool {
|
|
t.patternFilter = re
|
|
return t
|
|
}
|
|
|
|
// NewSearchKnowledgeTool creates a new search_knowledge tool.
|
|
func NewSearchKnowledgeTool(
|
|
knowledgeBaseService interfaces.KnowledgeBaseService,
|
|
knowledgeService interfaces.KnowledgeService,
|
|
chunkService interfaces.ChunkService,
|
|
searchTargets types.SearchTargets,
|
|
rerankModel rerank.Reranker,
|
|
cfg *config.Config,
|
|
) *SearchKnowledgeTool {
|
|
return &SearchKnowledgeTool{
|
|
BaseTool: searchKnowledgeTool,
|
|
knowledgeBaseService: knowledgeBaseService,
|
|
knowledgeService: knowledgeService,
|
|
chunkService: chunkService,
|
|
searchTargets: searchTargets,
|
|
rerankModel: rerankModel,
|
|
config: cfg,
|
|
}
|
|
}
|
|
|
|
// normalizeSearchMode validates the mode argument and applies the default.
|
|
func normalizeSearchMode(mode string) (string, error) {
|
|
mode = strings.ToLower(strings.TrimSpace(mode))
|
|
switch mode {
|
|
case "":
|
|
return SearchModeHybrid, nil
|
|
case SearchModeHybrid, SearchModeSemantic, SearchModeKeyword:
|
|
return mode, nil
|
|
default:
|
|
return "", fmt.Errorf("mode must be one of hybrid, semantic, keyword (got %q)", mode)
|
|
}
|
|
}
|
|
|
|
// clampSearchLimit applies the default and upper bound for limit.
|
|
func clampSearchLimit(limit int) int {
|
|
if limit <= 0 {
|
|
return searchKnowledgeDefaultLimit
|
|
}
|
|
if limit > searchKnowledgeMaxLimit {
|
|
return searchKnowledgeMaxLimit
|
|
}
|
|
return limit
|
|
}
|
|
|
|
// Execute executes the search.
|
|
func (t *SearchKnowledgeTool) Execute(ctx context.Context, args json.RawMessage) (*types.ToolResult, error) {
|
|
var input SearchKnowledgeInput
|
|
if err := json.Unmarshal(args, &input); err != nil {
|
|
return &types.ToolResult{Success: false, Error: fmt.Sprintf("Failed to parse args: %v", err)}, err
|
|
}
|
|
|
|
query := strings.TrimSpace(input.Query)
|
|
if query == "" {
|
|
return &types.ToolResult{Success: false, Error: "query is required"}, fmt.Errorf("no query provided")
|
|
}
|
|
mode, err := normalizeSearchMode(input.Mode)
|
|
if err != nil {
|
|
return &types.ToolResult{Success: false, Error: err.Error()}, err
|
|
}
|
|
limit := clampSearchLimit(input.Limit)
|
|
|
|
// Optional KB filter: reject hallucinated or out-of-scope handles.
|
|
searchTargets := t.searchTargets
|
|
if len(input.KnowledgeBaseIDs) > 0 {
|
|
if err := validateKnowledgeBaseIDsInSearchTargets(t.searchTargets, input.KnowledgeBaseIDs); err != nil {
|
|
return &types.ToolResult{Success: false, Error: err.Error()}, err
|
|
}
|
|
wanted := make(map[string]bool, len(input.KnowledgeBaseIDs))
|
|
for _, kbID := range input.KnowledgeBaseIDs {
|
|
wanted[kbID] = true
|
|
}
|
|
var filtered types.SearchTargets
|
|
for _, target := range t.searchTargets {
|
|
if target != nil && wanted[target.KnowledgeBaseID] {
|
|
filtered = append(filtered, target)
|
|
}
|
|
}
|
|
searchTargets = filtered
|
|
}
|
|
if len(searchTargets) == 0 {
|
|
return &types.ToolResult{
|
|
Success: false,
|
|
Error: "no knowledge base is in scope for this search",
|
|
}, fmt.Errorf("no search targets available")
|
|
}
|
|
|
|
kbIDs := searchTargets.GetAllKnowledgeBaseIDs()
|
|
kbList, err := t.knowledgeBaseService.GetKnowledgeBasesByIDsOnly(ctx, kbIDs)
|
|
if err != nil {
|
|
logger.Warnf(ctx, "[Tool][SearchKnowledge] Failed to load knowledge bases %v: %v", kbIDs, err)
|
|
}
|
|
kbModes, msg := resolveKBSearchModes(mode, kbList)
|
|
if msg != "" {
|
|
return &types.ToolResult{Success: false, Error: msg}, fmt.Errorf("%s", msg)
|
|
}
|
|
kbTypeMap := make(map[string]string, len(kbList))
|
|
for _, kb := range kbList {
|
|
if kb != nil {
|
|
kbTypeMap[kb.ID] = kb.Type
|
|
}
|
|
}
|
|
|
|
retrievalLimit := limit
|
|
if t.patternFilter != nil {
|
|
retrievalLimit = searchKnowledgeMaxLimit
|
|
}
|
|
topK, vectorThreshold, keywordThreshold := t.retrievalParams(retrievalLimit)
|
|
logger.Infof(ctx,
|
|
"[Tool][SearchKnowledge] query=%q mode=%s limit=%d top_k=%d targets=%d kbs=%d",
|
|
query, mode, limit, topK, len(searchTargets), len(kbIDs))
|
|
|
|
allResults := t.concurrentSearchByTargets(ctx, query, mode, kbModes, searchTargets, kbList,
|
|
topK, vectorThreshold, keywordThreshold, kbTypeMap)
|
|
|
|
deduplicated := t.deduplicateResults(allResults)
|
|
|
|
ranked := deduplicated
|
|
rerankRejected := 0
|
|
if t.rerankModel != nil && len(deduplicated) > 0 {
|
|
reranked, err := t.rerankResults(ctx, query, deduplicated)
|
|
if err != nil {
|
|
logger.Warnf(ctx, "[Tool][SearchKnowledge] Rerank failed, using retrieval order: %v", err)
|
|
} else {
|
|
ranked = reranked
|
|
if len(reranked) == 0 {
|
|
rerankRejected = len(deduplicated)
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(ranked) > 0 {
|
|
mmrK := len(ranked)
|
|
if mmrK > retrievalLimit {
|
|
mmrK = retrievalLimit
|
|
}
|
|
if selected := t.applyMMR(ctx, ranked, mmrK, 0.7); len(selected) > 0 {
|
|
ranked = selected
|
|
}
|
|
}
|
|
|
|
final := t.deduplicateResults(ranked)
|
|
sort.Slice(final, func(i, j int) bool {
|
|
if final[i].Score != final[j].Score {
|
|
return final[i].Score > final[j].Score
|
|
}
|
|
return final[i].KnowledgeID < final[j].KnowledgeID
|
|
})
|
|
if t.patternFilter != nil {
|
|
matched := final[:0]
|
|
for _, r := range final {
|
|
if t.patternFilter.MatchString(t.getEnrichedPassage(ctx, r.SearchResult)) {
|
|
matched = append(matched, r)
|
|
}
|
|
}
|
|
final = matched
|
|
}
|
|
if len(final) > limit {
|
|
final = final[:limit]
|
|
}
|
|
|
|
// Enrich image info for search results (lazy-loaded from child image chunks)
|
|
if t.chunkService != nil && len(final) > 0 {
|
|
byTenant := make(map[uint64][]*types.SearchResult)
|
|
for _, r := range final {
|
|
tid := t.searchTargets.GetTenantIDForKB(r.KnowledgeBaseID)
|
|
if tid == 0 {
|
|
continue
|
|
}
|
|
byTenant[tid] = append(byTenant[tid], r.SearchResult)
|
|
}
|
|
for tid, batch := range byTenant {
|
|
searchutil.EnrichSearchResultsImageInfo(ctx, t.chunkService.GetRepository(), tid, batch)
|
|
}
|
|
}
|
|
|
|
result := t.formatOutput(ctx, final, kbIDs, query, mode)
|
|
annotateModeFallback(result.Data, mode, kbModes)
|
|
if rerankRejected > 0 {
|
|
result.Data["rerank_rejected"] = rerankRejected
|
|
}
|
|
if len(final) == 0 {
|
|
result.Output = emptySearchStatement(query, result.Data, len(kbIDs))
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
// emptySearchStatement describes an empty result, including any mode
|
|
// fallback, so the model does not read "no semantic neighbours" as "the
|
|
// exact term does not occur". When retrieval found candidates and the rerank
|
|
// model rejected all of them, it says so: repeating the same query in
|
|
// another mode is usually wasted, but keyword mode can still surface a
|
|
// different candidate set for identifiers and exact terms.
|
|
func emptySearchStatement(query string, data map[string]interface{}, kbCount int) string {
|
|
mode, _ := data["mode"].(string)
|
|
msg := fmt.Sprintf("No matching chunks for %q (mode=%s) in %d knowledge base(s).", query, mode, kbCount)
|
|
if rejected, _ := data["rerank_rejected"].(int); rejected > 0 {
|
|
msg = fmt.Sprintf("No chunk passed the relevance check for %q (mode=%s) in %d knowledge base(s): "+
|
|
"retrieval found %d candidate chunks but the relevance model scored none of them as answering the "+
|
|
"query. Repeating the same query in another mode usually will not help; rephrase with terms the "+
|
|
"documents would use. For identifiers, error codes or exact names retry with mode=keyword.",
|
|
query, mode, kbCount, rejected)
|
|
}
|
|
fallbacks, _ := data["mode_fallbacks"].([]map[string]interface{})
|
|
for _, fb := range fallbacks {
|
|
msg += fmt.Sprintf(" Knowledge base %v was searched with mode=%v (%v).",
|
|
fb["knowledge_base_id"], fb["mode"], fb["reason"])
|
|
}
|
|
return msg
|
|
}
|
|
|
|
// kbSearchMode is the retrieval path used for one knowledge base.
|
|
type kbSearchMode struct {
|
|
mode string
|
|
fallback bool // mode differs from the requested one
|
|
reason string // why the requested mode could not be served
|
|
}
|
|
|
|
// resolveKBSearchModes decides, per knowledge base, which retrieval path
|
|
// serves the requested mode. A base that lacks the requested index falls
|
|
// back to the one it has (FAQ bases have no keyword index; vector-only bases
|
|
// have no keyword index; keyword-only bases have no vector index) instead of
|
|
// failing the whole call. Bases with no chunk index at all (wiki-only /
|
|
// graph-only) are left out. The call is refused only when no base in scope
|
|
// can be searched. An empty kbs list (lookup failed) resolves to nil, which
|
|
// callers treat as "use the requested mode everywhere".
|
|
func resolveKBSearchModes(mode string, kbs []*types.KnowledgeBase) (map[string]kbSearchMode, string) {
|
|
if len(kbs) == 0 {
|
|
return nil, ""
|
|
}
|
|
modes := make(map[string]kbSearchMode, len(kbs))
|
|
for _, kb := range kbs {
|
|
if kb == nil {
|
|
continue
|
|
}
|
|
hasVector := kb.IsVectorEnabled()
|
|
hasKeyword := kb.IsKeywordEnabled() && kb.Type != types.KnowledgeBaseTypeFAQ
|
|
switch {
|
|
case !hasVector && !hasKeyword:
|
|
continue
|
|
case mode == SearchModeKeyword && !hasKeyword:
|
|
reason := "no keyword index"
|
|
if kb.Type == types.KnowledgeBaseTypeFAQ {
|
|
reason = "FAQ bases are indexed for semantic search only"
|
|
}
|
|
modes[kb.ID] = kbSearchMode{mode: SearchModeSemantic, fallback: true, reason: reason}
|
|
case mode == SearchModeSemantic && !hasVector:
|
|
modes[kb.ID] = kbSearchMode{mode: SearchModeKeyword, fallback: true, reason: "no vector index"}
|
|
default:
|
|
modes[kb.ID] = kbSearchMode{mode: mode}
|
|
}
|
|
}
|
|
if len(modes) == 0 {
|
|
return nil, "none of the selected knowledge bases has a chunk index; use a wiki or graph tool for this scope"
|
|
}
|
|
return modes, ""
|
|
}
|
|
|
|
// annotateModeFallback records in the result which bases were searched with
|
|
// a different mode than requested, so neither the model nor the UI mistakes
|
|
// a semantic fallback for literal keyword matches.
|
|
func annotateModeFallback(data map[string]interface{}, requested string, kbModes map[string]kbSearchMode) {
|
|
if data == nil {
|
|
return
|
|
}
|
|
data["requested_mode"] = requested
|
|
var fallbacks []map[string]interface{}
|
|
used := make(map[string]bool)
|
|
ids := make([]string, 0, len(kbModes))
|
|
for kbID := range kbModes {
|
|
ids = append(ids, kbID)
|
|
}
|
|
sort.Strings(ids)
|
|
for _, kbID := range ids {
|
|
m := kbModes[kbID]
|
|
used[m.mode] = true
|
|
if m.fallback {
|
|
fallbacks = append(fallbacks, map[string]interface{}{
|
|
"knowledge_base_id": kbID, "mode": m.mode, "reason": m.reason,
|
|
})
|
|
}
|
|
}
|
|
if len(fallbacks) == 0 {
|
|
return
|
|
}
|
|
data["mode_fallbacks"] = fallbacks
|
|
if len(used) == 1 {
|
|
for effective := range used {
|
|
data["mode"] = effective
|
|
}
|
|
}
|
|
}
|
|
|
|
// retrievalParams resolves the candidate pool size and recall thresholds from
|
|
// the global configuration, with hard-coded fallbacks.
|
|
func (t *SearchKnowledgeTool) retrievalParams(limit int) (topK int, vectorThreshold, keywordThreshold float64) {
|
|
if t.config != nil && t.config.Conversation != nil {
|
|
topK = t.config.Conversation.EmbeddingTopK
|
|
vectorThreshold = t.config.Conversation.VectorThreshold
|
|
keywordThreshold = t.config.Conversation.KeywordThreshold
|
|
}
|
|
if topK <= 0 {
|
|
topK = searchKnowledgeMaxLimit
|
|
}
|
|
if topK < limit {
|
|
topK = limit
|
|
}
|
|
if vectorThreshold == 0 {
|
|
vectorThreshold = 0.6
|
|
}
|
|
if keywordThreshold == 0 {
|
|
keywordThreshold = 0.5
|
|
}
|
|
return topK, vectorThreshold, keywordThreshold
|
|
}
|
|
|
|
// concurrentSearchByTargets executes retrieval using pre-computed search targets.
|
|
// Targets sharing the same underlying embedding model (identified by model name + endpoint)
|
|
// are grouped so the query embedding is computed once per model, and all
|
|
// full-KB targets in a group are combined into a single retrieval call.
|
|
func (t *SearchKnowledgeTool) concurrentSearchByTargets(
|
|
ctx context.Context,
|
|
query string,
|
|
mode string,
|
|
kbModes map[string]kbSearchMode,
|
|
searchTargets types.SearchTargets,
|
|
kbList []*types.KnowledgeBase,
|
|
topK int,
|
|
vectorThreshold, keywordThreshold float64,
|
|
kbTypeMap map[string]string,
|
|
) []*searchResultWithMeta {
|
|
// Filter out non-searchable KBs (wiki-only / graph-only). Feeding a
|
|
// wiki-only KB into HybridSearch causes spurious "model ID cannot be
|
|
// empty" errors because such KBs have no EmbeddingModelID configured.
|
|
// KBs that we couldn't fetch (not in kbList) are kept so the downstream
|
|
// HybridSearch path can still surface the real error.
|
|
knownKBs := make(map[string]bool, len(kbList))
|
|
for _, kb := range kbList {
|
|
if kb != nil {
|
|
knownKBs[kb.ID] = true
|
|
}
|
|
}
|
|
modeFor := func(kbID string) (string, bool) {
|
|
if m, ok := kbModes[kbID]; ok {
|
|
return m.mode, true
|
|
}
|
|
// Unknown to the lookup: keep it so HybridSearch surfaces the real error.
|
|
return mode, !knownKBs[kbID]
|
|
}
|
|
filteredTargets := make(types.SearchTargets, 0, len(searchTargets))
|
|
for _, st := range searchTargets {
|
|
if st == nil || st.KnowledgeBaseID == "" {
|
|
continue
|
|
}
|
|
if _, ok := modeFor(st.KnowledgeBaseID); ok {
|
|
filteredTargets = append(filteredTargets, st)
|
|
continue
|
|
}
|
|
logger.Infof(ctx, "[Tool][SearchKnowledge] Skipping non-searchable KB %s (no vector/keyword index)",
|
|
st.KnowledgeBaseID)
|
|
}
|
|
if len(filteredTargets) == 0 {
|
|
return nil
|
|
}
|
|
searchTargets = filteredTargets
|
|
|
|
// Resolve actual model identities (name + endpoint) for cross-tenant grouping
|
|
modelKeyMap := t.knowledgeBaseService.ResolveEmbeddingModelKeys(ctx, kbList)
|
|
|
|
groups := make(map[string][]*types.SearchTarget)
|
|
for _, st := range searchTargets {
|
|
key := modelKeyMap[st.KnowledgeBaseID]
|
|
groups[key] = append(groups[key], st)
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
var mu sync.Mutex
|
|
allResults := make([]*searchResultWithMeta, 0)
|
|
collect := func(rows []*types.SearchResult, usedMode string) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
for _, r := range rows {
|
|
allResults = append(allResults, &searchResultWithMeta{
|
|
SearchResult: r,
|
|
SourceQuery: query,
|
|
QueryType: usedMode,
|
|
KnowledgeBaseID: r.KnowledgeBaseID,
|
|
KnowledgeBaseType: kbTypeMap[r.KnowledgeBaseID],
|
|
})
|
|
}
|
|
}
|
|
|
|
for modelKey, targets := range groups {
|
|
wg.Add(1)
|
|
go func(modelKey string, targets []*types.SearchTarget) {
|
|
defer wg.Done()
|
|
|
|
needsEmbedding := false
|
|
for _, st := range targets {
|
|
if m, _ := modeFor(st.KnowledgeBaseID); m != SearchModeKeyword {
|
|
needsEmbedding = true
|
|
break
|
|
}
|
|
}
|
|
// Compute embedding once for this (model, query) pair
|
|
var queryEmbedding []float32
|
|
if modelKey != "" && needsEmbedding {
|
|
emb, err := t.knowledgeBaseService.GetQueryEmbedding(ctx, targets[0].KnowledgeBaseID, query)
|
|
if err != nil {
|
|
logger.Warnf(ctx, "[Tool][SearchKnowledge] Failed to pre-compute embedding for model %s: %v",
|
|
modelKey, err)
|
|
} else {
|
|
queryEmbedding = emb
|
|
}
|
|
}
|
|
|
|
// Separate full-KB targets (combinable per retrieval mode) from
|
|
// specific-knowledge targets.
|
|
fullKBIDsByMode := make(map[string][]string)
|
|
var knowledgeTargets []*types.SearchTarget
|
|
for _, st := range targets {
|
|
if st.Type == types.SearchTargetTypeKnowledgeBase && len(st.TagIDs) == 0 {
|
|
m, _ := modeFor(st.KnowledgeBaseID)
|
|
fullKBIDsByMode[m] = append(fullKBIDsByMode[m], st.KnowledgeBaseID)
|
|
} else {
|
|
knowledgeTargets = append(knowledgeTargets, st)
|
|
}
|
|
}
|
|
|
|
var innerWg sync.WaitGroup
|
|
for usedMode, fullKBIDs := range fullKBIDsByMode {
|
|
innerWg.Add(1)
|
|
go func(usedMode string, fullKBIDs []string) {
|
|
defer innerWg.Done()
|
|
kbResults, err := t.knowledgeBaseService.HybridSearch(ctx, fullKBIDs[0], types.SearchParams{
|
|
QueryText: query,
|
|
QueryEmbedding: queryEmbedding,
|
|
KnowledgeBaseIDs: fullKBIDs,
|
|
MatchCount: topK,
|
|
VectorThreshold: vectorThreshold,
|
|
KeywordThreshold: keywordThreshold,
|
|
DisableVectorMatch: usedMode == SearchModeKeyword,
|
|
DisableKeywordsMatch: usedMode == SearchModeSemantic,
|
|
})
|
|
if err != nil {
|
|
logger.Warnf(ctx, "[Tool][SearchKnowledge] Combined search failed for KBs %v: %v",
|
|
fullKBIDs, err)
|
|
return
|
|
}
|
|
collect(kbResults, usedMode)
|
|
}(usedMode, fullKBIDs)
|
|
}
|
|
|
|
for _, target := range knowledgeTargets {
|
|
st := target
|
|
innerWg.Add(1)
|
|
go func() {
|
|
defer innerWg.Done()
|
|
usedMode, _ := modeFor(st.KnowledgeBaseID)
|
|
stVectorThreshold, stKeywordThreshold := st.RecallThresholds(vectorThreshold, keywordThreshold)
|
|
kbResults, err := t.knowledgeBaseService.HybridSearch(ctx, st.KnowledgeBaseID, types.SearchParams{
|
|
QueryText: query,
|
|
QueryEmbedding: queryEmbedding,
|
|
MatchCount: topK,
|
|
VectorThreshold: stVectorThreshold,
|
|
KeywordThreshold: stKeywordThreshold,
|
|
KnowledgeIDs: st.KnowledgeIDs,
|
|
TagIDs: st.TagIDs,
|
|
ScopeTagIDs: st.ScopeTagIDs,
|
|
DisableVectorMatch: usedMode == SearchModeKeyword,
|
|
DisableKeywordsMatch: usedMode == SearchModeSemantic,
|
|
})
|
|
if err != nil {
|
|
logger.Warnf(ctx, "[Tool][SearchKnowledge] Failed to search KB %s: %v", st.KnowledgeBaseID, err)
|
|
return
|
|
}
|
|
collect(kbResults, usedMode)
|
|
}()
|
|
}
|
|
innerWg.Wait()
|
|
}(modelKey, targets)
|
|
}
|
|
wg.Wait()
|
|
return allResults
|
|
}
|
|
|
|
// rerankResults applies reranking to all search results (including FAQ entries)
|
|
// using the configured rerank model, then filters by threshold and applies
|
|
// composite scoring so MMR/sorting uses a single score scale.
|
|
//
|
|
// A failed rerank call degrades to the raw retrieval order. An empty result
|
|
// after threshold filtering is kept empty: filterRerankRankResults already
|
|
// preserves the top candidate down to agentRerankFallbackMinScore, so reaching
|
|
// zero means even the best match is below that floor.
|
|
func (t *SearchKnowledgeTool) rerankResults(
|
|
ctx context.Context,
|
|
query string,
|
|
results []*searchResultWithMeta,
|
|
) ([]*searchResultWithMeta, error) {
|
|
if len(results) == 0 || t.rerankModel == nil {
|
|
return results, nil
|
|
}
|
|
|
|
rankResults, err := t.rerankScores(ctx, query, results)
|
|
if err != nil {
|
|
logger.Warnf(ctx, "[Tool][SearchKnowledge] Rerank model failed, using raw retrieval results: %v", err)
|
|
return results, nil
|
|
}
|
|
|
|
threshold := t.rerankThreshold()
|
|
reranked := t.applyModelRerankScores(
|
|
results,
|
|
rankResults,
|
|
threshold,
|
|
t.searchTargets.HasRecallThresholdOverride(),
|
|
)
|
|
logger.Infof(ctx, "[Tool][SearchKnowledge] Reranked %d/%d results above threshold %.2f",
|
|
len(reranked), len(results), threshold)
|
|
return reranked, nil
|
|
}
|
|
|
|
func (t *SearchKnowledgeTool) getFAQMetadata(
|
|
ctx context.Context,
|
|
chunkID string,
|
|
cache map[string]*types.FAQChunkMetadata,
|
|
) (*types.FAQChunkMetadata, error) {
|
|
if chunkID == "" || t.chunkService == nil {
|
|
return nil, nil
|
|
}
|
|
if meta, ok := cache[chunkID]; ok {
|
|
return meta, nil
|
|
}
|
|
chunk, err := t.chunkService.GetChunkByID(ctx, chunkID)
|
|
if err != nil {
|
|
cache[chunkID] = nil
|
|
return nil, err
|
|
}
|
|
if chunk == nil {
|
|
cache[chunkID] = nil
|
|
return nil, nil
|
|
}
|
|
meta, err := chunk.FAQMetadata()
|
|
if err != nil {
|
|
cache[chunkID] = nil
|
|
return nil, err
|
|
}
|
|
cache[chunkID] = meta
|
|
return meta, nil
|
|
}
|
|
|
|
// rerankScores scores the candidates with the configured rerank model and
|
|
// returns the raw relevance scores, leaving threshold filtering and composite
|
|
// scoring to the caller.
|
|
func (t *SearchKnowledgeTool) rerankScores(
|
|
ctx context.Context,
|
|
query string,
|
|
results []*searchResultWithMeta,
|
|
) ([]rerank.RankResult, error) {
|
|
passages := make([]string, len(results))
|
|
for i, result := range results {
|
|
passages[i] = t.rerankPassage(ctx, result.SearchResult)
|
|
}
|
|
rerankResp, err := t.rerankModel.Rerank(ctx, query, passages)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("rerank call failed: %w", err)
|
|
}
|
|
return rerankResp, nil
|
|
}
|
|
|
|
// rerankPassage is the text the rerank model scores: the document title
|
|
// followed by the enriched chunk. A chunk rarely restates what its document
|
|
// is about, so without the title a passage from "Show HN: Echo - ... using
|
|
// open-weight models" scored 0.002 against "Echo open-weight models reduce
|
|
// cost" and 0.45 with it. FAQ entries carry their own question instead.
|
|
func (t *SearchKnowledgeTool) rerankPassage(ctx context.Context, result *types.SearchResult) string {
|
|
passage := t.getEnrichedPassage(ctx, result)
|
|
title := strings.TrimSpace(result.KnowledgeTitle)
|
|
if title == "" || result.ChunkType == string(types.ChunkTypeFAQ) {
|
|
return passage
|
|
}
|
|
return title + "\n\n" + passage
|
|
}
|
|
|
|
func (t *SearchKnowledgeTool) rerankThreshold() float64 {
|
|
if t.config != nil && t.config.Conversation != nil && t.config.Conversation.RerankThreshold > 0 {
|
|
return t.config.Conversation.RerankThreshold
|
|
}
|
|
return 0.3
|
|
}
|
|
|
|
const agentRerankFallbackMinScore = 0.15
|
|
|
|
func filterRerankRankResults(
|
|
rankResults []rerank.RankResult,
|
|
threshold float64,
|
|
preserveTop bool,
|
|
) []rerank.RankResult {
|
|
if len(rankResults) == 0 {
|
|
return nil
|
|
}
|
|
filtered := make([]rerank.RankResult, 0, len(rankResults))
|
|
for _, r := range rankResults {
|
|
if r.RelevanceScore >= threshold {
|
|
filtered = append(filtered, r)
|
|
}
|
|
}
|
|
if len(filtered) == 0 {
|
|
top := rankResults[0]
|
|
for _, r := range rankResults[1:] {
|
|
if r.RelevanceScore > top.RelevanceScore {
|
|
top = r
|
|
}
|
|
}
|
|
if preserveTop || top.RelevanceScore >= agentRerankFallbackMinScore {
|
|
return []rerank.RankResult{top}
|
|
}
|
|
}
|
|
return filtered
|
|
}
|
|
|
|
func (t *SearchKnowledgeTool) applyModelRerankScores(
|
|
originals []*searchResultWithMeta,
|
|
rankResults []rerank.RankResult,
|
|
threshold float64,
|
|
preserveTop bool,
|
|
) []*searchResultWithMeta {
|
|
filtered := filterRerankRankResults(rankResults, threshold, preserveTop)
|
|
out := make([]*searchResultWithMeta, 0, len(filtered))
|
|
for _, rr := range filtered {
|
|
if rr.Index < 0 || rr.Index >= len(originals) {
|
|
continue
|
|
}
|
|
newResult := *originals[rr.Index]
|
|
baseScore := newResult.Score
|
|
modelScore := rr.RelevanceScore
|
|
newResult.Score = t.compositeScore(&newResult, modelScore, baseScore)
|
|
out = append(out, &newResult)
|
|
}
|
|
sort.Slice(out, func(i, j int) bool {
|
|
return out[i].Score > out[j].Score
|
|
})
|
|
return out
|
|
}
|
|
|
|
// deduplicateResults removes duplicate chunks, keeping the first occurrence.
|
|
// Uses multiple keys (ID, parent chunk ID, knowledge+index) and a content
|
|
// signature for near-duplicate detection. Input order is preserved.
|
|
func (t *SearchKnowledgeTool) deduplicateResults(results []*searchResultWithMeta) []*searchResultWithMeta {
|
|
seen := make(map[string]bool)
|
|
contentSig := make(map[string]bool)
|
|
uniqueResults := make([]*searchResultWithMeta, 0, len(results))
|
|
|
|
for _, r := range results {
|
|
if r == nil || r.SearchResult == nil {
|
|
continue
|
|
}
|
|
keys := []string{r.ID}
|
|
if r.ParentChunkID != "" {
|
|
keys = append(keys, "parent:"+r.ParentChunkID)
|
|
}
|
|
if r.KnowledgeID != "" {
|
|
keys = append(keys, fmt.Sprintf("kb:%s#%d", r.KnowledgeID, r.ChunkIndex))
|
|
}
|
|
dup := false
|
|
for _, k := range keys {
|
|
if seen[k] {
|
|
dup = true
|
|
break
|
|
}
|
|
}
|
|
if dup {
|
|
continue
|
|
}
|
|
sig := searchutil.BuildContentSignature(r.Content)
|
|
if sig != "" {
|
|
if contentSig[sig] {
|
|
continue
|
|
}
|
|
contentSig[sig] = true
|
|
}
|
|
for _, k := range keys {
|
|
seen[k] = true
|
|
}
|
|
uniqueResults = append(uniqueResults, r)
|
|
}
|
|
return uniqueResults
|
|
}
|
|
|
|
// writeKnowledgeMetadataHeader emits document-scoped metadata once per
|
|
// knowledge item. Chunk entries keep only chunk-specific content so repeated
|
|
// results from the same document do not waste context.
|
|
func writeKnowledgeMetadataHeader(ob *strings.Builder, results []*searchResultWithMeta) {
|
|
seen := make(map[string]struct{}, len(results))
|
|
hasMetadata := false
|
|
var documents strings.Builder
|
|
for _, result := range results {
|
|
if result == nil || result.SearchResult == nil || result.KnowledgeID == "" ||
|
|
result.KnowledgeCustomMetadata == "" {
|
|
continue
|
|
}
|
|
if _, ok := seen[result.KnowledgeID]; ok {
|
|
continue
|
|
}
|
|
seen[result.KnowledgeID] = struct{}{}
|
|
hasMetadata = true
|
|
fmt.Fprintf(&documents,
|
|
"<document knowledge_id=\"%s\" knowledge_base_id=\"%s\" title=\"%s\">\n",
|
|
xmlEscape(result.KnowledgeID),
|
|
xmlEscape(result.KnowledgeBaseID),
|
|
xmlEscape(result.KnowledgeTitle),
|
|
)
|
|
fmt.Fprintf(&documents, "<metadata>%s</metadata>\n", xmlEscape(result.KnowledgeCustomMetadata))
|
|
documents.WriteString("</document>\n")
|
|
}
|
|
if !hasMetadata {
|
|
return
|
|
}
|
|
ob.WriteString("<documents>\n")
|
|
ob.WriteString(documents.String())
|
|
ob.WriteString("</documents>\n")
|
|
}
|
|
|
|
// formatOutput builds the structured Data (the source of truth for the UI and
|
|
// the model rendering) and a compact XML Output for logs.
|
|
func (t *SearchKnowledgeTool) formatOutput(
|
|
ctx context.Context,
|
|
results []*searchResultWithMeta,
|
|
kbsToSearch []string,
|
|
query string,
|
|
mode string,
|
|
) *types.ToolResult {
|
|
data := map[string]interface{}{
|
|
"knowledge_base_ids": kbsToSearch,
|
|
"query": query,
|
|
"queries": []string{query}, // legacy alias for older frontends
|
|
"mode": mode,
|
|
"results": []interface{}{},
|
|
"count": 0,
|
|
"display_type": "search_results",
|
|
}
|
|
if len(results) == 0 {
|
|
return &types.ToolResult{
|
|
Success: true,
|
|
Output: fmt.Sprintf("No matching chunks for %q (mode=%s) in %d knowledge base(s).",
|
|
query, mode, len(kbsToSearch)),
|
|
Data: data,
|
|
}
|
|
}
|
|
|
|
kbCounts := make(map[string]int)
|
|
for _, r := range results {
|
|
kbCounts[r.KnowledgeBaseID]++
|
|
}
|
|
|
|
var ob strings.Builder
|
|
fmt.Fprintf(&ob, "<search_results count=\"%d\" mode=\"%s\">\n", len(results), xmlEscape(mode))
|
|
fmt.Fprintf(&ob, "<query>%s</query>\n", xmlEscape(query))
|
|
writeKnowledgeMetadataHeader(&ob, results)
|
|
|
|
formattedResults := make([]map[string]interface{}, 0, len(results))
|
|
faqMetadataCache := make(map[string]*types.FAQChunkMetadata)
|
|
queries := []string{query}
|
|
|
|
for i, result := range results {
|
|
var faqMeta *types.FAQChunkMetadata
|
|
if result.KnowledgeBaseType == types.KnowledgeBaseTypeFAQ {
|
|
meta, err := t.getFAQMetadata(ctx, result.ID, faqMetadataCache)
|
|
if err != nil {
|
|
logger.Warnf(ctx, "[Tool][SearchKnowledge] Failed to load FAQ metadata for chunk %s: %v",
|
|
result.ID, err)
|
|
} else {
|
|
faqMeta = meta
|
|
}
|
|
}
|
|
isFAQ := faqMeta != nil
|
|
|
|
snippet := ""
|
|
if isFAQ {
|
|
snippet = faqMatchSnippetFromQueries(faqMeta, queries)
|
|
}
|
|
if snippet == "" && t.patternFilter != nil {
|
|
snippet = extractSnippetRegex(result.Content, []*regexp.Regexp{t.patternFilter})
|
|
}
|
|
if snippet == "" {
|
|
snippet = extractSnippetForQueries(result.Content, queries)
|
|
}
|
|
|
|
tag := "chunk"
|
|
idAttr := fmt.Sprintf("chunk_id=\"%s\" chunk_index=\"%d\" knowledge_id=\"%s\"",
|
|
xmlEscape(result.ID), result.ChunkIndex, xmlEscape(result.KnowledgeID))
|
|
if isFAQ {
|
|
tag = "faq"
|
|
idAttr = fmt.Sprintf("faq_id=\"%s\" index=\"%d\"", xmlEscape(result.ID), result.ChunkIndex)
|
|
}
|
|
fmt.Fprintf(&ob, "<%s rank=\"%d\" %s knowledge_base_id=\"%s\" knowledge_title=\"%s\" score=\"%.3f\">\n",
|
|
tag, i+1, idAttr, xmlEscape(result.KnowledgeBaseID), xmlEscape(result.KnowledgeTitle), result.Score)
|
|
if snippet != "" {
|
|
fmt.Fprintf(&ob, "<match_snippet>%s</match_snippet>\n", xmlEscape(snippet))
|
|
}
|
|
fmt.Fprintf(&ob, "<content>%s</content>\n", result.Content)
|
|
if result.ImageInfo != "" {
|
|
var imageInfos []types.ImageInfo
|
|
if err := json.Unmarshal([]byte(result.ImageInfo), &imageInfos); err == nil {
|
|
for _, img := range imageInfos {
|
|
if md := searchutil.BuildImageInfoMarkdownWithURL(img.URL, &img); md != "" {
|
|
ob.WriteString(md)
|
|
ob.WriteString("\n")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if isFAQ {
|
|
writeFAQFieldsXML(&ob, faqMeta)
|
|
}
|
|
fmt.Fprintf(&ob, "</%s>\n", tag)
|
|
|
|
row := map[string]interface{}{
|
|
"result_index": i + 1,
|
|
"content": result.Content,
|
|
"knowledge_id": result.KnowledgeID,
|
|
"knowledge_base_id": result.KnowledgeBaseID,
|
|
"knowledge_title": result.KnowledgeTitle,
|
|
"knowledge_metadata": result.KnowledgeCustomMetadata,
|
|
"match_type": result.MatchType,
|
|
"match_snippet": snippet,
|
|
"score": result.Score,
|
|
"source_query": result.SourceQuery,
|
|
"query_type": result.QueryType,
|
|
"knowledge_base_type": result.KnowledgeBaseType,
|
|
}
|
|
if images := chunkImageList(result.ImageInfo); len(images) > 0 {
|
|
row["images"] = images
|
|
}
|
|
if isFAQ {
|
|
row["faq_id"] = result.ID
|
|
row["index"] = result.ChunkIndex
|
|
if faqMeta.StandardQuestion != "" {
|
|
row["faq_standard_question"] = faqMeta.StandardQuestion
|
|
}
|
|
appendSimilarQuestionsToChunkData(row, faqMeta.SimilarQuestions)
|
|
if len(faqMeta.Answers) > 0 {
|
|
row["faq_answers"] = faqMeta.Answers
|
|
}
|
|
} else {
|
|
row["chunk_id"] = result.ID
|
|
row["chunk_index"] = result.ChunkIndex
|
|
}
|
|
formattedResults = append(formattedResults, row)
|
|
}
|
|
ob.WriteString("</search_results>")
|
|
|
|
data["results"] = formattedResults
|
|
data["count"] = len(formattedResults)
|
|
data["kb_counts"] = kbCounts
|
|
return &types.ToolResult{Success: true, Output: ob.String(), Data: data}
|
|
}
|
|
|
|
// getEnrichedPassage merges chunk content with image captions / OCR text so
|
|
// the reranker and MMR see the same evidence the model will.
|
|
func (t *SearchKnowledgeTool) getEnrichedPassage(ctx context.Context, result *types.SearchResult) string {
|
|
if result.ImageInfo == "" {
|
|
return result.Content
|
|
}
|
|
var imageInfos []types.ImageInfo
|
|
if err := json.Unmarshal([]byte(result.ImageInfo), &imageInfos); err != nil {
|
|
logger.Warnf(ctx, "[Tool][SearchKnowledge] Failed to parse image info: %v", err)
|
|
return result.Content
|
|
}
|
|
if len(imageInfos) == 0 {
|
|
return result.Content
|
|
}
|
|
var imageTexts []string
|
|
for _, img := range imageInfos {
|
|
if img.Caption != "" {
|
|
imageTexts = append(imageTexts, fmt.Sprintf("Image Caption: %s", img.Caption))
|
|
}
|
|
if img.OCRText != "" {
|
|
imageTexts = append(imageTexts, fmt.Sprintf("Image Text: %s", img.OCRText))
|
|
}
|
|
}
|
|
if len(imageTexts) == 0 {
|
|
return result.Content
|
|
}
|
|
combinedText := result.Content
|
|
if combinedText != "" {
|
|
combinedText += "\n\n"
|
|
}
|
|
return combinedText + strings.Join(imageTexts, "\n")
|
|
}
|
|
|
|
// compositeScore calculates a composite score considering multiple factors
|
|
func (t *SearchKnowledgeTool) compositeScore(
|
|
result *searchResultWithMeta,
|
|
modelScore, baseScore float64,
|
|
) float64 {
|
|
// Source weight: web_search results get slightly lower weight
|
|
sourceWeight := 1.0
|
|
if strings.ToLower(result.KnowledgeSource) == "web_search" {
|
|
sourceWeight = 0.95
|
|
}
|
|
|
|
// Position prior: slightly favor chunks earlier in the document
|
|
positionPrior := 1.0
|
|
if result.StartAt >= 0 && result.EndAt > result.StartAt {
|
|
positionRatio := 1.0 - float64(result.StartAt)/float64(result.EndAt+1)
|
|
positionPrior += t.clampFloat(positionRatio, -0.05, 0.05)
|
|
}
|
|
|
|
composite := 0.6*modelScore + 0.3*baseScore + 0.1*sourceWeight
|
|
composite *= positionPrior
|
|
if composite < 0 {
|
|
composite = 0
|
|
}
|
|
if composite > 1 {
|
|
composite = 1
|
|
}
|
|
return composite
|
|
}
|
|
|
|
func (t *SearchKnowledgeTool) clampFloat(v, minV, maxV float64) float64 {
|
|
return searchutil.ClampFloat(v, minV, maxV)
|
|
}
|
|
|
|
// applyMMR applies Maximal Marginal Relevance to reduce redundancy.
|
|
func (t *SearchKnowledgeTool) applyMMR(
|
|
ctx context.Context,
|
|
results []*searchResultWithMeta,
|
|
k int,
|
|
lambda float64,
|
|
) []*searchResultWithMeta {
|
|
if k <= 0 || len(results) == 0 {
|
|
return nil
|
|
}
|
|
|
|
selected := make([]*searchResultWithMeta, 0, k)
|
|
candidates := make([]*searchResultWithMeta, len(results))
|
|
copy(candidates, results)
|
|
|
|
tokenSets := make([]map[string]struct{}, len(candidates))
|
|
for i, r := range candidates {
|
|
tokenSets[i] = t.tokenizeSimple(t.getEnrichedPassage(ctx, r.SearchResult))
|
|
}
|
|
|
|
// Incremental form: maxRedundancy[i] caches candidate i's maximum jaccard
|
|
// against everything selected so far, so each round only needs one
|
|
// comparison per remaining candidate. Selection output is identical to
|
|
// the naive form, including tie-breaking.
|
|
maxRedundancy := make([]float64, len(candidates))
|
|
for len(selected) < k && len(candidates) > 0 {
|
|
bestIdx := 0
|
|
bestScore := -1.0
|
|
for i, r := range candidates {
|
|
mmr := lambda*r.Score - (1.0-lambda)*maxRedundancy[i]
|
|
if mmr > bestScore {
|
|
bestScore = mmr
|
|
bestIdx = i
|
|
}
|
|
}
|
|
selected = append(selected, candidates[bestIdx])
|
|
chosenTokens := tokenSets[bestIdx]
|
|
candidates = append(candidates[:bestIdx], candidates[bestIdx+1:]...)
|
|
tokenSets = append(tokenSets[:bestIdx], tokenSets[bestIdx+1:]...)
|
|
maxRedundancy = append(maxRedundancy[:bestIdx], maxRedundancy[bestIdx+1:]...)
|
|
for i := range candidates {
|
|
maxRedundancy[i] = math.Max(maxRedundancy[i], t.jaccard(tokenSets[i], chosenTokens))
|
|
}
|
|
}
|
|
return selected
|
|
}
|
|
|
|
func (t *SearchKnowledgeTool) tokenizeSimple(text string) map[string]struct{} {
|
|
return searchutil.TokenizeSimple(text)
|
|
}
|
|
|
|
func (t *SearchKnowledgeTool) jaccard(a, b map[string]struct{}) float64 {
|
|
return searchutil.Jaccard(a, b)
|
|
}
|