277 lines
8.7 KiB
Go
277 lines
8.7 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
|
|
"github.com/Tencent/WeKnora/internal/application/service/retriever"
|
|
"github.com/Tencent/WeKnora/internal/logger"
|
|
"github.com/Tencent/WeKnora/internal/types"
|
|
"slices"
|
|
)
|
|
|
|
// applyFAQPostProcessing handles FAQ-specific post-processing: iterative retrieval
|
|
// when not enough unique chunks are found, or negative question filtering otherwise.
|
|
// For non-FAQ knowledge bases, returns the input unchanged.
|
|
func (s *knowledgeBaseService) applyFAQPostProcessing(
|
|
ctx context.Context,
|
|
kb *types.KnowledgeBase,
|
|
chunks []*types.IndexWithScore,
|
|
vectorResults []*types.IndexWithScore,
|
|
retrieveEngine *retriever.CompositeRetrieveEngine,
|
|
retrieveParams []types.RetrieveParams,
|
|
params types.SearchParams,
|
|
matchCount int,
|
|
) []*types.IndexWithScore {
|
|
if kb.Type != types.KnowledgeBaseTypeFAQ {
|
|
return chunks
|
|
}
|
|
|
|
// Check if we need iterative retrieval for FAQ with separate indexing
|
|
// Only use iterative retrieval if we don't have enough unique chunks after first deduplication
|
|
needsIterativeRetrieval := len(chunks) < params.MatchCount && len(vectorResults) == matchCount
|
|
if needsIterativeRetrieval {
|
|
logger.Info(ctx, "Not enough unique chunks, using iterative retrieval for FAQ")
|
|
return s.iterativeRetrieveWithDeduplication(
|
|
ctx,
|
|
retrieveEngine,
|
|
retrieveParams,
|
|
params.MatchCount,
|
|
params.QueryText,
|
|
)
|
|
}
|
|
|
|
// Filter by negative questions if not using iterative retrieval
|
|
result := s.filterByNegativeQuestions(ctx, chunks, params.QueryText)
|
|
logger.Infof(ctx, "Result count after negative question filtering: %d", len(result))
|
|
return result
|
|
}
|
|
|
|
// iterativeRetrieveWithDeduplication performs iterative retrieval until enough unique chunks are found.
|
|
// This is used for FAQ knowledge bases with separate indexing mode.
|
|
// Negative question filtering is applied after each iteration with chunk data caching.
|
|
func (s *knowledgeBaseService) iterativeRetrieveWithDeduplication(ctx context.Context,
|
|
retrieveEngine *retriever.CompositeRetrieveEngine,
|
|
retrieveParams []types.RetrieveParams,
|
|
matchCount int,
|
|
queryText string,
|
|
) []*types.IndexWithScore {
|
|
maxIterations := 5
|
|
// Start with a larger TopK since we're called when first retrieval wasn't enough
|
|
// The first retrieval already used matchCount*3, so start from there
|
|
currentTopK := matchCount * 3
|
|
uniqueChunks := make(map[string]*types.IndexWithScore)
|
|
// Cache chunk data to avoid repeated DB queries across iterations
|
|
chunkDataCache := make(map[string]*types.Chunk)
|
|
// Track chunks that have been filtered out by negative questions
|
|
filteredOutChunks := make(map[string]struct{})
|
|
|
|
queryTextLower := strings.ToLower(strings.TrimSpace(queryText))
|
|
tenantID := types.MustTenantIDFromContext(ctx)
|
|
|
|
for i := 0; i < maxIterations; i++ {
|
|
// Update TopK in retrieve params
|
|
updatedParams := make([]types.RetrieveParams, len(retrieveParams))
|
|
for j := range retrieveParams {
|
|
updatedParams[j] = retrieveParams[j]
|
|
updatedParams[j].TopK = currentTopK
|
|
}
|
|
|
|
// Execute retrieval
|
|
retrieveResults, err := retrieveEngine.Retrieve(ctx, updatedParams)
|
|
if err != nil {
|
|
logger.Warnf(ctx, "Iterative retrieval failed at iteration %d: %v", i+1, err)
|
|
break
|
|
}
|
|
|
|
// Collect results
|
|
iterationResults := []*types.IndexWithScore{}
|
|
for _, retrieveResult := range retrieveResults {
|
|
iterationResults = append(iterationResults, retrieveResult.Results...)
|
|
}
|
|
|
|
if len(iterationResults) == 0 {
|
|
logger.Infof(ctx, "No results found at iteration %d", i+1)
|
|
break
|
|
}
|
|
|
|
totalRetrieved := len(iterationResults)
|
|
|
|
// Collect new chunk IDs that need to be fetched from DB
|
|
newChunkIDs := make([]string, 0)
|
|
for _, result := range iterationResults {
|
|
if _, cached := chunkDataCache[result.ChunkID]; !cached {
|
|
if _, filtered := filteredOutChunks[result.ChunkID]; !filtered {
|
|
newChunkIDs = append(newChunkIDs, result.ChunkID)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Batch fetch only new chunks
|
|
if len(newChunkIDs) > 0 {
|
|
newChunks, err := s.chunkRepo.ListChunksByID(ctx, tenantID, newChunkIDs)
|
|
if err != nil {
|
|
logger.Warnf(ctx, "Failed to fetch chunks at iteration %d: %v", i+1, err)
|
|
} else {
|
|
for _, chunk := range newChunks {
|
|
chunkDataCache[chunk.ID] = chunk
|
|
}
|
|
}
|
|
}
|
|
|
|
// Deduplicate, merge, and filter in one pass
|
|
for _, result := range iterationResults {
|
|
// Skip if already filtered out
|
|
if _, filtered := filteredOutChunks[result.ChunkID]; filtered {
|
|
continue
|
|
}
|
|
|
|
// Check negative questions using cached data
|
|
if chunkData, ok := chunkDataCache[result.ChunkID]; ok {
|
|
if chunkData.ChunkType == types.ChunkTypeFAQ {
|
|
if meta, err := chunkData.FAQMetadata(); err == nil && meta != nil {
|
|
if s.matchesNegativeQuestions(queryTextLower, meta.NegativeQuestions) {
|
|
filteredOutChunks[result.ChunkID] = struct{}{}
|
|
delete(uniqueChunks, result.ChunkID)
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Keep highest score for each chunk
|
|
if existing, ok := uniqueChunks[result.ChunkID]; !ok || result.Score > existing.Score {
|
|
uniqueChunks[result.ChunkID] = result
|
|
}
|
|
}
|
|
|
|
logger.Infof(
|
|
ctx,
|
|
"After iteration %d: retrieved %d results, found %d valid unique chunks (target: %d)",
|
|
i+1,
|
|
totalRetrieved,
|
|
len(uniqueChunks),
|
|
matchCount,
|
|
)
|
|
|
|
// Early stop: Check if we have enough unique chunks after deduplication and filtering
|
|
if len(uniqueChunks) >= matchCount {
|
|
logger.Infof(ctx, "Found enough unique chunks after %d iterations", i+1)
|
|
break
|
|
}
|
|
|
|
// Early stop: If we got fewer results than TopK, there are no more results to retrieve
|
|
if totalRetrieved < currentTopK {
|
|
logger.Infof(ctx, "No more results available (got %d < %d), stopping iteration", totalRetrieved, currentTopK)
|
|
break
|
|
}
|
|
|
|
// Increase TopK for next iteration
|
|
currentTopK *= 2
|
|
}
|
|
|
|
// Convert map to slice and sort by score
|
|
result := make([]*types.IndexWithScore, 0, len(uniqueChunks))
|
|
for _, chunk := range uniqueChunks {
|
|
result = append(result, chunk)
|
|
}
|
|
|
|
slices.SortFunc(result, sortByScoreDesc)
|
|
|
|
logger.Infof(ctx, "Iterative retrieval completed: %d unique chunks found after filtering", len(result))
|
|
return result
|
|
}
|
|
|
|
// filterByNegativeQuestions filters out chunks that match negative questions for FAQ knowledge bases.
|
|
func (s *knowledgeBaseService) filterByNegativeQuestions(ctx context.Context,
|
|
chunks []*types.IndexWithScore,
|
|
queryText string,
|
|
) []*types.IndexWithScore {
|
|
if len(chunks) == 0 {
|
|
return chunks
|
|
}
|
|
|
|
queryTextLower := strings.ToLower(strings.TrimSpace(queryText))
|
|
if queryTextLower == "" {
|
|
return chunks
|
|
}
|
|
|
|
tenantID := types.MustTenantIDFromContext(ctx)
|
|
|
|
// Collect chunk IDs
|
|
chunkIDs := make([]string, 0, len(chunks))
|
|
for _, chunk := range chunks {
|
|
chunkIDs = append(chunkIDs, chunk.ChunkID)
|
|
}
|
|
|
|
// Batch fetch chunks to get negative questions
|
|
allChunks, err := s.chunkRepo.ListChunksByID(ctx, tenantID, chunkIDs)
|
|
if err != nil {
|
|
logger.Warnf(ctx, "Failed to fetch chunks for negative question filtering: %v", err)
|
|
// If we can't fetch chunks, return original results
|
|
return chunks
|
|
}
|
|
|
|
// Build chunk map for quick lookup
|
|
chunkMap := make(map[string]*types.Chunk, len(allChunks))
|
|
for _, chunk := range allChunks {
|
|
chunkMap[chunk.ID] = chunk
|
|
}
|
|
|
|
// Filter out chunks that match negative questions
|
|
filteredChunks := make([]*types.IndexWithScore, 0, len(chunks))
|
|
for _, chunk := range chunks {
|
|
chunkData, ok := chunkMap[chunk.ChunkID]
|
|
if !ok {
|
|
// If chunk not found, keep it (shouldn't happen, but be safe)
|
|
filteredChunks = append(filteredChunks, chunk)
|
|
continue
|
|
}
|
|
|
|
// Only filter FAQ type chunks
|
|
if chunkData.ChunkType != types.ChunkTypeFAQ {
|
|
filteredChunks = append(filteredChunks, chunk)
|
|
continue
|
|
}
|
|
|
|
// Get FAQ metadata and check negative questions
|
|
meta, err := chunkData.FAQMetadata()
|
|
if err != nil || meta == nil {
|
|
// If we can't parse metadata, keep the chunk
|
|
filteredChunks = append(filteredChunks, chunk)
|
|
continue
|
|
}
|
|
|
|
// Check if query matches any negative question
|
|
if s.matchesNegativeQuestions(queryTextLower, meta.NegativeQuestions) {
|
|
logger.Debugf(ctx, "Filtered FAQ chunk %s due to negative question match", chunk.ChunkID)
|
|
continue
|
|
}
|
|
|
|
// Keep the chunk
|
|
filteredChunks = append(filteredChunks, chunk)
|
|
}
|
|
|
|
return filteredChunks
|
|
}
|
|
|
|
// matchesNegativeQuestions checks if the query text matches any negative questions.
|
|
// Returns true if the query matches any negative question, false otherwise.
|
|
func (s *knowledgeBaseService) matchesNegativeQuestions(queryTextLower string, negativeQuestions []string) bool {
|
|
if len(negativeQuestions) == 0 {
|
|
return false
|
|
}
|
|
|
|
for _, negativeQ := range negativeQuestions {
|
|
negativeQLower := strings.ToLower(strings.TrimSpace(negativeQ))
|
|
if negativeQLower == "" {
|
|
continue
|
|
}
|
|
// Check if query text is exactly the same as the negative question
|
|
if queryTextLower == negativeQLower {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|