WeKnora/internal/handler/chunker_debug_test.go

343 lines
12 KiB
Go
Raw Normal View History

fix(chunker): post-review fixes for preview endpoint robustness Resolves issues from the review of be326aa..119f5e4. Each fix has a regression test attached and overclaimed findings (parserEngineRules defensive copy, runTier dead-code branch, init-vars architecture) were intentionally not touched after re-evaluation. Goroutine leak mitigation - previewMaxChars dropped from 256k to 64k runes. The splitter does not accept a context.Context, so when previewTimeout fires the worker keeps running. Bounding input size keeps worst-case CPU per request well under a second on commodity hardware. Sized so 10 concurrent timeouts don't pile up faster than they finish. - Frontend MAX_CHARS lowered to match. - Comment in handler explains the trade-off and points at the follow-up: real cancellation needs the splitter to take a ctx. Performance - ApproxTokenCountFromRuneLen variant lets the preview handler reuse a single rune-count per chunk for stats + size + token estimation. Eliminates the previous triple []rune allocation per chunk in the response loop. - computeChunkSizeStats now takes []int (pre-computed rune lens) instead of []chunker.Chunk; sumSq computed in float64 to avoid the int*int overflow at l > ~46k. Correctness / UX - Preview panel sends strategy / token_limit / languages unconditionally, mirroring the buildSubmitData convention so the preview faithfully reflects what would happen on save. - Empty-text returns a friendly 400 ("paste a sample…") instead of gin's cryptic 'Field validation failed on the required tag'. Tests - TestSplit_DelegatesToSplitWithDiagnostics renamed to TestSplit_AndDiagnostics_AgreeOnChunks (the post-audit refactor made the original name a misnomer; the test still asserts the right invariant under the new name). - New TestSplitWithDiagnostics_ProfileSetForAuto and TestSplitWithDiagnostics_ProfileNilForExplicit lock in the profile-reuse contract that the preview endpoint depends on. - New chunker_debug_test.go covers computeChunkSizeStats edge cases (empty / single / varying / no-variance underflow) plus PreviewChunking httptest scenarios (auto path, legacy strategy, empty-text rejection, oversize rejection, chunk truncation with full-set stats). Doc cleanup - runTier comment updated; the "stubbed in this scaffold" line was obsolete since the heading and heuristic splitters shipped. https://claude.ai/code/session_01XADhx6mtu2ZYW3DE9Lun6k
2026-05-03 07:16:47 +00:00
package handler
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
"github.com/gin-gonic/gin"
)
func init() {
gin.SetMode(gin.TestMode)
}
func TestComputeChunkSizeStats_Empty(t *testing.T) {
stats := computeChunkSizeStats(nil)
if stats.Count != 0 || stats.AvgChars != 0 || stats.MaxChars != 0 {
t.Errorf("empty input should yield zero stats, got %+v", stats)
}
}
func TestComputeChunkSizeStats_SingleChunk(t *testing.T) {
stats := computeChunkSizeStats([]int{500})
if stats.Count != 1 {
t.Errorf("count: got %d want 1", stats.Count)
}
if stats.AvgChars != 500 || stats.MinChars != 500 || stats.MaxChars != 500 {
t.Errorf("single-chunk stats should all equal 500, got %+v", stats)
}
if stats.StddevChars != 0 {
t.Errorf("stddev for one element should be 0, got %d", stats.StddevChars)
}
}
func TestComputeChunkSizeStats_VaryingSizes(t *testing.T) {
// 100, 200, 300, 400, 500 → avg 300, stddev ≈ 141
stats := computeChunkSizeStats([]int{100, 200, 300, 400, 500})
if stats.Count != 5 {
t.Errorf("count: got %d want 5", stats.Count)
}
if stats.AvgChars != 300 {
t.Errorf("avg: got %d want 300", stats.AvgChars)
}
if stats.MinChars != 100 || stats.MaxChars != 500 {
t.Errorf("min/max: got %d/%d want 100/500", stats.MinChars, stats.MaxChars)
}
if stats.StddevChars < 130 || stats.StddevChars > 150 {
t.Errorf("stddev: got %d, want ~141", stats.StddevChars)
}
}
func TestComputeChunkSizeStats_NoVarianceUnderflow(t *testing.T) {
// All identical — variance must clamp to 0 not flip negative on
// float-precision rounding.
stats := computeChunkSizeStats([]int{1234, 1234, 1234, 1234})
if stats.StddevChars != 0 {
t.Errorf("identical values must yield stddev=0, got %d", stats.StddevChars)
}
}
// --- PreviewChunking httptest -------------------------------------------------
func newPreviewRouter() *gin.Engine {
r := gin.New()
r.POST("/chunker/preview", PreviewChunking)
return r
}
func postPreview(t *testing.T, body any) (*httptest.ResponseRecorder, map[string]any) {
t.Helper()
r := newPreviewRouter()
buf := &bytes.Buffer{}
if err := json.NewEncoder(buf).Encode(body); err != nil {
t.Fatalf("encode body: %v", err)
}
req := httptest.NewRequest(http.MethodPost, "/chunker/preview", buf)
req.Header.Set("Content-Type", "application/json")
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
var parsed map[string]any
if w.Body.Len() > 0 {
_ = json.Unmarshal(w.Body.Bytes(), &parsed)
}
return w, parsed
}
func TestPreviewChunking_HappyPath_AutoStrategy(t *testing.T) {
body := PreviewChunkingRequest{
Text: "# Top\nintro paragraph here.\n\n## Section A\nbody A.\n\n## Section B\nbody B.",
ChunkingConfig: PreviewChunkingPayload{
ChunkSize: 200,
ChunkOverlap: 20,
Separators: []string{"\n\n", "\n"},
Strategy: "auto",
},
}
w, parsed := postPreview(t, body)
if w.Code != http.StatusOK {
t.Fatalf("status: got %d want 200; body=%s", w.Code, w.Body.String())
}
if parsed["success"] != true {
t.Fatalf("success flag missing or false: %v", parsed)
}
data, ok := parsed["data"].(map[string]any)
if !ok {
t.Fatalf("data missing: %v", parsed)
}
if data["selected_tier"] == "" {
t.Errorf("selected_tier must be set, got %v", data["selected_tier"])
}
if _, ok := data["chunks"].([]any); !ok {
t.Errorf("chunks must be an array, got %T", data["chunks"])
}
stats, ok := data["stats"].(map[string]any)
if !ok {
t.Fatalf("stats must be an object, got %T", data["stats"])
}
if c, _ := stats["count"].(float64); c <= 0 {
t.Errorf("stats.count should be > 0, got %v", stats["count"])
}
}
func TestPreviewChunking_RejectsEmptyText(t *testing.T) {
w, parsed := postPreview(t, PreviewChunkingRequest{Text: " \n\t "})
if w.Code != http.StatusBadRequest {
t.Errorf("status: got %d want 400", w.Code)
}
if errStr, _ := parsed["error"].(string); !strings.Contains(errStr, "empty") {
t.Errorf("error should mention 'empty', got %q", errStr)
}
}
func TestPreviewChunking_RejectsOversizedText(t *testing.T) {
body := PreviewChunkingRequest{Text: strings.Repeat("a", previewMaxChars+1)}
w, parsed := postPreview(t, body)
if w.Code != http.StatusRequestEntityTooLarge {
t.Errorf("status: got %d want 413", w.Code)
}
if parsed["limit"] == nil {
t.Errorf("response should include limit hint, got %v", parsed)
}
}
func TestPreviewChunking_LegacyStrategy_NoProfile(t *testing.T) {
// Auto-strategy is the only path that produces a profile inside
// SplitWithDiagnostics. For explicit strategies the handler
// materializes one itself so the UI always sees stats.
body := PreviewChunkingRequest{
Text: "para one.\n\npara two.\n\npara three.\n\npara four.",
ChunkingConfig: PreviewChunkingPayload{
ChunkSize: 100,
ChunkOverlap: 10,
Separators: []string{"\n\n"},
Strategy: "legacy",
},
}
w, parsed := postPreview(t, body)
if w.Code != http.StatusOK {
t.Fatalf("status %d body=%s", w.Code, w.Body.String())
}
data := parsed["data"].(map[string]any)
if data["profile"] == nil {
t.Error("profile should be materialized for explicit strategy too")
}
if string(chunker.StrategyTier(data["selected_tier"].(string))) != string(chunker.TierLegacy) {
t.Errorf("selected_tier: got %v want %s", data["selected_tier"], chunker.TierLegacy)
}
}
func TestPreviewChunking_ChunkTruncation(t *testing.T) {
// Build text that produces > previewMaxChunks chunks.
body := PreviewChunkingRequest{
Text: strings.Repeat("x.\n\n", previewMaxChunks+50),
ChunkingConfig: PreviewChunkingPayload{
ChunkSize: 3,
ChunkOverlap: 0,
Separators: []string{"\n\n"},
Strategy: "legacy",
},
}
w, parsed := postPreview(t, body)
if w.Code != http.StatusOK {
t.Fatalf("status %d", w.Code)
}
data := parsed["data"].(map[string]any)
chunks := data["chunks"].([]any)
if len(chunks) > previewMaxChunks {
t.Errorf("chunks should be truncated to ≤%d, got %d", previewMaxChunks, len(chunks))
}
stats := data["stats"].(map[string]any)
if truncated, _ := stats["truncated_to"].(float64); int(truncated) <= previewMaxChunks {
t.Errorf("stats.truncated_to should reflect ORIGINAL count > %d, got %v", previewMaxChunks, truncated)
}
}
func TestPreviewChunking_ParentChildMatchesIngestion(t *testing.T) {
text := strings.Repeat("## Record\n"+strings.Repeat("A sufficiently long entry body. ", 10)+"\n\n", 12)
payload := PreviewChunkingPayload{
ChunkSize: 300,
ChunkOverlap: 30,
Separators: []string{"\n\n", "\n"},
EnableParentChild: true,
ParentChunkSize: 300,
ChildChunkSize: 100,
Strategy: chunker.StrategyHeading,
}
w, parsed := postPreview(t, PreviewChunkingRequest{Text: text, ChunkingConfig: payload})
if w.Code != http.StatusOK {
t.Fatalf("status %d body=%s", w.Code, w.Body.String())
}
base := chunker.NormalizeSplitterConfig(chunker.SplitterConfig{
ChunkSize: payload.ChunkSize,
ChunkOverlap: payload.ChunkOverlap,
Separators: payload.Separators,
Strategy: payload.Strategy,
})
parentCfg, childCfg := chunker.DeriveParentChildConfigs(base, payload.ParentChunkSize, payload.ChildChunkSize)
want := chunker.SplitParentChild(text, parentCfg, childCfg)
data := parsed["data"].(map[string]any)
got := data["chunks"].([]any)
if len(got) != len(want.Children) {
t.Fatalf("chunk count: got %d want %d", len(got), len(want.Children))
}
for i, child := range want.Children {
previewChunk := got[i].(map[string]any)
if previewChunk["content"] != child.Content {
t.Errorf("chunk %d content differs", i)
}
if int(previewChunk["start"].(float64)) != child.Start || int(previewChunk["end"].(float64)) != child.End {
t.Errorf("chunk %d span: got %v-%v want %d-%d", i, previewChunk["start"], previewChunk["end"], child.Start, child.End)
}
gotHeader, _ := previewChunk["context_header"].(string)
if gotHeader != child.ContextHeader {
t.Errorf("chunk %d context_header: got %q want %q", i, gotHeader, child.ContextHeader)
}
}
}
func TestPreviewChunking_SingleLevelUnchanged(t *testing.T) {
text := strings.Repeat("Paragraph one.\n\nParagraph two.\n\n", 20)
payload := PreviewChunkingPayload{
ChunkSize: 200,
ChunkOverlap: 20,
Separators: []string{"\n\n", "\n"},
Strategy: chunker.StrategyLegacy,
}
w, parsed := postPreview(t, PreviewChunkingRequest{Text: text, ChunkingConfig: payload})
if w.Code != http.StatusOK {
t.Fatalf("status %d body=%s", w.Code, w.Body.String())
}
want, _ := chunker.SplitWithDiagnostics(text, chunker.NormalizeSplitterConfig(chunker.SplitterConfig{
ChunkSize: payload.ChunkSize,
ChunkOverlap: payload.ChunkOverlap,
Separators: payload.Separators,
Strategy: payload.Strategy,
}))
data := parsed["data"].(map[string]any)
got := data["chunks"].([]any)
if len(got) != len(want) {
t.Fatalf("chunk count: got %d want %d", len(got), len(want))
}
}
func TestPreviewChunking_ParentChildDefaultSizes(t *testing.T) {
text := strings.Repeat("## Record\n"+strings.Repeat("A sufficiently long entry body. ", 10)+"\n\n", 12)
payload := PreviewChunkingPayload{
ChunkSize: 300,
ChunkOverlap: 30,
Separators: []string{"\n\n", "\n"},
EnableParentChild: true,
Strategy: chunker.StrategyHeading,
}
w, parsed := postPreview(t, PreviewChunkingRequest{Text: text, ChunkingConfig: payload})
if w.Code != http.StatusOK {
t.Fatalf("status %d body=%s", w.Code, w.Body.String())
}
base := chunker.NormalizeSplitterConfig(chunker.SplitterConfig{
ChunkSize: payload.ChunkSize,
ChunkOverlap: payload.ChunkOverlap,
Separators: payload.Separators,
Strategy: payload.Strategy,
})
parentCfg, childCfg := chunker.DeriveParentChildConfigs(base, 0, 0)
want := chunker.SplitParentChild(text, parentCfg, childCfg)
data := parsed["data"].(map[string]any)
got := data["chunks"].([]any)
if len(got) != len(want.Children) {
t.Fatalf("chunk count: got %d want %d", len(got), len(want.Children))
}
}
func TestPreviewChunking_LineEndingsMatchUpload(t *testing.T) {
uploaded := strings.Repeat("## Record\r\n"+strings.Repeat("A sufficiently long entry body. ", 10)+"\r\n\r\n", 12)
pasted := strings.ReplaceAll(uploaded, "\r\n", "\n") // HTML textarea normalization
payload := PreviewChunkingPayload{
ChunkSize: 500,
ChunkOverlap: 20,
Separators: []string{"\n\n", "\n", "。", "!", "?", ";", ";"},
Strategy: chunker.StrategyHeading,
}
actual := chunker.Split(chunker.NormalizeLineEndings(uploaded), chunker.NormalizeSplitterConfig(chunker.SplitterConfig{
ChunkSize: payload.ChunkSize,
ChunkOverlap: payload.ChunkOverlap,
Separators: payload.Separators,
Strategy: payload.Strategy,
}))
for name, text := range map[string]string{
"uploaded CRLF": uploaded,
"pasted LF": pasted,
} {
t.Run(name, func(t *testing.T) {
w, parsed := postPreview(t, PreviewChunkingRequest{Text: text, ChunkingConfig: payload})
if w.Code != http.StatusOK {
t.Fatalf("status %d body=%s", w.Code, w.Body.String())
}
data := parsed["data"].(map[string]any)
preview := data["chunks"].([]any)
if len(preview) != len(actual) {
t.Fatalf("chunk count: got %d want %d", len(preview), len(actual))
}
for i, chunk := range actual {
previewChunk := preview[i].(map[string]any)
if previewChunk["content"] != chunk.Content {
t.Errorf("chunk %d content differs", i)
}
if int(previewChunk["start"].(float64)) != chunk.Start || int(previewChunk["end"].(float64)) != chunk.End {
t.Errorf("chunk %d span: got %v-%v want %d-%d", i, previewChunk["start"], previewChunk["end"], chunk.Start, chunk.End)
}
}
})
}
}