WeKnora/internal/handler/system_runtime_test.go

610 lines
21 KiB
Go
Raw Normal View History

package handler
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"testing"
"github.com/Tencent/WeKnora/internal/application/repository"
"github.com/Tencent/WeKnora/internal/types"
"github.com/gin-gonic/gin"
)
type runtimeTestSettings struct{}
func (runtimeTestSettings) GetInt(_ context.Context, key, _ string, def int64) int64 {
switch key {
case "asynq.core_concurrency":
return 8
case "asynq.postprocess_concurrency":
return 2
case "asynq.enrichment_concurrency":
return 12
case "asynq.maintenance_concurrency":
return 4
case "asynq.shared_concurrency":
return 6
case "asynq.wiki_concurrency":
return 8
default:
return def
}
}
func (runtimeTestSettings) GetString(_ context.Context, _, _, def string) string { return def }
func (runtimeTestSettings) GetBool(_ context.Context, _, _ string, def bool) bool { return def }
func (runtimeTestSettings) GetStringList(_ context.Context, _, _ string, def []string) []string {
return def
}
func (runtimeTestSettings) List(context.Context) ([]*types.SystemSetting, error) { return nil, nil }
func (runtimeTestSettings) Get(context.Context, string) (*types.SystemSetting, error) {
return nil, nil
}
func (runtimeTestSettings) Update(context.Context, string, any) (*types.SystemSetting, error) {
return nil, nil
}
func (runtimeTestSettings) Reset(context.Context, string) error { return nil }
func (runtimeTestSettings) SubscribeRedis(context.Context) error { return nil }
type runtimeInvalidSettings struct{ runtimeTestSettings }
func (runtimeInvalidSettings) GetInt(_ context.Context, _ string, _ string, _ int64) int64 {
return 0
}
type runtimeTestInspector struct{}
func (runtimeTestInspector) CancelTasksForKnowledge(context.Context, string) (int, int, error) {
return 0, 0, nil
}
fix(knowledge): recover stranded deleting rows and align the KB count (#3367) Second slice of #3338 (tracked as #3345). Count alignment: CountKnowledgeByKnowledgeBaseID now excludes rows mid-deletion, mirroring the document list's existing filter — a stranded delete no longer shows as '4 documents, 3 listed' on the KB card. Failed rows stay counted: they are visible, actionable cards. Sweep C: housekeeping now recovers rows stuck at 'deleting' whose delete task no longer exists. The delete dead-letter callback only flips rows that are still 'deleting' when retries exhaust, so a worker death mid-delete or a lost queue left the row hidden from the list forever. Same recovery contract as the parse sweep: flip to failed with a clear error so the row becomes visible and actionable; the delete executor is idempotent and plans by row ID, so a late-arriving task still finishes cleanly. Liveness gate: a new TaskInspector probe (HasQueuedDeleteTasksForKnowledge) matches knowledge:list_delete batch payloads — the per-parse matcher cannot see them. A live queued/active delete defers the row (backlog, not stranded); a probe error defers that row; a nil inspector defers the whole sweep. Lite mode's noop returns false by design: inline deletes never queue, so a stale deleting row there genuinely has no task. Tests: sweep matrix (stranded recovered / backlogged kept / fresh untouched / probe-error deferred / nil-inspector deferred), count alignment, and the payload matcher table. Refs #3345, #3338
2026-09-17 16:55:44 +00:00
func (runtimeTestInspector) HasQueuedDeleteTasksForKnowledge(context.Context, string) (bool, error) {
return false, nil
}
func (runtimeTestInspector) HasQueuedTasksForKnowledge(context.Context, string) (bool, error) {
return false, nil
}
feat(knowledge): flag documents that stop making progress (#3557) * feat(knowledge): flag documents that stop making progress A document stuck in processing looked the same as one still working: a spinner that polled forever, no hint of where it stopped, and, hours later, a generic housekeeping failure. - The batch and spans endpoints return last_activity_at for in-flight rows: the later of updated_at and the newest span write. updated_at only moves at stage transitions; span writes keep moving while a long stage still works. - After 20 minutes with no progress, the list and card views show "may be stuck" with a tooltip. The timeline shows the stage it stopped at and a stop-parsing button, plus a "last progress" stat. Both pollers back off to 15s while everything in flight is stalled. - When housekeeping fails a row, the error names the stalled stage and the time of the last progress. It closes the row's open spans with TASK_STALLED (the running stage fails, the rest are cancelled), so the timeline shows where the run stopped instead of a spinner. * fix(knowledge): locate stalls in finalizing and tell backlog from stuck - Housekeeping reads the stall site from the latest attempt. It takes the running stages, or, when none runs (post-process closes its stage once enrichment is fanned out), the innermost running spans, mapped back to their stage. Those spans fail with TASK_STALLED and are named in the message; every stalled stage is listed in pipeline order. The other open spans are cancelled. Every closed span gets a duration, and the message leaves out the time when the heartbeat query failed. - The spans tree prefers a TASK_STALLED failure for last_error over a later subtask failure. current_stage falls back to the stage owning a running subspan. - Batch and spans responses flag rows quiet past the stall hint whose work is still queued (waiting_in_queue), using the sweep's own probes. At most 20 rows are probed per request, cached for a minute. The UI shows these as queued, with no stop button in the banner. * fix(knowledge): answer stall verdicts from one shared queue scan - Probing a quiet document for a backlog rescanned the whole queue once per document. When the document was absent, which is exactly the stuck case, the scan ran to the end, and it read a Redis error as "nothing queued". QueuedKnowledgeIDs now collects every referenced knowledge ID in one pass and returns list errors. HousekeepingService keeps that snapshot for a minute, shared across requests and single-flight. A failed scan is not cached. - With the per-request cap of 20 gone, every quiet row on a page gets a verdict. The response carries it as stall_state (queued or stalled), replacing waiting_in_queue. The UI marks a row as stuck only on a "stalled" verdict. A row with no verdict, because the probe failed, shows as ordinary processing instead of inviting a stop.
2026-09-22 13:34:08 +00:00
func (runtimeTestInspector) QueuedKnowledgeIDs(context.Context) (map[string]struct{}, error) {
return map[string]struct{}{}, nil
}
func (runtimeTestInspector) QueueStats(context.Context) ([]types.QueueStat, bool, error) {
return []types.QueueStat{}, true, nil
}
func (runtimeTestInspector) WorkerServerStats(context.Context) ([]types.WorkerServerStat, bool, error) {
return []types.WorkerServerStat{
{Concurrency: 8, Active: 4, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolCore)},
{Concurrency: 2, Active: 1, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolPostProcess)},
{Concurrency: 12, Active: 6, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolEnrichment)},
{Concurrency: 4, Active: 1, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolMaintenance)},
{Concurrency: 6, Active: 3, Status: "active", Queues: types.QueueWeightsForSharedPool()},
{Concurrency: 8, Active: 2, Status: "active", Queues: types.QueueWeightsForPool(types.WorkerPoolWiki)},
{Concurrency: 99, Active: 0, Status: "stopped", Queues: types.QueueWeightsForPool(types.WorkerPoolCore)},
}, true, nil
}
type runtimeTaskTestInspector struct {
runtimeTestInspector
tasks []types.RuntimeTaskInfo
retriedTask string
deletedTask string
forceDeleted string
purgedQueue string
purgedCount int
purgeErr error
cancelKnowledge string
cancelDeleted int
mutatedQueue string
nextCursor string
hasMore bool
inputCursor string
inputPageSize int
listErr error
forceDeleteErr error
getRuntimeErr error
getRuntimeErrFrom int
getRuntimeTaskCalls int
}
func (r *runtimeTaskTestInspector) CancelTasksForKnowledge(
_ context.Context, knowledgeID string,
) (int, int, error) {
r.cancelKnowledge = knowledgeID
if r.cancelDeleted > 0 {
return r.cancelDeleted, 0, nil
}
return 0, 0, nil
}
type runtimeKnowledgeCancelTest struct {
tenantID uint64
knowledgeID string
err error
}
func (r *runtimeKnowledgeCancelTest) CancelKnowledgeParse(
ctx context.Context, knowledgeID string,
) (*types.Knowledge, error) {
r.tenantID, _ = ctx.Value(types.TenantIDContextKey).(uint64)
r.knowledgeID = knowledgeID
if r.err != nil {
return nil, r.err
}
return &types.Knowledge{ID: knowledgeID, TenantID: r.tenantID}, nil
}
func (r *runtimeTaskTestInspector) ListRuntimeTasks(
_ context.Context, _ string, state types.RuntimeTaskState, cursor string, pageSize int,
) (types.RuntimeTaskPage, bool, error) {
r.inputCursor = cursor
r.inputPageSize = pageSize
if r.listErr != nil {
return types.RuntimeTaskPage{}, true, r.listErr
}
for i := range r.tasks {
if r.tasks[i].State == "" {
r.tasks[i].State = state
}
if r.tasks[i].AllowedActions == nil && state == types.RuntimeTaskArchived {
r.tasks[i].AllowedActions = []types.RuntimeTaskAction{
types.RuntimeTaskActionRunNow,
types.RuntimeTaskActionDelete,
}
}
}
return types.RuntimeTaskPage{
Tasks: r.tasks, NextCursor: r.nextCursor, HasMore: r.hasMore,
}, true, nil
}
func (r *runtimeTaskTestInspector) GetRuntimeTask(
_ context.Context, queue, taskID string,
) (*types.RuntimeTaskInfo, bool, error) {
r.getRuntimeTaskCalls++
if r.getRuntimeErr != nil && r.getRuntimeTaskCalls >= r.getRuntimeErrFrom {
return nil, true, r.getRuntimeErr
}
for i := range r.tasks {
if r.tasks[i].ID == taskID {
return &r.tasks[i], true, nil
}
}
return &types.RuntimeTaskInfo{
ID: taskID, Queue: queue, State: types.RuntimeTaskArchived,
AllowedActions: []types.RuntimeTaskAction{
types.RuntimeTaskActionRunNow,
types.RuntimeTaskActionDelete,
},
}, true, nil
}
func (r *runtimeTaskTestInspector) RunRuntimeTask(
_ context.Context, queue, taskID string,
) (bool, error) {
r.mutatedQueue = queue
r.retriedTask = taskID
return true, nil
}
func (r *runtimeTaskTestInspector) DeleteRuntimeTask(
_ context.Context, queue, taskID string,
) (bool, error) {
r.mutatedQueue = queue
r.deletedTask = taskID
return true, nil
}
func (r *runtimeTaskTestInspector) ForceDeleteRuntimeTask(
_ context.Context, queue, taskID string,
) (bool, error) {
r.mutatedQueue = queue
r.forceDeleted = taskID
return true, r.forceDeleteErr
}
func (r *runtimeTaskTestInspector) PurgeArchivedRuntimeTasks(
_ context.Context, queue string,
) (int, bool, error) {
r.purgedQueue = queue
if r.purgeErr != nil {
return 0, true, r.purgeErr
}
return r.purgedCount, true, nil
}
func TestGetRuntimeQueuesReportsIsolatedPoolCapacity(t *testing.T) {
gin.SetMode(gin.TestMode)
handler := &SystemHandler{
systemSettingSvc: runtimeTestSettings{},
taskInspector: runtimeTestInspector{},
}
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Request = httptest.NewRequest(http.MethodGet, "/api/v1/system/admin/runtime/queues", nil)
handler.GetRuntimeQueues(ctx)
if recorder.Code != http.StatusOK {
t.Fatalf("status = %d, want %d; body=%s", recorder.Code, http.StatusOK, recorder.Body.String())
}
var response RuntimeQueuesResponse
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
t.Fatalf("decode response: %v", err)
}
if !response.Available {
t.Fatal("queue inspection should be available")
}
if response.UpstreamConcurrency != 32 || response.ParseConcurrency != 32 {
t.Fatalf("upstream compatibility values are wrong: %+v", response)
}
want := map[string]struct {
concurrency int
queueCount int
}{
types.WorkerPoolCore: {8, 2},
types.WorkerPoolPostProcess: {2, 1},
types.WorkerPoolEnrichment: {12, 5},
types.WorkerPoolMaintenance: {4, 2},
types.WorkerPoolShared: {6, 7},
types.WorkerPoolWiki: {8, 1},
}
if len(response.Pools) != len(want) {
t.Fatalf("pool count = %d, want %d", len(response.Pools), len(want))
}
for _, pool := range response.Pools {
expected, ok := want[pool.Name]
if !ok {
t.Fatalf("unexpected pool %q", pool.Name)
}
if pool.Concurrency != expected.concurrency || pool.QueueCount != expected.queueCount {
t.Fatalf("pool %q = %+v, want concurrency=%d queue_count=%d",
pool.Name, pool, expected.concurrency, expected.queueCount)
}
if pool.Instances != 1 || pool.ClusterCapacity != expected.concurrency {
t.Fatalf("pool %q live capacity = %+v", pool.Name, pool)
}
}
}
func TestGetRuntimeQueuesFallsBackFromInvalidHistoricalConcurrency(t *testing.T) {
gin.SetMode(gin.TestMode)
handler := &SystemHandler{
systemSettingSvc: runtimeInvalidSettings{},
taskInspector: runtimeTestInspector{},
}
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Request = httptest.NewRequest(http.MethodGet, "/api/v1/system/admin/runtime/queues", nil)
handler.GetRuntimeQueues(ctx)
var response RuntimeQueuesResponse
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
t.Fatalf("decode response: %v", err)
}
if response.UpstreamConcurrency != types.DefaultUpstreamWorkerConcurrency ||
response.WikiConcurrency != types.DefaultWikiWorkerConcurrency {
t.Fatalf("invalid stored values should use worker defaults: %+v", response)
}
for _, pool := range response.Pools {
if pool.Concurrency < 1 {
t.Fatalf("pool %q reported non-positive concurrency: %+v", pool.Name, pool)
}
}
}
func TestListRuntimeTasksReturnsSafeTaskDetails(t *testing.T) {
gin.SetMode(gin.TestMode)
inspector := &runtimeTaskTestInspector{tasks: []types.RuntimeTaskInfo{{
ID: "task-1",
Queue: types.QueueDefault,
Type: types.TypeDocumentProcess,
LastError: "model unavailable",
Retried: 5,
MaxRetry: 5,
KnowledgeBaseID: "kb-1",
KnowledgeID: "knowledge-1",
}}}
handler := &SystemHandler{taskInspector: inspector}
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Params = gin.Params{{Key: "queue", Value: types.QueueDefault}}
ctx.Request = httptest.NewRequest(http.MethodGet, "/api/v1/system/admin/runtime/queues/default/tasks?state=archived", nil)
handler.ListRuntimeTasks(ctx)
if recorder.Code != http.StatusOK {
t.Fatalf("status = %d, body=%s", recorder.Code, recorder.Body.String())
}
var response RuntimeTasksResponse
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
t.Fatalf("decode response: %v", err)
}
if !response.Available || len(response.Tasks) != 1 {
t.Fatalf("unexpected response: %+v", response)
}
if response.Tasks[0].KnowledgeID != "knowledge-1" || response.Tasks[0].LastError != "model unavailable" {
t.Fatalf("task details missing: %+v", response.Tasks[0])
}
}
func TestListRuntimeTasksReturnsAndForwardsCursor(t *testing.T) {
gin.SetMode(gin.TestMode)
inspector := &runtimeTaskTestInspector{
tasks: []types.RuntimeTaskInfo{{ID: "task-1"}},
nextCursor: "next-page", hasMore: true,
}
handler := &SystemHandler{taskInspector: inspector}
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Params = gin.Params{{Key: "queue", Value: types.QueueDefault}}
ctx.Request = httptest.NewRequest(
http.MethodGet,
"/api/v1/system/admin/runtime/queues/default/tasks?state=archived&cursor=previous-page&page_size=25",
nil,
)
handler.ListRuntimeTasks(ctx)
if recorder.Code != http.StatusOK {
t.Fatalf("status = %d, body=%s", recorder.Code, recorder.Body.String())
}
var response RuntimeTasksResponse
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
t.Fatalf("decode response: %v", err)
}
if inspector.inputCursor != "previous-page" || inspector.inputPageSize != 25 {
t.Fatalf("cursor request not forwarded: cursor=%q size=%d", inspector.inputCursor, inspector.inputPageSize)
}
if !response.HasMore || response.NextCursor != "next-page" {
t.Fatalf("cursor response missing: %+v", response)
}
}
func TestListRuntimeTasksReportsExpiredCursorForFrontendRefresh(t *testing.T) {
gin.SetMode(gin.TestMode)
handler := &SystemHandler{taskInspector: &runtimeTaskTestInspector{
listErr: types.ErrExpiredRuntimeTaskCursor,
}}
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Params = gin.Params{{Key: "queue", Value: types.QueueDefault}}
ctx.Request = httptest.NewRequest(
http.MethodGet,
"/api/v1/system/admin/runtime/queues/default/tasks?state=pending&cursor=expired",
nil,
)
handler.ListRuntimeTasks(ctx)
if recorder.Code != http.StatusConflict {
t.Fatalf("status = %d, body=%s", recorder.Code, recorder.Body.String())
}
var response map[string]any
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
t.Fatalf("decode response: %v", err)
}
if response["code"] != "runtime_task_cursor_expired" {
t.Fatalf("unexpected error response: %+v", response)
}
}
func TestRuntimeTaskMutationsDelegateToInspector(t *testing.T) {
gin.SetMode(gin.TestMode)
inspector := &runtimeTaskTestInspector{}
handler := &SystemHandler{taskInspector: inspector}
retryRecorder := httptest.NewRecorder()
retryCtx, _ := gin.CreateTestContext(retryRecorder)
retryCtx.Params = gin.Params{
{Key: "queue", Value: types.QueueDefault},
{Key: "task_id", Value: "task-1"},
{Key: "action", Value: string(types.RuntimeTaskActionRunNow)},
}
retryCtx.Request = httptest.NewRequest(http.MethodPost, "/retry", nil)
handler.MutateRuntimeTask(retryCtx)
if retryRecorder.Code != http.StatusOK || inspector.retriedTask != "task-1" {
t.Fatalf("retry failed: status=%d inspector=%+v", retryRecorder.Code, inspector)
}
deleteRecorder := httptest.NewRecorder()
deleteCtx, _ := gin.CreateTestContext(deleteRecorder)
deleteCtx.Params = gin.Params{
{Key: "queue", Value: types.QueueDefault},
{Key: "task_id", Value: "task-2"},
{Key: "action", Value: string(types.RuntimeTaskActionDelete)},
}
deleteCtx.Request = httptest.NewRequest(http.MethodDelete, "/task-2", nil)
handler.MutateRuntimeTask(deleteCtx)
if deleteRecorder.Code != http.StatusOK || inspector.deletedTask != "task-2" {
t.Fatalf("delete failed: status=%d inspector=%+v", deleteRecorder.Code, inspector)
}
}
func TestPurgeArchivedRuntimeTasksDelegatesToInspector(t *testing.T) {
gin.SetMode(gin.TestMode)
inspector := &runtimeTaskTestInspector{purgedCount: 7}
handler := &SystemHandler{taskInspector: inspector}
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Params = gin.Params{{Key: "queue", Value: types.QueueChatAttachment}}
ctx.Request = httptest.NewRequest(http.MethodDelete, "/archived", nil)
handler.PurgeArchivedRuntimeTasks(ctx)
if recorder.Code != http.StatusOK {
t.Fatalf("status = %d, body=%s", recorder.Code, recorder.Body.String())
}
if inspector.purgedQueue != types.QueueChatAttachment {
t.Fatalf("purged queue = %q, want %q", inspector.purgedQueue, types.QueueChatAttachment)
}
var response map[string]any
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
t.Fatalf("decode response: %v", err)
}
if response["deleted"] != float64(7) {
t.Fatalf("deleted = %v, want 7", response["deleted"])
}
}
func TestPurgeArchivedRuntimeTasksRejectsUnknownQueue(t *testing.T) {
gin.SetMode(gin.TestMode)
handler := &SystemHandler{taskInspector: &runtimeTaskTestInspector{}}
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Params = gin.Params{{Key: "queue", Value: "unknown"}}
ctx.Request = httptest.NewRequest(http.MethodDelete, "/archived", nil)
handler.PurgeArchivedRuntimeTasks(ctx)
if recorder.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want %d", recorder.Code, http.StatusBadRequest)
}
}
func TestListRuntimeTasksRejectsUnknownQueue(t *testing.T) {
gin.SetMode(gin.TestMode)
handler := &SystemHandler{taskInspector: &runtimeTaskTestInspector{}}
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Params = gin.Params{{Key: "queue", Value: "unknown"}}
ctx.Request = httptest.NewRequest(http.MethodGet, "/tasks?state=archived", nil)
handler.ListRuntimeTasks(ctx)
if recorder.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want %d", recorder.Code, http.StatusBadRequest)
}
}
func TestListRuntimeTasksRejectsUnknownState(t *testing.T) {
gin.SetMode(gin.TestMode)
handler := &SystemHandler{taskInspector: &runtimeTaskTestInspector{}}
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Params = gin.Params{{Key: "queue", Value: types.QueueDefault}}
ctx.Request = httptest.NewRequest(http.MethodGet, "/tasks?state=unknown", nil)
handler.ListRuntimeTasks(ctx)
if recorder.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want %d", recorder.Code, http.StatusBadRequest)
}
}
func TestRuntimeTaskCancelUsesDomainCancellationWithTaskTenant(t *testing.T) {
gin.SetMode(gin.TestMode)
inspector := &runtimeTaskTestInspector{tasks: []types.RuntimeTaskInfo{{
ID: "task-cancel", Queue: types.QueueDefault, Type: types.TypeDocumentProcess,
State: types.RuntimeTaskActive, TenantID: 42, KnowledgeID: "knowledge-42",
AllowedActions: []types.RuntimeTaskAction{types.RuntimeTaskActionCancel},
}}}
canceller := &runtimeKnowledgeCancelTest{}
handler := &SystemHandler{taskInspector: inspector, knowledgeSvc: canceller}
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Params = gin.Params{
{Key: "queue", Value: types.QueueDefault},
{Key: "task_id", Value: "task-cancel"},
{Key: "action", Value: string(types.RuntimeTaskActionCancel)},
}
ctx.Request = httptest.NewRequest(http.MethodPost, "/cancel", nil)
handler.MutateRuntimeTask(ctx)
if recorder.Code != http.StatusOK {
t.Fatalf("status = %d, body=%s", recorder.Code, recorder.Body.String())
}
if canceller.tenantID != 42 || canceller.knowledgeID != "knowledge-42" {
t.Fatalf("domain cancellation context mismatch: %+v", canceller)
}
}
func TestRuntimeTaskCancelPurgesOrphanWhenKnowledgeGone(t *testing.T) {
gin.SetMode(gin.TestMode)
inspector := &runtimeTaskTestInspector{tasks: []types.RuntimeTaskInfo{{
ID: "task-orphan", Queue: types.QueueMultimodal, Type: types.TypeImageMultimodal,
State: types.RuntimeTaskRetry, TenantID: 42, KnowledgeID: "knowledge-gone",
AllowedActions: []types.RuntimeTaskAction{types.RuntimeTaskActionCancel},
}}}
canceller := &runtimeKnowledgeCancelTest{err: repository.ErrKnowledgeNotFound}
handler := &SystemHandler{taskInspector: inspector, knowledgeSvc: canceller}
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Params = gin.Params{
{Key: "queue", Value: types.QueueMultimodal},
{Key: "task_id", Value: "task-orphan"},
{Key: "action", Value: string(types.RuntimeTaskActionCancel)},
}
ctx.Request = httptest.NewRequest(http.MethodPost, "/cancel", nil)
handler.MutateRuntimeTask(ctx)
if recorder.Code != http.StatusOK {
t.Fatalf("status = %d, body=%s", recorder.Code, recorder.Body.String())
}
if inspector.forceDeleted != "task-orphan" {
t.Fatalf("expected force delete, got deleted=%q force=%q cancel=%q",
inspector.deletedTask, inspector.forceDeleted, inspector.cancelKnowledge)
}
}
func TestRuntimeTaskCancelPurgesOrphanAfterSiblingSweep(t *testing.T) {
gin.SetMode(gin.TestMode)
inspector := &runtimeTaskTestInspector{
cancelDeleted: 2,
tasks: []types.RuntimeTaskInfo{{
ID: "task-orphan", Queue: types.QueueMultimodal, Type: types.TypeImageMultimodal,
State: types.RuntimeTaskArchived, TenantID: 42, KnowledgeID: "knowledge-gone",
AllowedActions: []types.RuntimeTaskAction{types.RuntimeTaskActionCancel},
}},
}
canceller := &runtimeKnowledgeCancelTest{err: repository.ErrKnowledgeNotFound}
handler := &SystemHandler{taskInspector: inspector, knowledgeSvc: canceller}
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Params = gin.Params{
{Key: "queue", Value: types.QueueMultimodal},
{Key: "task_id", Value: "task-orphan"},
{Key: "action", Value: string(types.RuntimeTaskActionCancel)},
}
ctx.Request = httptest.NewRequest(http.MethodPost, "/cancel", nil)
handler.MutateRuntimeTask(ctx)
if recorder.Code != http.StatusOK {
t.Fatalf("status = %d, body=%s", recorder.Code, recorder.Body.String())
}
if inspector.cancelKnowledge != "knowledge-gone" || inspector.forceDeleted != "task-orphan" {
t.Fatalf("expected sibling sweep then force delete, got cancel=%q force=%q",
inspector.cancelKnowledge, inspector.forceDeleted)
}
}
func TestRuntimeTaskCancelPurgesOrphanWhenSweepAlreadyRemovedTask(t *testing.T) {
gin.SetMode(gin.TestMode)
inspector := &runtimeTaskTestInspector{
cancelDeleted: 1,
forceDeleteErr: errors.New("task not found"),
getRuntimeErr: errors.New("task not found"),
getRuntimeErrFrom: 2,
tasks: []types.RuntimeTaskInfo{{
ID: "task-orphan", Queue: types.QueueMultimodal, Type: types.TypeImageMultimodal,
State: types.RuntimeTaskRetry, TenantID: 42, KnowledgeID: "knowledge-gone",
AllowedActions: []types.RuntimeTaskAction{types.RuntimeTaskActionCancel},
}},
}
canceller := &runtimeKnowledgeCancelTest{err: repository.ErrKnowledgeNotFound}
handler := &SystemHandler{taskInspector: inspector, knowledgeSvc: canceller}
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Params = gin.Params{
{Key: "queue", Value: types.QueueMultimodal},
{Key: "task_id", Value: "task-orphan"},
{Key: "action", Value: string(types.RuntimeTaskActionCancel)},
}
ctx.Request = httptest.NewRequest(http.MethodPost, "/cancel", nil)
handler.MutateRuntimeTask(ctx)
if recorder.Code != http.StatusOK {
t.Fatalf("status = %d, body=%s", recorder.Code, recorder.Body.String())
}
if inspector.forceDeleted != "task-orphan" {
t.Fatalf("expected force delete attempt, got %q", inspector.forceDeleted)
}
}