WeKnora/internal/handler/knowledge.go

2674 lines
95 KiB
Go
Raw Normal View History

2025-08-05 07:08:07 +00:00
package handler
import (
"context"
2025-08-05 07:08:07 +00:00
"encoding/json"
"fmt"
"io"
"mime"
2025-08-05 07:08:07 +00:00
"net/http"
"strconv"
"strings"
"time"
2025-08-05 07:08:07 +00:00
goerrors "errors"
"github.com/Tencent/WeKnora/internal/agent/tools"
"github.com/Tencent/WeKnora/internal/application/repository"
"github.com/Tencent/WeKnora/internal/application/service"
"github.com/Tencent/WeKnora/internal/config"
2025-08-05 07:08:07 +00:00
"github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/middleware"
feat(observability): extend Langfuse tracing across asynq pipeline Previously the Langfuse integration only traced in-process HTTP requests (chat / search / eval), so file uploads and every downstream asynq task (document parse, chunk embedding, OCR/VLM, summary / question gen, wiki ingest, datasource sync, etc.) produced either disconnected shallow traces or no observation at all. This change threads one trace end-to-end: - tracer: add SPAN observation type and StartSpan; add ResumeTrace so a worker can attach to an upstream trace without emitting a duplicate trace-create; StartGeneration now auto-picks parentObservationId from ctx so nested trace -> span -> generation trees render correctly. - types.TracingContext + LangfuseTracingCarrier: embed on all 17 asynq payloads so trace_id / parent_obs_id / user_id / session_id serialise into every job. - langfuse.InjectTracing: injected at 28 enqueue sites before json.Marshal so the HTTP-layer trace survives the Redis hop. - langfuse.AsynqMiddleware: mux.Use hook that peeks the payload, either resumes the upstream trace or opens a standalone asynq.<type> trace for scheduled jobs, and wraps the handler in a SPAN with task metadata (id / queue / retry / payload_bytes) plus ERROR level on failure. - GinMiddleware.shouldTrace: whitelist ingestion / knowledge-mutation / FAQ / wiki / datasource endpoints so the root trace actually starts. - Tests: tracer_test.go covers span nesting, error status, and ResumeTrace no-trace-create guarantee; asynq_test.go covers InjectTracing round-trip, middleware resume path, and standalone trace fallback. - Docs: docs/Langfuse\u96c6\u6210.md now lists the covered task types and documents the cross-process propagation model. No behavioural change when Langfuse is disabled (all new code paths are no-ops and carriers serialise to empty strings with omitempty).
2026-04-24 04:23:30 +00:00
"github.com/Tencent/WeKnora/internal/tracing/langfuse"
2025-08-05 07:08:07 +00:00
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/Tencent/WeKnora/internal/utils"
secutils "github.com/Tencent/WeKnora/internal/utils"
2025-08-05 07:08:07 +00:00
"github.com/gin-gonic/gin"
"github.com/hibiken/asynq"
2025-08-05 07:08:07 +00:00
)
// KnowledgeHandler processes HTTP requests related to knowledge resources
type KnowledgeHandler struct {
cfg *config.Config
kgService interfaces.KnowledgeService
kbService interfaces.KnowledgeBaseService
kbShareService interfaces.KBShareService
agentShareService interfaces.AgentShareService
asynqClient interfaces.TaskEnqueuer
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
spanRepo repository.KnowledgeSpanRepository
2025-08-05 07:08:07 +00:00
}
// NewKnowledgeHandler creates a new knowledge handler instance
func NewKnowledgeHandler(
cfg *config.Config,
2025-08-05 07:08:07 +00:00
kgService interfaces.KnowledgeService,
kbService interfaces.KnowledgeBaseService,
kbShareService interfaces.KBShareService,
agentShareService interfaces.AgentShareService,
asynqClient interfaces.TaskEnqueuer,
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
spanRepo repository.KnowledgeSpanRepository,
2025-08-05 07:08:07 +00:00
) *KnowledgeHandler {
return &KnowledgeHandler{
cfg: cfg,
kgService: kgService,
kbService: kbService,
kbShareService: kbShareService,
agentShareService: agentShareService,
asynqClient: asynqClient,
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
spanRepo: spanRepo,
}
2025-08-05 07:08:07 +00:00
}
// requireKBOwnershipOrAdmin enforces the same "KB creator OR Admin+" matrix
// used by OwnedKBOrAdmin for routes whose KB id comes from the request body.
func (h *KnowledgeHandler) requireKBOwnershipOrAdmin(c *gin.Context, kbID string) error {
creator, err := resolveKBCreatorByKBID(c, h.kbService, kbID)
evalErr := middleware.EvaluateOwnershipOrRole(
c.Request.Context(),
h.cfg,
types.TenantRoleAdmin,
creator,
err,
)
if evalErr == nil {
return nil
}
if goerrors.Is(evalErr, middleware.ErrResourceNotFound) {
return errors.NewNotFoundError("knowledge base not found")
}
if goerrors.Is(evalErr, middleware.ErrOwnershipForbidden) {
return errors.NewForbiddenError("No permission to operate on this knowledge base")
}
logger.ErrorWithFields(c.Request.Context(), evalErr, map[string]interface{}{
"kb_id": secutils.SanitizeForLog(kbID),
})
return errors.NewInternalServerError("cannot verify knowledge base ownership")
}
2025-08-05 07:08:07 +00:00
// validateKnowledgeBaseAccess validates access permissions to a knowledge base
// using the ":id" URL path parameter. It delegates to validateKnowledgeBaseAccessWithKBID.
func (h *KnowledgeHandler) validateKnowledgeBaseAccess(c *gin.Context) (*types.KnowledgeBase, string, uint64, types.OrgMemberRole, error) {
kbID := secutils.SanitizeForLog(c.Param("id"))
return h.validateKnowledgeBaseAccessWithKBID(c, kbID)
}
// validateKnowledgeBaseAccessWithKBID validates access to the given knowledge base ID (e.g. from query or body).
// Enforces per-API-key KB scope before tenant/share/agent resolution.
// Returns the knowledge base, kbID, effective tenant ID, permission, and error.
func (h *KnowledgeHandler) validateKnowledgeBaseAccessWithKBID(c *gin.Context, kbID string) (*types.KnowledgeBase, string, uint64, types.OrgMemberRole, error) {
ctx := c.Request.Context()
tenantID := c.GetUint64(types.TenantIDContextKey.String())
if tenantID == 0 {
logger.Error(ctx, "Failed to get tenant ID")
return nil, "", 0, "", errors.NewUnauthorizedError("Unauthorized")
}
userID, userExists := c.Get(types.UserIDContextKey.String())
feat(rbac): lift org membership from per-user to per-tenant (Plan 3, #1303) Before this change, the `organization_members` table keyed on user_id — joining an org as a Viewer in tenant T meant only that one user got visibility on shared KBs/agents, while the rest of T (including its Owner) saw nothing. The mismatch surfaced as confusing UX bugs: "my colleague shared this with us, why can't I see it?" This change replaces `organization_members` with `organization_tenant_members`. Membership and Org-RBAC role now key on (org_id, tenant_id); `representative_user_id` stays as a display-only label of the user who first brought the tenant in. Permission resolution gains a 3-D cap so tenant-RBAC stays honest under cross-tenant sharing: effective = min(share.Permission, tenant_org_role, tenant_role_cap) where tenant_role_cap pins tenant Viewers to OrgRoleViewer regardless of the org-level grant. A Viewer in their own tenant cannot edit a shared KB even if the org granted Editor — preserves the "Viewer = read-only everywhere" promise. Migration 000045 backfills `organization_tenant_members` from the old table by collapsing each (org, tenant) group to its max role, then renames the old table to `organization_members_pre_plan3` for safe rollback. Sqlite mirror (Lite mode) is fresh-bootstrap: the init schema replaces the old table directly. Routes: member-management URLs change from `:user_id` to `:tenant_id`. Org create / join / leave now require Admin+ in the caller's tenant — joining an Org affects everyone in T, so the decision can't sit with a Viewer. Frontend: OrganizationSettingsModal + store update to address members by tenant_id.
2026-05-15 13:49:54 +00:00
callerTenantRole := types.TenantRoleFromContext(ctx)
kbID = secutils.SanitizeForLog(kbID)
if kbID == "" {
return nil, "", 0, "", errors.NewBadRequestError("Knowledge base ID cannot be empty")
}
if err := requireTenantAPIKeyKnowledgeBase(ctx, kbID); err != nil {
return nil, kbID, 0, "", err
}
kb, err := h.kbService.GetKnowledgeBaseByID(ctx, kbID)
if err != nil {
// Same not-found-vs-real-error split as knowledgebase.go's
// validateAndGetKnowledgeBase: ErrKnowledgeBaseNotFound is the
// expected outcome for a probed/stale kb id and must surface as
// 404, not the generic 500 the original code produced.
if goerrors.Is(err, repository.ErrKnowledgeBaseNotFound) {
return nil, kbID, 0, "", errors.NewNotFoundError("knowledge base not found")
}
logger.ErrorWithFields(ctx, err, nil)
return nil, kbID, 0, "", errors.NewInternalServerError(err.Error())
}
if kb.TenantID == tenantID {
return kb, kbID, tenantID, types.OrgRoleAdmin, nil
}
feat(rbac): lift org membership from per-user to per-tenant (Plan 3, #1303) Before this change, the `organization_members` table keyed on user_id — joining an org as a Viewer in tenant T meant only that one user got visibility on shared KBs/agents, while the rest of T (including its Owner) saw nothing. The mismatch surfaced as confusing UX bugs: "my colleague shared this with us, why can't I see it?" This change replaces `organization_members` with `organization_tenant_members`. Membership and Org-RBAC role now key on (org_id, tenant_id); `representative_user_id` stays as a display-only label of the user who first brought the tenant in. Permission resolution gains a 3-D cap so tenant-RBAC stays honest under cross-tenant sharing: effective = min(share.Permission, tenant_org_role, tenant_role_cap) where tenant_role_cap pins tenant Viewers to OrgRoleViewer regardless of the org-level grant. A Viewer in their own tenant cannot edit a shared KB even if the org granted Editor — preserves the "Viewer = read-only everywhere" promise. Migration 000045 backfills `organization_tenant_members` from the old table by collapsing each (org, tenant) group to its max role, then renames the old table to `organization_members_pre_plan3` for safe rollback. Sqlite mirror (Lite mode) is fresh-bootstrap: the init schema replaces the old table directly. Routes: member-management URLs change from `:user_id` to `:tenant_id`. Org create / join / leave now require Admin+ in the caller's tenant — joining an Org affects everyone in T, so the decision can't sit with a Viewer. Frontend: OrganizationSettingsModal + store update to address members by tenant_id.
2026-05-15 13:49:54 +00:00
if h.kbShareService != nil {
permission, isShared, permErr := h.kbShareService.CheckTenantKBPermission(ctx, kbID, tenantID, callerTenantRole)
if permErr == nil && isShared {
sourceTenantID, srcErr := h.kbShareService.GetKBSourceTenant(ctx, kbID)
if srcErr == nil {
feat(rbac): lift org membership from per-user to per-tenant (Plan 3, #1303) Before this change, the `organization_members` table keyed on user_id — joining an org as a Viewer in tenant T meant only that one user got visibility on shared KBs/agents, while the rest of T (including its Owner) saw nothing. The mismatch surfaced as confusing UX bugs: "my colleague shared this with us, why can't I see it?" This change replaces `organization_members` with `organization_tenant_members`. Membership and Org-RBAC role now key on (org_id, tenant_id); `representative_user_id` stays as a display-only label of the user who first brought the tenant in. Permission resolution gains a 3-D cap so tenant-RBAC stays honest under cross-tenant sharing: effective = min(share.Permission, tenant_org_role, tenant_role_cap) where tenant_role_cap pins tenant Viewers to OrgRoleViewer regardless of the org-level grant. A Viewer in their own tenant cannot edit a shared KB even if the org granted Editor — preserves the "Viewer = read-only everywhere" promise. Migration 000045 backfills `organization_tenant_members` from the old table by collapsing each (org, tenant) group to its max role, then renames the old table to `organization_members_pre_plan3` for safe rollback. Sqlite mirror (Lite mode) is fresh-bootstrap: the init schema replaces the old table directly. Routes: member-management URLs change from `:user_id` to `:tenant_id`. Org create / join / leave now require Admin+ in the caller's tenant — joining an Org affects everyone in T, so the decision can't sit with a Viewer. Frontend: OrganizationSettingsModal + store update to address members by tenant_id.
2026-05-15 13:49:54 +00:00
logger.Infof(ctx, "Tenant %d accessing shared KB %s with permission %s, source tenant: %d",
tenantID, kbID, permission, sourceTenantID)
return kb, kbID, sourceTenantID, permission, nil
}
}
2025-08-05 07:08:07 +00:00
}
feat(rbac): lift org membership from per-user to per-tenant (Plan 3, #1303) Before this change, the `organization_members` table keyed on user_id — joining an org as a Viewer in tenant T meant only that one user got visibility on shared KBs/agents, while the rest of T (including its Owner) saw nothing. The mismatch surfaced as confusing UX bugs: "my colleague shared this with us, why can't I see it?" This change replaces `organization_members` with `organization_tenant_members`. Membership and Org-RBAC role now key on (org_id, tenant_id); `representative_user_id` stays as a display-only label of the user who first brought the tenant in. Permission resolution gains a 3-D cap so tenant-RBAC stays honest under cross-tenant sharing: effective = min(share.Permission, tenant_org_role, tenant_role_cap) where tenant_role_cap pins tenant Viewers to OrgRoleViewer regardless of the org-level grant. A Viewer in their own tenant cannot edit a shared KB even if the org granted Editor — preserves the "Viewer = read-only everywhere" promise. Migration 000045 backfills `organization_tenant_members` from the old table by collapsing each (org, tenant) group to its max role, then renames the old table to `organization_members_pre_plan3` for safe rollback. Sqlite mirror (Lite mode) is fresh-bootstrap: the init schema replaces the old table directly. Routes: member-management URLs change from `:user_id` to `:tenant_id`. Org create / join / leave now require Admin+ in the caller's tenant — joining an Org affects everyone in T, so the decision can't sit with a Viewer. Frontend: OrganizationSettingsModal + store update to address members by tenant_id.
2026-05-15 13:49:54 +00:00
if h.agentShareService != nil {
can, err := h.agentShareService.TenantCanAccessKBViaSomeSharedAgent(ctx, tenantID, callerTenantRole, kb)
if err == nil && can {
feat(rbac): lift org membership from per-user to per-tenant (Plan 3, #1303) Before this change, the `organization_members` table keyed on user_id — joining an org as a Viewer in tenant T meant only that one user got visibility on shared KBs/agents, while the rest of T (including its Owner) saw nothing. The mismatch surfaced as confusing UX bugs: "my colleague shared this with us, why can't I see it?" This change replaces `organization_members` with `organization_tenant_members`. Membership and Org-RBAC role now key on (org_id, tenant_id); `representative_user_id` stays as a display-only label of the user who first brought the tenant in. Permission resolution gains a 3-D cap so tenant-RBAC stays honest under cross-tenant sharing: effective = min(share.Permission, tenant_org_role, tenant_role_cap) where tenant_role_cap pins tenant Viewers to OrgRoleViewer regardless of the org-level grant. A Viewer in their own tenant cannot edit a shared KB even if the org granted Editor — preserves the "Viewer = read-only everywhere" promise. Migration 000045 backfills `organization_tenant_members` from the old table by collapsing each (org, tenant) group to its max role, then renames the old table to `organization_members_pre_plan3` for safe rollback. Sqlite mirror (Lite mode) is fresh-bootstrap: the init schema replaces the old table directly. Routes: member-management URLs change from `:user_id` to `:tenant_id`. Org create / join / leave now require Admin+ in the caller's tenant — joining an Org affects everyone in T, so the decision can't sit with a Viewer. Frontend: OrganizationSettingsModal + store update to address members by tenant_id.
2026-05-15 13:49:54 +00:00
logger.Infof(ctx, "Tenant %d accessing KB %s via some shared agent", tenantID, kbID)
return kb, kbID, kb.TenantID, types.OrgRoleViewer, nil
}
}
feat(rbac): lift org membership from per-user to per-tenant (Plan 3, #1303) Before this change, the `organization_members` table keyed on user_id — joining an org as a Viewer in tenant T meant only that one user got visibility on shared KBs/agents, while the rest of T (including its Owner) saw nothing. The mismatch surfaced as confusing UX bugs: "my colleague shared this with us, why can't I see it?" This change replaces `organization_members` with `organization_tenant_members`. Membership and Org-RBAC role now key on (org_id, tenant_id); `representative_user_id` stays as a display-only label of the user who first brought the tenant in. Permission resolution gains a 3-D cap so tenant-RBAC stays honest under cross-tenant sharing: effective = min(share.Permission, tenant_org_role, tenant_role_cap) where tenant_role_cap pins tenant Viewers to OrgRoleViewer regardless of the org-level grant. A Viewer in their own tenant cannot edit a shared KB even if the org granted Editor — preserves the "Viewer = read-only everywhere" promise. Migration 000045 backfills `organization_tenant_members` from the old table by collapsing each (org, tenant) group to its max role, then renames the old table to `organization_members_pre_plan3` for safe rollback. Sqlite mirror (Lite mode) is fresh-bootstrap: the init schema replaces the old table directly. Routes: member-management URLs change from `:user_id` to `:tenant_id`. Org create / join / leave now require Admin+ in the caller's tenant — joining an Org affects everyone in T, so the decision can't sit with a Viewer. Frontend: OrganizationSettingsModal + store update to address members by tenant_id.
2026-05-15 13:49:54 +00:00
_ = userID
_ = userExists
logger.Warnf(ctx, "Permission denied to access KB %s, tenant ID: %d, KB tenant: %d", kbID, tenantID, kb.TenantID)
return nil, kbID, 0, "", errors.NewForbiddenError("Permission denied to access this knowledge base")
}
// resolveKnowledgeAndValidateKBAccess resolves knowledge by ID and validates KB access (owner or shared with required permission).
// Returns the knowledge, context with effectiveTenantID set for downstream service calls, and error.
func (h *KnowledgeHandler) resolveKnowledgeAndValidateKBAccess(c *gin.Context, knowledgeID string, requiredPermission types.OrgMemberRole) (*types.Knowledge, context.Context, error) {
ctx := c.Request.Context()
tenantID := c.GetUint64(types.TenantIDContextKey.String())
if tenantID == 0 {
return nil, ctx, errors.NewUnauthorizedError("Unauthorized")
}
userID, userExists := c.Get(types.UserIDContextKey.String())
feat(rbac): lift org membership from per-user to per-tenant (Plan 3, #1303) Before this change, the `organization_members` table keyed on user_id — joining an org as a Viewer in tenant T meant only that one user got visibility on shared KBs/agents, while the rest of T (including its Owner) saw nothing. The mismatch surfaced as confusing UX bugs: "my colleague shared this with us, why can't I see it?" This change replaces `organization_members` with `organization_tenant_members`. Membership and Org-RBAC role now key on (org_id, tenant_id); `representative_user_id` stays as a display-only label of the user who first brought the tenant in. Permission resolution gains a 3-D cap so tenant-RBAC stays honest under cross-tenant sharing: effective = min(share.Permission, tenant_org_role, tenant_role_cap) where tenant_role_cap pins tenant Viewers to OrgRoleViewer regardless of the org-level grant. A Viewer in their own tenant cannot edit a shared KB even if the org granted Editor — preserves the "Viewer = read-only everywhere" promise. Migration 000045 backfills `organization_tenant_members` from the old table by collapsing each (org, tenant) group to its max role, then renames the old table to `organization_members_pre_plan3` for safe rollback. Sqlite mirror (Lite mode) is fresh-bootstrap: the init schema replaces the old table directly. Routes: member-management URLs change from `:user_id` to `:tenant_id`. Org create / join / leave now require Admin+ in the caller's tenant — joining an Org affects everyone in T, so the decision can't sit with a Viewer. Frontend: OrganizationSettingsModal + store update to address members by tenant_id.
2026-05-15 13:49:54 +00:00
callerTenantRole := types.TenantRoleFromContext(ctx)
2025-08-05 07:08:07 +00:00
knowledge, err := h.kgService.GetKnowledgeByIDOnly(ctx, knowledgeID)
if err != nil {
return nil, ctx, errors.NewNotFoundError("Knowledge not found")
}
if err := requireTenantAPIKeyKnowledgeBase(ctx, knowledge.KnowledgeBaseID); err != nil {
return nil, ctx, err
}
// Owner: knowledge belongs to caller's tenant
if knowledge.TenantID == tenantID {
return knowledge, context.WithValue(ctx, types.TenantIDContextKey, tenantID), nil
2025-08-05 07:08:07 +00:00
}
// Shared KB: check organization permission
feat(rbac): lift org membership from per-user to per-tenant (Plan 3, #1303) Before this change, the `organization_members` table keyed on user_id — joining an org as a Viewer in tenant T meant only that one user got visibility on shared KBs/agents, while the rest of T (including its Owner) saw nothing. The mismatch surfaced as confusing UX bugs: "my colleague shared this with us, why can't I see it?" This change replaces `organization_members` with `organization_tenant_members`. Membership and Org-RBAC role now key on (org_id, tenant_id); `representative_user_id` stays as a display-only label of the user who first brought the tenant in. Permission resolution gains a 3-D cap so tenant-RBAC stays honest under cross-tenant sharing: effective = min(share.Permission, tenant_org_role, tenant_role_cap) where tenant_role_cap pins tenant Viewers to OrgRoleViewer regardless of the org-level grant. A Viewer in their own tenant cannot edit a shared KB even if the org granted Editor — preserves the "Viewer = read-only everywhere" promise. Migration 000045 backfills `organization_tenant_members` from the old table by collapsing each (org, tenant) group to its max role, then renames the old table to `organization_members_pre_plan3` for safe rollback. Sqlite mirror (Lite mode) is fresh-bootstrap: the init schema replaces the old table directly. Routes: member-management URLs change from `:user_id` to `:tenant_id`. Org create / join / leave now require Admin+ in the caller's tenant — joining an Org affects everyone in T, so the decision can't sit with a Viewer. Frontend: OrganizationSettingsModal + store update to address members by tenant_id.
2026-05-15 13:49:54 +00:00
if h.kbShareService != nil {
permission, isShared, permErr := h.kbShareService.CheckTenantKBPermission(ctx, knowledge.KnowledgeBaseID, tenantID, callerTenantRole)
if permErr == nil && isShared && permission.HasPermission(requiredPermission) {
effectiveTenantID := knowledge.TenantID
return knowledge, context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID), nil
}
}
// Shared agent: request passes agent_id, or user has any shared agent that can access this KB
feat(rbac): lift org membership from per-user to per-tenant (Plan 3, #1303) Before this change, the `organization_members` table keyed on user_id — joining an org as a Viewer in tenant T meant only that one user got visibility on shared KBs/agents, while the rest of T (including its Owner) saw nothing. The mismatch surfaced as confusing UX bugs: "my colleague shared this with us, why can't I see it?" This change replaces `organization_members` with `organization_tenant_members`. Membership and Org-RBAC role now key on (org_id, tenant_id); `representative_user_id` stays as a display-only label of the user who first brought the tenant in. Permission resolution gains a 3-D cap so tenant-RBAC stays honest under cross-tenant sharing: effective = min(share.Permission, tenant_org_role, tenant_role_cap) where tenant_role_cap pins tenant Viewers to OrgRoleViewer regardless of the org-level grant. A Viewer in their own tenant cannot edit a shared KB even if the org granted Editor — preserves the "Viewer = read-only everywhere" promise. Migration 000045 backfills `organization_tenant_members` from the old table by collapsing each (org, tenant) group to its max role, then renames the old table to `organization_members_pre_plan3` for safe rollback. Sqlite mirror (Lite mode) is fresh-bootstrap: the init schema replaces the old table directly. Routes: member-management URLs change from `:user_id` to `:tenant_id`. Org create / join / leave now require Admin+ in the caller's tenant — joining an Org affects everyone in T, so the decision can't sit with a Viewer. Frontend: OrganizationSettingsModal + store update to address members by tenant_id.
2026-05-15 13:49:54 +00:00
if h.agentShareService != nil && requiredPermission == types.OrgRoleViewer {
agentID := c.Query("agent_id")
if agentID != "" {
sourceTenantID, parseErr := types.ParseAgentSourceTenantID(c.Query(types.AgentSourceTenantIDParam))
if parseErr != nil {
return nil, ctx, errors.NewBadRequestError(parseErr.Error())
}
agent, err := h.agentShareService.GetSharedAgentForTenant(ctx, tenantID, callerTenantRole, agentID, sourceTenantID)
if err == nil && agent != nil {
if knowledge.TenantID != agent.TenantID {
return nil, ctx, errors.NewForbiddenError("Permission denied to access this knowledge")
}
mode := agent.Config.KBSelectionMode
if mode == "none" {
return nil, ctx, errors.NewForbiddenError("Permission denied to access this knowledge")
}
if mode == "all" {
return knowledge, context.WithValue(ctx, types.TenantIDContextKey, knowledge.TenantID), nil
}
if mode == "selected" {
for _, kbID := range agent.Config.KnowledgeBases {
if kbID == knowledge.KnowledgeBaseID {
return knowledge, context.WithValue(ctx, types.TenantIDContextKey, knowledge.TenantID), nil
}
}
return nil, ctx, errors.NewForbiddenError("Permission denied to access this knowledge")
}
}
} else {
kbRef := &types.KnowledgeBase{ID: knowledge.KnowledgeBaseID, TenantID: knowledge.TenantID}
feat(rbac): lift org membership from per-user to per-tenant (Plan 3, #1303) Before this change, the `organization_members` table keyed on user_id — joining an org as a Viewer in tenant T meant only that one user got visibility on shared KBs/agents, while the rest of T (including its Owner) saw nothing. The mismatch surfaced as confusing UX bugs: "my colleague shared this with us, why can't I see it?" This change replaces `organization_members` with `organization_tenant_members`. Membership and Org-RBAC role now key on (org_id, tenant_id); `representative_user_id` stays as a display-only label of the user who first brought the tenant in. Permission resolution gains a 3-D cap so tenant-RBAC stays honest under cross-tenant sharing: effective = min(share.Permission, tenant_org_role, tenant_role_cap) where tenant_role_cap pins tenant Viewers to OrgRoleViewer regardless of the org-level grant. A Viewer in their own tenant cannot edit a shared KB even if the org granted Editor — preserves the "Viewer = read-only everywhere" promise. Migration 000045 backfills `organization_tenant_members` from the old table by collapsing each (org, tenant) group to its max role, then renames the old table to `organization_members_pre_plan3` for safe rollback. Sqlite mirror (Lite mode) is fresh-bootstrap: the init schema replaces the old table directly. Routes: member-management URLs change from `:user_id` to `:tenant_id`. Org create / join / leave now require Admin+ in the caller's tenant — joining an Org affects everyone in T, so the decision can't sit with a Viewer. Frontend: OrganizationSettingsModal + store update to address members by tenant_id.
2026-05-15 13:49:54 +00:00
can, err := h.agentShareService.TenantCanAccessKBViaSomeSharedAgent(ctx, tenantID, callerTenantRole, kbRef)
if err == nil && can {
return knowledge, context.WithValue(ctx, types.TenantIDContextKey, knowledge.TenantID), nil
}
}
}
feat(rbac): lift org membership from per-user to per-tenant (Plan 3, #1303) Before this change, the `organization_members` table keyed on user_id — joining an org as a Viewer in tenant T meant only that one user got visibility on shared KBs/agents, while the rest of T (including its Owner) saw nothing. The mismatch surfaced as confusing UX bugs: "my colleague shared this with us, why can't I see it?" This change replaces `organization_members` with `organization_tenant_members`. Membership and Org-RBAC role now key on (org_id, tenant_id); `representative_user_id` stays as a display-only label of the user who first brought the tenant in. Permission resolution gains a 3-D cap so tenant-RBAC stays honest under cross-tenant sharing: effective = min(share.Permission, tenant_org_role, tenant_role_cap) where tenant_role_cap pins tenant Viewers to OrgRoleViewer regardless of the org-level grant. A Viewer in their own tenant cannot edit a shared KB even if the org granted Editor — preserves the "Viewer = read-only everywhere" promise. Migration 000045 backfills `organization_tenant_members` from the old table by collapsing each (org, tenant) group to its max role, then renames the old table to `organization_members_pre_plan3` for safe rollback. Sqlite mirror (Lite mode) is fresh-bootstrap: the init schema replaces the old table directly. Routes: member-management URLs change from `:user_id` to `:tenant_id`. Org create / join / leave now require Admin+ in the caller's tenant — joining an Org affects everyone in T, so the decision can't sit with a Viewer. Frontend: OrganizationSettingsModal + store update to address members by tenant_id.
2026-05-15 13:49:54 +00:00
_ = userID
_ = userExists
return nil, ctx, errors.NewForbiddenError("Permission denied to access this knowledge")
2025-08-05 07:08:07 +00:00
}
// handleDuplicateKnowledgeError handles cases where duplicate knowledge is detected
// Returns true if the error was a duplicate error and was handled, false otherwise
func (h *KnowledgeHandler) handleDuplicateKnowledgeError(c *gin.Context,
err error, knowledge *types.Knowledge, duplicateType string,
) bool {
if dupErr, ok := err.(*types.DuplicateKnowledgeError); ok {
ctx := c.Request.Context()
logger.Warnf(ctx, "Detected duplicate %s: %s", duplicateType, secutils.SanitizeForLog(dupErr.Error()))
2025-08-05 07:08:07 +00:00
c.JSON(http.StatusConflict, gin.H{
"success": false,
"message": dupErr.Error(),
"data": knowledge, // knowledge contains the existing document
"code": fmt.Sprintf("duplicate_%s", duplicateType),
})
return true
}
return false
}
// enqueueKnowledgeListDelete enqueues an async batch-delete task for the
// given knowledge IDs and returns the asynq task ID.
func (h *KnowledgeHandler) enqueueKnowledgeListDelete(
ctx context.Context, tenantID uint64, ids []string,
) (string, error) {
payload := types.KnowledgeListDeletePayload{
TenantID: tenantID,
KnowledgeIDs: ids,
Initiator: types.TaskInitiatorFromContext(ctx),
}
langfuse.InjectTracing(ctx, &payload)
payloadBytes, err := json.Marshal(payload)
if err != nil {
return "", fmt.Errorf("marshal payload: %w", err)
}
task := asynq.NewTask(types.TypeKnowledgeListDelete, payloadBytes,
asynq.Queue(types.QueueMaintenance), asynq.MaxRetry(3), asynq.Timeout(2*time.Hour))
info, err := h.asynqClient.Enqueue(task)
if err != nil {
return "", fmt.Errorf("enqueue task: %w", err)
}
return info.ID, nil
}
// enqueueKnowledgeListReparse enqueues an async batch-reparse task for the
// given knowledge IDs and returns the asynq task ID.
func (h *KnowledgeHandler) enqueueKnowledgeListReparse(
ctx context.Context, tenantID uint64, ids []string, processConfig *types.KnowledgeProcessOverrides,
) (string, error) {
payload := types.KnowledgeListReparsePayload{
TenantID: tenantID,
KnowledgeIDs: ids,
ProcessConfig: processConfig,
Initiator: types.TaskInitiatorFromContext(ctx),
}
langfuse.InjectTracing(ctx, &payload)
payloadBytes, err := json.Marshal(payload)
if err != nil {
return "", fmt.Errorf("marshal payload: %w", err)
}
task := asynq.NewTask(types.TypeKnowledgeListReparse, payloadBytes,
asynq.Queue(types.QueueMaintenance), asynq.MaxRetry(3), asynq.Timeout(time.Hour))
info, err := h.asynqClient.Enqueue(task)
if err != nil {
return "", fmt.Errorf("enqueue task: %w", err)
}
return info.ID, nil
}
// CreateKnowledgeFromFile godoc
// @Summary 从文件创建知识
// @Description 上传文件并创建知识条目
// @Tags 知识管理
// @Accept multipart/form-data
// @Produce json
// @Param id path string true "知识库ID"
// @Param file formData file true "上传的文件"
// @Param fileName formData string false "自定义文件名"
// @Param metadata formData string false "元数据JSON"
// @Param enable_multimodel formData bool false "启用多模态处理"
// @Param tag_ids formData string false "分类ID列表,逗号分隔"
// @Param process_config formData string false "处理配置JSON(KnowledgeProcessOverrides)"
// @Success 200 {object} map[string]interface{} "创建的知识"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Failure 409 {object} map[string]interface{} "文件重复"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge-bases/{id}/knowledge/file [post]
2025-08-05 07:08:07 +00:00
func (h *KnowledgeHandler) CreateKnowledgeFromFile(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start creating knowledge from file")
// Validate access to the knowledge base (only owner or admin/editor can create)
_, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseAccess(c)
2025-08-05 07:08:07 +00:00
if err != nil {
c.Error(err)
return
}
ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID)
// Check write permission
if permission != types.OrgRoleAdmin && permission != types.OrgRoleEditor {
c.Error(errors.NewForbiddenError("No permission to create knowledge"))
return
}
2025-08-05 07:08:07 +00:00
// Get the uploaded file
file, err := c.FormFile("file")
if err != nil {
logger.Error(ctx, "File upload failed", err)
c.Error(errors.NewBadRequestError("File upload failed").WithDetails(err.Error()))
return
}
feat(system): consolidate system admin and settings into one Settings panel Replace the standalone /platform/system/* routes with a single "系统设置" section inside the canonical Settings modal. The previous SystemLayout.vue / SystemAdmins.vue surfaces are removed and their functionality (system admin roster, global settings) is hosted directly in SystemSettings.vue under the standard `.section-header` / `.settings-group` skeleton. Legacy URLs redirect to the modal section so external bookmarks don't 404. Backend: - SystemSettingService.Reset + repo.Delete: drop the DB override for a key so the 3-tier resolver falls back to ENV / built-in default. Idempotent (resetting a never-persisted key returns nil); emits an audit row only on real deletions, invalidates the local cache, and publishes to peers via the existing pubsub channel. - TenantService.BulkSetStorageQuota: overwrite every tenant's storage_quota in one statement. Powers POST /system/admin/tenants/ apply-default-storage-quota; bypasses the per-tenant whitelist on PUT /tenants/:id which intentionally forbids storage_quota edits. - AuditAction{SystemAdminPromoted,SystemAdminRevoked} constants and emitAdminAudit() in SystemHandler — promote / revoke now leave an audit trail with TenantID=0 and {target_email,target_username, idempotent|changed} in details. - SystemSetting.LastModifiedByName: derived per-request display label (username, email fallback) so the UI shows "wizardchen" instead of a UUID prefix without storing a denormalised column. Frontend: - SystemSettings.vue rewritten against the Settings modal skeleton with auto-persisting controls (switch/select @change, input/inputnumber @blur, tag-input + per-delta popconfirm for SSRF whitelist and admin roster). auth.registration_mode change goes through an inline popconfirm; cancel rolls back. Reset / bulk-apply also inline popconfirms — no dialog modal for these per-row affordances. - Priority hint panel surfaces the DB > ENV > default resolver order so operators can reason about "I set the env but it doesn't show up". - Router: /platform/system, /platform/system/settings, /platform/system/ admins are now compatibility redirects to /platform/settings?section= system-global. - Settings modal sized 900x700 → 1080x780 and content-wrapper 600 → 760 so the wider tables (members, system settings) breathe; <1100px viewport still flexes to the screen. - i18n: system.globalSettings.* keys for title / description / loading / empty / badges / priority hint / per-key labels / reset / bulkApply / admins (label, placeholder, save messages, popconfirm copy) across zh-CN, en-US, ko-KR, ru-RU. Misc: - internal/utils/filesize.go doc: clarify MAX_FILE_SIZE_MB is a deploy-time-only knob (nginx + docreader + frontend each cache the env at startup); a SystemAdmin UI override would mislead operators because nginx would still 413. Until all four layers can hot-reload the limit in lockstep, this stays env-only. - internal/utils/security.go: SSRF whitelist parser/runtime now drives off SystemSettingService for live updates; ENV remains the fallback for never-overridden deployments.
2026-05-26 12:03:35 +00:00
// Validate file size — read MAX_FILE_SIZE_MB env (50MB default).
// Deliberately not a runtime system_setting; see filesize.go for the
// rationale (nginx / docreader / browser bundle all cache this at
// container startup, so a UI knob would silently mismatch).
maxSizeMB := utils.GetMaxFileSizeMB()
maxSize := maxSizeMB * 1024 * 1024
if file.Size > maxSize {
logger.Error(ctx, "File size too large")
c.Error(errors.NewBadRequestError(fmt.Sprintf("文件大小不能超过%dMB", maxSizeMB)))
return
}
// Get custom filename if provided (for folder uploads with path)
customFileName := c.PostForm("fileName")
customFileName = secutils.SanitizeForLog(customFileName)
displayFileName := file.Filename
displayFileName = secutils.SanitizeForLog(displayFileName)
if customFileName != "" {
displayFileName = customFileName
logger.Infof(ctx, "Using custom filename: %s (original: %s)", customFileName, displayFileName)
}
logger.Infof(ctx, "File upload successful, filename: %s, size: %.2f KB", displayFileName, float64(file.Size)/1024)
logger.Infof(ctx, "Creating knowledge, knowledge base ID: %s, filename: %s", kbID, displayFileName)
2025-08-05 07:08:07 +00:00
// Parse metadata if provided
var metadata map[string]string
metadataStr := c.PostForm("metadata")
if metadataStr != "" {
if err := json.Unmarshal([]byte(metadataStr), &metadata); err != nil {
logger.Error(ctx, "Failed to parse metadata", err)
c.Error(errors.NewBadRequestError("Invalid metadata format").WithDetails(err.Error()))
return
}
logger.Infof(ctx, "Received file metadata: %s", secutils.SanitizeForLog(fmt.Sprintf("%v", metadata)))
2025-08-05 07:08:07 +00:00
}
enableMultimodelForm := c.PostForm("enable_multimodel")
var enableMultimodel *bool
if enableMultimodelForm != "" {
parseBool, err := strconv.ParseBool(enableMultimodelForm)
if err != nil {
logger.Error(ctx, "Failed to parse enable_multimodel", err)
c.Error(errors.NewBadRequestError("Invalid enable_multimodel format").WithDetails(err.Error()))
return
}
enableMultimodel = &parseBool
}
var processOverrides *types.KnowledgeProcessOverrides
if raw := c.PostForm("process_config"); raw != "" {
processOverrides = &types.KnowledgeProcessOverrides{}
if err := json.Unmarshal([]byte(raw), processOverrides); err != nil {
logger.Error(ctx, "Failed to parse process_config", err)
c.Error(errors.NewBadRequestError("Invalid process_config format").WithDetails(err.Error()))
return
}
}
if enableMultimodel != nil && (processOverrides == nil || processOverrides.EnableMultimodel == nil) {
if processOverrides == nil {
processOverrides = &types.KnowledgeProcessOverrides{EnableMultimodel: enableMultimodel}
} else {
processOverrides.EnableMultimodel = enableMultimodel
}
}
feat: support multi-tag for document knowledge (#1790) * feat: support multi-tag for document knowledge - Add knowledge_tag_relations join table for many-to-many tag-knowledge mapping - Migrate existing single tag_id data to the new table - Update Knowledge struct, filter, payload types (TagID -> Tags/TagIDs) - Rewrite repository layer: SetKnowledgeTags, GetKnowledgeTags, ListIDsByTagIDs - Update tag count references to query knowledge_tag_relations - Update service/handler layers for multi-tag create/list/batch-update - Frontend: multi-select tag sidebar, multi-tag chip display, tag toggle in popup - FAQ tags remain single-tag model (unchanged) * feat(knowledge): tag edit dialog with dynamic overflow and inline creation - Add TagEditDialog component with search, multi-select, and confirm - Replace inline t-popup tag selector with dialog-based editing - Tags overflow with +N badge, visible count adapts to container width via ResizeObserver - Date/card layout: fixed date width, tags flex to fill remaining space - Add persistent new-tag input bar in dialog (Enter to create, auto-select) - Tag creation emits callback to refresh parent tag list immediately - Apply same overflow + dialog trigger to both card view and list view - Add i18n keys: tagEditDialogTitle, tagEditSearch, tagSelectedCount, tagOverflowTip, tagNewPlaceholder * fix: exclude soft-deleted knowledge from tag reference counts CountReferences, BatchCountReferences, and DeleteUnusedTags now JOIN with knowledges table to filter out soft-deleted records when counting knowledge_tag_relations. Previously they queried knowledge_tag_relations directly without the deleted_at check, causing stale tag counts after knowledge deletion. * fix: refresh tag counts after async knowledge deletion completes Add a second delayed tag list refresh (2.5s) after tag deletion to capture updated reference counts for remaining tags. The original immediate refresh handles removing the deleted tag from the list, while this delayed refresh ensures other tags' counts are correct after the async deletion finishes. --------- Co-authored-by: lyingbug <xiayula2014@gmail.com>
2026-06-25 16:18:48 +00:00
// 获取分类ID列表(如果提供),逗号分隔,用于知识多标签分类管理
tagIDs := parseCommaSeparatedTagIDs(c.PostForm("tag_ids"))
channel := c.PostForm("channel")
2025-08-05 07:08:07 +00:00
// Create knowledge entry from the file
feat: support multi-tag for document knowledge (#1790) * feat: support multi-tag for document knowledge - Add knowledge_tag_relations join table for many-to-many tag-knowledge mapping - Migrate existing single tag_id data to the new table - Update Knowledge struct, filter, payload types (TagID -> Tags/TagIDs) - Rewrite repository layer: SetKnowledgeTags, GetKnowledgeTags, ListIDsByTagIDs - Update tag count references to query knowledge_tag_relations - Update service/handler layers for multi-tag create/list/batch-update - Frontend: multi-select tag sidebar, multi-tag chip display, tag toggle in popup - FAQ tags remain single-tag model (unchanged) * feat(knowledge): tag edit dialog with dynamic overflow and inline creation - Add TagEditDialog component with search, multi-select, and confirm - Replace inline t-popup tag selector with dialog-based editing - Tags overflow with +N badge, visible count adapts to container width via ResizeObserver - Date/card layout: fixed date width, tags flex to fill remaining space - Add persistent new-tag input bar in dialog (Enter to create, auto-select) - Tag creation emits callback to refresh parent tag list immediately - Apply same overflow + dialog trigger to both card view and list view - Add i18n keys: tagEditDialogTitle, tagEditSearch, tagSelectedCount, tagOverflowTip, tagNewPlaceholder * fix: exclude soft-deleted knowledge from tag reference counts CountReferences, BatchCountReferences, and DeleteUnusedTags now JOIN with knowledges table to filter out soft-deleted records when counting knowledge_tag_relations. Previously they queried knowledge_tag_relations directly without the deleted_at check, causing stale tag counts after knowledge deletion. * fix: refresh tag counts after async knowledge deletion completes Add a second delayed tag list refresh (2.5s) after tag deletion to capture updated reference counts for remaining tags. The original immediate refresh handles removing the deleted tag from the list, while this delayed refresh ensures other tags' counts are correct after the async deletion finishes. --------- Co-authored-by: lyingbug <xiayula2014@gmail.com>
2026-06-25 16:18:48 +00:00
knowledge, err := h.kgService.CreateKnowledgeFromFile(ctx, kbID, file, metadata, enableMultimodel, customFileName, tagIDs, channel, processOverrides)
2025-08-05 07:08:07 +00:00
// Check for duplicate knowledge error
if err != nil {
if h.handleDuplicateKnowledgeError(c, err, knowledge, "file") {
return
}
if appErr, ok := errors.IsAppError(err); ok {
c.Error(appErr)
return
}
2025-08-05 07:08:07 +00:00
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError(err.Error()))
return
}
logger.Infof(
ctx,
"Knowledge created successfully, ID: %s, title: %s",
secutils.SanitizeForLog(knowledge.ID),
secutils.SanitizeForLog(knowledge.Title),
)
2025-08-05 07:08:07 +00:00
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": knowledge,
})
}
// CreateKnowledgeFromURL godoc
// @Summary 从URL创建知识
// @Description 从指定URL抓取内容并创建知识条目。当提供 file_name/file_type 或 URL 路径含已知文件扩展名时,自动切换为文件下载模式
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param id path string true "知识库ID"
// @Param request body object{url=string,file_name=string,file_type=string,enable_multimodel=bool,title=string,tag_ids=[]string} true "URL请求"
// @Success 201 {object} map[string]interface{} "创建的知识"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Failure 409 {object} map[string]interface{} "URL重复"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge-bases/{id}/knowledge/url [post]
2025-08-05 07:08:07 +00:00
func (h *KnowledgeHandler) CreateKnowledgeFromURL(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start creating knowledge from URL")
// Validate access to the knowledge base (only owner or admin/editor can create)
_, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseAccess(c)
2025-08-05 07:08:07 +00:00
if err != nil {
c.Error(err)
return
}
ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID)
// Check write permission
if permission != types.OrgRoleAdmin && permission != types.OrgRoleEditor {
c.Error(errors.NewForbiddenError("No permission to create knowledge"))
return
}
2025-08-05 07:08:07 +00:00
// Parse URL from request body
var req struct {
URL string `json:"url" binding:"required"`
FileName string `json:"file_name"`
FileType string `json:"file_type"`
EnableMultimodel *bool `json:"enable_multimodel"`
Title string `json:"title"`
feat: support multi-tag for document knowledge (#1790) * feat: support multi-tag for document knowledge - Add knowledge_tag_relations join table for many-to-many tag-knowledge mapping - Migrate existing single tag_id data to the new table - Update Knowledge struct, filter, payload types (TagID -> Tags/TagIDs) - Rewrite repository layer: SetKnowledgeTags, GetKnowledgeTags, ListIDsByTagIDs - Update tag count references to query knowledge_tag_relations - Update service/handler layers for multi-tag create/list/batch-update - Frontend: multi-select tag sidebar, multi-tag chip display, tag toggle in popup - FAQ tags remain single-tag model (unchanged) * feat(knowledge): tag edit dialog with dynamic overflow and inline creation - Add TagEditDialog component with search, multi-select, and confirm - Replace inline t-popup tag selector with dialog-based editing - Tags overflow with +N badge, visible count adapts to container width via ResizeObserver - Date/card layout: fixed date width, tags flex to fill remaining space - Add persistent new-tag input bar in dialog (Enter to create, auto-select) - Tag creation emits callback to refresh parent tag list immediately - Apply same overflow + dialog trigger to both card view and list view - Add i18n keys: tagEditDialogTitle, tagEditSearch, tagSelectedCount, tagOverflowTip, tagNewPlaceholder * fix: exclude soft-deleted knowledge from tag reference counts CountReferences, BatchCountReferences, and DeleteUnusedTags now JOIN with knowledges table to filter out soft-deleted records when counting knowledge_tag_relations. Previously they queried knowledge_tag_relations directly without the deleted_at check, causing stale tag counts after knowledge deletion. * fix: refresh tag counts after async knowledge deletion completes Add a second delayed tag list refresh (2.5s) after tag deletion to capture updated reference counts for remaining tags. The original immediate refresh handles removing the deleted tag from the list, while this delayed refresh ensures other tags' counts are correct after the async deletion finishes. --------- Co-authored-by: lyingbug <xiayula2014@gmail.com>
2026-06-25 16:18:48 +00:00
TagIDs []string `json:"tag_ids"`
Channel string `json:"channel"`
ProcessConfig *types.KnowledgeProcessOverrides `json:"process_config"`
2025-08-05 07:08:07 +00:00
}
if err := c.ShouldBindJSON(&req); err != nil {
logger.Error(ctx, "Failed to parse URL request", err)
c.Error(errors.NewBadRequestError(err.Error()))
return
}
logger.Infof(ctx, "Received URL request: %s, file_name: %s, file_type: %s",
secutils.SanitizeForLog(req.URL),
secutils.SanitizeForLog(req.FileName),
secutils.SanitizeForLog(req.FileType),
)
// SSRF validation for user-supplied URL
if err := secutils.ValidateURLForSSRF(req.URL); err != nil {
logger.Warnf(ctx, "SSRF validation failed for knowledge URL: %v", err)
fix(initialization): surface upstream and SSRF errors verbatim in test-connection responses Test-connection's "连接失败" was nearly useless for debugging because two layers conspired to swallow the actual upstream error: Backend swallow --------------- checkChatModelConnection and CheckASRModel pattern-matched on the upstream error string (401 / 404 / timeout / ...) and replaced it with a fixed Chinese hint. The original SDK message (request URL, response body, provider's error code description) was discarded. Refactor: extract classifyConnectionError(errMsg) — returns just the short Chinese hint. Callers combine "<hint>:<raw err>" so the hint indicates "where to start looking" and the raw error shows "what actually happened". The status-400 special-case (endpoint reachable, parameter mismatch only) still returns success. Frontend swallow ---------------- ModelEditorDialog.testConnection ignored result.message on success paths (fine), but also overwrote it with a generic i18n string on failure paths and 4xx rejections — even though axios's interceptor already lifts the backend's error.message to error.message. Failure branch now uses error?.message || result.message as the toast content; i18n fallback only when neither is present. SSRF allowlist hint ------------------- SSRF validation failures previously returned "Base URL 未通过安全校验: <reason>" with no mention of how to recover (the allowlist is server-side, not configurable in the UI). Operators hit it and were stuck. New utils.FormatSSRFError(label, rawURL, err) wraps the error with guidance to add the host to SSRF_WHITELIST_EXTRA (the operator-facing append-only escape hatch — SSRF_WHITELIST itself is the project's baseline). The hint echoes the failed host so it can be copy-pasted straight into the env var: Base URL 未通过安全校验:DNS resolution failed for hostname test.local.com ... 如该地址确实可信,请联系运维在服务端环境变量 SSRF_WHITELIST_EXTRA 中加入该主机(支持精确域名 / *.example.com 通配 / IP / CIDR),示例:SSRF_WHITELIST_EXTRA=test.local.com, *.example.com,10.0.0.0/8 All SSRF call sites switch to the helper: initialization (CheckRemoteModel / TestEmbedding / CheckRerank / CheckASR / TestMultimodal / batch validation), model CRUD, mcp_service CRUD, knowledge URL ingest, system ReconnectDocReader.
2026-05-17 07:10:42 +00:00
c.Error(errors.NewBadRequestError(secutils.FormatSSRFError("URL", req.URL, err)))
return
}
logger.Infof(ctx,
"Creating knowledge from URL, knowledge base ID: %s, URL: %s",
secutils.SanitizeForLog(kbID),
secutils.SanitizeForLog(req.URL),
)
2025-08-05 07:08:07 +00:00
// Create knowledge entry from the URL
knowledge, err := h.kgService.CreateKnowledgeFromURL(
feat: support multi-tag for document knowledge (#1790) * feat: support multi-tag for document knowledge - Add knowledge_tag_relations join table for many-to-many tag-knowledge mapping - Migrate existing single tag_id data to the new table - Update Knowledge struct, filter, payload types (TagID -> Tags/TagIDs) - Rewrite repository layer: SetKnowledgeTags, GetKnowledgeTags, ListIDsByTagIDs - Update tag count references to query knowledge_tag_relations - Update service/handler layers for multi-tag create/list/batch-update - Frontend: multi-select tag sidebar, multi-tag chip display, tag toggle in popup - FAQ tags remain single-tag model (unchanged) * feat(knowledge): tag edit dialog with dynamic overflow and inline creation - Add TagEditDialog component with search, multi-select, and confirm - Replace inline t-popup tag selector with dialog-based editing - Tags overflow with +N badge, visible count adapts to container width via ResizeObserver - Date/card layout: fixed date width, tags flex to fill remaining space - Add persistent new-tag input bar in dialog (Enter to create, auto-select) - Tag creation emits callback to refresh parent tag list immediately - Apply same overflow + dialog trigger to both card view and list view - Add i18n keys: tagEditDialogTitle, tagEditSearch, tagSelectedCount, tagOverflowTip, tagNewPlaceholder * fix: exclude soft-deleted knowledge from tag reference counts CountReferences, BatchCountReferences, and DeleteUnusedTags now JOIN with knowledges table to filter out soft-deleted records when counting knowledge_tag_relations. Previously they queried knowledge_tag_relations directly without the deleted_at check, causing stale tag counts after knowledge deletion. * fix: refresh tag counts after async knowledge deletion completes Add a second delayed tag list refresh (2.5s) after tag deletion to capture updated reference counts for remaining tags. The original immediate refresh handles removing the deleted tag from the list, while this delayed refresh ensures other tags' counts are correct after the async deletion finishes. --------- Co-authored-by: lyingbug <xiayula2014@gmail.com>
2026-06-25 16:18:48 +00:00
ctx, kbID, req.URL, req.FileName, req.FileType, req.EnableMultimodel, req.Title, req.TagIDs, req.Channel, req.ProcessConfig,
)
2025-08-05 07:08:07 +00:00
// Check for duplicate knowledge error
if err != nil {
if h.handleDuplicateKnowledgeError(c, err, knowledge, "url") {
return
}
if appErr, ok := errors.IsAppError(err); ok {
c.Error(appErr)
return
}
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError(err.Error()))
return
}
logger.Infof(
ctx,
"Knowledge created successfully from URL, ID: %s, title: %s",
secutils.SanitizeForLog(knowledge.ID),
secutils.SanitizeForLog(knowledge.Title),
)
c.JSON(http.StatusCreated, gin.H{
"success": true,
"data": knowledge,
})
}
// CreateManualKnowledge godoc
// @Summary 手工创建知识
// @Description 手工录入Markdown格式的知识内容
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param id path string true "知识库ID"
// @Param request body types.ManualKnowledgePayload true "手工知识内容"
// @Success 200 {object} map[string]interface{} "创建的知识"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge-bases/{id}/knowledge/manual [post]
func (h *KnowledgeHandler) CreateManualKnowledge(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start creating manual knowledge")
// Validate access to the knowledge base (only owner or admin/editor can create)
_, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseAccess(c)
if err != nil {
c.Error(err)
return
}
ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID)
// Check write permission
if permission != types.OrgRoleAdmin && permission != types.OrgRoleEditor {
c.Error(errors.NewForbiddenError("No permission to create knowledge"))
return
}
var req types.ManualKnowledgePayload
if err := c.ShouldBindJSON(&req); err != nil {
logger.Error(ctx, "Failed to parse manual knowledge request", err)
c.Error(errors.NewBadRequestError(err.Error()))
return
}
knowledge, err := h.kgService.CreateKnowledgeFromManual(ctx, kbID, &req, req.Channel)
if err != nil {
if appErr, ok := errors.IsAppError(err); ok {
c.Error(appErr)
return
}
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"kb_id": kbID,
})
c.Error(errors.NewInternalServerError(err.Error()))
return
}
logger.Infof(ctx, "Manual knowledge created successfully, knowledge ID: %s",
secutils.SanitizeForLog(knowledge.ID))
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": knowledge,
})
}
// GetKnowledge godoc
// @Summary 获取知识详情
// @Description 根据ID获取知识条目详情
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param id path string true "知识ID"
// @Success 200 {object} map[string]interface{} "知识详情"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Failure 404 {object} errors.AppError "知识不存在"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge/{id} [get]
2025-08-05 07:08:07 +00:00
func (h *KnowledgeHandler) GetKnowledge(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start retrieving knowledge")
id := secutils.SanitizeForLog(c.Param("id"))
2025-08-05 07:08:07 +00:00
if id == "" {
logger.Error(ctx, "Knowledge ID is empty")
c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty"))
return
}
// Resolve knowledge and validate KB access (at least viewer)
knowledge, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleViewer)
2025-08-05 07:08:07 +00:00
if err != nil {
c.Error(err)
2025-08-05 07:08:07 +00:00
return
}
// Re-fetch with tenant-scoped service so tags and other joined fields are populated.
if knowledge, err = h.kgService.GetKnowledgeByID(effCtx, id); err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewNotFoundError("Knowledge not found"))
return
}
logger.Infof(ctx, "Knowledge retrieved successfully, ID: %s, title: %s",
secutils.SanitizeForLog(knowledge.ID), secutils.SanitizeForLog(knowledge.Title))
2025-08-05 07:08:07 +00:00
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": knowledge,
})
}
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
// GetKnowledgeSpans godoc
// @Summary 获取知识文档解析的 Span 树(含历史尝试)
// @Description 返回该知识在解析流水线的 trace tree(root → stage → subspan):每段状态、耗时、input/output、错误码、langfuse_trace_id。支持 ?attempt=N 查看历史尝试;不传则返回最新尝试。前端用于渲染时间线 + 多模态/embedding 子节点 + 一键跳转 Langfuse。
// @Tags 知识管理
// @Accept json
// @Produce json
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
// @Param id path string true "知识ID"
// @Param attempt query int false "指定尝试号;省略=最新"
// @Success 200 {object} map[string]interface{}
// @Router /api/v1/knowledge/{id}/spans [get]
//
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
// Always returns the canonical 5-stage timeline; missing stage rows are
// synthesized as "pending" so the frontend timeline always renders five
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
// segments. Subspans (multimodal.image[i], generation.*) ride along under
// each stage as children when present.
func (h *KnowledgeHandler) GetKnowledgeSpans(c *gin.Context) {
ctx := c.Request.Context()
id := secutils.SanitizeForLog(c.Param("id"))
if id == "" {
c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty"))
return
}
knowledge, _, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleViewer)
if err != nil {
c.Error(err)
return
}
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
// Pick attempt: explicit ?attempt=N wins; otherwise pull the
// latest attempt from the spans table. Lite-mode / fresh installs
// with zero rows fall through to attempt=0, in which case we
// return a placeholder tree (5 pending stages, no root, no
// children) so the UI still renders.
requestedAttempt := 0
if v := strings.TrimSpace(c.Query("attempt")); v != "" {
if n, perr := strconv.Atoi(v); perr == nil && n > 0 {
requestedAttempt = n
}
}
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
rows := []types.KnowledgeProcessingSpan{}
currentAttempt := 0
latestAttempt := 0
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
if h.spanRepo != nil {
latest, lerr := h.spanRepo.LatestAttempt(ctx, knowledge.ID)
if lerr != nil {
logger.Warnf(ctx, "spans LatestAttempt failed for %s: %v", knowledge.ID, lerr)
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
} else {
latestAttempt = latest
}
if requestedAttempt > 0 {
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
currentAttempt = requestedAttempt
} else {
currentAttempt = latestAttempt
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
}
if currentAttempt > 0 {
rows, err = h.spanRepo.ListByAttempt(ctx, knowledge.ID, currentAttempt)
if err != nil {
logger.Warnf(ctx, "spans ListByAttempt failed kid=%s attempt=%d: %v",
knowledge.ID, currentAttempt, err)
rows = nil
}
}
}
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
// Build tree: index by SpanID, then attach to parents. Stages
// missing from the DB are synthesized as "pending" placeholders
// under a synthetic (or real, if present) root so the timeline
// always renders five segments. parse_status threads through so
// pre-tracker historical knowledge (no rows but parse_status is
// already terminal) renders as done/failed instead of pending —
// otherwise legacy completed documents would forever look like
// they're still waiting in the queue.
tree, currentStageName, lastErr := buildSpanTree(knowledge.ID, currentAttempt, rows, knowledge.ParseStatus)
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
resp := gin.H{
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
"knowledge_id": knowledge.ID,
"attempt": currentAttempt,
"latest_attempt": latestAttempt,
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
"parse_status": knowledge.ParseStatus,
"current_attempt": currentAttempt,
"current_stage": currentStageName,
"trace": tree,
}
if lastError := knowledgeSpansLastError(
currentAttempt,
latestAttempt,
knowledge.ParseStatus,
knowledge.ErrorMessage,
knowledge.UpdatedAt,
lastErr,
); lastError != nil {
resp["last_error"] = lastError
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": resp,
})
}
// knowledgeSpansLastError builds the last_error payload for GetKnowledgeSpans.
// Span failures win when present; otherwise a failed knowledge row without a
// matching span error (recovery / dead-letter paths) surfaces ErrorMessage.
func knowledgeSpansLastError(
currentAttempt, latestAttempt int,
parseStatus, knowledgeErrorMessage string,
knowledgeUpdatedAt time.Time,
spanFailure *types.KnowledgeProcessingSpan,
) gin.H {
if spanFailure != nil {
return gin.H{
"stage": spanFailure.Name,
"code": spanFailure.ErrorCode,
"message": spanFailure.ErrorMessage,
"name": spanFailure.Name,
"error_code": spanFailure.ErrorCode,
"error_message": spanFailure.ErrorMessage,
"finished_at": spanFailure.FinishedAt,
}
}
if currentAttempt != latestAttempt || parseStatus != types.ParseStatusFailed || knowledgeErrorMessage == "" {
return nil
}
errorCode := "UNKNOWN"
if strings.EqualFold(strings.TrimSpace(knowledgeErrorMessage),
"Task interrupted due to application restart") {
errorCode = "SERVER_RESTART"
}
return gin.H{
"stage": "knowledge_processing",
"code": errorCode,
"message": knowledgeErrorMessage,
"name": "knowledge_processing",
"error_code": errorCode,
"error_message": knowledgeErrorMessage,
"finished_at": knowledgeUpdatedAt,
}
}
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
// buildSpanTree assembles a flat list of span rows into a parent-child
// tree rooted at the (knowledge, attempt)'s root span. Missing canonical
// stages are filled in with pending placeholders so the UI always renders
// the five timeline segments. Returns the root, the current_stage name
// (the running stage if any), and the most recent failed span if one
// exists.
//
// parseStatus is the knowledge.parse_status string. When the spans table
// has zero rows for this attempt (legacy data parsed before tracking, or
// a fresh knowledge before the pipeline starts), the placeholder status
// is inferred from parseStatus: completed → done, failed → failed,
// otherwise pending. Without this, every historical knowledge would
// render as "all 5 stages pending" forever despite having actually
// completed parsing.
func buildSpanTree(knowledgeID string, attempt int, rows []types.KnowledgeProcessingSpan, parseStatus string) (
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
root *types.SpanTreeNode, currentStage string, lastFailure *types.KnowledgeProcessingSpan,
) {
now := time.Now()
// Build node lookup, identify root.
nodes := make(map[string]*types.SpanTreeNode, len(rows))
var rootRow *types.KnowledgeProcessingSpan
stageRowByName := map[string]*types.KnowledgeProcessingSpan{}
for i := range rows {
r := rows[i]
nodes[r.SpanID] = &types.SpanTreeNode{KnowledgeProcessingSpan: r}
if r.Kind == types.SpanKindRoot && rootRow == nil {
cp := r
rootRow = &cp
}
if r.Kind == types.SpanKindStage {
cp := r
stageRowByName[r.Name] = &cp
}
if r.Status == types.SpanStatusRunning && r.Kind == types.SpanKindStage && currentStage == "" {
currentStage = r.Name
}
if r.Status == types.SpanStatusFailed {
cp := r
lastFailure = &cp
}
}
// Pick the synthesized stage status from parse_status. Without this,
// historical knowledge that completed before span tracking was wired
// would render as "5 pending stages" forever — the rows simply
// weren't recorded, but parse_status correctly reads "completed".
// The synthesized stages don't carry duration/timing data; they
// just communicate the inferred terminal state.
syntheticStatus := types.SpanStatusPending
switch parseStatus {
case types.ParseStatusCompleted:
syntheticStatus = types.SpanStatusDone
case types.ParseStatusFailed:
syntheticStatus = types.SpanStatusFailed
}
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
// Synthesize root if no rows came back so the API contract stays
// stable (frontend always expects a `trace` object).
if rootRow == nil {
root = &types.SpanTreeNode{KnowledgeProcessingSpan: types.KnowledgeProcessingSpan{
KnowledgeID: knowledgeID,
Attempt: attempt,
SpanID: "",
Name: "knowledge_processing",
Kind: types.SpanKindRoot,
Status: syntheticStatus,
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
CreatedAt: now,
UpdatedAt: now,
}}
} else {
root = nodes[rootRow.SpanID]
}
// Link real children to their parents. Walk `rows` (not the
// `nodes` map!) so the append order matches the repo's stable
// `ORDER BY id ASC`. Iterating the map directly would give
// callers a different child ordering on every request — Go map
// iteration is intentionally randomised — and the UI would
// flicker subspans into a different order on each refresh.
for i := range rows {
r := &rows[i]
n := nodes[r.SpanID]
if n == nil || n == root {
continue
}
if r.ParentSpanID == "" {
// Real top-level row with no parent and not the root
// itself — attach to root so it doesn't dangle.
root.Children = append(root.Children, n)
continue
}
parent, ok := nodes[r.ParentSpanID]
if !ok {
// Unknown parent (orphan); attach to root.
root.Children = append(root.Children, n)
continue
}
parent.Children = append(parent.Children, n)
}
// Synthesize missing stage rows as children of root so the timeline
// always shows 5 segments. Status mirrors the synthesized root —
// pending while the pipeline is still running, done/failed for
// historical knowledge whose terminal state we know but whose
// per-stage timing was never recorded. Appended in AllStages order
// so the canonical stage layout is deterministic regardless of
// which rows are missing.
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
for _, name := range types.AllStages {
if _, ok := stageRowByName[name]; ok {
continue
}
placeholder := types.KnowledgeProcessingSpan{
KnowledgeID: knowledgeID,
Attempt: attempt,
Name: name,
Kind: types.SpanKindStage,
Status: syntheticStatus,
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
CreatedAt: now,
UpdatedAt: now,
}
root.Children = append(root.Children, &types.SpanTreeNode{KnowledgeProcessingSpan: placeholder})
refactor(knowledge): replace flat stage table with langfuse-style span tree Addresses review feedback that the PR② design had four shortcomings: 1. The pipeline is a DAG, not a sequence — Embedding and Multimodal are independent of each other, both downstream of Chunking, both upstream of PostProcess. The flat (knowledge_id, stage) table couldn't represent that, so a Chunking failure left dependents stranded as "pending" forever instead of being marked as impossible-to-run. 2. No history across attempts. A reparse erased the previous run's status before the new run started, leaving operators with no way to investigate "why did this fail twice?". 3. Stages had only status + duration. Operators want to know how big the work was — pages parsed, chunks created, tokens embedded, VLM calls made — to distinguish "slow because the file is huge" from "slow because the docreader is wedged". 4. Multimodal fans out N image tasks; Embedding fans out M batches; PostProcess fans out into Summary/Question/Wiki/Graph. Each unit is interesting on its own (Langfuse already captures this for LLM calls). The flat model couldn't express it. The redesign mirrors Langfuse's trace/span/generation hierarchy: * Migration 000053 supersedes 000052: knowledge_processing_spans table with (knowledge_id, attempt, span_id) primary key, plus parent_span_id, kind ∈ {root, stage, subspan, generation}, status ∈ {pending,running,done,failed,skipped,cancelled}, and JSONB input/output/metadata fields. * SpanTracker (replacing StageTracker) exposes OpenAttempt / BeginStage / BeginSubSpan / EndSpan / FailSpan / SkipSpan / LookupStage. Cross-process workers (image_multimodal) get the parent's attempt + span via payload + LookupStage so subspans attach correctly. * StageDependencies declares the DAG; FailSpan now cascades — descendants of the failed span and dependent stages are flipped to "cancelled" with a UPSTREAM_FAILED code. The UI sees a clear blast radius instead of orphan spinners. * Reparse now calls OpenAttempt up front so the timeline reflects "new attempt, all pending" instead of letting the previous run's status linger until the worker picks up the task. * Image_multimodal records each image as a generation subspan with its own success/failure on the parent attempt's multimodal stage. The finalize-on-last-attempt counter logic is preserved unchanged. * GET /api/v1/knowledge/:id/spans (also kept /stages alias) returns a tree shape with synthesized pending placeholders so the frontend always renders five timeline segments. ?attempt=N enables history navigation.
2026-05-27 06:05:47 +00:00
}
return root, currentStage, lastFailure
}
// ListKnowledge godoc
// @Summary 获取知识列表
// @Description 获取知识库下的知识列表,支持分页和筛选
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param id path string true "知识库ID"
// @Param page query int false "页码"
// @Param page_size query int false "每页数量"
// @Param tag_ids query string false "标签ID筛选,逗号分隔(OR语义)"
feat(knowledge-base): document list filters and explicit batch-management UX Add three new optional filters to the document list under a knowledge base detail page — parse status, source/channel, and updated time range — and rework multi-select to no longer cause the card title to jitter on hover. Backend - Introduce types.KnowledgeListFilter to aggregate optional filter dimensions (tag, keyword, file_type, parse_status, source, updated_from/to) and switch ListPagedKnowledgeByKnowledgeBaseID (repository/service/interface) to accept it instead of a growing positional parameter list. - The ListKnowledge HTTP handler accepts new parse_status, source, start_time and end_time query params; time params accept RFC3339, "YYYY-MM-DD HH:MM:SS" and "YYYY-MM-DD". The repository routes source="manual"/"url" onto the type column to stay consistent with file_type semantics; other source values match the channel column. - Update the four other callers (agent_service, initialization) to pass an empty filter struct, preserving prior behavior. Frontend - Add three controls in the doc-filter-bar (status select, source select, date-range picker with future-date disabled) wired through getKnowled / listKnowledgeFiles into the new backend params. - Replace the hover-triggered card checkbox with an explicit "批量管理" mode (mirrors the session list UX): in card view the checkbox only renders while batch mode is on, entered via the per-card "..." menu; the list view keeps its leading checkbox column. Switching from list to grid auto-enables batch mode when something is already selected, so the selection stays visible. - DocumentBatchBar now stays open whenever batch mode or selection > 0, and its "取消选择" button both clears the selection and exits batch mode. API surface sync - Regenerate Swagger artifacts (docs/docs.go / swagger.json / swagger.yaml). - Update docs/api/knowledge.md with the new query parameters. - Add backward-compatible ListKnowledgeWithFilter + KnowledgeListFilter to the Go SDK; the existing ListKnowledge keeps its signature. i18n - New filter labels in zh-CN / en-US / ko-KR / ru-RU; reuse existing menu.batchManage / batchManage.cancel for the multi-select strings.
2026-05-12 10:16:34 +00:00
// @Param keyword query string false "关键词搜索"
// @Param file_type query string false "文件类型筛选"
// @Param parse_status query string false "解析状态筛选 (pending/processing/completed/failed)"
// @Param source query string false "来源/渠道筛选 (web/api/feishu/notion/yuque/wechat/...,或 manual/url 按 type 过滤)"
// @Param start_time query string false "更新时间起点,RFC3339 格式"
// @Param end_time query string false "更新时间终点,RFC3339 格式"
// @Param folder_path query string false "文件夹路径筛选,空字符串表示知识库根目录;不传该参数则不按文件夹过滤"
// @Param folder_recursive query bool false "为 true 时同时返回子文件夹内的文档"
// @Success 200 {object} map[string]interface{} "知识列表"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge-bases/{id}/knowledge [get]
2025-08-05 07:08:07 +00:00
func (h *KnowledgeHandler) ListKnowledge(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start retrieving knowledge list")
// Validate access to the knowledge base (read access - any permission level)
_, kbID, effectiveTenantID, _, err := h.validateKnowledgeBaseAccess(c)
if err != nil {
c.Error(err)
2025-08-05 07:08:07 +00:00
return
}
// Update context with effective tenant ID for shared KB access
ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID)
2025-08-05 07:08:07 +00:00
// Parse pagination parameters from query string
var pagination types.Pagination
if err := c.ShouldBindQuery(&pagination); err != nil {
logger.Error(ctx, "Failed to parse pagination parameters", err)
c.Error(errors.NewBadRequestError(err.Error()))
return
}
feat(knowledge-base): document list filters and explicit batch-management UX Add three new optional filters to the document list under a knowledge base detail page — parse status, source/channel, and updated time range — and rework multi-select to no longer cause the card title to jitter on hover. Backend - Introduce types.KnowledgeListFilter to aggregate optional filter dimensions (tag, keyword, file_type, parse_status, source, updated_from/to) and switch ListPagedKnowledgeByKnowledgeBaseID (repository/service/interface) to accept it instead of a growing positional parameter list. - The ListKnowledge HTTP handler accepts new parse_status, source, start_time and end_time query params; time params accept RFC3339, "YYYY-MM-DD HH:MM:SS" and "YYYY-MM-DD". The repository routes source="manual"/"url" onto the type column to stay consistent with file_type semantics; other source values match the channel column. - Update the four other callers (agent_service, initialization) to pass an empty filter struct, preserving prior behavior. Frontend - Add three controls in the doc-filter-bar (status select, source select, date-range picker with future-date disabled) wired through getKnowled / listKnowledgeFiles into the new backend params. - Replace the hover-triggered card checkbox with an explicit "批量管理" mode (mirrors the session list UX): in card view the checkbox only renders while batch mode is on, entered via the per-card "..." menu; the list view keeps its leading checkbox column. Switching from list to grid auto-enables batch mode when something is already selected, so the selection stays visible. - DocumentBatchBar now stays open whenever batch mode or selection > 0, and its "取消选择" button both clears the selection and exits batch mode. API surface sync - Regenerate Swagger artifacts (docs/docs.go / swagger.json / swagger.yaml). - Update docs/api/knowledge.md with the new query parameters. - Add backward-compatible ListKnowledgeWithFilter + KnowledgeListFilter to the Go SDK; the existing ListKnowledge keeps its signature. i18n - New filter labels in zh-CN / en-US / ko-KR / ru-RU; reuse existing menu.batchManage / batchManage.cancel for the multi-select strings.
2026-05-12 10:16:34 +00:00
filter := types.KnowledgeListFilter{
feat: support multi-tag for document knowledge (#1790) * feat: support multi-tag for document knowledge - Add knowledge_tag_relations join table for many-to-many tag-knowledge mapping - Migrate existing single tag_id data to the new table - Update Knowledge struct, filter, payload types (TagID -> Tags/TagIDs) - Rewrite repository layer: SetKnowledgeTags, GetKnowledgeTags, ListIDsByTagIDs - Update tag count references to query knowledge_tag_relations - Update service/handler layers for multi-tag create/list/batch-update - Frontend: multi-select tag sidebar, multi-tag chip display, tag toggle in popup - FAQ tags remain single-tag model (unchanged) * feat(knowledge): tag edit dialog with dynamic overflow and inline creation - Add TagEditDialog component with search, multi-select, and confirm - Replace inline t-popup tag selector with dialog-based editing - Tags overflow with +N badge, visible count adapts to container width via ResizeObserver - Date/card layout: fixed date width, tags flex to fill remaining space - Add persistent new-tag input bar in dialog (Enter to create, auto-select) - Tag creation emits callback to refresh parent tag list immediately - Apply same overflow + dialog trigger to both card view and list view - Add i18n keys: tagEditDialogTitle, tagEditSearch, tagSelectedCount, tagOverflowTip, tagNewPlaceholder * fix: exclude soft-deleted knowledge from tag reference counts CountReferences, BatchCountReferences, and DeleteUnusedTags now JOIN with knowledges table to filter out soft-deleted records when counting knowledge_tag_relations. Previously they queried knowledge_tag_relations directly without the deleted_at check, causing stale tag counts after knowledge deletion. * fix: refresh tag counts after async knowledge deletion completes Add a second delayed tag list refresh (2.5s) after tag deletion to capture updated reference counts for remaining tags. The original immediate refresh handles removing the deleted tag from the list, while this delayed refresh ensures other tags' counts are correct after the async deletion finishes. --------- Co-authored-by: lyingbug <xiayula2014@gmail.com>
2026-06-25 16:18:48 +00:00
TagIDs: parseCommaSeparatedTagIDs(c.Query("tag_ids")),
feat(knowledge-base): document list filters and explicit batch-management UX Add three new optional filters to the document list under a knowledge base detail page — parse status, source/channel, and updated time range — and rework multi-select to no longer cause the card title to jitter on hover. Backend - Introduce types.KnowledgeListFilter to aggregate optional filter dimensions (tag, keyword, file_type, parse_status, source, updated_from/to) and switch ListPagedKnowledgeByKnowledgeBaseID (repository/service/interface) to accept it instead of a growing positional parameter list. - The ListKnowledge HTTP handler accepts new parse_status, source, start_time and end_time query params; time params accept RFC3339, "YYYY-MM-DD HH:MM:SS" and "YYYY-MM-DD". The repository routes source="manual"/"url" onto the type column to stay consistent with file_type semantics; other source values match the channel column. - Update the four other callers (agent_service, initialization) to pass an empty filter struct, preserving prior behavior. Frontend - Add three controls in the doc-filter-bar (status select, source select, date-range picker with future-date disabled) wired through getKnowled / listKnowledgeFiles into the new backend params. - Replace the hover-triggered card checkbox with an explicit "批量管理" mode (mirrors the session list UX): in card view the checkbox only renders while batch mode is on, entered via the per-card "..." menu; the list view keeps its leading checkbox column. Switching from list to grid auto-enables batch mode when something is already selected, so the selection stays visible. - DocumentBatchBar now stays open whenever batch mode or selection > 0, and its "取消选择" button both clears the selection and exits batch mode. API surface sync - Regenerate Swagger artifacts (docs/docs.go / swagger.json / swagger.yaml). - Update docs/api/knowledge.md with the new query parameters. - Add backward-compatible ListKnowledgeWithFilter + KnowledgeListFilter to the Go SDK; the existing ListKnowledge keeps its signature. i18n - New filter labels in zh-CN / en-US / ko-KR / ru-RU; reuse existing menu.batchManage / batchManage.cancel for the multi-select strings.
2026-05-12 10:16:34 +00:00
Keyword: c.Query("keyword"),
FileType: c.Query("file_type"),
ParseStatus: c.Query("parse_status"),
Source: c.Query("source"),
}
if raw := c.Query("start_time"); raw != "" {
t, err := parseFilterTime(raw)
if err != nil {
c.Error(errors.NewBadRequestError("invalid start_time: " + err.Error()))
return
}
filter.UpdatedFrom = t
}
if raw := c.Query("end_time"); raw != "" {
t, err := parseFilterTime(raw)
if err != nil {
c.Error(errors.NewBadRequestError("invalid end_time: " + err.Error()))
return
}
filter.UpdatedTo = t
}
// The folder dimension is opt-in by parameter *presence*: an empty
// folder_path is meaningful (the knowledge base root), so it cannot be
// distinguished from "no folder filter" by value alone.
if raw, ok := c.GetQuery("folder_path"); ok {
filter.FolderPath = types.NormalizeKnowledgeFolderPath(raw)
filter.FolderScope = types.FolderScopeExact
if recursive, err := strconv.ParseBool(c.DefaultQuery("folder_recursive", "false")); err == nil && recursive {
filter.FolderScope = types.FolderScopeSubtree
}
}
2025-11-17 10:15:02 +00:00
logger.Infof(
ctx,
"Retrieving knowledge list under knowledge base, kb_id=%s tag_ids=%s keyword=%s file_type=%s parse_status=%s source=%s start_time=%s end_time=%s folder_path=%s folder_scope=%s page=%d page_size=%d effectiveTenantID=%d",
secutils.SanitizeForLog(kbID),
feat: support multi-tag for document knowledge (#1790) * feat: support multi-tag for document knowledge - Add knowledge_tag_relations join table for many-to-many tag-knowledge mapping - Migrate existing single tag_id data to the new table - Update Knowledge struct, filter, payload types (TagID -> Tags/TagIDs) - Rewrite repository layer: SetKnowledgeTags, GetKnowledgeTags, ListIDsByTagIDs - Update tag count references to query knowledge_tag_relations - Update service/handler layers for multi-tag create/list/batch-update - Frontend: multi-select tag sidebar, multi-tag chip display, tag toggle in popup - FAQ tags remain single-tag model (unchanged) * feat(knowledge): tag edit dialog with dynamic overflow and inline creation - Add TagEditDialog component with search, multi-select, and confirm - Replace inline t-popup tag selector with dialog-based editing - Tags overflow with +N badge, visible count adapts to container width via ResizeObserver - Date/card layout: fixed date width, tags flex to fill remaining space - Add persistent new-tag input bar in dialog (Enter to create, auto-select) - Tag creation emits callback to refresh parent tag list immediately - Apply same overflow + dialog trigger to both card view and list view - Add i18n keys: tagEditDialogTitle, tagEditSearch, tagSelectedCount, tagOverflowTip, tagNewPlaceholder * fix: exclude soft-deleted knowledge from tag reference counts CountReferences, BatchCountReferences, and DeleteUnusedTags now JOIN with knowledges table to filter out soft-deleted records when counting knowledge_tag_relations. Previously they queried knowledge_tag_relations directly without the deleted_at check, causing stale tag counts after knowledge deletion. * fix: refresh tag counts after async knowledge deletion completes Add a second delayed tag list refresh (2.5s) after tag deletion to capture updated reference counts for remaining tags. The original immediate refresh handles removing the deleted tag from the list, while this delayed refresh ensures other tags' counts are correct after the async deletion finishes. --------- Co-authored-by: lyingbug <xiayula2014@gmail.com>
2026-06-25 16:18:48 +00:00
secutils.SanitizeForLog(strings.Join(filter.TagIDs, ",")),
feat(knowledge-base): document list filters and explicit batch-management UX Add three new optional filters to the document list under a knowledge base detail page — parse status, source/channel, and updated time range — and rework multi-select to no longer cause the card title to jitter on hover. Backend - Introduce types.KnowledgeListFilter to aggregate optional filter dimensions (tag, keyword, file_type, parse_status, source, updated_from/to) and switch ListPagedKnowledgeByKnowledgeBaseID (repository/service/interface) to accept it instead of a growing positional parameter list. - The ListKnowledge HTTP handler accepts new parse_status, source, start_time and end_time query params; time params accept RFC3339, "YYYY-MM-DD HH:MM:SS" and "YYYY-MM-DD". The repository routes source="manual"/"url" onto the type column to stay consistent with file_type semantics; other source values match the channel column. - Update the four other callers (agent_service, initialization) to pass an empty filter struct, preserving prior behavior. Frontend - Add three controls in the doc-filter-bar (status select, source select, date-range picker with future-date disabled) wired through getKnowled / listKnowledgeFiles into the new backend params. - Replace the hover-triggered card checkbox with an explicit "批量管理" mode (mirrors the session list UX): in card view the checkbox only renders while batch mode is on, entered via the per-card "..." menu; the list view keeps its leading checkbox column. Switching from list to grid auto-enables batch mode when something is already selected, so the selection stays visible. - DocumentBatchBar now stays open whenever batch mode or selection > 0, and its "取消选择" button both clears the selection and exits batch mode. API surface sync - Regenerate Swagger artifacts (docs/docs.go / swagger.json / swagger.yaml). - Update docs/api/knowledge.md with the new query parameters. - Add backward-compatible ListKnowledgeWithFilter + KnowledgeListFilter to the Go SDK; the existing ListKnowledge keeps its signature. i18n - New filter labels in zh-CN / en-US / ko-KR / ru-RU; reuse existing menu.batchManage / batchManage.cancel for the multi-select strings.
2026-05-12 10:16:34 +00:00
secutils.SanitizeForLog(filter.Keyword),
secutils.SanitizeForLog(filter.FileType),
secutils.SanitizeForLog(filter.ParseStatus),
secutils.SanitizeForLog(filter.Source),
secutils.SanitizeForLog(c.Query("start_time")),
secutils.SanitizeForLog(c.Query("end_time")),
secutils.SanitizeForLog(filter.FolderPath),
string(filter.FolderScope),
pagination.Page,
pagination.PageSize,
effectiveTenantID,
)
2025-08-05 07:08:07 +00:00
// Retrieve paginated knowledge entries
feat(knowledge-base): document list filters and explicit batch-management UX Add three new optional filters to the document list under a knowledge base detail page — parse status, source/channel, and updated time range — and rework multi-select to no longer cause the card title to jitter on hover. Backend - Introduce types.KnowledgeListFilter to aggregate optional filter dimensions (tag, keyword, file_type, parse_status, source, updated_from/to) and switch ListPagedKnowledgeByKnowledgeBaseID (repository/service/interface) to accept it instead of a growing positional parameter list. - The ListKnowledge HTTP handler accepts new parse_status, source, start_time and end_time query params; time params accept RFC3339, "YYYY-MM-DD HH:MM:SS" and "YYYY-MM-DD". The repository routes source="manual"/"url" onto the type column to stay consistent with file_type semantics; other source values match the channel column. - Update the four other callers (agent_service, initialization) to pass an empty filter struct, preserving prior behavior. Frontend - Add three controls in the doc-filter-bar (status select, source select, date-range picker with future-date disabled) wired through getKnowled / listKnowledgeFiles into the new backend params. - Replace the hover-triggered card checkbox with an explicit "批量管理" mode (mirrors the session list UX): in card view the checkbox only renders while batch mode is on, entered via the per-card "..." menu; the list view keeps its leading checkbox column. Switching from list to grid auto-enables batch mode when something is already selected, so the selection stays visible. - DocumentBatchBar now stays open whenever batch mode or selection > 0, and its "取消选择" button both clears the selection and exits batch mode. API surface sync - Regenerate Swagger artifacts (docs/docs.go / swagger.json / swagger.yaml). - Update docs/api/knowledge.md with the new query parameters. - Add backward-compatible ListKnowledgeWithFilter + KnowledgeListFilter to the Go SDK; the existing ListKnowledge keeps its signature. i18n - New filter labels in zh-CN / en-US / ko-KR / ru-RU; reuse existing menu.batchManage / batchManage.cancel for the multi-select strings.
2026-05-12 10:16:34 +00:00
result, err := h.kgService.ListPagedKnowledgeByKnowledgeBaseID(ctx, kbID, &pagination, filter)
2025-08-05 07:08:07 +00:00
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError(err.Error()))
return
}
logger.Infof(
ctx,
"Knowledge list retrieved successfully, knowledge base ID: %s, total: %d",
secutils.SanitizeForLog(kbID),
result.Total,
)
2025-08-05 07:08:07 +00:00
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": result.Data,
"total": result.Total,
"page": result.Page,
"page_size": result.PageSize,
})
}
// ListKnowledgeFolders godoc
// @Summary 获取知识库文件夹目录树
// @Description 返回知识库内由文件夹上传形成的目录树,包含每个文件夹的直接文档数与含子目录的总数
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param id path string true "知识库ID"
// @Success 200 {object} map[string]interface{} "目录树"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge-bases/{id}/knowledge/folders [get]
func (h *KnowledgeHandler) ListKnowledgeFolders(c *gin.Context) {
ctx := c.Request.Context()
// Read access mirrors ListKnowledge so the sidebar tree is available to
// every viewer of a shared knowledge base.
_, kbID, effectiveTenantID, _, err := h.validateKnowledgeBaseAccess(c)
if err != nil {
c.Error(err)
return
}
ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID)
tree, err := h.kgService.ListKnowledgeFolderTree(ctx, kbID)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError(err.Error()))
return
}
logger.Infof(ctx, "Knowledge folder tree retrieved, kb_id=%s folders=%d",
secutils.SanitizeForLog(kbID), len(tree.Folders))
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": tree,
})
}
// DeleteKnowledge godoc
// @Summary 删除知识
// @Description 根据ID异步删除知识条目。请求会被入队到与批量删除相同的异步管道(asynq);
// @Description 接口返回 200 仅表示任务已提交(响应 data.task_id 为任务 ID),实际删除由后台 worker 完成。
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param id path string true "知识ID"
// @Success 200 {object} map[string]interface{} "任务已提交,返回 task_id"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge/{id} [delete]
2025-08-05 07:08:07 +00:00
func (h *KnowledgeHandler) DeleteKnowledge(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start deleting knowledge")
id := secutils.SanitizeForLog(c.Param("id"))
2025-08-05 07:08:07 +00:00
if id == "" {
logger.Error(ctx, "Knowledge ID is empty")
c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty"))
return
}
_, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleEditor)
if err != nil {
c.Error(err)
return
}
// Reuse the batch async pipeline so single-item delete shares the same
// hardening (asynq retries, business-aware queue routing, marking-as-deleting
// inside the worker) as BatchDeleteKnowledge / ClearKnowledgeBaseContents.
effectiveTenantID, _ := effCtx.Value(types.TenantIDContextKey).(uint64)
if effectiveTenantID == 0 {
logger.Error(ctx, "Effective tenant ID missing after access validation")
c.Error(errors.NewInternalServerError("workspace context unavailable"))
return
}
logger.Infof(ctx, "Enqueuing knowledge delete, ID: %s", secutils.SanitizeForLog(id))
taskID, err := h.enqueueKnowledgeListDelete(effCtx, effectiveTenantID, []string{id})
2025-08-05 07:08:07 +00:00
if err != nil {
logger.Errorf(ctx, "Failed to enqueue knowledge delete task: %v", err)
c.Error(errors.NewInternalServerError("Failed to enqueue delete task"))
2025-08-05 07:08:07 +00:00
return
}
logger.Infof(ctx, "Knowledge delete task enqueued: %s, knowledge_id: %s", taskID, secutils.SanitizeForLog(id))
2025-08-05 07:08:07 +00:00
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "Delete task submitted",
"data": gin.H{
"task_id": taskID,
},
2025-08-05 07:08:07 +00:00
})
}
// BatchDeleteKnowledgeRequest is the body schema for POST /knowledge/batch-delete.
type BatchDeleteKnowledgeRequest struct {
KBID string `json:"kb_id" binding:"required"`
IDs []string `json:"ids" binding:"required"`
}
// BatchDeleteKnowledge godoc
// @Summary 批量删除知识
// @Description 按 ID 列表批量删除单个知识库下的多个知识条目
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param request body BatchDeleteKnowledgeRequest true "批量删除请求"
// @Success 200 {object} map[string]interface{} "删除成功"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Failure 403 {object} errors.AppError "权限不足"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge/batch-delete [post]
func (h *KnowledgeHandler) BatchDeleteKnowledge(c *gin.Context) {
ctx := c.Request.Context()
var req BatchDeleteKnowledgeRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.Error(errors.NewBadRequestError("Invalid request parameters: " + err.Error()))
return
}
// Deduplicate and drop empty IDs.
seen := make(map[string]struct{}, len(req.IDs))
ids := make([]string, 0, len(req.IDs))
for _, raw := range req.IDs {
id := strings.TrimSpace(raw)
if id == "" {
continue
}
if _, ok := seen[id]; ok {
continue
}
seen[id] = struct{}{}
ids = append(ids, id)
}
if len(ids) == 0 {
c.Error(errors.NewBadRequestError("ids cannot be empty"))
return
}
const maxBatch = 200
if len(ids) > maxBatch {
c.Error(errors.NewBadRequestError(fmt.Sprintf("too many ids (max %d per batch)", maxBatch)))
return
}
// Validate KB access (editor or admin) using the kb_id from body.
_, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseAccessWithKBID(c, req.KBID)
if err != nil {
c.Error(err)
return
}
if permission != types.OrgRoleAdmin && permission != types.OrgRoleEditor {
c.Error(errors.NewForbiddenError("No permission to delete knowledge"))
return
}
if err := h.requireKBOwnershipOrAdmin(c, kbID); err != nil {
c.Error(err)
return
}
ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID)
// Single batch fetch to validate that every id exists and belongs to the
// requested KB. The service-layer DeleteKnowledgeList only enforces tenant
// scope, not KB scope, so the handler must guard against cross-KB deletion.
knowledgeList, err := h.kgService.GetKnowledgeBatch(ctx, effectiveTenantID, ids)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError(err.Error()))
return
}
if len(knowledgeList) != len(ids) {
c.Error(errors.NewBadRequestError("One or more knowledge entries not found"))
return
}
for _, k := range knowledgeList {
if k.KnowledgeBaseID != kbID {
c.Error(errors.NewBadRequestError(
fmt.Sprintf("Knowledge %s does not belong to knowledge base %s",
secutils.SanitizeForLog(k.ID), secutils.SanitizeForLog(kbID))))
return
}
}
taskID, err := h.enqueueKnowledgeListDelete(ctx, effectiveTenantID, ids)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue batch knowledge delete task: %v", err)
c.Error(errors.NewInternalServerError("Failed to enqueue batch delete task"))
return
}
logger.Infof(ctx, "Batch knowledge delete task enqueued: %s, kb_id: %s, count: %d",
taskID, secutils.SanitizeForLog(kbID), len(ids))
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "Batch delete task submitted",
"data": gin.H{
"task_id": taskID,
"deleted_count": len(ids),
},
})
}
// ClearKnowledgeBaseContents godoc
// @Summary 清空知识库内容
// @Description 删除知识库下的所有知识条目(异步任务)。知识库本身保留,仅清空其中的内容
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param id path string true "知识库ID"
// @Success 200 {object} map[string]interface{} "清空任务已提交"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Failure 403 {object} errors.AppError "权限不足"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge-bases/{id}/knowledge [delete]
func (h *KnowledgeHandler) ClearKnowledgeBaseContents(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start clearing knowledge base contents")
kb, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseAccess(c)
if err != nil {
c.Error(err)
return
}
// Only owner (admin with matching tenant) can clear knowledge base contents
tenantID := c.GetUint64(types.TenantIDContextKey.String())
if kb.TenantID != tenantID || permission != types.OrgRoleAdmin {
c.Error(errors.NewForbiddenError("Only knowledge base owner can clear contents"))
return
}
ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID)
knowledgeList, err := h.kgService.ListKnowledgeByKnowledgeBaseID(ctx, kbID)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError("Failed to list knowledge entries").WithDetails(err.Error()))
return
}
if len(knowledgeList) == 0 {
logger.Infof(ctx, "Knowledge base %s is already empty", secutils.SanitizeForLog(kbID))
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "Knowledge base is already empty",
"data": gin.H{"deleted_count": 0},
})
return
}
knowledgeIDs := make([]string, 0, len(knowledgeList))
for _, knowledge := range knowledgeList {
knowledgeIDs = append(knowledgeIDs, knowledge.ID)
}
taskID, err := h.enqueueKnowledgeListDelete(ctx, effectiveTenantID, knowledgeIDs)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue knowledge list delete task: %v", err)
c.Error(errors.NewInternalServerError("Failed to enqueue cleanup task"))
return
}
logger.Infof(ctx, "Knowledge base contents clear task enqueued: %s, kb_id: %s, count: %d",
taskID, secutils.SanitizeForLog(kbID), len(knowledgeIDs))
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "Knowledge base contents clear task submitted",
"data": gin.H{"deleted_count": len(knowledgeIDs)},
})
}
// DownloadKnowledgeFile godoc
// @Summary 下载知识文件
// @Description 下载知识条目关联的原始文件
// @Tags 知识管理
// @Accept json
// @Produce application/octet-stream
// @Param id path string true "知识ID"
// @Success 200 {file} file "文件内容"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge/{id}/download [get]
2025-08-05 07:08:07 +00:00
func (h *KnowledgeHandler) DownloadKnowledgeFile(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start downloading knowledge file")
id := secutils.SanitizeForLog(c.Param("id"))
2025-08-05 07:08:07 +00:00
if id == "" {
logger.Error(ctx, "Knowledge ID is empty")
c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty"))
return
}
// Keep a handler-level Editor check in addition to the route guard. The
// original file is more sensitive than parsed-content reads and must not
// be downloadable through a read-only organization share.
_, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleEditor)
if err != nil {
c.Error(err)
return
}
logger.Infof(ctx, "Retrieving knowledge file, ID: %s", secutils.SanitizeForLog(id))
2025-08-05 07:08:07 +00:00
file, filename, err := h.kgService.GetKnowledgeFile(effCtx, id)
2025-08-05 07:08:07 +00:00
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError("Failed to retrieve file").WithDetails(err.Error()))
return
}
defer file.Close()
logger.Infof(
ctx,
"Knowledge file retrieved successfully, ID: %s, filename: %s",
secutils.SanitizeForLog(id),
secutils.SanitizeForLog(filename),
)
2025-08-05 07:08:07 +00:00
// Set response headers for file download
c.Header("Content-Description", "File Transfer")
c.Header("Content-Transfer-Encoding", "binary")
cd := mime.FormatMediaType("attachment", map[string]string{"filename": filename})
c.Header("Content-Disposition", cd)
2025-08-05 07:08:07 +00:00
c.Header("Content-Type", "application/octet-stream")
c.Header("Expires", "0")
c.Header("Cache-Control", "must-revalidate")
c.Header("Pragma", "public")
// Stream file content to response
c.Stream(func(w io.Writer) bool {
if _, err := io.Copy(w, file); err != nil {
logger.Errorf(ctx, "Failed to send file: %v", err)
return false
}
logger.Debug(ctx, "File sending completed")
return false
})
}
// mimeTypeByExt returns the MIME type for a given file extension.
func mimeTypeByExt(filename string) string {
ct, _ := secutils.SafeContentTypeByFilename(filename)
return ct
}
// PreviewKnowledgeFile godoc
// @Summary 预览知识文件
// @Description 返回知识条目关联的原始文件,Content-Type 根据文件类型设置,用于浏览器内嵌预览
// @Tags 知识管理
// @Accept json
// @Produce application/pdf,image/jpeg,image/png,text/plain
// @Param id path string true "知识ID"
// @Success 200 {file} file "文件内容"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge/{id}/preview [get]
func (h *KnowledgeHandler) PreviewKnowledgeFile(c *gin.Context) {
ctx := c.Request.Context()
id := secutils.SanitizeForLog(c.Param("id"))
if id == "" {
c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty"))
return
}
_, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleViewer)
if err != nil {
c.Error(err)
return
}
file, filename, err := h.kgService.GetKnowledgeFile(effCtx, id)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError("Failed to retrieve file").WithDetails(err.Error()))
return
}
defer file.Close()
contentType, inline := secutils.SafeContentTypeByFilename(filename)
c.Header("Content-Type", contentType)
c.Header("X-Content-Type-Options", "nosniff")
disposition := "inline"
if !inline {
disposition = "attachment"
}
c.Header("Content-Disposition", mime.FormatMediaType(disposition, map[string]string{"filename": filename}))
c.Header("Cache-Control", "private, max-age=3600")
c.Stream(func(w io.Writer) bool {
if _, err := io.Copy(w, file); err != nil {
logger.Errorf(ctx, "Failed to stream preview: %v", err)
return false
}
return false
})
}
2025-08-05 07:08:07 +00:00
// GetKnowledgeBatchRequest defines parameters for batch knowledge retrieval
type GetKnowledgeBatchRequest struct {
IDs []string `form:"ids" binding:"required"` // List of knowledge IDs
KBID string `form:"kb_id"` // Optional: scope to this KB (validates access and uses effective tenant for shared KB)
AgentID string `form:"agent_id"` // Optional: when using a shared agent, use agent's tenant for retrieval (validates shared agent access)
AgentSourceTenantID uint64 `form:"agent_source_tenant_id"` // Optional source selector, verified against the share relation
2025-08-05 07:08:07 +00:00
}
// GetKnowledgeBatch godoc
// @Summary 批量获取知识
// @Description 根据ID列表批量获取知识条目。可选 kb_id:指定时按该知识库校验权限并用于共享知识库的空间解析;可选 agent_id:使用共享智能体时传此参数,后端按智能体所属空间查询(用于刷新后恢复共享知识库下的文件)
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param ids query []string true "知识ID列表"
// @Param kb_id query string false "可选,知识库ID(用于共享知识库时指定范围)"
// @Param agent_id query string false "可选,共享智能体ID(用于按智能体空间批量拉取文件详情)"
// @Success 200 {object} map[string]interface{} "知识列表"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge/batch [get]
2025-08-05 07:08:07 +00:00
func (h *KnowledgeHandler) GetKnowledgeBatch(c *gin.Context) {
ctx := c.Request.Context()
tenantID, ok := c.Get(types.TenantIDContextKey.String())
if !ok {
logger.Error(ctx, "Failed to get tenant ID")
c.Error(errors.NewUnauthorizedError("Unauthorized"))
return
}
effectiveTenantID := tenantID.(uint64)
2025-08-05 07:08:07 +00:00
var req GetKnowledgeBatchRequest
if err := c.ShouldBindQuery(&req); err != nil {
logger.Error(ctx, "Failed to parse request parameters", err)
c.Error(errors.NewBadRequestError("Invalid request parameters").WithDetails(err.Error()))
return
}
if _, parseErr := types.ParseAgentSourceTenantID(c.Query(types.AgentSourceTenantIDParam)); parseErr != nil {
c.Error(errors.NewBadRequestError(parseErr.Error()))
return
}
2025-08-05 07:08:07 +00:00
// agentAllowedKBIDs restricts results to the agent's configured KB scope.
// nil = no agent restriction; empty slice = agent has no KB access (none mode).
var agentAllowedKBIDs []string
// Optional agent_id: when using shared agent, resolve agent and use its tenant for batch retrieval (so shared KB files can be loaded after refresh)
if agentID := secutils.SanitizeForLog(req.AgentID); agentID != "" && h.agentShareService != nil {
userIDVal, ok := c.Get(types.UserIDContextKey.String())
if !ok {
c.Error(errors.NewUnauthorizedError("Unauthorized"))
return
}
userID, _ := userIDVal.(string)
currentTenantID := c.GetUint64(types.TenantIDContextKey.String())
if currentTenantID == 0 {
c.Error(errors.NewUnauthorizedError("Unauthorized"))
return
}
feat(rbac): lift org membership from per-user to per-tenant (Plan 3, #1303) Before this change, the `organization_members` table keyed on user_id — joining an org as a Viewer in tenant T meant only that one user got visibility on shared KBs/agents, while the rest of T (including its Owner) saw nothing. The mismatch surfaced as confusing UX bugs: "my colleague shared this with us, why can't I see it?" This change replaces `organization_members` with `organization_tenant_members`. Membership and Org-RBAC role now key on (org_id, tenant_id); `representative_user_id` stays as a display-only label of the user who first brought the tenant in. Permission resolution gains a 3-D cap so tenant-RBAC stays honest under cross-tenant sharing: effective = min(share.Permission, tenant_org_role, tenant_role_cap) where tenant_role_cap pins tenant Viewers to OrgRoleViewer regardless of the org-level grant. A Viewer in their own tenant cannot edit a shared KB even if the org granted Editor — preserves the "Viewer = read-only everywhere" promise. Migration 000045 backfills `organization_tenant_members` from the old table by collapsing each (org, tenant) group to its max role, then renames the old table to `organization_members_pre_plan3` for safe rollback. Sqlite mirror (Lite mode) is fresh-bootstrap: the init schema replaces the old table directly. Routes: member-management URLs change from `:user_id` to `:tenant_id`. Org create / join / leave now require Admin+ in the caller's tenant — joining an Org affects everyone in T, so the decision can't sit with a Viewer. Frontend: OrganizationSettingsModal + store update to address members by tenant_id.
2026-05-15 13:49:54 +00:00
callerTenantRole := types.TenantRoleFromContext(ctx)
agent, err := h.agentShareService.GetSharedAgentForTenant(ctx, currentTenantID, callerTenantRole, agentID, req.AgentSourceTenantID)
if err != nil || agent == nil {
logger.Warnf(ctx, "GetKnowledgeBatch: invalid or inaccessible shared agent %s: %v", agentID, err)
c.Error(errors.NewForbiddenError("Invalid or inaccessible shared agent").WithDetails(err.Error()))
return
}
feat(rbac): lift org membership from per-user to per-tenant (Plan 3, #1303) Before this change, the `organization_members` table keyed on user_id — joining an org as a Viewer in tenant T meant only that one user got visibility on shared KBs/agents, while the rest of T (including its Owner) saw nothing. The mismatch surfaced as confusing UX bugs: "my colleague shared this with us, why can't I see it?" This change replaces `organization_members` with `organization_tenant_members`. Membership and Org-RBAC role now key on (org_id, tenant_id); `representative_user_id` stays as a display-only label of the user who first brought the tenant in. Permission resolution gains a 3-D cap so tenant-RBAC stays honest under cross-tenant sharing: effective = min(share.Permission, tenant_org_role, tenant_role_cap) where tenant_role_cap pins tenant Viewers to OrgRoleViewer regardless of the org-level grant. A Viewer in their own tenant cannot edit a shared KB even if the org granted Editor — preserves the "Viewer = read-only everywhere" promise. Migration 000045 backfills `organization_tenant_members` from the old table by collapsing each (org, tenant) group to its max role, then renames the old table to `organization_members_pre_plan3` for safe rollback. Sqlite mirror (Lite mode) is fresh-bootstrap: the init schema replaces the old table directly. Routes: member-management URLs change from `:user_id` to `:tenant_id`. Org create / join / leave now require Admin+ in the caller's tenant — joining an Org affects everyone in T, so the decision can't sit with a Viewer. Frontend: OrganizationSettingsModal + store update to address members by tenant_id.
2026-05-15 13:49:54 +00:00
_ = userID
effectiveTenantID = agent.TenantID
agentAllowedKBIDs = resolveAgentAllowedKBIDs(agent)
if agentAllowedKBIDs != nil && len(agentAllowedKBIDs) == 0 {
c.JSON(http.StatusOK, gin.H{"success": true, "data": []*types.Knowledge{}})
return
}
logger.Infof(ctx, "Batch retrieving knowledge with agent_id, effective tenant ID: %d, IDs count: %d, allowed KBs: %v",
effectiveTenantID, len(req.IDs), agentAllowedKBIDs)
}
var knowledges []*types.Knowledge
var err error
// scopeKBID tracks the single KB the results must belong to (set by explicit kb_id).
var scopeKBID string
// Optional kb_id: validate KB access and use effective tenant for shared KB
if kbID := secutils.SanitizeForLog(req.KBID); kbID != "" {
_, _, effID, _, err := h.validateKnowledgeBaseAccessWithKBID(c, kbID)
if err != nil {
c.Error(err)
return
}
if agentAllowedKBIDs != nil && !sliceContains(agentAllowedKBIDs, kbID) {
c.Error(errors.NewForbiddenError("Knowledge base not accessible through this agent"))
return
}
scopeKBID = kbID
effectiveTenantID = effID
ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID)
logger.Infof(ctx, "Batch retrieving knowledge with kb_id, effective tenant ID: %d, IDs count: %d",
effectiveTenantID, len(req.IDs))
knowledges, err = h.kgService.GetKnowledgeBatch(ctx, effectiveTenantID, req.IDs)
} else {
// No kb_id: use GetKnowledgeBatchWithSharedAccess (or effectiveTenantID may already be set by agent_id for shared agent)
logger.Infof(ctx, "Batch retrieving knowledge without kb_id, effective tenant ID: %d, IDs count: %d",
effectiveTenantID, len(req.IDs))
knowledges, err = h.kgService.GetKnowledgeBatchWithSharedAccess(ctx, effectiveTenantID, req.IDs)
}
2025-08-05 07:08:07 +00:00
// Build the effective allowed-KB set from explicit kb_id, shared agent
// scope, and per-API-key KB restrictions.
var allowedKBSet map[string]bool
if scopeKBID != "" {
allowedKBSet = map[string]bool{scopeKBID: true}
} else if agentAllowedKBIDs != nil {
allowedKBSet = make(map[string]bool, len(agentAllowedKBIDs))
for _, id := range agentAllowedKBIDs {
allowedKBSet[id] = true
}
}
if apiKeySet := tenantAPIKeyAllowedKBSet(ctx); apiKeySet != nil {
allowedKBSet = intersectKBAllowSet(allowedKBSet, apiKeySet)
}
if allowedKBSet != nil {
knowledges = filterKnowledgesByKBAllowSet(knowledges, allowedKBSet)
}
2025-08-05 07:08:07 +00:00
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError("Failed to retrieve knowledge list").WithDetails(err.Error()))
return
}
logger.Infof(ctx, "Batch knowledge retrieval successful, requested count: %d, returned count: %d",
len(req.IDs), len(knowledges))
2025-08-05 07:08:07 +00:00
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": knowledges,
})
}
// UpdateKnowledge godoc
// @Summary 更新知识
// @Description 更新知识条目信息
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param id path string true "知识ID"
// @Param request body types.Knowledge true "知识信息"
// @Success 200 {object} map[string]interface{} "更新成功"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge/{id} [put]
2025-08-05 07:08:07 +00:00
func (h *KnowledgeHandler) UpdateKnowledge(c *gin.Context) {
ctx := c.Request.Context()
id := secutils.SanitizeForLog(c.Param("id"))
2025-08-05 07:08:07 +00:00
if id == "" {
logger.Error(ctx, "Knowledge ID is empty")
c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty"))
return
}
_, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleEditor)
if err != nil {
c.Error(err)
return
}
2025-08-05 07:08:07 +00:00
var knowledge types.Knowledge
if err := c.ShouldBindJSON(&knowledge); err != nil {
logger.Error(ctx, "Failed to parse request parameters", err)
c.Error(errors.NewBadRequestError(err.Error()))
return
}
knowledge.ID = id
2025-08-05 07:08:07 +00:00
if err := h.kgService.UpdateKnowledge(effCtx, &knowledge); err != nil {
2025-08-05 07:08:07 +00:00
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError(err.Error()))
return
}
updated, getErr := h.kgService.GetKnowledgeByID(effCtx, id)
if getErr != nil {
logger.Warnf(ctx, "Knowledge updated but failed to reload status for %s: %v", id, getErr)
}
2025-08-05 07:08:07 +00:00
logger.Infof(ctx, "Knowledge updated successfully, knowledge ID: %s", id)
2025-08-05 07:08:07 +00:00
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "Knowledge updated successfully",
"data": updated,
2025-08-05 07:08:07 +00:00
})
}
// RegenerateKnowledgeSummary refreshes a stale summary after chunk or metadata edits.
func (h *KnowledgeHandler) RegenerateKnowledgeSummary(c *gin.Context) {
ctx := c.Request.Context()
id := secutils.SanitizeForLog(c.Param("id"))
if id == "" {
c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty"))
return
}
_, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleEditor)
if err != nil {
c.Error(err)
return
}
knowledge, err := h.kgService.GetKnowledgeByID(effCtx, id)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError(err.Error()))
return
}
if knowledge.SummaryStatus == "" || knowledge.SummaryStatus == types.SummaryStatusNone {
knowledge, err = h.kgService.RegenerateKnowledgeSummary(effCtx, id)
} else {
err = h.kgService.RequestKnowledgeSummaryRefresh(effCtx, id)
if err == nil {
knowledge, err = h.kgService.GetKnowledgeByID(effCtx, id)
}
}
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewBadRequestError(err.Error()))
return
}
c.JSON(http.StatusOK, gin.H{"success": true, "data": knowledge})
}
// UpdateManualKnowledge godoc
// @Summary 更新手工知识
// @Description 更新手工录入的Markdown知识内容
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param id path string true "知识ID"
// @Param request body types.ManualKnowledgePayload true "手工知识内容"
// @Success 200 {object} map[string]interface{} "更新后的知识"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge/manual/{id} [put]
func (h *KnowledgeHandler) UpdateManualKnowledge(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start updating manual knowledge")
id := secutils.SanitizeForLog(c.Param("id"))
if id == "" {
logger.Error(ctx, "Knowledge ID is empty")
c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty"))
return
}
_, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleEditor)
if err != nil {
c.Error(err)
return
}
var req types.ManualKnowledgePayload
if err := c.ShouldBindJSON(&req); err != nil {
logger.Error(ctx, "Failed to parse manual knowledge update request", err)
c.Error(errors.NewBadRequestError(err.Error()))
return
}
knowledge, err := h.kgService.UpdateManualKnowledge(effCtx, id, &req)
if err != nil {
if appErr, ok := errors.IsAppError(err); ok {
c.Error(appErr)
return
}
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"knowledge_id": id,
})
c.Error(errors.NewInternalServerError(err.Error()))
return
}
logger.Infof(ctx, "Manual knowledge updated successfully, knowledge ID: %s", id)
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": knowledge,
})
}
// ReparseKnowledge godoc
// @Summary 重新解析知识
// @Description 删除知识中现有的文档内容并重新解析,使用异步任务方式处理
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param id path string true "知识ID"
// @Param body body object false "可选的处理配置覆盖:{\"process_config\": KnowledgeProcessOverrides}"
// @Success 200 {object} map[string]interface{} "重新解析任务已提交"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Failure 403 {object} errors.AppError "权限不足"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge/{id}/reparse [post]
func (h *KnowledgeHandler) ReparseKnowledge(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start re-parsing knowledge")
id := secutils.SanitizeForLog(c.Param("id"))
if id == "" {
logger.Error(ctx, "Knowledge ID is empty")
c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty"))
return
}
// Validate KB access with editor permission (reparse requires write access)
_, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleEditor)
if err != nil {
c.Error(err)
return
}
// Optional per-reparse parse config override. Empty body keeps the
// overrides stored at upload time.
var processOverrides *types.KnowledgeProcessOverrides
if c.Request.ContentLength != 0 {
var req struct {
ProcessConfig *types.KnowledgeProcessOverrides `json:"process_config"`
}
if err := c.ShouldBindJSON(&req); err != nil {
logger.Error(ctx, "Failed to parse reparse request body", err)
c.Error(errors.NewBadRequestError("Invalid reparse request body").WithDetails(err.Error()))
return
}
processOverrides = req.ProcessConfig
}
// Call service to reparse knowledge
knowledge, err := h.kgService.ReparseKnowledge(effCtx, id, processOverrides)
if err != nil {
if appErr, ok := errors.IsAppError(err); ok {
c.Error(appErr)
return
}
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"knowledge_id": id,
})
c.Error(errors.NewInternalServerError(err.Error()))
return
}
logger.Infof(ctx, "Knowledge reparse task submitted successfully, knowledge ID: %s", id)
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "Knowledge reparse task submitted",
"data": knowledge,
})
}
feat: add knowledge parse cancellation with finalizing post-process state Lets users stop an in-flight document parse to free up LLM / worker resources without losing the chunks and index already written. The core insight is that the previous parse_status=completed flipped as soon as primary chunks landed, while the most expensive subtasks (graph extract = N LLM calls per chunk, plus summary, question generation) were still running in the background — so "completed" wasn't actually terminal from a resource standpoint. State machine pending -> processing -> finalizing -> completed | +-> cancelled (any of the three in-flight states) +-> failed +-> deleting `finalizing` is the new post-process fan-out window. parse_status only promotes to `completed` once pending_subtasks_count (a new column tracking summary + question + per-chunk graph extract) drains to zero via atomic FinalizeSubtask. Wiki ingest is intentionally excluded from the counter — it's a KB-scoped debounced batch and would otherwise pin parse_status in `finalizing` for the wiki batch window. Backend - New ParseStatusFinalizing + pending_subtasks_count column with migration 000056. - knowledgeRepository.SetFinalizing transitions processing -> finalizing conditionally so a racing cancel cannot be clobbered. - knowledgeRepository.FinalizeSubtask atomically decrements the counter and self-promotes the row to completed when it hits zero. - KnowledgePostProcess restructured to compute expected subtask count up front, flip to finalizing (or completed when no enrichment is enabled), and only then fan out subtasks. Subtask handlers (summary, question, graph extract) defer-decrement on terminal exit using the existing isFinalAsynqAttempt convention. - New POST /api/v1/knowledge/{id}/cancel-parse handler accepting pending / processing / finalizing. Marks the row cancelled, zeroes the counter, best-effort dequeues asynq tasks via a new TaskInspector abstraction (asynq-mode walks pending/scheduled/ retry queues; Lite-mode noop), and scrubs wiki ingest pending op. - SpanTracker.AbortAttempt flat-sweeps every still-running span for the attempt via a new repo.CancelAllOpenSpans helper so the trace viewer's striped bars all flip to cancelled, even leaf generations whose parent stage already EndSpan'd (multimodal fan-out pattern). knowledge_post_process closes its postSpan via SkipSpan on the cancel/deleting entry guard so a worker that opens a span AFTER the cancel sweep doesn't leak it. - Housekeeping and resetPendingTasks sweep finalizing rows identically to processing so a crash/restart can't strand them. - DeleteKnowledge/DeleteKnowledgeList proactively dequeue downstream tasks via the same TaskInspector path. - ChunkExtractService gets a cancel entry guard so the most expensive enrichment (graph extract) bails immediately when the parent knowledge is aborted. Frontend - New cancelKnowledgeParse API client + "Stop parsing" entry in both list view and card view more menus, gated on pending/processing/finalizing. - Polling predicate refactored to a shared isParseInFlight helper that recognises `finalizing` (previously the doc list silently stopped polling once parse_status flipped from processing). - Knowledge processing timeline: isPolling includes finalizing, new isHardTerminal short-circuits LIVE for cancelled/failed/ completed so stranded child spans cannot pin LIVE on. - DocumentListView.computeStatus distinguishes finalizing ("增强中") from completed and shows the previous "生成摘要中" copy when summary_status is still pending under finalizing. Added cancelled badge as well. - i18n: statusFinalizing / statusCancelled / cancelParse* keys across zh-CN, en-US, ko-KR, ru-RU. Docs / SDK - docs/api/knowledge.md: documents the new finalizing state, cancel-parse semantics, and which statuses accept cancel. - client (Go SDK): CancelKnowledgeParse with docstring listing the cancellable statuses.
2026-05-28 10:26:56 +00:00
// CancelKnowledgeParse godoc
// @Summary 取消知识解析
// @Description 取消进行中的知识解析任务。当前已写入的 chunk / 索引保留,可通过 reparse 接口重新触发解析。已完成 / 已失败 / 删除中的知识不支持取消。
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param id path string true "知识ID"
// @Success 200 {object} map[string]interface{} "取消已提交"
// @Failure 400 {object} errors.AppError "状态不支持取消"
// @Failure 403 {object} errors.AppError "权限不足"
// @Failure 404 {object} errors.AppError "知识不存在"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge/{id}/cancel-parse [post]
func (h *KnowledgeHandler) CancelKnowledgeParse(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start cancelling knowledge parse")
id := secutils.SanitizeForLog(c.Param("id"))
if id == "" {
logger.Error(ctx, "Knowledge ID is empty")
c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty"))
return
}
// Editor permission — same gate as ReparseKnowledge / DeleteKnowledge.
_, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleEditor)
if err != nil {
c.Error(err)
return
}
knowledge, err := h.kgService.CancelKnowledgeParse(effCtx, id)
if err != nil {
if appErr, ok := errors.IsAppError(err); ok {
c.Error(appErr)
return
}
logger.ErrorWithFields(ctx, err, map[string]interface{}{
"knowledge_id": id,
})
c.Error(errors.NewInternalServerError(err.Error()))
return
}
logger.Infof(ctx, "Knowledge parse cancelled successfully, knowledge ID: %s", id)
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "Knowledge parse cancelled",
"data": knowledge,
})
}
type knowledgeTagBatchRequest struct {
feat: support multi-tag for document knowledge (#1790) * feat: support multi-tag for document knowledge - Add knowledge_tag_relations join table for many-to-many tag-knowledge mapping - Migrate existing single tag_id data to the new table - Update Knowledge struct, filter, payload types (TagID -> Tags/TagIDs) - Rewrite repository layer: SetKnowledgeTags, GetKnowledgeTags, ListIDsByTagIDs - Update tag count references to query knowledge_tag_relations - Update service/handler layers for multi-tag create/list/batch-update - Frontend: multi-select tag sidebar, multi-tag chip display, tag toggle in popup - FAQ tags remain single-tag model (unchanged) * feat(knowledge): tag edit dialog with dynamic overflow and inline creation - Add TagEditDialog component with search, multi-select, and confirm - Replace inline t-popup tag selector with dialog-based editing - Tags overflow with +N badge, visible count adapts to container width via ResizeObserver - Date/card layout: fixed date width, tags flex to fill remaining space - Add persistent new-tag input bar in dialog (Enter to create, auto-select) - Tag creation emits callback to refresh parent tag list immediately - Apply same overflow + dialog trigger to both card view and list view - Add i18n keys: tagEditDialogTitle, tagEditSearch, tagSelectedCount, tagOverflowTip, tagNewPlaceholder * fix: exclude soft-deleted knowledge from tag reference counts CountReferences, BatchCountReferences, and DeleteUnusedTags now JOIN with knowledges table to filter out soft-deleted records when counting knowledge_tag_relations. Previously they queried knowledge_tag_relations directly without the deleted_at check, causing stale tag counts after knowledge deletion. * fix: refresh tag counts after async knowledge deletion completes Add a second delayed tag list refresh (2.5s) after tag deletion to capture updated reference counts for remaining tags. The original immediate refresh handles removing the deleted tag from the list, while this delayed refresh ensures other tags' counts are correct after the async deletion finishes. --------- Co-authored-by: lyingbug <xiayula2014@gmail.com>
2026-06-25 16:18:48 +00:00
Updates map[string][]string `json:"updates" binding:"required,min=1"`
KBID string `json:"kb_id"` // Optional: scope to this KB (validates editor access and uses effective tenant for shared KB)
}
// UpdateKnowledgeTagBatch godoc
// @Summary 批量更新知识标签
// @Description 批量更新知识条目的标签。可选 kb_id:指定时按该知识库校验编辑权限并用于共享知识库的空间解析
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param request body object true "标签更新请求(updates 必填,kb_id 可选)"
// @Success 200 {object} map[string]interface{} "更新成功"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge/tags [put]
func (h *KnowledgeHandler) UpdateKnowledgeTagBatch(c *gin.Context) {
ctx := c.Request.Context()
// Ensure tenant ID is in context (service reads it; may be missing if request context was not set by auth)
tenantID := c.GetUint64(types.TenantIDContextKey.String())
if tenantID == 0 {
c.Error(errors.NewUnauthorizedError("Unauthorized"))
return
}
ctx = context.WithValue(ctx, types.TenantIDContextKey, tenantID)
var req knowledgeTagBatchRequest
if err := c.ShouldBindJSON(&req); err != nil {
logger.Error(ctx, "Failed to parse knowledge tag batch request", err)
c.Error(errors.NewBadRequestError("请求参数不合法").WithDetails(err.Error()))
return
}
// Resolve effective tenant and the authorized KB scope.
var authorizedKBID string
if kbID := secutils.SanitizeForLog(req.KBID); kbID != "" {
_, _, effID, permission, err := h.validateKnowledgeBaseAccessWithKBID(c, kbID)
if err != nil {
c.Error(err)
return
}
if permission != types.OrgRoleAdmin && permission != types.OrgRoleEditor {
c.Error(errors.NewForbiddenError("No permission to update knowledge tags"))
return
}
authorizedKBID = kbID
ctx = context.WithValue(ctx, types.TenantIDContextKey, effID)
} else if len(req.Updates) > 0 {
// No kb_id: infer from first knowledge ID so shared-KB updates work without client sending kb_id
var firstKnowledgeID string
for id := range req.Updates {
firstKnowledgeID = id
break
}
if firstKnowledgeID != "" {
knowledge, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, firstKnowledgeID, types.OrgRoleEditor)
if err != nil {
c.Error(err)
return
}
authorizedKBID = knowledge.KnowledgeBaseID
ctx = effCtx
}
}
if err := h.kgService.UpdateKnowledgeTagBatch(ctx, authorizedKBID, req.Updates); err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(err)
return
}
c.JSON(http.StatusOK, gin.H{
"success": true,
})
}
// UpdateImageInfo godoc
// @Summary 更新图像信息
// @Description 更新知识分块的图像信息
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param id path string true "知识ID"
// @Param chunk_id path string true "分块ID"
// @Param request body object{image_info=string} true "图像信息"
// @Success 200 {object} map[string]interface{} "更新成功"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge/image/{id}/{chunk_id} [put]
2025-08-05 07:08:07 +00:00
func (h *KnowledgeHandler) UpdateImageInfo(c *gin.Context) {
ctx := c.Request.Context()
logger.Info(ctx, "Start updating image info")
id := secutils.SanitizeForLog(c.Param("id"))
2025-08-05 07:08:07 +00:00
if id == "" {
logger.Error(ctx, "Knowledge ID is empty")
c.Error(errors.NewBadRequestError("Knowledge ID cannot be empty"))
return
}
chunkID := secutils.SanitizeForLog(c.Param("chunk_id"))
if chunkID == "" {
2025-08-05 07:08:07 +00:00
logger.Error(ctx, "Chunk ID is empty")
c.Error(errors.NewBadRequestError("Chunk ID cannot be empty"))
return
}
_, effCtx, err := h.resolveKnowledgeAndValidateKBAccess(c, id, types.OrgRoleEditor)
if err != nil {
c.Error(err)
return
}
2025-08-05 07:08:07 +00:00
var request struct {
ImageInfo string `json:"image_info"`
}
if err := c.ShouldBindJSON(&request); err != nil {
logger.Error(ctx, "Failed to parse request parameters", err)
c.Error(errors.NewBadRequestError(err.Error()))
return
}
logger.Infof(ctx, "Updating knowledge chunk, knowledge ID: %s, chunk ID: %s", id, chunkID)
err = h.kgService.UpdateImageInfo(effCtx, id, chunkID, secutils.SanitizeForLog(request.ImageInfo))
2025-08-05 07:08:07 +00:00
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError(err.Error()))
return
}
logger.Infof(ctx, "Knowledge chunk updated successfully, knowledge ID: %s, chunk ID: %s", id, chunkID)
2025-08-05 07:08:07 +00:00
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "Knowledge chunk image updated successfully",
})
}
// SearchKnowledge godoc
// @Summary Search knowledge
// @Description Search knowledge files by keyword. Pass recent=true without a keyword to browse recent files. When agent_id is set (shared agent), scope is the agent's configured knowledge bases.
// @Tags Knowledge
// @Accept json
// @Produce json
// @Param keyword query string false "Keyword to search"
// @Param offset query int false "Offset for pagination"
// @Param limit query int false "Limit for pagination (default 20)"
// @Param file_types query string false "Comma-separated file extensions to filter (e.g., csv,xlsx)"
// @Param agent_id query string false "Shared agent ID (search within agent's KB scope)"
// @Param recent query bool false "Return recent files when keyword is empty"
// @Success 200 {object} map[string]interface{} "Search results"
// @Failure 400 {object} errors.AppError "Invalid request"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge/search [get]
func (h *KnowledgeHandler) SearchKnowledge(c *gin.Context) {
ctx := c.Request.Context()
if userID, ok := c.Get(types.UserIDContextKey.String()); ok {
ctx = context.WithValue(ctx, types.UserIDContextKey, userID)
}
fix(handler): /knowledge/search · accept ?query= and reject empty keyword Silent-failure footgun in SearchKnowledge handler: keyword := c.Query("keyword") // only "keyword" recognized // empty keyword → service falls back to default listing (created_at DESC) // → caller sees "all queries return the same N newest knowledges" This is exactly what happened to a downstream agent (Support GPT) — it was sending ?query=... (the parameter name every MCP / semantic-search tool uses) and getting a constant 5-card listing back regardless of input. The caller diagnosed it as "WeKnora embedding has degraded · vector channel returns constant rankings", but actually the parameter was never read, the search service got keyword="", and the SQL came back as a plain list-by-created_at. The handler stays a filename/title LIKE search (its real purpose is the document-management UI). Two tweaks: 1. Accept ?query= as an alias for ?keyword= · this is what MCP / agent integrations actually send. Cheap compatibility win, zero downside. 2. Empty keyword/query → 400 with a clear hint. Refusing the request is strictly better than silently returning a default listing that looks like "search results" but isn't query-driven. Separately worth noting: callers who actually want semantic recall should use POST /api/v1/knowledge-bases/{kb_id}/hybrid-search · which is the real vector + BM25 endpoint. This handler was never the right tool for RAG queries · the silent failure just made it look like one. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-06-05 00:01:02 +00:00
// Accept both ?keyword= (legacy / upstream name) and ?query= (what most
// MCP / agent integrations send). Empty input is only valid for an explicit
// recent-file browse request; ordinary callers still get a clear 400 instead
// of silently receiving the same newest cards for every missing query.
keyword := c.Query("keyword")
fix(handler): /knowledge/search · accept ?query= and reject empty keyword Silent-failure footgun in SearchKnowledge handler: keyword := c.Query("keyword") // only "keyword" recognized // empty keyword → service falls back to default listing (created_at DESC) // → caller sees "all queries return the same N newest knowledges" This is exactly what happened to a downstream agent (Support GPT) — it was sending ?query=... (the parameter name every MCP / semantic-search tool uses) and getting a constant 5-card listing back regardless of input. The caller diagnosed it as "WeKnora embedding has degraded · vector channel returns constant rankings", but actually the parameter was never read, the search service got keyword="", and the SQL came back as a plain list-by-created_at. The handler stays a filename/title LIKE search (its real purpose is the document-management UI). Two tweaks: 1. Accept ?query= as an alias for ?keyword= · this is what MCP / agent integrations actually send. Cheap compatibility win, zero downside. 2. Empty keyword/query → 400 with a clear hint. Refusing the request is strictly better than silently returning a default listing that looks like "search results" but isn't query-driven. Separately worth noting: callers who actually want semantic recall should use POST /api/v1/knowledge-bases/{kb_id}/hybrid-search · which is the real vector + BM25 endpoint. This handler was never the right tool for RAG queries · the silent failure just made it look like one. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-06-05 00:01:02 +00:00
if keyword == "" {
keyword = c.Query("query")
}
recent, _ := strconv.ParseBool(c.DefaultQuery("recent", "false"))
if strings.TrimSpace(keyword) == "" && !recent {
fix(handler): /knowledge/search · accept ?query= and reject empty keyword Silent-failure footgun in SearchKnowledge handler: keyword := c.Query("keyword") // only "keyword" recognized // empty keyword → service falls back to default listing (created_at DESC) // → caller sees "all queries return the same N newest knowledges" This is exactly what happened to a downstream agent (Support GPT) — it was sending ?query=... (the parameter name every MCP / semantic-search tool uses) and getting a constant 5-card listing back regardless of input. The caller diagnosed it as "WeKnora embedding has degraded · vector channel returns constant rankings", but actually the parameter was never read, the search service got keyword="", and the SQL came back as a plain list-by-created_at. The handler stays a filename/title LIKE search (its real purpose is the document-management UI). Two tweaks: 1. Accept ?query= as an alias for ?keyword= · this is what MCP / agent integrations actually send. Cheap compatibility win, zero downside. 2. Empty keyword/query → 400 with a clear hint. Refusing the request is strictly better than silently returning a default listing that looks like "search results" but isn't query-driven. Separately worth noting: callers who actually want semantic recall should use POST /api/v1/knowledge-bases/{kb_id}/hybrid-search · which is the real vector + BM25 endpoint. This handler was never the right tool for RAG queries · the silent failure just made it look like one. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-06-05 00:01:02 +00:00
c.Error(errors.NewBadRequestError("missing search keyword: pass ?keyword=... or ?query=..."))
return
}
keyword = strings.TrimSpace(keyword)
offset, _ := strconv.Atoi(c.DefaultQuery("offset", "0"))
limit, _ := strconv.Atoi(c.DefaultQuery("limit", "20"))
var fileTypes []string
if fileTypesStr := c.Query("file_types"); fileTypesStr != "" {
for _, ft := range strings.Split(fileTypesStr, ",") {
ft = strings.TrimSpace(ft)
if ft != "" {
fileTypes = append(fileTypes, ft)
}
}
}
agentID := c.Query("agent_id")
if agentID != "" {
userIDVal, ok := c.Get(types.UserIDContextKey.String())
if !ok {
c.Error(errors.NewUnauthorizedError("user ID not found"))
return
}
feat(rbac): lift org membership from per-user to per-tenant (Plan 3, #1303) Before this change, the `organization_members` table keyed on user_id — joining an org as a Viewer in tenant T meant only that one user got visibility on shared KBs/agents, while the rest of T (including its Owner) saw nothing. The mismatch surfaced as confusing UX bugs: "my colleague shared this with us, why can't I see it?" This change replaces `organization_members` with `organization_tenant_members`. Membership and Org-RBAC role now key on (org_id, tenant_id); `representative_user_id` stays as a display-only label of the user who first brought the tenant in. Permission resolution gains a 3-D cap so tenant-RBAC stays honest under cross-tenant sharing: effective = min(share.Permission, tenant_org_role, tenant_role_cap) where tenant_role_cap pins tenant Viewers to OrgRoleViewer regardless of the org-level grant. A Viewer in their own tenant cannot edit a shared KB even if the org granted Editor — preserves the "Viewer = read-only everywhere" promise. Migration 000045 backfills `organization_tenant_members` from the old table by collapsing each (org, tenant) group to its max role, then renames the old table to `organization_members_pre_plan3` for safe rollback. Sqlite mirror (Lite mode) is fresh-bootstrap: the init schema replaces the old table directly. Routes: member-management URLs change from `:user_id` to `:tenant_id`. Org create / join / leave now require Admin+ in the caller's tenant — joining an Org affects everyone in T, so the decision can't sit with a Viewer. Frontend: OrganizationSettingsModal + store update to address members by tenant_id.
2026-05-15 13:49:54 +00:00
_ = userIDVal
currentTenantID := c.GetUint64(types.TenantIDContextKey.String())
if currentTenantID == 0 {
c.Error(errors.NewUnauthorizedError("workspace ID not found"))
return
}
feat(rbac): lift org membership from per-user to per-tenant (Plan 3, #1303) Before this change, the `organization_members` table keyed on user_id — joining an org as a Viewer in tenant T meant only that one user got visibility on shared KBs/agents, while the rest of T (including its Owner) saw nothing. The mismatch surfaced as confusing UX bugs: "my colleague shared this with us, why can't I see it?" This change replaces `organization_members` with `organization_tenant_members`. Membership and Org-RBAC role now key on (org_id, tenant_id); `representative_user_id` stays as a display-only label of the user who first brought the tenant in. Permission resolution gains a 3-D cap so tenant-RBAC stays honest under cross-tenant sharing: effective = min(share.Permission, tenant_org_role, tenant_role_cap) where tenant_role_cap pins tenant Viewers to OrgRoleViewer regardless of the org-level grant. A Viewer in their own tenant cannot edit a shared KB even if the org granted Editor — preserves the "Viewer = read-only everywhere" promise. Migration 000045 backfills `organization_tenant_members` from the old table by collapsing each (org, tenant) group to its max role, then renames the old table to `organization_members_pre_plan3` for safe rollback. Sqlite mirror (Lite mode) is fresh-bootstrap: the init schema replaces the old table directly. Routes: member-management URLs change from `:user_id` to `:tenant_id`. Org create / join / leave now require Admin+ in the caller's tenant — joining an Org affects everyone in T, so the decision can't sit with a Viewer. Frontend: OrganizationSettingsModal + store update to address members by tenant_id.
2026-05-15 13:49:54 +00:00
callerTenantRole := types.TenantRoleFromContext(ctx)
requestedSourceTenantID, parseErr := types.ParseAgentSourceTenantID(c.Query(types.AgentSourceTenantIDParam))
if parseErr != nil {
c.Error(errors.NewBadRequestError(parseErr.Error()))
return
}
agent, err := h.agentShareService.GetSharedAgentForTenant(ctx, currentTenantID, callerTenantRole, agentID, requestedSourceTenantID)
if err != nil {
if goerrors.Is(err, service.ErrAgentShareNotFound) || goerrors.Is(err, service.ErrAgentSharePermission) || goerrors.Is(err, service.ErrAgentNotFoundForShare) {
c.Error(errors.NewForbiddenError("no permission for this shared agent"))
return
}
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError("Failed to verify shared agent access").WithDetails(err.Error()))
return
}
sourceTenantID := agent.TenantID
mode := agent.Config.KBSelectionMode
if mode == "none" {
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": []interface{}{},
"has_more": false,
"total": 0,
})
return
}
var scopes []types.KnowledgeSearchScope
if mode == "selected" && len(agent.Config.KnowledgeBases) > 0 {
for _, kbID := range agent.Config.KnowledgeBases {
if kbID != "" {
scopes = append(scopes, types.KnowledgeSearchScope{TenantID: sourceTenantID, KBID: kbID})
}
}
}
if len(scopes) == 0 {
kbs, err := h.kbService.ListKnowledgeBasesByTenantID(ctx, sourceTenantID)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError("Failed to list knowledge bases").WithDetails(err.Error()))
return
}
// `all` mode: authoritative server-side capability filter. Mirrors the
// logic in ListKnowledgeBases so @file search, KB listing, and runtime
fix(agent): exclude wiki-only KBs from quick-answer (RAG) mode Quick-answer agent mode retrieves purely through vector/keyword chunk search and ships with no `allowed_tools`, so the existing capability filter (which only reads from `allowed_tools`) let wiki-only KBs through in every entry point. End result: users could @-mention, select, and receive suggested questions from wiki-only KBs in quick-answer mode, but the underlying retrieval always returned empty. Treat "RAG-only" as an implicit property of `agent_mode = quick-answer` and union it with the tool-derived filter. The same predicate is now used everywhere the user can pick or be steered toward a KB: Backend - `tools.DeriveKBFilterForAgent` / `KBSatisfiesAgentRequirements` layer the implicit quick-answer requirement on top of tool derivation. - `ListKnowledgeBases`, `SearchKnowledge` (shared-agent `@file`), `resolveKnowledgeBasesFromAgent` (chat runtime), `/search` IM command, and `GetSuggestedQuestions` now all use the agent-mode-aware variant. - `GetSuggestedQuestions` also skips the wiki-page fallback for quick-answer agents to cover the `selected` / explicit-kb-ids paths where a wiki-only KB could still slip through. Frontend - `deriveKbFilterForAgent` / `kbSatisfiesAgentRequirements` mirror the Go helpers. - `@` mention dropdown (`Input-field.vue`) uses the new helper. - Agent editor's "specified KB" picker (`AgentEditorModal.vue`) grays out wiki-only KBs for quick-answer agents with a tooltip, and the pre-save warning fires for quick-answer mode too. - i18n: add `agentEditor.agentType.kbMismatch.quickAnswer` across all four locales.
2026-05-12 07:41:47 +00:00
// all agree on what "mode=all" actually means for this agent. The
// filter is agent-mode aware so quick-answer (RAG-only) skips
// wiki-only KBs even though it has no `allowed_tools`.
filter := tools.DeriveKBFilterForAgent(agent.Config.AgentMode, agent.Config.AllowedTools)
removed := 0
for _, kb := range kbs {
if kb == nil || kb.Type != types.KnowledgeBaseTypeDocument {
continue
}
fix(agent): exclude wiki-only KBs from quick-answer (RAG) mode Quick-answer agent mode retrieves purely through vector/keyword chunk search and ships with no `allowed_tools`, so the existing capability filter (which only reads from `allowed_tools`) let wiki-only KBs through in every entry point. End result: users could @-mention, select, and receive suggested questions from wiki-only KBs in quick-answer mode, but the underlying retrieval always returned empty. Treat "RAG-only" as an implicit property of `agent_mode = quick-answer` and union it with the tool-derived filter. The same predicate is now used everywhere the user can pick or be steered toward a KB: Backend - `tools.DeriveKBFilterForAgent` / `KBSatisfiesAgentRequirements` layer the implicit quick-answer requirement on top of tool derivation. - `ListKnowledgeBases`, `SearchKnowledge` (shared-agent `@file`), `resolveKnowledgeBasesFromAgent` (chat runtime), `/search` IM command, and `GetSuggestedQuestions` now all use the agent-mode-aware variant. - `GetSuggestedQuestions` also skips the wiki-page fallback for quick-answer agents to cover the `selected` / explicit-kb-ids paths where a wiki-only KB could still slip through. Frontend - `deriveKbFilterForAgent` / `kbSatisfiesAgentRequirements` mirror the Go helpers. - `@` mention dropdown (`Input-field.vue`) uses the new helper. - Agent editor's "specified KB" picker (`AgentEditorModal.vue`) grays out wiki-only KBs for quick-answer agents with a tooltip, and the pre-save warning fires for quick-answer mode too. - i18n: add `agentEditor.agentType.kbMismatch.quickAnswer` across all four locales.
2026-05-12 07:41:47 +00:00
if !filter.IsEmpty() && !tools.KBSatisfiesAgentRequirements(kb.Capabilities(), agent.Config.AgentMode, agent.Config.AllowedTools) {
removed++
continue
}
scopes = append(scopes, types.KnowledgeSearchScope{TenantID: sourceTenantID, KBID: kb.ID})
}
if removed > 0 {
logger.Infof(ctx,
fix(agent): exclude wiki-only KBs from quick-answer (RAG) mode Quick-answer agent mode retrieves purely through vector/keyword chunk search and ships with no `allowed_tools`, so the existing capability filter (which only reads from `allowed_tools`) let wiki-only KBs through in every entry point. End result: users could @-mention, select, and receive suggested questions from wiki-only KBs in quick-answer mode, but the underlying retrieval always returned empty. Treat "RAG-only" as an implicit property of `agent_mode = quick-answer` and union it with the tool-derived filter. The same predicate is now used everywhere the user can pick or be steered toward a KB: Backend - `tools.DeriveKBFilterForAgent` / `KBSatisfiesAgentRequirements` layer the implicit quick-answer requirement on top of tool derivation. - `ListKnowledgeBases`, `SearchKnowledge` (shared-agent `@file`), `resolveKnowledgeBasesFromAgent` (chat runtime), `/search` IM command, and `GetSuggestedQuestions` now all use the agent-mode-aware variant. - `GetSuggestedQuestions` also skips the wiki-page fallback for quick-answer agents to cover the `selected` / explicit-kb-ids paths where a wiki-only KB could still slip through. Frontend - `deriveKbFilterForAgent` / `kbSatisfiesAgentRequirements` mirror the Go helpers. - `@` mention dropdown (`Input-field.vue`) uses the new helper. - Agent editor's "specified KB" picker (`AgentEditorModal.vue`) grays out wiki-only KBs for quick-answer agents with a tooltip, and the pre-save warning fires for quick-answer mode too. - i18n: add `agentEditor.agentType.kbMismatch.quickAnswer` across all four locales.
2026-05-12 07:41:47 +00:00
"SearchKnowledge(agent=%s, mode=all): capability filter removed %d KBs",
agentID, removed)
}
}
scopes = filterKnowledgeSearchScopesForAPIKey(ctx, scopes)
if len(scopes) == 0 {
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": []interface{}{},
"has_more": false,
"total": 0,
})
return
}
knowledges, hasMore, total, err := h.kgService.SearchKnowledgeForScopes(ctx, scopes, keyword, offset, limit, fileTypes)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError("Failed to search knowledge").WithDetails(err.Error()))
return
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": knowledges,
"has_more": hasMore,
"total": total,
})
return
}
if scopes, restricted := tenantAPIKeySearchScopes(ctx); restricted {
if len(scopes) == 0 {
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": []interface{}{},
"has_more": false,
"total": 0,
})
return
}
knowledges, hasMore, total, err := h.kgService.SearchKnowledgeForScopes(ctx, scopes, keyword, offset, limit, fileTypes)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError("Failed to search knowledge").WithDetails(err.Error()))
return
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": knowledges,
"has_more": hasMore,
"total": total,
})
return
}
// Default: own + shared KBs
knowledges, hasMore, total, err := h.kgService.SearchKnowledge(ctx, keyword, offset, limit, fileTypes)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(errors.NewInternalServerError("Failed to search knowledge").WithDetails(err.Error()))
return
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": knowledges,
"has_more": hasMore,
"total": total,
})
}
// MoveKnowledgeRequest defines the request for moving knowledge items
type MoveKnowledgeRequest struct {
KnowledgeIDs []string `json:"knowledge_ids" binding:"required,min=1"`
SourceKBID string `json:"source_kb_id" binding:"required"`
TargetKBID string `json:"target_kb_id" binding:"required"`
Mode string `json:"mode" binding:"required,oneof=reuse_vectors reparse"`
}
// MoveKnowledgeResponse defines the response for move knowledge
type MoveKnowledgeResponse struct {
TaskID string `json:"task_id"`
SourceKBID string `json:"source_kb_id"`
TargetKBID string `json:"target_kb_id"`
KnowledgeCount int `json:"knowledge_count"`
Message string `json:"message"`
}
// MoveKnowledge moves knowledge items from one knowledge base to another (async task).
docs(api): add and fix swag annotations on handlers 将 swag 注解覆盖率从 ~88% 提升到 100%,并修复审计中发现的若干既有 注解 bug。 新增注解(24 个): - im.go: 4 (UpdateIMChannel/DeleteIMChannel/ToggleIMChannel/IMCallback) 其中 IMCallback 同时注册 GET 和 POST,两个方法都加了 @Router 声明。 - web_search.go: 1 (GetProviders) - web_search_provider.go: 6 - wechat_qrcode.go: 2 - weknoracloud.go: 2 - knowledge.go: 2 (MoveKnowledge/GetKnowledgeMoveProgress) - knowledgebase.go: 1 (ListMoveTargets) - organization.go: 3 (ListSharedAgents/ListOrgAgentShares/RemoveAgentShare) - mcp_service.go: 2 (SetMCPToolApproval/ResolveToolApproval) - chunker_debug.go: 1 (PreviewChunking) 修复既有注解 bug(12 处): - initialization.go: 10 个 @Router 路径与真实路由不符 (e.g. /initialization/kb/{kbId}/config → /initialization/config/{kbId} /initialization/fabri/tag GET → /initialization/extract/fabri-tag POST) - session/stream.go: ContinueStream 路径 /sessions/{session_id}/continue → /sessions/continue-stream/{session_id} - mcp_service.go: ResolveToolApproval 请求体 {approve: bool} → {decision: "approve"|"reject", reason?, modified_args?} 收敛响应 schema 到已有命名 struct(之前用 map[string]interface{}): - knowledge.go MoveKnowledge → handler.MoveKnowledgeResponse - knowledge.go GetKnowledgeMoveProgress → types.KnowledgeMoveProgress - web_search_provider.go GetProvider/UpdateProvider → types.WebSearchProviderEntity - initialization.go InitializeByKB → handler.InitializationRequest - initialization.go 三个 model-test endpoint → handler.ModelTestRequest 修正 organization.go 中 errors.AppError 应使用 apperrors 别名(该文件 stdlib errors 未别名化)。 不补 /files 和 /api/v1/files/presigned 内联闭包路由(文件服务,非业务 REST)。 Refs #890 #1049 #1168
2026-05-11 09:37:14 +00:00
//
// MoveKnowledge godoc
// @Summary 移动知识到其他知识库
// @Description 将一条或多条知识从源知识库移动到目标知识库(异步),返回任务 ID 用于查询进度
// @Tags 知识
// @Accept json
// @Produce json
// @Param request body handler.MoveKnowledgeRequest true "{source_kb_id, target_kb_id, knowledge_ids}"
// @Success 200 {object} handler.MoveKnowledgeResponse "任务信息"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge/move [post]
func (h *KnowledgeHandler) MoveKnowledge(c *gin.Context) {
ctx := c.Request.Context()
var req MoveKnowledgeRequest
if err := c.ShouldBindJSON(&req); err != nil {
logger.Error(ctx, "MoveKnowledge: failed to parse request", err)
c.Error(errors.NewBadRequestError("Invalid request parameters: " + err.Error()))
return
}
// Validate source != target
if req.SourceKBID == req.TargetKBID {
c.Error(errors.NewBadRequestError("Source and target knowledge base cannot be the same"))
return
}
tenantID, exists := c.Get(types.TenantIDContextKey.String())
if !exists {
c.Error(errors.NewUnauthorizedError("Unauthorized"))
return
}
if err := requireTenantAPIKeyKnowledgeBases(ctx, req.SourceKBID, req.TargetKBID); err != nil {
c.Error(err)
return
}
// Validate source KB
sourceKB, err := h.kbService.GetKnowledgeBaseByID(ctx, req.SourceKBID)
if err != nil {
if goerrors.Is(err, repository.ErrKnowledgeBaseNotFound) {
c.Error(errors.NewNotFoundError("Source knowledge base not found"))
return
}
c.Error(errors.NewInternalServerError(err.Error()))
return
}
if sourceKB.TenantID != tenantID.(uint64) {
c.Error(errors.NewForbiddenError("No permission to access source knowledge base"))
return
}
if err := h.requireKBOwnershipOrAdmin(c, req.SourceKBID); err != nil {
c.Error(err)
return
}
// Validate target KB
targetKB, err := h.kbService.GetKnowledgeBaseByID(ctx, req.TargetKBID)
if err != nil {
if goerrors.Is(err, repository.ErrKnowledgeBaseNotFound) {
c.Error(errors.NewNotFoundError("Target knowledge base not found"))
return
}
c.Error(errors.NewInternalServerError(err.Error()))
return
}
if targetKB.TenantID != tenantID.(uint64) {
c.Error(errors.NewForbiddenError("No permission to access target knowledge base"))
return
}
if err := h.requireKBOwnershipOrAdmin(c, req.TargetKBID); err != nil {
c.Error(err)
return
}
// Validate type match
if sourceKB.Type != targetKB.Type {
c.Error(errors.NewBadRequestError("Source and target knowledge bases must be the same type"))
return
}
// Validate embedding model match
if sourceKB.EmbeddingModelID != targetKB.EmbeddingModelID {
c.Error(errors.NewBadRequestError("Source and target must use the same embedding model"))
return
}
// reuse_vectors copies index entries directly between KBs, which only works
// inside the same VectorStore backend. A cross-store reuse_vectors move would
// route CopyIndices through the SOURCE store and then delete the source
// indices, corrupting the vector data. Reject it and point the caller at
// reparse mode, which re-indexes into the target store safely.
if req.Mode == "reuse_vectors" && !sourceKB.SharesStoreWith(targetKB) {
c.Error(errors.NewBadRequestError(
"reuse_vectors move across different vector stores is not supported; " +
"use reparse mode to move into a different store"))
return
}
// Validate all knowledge IDs belong to source KB and are in completed status
for _, kID := range req.KnowledgeIDs {
knowledge, err := h.kgService.GetKnowledgeByID(ctx, kID)
if err != nil {
c.Error(errors.NewBadRequestError(fmt.Sprintf("Knowledge item %s not found", kID)))
return
}
if knowledge.KnowledgeBaseID != req.SourceKBID {
c.Error(errors.NewBadRequestError(fmt.Sprintf("Knowledge item %s does not belong to the source knowledge base", kID)))
return
}
if knowledge.ParseStatus != types.ParseStatusCompleted {
c.Error(errors.NewBadRequestError(fmt.Sprintf("Knowledge item %s is not in completed status (current: %s)", kID, knowledge.ParseStatus)))
return
}
}
// Generate task ID
taskID := utils.GenerateTaskID("kg_move", tenantID.(uint64), req.SourceKBID)
// Create move payload
payload := types.KnowledgeMovePayload{
TenantID: tenantID.(uint64),
TaskID: taskID,
KnowledgeIDs: req.KnowledgeIDs,
SourceKBID: req.SourceKBID,
TargetKBID: req.TargetKBID,
Mode: req.Mode,
Initiator: types.TaskInitiatorFromContext(ctx),
}
feat(observability): extend Langfuse tracing across asynq pipeline Previously the Langfuse integration only traced in-process HTTP requests (chat / search / eval), so file uploads and every downstream asynq task (document parse, chunk embedding, OCR/VLM, summary / question gen, wiki ingest, datasource sync, etc.) produced either disconnected shallow traces or no observation at all. This change threads one trace end-to-end: - tracer: add SPAN observation type and StartSpan; add ResumeTrace so a worker can attach to an upstream trace without emitting a duplicate trace-create; StartGeneration now auto-picks parentObservationId from ctx so nested trace -> span -> generation trees render correctly. - types.TracingContext + LangfuseTracingCarrier: embed on all 17 asynq payloads so trace_id / parent_obs_id / user_id / session_id serialise into every job. - langfuse.InjectTracing: injected at 28 enqueue sites before json.Marshal so the HTTP-layer trace survives the Redis hop. - langfuse.AsynqMiddleware: mux.Use hook that peeks the payload, either resumes the upstream trace or opens a standalone asynq.<type> trace for scheduled jobs, and wraps the handler in a SPAN with task metadata (id / queue / retry / payload_bytes) plus ERROR level on failure. - GinMiddleware.shouldTrace: whitelist ingestion / knowledge-mutation / FAQ / wiki / datasource endpoints so the root trace actually starts. - Tests: tracer_test.go covers span nesting, error status, and ResumeTrace no-trace-create guarantee; asynq_test.go covers InjectTracing round-trip, middleware resume path, and standalone trace fallback. - Docs: docs/Langfuse\u96c6\u6210.md now lists the covered task types and documents the cross-process propagation model. No behavioural change when Langfuse is disabled (all new code paths are no-ops and carriers serialise to empty strings with omitempty).
2026-04-24 04:23:30 +00:00
langfuse.InjectTracing(ctx, &payload)
payloadBytes, err := json.Marshal(payload)
if err != nil {
logger.Errorf(ctx, "MoveKnowledge: failed to marshal payload: %v", err)
c.Error(errors.NewInternalServerError("Failed to create task"))
return
}
// Enqueue move task
task := asynq.NewTask(types.TypeKnowledgeMove, payloadBytes,
asynq.TaskID(taskID), asynq.Queue(types.QueueMaintenance),
asynq.MaxRetry(3), asynq.Timeout(2*time.Hour))
info, err := h.asynqClient.Enqueue(task)
if err != nil {
logger.Errorf(ctx, "MoveKnowledge: failed to enqueue task: %v", err)
c.Error(errors.NewInternalServerError("Failed to enqueue task"))
return
}
logger.Infof(ctx, "MoveKnowledge: task enqueued: %s, asynq_id: %s, source: %s, target: %s, count: %d",
taskID, info.ID, secutils.SanitizeForLog(req.SourceKBID), secutils.SanitizeForLog(req.TargetKBID), len(req.KnowledgeIDs))
// Save initial progress
initialProgress := &types.KnowledgeMoveProgress{
TaskID: taskID,
SourceKBID: req.SourceKBID,
TargetKBID: req.TargetKBID,
Status: types.KBCloneStatusPending,
Total: len(req.KnowledgeIDs),
Progress: 0,
Message: "Task queued, waiting to start...",
CreatedAt: time.Now().Unix(),
UpdatedAt: time.Now().Unix(),
}
if err := h.kgService.SaveKnowledgeMoveProgress(ctx, initialProgress); err != nil {
logger.Warnf(ctx, "MoveKnowledge: failed to save initial progress: %v", err)
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": MoveKnowledgeResponse{
TaskID: taskID,
SourceKBID: req.SourceKBID,
TargetKBID: req.TargetKBID,
KnowledgeCount: len(req.KnowledgeIDs),
Message: "Knowledge move task started",
},
})
}
// GetKnowledgeMoveProgress retrieves the progress of a knowledge move task.
docs(api): add and fix swag annotations on handlers 将 swag 注解覆盖率从 ~88% 提升到 100%,并修复审计中发现的若干既有 注解 bug。 新增注解(24 个): - im.go: 4 (UpdateIMChannel/DeleteIMChannel/ToggleIMChannel/IMCallback) 其中 IMCallback 同时注册 GET 和 POST,两个方法都加了 @Router 声明。 - web_search.go: 1 (GetProviders) - web_search_provider.go: 6 - wechat_qrcode.go: 2 - weknoracloud.go: 2 - knowledge.go: 2 (MoveKnowledge/GetKnowledgeMoveProgress) - knowledgebase.go: 1 (ListMoveTargets) - organization.go: 3 (ListSharedAgents/ListOrgAgentShares/RemoveAgentShare) - mcp_service.go: 2 (SetMCPToolApproval/ResolveToolApproval) - chunker_debug.go: 1 (PreviewChunking) 修复既有注解 bug(12 处): - initialization.go: 10 个 @Router 路径与真实路由不符 (e.g. /initialization/kb/{kbId}/config → /initialization/config/{kbId} /initialization/fabri/tag GET → /initialization/extract/fabri-tag POST) - session/stream.go: ContinueStream 路径 /sessions/{session_id}/continue → /sessions/continue-stream/{session_id} - mcp_service.go: ResolveToolApproval 请求体 {approve: bool} → {decision: "approve"|"reject", reason?, modified_args?} 收敛响应 schema 到已有命名 struct(之前用 map[string]interface{}): - knowledge.go MoveKnowledge → handler.MoveKnowledgeResponse - knowledge.go GetKnowledgeMoveProgress → types.KnowledgeMoveProgress - web_search_provider.go GetProvider/UpdateProvider → types.WebSearchProviderEntity - initialization.go InitializeByKB → handler.InitializationRequest - initialization.go 三个 model-test endpoint → handler.ModelTestRequest 修正 organization.go 中 errors.AppError 应使用 apperrors 别名(该文件 stdlib errors 未别名化)。 不补 /files 和 /api/v1/files/presigned 内联闭包路由(文件服务,非业务 REST)。 Refs #890 #1049 #1168
2026-05-11 09:37:14 +00:00
//
// GetKnowledgeMoveProgress godoc
// @Summary 获取知识移动进度
// @Description 按任务 ID 查询移动进度
// @Tags 知识
// @Produce json
// @Param task_id path string true "移动任务 ID"
// @Success 200 {object} types.KnowledgeMoveProgress "进度信息"
// @Failure 404 {object} errors.AppError "任务不存在"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge/move/progress/{task_id} [get]
func (h *KnowledgeHandler) GetKnowledgeMoveProgress(c *gin.Context) {
ctx := c.Request.Context()
taskID := c.Param("task_id")
if taskID == "" {
c.Error(errors.NewBadRequestError("Task ID cannot be empty"))
return
}
if err := requireTaskProgressTenant(ctx, taskID); err != nil {
c.Error(err)
return
}
progress, err := h.kgService.GetKnowledgeMoveProgress(ctx, taskID)
if err != nil {
logger.ErrorWithFields(ctx, err, nil)
c.Error(err)
return
}
c.JSON(http.StatusOK, gin.H{
"success": true,
"data": progress,
})
}
// resolveAgentAllowedKBIDs returns the set of knowledge base IDs that the
// shared agent is allowed to access based on its KBSelectionMode config.
// Returns nil when no restriction applies ("all" mode), or a concrete slice
// (possibly empty for "none" mode) when the results must be filtered.
func resolveAgentAllowedKBIDs(agent *types.CustomAgent) []string {
switch agent.Config.KBSelectionMode {
case "all":
return nil
case "none":
return []string{}
case "selected":
return agent.Config.KnowledgeBases
default:
if len(agent.Config.KnowledgeBases) > 0 {
return agent.Config.KnowledgeBases
}
return nil
}
}
feat: support multi-tag for document knowledge (#1790) * feat: support multi-tag for document knowledge - Add knowledge_tag_relations join table for many-to-many tag-knowledge mapping - Migrate existing single tag_id data to the new table - Update Knowledge struct, filter, payload types (TagID -> Tags/TagIDs) - Rewrite repository layer: SetKnowledgeTags, GetKnowledgeTags, ListIDsByTagIDs - Update tag count references to query knowledge_tag_relations - Update service/handler layers for multi-tag create/list/batch-update - Frontend: multi-select tag sidebar, multi-tag chip display, tag toggle in popup - FAQ tags remain single-tag model (unchanged) * feat(knowledge): tag edit dialog with dynamic overflow and inline creation - Add TagEditDialog component with search, multi-select, and confirm - Replace inline t-popup tag selector with dialog-based editing - Tags overflow with +N badge, visible count adapts to container width via ResizeObserver - Date/card layout: fixed date width, tags flex to fill remaining space - Add persistent new-tag input bar in dialog (Enter to create, auto-select) - Tag creation emits callback to refresh parent tag list immediately - Apply same overflow + dialog trigger to both card view and list view - Add i18n keys: tagEditDialogTitle, tagEditSearch, tagSelectedCount, tagOverflowTip, tagNewPlaceholder * fix: exclude soft-deleted knowledge from tag reference counts CountReferences, BatchCountReferences, and DeleteUnusedTags now JOIN with knowledges table to filter out soft-deleted records when counting knowledge_tag_relations. Previously they queried knowledge_tag_relations directly without the deleted_at check, causing stale tag counts after knowledge deletion. * fix: refresh tag counts after async knowledge deletion completes Add a second delayed tag list refresh (2.5s) after tag deletion to capture updated reference counts for remaining tags. The original immediate refresh handles removing the deleted tag from the list, while this delayed refresh ensures other tags' counts are correct after the async deletion finishes. --------- Co-authored-by: lyingbug <xiayula2014@gmail.com>
2026-06-25 16:18:48 +00:00
// parseCommaSeparatedTagIDs splits a comma-separated string of tag IDs and
// filters out empty strings and the "__untagged__" sentinel value.
func parseCommaSeparatedTagIDs(raw string) []string {
if raw == "" {
return nil
}
parts := strings.Split(raw, ",")
result := make([]string, 0, len(parts))
for _, p := range parts {
p = strings.TrimSpace(p)
if p == "" || p == "__untagged__" {
continue
}
result = append(result, p)
}
return result
}
feat(knowledge-base): document list filters and explicit batch-management UX Add three new optional filters to the document list under a knowledge base detail page — parse status, source/channel, and updated time range — and rework multi-select to no longer cause the card title to jitter on hover. Backend - Introduce types.KnowledgeListFilter to aggregate optional filter dimensions (tag, keyword, file_type, parse_status, source, updated_from/to) and switch ListPagedKnowledgeByKnowledgeBaseID (repository/service/interface) to accept it instead of a growing positional parameter list. - The ListKnowledge HTTP handler accepts new parse_status, source, start_time and end_time query params; time params accept RFC3339, "YYYY-MM-DD HH:MM:SS" and "YYYY-MM-DD". The repository routes source="manual"/"url" onto the type column to stay consistent with file_type semantics; other source values match the channel column. - Update the four other callers (agent_service, initialization) to pass an empty filter struct, preserving prior behavior. Frontend - Add three controls in the doc-filter-bar (status select, source select, date-range picker with future-date disabled) wired through getKnowled / listKnowledgeFiles into the new backend params. - Replace the hover-triggered card checkbox with an explicit "批量管理" mode (mirrors the session list UX): in card view the checkbox only renders while batch mode is on, entered via the per-card "..." menu; the list view keeps its leading checkbox column. Switching from list to grid auto-enables batch mode when something is already selected, so the selection stays visible. - DocumentBatchBar now stays open whenever batch mode or selection > 0, and its "取消选择" button both clears the selection and exits batch mode. API surface sync - Regenerate Swagger artifacts (docs/docs.go / swagger.json / swagger.yaml). - Update docs/api/knowledge.md with the new query parameters. - Add backward-compatible ListKnowledgeWithFilter + KnowledgeListFilter to the Go SDK; the existing ListKnowledge keeps its signature. i18n - New filter labels in zh-CN / en-US / ko-KR / ru-RU; reuse existing menu.batchManage / batchManage.cancel for the multi-select strings.
2026-05-12 10:16:34 +00:00
// parseFilterTime parses a query-string timestamp accepted by knowledge list
// filters. It supports RFC3339, RFC3339 with milliseconds, and the date-only
// "2006-01-02" form (interpreted at start of day in the local timezone).
func parseFilterTime(raw string) (time.Time, error) {
raw = strings.TrimSpace(raw)
if raw == "" {
return time.Time{}, nil
}
layouts := []string{time.RFC3339Nano, time.RFC3339, "2006-01-02 15:04:05", "2006-01-02"}
var lastErr error
for _, layout := range layouts {
if t, err := time.ParseInLocation(layout, raw, time.Local); err == nil {
return t, nil
} else {
lastErr = err
}
}
return time.Time{}, lastErr
}
func sliceContains(ss []string, target string) bool {
for _, s := range ss {
if s == target {
return true
}
}
return false
}
type batchReparseKnowledgeRequest struct {
KBID string `json:"kb_id" binding:"required"`
IDs []string `json:"ids" binding:"required"`
ProcessConfig *types.KnowledgeProcessOverrides `json:"process_config,omitempty"`
}
// BatchReparseKnowledge godoc
// @Summary 批量重新解析知识
// @Description 按 ID 列表批量重新解析单个知识库下的多个知识条目
// @Tags 知识管理
// @Accept json
// @Produce json
// @Param request body batchReparseKnowledgeRequest true "批量重解析请求"
// @Success 200 {object} map[string]interface{} "任务已提交"
// @Failure 400 {object} errors.AppError "请求参数错误"
// @Failure 403 {object} errors.AppError "权限不足"
// @Security Bearer
// @Security ApiKeyAuth
// @Router /knowledge/batch-reparse [post]
func (h *KnowledgeHandler) BatchReparseKnowledge(c *gin.Context) {
ctx := c.Request.Context()
var req batchReparseKnowledgeRequest
if err := c.ShouldBindJSON(&req); err != nil {
logger.Errorf(ctx, "failed to parse batch reparse knowledge request: %v", err)
c.Error(errors.NewBadRequestError("invalid batch reparse knowledge request parameters"))
return
}
seen := make(map[string]struct{}, len(req.IDs))
ids := make([]string, 0, len(req.IDs))
for _, raw := range req.IDs {
id := strings.TrimSpace(raw)
if id == "" {
continue
}
if _, ok := seen[id]; ok {
continue
}
seen[id] = struct{}{}
ids = append(ids, id)
}
if len(ids) == 0 {
c.Error(errors.NewBadRequestError("no knowledge IDs provided for batch reparse"))
return
}
const maxBatch = 200
if len(ids) > maxBatch {
c.Error(errors.NewBadRequestError(fmt.Sprintf("too many ids (max %d per batch)", maxBatch)))
return
}
_, kbID, effectiveTenantID, permission, err := h.validateKnowledgeBaseAccessWithKBID(c, req.KBID)
if err != nil {
c.Error(err)
return
}
if permission != types.OrgRoleAdmin && permission != types.OrgRoleEditor {
c.Error(errors.NewForbiddenError("no permission to reparse knowledge in this kb"))
return
}
ctx = context.WithValue(ctx, types.TenantIDContextKey, effectiveTenantID)
knowledgeList, err := h.kgService.GetKnowledgeBatch(ctx, effectiveTenantID, ids)
if err != nil {
logger.Errorf(ctx, "failed to get knowledge batch, kb_id: %s, size: %d, err: %v", kbID, len(ids), err)
c.Error(errors.NewInternalServerError("failed to get knowledge batch"))
return
}
if len(knowledgeList) != len(ids) {
c.Error(errors.NewBadRequestError("some knowledge entries were not found"))
return
}
for _, k := range knowledgeList {
if k.KnowledgeBaseID != kbID {
c.Error(errors.NewBadRequestError(
fmt.Sprintf("Knowledge %s does not belong to knowledge base %s",
secutils.SanitizeForLog(k.ID), secutils.SanitizeForLog(kbID))))
return
}
}
taskID, err := h.enqueueKnowledgeListReparse(ctx, effectiveTenantID, ids, req.ProcessConfig)
if err != nil {
logger.Errorf(ctx, "Failed to enqueue batch knowledge reparse task: %v", err)
c.Error(errors.NewInternalServerError("Failed to enqueue batch reparse task"))
return
}
logger.Infof(ctx, "Batch knowledge reparse task enqueued: %s, kb_id: %s, count: %d",
taskID, secutils.SanitizeForLog(kbID), len(ids))
c.JSON(http.StatusOK, gin.H{
"success": true,
"message": "Batch reparse task submitted",
"data": gin.H{
"task_id": taskID,
"reparse_count": len(ids),
},
})
}
func requireTenantAPIKeyKnowledgeBase(ctx context.Context, kbID string) error {
return requireTenantAPIKeyKnowledgeBases(ctx, kbID)
}
func requireTenantAPIKeyKnowledgeBases(ctx context.Context, kbIDs ...string) error {
return types.AuthorizeTenantAPIKeyKnowledgeBases(ctx, kbIDs...)
}
func tenantAPIKeyAllowedKBSet(ctx context.Context) map[string]bool {
scope, ok := types.TenantAPIKeyScopeFromContext(ctx)
if !ok || !scope.IsKnowledgeBaseRestricted() {
return nil
}
allowed := make(map[string]bool, len(scope.KnowledgeBaseIDs))
for _, id := range scope.KnowledgeBaseIDs {
allowed[id] = true
}
return allowed
}
func intersectKBAllowSet(base, restrict map[string]bool) map[string]bool {
if restrict == nil {
return base
}
if base == nil {
return restrict
}
out := make(map[string]bool)
for id := range base {
if restrict[id] {
out[id] = true
}
}
return out
}
func filterKnowledgesByKBAllowSet(knowledges []*types.Knowledge, allowed map[string]bool) []*types.Knowledge {
if allowed == nil {
return knowledges
}
filtered := make([]*types.Knowledge, 0, len(knowledges))
for _, k := range knowledges {
if k != nil && allowed[k.KnowledgeBaseID] {
filtered = append(filtered, k)
}
}
return filtered
}
func filterKnowledgeSearchScopesForAPIKey(ctx context.Context, scopes []types.KnowledgeSearchScope) []types.KnowledgeSearchScope {
allowed := tenantAPIKeyAllowedKBSet(ctx)
if allowed == nil {
return scopes
}
filtered := make([]types.KnowledgeSearchScope, 0, len(scopes))
for _, scope := range scopes {
if allowed[scope.KBID] {
filtered = append(filtered, scope)
}
}
return filtered
}
func tenantAPIKeySearchScopes(ctx context.Context) ([]types.KnowledgeSearchScope, bool) {
scope, ok := types.TenantAPIKeyScopeFromContext(ctx)
if !ok || !scope.IsKnowledgeBaseRestricted() {
return nil, false
}
tenantID, ok := types.TenantIDFromContext(ctx)
if !ok || tenantID == 0 {
return nil, true
}
scopes := make([]types.KnowledgeSearchScope, 0, len(scope.KnowledgeBaseIDs))
for _, kbID := range scope.KnowledgeBaseIDs {
scopes = append(scopes, types.KnowledgeSearchScope{TenantID: tenantID, KBID: kbID})
}
return scopes, true
}