WeKnora/cli/cmd/agenthelp_coverage_test.go

110 lines
3.3 KiB
Go
Raw Normal View History

package cmd
import (
"bytes"
"encoding/json"
"sort"
"testing"
"github.com/spf13/cobra"
"github.com/Tencent/WeKnora/cli/internal/cmdutil"
)
// TestEveryLeafCommandHasAgentHelp enforces the agent-first contract: every
// leaf (runnable, no subcommands) command must emit a structured AgentHelp JSON
// blob under WEKNORA_AGENT_HELP=1, so an agent never has to scrape human prose.
// Drift guard in the spirit of the K6/K7 skill-parity tests: a new leaf command
// added without SetAgentHelp fails CI here.
//
// Exemptions: cobra's generated `completion`/`help` subtrees carry no
// domain semantics worth a machine blob.
func TestEveryLeafCommandHasAgentHelp(t *testing.T) {
t.Setenv("WEKNORA_AGENT_HELP", "1")
root := NewRootCmd(cmdutil.New())
var missing []string
eachLeafCommand(root, func(c *cobra.Command) {
// Invoke the leaf's help func with the agent env set and require JSON.
var buf bytes.Buffer
c.SetOut(&buf)
c.Help()
var ah struct {
UsedFor string `json:"used_for"`
}
if err := json.Unmarshal(buf.Bytes(), &ah); err != nil || ah.UsedFor == "" {
missing = append(missing, c.CommandPath())
}
})
sort.Strings(missing)
if len(missing) > 0 {
t.Errorf("leaf commands missing agent-help JSON (register cmdutil.SetAgentHelp):\n %v\n(%d commands)",
missing, len(missing))
}
}
feat(cli)!: v0.10 reliability, agent-UX, and command-surface hardening A hardening + finalization pass over the agent-first CLI: correctness fixes, richer machine-readable signals, flag/naming consistency, and a symmetric config surface. Pre-1.0, so it includes breaking renames. Correctness: - agent update: resolve/validate --model/--rerank-model (was storing a bogus name verbatim, corrupting config.model_id). - doctor: honor WEKNORA_HOST / WEKNORA_API_KEY (headless path no longer reports "no host configured"). - session ask / MCP session_ask: text answer was empty on non-TTY — the agent stream sets Done=true on an intermediate agent_query frame before the answer, and AgentAccumulator treated the first Done as terminal. Terminate on the `complete` event (new sdk.AgentResponseTypeComplete), not a per-frame Done. - batch exit codes: any per-item failure collapses to operation.failed (exit 1), including `doc upload --recursive` partial failures — a permanent per-file failure (e.g. a duplicate) no longer surfaces as a retryable exit 7 an agent would loop on; per-item typed errors stay in the envelope. Agent-first signals & discovery: - error.exit_code in the envelope (type + exit_code disambiguate the input.invalid_argument exit-2-vs-5 split in one JSON read). - meta.hint on empty content search and on draft doc create; doc wait fails fast on a never-parsing draft instead of hanging to --timeout. - retrieval-readiness is visible in the natural flow: kb status / kb check emit retrieval_ready, and kb create hints the fix when no embedding model is bound — an unconfigured KB no longer looks silently healthy. - schema contract completeness: every leaf declares output + >=1 example (drift-guarded); output strings match the meta actually emitted; chunk list and search docs now emit meta.total_count (both previously dropped it). - schema tolerates a quoted multi-word command label; zero-state auth — and `link` with no profile — point at profile setup / the headless WEKNORA_KB_ID path instead of looping on `auth login`. - id-addressed reads tolerate a redundant --kb (doc view/wait, chunk list/view accept and ignore it, declared in schema) so a carried-over --kb doesn't exit 2; streaming commands warn that --jq does not apply to an NDJSON stream. - keep JSON-always as the default; --jq hints spell out the .data path. Consistency & gating: - doc create: drop the deprecated --name alias (--title only; pre-1.0 break). - chunk list --limit aligned to 1..10000; model list --limit/-L with has_more/total_count; api write-gates -X PUT/PATCH (exit 10); skills install expands a leading ~. - docs corrected: search docs / doc list --keyword help is case-insensitive (server does LOWER LIKE); AGENTS.md risk-action list (no phantom kb.init; add model.update / kb.config.set) and batch example (failed item carries `error`); session resume --message id comes from `message list`, not the stream. - auth/profile ergonomics: env credentials are now first-class — `auth token` prints the active WEKNORA_API_KEY / WEKNORA_TOKEN, and auth login/logout/refresh give an env-aware message instead of looping on "run auth login". `auth logout` clears credentials but keeps the profile registered (host preserved for re-login); deleting a profile is `profile remove`'s job (clean logout/remove separation, matching gh / lark). Config surface (symmetric read/write, in-place model edits): - kb config now returns a secret-free KBModelConfigView (was {}); `kb config` reads, new `kb config set` writes; `kb init` removed (misnomer). - kb create --chat-model: retrieval-ready in one step. - model update: edit a model in place (id preserved, references survive) — rotate --api-key-stdin, change base-url / display-name / etc. - session continue-stream renamed to session resume. Docs: AGENTS.md is the single wire-contract source; CHANGELOG slimmed; stale kb-init / continue-stream references removed; skill wire-vocab guard extended. AGENTS.md / weknora-shared SKILL document retrieval_ready (a KB needs an embedding model to be searchable), that --jq does not apply to NDJSON streams, and the env-credential-first auth path; the KB quickstart example now creates a retrieval-ready KB.
2026-07-02 07:53:51 +00:00
// renderAgentHelp runs a leaf's help under WEKNORA_AGENT_HELP=1 and decodes the
// machine blob (used_for / output / examples) an agent would read.
func renderAgentHelp(t *testing.T, c *cobra.Command) struct {
UsedFor string `json:"used_for"`
Output string `json:"output"`
Examples []string `json:"examples"`
} {
t.Helper()
var buf bytes.Buffer
c.SetOut(&buf)
c.Help()
var ah struct {
UsedFor string `json:"used_for"`
Output string `json:"output"`
Examples []string `json:"examples"`
}
if err := json.Unmarshal(buf.Bytes(), &ah); err != nil {
t.Fatalf("%s: agent-help is not JSON: %v", c.CommandPath(), err)
}
return ah
}
// TestEveryLeafCommandDeclaresOutput enforces that every leaf command tells an
// agent what its stdout carries. Even side-effect commands describe their
// envelope (e.g. deletes emit {id, deleted:true}); an empty Output is a contract
// gap, not a valid state. Sibling drift guard to the agent-help test above.
func TestEveryLeafCommandDeclaresOutput(t *testing.T) {
t.Setenv("WEKNORA_AGENT_HELP", "1")
root := NewRootCmd(cmdutil.New())
var missing []string
eachLeafCommand(root, func(c *cobra.Command) {
if renderAgentHelp(t, c).Output == "" {
missing = append(missing, c.CommandPath())
}
})
sort.Strings(missing)
if len(missing) > 0 {
t.Errorf("leaf commands missing agent-help Output (declare AgentHelp.Output):\n %v\n(%d commands)",
missing, len(missing))
}
}
// TestEveryLeafCommandHasExample enforces that every leaf ships at least one
// runnable example — agents learn invocation shape from examples, not prose.
func TestEveryLeafCommandHasExample(t *testing.T) {
t.Setenv("WEKNORA_AGENT_HELP", "1")
root := NewRootCmd(cmdutil.New())
var missing []string
eachLeafCommand(root, func(c *cobra.Command) {
if len(renderAgentHelp(t, c).Examples) == 0 {
missing = append(missing, c.CommandPath())
}
})
sort.Strings(missing)
if len(missing) > 0 {
t.Errorf("leaf commands missing agent-help Examples (declare AgentHelp.Examples):\n %v\n(%d commands)",
missing, len(missing))
}
}