2026-08-27 02:30:32 +00:00
|
|
|
package service
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
"bytes"
|
|
|
|
|
"os"
|
|
|
|
|
"os/exec"
|
|
|
|
|
"path/filepath"
|
|
|
|
|
"sort"
|
|
|
|
|
"strings"
|
|
|
|
|
"testing"
|
|
|
|
|
|
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
// The Go tests above pin which commands an install issues. These run the
|
|
|
|
|
// embedded checker itself, because its judgement is what decides whether a
|
|
|
|
|
// working skill installs — and a check that is too strict fails good skills
|
|
|
|
|
// just as surely as one that is too loose passes broken ones. Every case here
|
|
|
|
|
// is a shape a real skill ships.
|
|
|
|
|
func TestSkillPythonVerifier(t *testing.T) {
|
2026-08-31 03:54:04 +00:00
|
|
|
// A false environment marker is silent when packaging can evaluate it, and
|
|
|
|
|
// a note when it cannot. The install must succeed in both environments;
|
|
|
|
|
// only the note is conditional.
|
|
|
|
|
unevaluableMarkerNote := ""
|
|
|
|
|
if !pythonCanEvaluateMarkers(t) {
|
|
|
|
|
unevaluableMarkerNote = "requirements.txt declares pywin32 but it is not installed"
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-27 02:30:32 +00:00
|
|
|
cases := []struct {
|
|
|
|
|
name string
|
|
|
|
|
files map[string]string
|
2026-08-31 03:42:47 +00:00
|
|
|
// optional names the files whose findings must be reported instead of
|
|
|
|
|
// refusing the install. The install path fills this from the naming
|
|
|
|
|
// conventions the ecosystems share; here it is explicit.
|
|
|
|
|
optional []string
|
2026-08-27 02:30:32 +00:00
|
|
|
// wantProblem is a substring of the expected stderr. Empty means the
|
|
|
|
|
// tree must verify cleanly.
|
|
|
|
|
wantProblem string
|
2026-08-31 03:42:47 +00:00
|
|
|
// wantExit is the code a failing tree must exit with: 2 when installing
|
|
|
|
|
// a package would satisfy everything found, 1 when it would not. The
|
|
|
|
|
// install flow reads it to decide whether another installer round is
|
|
|
|
|
// worth its minutes.
|
|
|
|
|
wantExit int
|
|
|
|
|
// wantNote is a substring of a stdout note - something the checker
|
|
|
|
|
// reported without refusing the install.
|
|
|
|
|
wantNote string
|
2026-08-27 02:30:32 +00:00
|
|
|
}{{
|
|
|
|
|
name: "a syntax error",
|
|
|
|
|
files: map[string]string{
|
|
|
|
|
"scripts/run.py": "def broken(:\n pass\n",
|
|
|
|
|
},
|
|
|
|
|
wantProblem: "scripts/run.py has a syntax error on line 1",
|
2026-08-31 03:42:47 +00:00
|
|
|
wantExit: 1,
|
|
|
|
|
}, {
|
2026-09-02 06:33:26 +00:00
|
|
|
// Every file is parsed, not a guessed entry point: a library module the
|
|
|
|
|
// runtime only ever imports is just as fatal when it does not parse.
|
|
|
|
|
name: "a syntax error in a library module rather than an entry script",
|
2026-08-31 03:42:47 +00:00
|
|
|
files: map[string]string{
|
2026-09-02 06:33:26 +00:00
|
|
|
"scripts/run.py": "x = 1\n",
|
|
|
|
|
"scripts/helper.py": "def broken(:\n",
|
2026-08-31 03:42:47 +00:00
|
|
|
},
|
2026-09-02 06:33:26 +00:00
|
|
|
wantProblem: "scripts/helper.py has a syntax error",
|
2026-08-31 03:42:47 +00:00
|
|
|
wantExit: 1,
|
2026-08-27 02:30:32 +00:00
|
|
|
}, {
|
2026-09-02 06:33:26 +00:00
|
|
|
// The exit code is the whole verdict, so one unfixable finding among
|
|
|
|
|
// fixable ones has to sink the batch: sending the installer back for a
|
|
|
|
|
// package it can install would only delay a failure it cannot.
|
|
|
|
|
name: "a syntax error alongside a missing requirement",
|
2026-08-27 02:30:32 +00:00
|
|
|
files: map[string]string{
|
2026-09-02 06:33:26 +00:00
|
|
|
"requirements.txt": "pandas==3.0.1\n",
|
|
|
|
|
"scripts/bad.py": "def broken(:\n",
|
2026-08-27 02:30:32 +00:00
|
|
|
},
|
2026-09-02 06:33:26 +00:00
|
|
|
wantProblem: "scripts/bad.py has a syntax error",
|
2026-08-31 03:42:47 +00:00
|
|
|
wantExit: 1,
|
2026-08-27 02:30:32 +00:00
|
|
|
}, {
|
|
|
|
|
name: "a requirement the venv does not carry",
|
|
|
|
|
files: map[string]string{
|
2026-08-31 03:42:47 +00:00
|
|
|
"requirements.txt": "# pinned\npandas==3.0.1\n-r other.txt\n",
|
|
|
|
|
"scripts/run.py": "x = 1\n",
|
|
|
|
|
},
|
|
|
|
|
wantProblem: "requirements.txt declares pandas but it is not installed",
|
|
|
|
|
wantExit: 2,
|
|
|
|
|
}, {
|
|
|
|
|
// pip skips a line whose marker is false here, so refusing the install
|
|
|
|
|
// over it rejects a skill whose requirements are all present. Markers
|
|
|
|
|
// compare versions with version semantics, so they are evaluated by
|
2026-08-31 03:54:04 +00:00
|
|
|
// `packaging` or not at all. CI's system Python usually has it (the
|
|
|
|
|
// marker is false, the line is silent); a bare skill venv does not
|
|
|
|
|
// (the line is a note, not a failure). Either way the install proceeds.
|
2026-08-31 03:42:47 +00:00
|
|
|
name: "a requirement gated by an environment marker",
|
|
|
|
|
files: map[string]string{
|
|
|
|
|
"requirements.txt": "pywin32; sys_platform == \"win32\"\n" +
|
|
|
|
|
"totally_absent_package; extra == \"dev\"\n",
|
|
|
|
|
"scripts/run.py": "x = 1\n",
|
|
|
|
|
},
|
2026-08-31 03:54:04 +00:00
|
|
|
wantNote: unevaluableMarkerNote,
|
2026-08-31 03:42:47 +00:00
|
|
|
}, {
|
|
|
|
|
// An extras-gated dependency is never installed unless the extra is
|
|
|
|
|
// requested, so it is not even worth a note.
|
|
|
|
|
name: "an extras-gated requirement is not reported at all",
|
|
|
|
|
files: map[string]string{
|
|
|
|
|
"requirements.txt": "totally_absent_package; extra == \"dev\"\n",
|
|
|
|
|
"scripts/run.py": "x = 1\n",
|
|
|
|
|
},
|
|
|
|
|
}, {
|
|
|
|
|
// poetry tables carrying optional, markers or a python constraint are
|
|
|
|
|
// conditional, and poetry would not have installed them here either.
|
|
|
|
|
name: "a poetry dependency marked optional",
|
|
|
|
|
files: map[string]string{
|
|
|
|
|
"pyproject.toml": "[tool.poetry.dependencies]\npython = \"^3.11\"\n" +
|
|
|
|
|
"totally_absent_package = { version = \"^1.0\", optional = true }\n",
|
2026-08-27 02:30:32 +00:00
|
|
|
"scripts/run.py": "x = 1\n",
|
|
|
|
|
},
|
|
|
|
|
}, {
|
|
|
|
|
name: "a pyproject.toml dependency the venv does not carry",
|
|
|
|
|
files: map[string]string{
|
|
|
|
|
"pyproject.toml": "[project]\nname = \"demo\"\ndependencies = [\n" +
|
|
|
|
|
" \"totally_absent_package>=1.0\",\n]\n",
|
|
|
|
|
"scripts/run.py": "x = 1\n",
|
|
|
|
|
},
|
|
|
|
|
wantProblem: "pyproject.toml declares totally_absent_package but it is not installed",
|
2026-08-31 03:42:47 +00:00
|
|
|
wantExit: 2,
|
2026-08-27 02:30:32 +00:00
|
|
|
}, {
|
|
|
|
|
// Lines that name a distribution only indirectly cannot be checked by
|
|
|
|
|
// name, and inventing one from the URL would fail installs whose
|
|
|
|
|
// requirements are all present.
|
|
|
|
|
name: "requirements that point at a VCS, an archive or a local path",
|
|
|
|
|
files: map[string]string{
|
|
|
|
|
"requirements.txt": "git+https://example.com/x/y.git#egg=y\n" +
|
|
|
|
|
"./vendor/local-wheel.whl\n" +
|
|
|
|
|
"https://example.com/pkg-1.0.tar.gz\n" +
|
|
|
|
|
"--index-url https://example.com/simple\n",
|
|
|
|
|
"scripts/run.py": "x = 1\n",
|
|
|
|
|
},
|
perf(skills): batch install verification and live installer progress (#2960)
* perf(skills): batch install verification and keep caches under budget
The tail of an install (agent_done -> verified) issued one remote exec per
check: two for permission normalisation, one per bundled script for the tree
check, three for the scratch wipe. A skill with twenty scripts paid thirty-
odd round trips, each carrying a hidden bootstrap exec and a distributed-
lock resolve, while the console sat on a finished agent with no feedback.
- normalizeSkillPermissions sends chmod and chown as one command.
- verifySkillTree folds the SKILL.md probe and one test per script into a
single command that reports every missing file instead of stopping at the
first. Exit codes 0/1/3 carry the verdict, stderr lines are the final
messages, and the two existing error texts survive verbatim.
- cleanImageScratch merges the workspace reset, the directory restore and
the cache pass into one command whose exit code is still owned by the
strict workspace half. Package caches are no longer wiped wholesale: uv,
npm and pnpm trim their own entries, and the pip/uv/npm/pnpm totals of
both accounts are kept while under 256MB and wiped whole over it, so the
next install resolves a previous download instead of the network.
- The install prompt hands the agent absolute paths for uv, npm, pnpm, pip,
python3 and node from the same one-command probe that used to ask only
whether uv exists, so it neither burns a turn on `which` nor gambles on
the shell PATH.
Tests pin the new command shapes exactly, cover quoting of archive-chosen
file names, assert the tree check stays one command regardless of script
count, exercise the budget guard against a real shell, and keep the
workspace-restore failure semantics intact.
* fix(skills): credit a script's own sys.path bootstrap for shipped modules
The python check resolved every import against the script's directory, its
ancestors up to the skill root and the venv - and nothing else. A module the
skill ships in a sibling subtree (lib/image_video.py, imported by
scripts/generate.py) was invisible to that set, so a skill whose scripts
bootstrap their own library directory was refused for a dependency no
package provides, and the repair round that followed could not fix it
either: it installs packages and is forbidden from editing the tree. One
real install spent seven minutes discovering that, three LLM stream
timeouts included.
The verifier now scans what the tree ships and statically evaluates the
sys.path.insert/append calls each script raises before its imports - the
os.path and pathlib idioms, with variables tracked in source order. A
bootstrap proven to raise the directory that provides the module turns the
finding into a note; anything unprovable stays a refusal, now naming the
shipped file instead of pretending a dependency is missing. The refusal is
unrepairable on purpose: no installer round can fix a shipped module the
import cannot reach.
Tests pin the incident's exact shape, the pathlib idiom, and the guarantee
that makes the evaluator safe: a vendored module still cannot satisfy an
import unless the script's own bootstrap provably reaches it, and modules
pip placed under .venv never count as shipped.
* feat(skills): move the install progress bar while the installer works
The progress card sat at the seeded 35% for the whole agent phase and
only jumped at agent_done. Every tool call the transcript already sees
now advances an asymptotic curve 35 + 44·(1 − e^(−k/12)): monotonic for
any round count, converging below the agent_done anchor at 80, topping
out around 75 even when max_iterations is exhausted.
Each step publishes a one-line command summary (engine hint preferred,
80-rune cap) so the admin can see what the installer is doing. After
the first round ends the activity progress is muted — verification and
repair rounds own the stage anchors (80/82) from there, and a repair
round's tool calls must not drag the bar back under them.
* fix(skills): pin the installer agent's thinking mode off
installerAgentConfig left AgentConfig.Thinking nil, which most providers
read as "defer to the model's own default" — a thinking-capable model
would reason on every ReAct round of an install, burning latency and
completion tokens on what are short shell-command turns. Set an explicit
false so every provider path (Qwen enable_thinking, Ollama think, LKEAP
thinking type) receives a deterministic off instead of a default.
* style(skills): gofmt tenant_skill_verify_python_test.go
CI rejects aligned map keys in this table-driven test; gofmt drops the extra spaces.
* docs(skills): drop the duplicated exit 1 line in the python verifier
The shipped-module case was appended by repeating the whole exit 1 block
instead of extending the existing one.
---------
Co-authored-by: hailongzhao <hailongzhao@tencent.com>
2026-09-02 04:23:42 +00:00
|
|
|
}, {
|
2026-09-02 06:33:26 +00:00
|
|
|
// Skills ship their tests. Nothing the skill offers loads them, so a
|
|
|
|
|
// bundled tests/ directory must not decide whether the skill installs -
|
|
|
|
|
// and refusing over one throws away the dependency work that succeeded.
|
|
|
|
|
name: "a bundled test file that does not parse",
|
perf(skills): batch install verification and live installer progress (#2960)
* perf(skills): batch install verification and keep caches under budget
The tail of an install (agent_done -> verified) issued one remote exec per
check: two for permission normalisation, one per bundled script for the tree
check, three for the scratch wipe. A skill with twenty scripts paid thirty-
odd round trips, each carrying a hidden bootstrap exec and a distributed-
lock resolve, while the console sat on a finished agent with no feedback.
- normalizeSkillPermissions sends chmod and chown as one command.
- verifySkillTree folds the SKILL.md probe and one test per script into a
single command that reports every missing file instead of stopping at the
first. Exit codes 0/1/3 carry the verdict, stderr lines are the final
messages, and the two existing error texts survive verbatim.
- cleanImageScratch merges the workspace reset, the directory restore and
the cache pass into one command whose exit code is still owned by the
strict workspace half. Package caches are no longer wiped wholesale: uv,
npm and pnpm trim their own entries, and the pip/uv/npm/pnpm totals of
both accounts are kept while under 256MB and wiped whole over it, so the
next install resolves a previous download instead of the network.
- The install prompt hands the agent absolute paths for uv, npm, pnpm, pip,
python3 and node from the same one-command probe that used to ask only
whether uv exists, so it neither burns a turn on `which` nor gambles on
the shell PATH.
Tests pin the new command shapes exactly, cover quoting of archive-chosen
file names, assert the tree check stays one command regardless of script
count, exercise the budget guard against a real shell, and keep the
workspace-restore failure semantics intact.
* fix(skills): credit a script's own sys.path bootstrap for shipped modules
The python check resolved every import against the script's directory, its
ancestors up to the skill root and the venv - and nothing else. A module the
skill ships in a sibling subtree (lib/image_video.py, imported by
scripts/generate.py) was invisible to that set, so a skill whose scripts
bootstrap their own library directory was refused for a dependency no
package provides, and the repair round that followed could not fix it
either: it installs packages and is forbidden from editing the tree. One
real install spent seven minutes discovering that, three LLM stream
timeouts included.
The verifier now scans what the tree ships and statically evaluates the
sys.path.insert/append calls each script raises before its imports - the
os.path and pathlib idioms, with variables tracked in source order. A
bootstrap proven to raise the directory that provides the module turns the
finding into a note; anything unprovable stays a refusal, now naming the
shipped file instead of pretending a dependency is missing. The refusal is
unrepairable on purpose: no installer round can fix a shipped module the
import cannot reach.
Tests pin the incident's exact shape, the pathlib idiom, and the guarantee
that makes the evaluator safe: a vendored module still cannot satisfy an
import unless the script's own bootstrap provably reaches it, and modules
pip placed under .venv never count as shipped.
* feat(skills): move the install progress bar while the installer works
The progress card sat at the seeded 35% for the whole agent phase and
only jumped at agent_done. Every tool call the transcript already sees
now advances an asymptotic curve 35 + 44·(1 − e^(−k/12)): monotonic for
any round count, converging below the agent_done anchor at 80, topping
out around 75 even when max_iterations is exhausted.
Each step publishes a one-line command summary (engine hint preferred,
80-rune cap) so the admin can see what the installer is doing. After
the first round ends the activity progress is muted — verification and
repair rounds own the stage anchors (80/82) from there, and a repair
round's tool calls must not drag the bar back under them.
* fix(skills): pin the installer agent's thinking mode off
installerAgentConfig left AgentConfig.Thinking nil, which most providers
read as "defer to the model's own default" — a thinking-capable model
would reason on every ReAct round of an install, burning latency and
completion tokens on what are short shell-command turns. Set an explicit
false so every provider path (Qwen enable_thinking, Ollama think, LKEAP
thinking type) receives a deterministic off instead of a default.
* style(skills): gofmt tenant_skill_verify_python_test.go
CI rejects aligned map keys in this table-driven test; gofmt drops the extra spaces.
* docs(skills): drop the duplicated exit 1 line in the python verifier
The shipped-module case was appended by repeating the whole exit 1 block
instead of extending the existing one.
---------
Co-authored-by: hailongzhao <hailongzhao@tencent.com>
2026-09-02 04:23:42 +00:00
|
|
|
files: map[string]string{
|
2026-09-02 06:33:26 +00:00
|
|
|
"scripts/run.py": "x = 1\n",
|
|
|
|
|
"tests/conftest.py": "def broken(:\n",
|
|
|
|
|
"examples/demo.py": "def also_broken(:\n",
|
perf(skills): batch install verification and live installer progress (#2960)
* perf(skills): batch install verification and keep caches under budget
The tail of an install (agent_done -> verified) issued one remote exec per
check: two for permission normalisation, one per bundled script for the tree
check, three for the scratch wipe. A skill with twenty scripts paid thirty-
odd round trips, each carrying a hidden bootstrap exec and a distributed-
lock resolve, while the console sat on a finished agent with no feedback.
- normalizeSkillPermissions sends chmod and chown as one command.
- verifySkillTree folds the SKILL.md probe and one test per script into a
single command that reports every missing file instead of stopping at the
first. Exit codes 0/1/3 carry the verdict, stderr lines are the final
messages, and the two existing error texts survive verbatim.
- cleanImageScratch merges the workspace reset, the directory restore and
the cache pass into one command whose exit code is still owned by the
strict workspace half. Package caches are no longer wiped wholesale: uv,
npm and pnpm trim their own entries, and the pip/uv/npm/pnpm totals of
both accounts are kept while under 256MB and wiped whole over it, so the
next install resolves a previous download instead of the network.
- The install prompt hands the agent absolute paths for uv, npm, pnpm, pip,
python3 and node from the same one-command probe that used to ask only
whether uv exists, so it neither burns a turn on `which` nor gambles on
the shell PATH.
Tests pin the new command shapes exactly, cover quoting of archive-chosen
file names, assert the tree check stays one command regardless of script
count, exercise the budget guard against a real shell, and keep the
workspace-restore failure semantics intact.
* fix(skills): credit a script's own sys.path bootstrap for shipped modules
The python check resolved every import against the script's directory, its
ancestors up to the skill root and the venv - and nothing else. A module the
skill ships in a sibling subtree (lib/image_video.py, imported by
scripts/generate.py) was invisible to that set, so a skill whose scripts
bootstrap their own library directory was refused for a dependency no
package provides, and the repair round that followed could not fix it
either: it installs packages and is forbidden from editing the tree. One
real install spent seven minutes discovering that, three LLM stream
timeouts included.
The verifier now scans what the tree ships and statically evaluates the
sys.path.insert/append calls each script raises before its imports - the
os.path and pathlib idioms, with variables tracked in source order. A
bootstrap proven to raise the directory that provides the module turns the
finding into a note; anything unprovable stays a refusal, now naming the
shipped file instead of pretending a dependency is missing. The refusal is
unrepairable on purpose: no installer round can fix a shipped module the
import cannot reach.
Tests pin the incident's exact shape, the pathlib idiom, and the guarantee
that makes the evaluator safe: a vendored module still cannot satisfy an
import unless the script's own bootstrap provably reaches it, and modules
pip placed under .venv never count as shipped.
* feat(skills): move the install progress bar while the installer works
The progress card sat at the seeded 35% for the whole agent phase and
only jumped at agent_done. Every tool call the transcript already sees
now advances an asymptotic curve 35 + 44·(1 − e^(−k/12)): monotonic for
any round count, converging below the agent_done anchor at 80, topping
out around 75 even when max_iterations is exhausted.
Each step publishes a one-line command summary (engine hint preferred,
80-rune cap) so the admin can see what the installer is doing. After
the first round ends the activity progress is muted — verification and
repair rounds own the stage anchors (80/82) from there, and a repair
round's tool calls must not drag the bar back under them.
* fix(skills): pin the installer agent's thinking mode off
installerAgentConfig left AgentConfig.Thinking nil, which most providers
read as "defer to the model's own default" — a thinking-capable model
would reason on every ReAct round of an install, burning latency and
completion tokens on what are short shell-command turns. Set an explicit
false so every provider path (Qwen enable_thinking, Ollama think, LKEAP
thinking type) receives a deterministic off instead of a default.
* style(skills): gofmt tenant_skill_verify_python_test.go
CI rejects aligned map keys in this table-driven test; gofmt drops the extra spaces.
* docs(skills): drop the duplicated exit 1 line in the python verifier
The shipped-module case was appended by repeating the whole exit 1 block
instead of extending the existing one.
---------
Co-authored-by: hailongzhao <hailongzhao@tencent.com>
2026-09-02 04:23:42 +00:00
|
|
|
},
|
2026-09-02 06:33:26 +00:00
|
|
|
optional: []string{"examples/demo.py", "tests/conftest.py"},
|
|
|
|
|
wantNote: "auxiliary file; this does not fail the install",
|
2026-08-27 02:30:32 +00:00
|
|
|
}}
|
|
|
|
|
|
|
|
|
|
for _, tc := range cases {
|
|
|
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
|
|
|
root := writeSkillTree(t, tc.files)
|
2026-08-31 03:42:47 +00:00
|
|
|
entry := pythonFiles(tc.files)
|
|
|
|
|
if len(tc.optional) > 0 {
|
|
|
|
|
entry = withoutPaths(entry, tc.optional)
|
|
|
|
|
}
|
|
|
|
|
stdout, stderr, err := runSkillPythonVerifier(t, root, entry, tc.optional)
|
2026-08-27 02:30:32 +00:00
|
|
|
|
|
|
|
|
if tc.wantProblem == "" {
|
|
|
|
|
require.NoError(t, err, "this skill must install; stderr: %s", stderr)
|
|
|
|
|
require.Contains(t, stdout, "verified")
|
2026-08-31 03:42:47 +00:00
|
|
|
} else {
|
|
|
|
|
require.Error(t, err, "this skill is broken and must not reach a snapshot")
|
|
|
|
|
require.Contains(t, stderr, tc.wantProblem)
|
|
|
|
|
require.Equal(t, tc.wantExit, verifierExitCode(t, err),
|
|
|
|
|
"the exit code is what decides whether an installer round can fix this; "+
|
|
|
|
|
"stderr: %s", stderr)
|
|
|
|
|
}
|
|
|
|
|
if tc.wantNote != "" {
|
|
|
|
|
require.Contains(t, stdout, "note: ",
|
|
|
|
|
"a finding that does not refuse the install must still be reported")
|
|
|
|
|
require.Contains(t, stdout, tc.wantNote)
|
2026-08-27 02:30:32 +00:00
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-31 03:42:47 +00:00
|
|
|
func verifierExitCode(t *testing.T, err error) int {
|
|
|
|
|
t.Helper()
|
|
|
|
|
var exit *exec.ExitError
|
|
|
|
|
require.ErrorAs(t, err, &exit, "the checker must fail by exiting, not by crashing")
|
|
|
|
|
return exit.ExitCode()
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func withoutPaths(all, drop []string) []string {
|
|
|
|
|
dropped := make(map[string]struct{}, len(drop))
|
|
|
|
|
for _, rel := range drop {
|
|
|
|
|
dropped[rel] = struct{}{}
|
|
|
|
|
}
|
|
|
|
|
kept := make([]string, 0, len(all))
|
|
|
|
|
for _, rel := range all {
|
|
|
|
|
if _, ok := dropped[rel]; !ok {
|
|
|
|
|
kept = append(kept, rel)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return kept
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-27 02:30:32 +00:00
|
|
|
// Verification must be able to read a skill that writes files or opens sockets
|
|
|
|
|
// the moment it is imported, without doing either.
|
|
|
|
|
func TestSkillPythonVerifierNeverExecutesTheSkill(t *testing.T) {
|
|
|
|
|
root := writeSkillTree(t, map[string]string{
|
|
|
|
|
"scripts/run.py": "import os\n" +
|
|
|
|
|
"open(os.path.join(os.path.dirname(__file__), 'SIDE_EFFECT'), 'w').close()\n",
|
|
|
|
|
})
|
|
|
|
|
|
2026-08-31 03:42:47 +00:00
|
|
|
_, stderr, err := runSkillPythonVerifier(t, root, []string{"scripts/run.py"}, nil)
|
2026-08-27 02:30:32 +00:00
|
|
|
|
|
|
|
|
require.NoError(t, err, stderr)
|
|
|
|
|
_, statErr := os.Stat(filepath.Join(root, "scripts", "SIDE_EFFECT"))
|
|
|
|
|
require.True(t, os.IsNotExist(statErr),
|
|
|
|
|
"the checker ran the skill's module body instead of reading it")
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// The skill tree is owned by root and readable by everyone; a file the
|
|
|
|
|
// execution user cannot open is an install that would fail on first use.
|
|
|
|
|
func TestSkillPythonVerifierReportsAnUnreadableScript(t *testing.T) {
|
|
|
|
|
if os.Geteuid() == 0 {
|
|
|
|
|
t.Skip("root can read a 000 file, so this states nothing when tests run as root")
|
|
|
|
|
}
|
|
|
|
|
root := writeSkillTree(t, map[string]string{"scripts/run.py": "x = 1\n"})
|
|
|
|
|
require.NoError(t, os.Chmod(filepath.Join(root, "scripts", "run.py"), 0o000))
|
|
|
|
|
|
2026-08-31 03:42:47 +00:00
|
|
|
_, stderr, err := runSkillPythonVerifier(t, root, []string{"scripts/run.py"}, nil)
|
2026-08-27 02:30:32 +00:00
|
|
|
|
|
|
|
|
require.Error(t, err)
|
|
|
|
|
require.Contains(t, stderr, "cannot be read by the skill execution user")
|
2026-08-31 03:42:47 +00:00
|
|
|
require.Equal(t, 1, verifierExitCode(t, err),
|
|
|
|
|
"a file the execution user cannot read is not something installing a package fixes")
|
|
|
|
|
}
|
|
|
|
|
|
2026-09-02 06:33:26 +00:00
|
|
|
// The contract this checker now keeps: an import shape is never a verdict.
|
|
|
|
|
//
|
|
|
|
|
// Whether `import helper` resolves depends on what the file does to sys.path
|
|
|
|
|
// before the import runs, and no static evaluator can enumerate those idioms —
|
|
|
|
|
// a guarded insert, a path constant imported from a sibling module, a value
|
|
|
|
|
// read from the environment, a mutation inside a helper function. Every
|
|
|
|
|
// approximation refused skills that run perfectly. Proving an import resolves
|
|
|
|
|
// belongs to the installer agent, which has a root shell and the real
|
|
|
|
|
// interpreter; this pass only proves the file parses.
|
|
|
|
|
//
|
|
|
|
|
// Each case below was once a refused install. All of them must now install.
|
|
|
|
|
func TestSkillPythonVerifierNeverJudgesImports(t *testing.T) {
|
|
|
|
|
shapes := map[string]map[string]string{
|
|
|
|
|
"a package the image genuinely does not carry": {
|
|
|
|
|
"scripts/run.py": "import totally_absent_package\n",
|
|
|
|
|
},
|
|
|
|
|
"a sibling module reached only by a sys.path bootstrap": {
|
|
|
|
|
"lib/image_video.py": "def generate_image():\n pass\n",
|
|
|
|
|
"scripts/generate.py": "import sys, os\n" +
|
|
|
|
|
"sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'lib'))\n" +
|
|
|
|
|
"from image_video import generate_image\n",
|
|
|
|
|
},
|
|
|
|
|
"a sibling module with no bootstrap at all": {
|
|
|
|
|
"lib/image_video.py": "def generate_image():\n pass\n",
|
|
|
|
|
"scripts/generate.py": "from image_video import generate_image\n",
|
|
|
|
|
},
|
|
|
|
|
"a bootstrap whose argument cannot be evaluated statically": {
|
|
|
|
|
"lib/helper.py": "x = 1\n",
|
|
|
|
|
"scripts/run.py": "import sys, os\n" +
|
|
|
|
|
"sys.path.insert(0, os.environ['LIB_DIR'])\n" +
|
|
|
|
|
"import helper\n",
|
|
|
|
|
},
|
|
|
|
|
"a bootstrap written inside a helper function": {
|
|
|
|
|
"lib/helper.py": "x = 1\n",
|
|
|
|
|
"scripts/run.py": "import sys\n" +
|
|
|
|
|
"from pathlib import Path\n" +
|
|
|
|
|
"def _setup():\n" +
|
|
|
|
|
" sys.path.insert(0, str(Path(__file__).parent.parent / 'lib'))\n" +
|
|
|
|
|
"_setup()\n" +
|
|
|
|
|
"import helper\n",
|
|
|
|
|
},
|
|
|
|
|
"a path constant imported from a sibling module": {
|
|
|
|
|
"scripts/paths.py": "from pathlib import Path\n" +
|
|
|
|
|
"LIB = Path(__file__).resolve().parent.parent / 'lib'\n",
|
|
|
|
|
"lib/helper.py": "x = 1\n",
|
|
|
|
|
"scripts/run.py": "import sys\n" +
|
|
|
|
|
"from paths import LIB\n" +
|
|
|
|
|
"if str(LIB) not in sys.path:\n" +
|
|
|
|
|
" sys.path.insert(0, str(LIB))\n" +
|
|
|
|
|
"import helper\n",
|
|
|
|
|
},
|
|
|
|
|
"a vendored module sharing a distribution's name": {
|
|
|
|
|
"vendor/totally_absent_package.py": "x = 1\n",
|
|
|
|
|
"scripts/run.py": "import totally_absent_package\n",
|
|
|
|
|
},
|
|
|
|
|
"a relative import in a directory with no __init__.py": {
|
|
|
|
|
"scripts/run.py": "from .helper import go\n",
|
|
|
|
|
"scripts/helper.py": "def go():\n pass\n",
|
|
|
|
|
},
|
|
|
|
|
"a relative import reaching a module the skill does not ship": {
|
|
|
|
|
"pkg/__init__.py": "",
|
|
|
|
|
"pkg/sub/__init__.py": "",
|
|
|
|
|
"pkg/sub/run.py": "from ..missing import go\n",
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for name, files := range shapes {
|
|
|
|
|
t.Run(name, func(t *testing.T) {
|
|
|
|
|
root := writeSkillTree(t, files)
|
|
|
|
|
|
|
|
|
|
stdout, stderr, err := runSkillPythonVerifier(t, root, pythonFiles(files), nil)
|
|
|
|
|
|
|
|
|
|
require.NoError(t, err,
|
|
|
|
|
"an import shape must not refuse an install; stderr: %s", stderr)
|
|
|
|
|
require.Contains(t, stdout, "verified")
|
|
|
|
|
})
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-31 03:42:47 +00:00
|
|
|
// The layout that started this: the official office toolkit ships entry scripts
|
|
|
|
|
// beside sibling packages, and its library modules import those siblings by
|
2026-09-02 06:33:26 +00:00
|
|
|
// short name. It parses, so it installs.
|
2026-08-31 03:42:47 +00:00
|
|
|
func TestSkillPythonVerifierAcceptsTheOfficeToolkitLayout(t *testing.T) {
|
|
|
|
|
files := map[string]string{
|
|
|
|
|
"SKILL.md": "# xlsx\n",
|
|
|
|
|
"scripts/recalc.py": "import json\nimport sys\nfrom pathlib import Path\n" +
|
|
|
|
|
"from office.soffice import run_soffice\n",
|
|
|
|
|
"scripts/office/soffice.py": "import subprocess\nimport tempfile\n" +
|
|
|
|
|
"def run_soffice():\n pass\n",
|
|
|
|
|
"scripts/office/validate.py": "import argparse\n" +
|
|
|
|
|
"from helpers import safe_extract\n" +
|
|
|
|
|
"from validators import DOCXSchemaValidator\n",
|
|
|
|
|
"scripts/office/helpers/__init__.py": "import zipfile\n" +
|
|
|
|
|
"def safe_extract():\n pass\n",
|
|
|
|
|
"scripts/office/validators/__init__.py": "from .docx import DOCXSchemaValidator\n",
|
|
|
|
|
"scripts/office/validators/base.py": "import re\n" +
|
|
|
|
|
"from helpers import safe_extract\n" +
|
|
|
|
|
"class BaseSchemaValidator:\n pass\n",
|
|
|
|
|
"scripts/office/validators/docx.py": "from helpers import safe_extract\n" +
|
|
|
|
|
"from .base import BaseSchemaValidator\n" +
|
|
|
|
|
"class DOCXSchemaValidator(BaseSchemaValidator):\n pass\n",
|
|
|
|
|
"scripts/office/helpers/pptx_chart.py": "from __future__ import annotations\n" +
|
|
|
|
|
"import re\nfrom . import part_text\n",
|
|
|
|
|
}
|
|
|
|
|
root := writeSkillTree(t, files)
|
|
|
|
|
|
|
|
|
|
stdout, stderr, err := runSkillPythonVerifier(t, root, pythonFiles(files), nil)
|
|
|
|
|
|
|
|
|
|
require.NoError(t, err, "the toolkit's own layout must not be a failed install; stderr: %s", stderr)
|
|
|
|
|
require.Contains(t, stdout, "verified")
|
|
|
|
|
require.NotContains(t, stderr, "helpers",
|
|
|
|
|
"helpers is a sibling package of validators/, reachable from scripts/office/")
|
2026-08-27 02:30:32 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func writeSkillTree(t *testing.T, files map[string]string) string {
|
|
|
|
|
t.Helper()
|
|
|
|
|
root := t.TempDir()
|
|
|
|
|
for rel, content := range files {
|
|
|
|
|
full := filepath.Join(root, filepath.FromSlash(rel))
|
|
|
|
|
require.NoError(t, os.MkdirAll(filepath.Dir(full), 0o755))
|
|
|
|
|
require.NoError(t, os.WriteFile(full, []byte(content), 0o644))
|
|
|
|
|
}
|
|
|
|
|
return root
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
func pythonFiles(files map[string]string) []string {
|
|
|
|
|
var scripts []string
|
|
|
|
|
for rel := range files {
|
|
|
|
|
if strings.HasSuffix(rel, ".py") {
|
|
|
|
|
scripts = append(scripts, rel)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
sort.Strings(scripts)
|
|
|
|
|
return scripts
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-31 03:54:04 +00:00
|
|
|
// pythonCanEvaluateMarkers reports whether this interpreter has packaging, the
|
|
|
|
|
// library the checker uses for PEP 508 markers. Without it a false marker is a
|
|
|
|
|
// note rather than a skip, which is the contract a bare venv relies on.
|
|
|
|
|
func pythonCanEvaluateMarkers(t *testing.T) bool {
|
|
|
|
|
t.Helper()
|
|
|
|
|
python, err := exec.LookPath("python3")
|
|
|
|
|
if err != nil {
|
|
|
|
|
return false
|
|
|
|
|
}
|
|
|
|
|
return exec.Command(python, "-c", "from packaging.markers import Marker").Run() == nil
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-27 02:30:32 +00:00
|
|
|
// runSkillPythonVerifier feeds the embedded checker to a real interpreter the
|
|
|
|
|
// same way the sandbox command does: on stdin, with the tree and the files to
|
2026-08-31 03:42:47 +00:00
|
|
|
// check as argv, auxiliary files last behind the separator.
|
|
|
|
|
func runSkillPythonVerifier(
|
|
|
|
|
t *testing.T, root string, scripts, optional []string,
|
|
|
|
|
) (string, string, error) {
|
2026-08-27 02:30:32 +00:00
|
|
|
t.Helper()
|
|
|
|
|
python, err := exec.LookPath("python3")
|
|
|
|
|
if err != nil {
|
|
|
|
|
t.Skip("python3 is not on PATH")
|
|
|
|
|
}
|
2026-08-31 03:42:47 +00:00
|
|
|
argv := append([]string{"-", root}, scripts...)
|
|
|
|
|
if len(optional) > 0 {
|
|
|
|
|
argv = append(append(argv, skillVerifyOptionalFlag), optional...)
|
|
|
|
|
}
|
|
|
|
|
cmd := exec.Command(python, argv...)
|
2026-08-27 02:30:32 +00:00
|
|
|
cmd.Stdin = strings.NewReader(skillPythonVerifier)
|
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
|
|
|
cmd.Stdout = &stdout
|
|
|
|
|
cmd.Stderr = &stderr
|
|
|
|
|
runErr := cmd.Run()
|
|
|
|
|
return stdout.String(), stderr.String(), runErr
|
|
|
|
|
}
|