Revision control

+ #!/usr/bin/env python3
+ # Copyright (C) 2026 The Qt Company Ltd.
+ # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR BSD-3-Clause
+
+ """Prove that check_artifact.py catches the defects it claims to catch.
+
+ Copies the framework to a temporary directory, adds an agent and a
+ skill with one seeded defect per check, runs check_artifact.py with
+ --root on the copy, and compares the findings against the list below.
+ A check that stays silent on its seeded defect fails the self-test.
+
+ Exit status: 0 when every seeded defect is found and the shipped
+ artifacts have no blocking finding, 1 otherwise.
+ """
+
+ import json
+ import shutil
+ import subprocess
+ import sys
+ import tempfile
+ from pathlib import Path
+
+ HERE = Path(__file__).resolve().parent
+ FRAMEWORK = HERE.parent.parent
+ CHECKER = HERE / "check_artifact.py"
+
+ # Assembled at run time so this file does not itself trip the checks.
+ HOME = "/" + "Users/someone/"
+ TELL = "del" + "ve"
+
+ BAD_AGENT = f"""---
+ name: wrong-name
+ description: {"x" * 160}
+ model: opus
+ tools: Read, Agent
+ ---
+
+ # Bad agent
+
+ Uses [a missing page](no-such-page.md) and loads
+ `../skills/skill-not-there/SKILL.md`. Reads {HOME}code/file.txt.
+ The agent will {TELL} into the details.
+ """
+
+ BAD_SKILL = """---
+ name: skill-other
+ description: A seeded skill for the checker self-test.
+ model: claude-opus-5-5
+ ---
+
+ # Bad skill
+ """
+
+ # (artifact, check, severity) triples the checker must report.
+ EXPECTED = {
+ ("agents/bad-agent.md", "frontmatter", "High"), # name differs from file
+ ("agents/bad-agent.md", "model-pin", "High"), # alias, not pinned
+ ("agents/bad-agent.md", "description-length", "Low"), # 160 > 150
+ ("agents/bad-agent.md", "overview-fields", "Medium"), # no overview table
+ ("agents/bad-agent.md", "verification-gate", "Medium"), # no gate section
+ ("agents/bad-agent.md", "registered-agents-readme", "Medium"),
+ ("agents/bad-agent.md", "registered-orchestrator", "Medium"),
+ ("agents/bad-agent.md", "instruction", "Low"),
+ ("agents/bad-agent.md", "scenario-coverage", "Medium"),
+ ("agents/bad-agent.md", "skill-path", "Medium"),
+ ("agents/bad-agent.md", "link", "Medium"),
+ ("agents/bad-agent.md", "boundary-home-path", "High"),
+ ("agents/bad-agent.md", "ai-tell-tier1", "Medium"),
+ ("skills/skill-bad/SKILL.md", "frontmatter", "High"), # name differs from dir
+ ("skills/skill-bad/SKILL.md", "model-pin", "Medium"), # skills carry no model
+ ("skills/skill-bad/SKILL.md", "registered-skills-readme", "Medium"),
+ ("skills/skill-bad/SKILL.md", "loaded-by-agent", "Medium"),
+ }
+
+
+ def run(root, *paths):
+ result = subprocess.run([sys.executable, str(CHECKER), "--root", str(root), "--json", *paths],
+ capture_output=True, text=True)
+ if result.returncode == 2:
+ sys.exit(f"checker error:\n{result.stderr}")
+ return json.loads(result.stdout)
+
+
+ def main():
+ failures = []
+ with tempfile.TemporaryDirectory(prefix="check-artifact-") as tmp:
+ root = Path(tmp) / "fw"
+ shutil.copytree(FRAMEWORK, root, ignore=shutil.ignore_patterns(".DS_Store", "__pycache__"))
+ (root / "agents" / "bad-agent.md").write_text(BAD_AGENT)
+ (root / "skills" / "skill-bad").mkdir()
+ (root / "skills" / "skill-bad" / "SKILL.md").write_text(BAD_SKILL)
+
+ report = run(root, str(root / "agents/bad-agent.md"), str(root / "skills/skill-bad/SKILL.md"))
+ found = {(r["artifact"], f["check"], f["severity"])
+ for r in report["results"] for f in r["findings"]}
+ for item in sorted(EXPECTED - found):
+ failures.append(f"missed seeded defect: {item}")
+
+ # The fan-out check needs a registered agent: list bad-agent as
+ # "no" in agents/README.md while its tools: include Agent.
+ readme = root / "agents" / "README.md"
+ readme.write_text(readme.read_text().replace(
+ "| (all) | Dispatch | orchestrator | yes |",
+ "| (all) | Dispatch | orchestrator | yes |\n| A | Seeded | bad-agent | no |"))
+ report = run(root, str(root / "agents/bad-agent.md"))
+ if not any(f["check"] == "fan-out-consistency" for f in report["results"][0]["findings"]):
+ failures.append("missed seeded defect: fan-out-consistency")
+
+ shipped = [str(p) for p in sorted((FRAMEWORK / "agents").glob("*.md")) if p.name != "README.md"]
+ shipped += [str(p) for p in sorted((FRAMEWORK / "skills").rglob("SKILL.md"))]
+ report = run(FRAMEWORK, *shipped)
+ for r in report["results"]:
+ for f in r["findings"]:
+ if f["severity"] in ("Critical", "High"):
+ failures.append(f"false blocker on shipped artifact: {r['artifact']}: "
+ f"{f['check']} {f['message']}")
+
+ checked = len(EXPECTED) + 1
+ if failures:
+ print("\n".join(failures))
+ print(f"FAIL: {len(failures)} problem(s)")
+ return 1
+ print(f"PASS: {checked} seeded defects found; {len(shipped)} shipped artifacts, 0 blocking")
+ return 0
+
+
+ if __name__ == "__main__":
+ sys.exit(main())