Revision control
Copy as Markdown
#!/usr/bin/env python3
# Copyright (C) 2026 The Qt Company Ltd.
# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR BSD-3-Clause
"""Run the deterministic checks on a framework artifact.
Takes one or more paths inside doc_agent_framework/ and reports every
check that a script can decide without judgment: frontmatter, model pin,
registration in the subdir READMEs and the orchestrator, skill paths,
relative links, test-scenario coverage, the public/internal boundary,
secrets, and Tier 1 AI-tell patterns.
The artifact kind comes from its location:
agents/<name>.md agent
skills/**/SKILL.md skill
tests/scenarios/<name>/ scenario (pass the directory)
tools/<name>/ tool (pass the directory)
anything else file (common checks only)
Each finding prints as "SEVERITY CHECK path[:line] message". Exit
status: 0 when no finding is High or Critical, 1 when at least one is,
2 on a usage error.
"""
import argparse
import json
import re
import subprocess
import sys
from pathlib import Path
FRAMEWORK = Path(__file__).resolve().parent.parent.parent
MODEL = "claude-opus-5-5"
SEVERITIES = ["Critical", "High", "Medium", "Low", "Info"]
BLOCKING = {"Critical", "High"}
DESC_SOFT, DESC_HARD = 150, 1536
BODY_LINES_SOFT, BODY_CHARS_HARD = 200, 20000
# Patterns from skill-security-hygiene, "Quick grep patterns".
SECRET_RE = re.compile(
r"(-----BEGIN [A-Z ]+PRIVATE KEY-----|AKIA[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{30,}"
r"|xox[bp]-[A-Za-z0-9-]+|AIza[0-9A-Za-z_-]{35}|sk_live_[A-Za-z0-9]{20,}"
r"|ssh-(rsa|ed25519|dss) AAAA[0-9A-Za-z+/=]+)")
HOME_RE = re.compile(r"(/Users/[^/\s]+/|/home/[^/\s]+/|[A-Z]:\\Users\\[^\\]+\\|~/code/)")
QT_HOST_RE = re.compile(r"\b([a-z0-9-]+(?:\.[a-z0-9-]+)*\.(?:qt-project\.org|qt\.io))\b")
PUBLIC_HOSTS = {"doc.qt.io", "doc-snapshots.qt.io", "wiki.qt.io", "code.qt.io",
"contribute.qt-project.org", "codereview.qt-project.org",
"bugreports.qt.io", "qt-project.atlassian.net", "www.qt.io", "qt.io"}
# Tier 1 patterns from skill-ai-tells. Each hit is a finding; a person
# confirms it is prose and not a quoted pattern.
AI_TELLS = [
("T1a", re.compile(r" —[^—]*— ")),
("T6", re.compile(r"not only|on the other hand|on one hand", re.I)),
("T7", re.compile(r"\b(in conclusion|overall|ultimately|in summary|to sum up)\b", re.I)),
("T13", re.compile(r"\b(cutting-edge|state-of-the-art|world-class|next-generation"
r"|pivotal|holistic)\b", re.I)),
("T14", re.compile(r"it depends|both have|some (developers|users).*others", re.I)),
("T16", re.compile(r"\b(page|class|function|api|method|code|type|module|docs|widget|reader)s? "
r"(speaks|hears|knows|wants|understands|cares|thinks|remembers|decides"
r"|lives|sits|feels|asks)\b", re.I)),
("T17", re.compile(r"(is|are) not [^.]{2,40}\. *(It|They|That) (is|are)"
r"|(isn't|aren't)[^.]{0,45}(it's|they're)", re.I)),
("T18", re.compile(r"\b(delve|delves|plethora|tapestry|testament|nuanced|elevate|empower"
r"|intricate|effortless|effortlessly|boasts|synergy|undoubtedly)\b", re.I)),
]
LINK_RE = re.compile(r"\[[^\]]*\]\(([^)\s]+)\)")
SKILL_REF_RE = re.compile(r"\.\./skills/([A-Za-z0-9_./-]+?)/SKILL\.md")
class Report:
def __init__(self):
self.findings = []
self.checks = []
def ran(self, check):
if check not in self.checks:
self.checks.append(check)
def add(self, severity, check, path, message, line=None):
self.ran(check)
self.findings.append({"severity": severity, "check": check,
"path": rel(path), "line": line, "message": message})
def rel(path):
try:
return str(Path(path).resolve().relative_to(FRAMEWORK))
except ValueError:
return str(path)
def read(path):
return Path(path).read_text(encoding="utf-8", errors="replace")
def frontmatter(text):
"""Parse the simple YAML subset the framework uses. Returns (dict, body)."""
if not text.startswith("---\n"):
return None, text
end = text.find("\n---", 4)
if end < 0:
return None, text
block, body = text[4:end], text[end + 4:].lstrip("\n")
data, key, folded = {}, None, None
for raw in block.splitlines():
if folded is not None and (raw.startswith(" ") or not raw.strip()):
folded.append(raw.strip())
data[key] = " ".join(p for p in folded if p)
continue
folded = None
match = re.match(r"^(\s*)([A-Za-z_]+):\s*(.*)$", raw)
if not match:
continue
indent, name, value = match.groups()
if indent and key == "metadata":
data[f"metadata.{name}"] = value.strip().strip("\"'")
continue
key = name
if value in (">-", ">", "|", "|-"):
folded = []
data[key] = ""
else:
data[key] = value.strip().strip("\"'")
return data, body
def table_rows(text, heading):
"""Return the cells of each row in the first table under a heading."""
lines = text.splitlines()
rows, inside = [], False
for line in lines:
if line.startswith("#"):
if inside and rows:
break
inside = line.lstrip("#").strip().lower() == heading.lower()
continue
if inside and line.startswith("|"):
cells = [c.strip() for c in line.strip().strip("|").split("|")]
if not all(set(c) <= set("-: ") for c in cells):
rows.append(cells)
return rows[1:] if rows else []
def strip_md(cell):
return re.sub(r"[`*\[\]]|\(.*?\)", "", cell).strip()
def prose_lines(text):
"""Yield (line number, line, inside fenced code) for a Markdown file."""
fenced = False
for number, line in enumerate(text.splitlines(), 1):
if line.lstrip().startswith("```"):
fenced = not fenced
continue
yield number, line, fenced
# Common checks, every file ------------------------------------------------
def check_common(path, report):
text = read(path)
is_md = path.suffix == ".md"
for check in ("boundary-home-path", "boundary-host", "secret", "ai-tell-tier1"):
report.ran(check)
if is_md:
report.ran("link")
for number, line, fenced in (prose_lines(text) if is_md else
((n, l, False) for n, l in enumerate(text.splitlines(), 1))):
if SECRET_RE.search(line):
report.add("Critical", "secret", path, "possible secret or key", number)
for match in HOME_RE.finditer(line):
if is_placeholder(match, line):
continue
report.add("High", "boundary-home-path", path,
f"user-specific path: {match.group(0)}", number)
for host in QT_HOST_RE.findall(line):
if host not in PUBLIC_HOSTS:
report.add("Medium", "boundary-host", path,
f"Qt host not on the public list, needs a human check: {host}", number)
if is_md and not fenced:
for target in LINK_RE.findall(line):
check_link(path, target, number, report)
if is_md:
for tell, pattern in AI_TELLS:
if pattern.search(line):
report.add("Medium", "ai-tell-tier1", path,
f"{tell} pattern: {pattern.search(line).group(0)!r} "
f"(confirm it is a tell, then treat as High)", number)
def is_placeholder(match, line):
"""True for rule text such as /Users/<name>/, ~/code/..., or a regex."""
text, after = match.group(0), line[match.end():match.end() + 1]
return (any(c in text for c in "<[{") or after in ("…", ".")
or "USERNAME" in line or "[^" in line)
def check_link(path, target, number, report):
if re.match(r"^[a-z]+:", target) or target.startswith("#") or "{" in target:
return
target = target.split("#", 1)[0]
if target and not (path.parent / target).exists():
report.add("Medium", "link", path, f"relative link does not resolve: {target}", number)
# Agents -------------------------------------------------------------------
def check_agent(path, report):
text = read(path)
data, body = frontmatter(text)
name = path.stem
report.ran("frontmatter")
if data is None:
report.add("High", "frontmatter", path, "no YAML frontmatter")
data = {}
for field in ("name", "description", "model"):
if not data.get(field):
report.add("High", "frontmatter", path, f"missing '{field}'")
if data.get("name") and data["name"] != name:
report.add("High", "frontmatter", path, f"name '{data['name']}' differs from file name '{name}'")
report.ran("model-pin")
if data.get("model") and data["model"] != MODEL:
report.add("High", "model-pin", path, f"model is '{data['model']}', expected '{MODEL}'")
check_description(path, data.get("description", ""), report)
check_size(path, body, report, lines_soft=None)
report.ran("overview-fields")
for field in ("Pipeline", "Fan-out", "Loading pattern"):
if not re.search(rf"^\|\s*{field}\s*\|", body, re.M) and name != "orchestrator":
report.add("Medium", "overview-fields", path, f"overview table has no '{field}' row")
report.ran("verification-gate")
if not re.search(r"^#+ .*verification", body, re.I | re.M):
report.add("Medium", "verification-gate", path, "no verification gate section")
agents_readme = FRAMEWORK / "agents" / "README.md"
rows = [r for r in table_rows(read(agents_readme), "Pipeline mapping") if len(r) >= 4]
mine = [r for r in rows if strip_md(r[2]) == name]
report.ran("registered-agents-readme")
if not mine:
report.add("Medium", "registered-agents-readme", agents_readme,
f"no row for '{name}' in the pipeline mapping table")
report.ran("fan-out-consistency")
tools = data.get("tools")
if mine and tools is not None:
listed_yes = any(strip_md(r[3]).lower().startswith("yes") for r in mine)
has_agent = re.search(r"\bAgent\b", tools) is not None
if listed_yes != has_agent:
report.add("High", "fan-out-consistency", path,
f"agents/README.md says fan-out {'yes' if listed_yes else 'no'}, "
f"but tools: {'includes' if has_agent else 'omits'} Agent")
if name != "orchestrator":
orchestrator = FRAMEWORK / "agents" / "orchestrator.md"
report.ran("registered-orchestrator")
dispatch = table_rows(read(orchestrator), "Dispatch table")
if not any(len(r) > 1 and strip_md(r[1]) == name for r in dispatch):
report.add("Medium", "registered-orchestrator", orchestrator,
f"no dispatch table row for '{name}'")
report.ran("instruction")
howto = FRAMEWORK / "doc" / "instructions" / f"how-to-use-{name}.md"
if not howto.exists():
report.add("Low", "instruction", howto, "no how-to instruction (CONTRIBUTING step 6)")
check_scenarios(path, name, report)
report.ran("skill-path")
seen = set()
for number, line in enumerate(text.splitlines(), 1):
for skill in SKILL_REF_RE.findall(line):
if "{" in skill or skill in seen:
continue
seen.add(skill)
if not (FRAMEWORK / "skills" / skill / "SKILL.md").exists():
report.add("Medium", "skill-path", path,
f"loads ../skills/{skill}/SKILL.md, which is not in the framework",
number)
def check_scenarios(path, name, report):
report.ran("scenario-coverage")
tests_readme = FRAMEWORK / "tests" / "README.md"
rows = table_rows(read(tests_readme), "Scenarios")
found = [strip_md(r[0]) for r in rows if len(r) > 1 and strip_md(r[1]) == name]
if found:
report.add("Info", "scenario-coverage", path, f"scenarios: {', '.join(found)}")
else:
report.add("Medium", "scenario-coverage", path,
"no test scenario lists this agent in tests/README.md")
# Skills -------------------------------------------------------------------
def check_skill(path, report):
text = read(path)
data, body = frontmatter(text)
name = path.parent.name
report.ran("frontmatter")
if data is None:
report.add("High", "frontmatter", path, "no YAML frontmatter")
data = {}
for field in ("name", "description", "metadata.version"):
if not data.get(field):
report.add("High" if field != "metadata.version" else "Medium",
"frontmatter", path, f"missing '{field}'")
if data.get("name") and data["name"] != name:
report.add("High", "frontmatter", path, f"name '{data['name']}' differs from directory '{name}'")
report.ran("model-pin")
if "model" in data:
report.add("Medium", "model-pin", path, "skills carry no model field")
check_description(path, data.get("description", ""), report)
check_size(path, body, report, lines_soft=BODY_LINES_SOFT)
report.ran("registered-skills-readme")
# A skill in a group directory (skills/personas/) may be listed in
# the group README instead of skills/README.md.
skills_dir = FRAMEWORK / "skills"
readmes = [skills_dir / "README.md"]
readmes += [p / "README.md" for p in path.parent.parents
if skills_dir in p.parents and (p / "README.md").exists()]
if not any(name in read(r) for r in readmes):
report.add("Medium", "registered-skills-readme", readmes[0],
f"'{name}' is not listed in skills/README.md")
report.ran("loaded-by-agent")
# A skill in a group directory can be loaded through a template
# path such as ../skills/personas/skill-persona-{name}/SKILL.md.
group = path.parent.parent
group_ref = (f"skills/{group.relative_to(skills_dir)}/" if group != skills_dir else None)
loaders = [a.stem for a in (FRAMEWORK / "agents").glob("*.md")
if a.name != "README.md" and (name in read(a) or
(group_ref and group_ref in read(a)))]
if loaders:
report.add("Info", "loaded-by-agent", path, f"referenced by: {', '.join(sorted(loaders))}")
else:
report.add("Medium", "loaded-by-agent", path, "no agent in agents/ references this skill")
# Scenarios and tools ------------------------------------------------------
def check_scenario(path, report):
name = path.name
report.ran("scenario-files")
for required in ("input.patch", "README.md"):
if not (path / required).is_file():
report.add("High", "scenario-files", path, f"missing {required}")
if not any((path / f).is_file() for f in ("expected-warnings.txt", "expected-findings.txt")):
report.add("High", "scenario-files", path, "no expected output file")
report.ran("registered-tests-readme")
rows = table_rows(read(FRAMEWORK / "tests" / "README.md"), "Scenarios")
row = [r for r in rows if strip_md(r[0]) == name]
if not row:
report.add("Medium", "registered-tests-readme", FRAMEWORK / "tests" / "README.md",
f"no row for scenario '{name}'")
else:
agent = strip_md(row[0][1])
report.ran("scenario-agent-exists")
if not (FRAMEWORK / "agents" / f"{agent}.md").exists():
report.add("Medium", "scenario-agent-exists", path,
f"scenario names agent '{agent}', which is not in agents/")
for file in sorted(path.iterdir()):
if file.suffix == ".md":
check_common(file, report)
def check_tool(path, report):
report.ran("tool-readme")
if not (path / "README.md").is_file():
report.add("Medium", "tool-readme", path, "no README.md in the tool directory")
report.ran("spdx")
report.ran("registered-tools-readme")
if path.name not in read(FRAMEWORK / "tools" / "README.md"):
report.add("Medium", "registered-tools-readme", FRAMEWORK / "tools" / "README.md",
f"'{path.name}' is not listed in tools/README.md")
for file in sorted(path.rglob("*")):
if file.suffix in (".py", ".sh") and "SPDX-License-Identifier" not in read(file)[:500]:
report.add("Medium", "spdx", file, "no SPDX-License-Identifier header")
if file.suffix in (".md", ".py", ".sh"):
check_common(file, report)
def check_description(path, description, report):
report.ran("description-length")
size = len(description)
if size > DESC_HARD:
report.add("High", "description-length", path, f"description is {size} chars (cap {DESC_HARD})")
elif size > DESC_SOFT:
report.add("Low", "description-length", path, f"description is {size} chars (target {DESC_SOFT})")
def check_size(path, body, report, lines_soft):
report.ran("body-size")
chars, lines = len(body), body.count("\n") + 1
if chars > BODY_CHARS_HARD:
report.add("High", "body-size", path,
f"body is {chars} chars; content past ~{BODY_CHARS_HARD} is dropped on re-attach")
if lines_soft and lines > lines_soft:
report.add("Low", "body-size", path, f"body is {lines} lines (split threshold {lines_soft})")
# Driver -------------------------------------------------------------------
def classify(path):
parts = path.resolve().relative_to(FRAMEWORK).parts
if parts[0] == "agents" and len(parts) == 2 and path.suffix == ".md" and path.name != "README.md":
return "agent"
if parts[0] == "skills" and path.name == "SKILL.md":
return "skill"
if parts[:2] == ("tests", "scenarios") and len(parts) == 3 and path.is_dir():
return "scenario"
if parts[0] == "tools" and len(parts) == 2 and path.is_dir():
return "tool"
return "file"
def main():
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("paths", nargs="+", help="artifacts inside doc_agent_framework/")
parser.add_argument("--root", help="framework root to check against "
"(default: the tree this script lives in)")
parser.add_argument("--json", action="store_true", help="print the report as JSON")
parser.add_argument("--min-severity", choices=SEVERITIES, default="Info",
help="hide findings below this severity (text output only)")
args = parser.parse_args()
if args.root:
global FRAMEWORK
FRAMEWORK = Path(args.root).resolve()
if not (FRAMEWORK / "agents" / "orchestrator.md").is_file():
print(f"error: {args.root} has no agents/orchestrator.md", file=sys.stderr)
return 2
results = []
for arg in args.paths:
path = Path(arg).resolve()
if not path.exists():
print(f"error: no such path: {arg}", file=sys.stderr)
return 2
try:
path.relative_to(FRAMEWORK)
except ValueError:
print(f"error: {arg} is not inside {FRAMEWORK}", file=sys.stderr)
return 2
kind = classify(path)
report = Report()
if kind == "agent":
check_agent(path, report)
elif kind == "skill":
check_skill(path, report)
elif kind == "scenario":
check_scenario(path, report)
elif kind == "tool":
check_tool(path, report)
if kind in ("agent", "skill", "file"):
if path.is_dir():
print(f"error: pass a file, not a directory: {arg}", file=sys.stderr)
return 2
check_common(path, report)
results.append({"artifact": rel(path), "kind": kind,
"checks": report.checks, "findings": report.findings})
blocking = sum(f["severity"] in BLOCKING for r in results for f in r["findings"])
if args.json:
print(json.dumps({"framework": str(FRAMEWORK), "results": results,
"blocking": blocking}, indent=2))
else:
cutoff = SEVERITIES.index(args.min_severity)
for r in results:
print(f"== {r['artifact']} ({r['kind']}): {len(r['checks'])} checks ran")
order = sorted(r["findings"], key=lambda f: (SEVERITIES.index(f["severity"]),
f["check"], f["path"], f["line"] or 0))
for f in order:
if SEVERITIES.index(f["severity"]) > cutoff:
continue
loc = f"{f['path']}:{f['line']}" if f["line"] else f["path"]
print(f"{f['severity']:<8} {f['check']:<26} {loc} {f['message']}")
print(f"{blocking} blocking (High or Critical) finding(s)")
return 1 if blocking else 0
if __name__ == "__main__":
sys.exit(main())