Revision control
++ #!/usr/bin/env python3
++ # Copyright (C) 2026 The Qt Company Ltd.
++ # SPDX-License-Identifier: LicenseRef-Qt-Commercial OR BSD-3-Clause
++
++ """Run the deterministic checks on a framework artifact.
++
++ Takes one or more paths inside doc_agent_framework/ and reports every
++ check that a script can decide without judgment: frontmatter, model pin,
++ registration in the subdir READMEs and the orchestrator, skill paths,
++ relative links, test-scenario coverage, the public/internal boundary,
++ secrets, and Tier 1 AI-tell patterns.
++
++ The artifact kind comes from its location:
++
++ agents/<name>.md agent
++ skills/**/SKILL.md skill
++ tests/scenarios/<name>/ scenario (pass the directory)
++ tools/<name>/ tool (pass the directory)
++ anything else file (common checks only)
++
++ Each finding prints as "SEVERITY CHECK path[:line] message". Exit
++ status: 0 when no finding is High or Critical, 1 when at least one is,
++ 2 on a usage error.
++ """
++
++ import argparse
++ import json
++ import re
++ import subprocess
++ import sys
++ from pathlib import Path
++
++ FRAMEWORK = Path(__file__).resolve().parent.parent.parent
++ MODEL = "claude-opus-5-5"
++ SEVERITIES = ["Critical", "High", "Medium", "Low", "Info"]
++ BLOCKING = {"Critical", "High"}
++
++ DESC_SOFT, DESC_HARD = 150, 1536
++ BODY_LINES_SOFT, BODY_CHARS_HARD = 200, 20000
++
++ # Patterns from skill-security-hygiene, "Quick grep patterns".
++ SECRET_RE = re.compile(
++ r"(-----BEGIN [A-Z ]+PRIVATE KEY-----|AKIA[0-9A-Z]{16}|gh[pousr]_[A-Za-z0-9]{30,}"
++ r"|xox[bp]-[A-Za-z0-9-]+|AIza[0-9A-Za-z_-]{35}|sk_live_[A-Za-z0-9]{20,}"
++ r"|ssh-(rsa|ed25519|dss) AAAA[0-9A-Za-z+/=]+)")
++ HOME_RE = re.compile(r"(/Users/[^/\s]+/|/home/[^/\s]+/|[A-Z]:\\Users\\[^\\]+\\|~/code/)")
++ QT_HOST_RE = re.compile(r"\b([a-z0-9-]+(?:\.[a-z0-9-]+)*\.(?:qt-project\.org|qt\.io))\b")
++ PUBLIC_HOSTS = {"doc.qt.io", "doc-snapshots.qt.io", "wiki.qt.io", "code.qt.io",
++ "contribute.qt-project.org", "codereview.qt-project.org",
++ "bugreports.qt.io", "qt-project.atlassian.net", "www.qt.io", "qt.io"}
++
++ # Tier 1 patterns from skill-ai-tells. Each hit is a finding; a person
++ # confirms it is prose and not a quoted pattern.
++ AI_TELLS = [
++ ("T1a", re.compile(r" —[^—]*— ")),
++ ("T6", re.compile(r"not only|on the other hand|on one hand", re.I)),
++ ("T7", re.compile(r"\b(in conclusion|overall|ultimately|in summary|to sum up)\b", re.I)),
++ ("T13", re.compile(r"\b(cutting-edge|state-of-the-art|world-class|next-generation"
++ r"|pivotal|holistic)\b", re.I)),
++ ("T14", re.compile(r"it depends|both have|some (developers|users).*others", re.I)),
++ ("T16", re.compile(r"\b(page|class|function|api|method|code|type|module|docs|widget|reader)s? "
++ r"(speaks|hears|knows|wants|understands|cares|thinks|remembers|decides"
++ r"|lives|sits|feels|asks)\b", re.I)),
++ ("T17", re.compile(r"(is|are) not [^.]{2,40}\. *(It|They|That) (is|are)"
++ r"|(isn't|aren't)[^.]{0,45}(it's|they're)", re.I)),
++ ("T18", re.compile(r"\b(delve|delves|plethora|tapestry|testament|nuanced|elevate|empower"
++ r"|intricate|effortless|effortlessly|boasts|synergy|undoubtedly)\b", re.I)),
++ ]
++
++ LINK_RE = re.compile(r"\[[^\]]*\]\(([^)\s]+)\)")
++ SKILL_REF_RE = re.compile(r"\.\./skills/([A-Za-z0-9_./-]+?)/SKILL\.md")
++
++
++ class Report:
++ def __init__(self):
++ self.findings = []
++ self.checks = []
++
++ def ran(self, check):
++ if check not in self.checks:
++ self.checks.append(check)
++
++ def add(self, severity, check, path, message, line=None):
++ self.ran(check)
++ self.findings.append({"severity": severity, "check": check,
++ "path": rel(path), "line": line, "message": message})
++
++
++ def rel(path):
++ try:
++ return str(Path(path).resolve().relative_to(FRAMEWORK))
++ except ValueError:
++ return str(path)
++
++
++ def read(path):
++ return Path(path).read_text(encoding="utf-8", errors="replace")
++
++
++ def frontmatter(text):
++ """Parse the simple YAML subset the framework uses. Returns (dict, body)."""
++ if not text.startswith("---\n"):
++ return None, text
++ end = text.find("\n---", 4)
++ if end < 0:
++ return None, text
++ block, body = text[4:end], text[end + 4:].lstrip("\n")
++ data, key, folded = {}, None, None
++ for raw in block.splitlines():
++ if folded is not None and (raw.startswith(" ") or not raw.strip()):
++ folded.append(raw.strip())
++ data[key] = " ".join(p for p in folded if p)
++ continue
++ folded = None
++ match = re.match(r"^(\s*)([A-Za-z_]+):\s*(.*)$", raw)
++ if not match:
++ continue
++ indent, name, value = match.groups()
++ if indent and key == "metadata":
++ data[f"metadata.{name}"] = value.strip().strip("\"'")
++ continue
++ key = name
++ if value in (">-", ">", "|", "|-"):
++ folded = []
++ data[key] = ""
++ else:
++ data[key] = value.strip().strip("\"'")
++ return data, body
++
++
++ def table_rows(text, heading):
++ """Return the cells of each row in the first table under a heading."""
++ lines = text.splitlines()
++ rows, inside = [], False
++ for line in lines:
++ if line.startswith("#"):
++ if inside and rows:
++ break
++ inside = line.lstrip("#").strip().lower() == heading.lower()
++ continue
++ if inside and line.startswith("|"):
++ cells = [c.strip() for c in line.strip().strip("|").split("|")]
++ if not all(set(c) <= set("-: ") for c in cells):
++ rows.append(cells)
++ return rows[1:] if rows else []
++
++
++ def strip_md(cell):
++ return re.sub(r"[`*\[\]]|\(.*?\)", "", cell).strip()
++
++
++ def prose_lines(text):
++ """Yield (line number, line, inside fenced code) for a Markdown file."""
++ fenced = False
++ for number, line in enumerate(text.splitlines(), 1):
++ if line.lstrip().startswith("```"):
++ fenced = not fenced
++ continue
++ yield number, line, fenced
++
++
++ # Common checks, every file ------------------------------------------------
++
++ def check_common(path, report):
++ text = read(path)
++ is_md = path.suffix == ".md"
++ for check in ("boundary-home-path", "boundary-host", "secret", "ai-tell-tier1"):
++ report.ran(check)
++ if is_md:
++ report.ran("link")
++ for number, line, fenced in (prose_lines(text) if is_md else
++ ((n, l, False) for n, l in enumerate(text.splitlines(), 1))):
++ if SECRET_RE.search(line):
++ report.add("Critical", "secret", path, "possible secret or key", number)
++ for match in HOME_RE.finditer(line):
++ if is_placeholder(match, line):
++ continue
++ report.add("High", "boundary-home-path", path,
++ f"user-specific path: {match.group(0)}", number)
++ for host in QT_HOST_RE.findall(line):
++ if host not in PUBLIC_HOSTS:
++ report.add("Medium", "boundary-host", path,
++ f"Qt host not on the public list, needs a human check: {host}", number)
++ if is_md and not fenced:
++ for target in LINK_RE.findall(line):
++ check_link(path, target, number, report)
++ if is_md:
++ for tell, pattern in AI_TELLS:
++ if pattern.search(line):
++ report.add("Medium", "ai-tell-tier1", path,
++ f"{tell} pattern: {pattern.search(line).group(0)!r} "
++ f"(confirm it is a tell, then treat as High)", number)
++
++
++ def is_placeholder(match, line):
++ """True for rule text such as /Users/<name>/, ~/code/..., or a regex."""
++ text, after = match.group(0), line[match.end():match.end() + 1]
++ return (any(c in text for c in "<[{") or after in ("…", ".")
++ or "USERNAME" in line or "[^" in line)
++
++
++ def check_link(path, target, number, report):
++ if re.match(r"^[a-z]+:", target) or target.startswith("#") or "{" in target:
++ return
++ target = target.split("#", 1)[0]
++ if target and not (path.parent / target).exists():
++ report.add("Medium", "link", path, f"relative link does not resolve: {target}", number)
++
++
++ # Agents -------------------------------------------------------------------
++
++ def check_agent(path, report):
++ text = read(path)
++ data, body = frontmatter(text)
++ name = path.stem
++ report.ran("frontmatter")
++ if data is None:
++ report.add("High", "frontmatter", path, "no YAML frontmatter")
++ data = {}
++ for field in ("name", "description", "model"):
++ if not data.get(field):
++ report.add("High", "frontmatter", path, f"missing '{field}'")
++ if data.get("name") and data["name"] != name:
++ report.add("High", "frontmatter", path, f"name '{data['name']}' differs from file name '{name}'")
++ report.ran("model-pin")
++ if data.get("model") and data["model"] != MODEL:
++ report.add("High", "model-pin", path, f"model is '{data['model']}', expected '{MODEL}'")
++ check_description(path, data.get("description", ""), report)
++ check_size(path, body, report, lines_soft=None)
++
++ report.ran("overview-fields")
++ for field in ("Pipeline", "Fan-out", "Loading pattern"):
++ if not re.search(rf"^\|\s*{field}\s*\|", body, re.M) and name != "orchestrator":
++ report.add("Medium", "overview-fields", path, f"overview table has no '{field}' row")
++ report.ran("verification-gate")
++ if not re.search(r"^#+ .*verification", body, re.I | re.M):
++ report.add("Medium", "verification-gate", path, "no verification gate section")
++
++ agents_readme = FRAMEWORK / "agents" / "README.md"
++ rows = [r for r in table_rows(read(agents_readme), "Pipeline mapping") if len(r) >= 4]
++ mine = [r for r in rows if strip_md(r[2]) == name]
++ report.ran("registered-agents-readme")
++ if not mine:
++ report.add("Medium", "registered-agents-readme", agents_readme,
++ f"no row for '{name}' in the pipeline mapping table")
++ report.ran("fan-out-consistency")
++ tools = data.get("tools")
++ if mine and tools is not None:
++ listed_yes = any(strip_md(r[3]).lower().startswith("yes") for r in mine)
++ has_agent = re.search(r"\bAgent\b", tools) is not None
++ if listed_yes != has_agent:
++ report.add("High", "fan-out-consistency", path,
++ f"agents/README.md says fan-out {'yes' if listed_yes else 'no'}, "
++ f"but tools: {'includes' if has_agent else 'omits'} Agent")
++
++ if name != "orchestrator":
++ orchestrator = FRAMEWORK / "agents" / "orchestrator.md"
++ report.ran("registered-orchestrator")
++ dispatch = table_rows(read(orchestrator), "Dispatch table")
++ if not any(len(r) > 1 and strip_md(r[1]) == name for r in dispatch):
++ report.add("Medium", "registered-orchestrator", orchestrator,
++ f"no dispatch table row for '{name}'")
++ report.ran("instruction")
++ howto = FRAMEWORK / "doc" / "instructions" / f"how-to-use-{name}.md"
++ if not howto.exists():
++ report.add("Low", "instruction", howto, "no how-to instruction (CONTRIBUTING step 6)")
++ check_scenarios(path, name, report)
++
++ report.ran("skill-path")
++ seen = set()
++ for number, line in enumerate(text.splitlines(), 1):
++ for skill in SKILL_REF_RE.findall(line):
++ if "{" in skill or skill in seen:
++ continue
++ seen.add(skill)
++ if not (FRAMEWORK / "skills" / skill / "SKILL.md").exists():
++ report.add("Medium", "skill-path", path,
++ f"loads ../skills/{skill}/SKILL.md, which is not in the framework",
++ number)
++
++
++ def check_scenarios(path, name, report):
++ report.ran("scenario-coverage")
++ tests_readme = FRAMEWORK / "tests" / "README.md"
++ rows = table_rows(read(tests_readme), "Scenarios")
++ found = [strip_md(r[0]) for r in rows if len(r) > 1 and strip_md(r[1]) == name]
++ if found:
++ report.add("Info", "scenario-coverage", path, f"scenarios: {', '.join(found)}")
++ else:
++ report.add("Medium", "scenario-coverage", path,
++ "no test scenario lists this agent in tests/README.md")
++
++
++ # Skills -------------------------------------------------------------------
++
++ def check_skill(path, report):
++ text = read(path)
++ data, body = frontmatter(text)
++ name = path.parent.name
++ report.ran("frontmatter")
++ if data is None:
++ report.add("High", "frontmatter", path, "no YAML frontmatter")
++ data = {}
++ for field in ("name", "description", "metadata.version"):
++ if not data.get(field):
++ report.add("High" if field != "metadata.version" else "Medium",
++ "frontmatter", path, f"missing '{field}'")
++ if data.get("name") and data["name"] != name:
++ report.add("High", "frontmatter", path, f"name '{data['name']}' differs from directory '{name}'")
++ report.ran("model-pin")
++ if "model" in data:
++ report.add("Medium", "model-pin", path, "skills carry no model field")
++ check_description(path, data.get("description", ""), report)
++ check_size(path, body, report, lines_soft=BODY_LINES_SOFT)
++
++ report.ran("registered-skills-readme")
++ # A skill in a group directory (skills/personas/) may be listed in
++ # the group README instead of skills/README.md.
++ skills_dir = FRAMEWORK / "skills"
++ readmes = [skills_dir / "README.md"]
++ readmes += [p / "README.md" for p in path.parent.parents
++ if skills_dir in p.parents and (p / "README.md").exists()]
++ if not any(name in read(r) for r in readmes):
++ report.add("Medium", "registered-skills-readme", readmes[0],
++ f"'{name}' is not listed in skills/README.md")
++ report.ran("loaded-by-agent")
++ # A skill in a group directory can be loaded through a template
++ # path such as ../skills/personas/skill-persona-{name}/SKILL.md.
++ group = path.parent.parent
++ group_ref = (f"skills/{group.relative_to(skills_dir)}/" if group != skills_dir else None)
++ loaders = [a.stem for a in (FRAMEWORK / "agents").glob("*.md")
++ if a.name != "README.md" and (name in read(a) or
++ (group_ref and group_ref in read(a)))]
++ if loaders:
++ report.add("Info", "loaded-by-agent", path, f"referenced by: {', '.join(sorted(loaders))}")
++ else:
++ report.add("Medium", "loaded-by-agent", path, "no agent in agents/ references this skill")
++
++
++
++ # Scenarios and tools ------------------------------------------------------
++
++ def check_scenario(path, report):
++ name = path.name
++ report.ran("scenario-files")
++ for required in ("input.patch", "README.md"):
++ if not (path / required).is_file():
++ report.add("High", "scenario-files", path, f"missing {required}")
++ if not any((path / f).is_file() for f in ("expected-warnings.txt", "expected-findings.txt")):
++ report.add("High", "scenario-files", path, "no expected output file")
++ report.ran("registered-tests-readme")
++ rows = table_rows(read(FRAMEWORK / "tests" / "README.md"), "Scenarios")
++ row = [r for r in rows if strip_md(r[0]) == name]
++ if not row:
++ report.add("Medium", "registered-tests-readme", FRAMEWORK / "tests" / "README.md",
++ f"no row for scenario '{name}'")
++ else:
++ agent = strip_md(row[0][1])
++ report.ran("scenario-agent-exists")
++ if not (FRAMEWORK / "agents" / f"{agent}.md").exists():
++ report.add("Medium", "scenario-agent-exists", path,
++ f"scenario names agent '{agent}', which is not in agents/")
++ for file in sorted(path.iterdir()):
++ if file.suffix == ".md":
++ check_common(file, report)
++
++
++ def check_tool(path, report):
++ report.ran("tool-readme")
++ if not (path / "README.md").is_file():
++ report.add("Medium", "tool-readme", path, "no README.md in the tool directory")
++ report.ran("spdx")
++ report.ran("registered-tools-readme")
++ if path.name not in read(FRAMEWORK / "tools" / "README.md"):
++ report.add("Medium", "registered-tools-readme", FRAMEWORK / "tools" / "README.md",
++ f"'{path.name}' is not listed in tools/README.md")
++ for file in sorted(path.rglob("*")):
++ if file.suffix in (".py", ".sh") and "SPDX-License-Identifier" not in read(file)[:500]:
++ report.add("Medium", "spdx", file, "no SPDX-License-Identifier header")
++ if file.suffix in (".md", ".py", ".sh"):
++ check_common(file, report)
++
++
++ def check_description(path, description, report):
++ report.ran("description-length")
++ size = len(description)
++ if size > DESC_HARD:
++ report.add("High", "description-length", path, f"description is {size} chars (cap {DESC_HARD})")
++ elif size > DESC_SOFT:
++ report.add("Low", "description-length", path, f"description is {size} chars (target {DESC_SOFT})")
++
++
++ def check_size(path, body, report, lines_soft):
++ report.ran("body-size")
++ chars, lines = len(body), body.count("\n") + 1
++ if chars > BODY_CHARS_HARD:
++ report.add("High", "body-size", path,
++ f"body is {chars} chars; content past ~{BODY_CHARS_HARD} is dropped on re-attach")
++ if lines_soft and lines > lines_soft:
++ report.add("Low", "body-size", path, f"body is {lines} lines (split threshold {lines_soft})")
++
++
++ # Driver -------------------------------------------------------------------
++
++ def classify(path):
++ parts = path.resolve().relative_to(FRAMEWORK).parts
++ if parts[0] == "agents" and len(parts) == 2 and path.suffix == ".md" and path.name != "README.md":
++ return "agent"
++ if parts[0] == "skills" and path.name == "SKILL.md":
++ return "skill"
++ if parts[:2] == ("tests", "scenarios") and len(parts) == 3 and path.is_dir():
++ return "scenario"
++ if parts[0] == "tools" and len(parts) == 2 and path.is_dir():
++ return "tool"
++ return "file"
++
++
++ def main():
++ parser = argparse.ArgumentParser(description=__doc__,
++ formatter_class=argparse.RawDescriptionHelpFormatter)
++ parser.add_argument("paths", nargs="+", help="artifacts inside doc_agent_framework/")
++ parser.add_argument("--root", help="framework root to check against "
++ "(default: the tree this script lives in)")
++ parser.add_argument("--json", action="store_true", help="print the report as JSON")
++ parser.add_argument("--min-severity", choices=SEVERITIES, default="Info",
++ help="hide findings below this severity (text output only)")
++ args = parser.parse_args()
++ if args.root:
++ global FRAMEWORK
++ FRAMEWORK = Path(args.root).resolve()
++ if not (FRAMEWORK / "agents" / "orchestrator.md").is_file():
++ print(f"error: {args.root} has no agents/orchestrator.md", file=sys.stderr)
++ return 2
++
++ results = []
++ for arg in args.paths:
++ path = Path(arg).resolve()
++ if not path.exists():
++ print(f"error: no such path: {arg}", file=sys.stderr)
++ return 2
++ try:
++ path.relative_to(FRAMEWORK)
++ except ValueError:
++ print(f"error: {arg} is not inside {FRAMEWORK}", file=sys.stderr)
++ return 2
++ kind = classify(path)
++ report = Report()
++ if kind == "agent":
++ check_agent(path, report)
++ elif kind == "skill":
++ check_skill(path, report)
++ elif kind == "scenario":
++ check_scenario(path, report)
++ elif kind == "tool":
++ check_tool(path, report)
++ if kind in ("agent", "skill", "file"):
++ if path.is_dir():
++ print(f"error: pass a file, not a directory: {arg}", file=sys.stderr)
++ return 2
++ check_common(path, report)
++ results.append({"artifact": rel(path), "kind": kind,
++ "checks": report.checks, "findings": report.findings})
++
++ blocking = sum(f["severity"] in BLOCKING for r in results for f in r["findings"])
++ if args.json:
++ print(json.dumps({"framework": str(FRAMEWORK), "results": results,
++ "blocking": blocking}, indent=2))
++ else:
++ cutoff = SEVERITIES.index(args.min_severity)
++ for r in results:
++ print(f"== {r['artifact']} ({r['kind']}): {len(r['checks'])} checks ran")
++ order = sorted(r["findings"], key=lambda f: (SEVERITIES.index(f["severity"]),
++ f["check"], f["path"], f["line"] or 0))
++ for f in order:
++ if SEVERITIES.index(f["severity"]) > cutoff:
++ continue
++ loc = f"{f['path']}:{f['line']}" if f["line"] else f["path"]
++ print(f"{f['severity']:<8} {f['check']:<26} {loc} {f['message']}")
++ print(f"{blocking} blocking (High or Critical) finding(s)")
++ return 1 if blocking else 0
++
++
++ if __name__ == "__main__":
++ sys.exit(main())