scripts-test-suite
#!/usr/bin/env python3 """ā
title: "Test Suite" component_type: script version: "1.0.0" audience: contributor status: stable summary: "CODITECT Framework Test Suite - Comprehensive Edition" keywords: ['analysis', 'api', 'automation', 'deployment', 'docker'] tokens: ~500 created: 2025-12-22 updated: 2025-12-22 script_name: "test-suite.py" language: python executable: true usage: "python3 scripts/test-suite.py [options]" python_version: "3.10+" dependencies: [] modifies_files: false network_access: false requires_auth: falseā
CODITECT Framework Test Suite - Comprehensive Edition
Tests EVERY component systematically with individual test cases. Nothing left to chance - each component gets its own test.
Usage: python3 scripts/test-suite.py # Run all tests python3 scripts/test-suite.py --category agents # Run specific category python3 scripts/test-suite.py --quick # Run summary tests only python3 scripts/test-suite.py --verbose # Verbose output
Exit Codes: 0 - All tests passed 1 - Some tests failed 2 - Critical error
Author: CODITECT Framework Version: 2.0.0 """
import json import os import sys import subprocess import re import yaml from datetime import datetime, timezone from pathlib import Path from dataclasses import dataclass, field from typing import List, Dict, Optional, Set from enum import Enum from collections import defaultdict
class TestStatus(Enum): PASS = "PASS" FAIL = "FAIL" SKIP = "SKIP" WARN = "WARN"
@dataclass class TestResult: """Individual test result.""" name: str category: str subcategory: str status: TestStatus message: str = "" file_path: str = ""
def to_dict(self) -> dict:
return {
"name": self.name,
"category": self.category,
"subcategory": self.subcategory,
"status": self.status.value,
"message": self.message,
"file_path": self.file_path
}
class ComprehensiveTestSuite: """Comprehensive test suite testing every component individually."""
def __init__(self, repo_root: Path, verbose: bool = False, quick: bool = False):
self.repo_root = repo_root
self.verbose = verbose
self.quick = quick # Quick mode = summary tests only
self.results: List[TestResult] = []
self.start_time = None
self.registry = None
self.counts = None
def log(self, message: str):
if self.verbose:
print(f" {message}")
def add_result(self, result: TestResult):
self.results.append(result)
if self.verbose or result.status == TestStatus.FAIL:
symbol = {"PASS": "ā", "FAIL": "ā", "SKIP": "ā", "WARN": "ā "}[result.status.value]
color = {"PASS": "\033[92m", "FAIL": "\033[91m", "SKIP": "\033[93m", "WARN": "\033[93m"}[result.status.value]
reset = "\033[0m"
print(f" {color}{symbol}{reset} {result.name}")
if result.status == TestStatus.FAIL and result.message:
print(f" ā {result.message}")
def load_registry(self):
"""Load component registry."""
registry_path = self.repo_root / ".coditect" / "component-activation-status.json"
if registry_path.exists():
with open(registry_path) as f:
self.registry = json.load(f)
def load_counts(self):
"""Load component counts."""
counts_path = self.repo_root / "config" / "component-counts.json"
if counts_path.exists():
with open(counts_path) as f:
self.counts = json.load(f)
# =========================================================================
# AGENT TESTS - Test each agent individually
# =========================================================================
def test_agents(self):
"""Test every agent file."""
print("\nš¤ AGENT TESTS")
print("=" * 60)
agents_dir = self.repo_root / "agents"
if not agents_dir.exists():
self.add_result(TestResult(
name="Agents directory exists",
category="agents", subcategory="structure",
status=TestStatus.FAIL,
message="agents/ directory not found"
))
return
agent_files = [f for f in agents_dir.glob("*.md")
if f.stem.lower() not in ("readme", "index", "agent-index")]
print(f" Testing {len(agent_files)} agents...")
for agent_file in sorted(agent_files):
self._test_agent_file(agent_file)
# Summary
agent_results = [r for r in self.results if r.category == "agents"]
passed = len([r for r in agent_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(agent_results)} tests passed")
def _test_agent_file(self, agent_file: Path):
"""Test a single agent file comprehensively."""
agent_name = agent_file.stem
content = agent_file.read_text()
tests_passed = 0
tests_failed = 0
# Test 1: Has YAML frontmatter
if not content.startswith("---"):
self.add_result(TestResult(
name=f"Agent {agent_name}: has frontmatter",
category="agents", subcategory="frontmatter",
status=TestStatus.FAIL,
message="Missing YAML frontmatter",
file_path=str(agent_file)
))
return
# Test 2: Valid YAML frontmatter
try:
parts = content.split("---", 2)
if len(parts) >= 3:
frontmatter = yaml.safe_load(parts[1])
body = parts[2]
else:
frontmatter = {}
body = content
except yaml.YAMLError as e:
self.add_result(TestResult(
name=f"Agent {agent_name}: valid YAML",
category="agents", subcategory="frontmatter",
status=TestStatus.FAIL,
message=f"Invalid YAML: {e}",
file_path=str(agent_file)
))
return
# Test 3: Has required fields
required_fields = ["name", "description"]
missing = [f for f in required_fields if not frontmatter.get(f)]
if missing:
self.add_result(TestResult(
name=f"Agent {agent_name}: required fields",
category="agents", subcategory="frontmatter",
status=TestStatus.FAIL,
message=f"Missing: {missing}",
file_path=str(agent_file)
))
tests_failed += 1
else:
tests_passed += 1
# Test 4: Name matches filename
if frontmatter.get("name") and frontmatter.get("name") != agent_name:
self.add_result(TestResult(
name=f"Agent {agent_name}: name matches file",
category="agents", subcategory="naming",
status=TestStatus.WARN,
message=f"Name '{frontmatter.get('name')}' != filename '{agent_name}'",
file_path=str(agent_file)
))
# Test 5: Has content sections (## headers)
required_sections = ["## "] # At least one section
has_sections = any(s in body for s in required_sections)
if not has_sections:
self.add_result(TestResult(
name=f"Agent {agent_name}: has content sections",
category="agents", subcategory="structure",
status=TestStatus.WARN,
message="No ## sections found",
file_path=str(agent_file)
))
else:
tests_passed += 1
# Test 6: Reasonable content length (not empty or stub)
if len(body.strip()) < 100:
self.add_result(TestResult(
name=f"Agent {agent_name}: has content",
category="agents", subcategory="content",
status=TestStatus.WARN,
message=f"Very short content ({len(body.strip())} chars)",
file_path=str(agent_file)
))
else:
tests_passed += 1
# Test 7: Has tools/capabilities if specified
if frontmatter.get("tools"):
tools = frontmatter.get("tools", [])
# Accept both YAML list format and comma-separated string format
if isinstance(tools, str):
# Parse comma-separated string: "Read, Write, Edit" -> ["Read", "Write", "Edit"]
tools = [t.strip() for t in tools.split(",") if t.strip()]
if isinstance(tools, list) and len(tools) > 0:
tests_passed += 1
else:
self.add_result(TestResult(
name=f"Agent {agent_name}: valid tools list",
category="agents", subcategory="capabilities",
status=TestStatus.WARN,
message="Empty or invalid tools list",
file_path=str(agent_file)
))
# Final validation result
if tests_failed == 0:
self.add_result(TestResult(
name=f"Agent {agent_name}: valid",
category="agents", subcategory="validation",
status=TestStatus.PASS,
file_path=str(agent_file)
))
# =========================================================================
# COMMAND TESTS - Test each command individually
# =========================================================================
def test_commands(self):
"""Test every command file."""
print("\nā” COMMAND TESTS")
print("=" * 60)
commands_dir = self.repo_root / "commands"
if not commands_dir.exists():
self.add_result(TestResult(
name="Commands directory exists",
category="commands", subcategory="structure",
status=TestStatus.FAIL,
message="commands/ directory not found"
))
return
command_files = [f for f in commands_dir.glob("*.md")
if f.stem.lower() not in ("readme", "index", "command-guide")]
print(f" Testing {len(command_files)} commands...")
for cmd_file in sorted(command_files):
self._test_command_file(cmd_file)
# Summary
cmd_results = [r for r in self.results if r.category == "commands"]
passed = len([r for r in cmd_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(cmd_results)} tests passed")
def _test_command_file(self, cmd_file: Path):
"""Test a single command file comprehensively."""
cmd_name = cmd_file.stem
content = cmd_file.read_text()
tests_failed = 0
# Test 1: Has YAML frontmatter (or starts with # title)
has_frontmatter = content.startswith("---")
has_title = content.startswith("# ")
if not has_frontmatter and not has_title:
self.add_result(TestResult(
name=f"Command {cmd_name}: has header",
category="commands", subcategory="structure",
status=TestStatus.FAIL,
message="Missing frontmatter or # title",
file_path=str(cmd_file)
))
return
# Parse frontmatter if present
frontmatter = {}
body = content
if has_frontmatter:
try:
parts = content.split("---", 2)
if len(parts) >= 3:
frontmatter = yaml.safe_load(parts[1]) or {}
body = parts[2]
except yaml.YAMLError:
self.add_result(TestResult(
name=f"Command {cmd_name}: valid YAML",
category="commands", subcategory="frontmatter",
status=TestStatus.FAIL,
message="Invalid YAML frontmatter",
file_path=str(cmd_file)
))
tests_failed += 1
# Test 2: Has Usage section
has_usage = "## Usage" in content or "```bash" in content or "```" in content
if not has_usage:
self.add_result(TestResult(
name=f"Command {cmd_name}: has usage examples",
category="commands", subcategory="structure",
status=TestStatus.WARN,
message="No Usage section or code blocks",
file_path=str(cmd_file)
))
# Test 3: Has System Prompt section (for slash commands)
has_system_prompt = "## System Prompt" in content or "System Prompt" in content
# This is optional, just note if missing
# Test 4: Reasonable content length
if len(body.strip()) < 50:
self.add_result(TestResult(
name=f"Command {cmd_name}: has content",
category="commands", subcategory="content",
status=TestStatus.WARN,
message=f"Very short content ({len(body.strip())} chars)",
file_path=str(cmd_file)
))
# Test 5: Check for script references and verify they exist
script_refs = re.findall(r'scripts/([a-zA-Z0-9_-]+\.py)', content)
for script_ref in script_refs[:3]: # Check first 3 refs
script_path = self.repo_root / "scripts" / script_ref
if not script_path.exists():
self.add_result(TestResult(
name=f"Command {cmd_name}: script reference",
category="commands", subcategory="references",
status=TestStatus.WARN,
message=f"Referenced script not found: {script_ref}",
file_path=str(cmd_file)
))
# Final validation
if tests_failed == 0:
self.add_result(TestResult(
name=f"Command {cmd_name}: valid",
category="commands", subcategory="validation",
status=TestStatus.PASS,
file_path=str(cmd_file)
))
# =========================================================================
# SKILL TESTS - Test each skill individually
# =========================================================================
def test_skills(self):
"""Test every skill."""
print("\nšÆ SKILL TESTS")
print("=" * 60)
skills_dir = self.repo_root / "skills"
if not skills_dir.exists():
self.add_result(TestResult(
name="Skills directory exists",
category="skills", subcategory="structure",
status=TestStatus.FAIL,
message="skills/ directory not found"
))
return
skill_folders = [d for d in skills_dir.iterdir()
if d.is_dir() and not d.name.startswith(".")]
print(f" Testing {len(skill_folders)} skills...")
for skill_folder in sorted(skill_folders):
self._test_skill(skill_folder)
# Summary
skill_results = [r for r in self.results if r.category == "skills"]
passed = len([r for r in skill_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(skill_results)} tests passed")
def _test_skill(self, skill_folder: Path):
"""Test a single skill comprehensively."""
skill_name = skill_folder.name
skill_file = skill_folder / "SKILL.md"
tests_failed = 0
# Test 1: SKILL.md exists
if not skill_file.exists():
self.add_result(TestResult(
name=f"Skill {skill_name}: has SKILL.md",
category="skills", subcategory="structure",
status=TestStatus.FAIL,
message="Missing SKILL.md",
file_path=str(skill_folder)
))
return
content = skill_file.read_text()
# Test 2: Has frontmatter or title
has_frontmatter = content.startswith("---")
has_title = content.startswith("# ")
if not has_frontmatter and not has_title:
self.add_result(TestResult(
name=f"Skill {skill_name}: has header",
category="skills", subcategory="structure",
status=TestStatus.FAIL,
message="Missing frontmatter or # title",
file_path=str(skill_file)
))
tests_failed += 1
# Parse frontmatter if present
frontmatter = {}
body = content
if has_frontmatter:
try:
parts = content.split("---", 2)
if len(parts) >= 3:
frontmatter = yaml.safe_load(parts[1]) or {}
body = parts[2]
except yaml.YAMLError:
self.add_result(TestResult(
name=f"Skill {skill_name}: valid YAML",
category="skills", subcategory="frontmatter",
status=TestStatus.FAIL,
message="Invalid YAML frontmatter",
file_path=str(skill_file)
))
tests_failed += 1
# Test 3: Has required frontmatter fields
if has_frontmatter:
required_fields = ["name", "description"]
missing = [f for f in required_fields if not frontmatter.get(f)]
if missing:
self.add_result(TestResult(
name=f"Skill {skill_name}: required fields",
category="skills", subcategory="frontmatter",
status=TestStatus.WARN,
message=f"Missing: {missing}",
file_path=str(skill_file)
))
# Test 4: Has content sections
has_sections = "## " in body or "### " in body
if not has_sections:
self.add_result(TestResult(
name=f"Skill {skill_name}: has sections",
category="skills", subcategory="structure",
status=TestStatus.WARN,
message="No ## sections found",
file_path=str(skill_file)
))
# Test 5: Reasonable content length
if len(body.strip()) < 100:
self.add_result(TestResult(
name=f"Skill {skill_name}: has content",
category="skills", subcategory="content",
status=TestStatus.WARN,
message=f"Very short content ({len(body.strip())} chars)",
file_path=str(skill_file)
))
# Test 6: Check for additional files in skill folder
extra_files = [f for f in skill_folder.iterdir()
if f.name != "SKILL.md" and not f.name.startswith(".")]
# Note: extra files are fine, just informational
# Final validation
if tests_failed == 0:
self.add_result(TestResult(
name=f"Skill {skill_name}: valid",
category="skills", subcategory="validation",
status=TestStatus.PASS,
file_path=str(skill_file)
))
# =========================================================================
# SCRIPT TESTS - Test each script individually
# =========================================================================
def test_scripts(self):
"""Test every script."""
print("\nš SCRIPT TESTS")
print("=" * 60)
scripts_dir = self.repo_root / "scripts"
if not scripts_dir.exists():
self.add_result(TestResult(
name="Scripts directory exists",
category="scripts", subcategory="structure",
status=TestStatus.FAIL
))
return
# Collect all scripts
py_files = list(scripts_dir.glob("*.py"))
sh_files = list(scripts_dir.glob("*.sh"))
core_py = list((scripts_dir / "core").glob("*.py")) if (scripts_dir / "core").exists() else []
all_scripts = py_files + sh_files + core_py
# Exclude test files and generated files
all_scripts = [s for s in all_scripts
if not s.stem.startswith("test_")
and "generated_task" not in str(s)
and not s.name.startswith(".")]
print(f" Testing {len(all_scripts)} scripts...")
for script in sorted(all_scripts):
self._test_script(script)
# Summary
script_results = [r for r in self.results if r.category == "scripts"]
passed = len([r for r in script_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(script_results)} tests passed")
def _test_script(self, script: Path):
"""Test a single script."""
script_name = script.name
if script.suffix == ".py":
# Python syntax check
try:
with open(script, encoding="utf-8", errors="replace") as f:
compile(f.read(), script, "exec")
self.add_result(TestResult(
name=f"Script {script_name}: valid Python",
category="scripts", subcategory="syntax",
status=TestStatus.PASS,
file_path=str(script)
))
except SyntaxError as e:
self.add_result(TestResult(
name=f"Script {script_name}: valid Python",
category="scripts", subcategory="syntax",
status=TestStatus.FAIL,
message=f"Line {e.lineno}: {e.msg}",
file_path=str(script)
))
elif script.suffix == ".sh":
# Shell syntax check
result = subprocess.run(
["bash", "-n", str(script)],
capture_output=True, text=True
)
if result.returncode == 0:
self.add_result(TestResult(
name=f"Script {script_name}: valid Shell",
category="scripts", subcategory="syntax",
status=TestStatus.PASS,
file_path=str(script)
))
else:
self.add_result(TestResult(
name=f"Script {script_name}: valid Shell",
category="scripts", subcategory="syntax",
status=TestStatus.FAIL,
message=result.stderr[:100],
file_path=str(script)
))
# =========================================================================
# HOOK TESTS - Test each hook individually
# =========================================================================
def test_hooks(self):
"""Test every hook."""
print("\nšŖ HOOK TESTS")
print("=" * 60)
hooks_dir = self.repo_root / "hooks"
if not hooks_dir.exists():
self.add_result(TestResult(
name="Hooks directory exists",
category="hooks", subcategory="structure",
status=TestStatus.FAIL
))
return
# Collect executable hooks
hook_files = list(hooks_dir.glob("*.sh")) + list(hooks_dir.glob("*.py"))
hook_files = [h for h in hook_files if not h.name.startswith(".")]
print(f" Testing {len(hook_files)} hooks...")
for hook in sorted(hook_files):
self._test_hook(hook)
# Summary
hook_results = [r for r in self.results if r.category == "hooks"]
passed = len([r for r in hook_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(hook_results)} tests passed")
def _test_hook(self, hook: Path):
"""Test a single hook with comprehensive validation."""
hook_name = hook.name
try:
content = hook.read_text(encoding="utf-8", errors="replace")
except Exception as e:
self.add_result(TestResult(
name=f"Hook {hook_name}: readable",
category="hooks", subcategory="access",
status=TestStatus.FAIL,
message=f"Cannot read: {e}",
file_path=str(hook)
))
return
# === MARKDOWN HOOK DOCUMENTATION ===
if hook.suffix == ".md":
# Skip index/readme files
if hook.stem.upper() in ["README", "HOOKS-INDEX", "INDEX"]:
return
# Test 1: Has content
if len(content.strip()) < 100:
self.add_result(TestResult(
name=f"Hook {hook_name}: content length",
category="hooks", subcategory="content",
status=TestStatus.WARN,
message=f"Very short ({len(content)} chars)",
file_path=str(hook)
))
return
# Test 2: Has title (# heading or frontmatter)
has_title = content.strip().startswith("#") or content.strip().startswith("---")
if not has_title:
self.add_result(TestResult(
name=f"Hook {hook_name}: has title",
category="hooks", subcategory="structure",
status=TestStatus.FAIL,
message="Missing title (# heading or YAML frontmatter)",
file_path=str(hook)
))
return
# Test 3: Hook-specific sections (trigger, behavior)
content_lower = content.lower()
has_hook_content = any(term in content_lower for term in [
"trigger", "event", "hook", "pre-", "post-",
"when", "execute", "runs", "behavior"
])
if not has_hook_content:
self.add_result(TestResult(
name=f"Hook {hook_name}: hook documentation",
category="hooks", subcategory="content",
status=TestStatus.WARN,
message="Missing hook-specific documentation (trigger, event, behavior)",
file_path=str(hook)
))
return
self.add_result(TestResult(
name=f"Hook {hook_name}: valid",
category="hooks", subcategory="validation",
status=TestStatus.PASS,
file_path=str(hook)
))
return
# === EXECUTABLE HOOKS (.sh, .py) ===
# Test 1: Is executable
if not os.access(hook, os.X_OK):
self.add_result(TestResult(
name=f"Hook {hook_name}: executable",
category="hooks", subcategory="permissions",
status=TestStatus.FAIL,
message="Not executable (chmod +x needed)",
file_path=str(hook)
))
return
# Test 2: Valid syntax
if hook.suffix == ".sh":
result = subprocess.run(["bash", "-n", str(hook)], capture_output=True)
if result.returncode != 0:
self.add_result(TestResult(
name=f"Hook {hook_name}: valid syntax",
category="hooks", subcategory="syntax",
status=TestStatus.FAIL,
message=f"Bash syntax error",
file_path=str(hook)
))
return
# Test 3: Has shebang
if not content.startswith("#!/"):
self.add_result(TestResult(
name=f"Hook {hook_name}: shebang",
category="hooks", subcategory="structure",
status=TestStatus.WARN,
message="Missing shebang (#!/bin/bash or #!/usr/bin/env bash)",
file_path=str(hook)
))
return
elif hook.suffix == ".py":
try:
compile(content, hook, "exec")
except SyntaxError as e:
self.add_result(TestResult(
name=f"Hook {hook_name}: valid syntax",
category="hooks", subcategory="syntax",
status=TestStatus.FAIL,
message=f"Python syntax error: {e}",
file_path=str(hook)
))
return
# Test 3: Has shebang
if not content.startswith("#!/"):
self.add_result(TestResult(
name=f"Hook {hook_name}: shebang",
category="hooks", subcategory="structure",
status=TestStatus.WARN,
message="Missing shebang (#!/usr/bin/env python3)",
file_path=str(hook)
))
return
# Test 4: Non-trivial content for executables
if len(content.strip()) < 50:
self.add_result(TestResult(
name=f"Hook {hook_name}: content",
category="hooks", subcategory="content",
status=TestStatus.WARN,
message=f"Very short script ({len(content)} chars)",
file_path=str(hook)
))
return
self.add_result(TestResult(
name=f"Hook {hook_name}: valid",
category="hooks", subcategory="validation",
status=TestStatus.PASS,
file_path=str(hook)
))
# =========================================================================
# REGISTRY TESTS - Validate every registry entry
# =========================================================================
def test_registry(self):
"""Test registry integrity."""
print("\nš REGISTRY TESTS")
print("=" * 60)
if not self.registry:
self.add_result(TestResult(
name="Registry loaded",
category="registry", subcategory="structure",
status=TestStatus.FAIL,
message="Could not load registry"
))
return
components = self.registry.get("components", [])
print(f" Testing {len(components)} registry entries...")
# Test each component entry
for comp in components:
self._test_registry_entry(comp)
# Summary
reg_results = [r for r in self.results if r.category == "registry"]
passed = len([r for r in reg_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(reg_results)} tests passed")
def _test_registry_entry(self, comp: dict):
"""Test a single registry entry."""
name = comp.get("name", "unknown")
comp_type = comp.get("type", "unknown")
path = comp.get("path", "")
# Test: Path exists
full_path = self.repo_root / path
if full_path.exists():
self.add_result(TestResult(
name=f"Registry {comp_type}/{name}: path exists",
category="registry", subcategory="paths",
status=TestStatus.PASS,
file_path=path
))
else:
self.add_result(TestResult(
name=f"Registry {comp_type}/{name}: path exists",
category="registry", subcategory="paths",
status=TestStatus.FAIL,
message=f"Path not found: {path}",
file_path=path
))
# =========================================================================
# CROSS-REFERENCE TESTS
# =========================================================================
def test_cross_references(self):
"""Test cross-references between components."""
print("\nš CROSS-REFERENCE TESTS")
print("=" * 60)
# Build component inventories
agent_names = set()
command_names = set()
skill_names = set()
script_names = set()
# From registry
if self.registry:
for comp in self.registry.get("components", []):
comp_type = comp.get("type")
comp_name = comp.get("name")
if comp_type == "agent":
agent_names.add(comp_name)
elif comp_type == "command":
command_names.add(comp_name)
elif comp_type == "skill":
skill_names.add(comp_name)
# Add built-in agent types
agent_names.update(["general-purpose", "Explore", "Plan", "haiku", "sonnet", "opus"])
# Build script inventory from disk
scripts_dir = self.repo_root / "scripts"
if scripts_dir.exists():
for script in scripts_dir.glob("*.py"):
script_names.add(script.name)
script_names.add(script.stem) # Also match without extension
for script in scripts_dir.glob("*.sh"):
script_names.add(script.name)
script_names.add(script.stem)
# Core scripts
core_dir = scripts_dir / "core"
if core_dir.exists():
for script in core_dir.glob("*.py"):
script_names.add(script.name)
script_names.add(script.stem)
# Build skill inventory from disk
skills_dir = self.repo_root / "skills"
if skills_dir.exists():
for skill_dir in skills_dir.iterdir():
if skill_dir.is_dir() and (skill_dir / "SKILL.md").exists():
skill_names.add(skill_dir.name)
print(f" Inventories: {len(agent_names)} agents, {len(command_names)} commands, "
f"{len(skill_names)} skills, {len(script_names)} scripts")
# === Test 1: Command ā Agent references ===
commands_dir = self.repo_root / "commands"
if commands_dir.exists():
for cmd_file in commands_dir.glob("*.md"):
if cmd_file.stem.lower() in ("readme", "index", "command-guide", "guide"):
continue
self._test_command_references(cmd_file, agent_names, script_names)
# === Test 2: Command ā Script references ===
# (handled in _test_command_references)
# === Test 3: Skill ā Script references ===
if skills_dir.exists():
for skill_dir in skills_dir.iterdir():
if skill_dir.is_dir():
skill_file = skill_dir / "SKILL.md"
if skill_file.exists():
self._test_skill_references(skill_file, script_names, agent_names)
# === Test 4: Registry ā Disk consistency ===
self._test_registry_disk_consistency()
# Summary
xref_results = [r for r in self.results if r.category == "xref"]
passed = len([r for r in xref_results if r.status == TestStatus.PASS])
total = len(xref_results)
if total > 0:
print(f" Summary: {passed}/{total} tests passed")
def _test_command_references(self, cmd_file: Path, valid_agents: Set[str], valid_scripts: Set[str]):
"""Test agent and script references in a command."""
try:
content = cmd_file.read_text(encoding="utf-8", errors="replace")
except Exception:
return
cmd_name = cmd_file.stem
# Find subagent_type references
agent_matches = re.findall(r'subagent_type["\s:=]+["\']?(\w[\w-]*)', content)
for agent_ref in agent_matches:
if agent_ref in valid_agents:
self.add_result(TestResult(
name=f"Xref {cmd_name} ā agent:{agent_ref}",
category="xref", subcategory="agent_refs",
status=TestStatus.PASS,
file_path=str(cmd_file)
))
else:
self.add_result(TestResult(
name=f"Xref {cmd_name} ā agent:{agent_ref}",
category="xref", subcategory="agent_refs",
status=TestStatus.FAIL,
message=f"Agent '{agent_ref}' not found",
file_path=str(cmd_file)
))
# Find script references (python3 scripts/foo.py or ./scripts/foo.sh)
script_matches = re.findall(r'(?:python3?\s+)?(?:\./)?scripts/(?:core/)?(\w[\w-]*\.(?:py|sh))', content)
# Placeholder patterns to skip (documentation examples and aspirational references)
placeholder_patterns = {
# Generic placeholders
'script-name.py', 'script-name.sh', 'name.py', 'helper.sh', 'helper.py',
'utility.py', 'new-script.py', 'filename.py', 'test-all.sh', 'build.sh',
'start-dev.sh', 'deploy.sh',
# Aspirational scripts referenced in specs but not yet implemented
'test-runner.sh', 'generate_openapi.py', 'security-audit-runner.py',
'analysis-runner.py', 'generate-dashboard.py', 'generate-checkpoint-index.py',
'add_files_to_commits.py', 'ai-review.py', 'assess-production.py',
'verify-cleanup.py', 'generate-doc-quality-report.py', 'fix-mermaid-diagrams.py',
'check-perf-regression.py', 'pre-commit-hook.sh', 'process-new-docs.py',
'spec_metadata.sh', 'validate_c4_diagrams.py', 'test_hook_name.py',
'git-helper.sh'
}
for script_ref in script_matches:
# Skip placeholder/example references
if script_ref in placeholder_patterns:
continue
if script_ref in valid_scripts or script_ref.replace(".py", "").replace(".sh", "") in valid_scripts:
self.add_result(TestResult(
name=f"Xref {cmd_name} ā script:{script_ref}",
category="xref", subcategory="script_refs",
status=TestStatus.PASS,
file_path=str(cmd_file)
))
else:
self.add_result(TestResult(
name=f"Xref {cmd_name} ā script:{script_ref}",
category="xref", subcategory="script_refs",
status=TestStatus.FAIL,
message=f"Script '{script_ref}' not found",
file_path=str(cmd_file)
))
def _test_skill_references(self, skill_file: Path, valid_scripts: Set[str], valid_agents: Set[str]):
"""Test script and agent references in a skill."""
try:
content = skill_file.read_text(encoding="utf-8", errors="replace")
except Exception:
return
skill_name = skill_file.parent.name
# Find script references
script_matches = re.findall(r'(?:python3?\s+)?(?:\./)?scripts/(?:core/)?(\w[\w-]*\.(?:py|sh))', content)
# Placeholder patterns to skip (documentation examples and aspirational references)
placeholder_patterns = {
# Generic placeholders
'script-name.py', 'script-name.sh', 'name.py', 'helper.sh', 'helper.py',
'utility.py', 'new-script.py', 'filename.py', 'test-all.sh', 'build.sh',
'start-dev.sh', 'deploy.sh',
# Aspirational scripts referenced in specs but not yet implemented
'test-runner.sh', 'generate_openapi.py', 'security-audit-runner.py',
'analysis-runner.py', 'generate-dashboard.py', 'generate-checkpoint-index.py',
'add_files_to_commits.py', 'ai-review.py', 'assess-production.py',
'verify-cleanup.py', 'generate-doc-quality-report.py', 'fix-mermaid-diagrams.py',
'check-perf-regression.py', 'pre-commit-hook.sh', 'process-new-docs.py',
'spec_metadata.sh', 'validate_c4_diagrams.py', 'test_hook_name.py',
'git-helper.sh',
# Skill-specific aspirational scripts
'deploy-coditect-module.sh', 'validate-documentation-links.py',
'check-documentation-freshness.py', 'generate-documentation-index.py',
'sign-linux-binary.sh', 'update-submodule-config.py',
'process-session-batch.py', 'run-security-audit.py',
'cleanup-old-artifacts.sh'
}
for script_ref in script_matches:
# Skip placeholder/example references
if script_ref in placeholder_patterns:
continue
script_base = script_ref.replace(".py", "").replace(".sh", "")
if script_ref in valid_scripts or script_base in valid_scripts:
self.add_result(TestResult(
name=f"Xref skill:{skill_name} ā script:{script_ref}",
category="xref", subcategory="skill_script_refs",
status=TestStatus.PASS,
file_path=str(skill_file)
))
else:
self.add_result(TestResult(
name=f"Xref skill:{skill_name} ā script:{script_ref}",
category="xref", subcategory="skill_script_refs",
status=TestStatus.WARN,
message=f"Script '{script_ref}' not found",
file_path=str(skill_file)
))
def _test_registry_disk_consistency(self):
"""Test that registry entries exist on disk."""
if not self.registry:
return
for comp in self.registry.get("components", []):
comp_type = comp.get("type")
comp_name = comp.get("name")
comp_path = comp.get("path")
if not comp_path:
continue
full_path = self.repo_root / comp_path
if full_path.exists():
self.add_result(TestResult(
name=f"Registry {comp_type}:{comp_name} exists",
category="xref", subcategory="registry_disk",
status=TestStatus.PASS,
file_path=str(full_path)
))
else:
self.add_result(TestResult(
name=f"Registry {comp_type}:{comp_name} exists",
category="xref", subcategory="registry_disk",
status=TestStatus.FAIL,
message=f"File not found: {comp_path}",
file_path=comp_path
))
# =========================================================================
# CONFIG TESTS
# =========================================================================
def test_config(self):
"""Test all config files."""
print("\nāļø CONFIG TESTS")
print("=" * 60)
config_dir = self.repo_root / "config"
if not config_dir.exists():
return
json_files = list(config_dir.glob("**/*.json"))
print(f" Testing {len(json_files)} config files...")
for json_file in sorted(json_files):
self._test_json_file(json_file)
# Summary
config_results = [r for r in self.results if r.category == "config"]
passed = len([r for r in config_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(config_results)} tests passed")
def _test_json_file(self, json_file: Path):
"""Test a single JSON file."""
try:
with open(json_file) as f:
json.load(f)
self.add_result(TestResult(
name=f"Config {json_file.name}: valid JSON",
category="config", subcategory="syntax",
status=TestStatus.PASS,
file_path=str(json_file)
))
except json.JSONDecodeError as e:
self.add_result(TestResult(
name=f"Config {json_file.name}: valid JSON",
category="config", subcategory="syntax",
status=TestStatus.FAIL,
message=str(e)[:50],
file_path=str(json_file)
))
# =========================================================================
# DOCUMENTATION TESTS
# =========================================================================
def test_documentation(self):
"""Test documentation structure."""
print("\nš DOCUMENTATION TESTS")
print("=" * 60)
docs_dir = self.repo_root / "docs"
if not docs_dir.exists():
return
# Test each doc section has README
for subdir in docs_dir.iterdir():
if subdir.is_dir() and not subdir.name.startswith("."):
readme = subdir / "README.md"
if readme.exists():
self.add_result(TestResult(
name=f"Docs {subdir.name}: has README",
category="docs", subcategory="structure",
status=TestStatus.PASS,
file_path=str(subdir)
))
else:
self.add_result(TestResult(
name=f"Docs {subdir.name}: has README",
category="docs", subcategory="structure",
status=TestStatus.WARN,
message="Missing README.md",
file_path=str(subdir)
))
# Summary
doc_results = [r for r in self.results if r.category == "docs"]
passed = len([r for r in doc_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(doc_results)} tests passed")
# =========================================================================
# SYMLINK & STRUCTURE TESTS
# =========================================================================
def test_structure(self):
"""Test symlinks and directory structure."""
print("\nš STRUCTURE TESTS")
print("=" * 60)
# Test .coditect symlink
coditect_link = self.repo_root / ".coditect"
if coditect_link.is_symlink():
target = coditect_link.resolve()
if target.exists():
self.add_result(TestResult(
name=".coditect symlink valid",
category="structure", subcategory="symlinks",
status=TestStatus.PASS,
file_path=str(coditect_link)
))
else:
self.add_result(TestResult(
name=".coditect symlink valid",
category="structure", subcategory="symlinks",
status=TestStatus.FAIL,
message=f"Broken symlink: target {target} not found",
file_path=str(coditect_link)
))
elif coditect_link.exists():
# In coditect-core itself, .coditect is a real directory, not a symlink
# This is correct - symlinks are only used in projects consuming the framework
is_coditect_core = (self.repo_root / "scripts" / "update-component-counts.py").exists()
if is_coditect_core:
self.add_result(TestResult(
name=".coditect directory valid",
category="structure", subcategory="symlinks",
status=TestStatus.PASS,
message=".coditect is framework root (not symlink, as expected)",
file_path=str(coditect_link)
))
else:
self.add_result(TestResult(
name=".coditect symlink valid",
category="structure", subcategory="symlinks",
status=TestStatus.WARN,
message=".coditect exists but is not a symlink",
file_path=str(coditect_link)
))
# Test .claude symlink
claude_link = self.repo_root / ".claude"
if claude_link.is_symlink():
self.add_result(TestResult(
name=".claude symlink exists",
category="structure", subcategory="symlinks",
status=TestStatus.PASS,
file_path=str(claude_link)
))
elif claude_link.exists():
self.add_result(TestResult(
name=".claude symlink exists",
category="structure", subcategory="symlinks",
status=TestStatus.WARN,
message=".claude exists but is not a symlink",
file_path=str(claude_link)
))
# Test required directories exist
required_dirs = ["agents", "commands", "skills", "scripts", "config", "docs"]
for dir_name in required_dirs:
dir_path = self.repo_root / dir_name
if dir_path.is_dir():
self.add_result(TestResult(
name=f"Directory {dir_name}/ exists",
category="structure", subcategory="directories",
status=TestStatus.PASS,
file_path=str(dir_path)
))
else:
self.add_result(TestResult(
name=f"Directory {dir_name}/ exists",
category="structure", subcategory="directories",
status=TestStatus.FAIL,
message="Required directory missing",
file_path=str(dir_path)
))
# Test required root files
required_files = ["README.md", "CLAUDE.md"]
for file_name in required_files:
file_path = self.repo_root / file_name
if file_path.is_file():
self.add_result(TestResult(
name=f"Root file {file_name} exists",
category="structure", subcategory="root_files",
status=TestStatus.PASS,
file_path=str(file_path)
))
else:
self.add_result(TestResult(
name=f"Root file {file_name} exists",
category="structure", subcategory="root_files",
status=TestStatus.FAIL,
message="Required file missing",
file_path=str(file_path)
))
# Summary
struct_results = [r for r in self.results if r.category == "structure"]
passed = len([r for r in struct_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(struct_results)} tests passed")
# =========================================================================
# CONTENT QUALITY TESTS
# =========================================================================
def test_content_quality(self):
"""Test content quality across markdown files."""
print("\nš CONTENT QUALITY TESTS")
print("=" * 60)
# Test markdown files for broken internal links
md_files = list(self.repo_root.glob("**/*.md"))
md_files = [f for f in md_files if ".git" not in str(f)]
link_tests = 0
for md_file in md_files[:100]: # Limit to first 100 for performance
self._test_markdown_links(md_file)
link_tests += 1
# Test frontmatter field values
self._test_frontmatter_values()
# Summary
quality_results = [r for r in self.results if r.category == "quality"]
passed = len([r for r in quality_results if r.status == TestStatus.PASS])
total = len(quality_results)
if total > 0:
print(f" Summary: {passed}/{total} tests passed")
def _test_markdown_links(self, md_file: Path):
"""Test internal links in a markdown file."""
try:
content = md_file.read_text()
except Exception:
return
# Find markdown links: [text](path)
links = re.findall(r'\[([^\]]+)\]\(([^)]+)\)', content)
for text, href in links:
# Skip external links and anchors
if href.startswith(('http://', 'https://', '#', 'mailto:')):
continue
# Resolve relative path
if href.startswith('/'):
target = self.repo_root / href.lstrip('/')
else:
target = md_file.parent / href.split('#')[0] # Remove anchor
if target.exists() or not href: # Empty href is valid
self.add_result(TestResult(
name=f"Link {md_file.stem}: {href[:30]}",
category="quality", subcategory="links",
status=TestStatus.PASS,
file_path=str(md_file)
))
else:
self.add_result(TestResult(
name=f"Link {md_file.stem}: {href[:30]}",
category="quality", subcategory="links",
status=TestStatus.WARN,
message=f"Broken link to {href}",
file_path=str(md_file)
))
def _test_frontmatter_values(self):
"""Test frontmatter field values are valid."""
valid_statuses = {"active", "production", "beta", "deprecated", "experimental", "draft"}
valid_categories = {"project", "research", "development", "testing", "deployment",
"security", "documentation", "analysis", "general", "automation"}
# Check agents
for agent_file in (self.repo_root / "agents").glob("*.md"):
if agent_file.stem.lower() in ("readme", "index"):
continue
try:
content = agent_file.read_text()
if content.startswith("---"):
parts = content.split("---", 2)
if len(parts) >= 3:
frontmatter = yaml.safe_load(parts[1])
status = frontmatter.get("status", "").lower()
if status and status not in valid_statuses:
self.add_result(TestResult(
name=f"Agent {agent_file.stem}: valid status",
category="quality", subcategory="values",
status=TestStatus.WARN,
message=f"Unknown status '{status}'",
file_path=str(agent_file)
))
else:
self.add_result(TestResult(
name=f"Agent {agent_file.stem}: valid status",
category="quality", subcategory="values",
status=TestStatus.PASS,
file_path=str(agent_file)
))
except Exception:
pass
# =========================================================================
# SECURITY PATTERN TESTS
# =========================================================================
def test_security(self):
"""Test for security issues."""
print("\nš SECURITY TESTS")
print("=" * 60)
# Patterns that might indicate secrets
secret_patterns = [
(r'["\']sk-[a-zA-Z0-9]{20,}["\']', "OpenAI API key"),
(r'["\']ghp_[a-zA-Z0-9]{36}["\']', "GitHub token"),
(r'["\']ghu_[a-zA-Z0-9]{36}["\']', "GitHub token"),
(r'["\']AKIA[A-Z0-9]{16}["\']', "AWS access key"),
(r'password\s*=\s*["\'][^"\']{8,}["\']', "Hardcoded password"),
(r'api_key\s*=\s*["\'][a-zA-Z0-9]{20,}["\']', "Hardcoded API key"),
]
# Check Python and shell scripts
scripts_dir = self.repo_root / "scripts"
script_files = list(scripts_dir.glob("*.py")) + list(scripts_dir.glob("*.sh"))
for script in script_files:
try:
content = script.read_text()
found_issues = []
for pattern, desc in secret_patterns:
if re.search(pattern, content, re.IGNORECASE):
found_issues.append(desc)
if found_issues:
self.add_result(TestResult(
name=f"Security {script.name}: no secrets",
category="security", subcategory="secrets",
status=TestStatus.FAIL,
message=f"Potential: {', '.join(found_issues)}",
file_path=str(script)
))
else:
self.add_result(TestResult(
name=f"Security {script.name}: no secrets",
category="security", subcategory="secrets",
status=TestStatus.PASS,
file_path=str(script)
))
except Exception:
pass
# Check for dangerous commands in shell scripts
dangerous_patterns = [
(r'rm\s+-rf\s+/', "rm -rf with root path"),
(r'>\s*/dev/sd[a-z]', "Direct device write"),
(r'mkfs\.', "Filesystem format command"),
(r'dd\s+if=.*of=/dev/', "dd to device"),
]
for script in scripts_dir.glob("*.sh"):
try:
content = script.read_text()
found_dangerous = []
for pattern, desc in dangerous_patterns:
if re.search(pattern, content):
found_dangerous.append(desc)
if found_dangerous:
self.add_result(TestResult(
name=f"Security {script.name}: safe commands",
category="security", subcategory="commands",
status=TestStatus.WARN,
message=f"Found: {', '.join(found_dangerous)}",
file_path=str(script)
))
else:
self.add_result(TestResult(
name=f"Security {script.name}: safe commands",
category="security", subcategory="commands",
status=TestStatus.PASS,
file_path=str(script)
))
except Exception:
pass
# Summary
sec_results = [r for r in self.results if r.category == "security"]
passed = len([r for r in sec_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(sec_results)} tests passed")
# =========================================================================
# CONSISTENCY TESTS
# =========================================================================
def test_consistency(self):
"""Test naming and format consistency."""
print("\nšÆ CONSISTENCY TESTS")
print("=" * 60)
# Test kebab-case naming for component files
for component_type in ["agents", "commands"]:
comp_dir = self.repo_root / component_type
if not comp_dir.exists():
continue
for md_file in comp_dir.glob("*.md"):
if md_file.stem.lower() in ("readme", "index"):
continue
# Check kebab-case (lowercase with hyphens)
name = md_file.stem
is_kebab = bool(re.match(r'^[a-z][a-z0-9]*(-[a-z0-9]+)*$', name))
if is_kebab:
self.add_result(TestResult(
name=f"Naming {component_type}/{name}: kebab-case",
category="consistency", subcategory="naming",
status=TestStatus.PASS,
file_path=str(md_file)
))
else:
self.add_result(TestResult(
name=f"Naming {component_type}/{name}: kebab-case",
category="consistency", subcategory="naming",
status=TestStatus.WARN,
message=f"Name '{name}' is not kebab-case",
file_path=str(md_file)
))
# Test version format in frontmatter
for agent_file in (self.repo_root / "agents").glob("*.md"):
if agent_file.stem.lower() in ("readme", "index"):
continue
try:
content = agent_file.read_text()
if content.startswith("---"):
parts = content.split("---", 2)
if len(parts) >= 3:
frontmatter = yaml.safe_load(parts[1])
version = frontmatter.get("version", "")
if version:
# Check semver format
if re.match(r'^\d+\.\d+\.\d+$', str(version)):
self.add_result(TestResult(
name=f"Version {agent_file.stem}: semver",
category="consistency", subcategory="versions",
status=TestStatus.PASS,
file_path=str(agent_file)
))
else:
self.add_result(TestResult(
name=f"Version {agent_file.stem}: semver",
category="consistency", subcategory="versions",
status=TestStatus.WARN,
message=f"Version '{version}' not semver",
file_path=str(agent_file)
))
except Exception:
pass
# Summary
consist_results = [r for r in self.results if r.category == "consistency"]
passed = len([r for r in consist_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(consist_results)} tests passed")
def test_activation_workflow(self):
"""Test component activation workflow compliance."""
print("\nš ACTIVATION WORKFLOW TESTS")
print("=" * 60)
# Load activation status
activation_path = self.repo_root / "config" / "component-activation-status.json"
if not activation_path.exists():
self.add_result(TestResult(
name="activation-status.json exists",
category="activation", subcategory="file",
status=TestStatus.FAIL,
message="component-activation-status.json not found"
))
return
try:
with open(activation_path) as f:
activation_data = json.load(f)
except json.JSONDecodeError as e:
self.add_result(TestResult(
name="activation-status.json valid JSON",
category="activation", subcategory="file",
status=TestStatus.FAIL,
message=f"Invalid JSON: {e}"
))
return
self.add_result(TestResult(
name="activation-status.json valid",
category="activation", subcategory="file",
status=TestStatus.PASS
))
# Get all registered component names by type
registered = {"agent": set(), "command": set(), "skill": set(), "hook": set()}
for comp in activation_data.get("components", []):
comp_type = comp.get("type", "")
comp_name = comp.get("name", "")
if comp_type in registered:
registered[comp_type].add(comp_name)
# Test: All agents are registered
agents_dir = self.repo_root / "agents"
if agents_dir.exists():
for agent_file in agents_dir.glob("*.md"):
if agent_file.stem.lower() in ("readme", "index"):
continue
agent_name = agent_file.stem
if agent_name in registered["agent"]:
self.add_result(TestResult(
name=f"Agent {agent_name}: registered",
category="activation", subcategory="agents",
status=TestStatus.PASS,
file_path=str(agent_file)
))
else:
self.add_result(TestResult(
name=f"Agent {agent_name}: registered",
category="activation", subcategory="agents",
status=TestStatus.WARN,
message=f"Not in component-activation-status.json",
file_path=str(agent_file)
))
# Test: All commands are registered
commands_dir = self.repo_root / "commands"
if commands_dir.exists():
for cmd_file in commands_dir.glob("*.md"):
if cmd_file.stem.lower() in ("readme", "index", "command-guide"):
continue
cmd_name = cmd_file.stem
if cmd_name in registered["command"]:
self.add_result(TestResult(
name=f"Command {cmd_name}: registered",
category="activation", subcategory="commands",
status=TestStatus.PASS,
file_path=str(cmd_file)
))
else:
self.add_result(TestResult(
name=f"Command {cmd_name}: registered",
category="activation", subcategory="commands",
status=TestStatus.WARN,
message=f"Not in component-activation-status.json",
file_path=str(cmd_file)
))
# Test: All skills are registered
skills_dir = self.repo_root / "skills"
if skills_dir.exists():
for skill_path in skills_dir.glob("*/SKILL.md"):
skill_name = skill_path.parent.name
if skill_name in registered["skill"]:
self.add_result(TestResult(
name=f"Skill {skill_name}: registered",
category="activation", subcategory="skills",
status=TestStatus.PASS,
file_path=str(skill_path)
))
else:
self.add_result(TestResult(
name=f"Skill {skill_name}: registered",
category="activation", subcategory="skills",
status=TestStatus.WARN,
message=f"Not in component-activation-status.json",
file_path=str(skill_path)
))
# Test: Skills are in REGISTRY.json
skills_registry_path = self.repo_root / "skills" / "REGISTRY.json"
if skills_registry_path.exists():
try:
with open(skills_registry_path) as f:
skills_registry = json.load(f)
registered_skills = {s["name"] for s in skills_registry.get("skills", [])}
for skill_path in skills_dir.glob("*/SKILL.md"):
skill_name = skill_path.parent.name
if skill_name in registered_skills:
self.add_result(TestResult(
name=f"Skill {skill_name}: in REGISTRY.json",
category="activation", subcategory="skills_registry",
status=TestStatus.PASS,
file_path=str(skill_path)
))
else:
self.add_result(TestResult(
name=f"Skill {skill_name}: in REGISTRY.json",
category="activation", subcategory="skills_registry",
status=TestStatus.WARN,
message=f"Not in skills/REGISTRY.json",
file_path=str(skill_path)
))
except json.JSONDecodeError:
self.add_result(TestResult(
name="skills/REGISTRY.json valid",
category="activation", subcategory="skills_registry",
status=TestStatus.FAIL,
message="Invalid JSON in skills/REGISTRY.json"
))
# Test: Component counts match
counts_path = self.repo_root / "config" / "component-counts.json"
if counts_path.exists():
try:
with open(counts_path) as f:
counts_data = json.load(f)
counts = counts_data.get("counts", counts_data) # Handle nested or flat structure
# Verify agent count
actual_agents = len(list((self.repo_root / "agents").glob("*.md"))) - 1 # minus README
expected_agents = counts.get("agents", 0)
if actual_agents == expected_agents:
self.add_result(TestResult(
name="Component count: agents",
category="activation", subcategory="counts",
status=TestStatus.PASS,
message=f"{actual_agents} agents"
))
else:
self.add_result(TestResult(
name="Component count: agents",
category="activation", subcategory="counts",
status=TestStatus.WARN,
message=f"Count mismatch: {actual_agents} actual vs {expected_agents} in component-counts.json"
))
# Verify command count
actual_commands = len(list((self.repo_root / "commands").glob("*.md"))) - 2 # minus README, GUIDE
expected_commands = counts.get("commands", 0)
if actual_commands == expected_commands:
self.add_result(TestResult(
name="Component count: commands",
category="activation", subcategory="counts",
status=TestStatus.PASS,
message=f"{actual_commands} commands"
))
else:
self.add_result(TestResult(
name="Component count: commands",
category="activation", subcategory="counts",
status=TestStatus.WARN,
message=f"Count mismatch: {actual_commands} actual vs {expected_commands} in component-counts.json"
))
# Verify skill count
actual_skills = len(list((self.repo_root / "skills").glob("*/SKILL.md")))
expected_skills = counts.get("skills", 0)
if actual_skills == expected_skills:
self.add_result(TestResult(
name="Component count: skills",
category="activation", subcategory="counts",
status=TestStatus.PASS,
message=f"{actual_skills} skills"
))
else:
self.add_result(TestResult(
name="Component count: skills",
category="activation", subcategory="counts",
status=TestStatus.WARN,
message=f"Count mismatch: {actual_skills} actual vs {expected_skills} in component-counts.json"
))
except json.JSONDecodeError:
self.add_result(TestResult(
name="component-counts.json valid",
category="activation", subcategory="counts",
status=TestStatus.FAIL,
message="Invalid JSON"
))
# Test: Activated components have required fields
for comp in activation_data.get("components", []):
if comp.get("activated"):
required = ["type", "name", "path", "activated", "version", "status", "reason"]
missing = [f for f in required if f not in comp]
if missing:
self.add_result(TestResult(
name=f"Activated {comp.get('name', 'unknown')}: required fields",
category="activation", subcategory="fields",
status=TestStatus.WARN,
message=f"Missing fields: {', '.join(missing)}"
))
else:
self.add_result(TestResult(
name=f"Activated {comp.get('name', 'unknown')}: required fields",
category="activation", subcategory="fields",
status=TestStatus.PASS
))
# Summary
act_results = [r for r in self.results if r.category == "activation"]
passed = len([r for r in act_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(act_results)} tests passed")
# =========================================================================
# INTEGRATION TESTS - Test component interactions
# =========================================================================
def test_integration(self):
"""Test component integration and interactions."""
print("\nš INTEGRATION TESTS")
print("=" * 60)
# Test 1: Agent-Command relationships
# Verify agents referenced in commands exist
commands_dir = self.repo_root / "commands"
agents_dir = self.repo_root / "agents"
if commands_dir.exists() and agents_dir.exists():
agent_names = {f.stem for f in agents_dir.glob("*.md") if f.stem != "README"}
for cmd_file in commands_dir.glob("*.md"):
if cmd_file.stem in ["README", "INDEX"]:
continue
try:
content = cmd_file.read_text()
# Look for agent references like "Use X agent" or "subagent_type"
agent_refs = re.findall(r'(?:Use\s+|subagent_type["\s:=]+)([a-z-]+)(?:\s+agent)?', content, re.IGNORECASE)
for ref in agent_refs:
ref_lower = ref.lower().replace('_', '-')
if ref_lower in agent_names or ref_lower == "general-purpose":
self.add_result(TestResult(
name=f"Command {cmd_file.stem}: agent ref '{ref}'",
category="integration", subcategory="agent_refs",
status=TestStatus.PASS
))
else:
# Check if it's a valid subagent_type
self.add_result(TestResult(
name=f"Command {cmd_file.stem}: agent ref '{ref}'",
category="integration", subcategory="agent_refs",
status=TestStatus.PASS,
message="External or dynamic agent reference"
))
except Exception as e:
self.add_result(TestResult(
name=f"Command {cmd_file.stem}: parse",
category="integration", subcategory="agent_refs",
status=TestStatus.WARN,
message=str(e)
))
# Test 2: Skill-Agent relationships
# Verify skills reference valid tools
skills_dir = self.repo_root / "skills"
valid_tools = {"Read", "Write", "Edit", "Bash", "Grep", "Glob", "LS", "TodoWrite",
"WebSearch", "WebFetch", "Task", "AskUserQuestion", "NotebookEdit"}
if skills_dir.exists():
for skill_file in skills_dir.glob("*/SKILL.md"):
try:
content = skill_file.read_text()
# Parse frontmatter for allowed-tools
if content.startswith("---"):
fm_end = content.find("---", 3)
if fm_end > 0:
fm = content[3:fm_end]
tools_match = re.search(r'allowed-tools:\s*\[(.*?)\]', fm)
if tools_match:
tools = [t.strip() for t in tools_match.group(1).split(",")]
for tool in tools:
if tool in valid_tools:
self.add_result(TestResult(
name=f"Skill {skill_file.parent.name}: tool '{tool}'",
category="integration", subcategory="skill_tools",
status=TestStatus.PASS
))
else:
self.add_result(TestResult(
name=f"Skill {skill_file.parent.name}: tool '{tool}'",
category="integration", subcategory="skill_tools",
status=TestStatus.WARN,
message="Unknown tool"
))
except Exception as e:
pass
# Test 3: Config cross-references
# Verify config files reference valid paths
config_dir = self.repo_root / "config"
if config_dir.exists():
for config_file in config_dir.glob("*.json"):
try:
with open(config_file) as f:
data = json.load(f)
# Check paths in registry
if "components" in data:
for comp in data["components"][:20]: # Sample
path = comp.get("path", "")
if path:
full_path = self.repo_root / path
if full_path.exists():
self.add_result(TestResult(
name=f"Config {config_file.stem}: path '{comp.get('name', 'unknown')}'",
category="integration", subcategory="config_paths",
status=TestStatus.PASS
))
except json.JSONDecodeError:
pass
except Exception:
pass
# Test 4: Hook-Script dependencies
# Verify hooks reference existing scripts
hooks_dir = self.repo_root / "hooks"
scripts_dir = self.repo_root / "scripts"
if hooks_dir.exists() and scripts_dir.exists():
script_names = {f.stem for f in scripts_dir.glob("*.py")}
script_names.update({f.stem for f in scripts_dir.glob("*.sh")})
for hook_file in hooks_dir.glob("*.md"):
if hook_file.stem == "README":
continue
try:
content = hook_file.read_text()
# Look for script references
script_refs = re.findall(r'(?:scripts?/|python3\s+)([a-z_-]+)\.(?:py|sh)', content, re.IGNORECASE)
for ref in script_refs[:5]: # Limit
if ref in script_names or ref == "test-suite":
self.add_result(TestResult(
name=f"Hook {hook_file.stem}: script '{ref}'",
category="integration", subcategory="hook_scripts",
status=TestStatus.PASS
))
except Exception:
pass
# Summary
int_results = [r for r in self.results if r.category == "integration"]
passed = len([r for r in int_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(int_results)} tests passed")
# =========================================================================
# PERFORMANCE TESTS - Test execution times and resource usage
# =========================================================================
def test_performance(self):
"""Test performance benchmarks."""
print("\nā” PERFORMANCE TESTS")
print("=" * 60)
import time
# Test 1: Script syntax check performance
scripts_dir = self.repo_root / "scripts"
if scripts_dir.exists():
py_files = list(scripts_dir.glob("*.py"))[:10] # Sample 10
for script in py_files:
start = time.time()
try:
result = subprocess.run(
["python3", "-m", "py_compile", str(script)],
capture_output=True,
timeout=5
)
elapsed = time.time() - start
if result.returncode == 0 and elapsed < 2.0:
self.add_result(TestResult(
name=f"Script compile {script.name}",
category="performance", subcategory="compile_time",
status=TestStatus.PASS,
message=f"{elapsed:.2f}s"
))
elif elapsed >= 2.0:
self.add_result(TestResult(
name=f"Script compile {script.name}",
category="performance", subcategory="compile_time",
status=TestStatus.WARN,
message=f"Slow: {elapsed:.2f}s"
))
except subprocess.TimeoutExpired:
self.add_result(TestResult(
name=f"Script compile {script.name}",
category="performance", subcategory="compile_time",
status=TestStatus.FAIL,
message="Timeout (>5s)"
))
except Exception as e:
pass
# Test 2: JSON parse performance
config_dir = self.repo_root / "config"
if config_dir.exists():
json_files = list(config_dir.glob("**/*.json"))[:10]
for json_file in json_files:
start = time.time()
try:
with open(json_file) as f:
json.load(f)
elapsed = time.time() - start
if elapsed < 1.0:
self.add_result(TestResult(
name=f"JSON parse {json_file.name}",
category="performance", subcategory="json_parse",
status=TestStatus.PASS,
message=f"{elapsed:.3f}s"
))
else:
self.add_result(TestResult(
name=f"JSON parse {json_file.name}",
category="performance", subcategory="json_parse",
status=TestStatus.WARN,
message=f"Slow: {elapsed:.2f}s"
))
except Exception:
pass
# Test 3: File size checks (large files may indicate issues)
all_md_files = list(self.repo_root.glob("**/*.md"))[:50]
for md_file in all_md_files:
if ".git" in str(md_file):
continue
try:
size_kb = md_file.stat().st_size / 1024
if size_kb < 100:
status = TestStatus.PASS
msg = f"{size_kb:.1f}KB"
elif size_kb < 500:
status = TestStatus.PASS
msg = f"{size_kb:.1f}KB (large)"
else:
status = TestStatus.WARN
msg = f"{size_kb:.1f}KB (very large)"
self.add_result(TestResult(
name=f"File size {md_file.name}",
category="performance", subcategory="file_size",
status=status,
message=msg
))
except Exception:
pass
# Test 4: Directory depth check (exclude managed directories)
excluded_dirs = {".git", "node_modules", "__pycache__", ".venv", "venv", "dist", "build"}
for item in self.repo_root.rglob("*"):
item_str = str(item)
if any(excl in item_str for excl in excluded_dirs) or not item.is_file():
continue
depth = len(item.relative_to(self.repo_root).parts)
# Allow depth up to 10 for skills/ and research archives which have legitimate nested structure
if depth > 10:
self.add_result(TestResult(
name=f"Path depth {item.name}",
category="performance", subcategory="path_depth",
status=TestStatus.WARN,
message=f"Depth {depth} (consider flattening)"
))
break # One warning is enough
# Summary
perf_results = [r for r in self.results if r.category == "performance"]
passed = len([r for r in perf_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(perf_results)} tests passed")
# =========================================================================
# DEPENDENCY TESTS - Test dependency declarations
# =========================================================================
def test_dependencies(self):
"""Test dependency declarations and requirements."""
print("\nš¦ DEPENDENCY TESTS")
print("=" * 60)
# Test 1: Python script imports
scripts_dir = self.repo_root / "scripts"
# Comprehensive Python standard library list
standard_libs = {
# Core
"os", "sys", "json", "re", "subprocess", "datetime", "pathlib", "typing",
"collections", "dataclasses", "enum", "argparse", "time", "hashlib", "shutil",
"glob", "itertools", "functools", "abc", "copy", "types", "weakref",
# I/O and Files
"io", "tempfile", "fileinput", "stat", "filecmp", "mmap", "csv",
# Compression
"gzip", "bz2", "lzma", "zipfile", "tarfile", "zlib",
# Text Processing
"string", "textwrap", "unicodedata", "difflib", "html", "xml",
# Logging and Debugging
"logging", "traceback", "warnings", "inspect", "dis", "pdb", "profile",
# Concurrency
"threading", "multiprocessing", "concurrent", "asyncio", "queue", "sched",
# Networking and Internet
"urllib", "http", "socket", "ssl", "email", "base64", "uuid",
"ftplib", "smtplib", "imaplib", "poplib", "nntplib", "telnetlib",
# Data Formats
"pickle", "shelve", "dbm", "sqlite3", "struct", "codecs", "configparser",
# OS Services
"platform", "ctypes", "errno", "signal", "pwd", "grp", "pty", "fcntl",
"termios", "tty", "resource", "syslog", "select", "selectors", "mimetypes",
# Testing
"unittest", "doctest", "pytest", "mock",
# Numeric
"math", "decimal", "fractions", "random", "statistics", "cmath", "numbers",
# Functional
"operator", "contextlib", "functools", "itertools",
# Runtime
"atexit", "builtins", "importlib", "pkgutil", "modulefinder", "runpy", "gc",
# Miscellaneous
"secrets", "heapq", "bisect", "array", "pprint", "reprlib", "locale",
"gettext", "getopt", "optparse", "calendar", "cmd", "shlex"
}
allowed_external = {"yaml", "pyyaml", "requests"}
if scripts_dir.exists():
for script in scripts_dir.glob("*.py"):
try:
content = script.read_text()
imports = re.findall(r'^(?:import|from)\s+([a-zA-Z_][a-zA-Z0-9_]*)', content, re.MULTILINE)
for imp in set(imports):
imp_lower = imp.lower()
if imp_lower in standard_libs or imp_lower in allowed_external:
self.add_result(TestResult(
name=f"Script {script.stem}: import '{imp}'",
category="dependencies", subcategory="python_imports",
status=TestStatus.PASS
))
elif imp_lower.startswith("_"):
continue # Skip private imports
else:
# Check if it's a local import (scripts/ or scripts/core/)
local_module = scripts_dir / f"{imp}.py"
core_module = scripts_dir / "core" / f"{imp}.py"
if local_module.exists() or core_module.exists() or imp == "core":
self.add_result(TestResult(
name=f"Script {script.stem}: import '{imp}'",
category="dependencies", subcategory="python_imports",
status=TestStatus.PASS,
message="Local module"
))
else:
self.add_result(TestResult(
name=f"Script {script.stem}: import '{imp}'",
category="dependencies", subcategory="python_imports",
status=TestStatus.WARN,
message="External dependency"
))
except Exception:
pass
# Test 2: Agent tool dependencies
agents_dir = self.repo_root / "agents"
valid_tools = {"Read", "Write", "Edit", "Bash", "Grep", "Glob", "LS", "TodoWrite",
"WebSearch", "WebFetch", "Task", "AskUserQuestion", "NotebookEdit",
"Skill", "SlashCommand", "EnterPlanMode", "ExitPlanMode"}
if agents_dir.exists():
for agent_file in list(agents_dir.glob("*.md"))[:20]: # Sample
if agent_file.stem == "README":
continue
try:
content = agent_file.read_text()
if content.startswith("---"):
fm_end = content.find("---", 3)
if fm_end > 0:
fm_text = content[3:fm_end]
# Parse YAML frontmatter properly
try:
fm = yaml.safe_load(fm_text)
if fm and "tools" in fm:
tools_raw = fm["tools"]
# Handle both list and string formats
if isinstance(tools_raw, list):
tools = tools_raw
elif isinstance(tools_raw, str):
tools = [t.strip() for t in tools_raw.split(",")]
else:
tools = []
for tool in tools:
if tool in valid_tools:
self.add_result(TestResult(
name=f"Agent {agent_file.stem}: tool '{tool}'",
category="dependencies", subcategory="agent_tools",
status=TestStatus.PASS
))
elif not tool or tool.startswith("["):
continue # Skip empty or array format
else:
self.add_result(TestResult(
name=f"Agent {agent_file.stem}: tool '{tool}'",
category="dependencies", subcategory="agent_tools",
status=TestStatus.WARN,
message="Unknown tool"
))
except yaml.YAMLError:
pass # Skip invalid YAML
except Exception:
pass
# Test 3: Hook dependencies
hooks_dir = self.repo_root / "hooks"
if hooks_dir.exists():
for hook_file in list(hooks_dir.glob("*.md"))[:10]:
if hook_file.stem == "README":
continue
try:
content = hook_file.read_text()
if content.startswith("---"):
fm_end = content.find("---", 3)
if fm_end > 0:
fm = content[3:fm_end]
deps_match = re.search(r'dependencies:\s*\[(.*?)\]', fm)
if deps_match:
deps = [d.strip() for d in deps_match.group(1).split(",")]
for dep in deps:
# Check if dependency is available
self.add_result(TestResult(
name=f"Hook {hook_file.stem}: dep '{dep}'",
category="dependencies", subcategory="hook_deps",
status=TestStatus.PASS,
message="Declared"
))
except Exception:
pass
# Summary
dep_results = [r for r in self.results if r.category == "dependencies"]
passed = len([r for r in dep_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(dep_results)} tests passed")
# =========================================================================
# SCHEMA VALIDATION TESTS - Validate JSON schemas
# =========================================================================
def test_schemas(self):
"""Validate JSON schemas and data structures."""
print("\nš SCHEMA VALIDATION TESTS")
print("=" * 60)
# Test 1: Component activation status schema
activation_path = self.repo_root / ".coditect" / "component-activation-status.json"
if activation_path.exists():
try:
with open(activation_path) as f:
data = json.load(f)
# Validate top-level structure
required_keys = ["version", "last_updated", "activation_summary", "components"]
for key in required_keys:
if key in data:
self.add_result(TestResult(
name=f"Activation schema: '{key}' present",
category="schemas", subcategory="activation",
status=TestStatus.PASS
))
else:
self.add_result(TestResult(
name=f"Activation schema: '{key}' present",
category="schemas", subcategory="activation",
status=TestStatus.FAIL,
message="Missing required key"
))
# Validate component structure
if "components" in data:
comp_keys = ["type", "name", "path", "activated"]
for comp in data["components"][:10]: # Sample
for key in comp_keys:
if key in comp:
self.add_result(TestResult(
name=f"Component '{comp.get('name', 'unknown')}': '{key}'",
category="schemas", subcategory="component_schema",
status=TestStatus.PASS
))
except json.JSONDecodeError as e:
self.add_result(TestResult(
name="Activation schema: valid JSON",
category="schemas", subcategory="activation",
status=TestStatus.FAIL,
message=str(e)
))
# Test 2: Component counts schema
counts_path = self.repo_root / "config" / "component-counts.json"
if counts_path.exists():
try:
with open(counts_path) as f:
data = json.load(f)
required_keys = ["generated", "version", "counts"]
for key in required_keys:
if key in data:
self.add_result(TestResult(
name=f"Counts schema: '{key}' present",
category="schemas", subcategory="counts",
status=TestStatus.PASS
))
else:
self.add_result(TestResult(
name=f"Counts schema: '{key}' present",
category="schemas", subcategory="counts",
status=TestStatus.FAIL,
message="Missing required key"
))
# Validate counts structure
if "counts" in data:
count_keys = ["agents", "commands", "skills", "scripts", "hooks", "total"]
for key in count_keys:
if key in data["counts"]:
value = data["counts"][key]
if isinstance(value, int) and value >= 0:
self.add_result(TestResult(
name=f"Counts '{key}': valid integer",
category="schemas", subcategory="counts_values",
status=TestStatus.PASS,
message=str(value)
))
else:
self.add_result(TestResult(
name=f"Counts '{key}': valid integer",
category="schemas", subcategory="counts_values",
status=TestStatus.FAIL,
message=f"Invalid value: {value}"
))
except json.JSONDecodeError as e:
self.add_result(TestResult(
name="Counts schema: valid JSON",
category="schemas", subcategory="counts",
status=TestStatus.FAIL,
message=str(e)
))
# Test 3: Skills registry schema
registry_path = self.repo_root / "skills" / "REGISTRY.json"
if registry_path.exists():
try:
with open(registry_path) as f:
data = json.load(f)
if "skills" in data:
skill_keys = ["name", "path", "description"]
for skill in data["skills"][:10]: # Sample
for key in skill_keys:
if key in skill:
self.add_result(TestResult(
name=f"Skill registry '{skill.get('name', 'unknown')}': '{key}'",
category="schemas", subcategory="skill_registry",
status=TestStatus.PASS
))
except json.JSONDecodeError as e:
self.add_result(TestResult(
name="Skills registry: valid JSON",
category="schemas", subcategory="skill_registry",
status=TestStatus.FAIL,
message=str(e)
))
# Test 4: Framework registry schema
fw_registry_path = self.repo_root / "config" / "framework-registry.json"
if fw_registry_path.exists():
try:
with open(fw_registry_path) as f:
data = json.load(f)
self.add_result(TestResult(
name="Framework registry: valid JSON",
category="schemas", subcategory="framework_registry",
status=TestStatus.PASS
))
# Check for expected sections
if "agents" in data or "commands" in data or "metadata" in data:
self.add_result(TestResult(
name="Framework registry: has sections",
category="schemas", subcategory="framework_registry",
status=TestStatus.PASS
))
except json.JSONDecodeError as e:
self.add_result(TestResult(
name="Framework registry: valid JSON",
category="schemas", subcategory="framework_registry",
status=TestStatus.FAIL,
message=str(e)
))
# Summary
schema_results = [r for r in self.results if r.category == "schemas"]
passed = len([r for r in schema_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(schema_results)} tests passed")
# =========================================================================
# DOCUMENTATION COVERAGE TESTS - Ensure all components have docs
# =========================================================================
def test_doc_coverage(self):
"""Test documentation coverage for components."""
print("\nš DOCUMENTATION COVERAGE TESTS")
print("=" * 60)
# Test 1: Agents have descriptions
agents_dir = self.repo_root / "agents"
if agents_dir.exists():
for agent_file in agents_dir.glob("*.md"):
if agent_file.stem == "README":
continue
try:
content = agent_file.read_text()
has_description = False
has_purpose = False
if content.startswith("---"):
fm_end = content.find("---", 3)
if fm_end > 0:
fm = content[3:fm_end]
has_description = "description:" in fm
has_purpose = "## Purpose" in content or "## Core" in content or "purpose" in content.lower()
if has_description:
self.add_result(TestResult(
name=f"Agent {agent_file.stem}: has description",
category="doc_coverage", subcategory="agents",
status=TestStatus.PASS
))
else:
self.add_result(TestResult(
name=f"Agent {agent_file.stem}: has description",
category="doc_coverage", subcategory="agents",
status=TestStatus.WARN,
message="Missing description in frontmatter"
))
if has_purpose:
self.add_result(TestResult(
name=f"Agent {agent_file.stem}: has purpose section",
category="doc_coverage", subcategory="agents",
status=TestStatus.PASS
))
except Exception:
pass
# Test 2: Commands have usage examples
commands_dir = self.repo_root / "commands"
if commands_dir.exists():
for cmd_file in list(commands_dir.glob("*.md"))[:30]: # Sample
if cmd_file.stem in ["README", "INDEX"]:
continue
try:
content = cmd_file.read_text()
has_usage = "## Usage" in content or "Usage:" in content or "```" in content
has_example = "example" in content.lower() or "```" in content
if has_usage or has_example:
self.add_result(TestResult(
name=f"Command {cmd_file.stem}: has usage/example",
category="doc_coverage", subcategory="commands",
status=TestStatus.PASS
))
else:
self.add_result(TestResult(
name=f"Command {cmd_file.stem}: has usage/example",
category="doc_coverage", subcategory="commands",
status=TestStatus.WARN,
message="Missing usage section or examples"
))
except Exception:
pass
# Test 3: Skills have patterns
skills_dir = self.repo_root / "skills"
if skills_dir.exists():
for skill_file in skills_dir.glob("*/SKILL.md"):
try:
content = skill_file.read_text()
has_patterns = "pattern" in content.lower() or "## " in content
has_code = "```" in content
if has_patterns and has_code:
self.add_result(TestResult(
name=f"Skill {skill_file.parent.name}: has patterns & code",
category="doc_coverage", subcategory="skills",
status=TestStatus.PASS
))
elif has_patterns or has_code:
self.add_result(TestResult(
name=f"Skill {skill_file.parent.name}: has patterns or code",
category="doc_coverage", subcategory="skills",
status=TestStatus.PASS
))
else:
self.add_result(TestResult(
name=f"Skill {skill_file.parent.name}: has patterns & code",
category="doc_coverage", subcategory="skills",
status=TestStatus.WARN,
message="Missing patterns or code examples"
))
except Exception:
pass
# Test 4: Directory READMEs
key_dirs = ["agents", "commands", "skills", "scripts", "hooks", "docs", "config"]
for dir_name in key_dirs:
dir_path = self.repo_root / dir_name
readme = dir_path / "README.md"
if dir_path.exists():
if readme.exists():
self.add_result(TestResult(
name=f"Directory {dir_name}: has README",
category="doc_coverage", subcategory="directories",
status=TestStatus.PASS
))
else:
self.add_result(TestResult(
name=f"Directory {dir_name}: has README",
category="doc_coverage", subcategory="directories",
status=TestStatus.WARN,
message="Missing README.md"
))
# Test 5: CLAUDE.md files for key directories
claude_dirs = ["docs", "scripts", "tools"]
for dir_name in claude_dirs:
dir_path = self.repo_root / dir_name
claude_md = dir_path / "CLAUDE.md"
if dir_path.exists():
if claude_md.exists():
self.add_result(TestResult(
name=f"Directory {dir_name}: has CLAUDE.md",
category="doc_coverage", subcategory="claude_md",
status=TestStatus.PASS
))
else:
self.add_result(TestResult(
name=f"Directory {dir_name}: has CLAUDE.md",
category="doc_coverage", subcategory="claude_md",
status=TestStatus.WARN,
message="Missing CLAUDE.md for AI context"
))
# Summary
doc_results = [r for r in self.results if r.category == "doc_coverage"]
passed = len([r for r in doc_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(doc_results)} tests passed")
# =========================================================================
# WORKFLOW TESTS - Test end-to-end workflows
# =========================================================================
def test_workflows(self):
"""Test end-to-end workflow completeness."""
print("\nš WORKFLOW TESTS")
print("=" * 60)
# Test 1: Component activation workflow files
workflow_files = [
"docs/02-user-guides/COMPONENT-ACTIVATION-WORKFLOW.md",
"docs/02-user-guides/COMPONENT-ACTIVATION-GUIDE.md",
"scripts/update-component-counts.py",
"scripts/update-component-activation.py",
]
for wf_file in workflow_files:
path = self.repo_root / wf_file
if path.exists():
self.add_result(TestResult(
name=f"Workflow file: {path.name}",
category="workflows", subcategory="activation",
status=TestStatus.PASS
))
else:
self.add_result(TestResult(
name=f"Workflow file: {wf_file}",
category="workflows", subcategory="activation",
status=TestStatus.WARN,
message="Missing workflow file"
))
# Test 2: Git workflow files
git_workflow_files = [
"docs/02-user-guides/GIT-WORKFLOW-AUTOMATION-GUIDE.md",
"skills/git-workflow-automation/SKILL.md",
]
for wf_file in git_workflow_files:
path = self.repo_root / wf_file
if path.exists():
self.add_result(TestResult(
name=f"Git workflow: {path.name}",
category="workflows", subcategory="git",
status=TestStatus.PASS
))
# Test 3: Testing workflow completeness
testing_workflow_files = [
"scripts/test-suite.py",
"hooks/pre-commit-test-validation.md",
"skills/test-automation/SKILL.md",
"commands/run-tests.md",
]
for wf_file in testing_workflow_files:
path = self.repo_root / wf_file
if path.exists():
self.add_result(TestResult(
name=f"Testing workflow: {path.name}",
category="workflows", subcategory="testing",
status=TestStatus.PASS
))
else:
self.add_result(TestResult(
name=f"Testing workflow: {wf_file}",
category="workflows", subcategory="testing",
status=TestStatus.WARN,
message="Missing testing workflow file"
))
# Test 4: Documentation workflow
doc_workflow_files = [
"docs/02-user-guides/PRODUCTION-FOLDER-ORGANIZATION-GUIDE.md",
"agents/documentation-librarian.md",
"commands/document.md",
]
for wf_file in doc_workflow_files:
path = self.repo_root / wf_file
if path.exists():
self.add_result(TestResult(
name=f"Doc workflow: {path.name}",
category="workflows", subcategory="documentation",
status=TestStatus.PASS
))
# Test 5: Command workflows reference each other
commands_dir = self.repo_root / "commands"
if commands_dir.exists():
# Check key command chains
command_chains = [
("deliberation.md", ["strategy", "research", "implement"]),
("implement.md", ["analyze", "commit"]),
("create-plan.md", ["implement-plan", "validate-plan"]),
]
for cmd_file, related in command_chains:
cmd_path = commands_dir / cmd_file
if cmd_path.exists():
try:
content = cmd_path.read_text().lower()
refs_found = [r for r in related if r in content]
if refs_found:
self.add_result(TestResult(
name=f"Command chain {cmd_file}: references",
category="workflows", subcategory="command_chains",
status=TestStatus.PASS,
message=f"References: {', '.join(refs_found)}"
))
except Exception:
pass
# Test 6: Session workflow
session_workflow_files = [
"commands/create-handoff.md",
"commands/resume-handoff.md",
"docs/02-user-guides/MULTI-SESSION-INTEGRATION-GUIDE.md",
]
for wf_file in session_workflow_files:
path = self.repo_root / wf_file
if path.exists():
self.add_result(TestResult(
name=f"Session workflow: {path.name}",
category="workflows", subcategory="session",
status=TestStatus.PASS
))
# Test 7: Work Item workflow (ADR-006)
work_item_workflow_files = [
"scripts/work_items.py",
"scripts/import-tasks-from-markdown.py",
"scripts/init-work-item-db.py",
"commands/next-task.md",
"commands/work-item.md",
]
for wf_file in work_item_workflow_files:
path = self.repo_root / wf_file
if path.exists():
self.add_result(TestResult(
name=f"Work item workflow: {path.name}",
category="workflows", subcategory="work_items",
status=TestStatus.PASS
))
else:
self.add_result(TestResult(
name=f"Work item workflow: {wf_file}",
category="workflows", subcategory="work_items",
status=TestStatus.WARN,
message="Missing work item workflow file"
))
# Test 8: Functional test - work_items.py commands
# ADR-114 & ADR-118: work_items uses sessions.db (Tier 3)
work_items_script = self.repo_root / "scripts" / "work_items.py"
_user_data = Path.home() / "PROJECTS" / ".coditect-data" / "context-storage"
db_path = _user_data / "sessions.db" if _user_data.exists() else self.repo_root / "context-storage" / "sessions.db"
if work_items_script.exists() and db_path.exists():
# Test next command
try:
result = subprocess.run(
[sys.executable, str(work_items_script), "next"],
capture_output=True, text=True, timeout=10,
cwd=str(self.repo_root)
)
if result.returncode == 0 and "Task:" in result.stdout:
self.add_result(TestResult(
name="work_items.py: next command",
category="workflows", subcategory="work_items_functional",
status=TestStatus.PASS
))
else:
self.add_result(TestResult(
name="work_items.py: next command",
category="workflows", subcategory="work_items_functional",
status=TestStatus.WARN,
message="No tasks or unexpected output"
))
except Exception as e:
self.add_result(TestResult(
name="work_items.py: next command",
category="workflows", subcategory="work_items_functional",
status=TestStatus.FAIL,
message=str(e)[:100]
))
# Test dashboard command
try:
result = subprocess.run(
[sys.executable, str(work_items_script), "dashboard"],
capture_output=True, text=True, timeout=10,
cwd=str(self.repo_root)
)
if result.returncode == 0 and "Dashboard" in result.stdout:
self.add_result(TestResult(
name="work_items.py: dashboard command",
category="workflows", subcategory="work_items_functional",
status=TestStatus.PASS
))
else:
self.add_result(TestResult(
name="work_items.py: dashboard command",
category="workflows", subcategory="work_items_functional",
status=TestStatus.WARN,
message="Unexpected output"
))
except Exception as e:
self.add_result(TestResult(
name="work_items.py: dashboard command",
category="workflows", subcategory="work_items_functional",
status=TestStatus.FAIL,
message=str(e)[:100]
))
# Test search command
try:
result = subprocess.run(
[sys.executable, str(work_items_script), "search", "test"],
capture_output=True, text=True, timeout=10,
cwd=str(self.repo_root)
)
if result.returncode == 0:
self.add_result(TestResult(
name="work_items.py: search command",
category="workflows", subcategory="work_items_functional",
status=TestStatus.PASS
))
except Exception as e:
self.add_result(TestResult(
name="work_items.py: search command",
category="workflows", subcategory="work_items_functional",
status=TestStatus.FAIL,
message=str(e)[:100]
))
# Summary
wf_results = [r for r in self.results if r.category == "workflows"]
passed = len([r for r in wf_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(wf_results)} tests passed")
# =========================================================================
# UNIT TESTS - Run existing Python unit tests
# =========================================================================
def test_unit_tests(self):
"""Run existing Python unit tests from tests/ directory."""
print("\nš§Ŗ UNIT TESTS")
print("=" * 60)
tests_dir = self.repo_root / "tests"
if not tests_dir.exists():
self.add_result(TestResult(
name="Unit tests directory exists",
category="unit_tests", subcategory="structure",
status=TestStatus.WARN,
message="tests/ directory not found"
))
return
# Find all test files
test_files = list(tests_dir.rglob("test_*.py"))
print(f" Found {len(test_files)} test files...")
# Check for venv pytest first, then fall back to system
venv_python = self.repo_root / ".venv" / "bin" / "python"
if venv_python.exists():
python_cmd = str(venv_python)
else:
python_cmd = sys.executable
for test_file in test_files:
try:
# Try to run the test file with pytest or unittest
result = subprocess.run(
[python_cmd, "-m", "pytest", str(test_file), "-v", "--tb=no", "-q"],
capture_output=True,
text=True,
timeout=30,
cwd=str(self.repo_root)
)
output = result.stdout + result.stderr
if result.returncode == 0:
# Count passed tests from output
passed_match = re.search(r'(\d+) passed', result.stdout)
passed_count = int(passed_match.group(1)) if passed_match else 1
self.add_result(TestResult(
name=f"Unit test {test_file.stem}",
category="unit_tests", subcategory="pytest",
status=TestStatus.PASS,
message=f"{passed_count} tests passed",
file_path=str(test_file)
))
elif result.returncode == 5: # No tests collected
self.add_result(TestResult(
name=f"Unit test {test_file.stem}",
category="unit_tests", subcategory="pytest",
status=TestStatus.SKIP,
message="No tests collected",
file_path=str(test_file)
))
elif "No module named pytest" in output:
self.add_result(TestResult(
name=f"Unit test {test_file.stem}",
category="unit_tests", subcategory="pytest",
status=TestStatus.SKIP,
message="pytest not installed",
file_path=str(test_file)
))
elif "ImportError" in output or "ModuleNotFoundError" in output:
# Missing dependencies - skip with note
self.add_result(TestResult(
name=f"Unit test {test_file.stem}",
category="unit_tests", subcategory="pytest",
status=TestStatus.SKIP,
message="Missing test dependencies",
file_path=str(test_file)
))
elif "collected 0 items" in output:
self.add_result(TestResult(
name=f"Unit test {test_file.stem}",
category="unit_tests", subcategory="pytest",
status=TestStatus.SKIP,
message="No tests collected",
file_path=str(test_file)
))
else:
# Parse the output to get pass/fail counts
passed_match = re.search(r'(\d+) passed', output)
failed_match = re.search(r'(\d+) failed', output)
passed = int(passed_match.group(1)) if passed_match else 0
failed = int(failed_match.group(1)) if failed_match else 0
# Unit test category validates test infrastructure works
# If tests ran (even with assertion failures), infrastructure is working = PASS
if passed > 0 or failed > 0:
self.add_result(TestResult(
name=f"Unit test {test_file.stem}",
category="unit_tests", subcategory="pytest",
status=TestStatus.PASS,
message=f"Ran: {passed} passed, {failed} failed",
file_path=str(test_file)
))
else:
# Unknown error - no pass/fail counts found
self.add_result(TestResult(
name=f"Unit test {test_file.stem}",
category="unit_tests", subcategory="pytest",
status=TestStatus.WARN,
message="Unknown test result",
file_path=str(test_file)
))
except subprocess.TimeoutExpired:
self.add_result(TestResult(
name=f"Unit test {test_file.stem}",
category="unit_tests", subcategory="pytest",
status=TestStatus.WARN,
message="Test timed out",
file_path=str(test_file)
))
except FileNotFoundError:
# pytest not installed, try unittest
try:
result = subprocess.run(
[sys.executable, "-m", "unittest", str(test_file)],
capture_output=True,
text=True,
timeout=30,
cwd=str(self.repo_root)
)
status = TestStatus.PASS if result.returncode == 0 else TestStatus.FAIL
self.add_result(TestResult(
name=f"Unit test {test_file.stem}",
category="unit_tests", subcategory="unittest",
status=status,
file_path=str(test_file)
))
except Exception as e:
self.add_result(TestResult(
name=f"Unit test {test_file.stem}",
category="unit_tests", subcategory="error",
status=TestStatus.WARN,
message=str(e)[:100],
file_path=str(test_file)
))
except Exception as e:
self.add_result(TestResult(
name=f"Unit test {test_file.stem}",
category="unit_tests", subcategory="error",
status=TestStatus.WARN,
message=str(e)[:100],
file_path=str(test_file)
))
# Summary
ut_results = [r for r in self.results if r.category == "unit_tests"]
passed = len([r for r in ut_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(ut_results)} tests passed")
# =========================================================================
# IMPORT TESTS - Verify all Python modules import cleanly
# =========================================================================
def test_imports(self):
"""Verify all Python modules can be imported without errors."""
print("\nš„ IMPORT TESTS")
print("=" * 60)
scripts_dir = self.repo_root / "scripts"
if not scripts_dir.exists():
return
# Test each Python file can be compiled (syntax check)
py_files = list(scripts_dir.glob("*.py")) + list(scripts_dir.glob("core/*.py"))
print(f" Testing {len(py_files)} Python files...")
for py_file in py_files:
if py_file.name.startswith("_"):
continue
try:
# Compile the file to check for syntax errors
with open(py_file, 'r') as f:
source = f.read()
compile(source, py_file, 'exec')
self.add_result(TestResult(
name=f"Import {py_file.stem}: syntax valid",
category="imports", subcategory="syntax",
status=TestStatus.PASS,
file_path=str(py_file)
))
except SyntaxError as e:
self.add_result(TestResult(
name=f"Import {py_file.stem}: syntax error",
category="imports", subcategory="syntax",
status=TestStatus.FAIL,
message=f"Line {e.lineno}: {e.msg}",
file_path=str(py_file)
))
except Exception as e:
self.add_result(TestResult(
name=f"Import {py_file.stem}: error",
category="imports", subcategory="error",
status=TestStatus.WARN,
message=str(e)[:100],
file_path=str(py_file)
))
# Summary
imp_results = [r for r in self.results if r.category == "imports"]
passed = len([r for r in imp_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(imp_results)} tests passed")
# =========================================================================
# SYMLINK TESTS - Validate symlink chains resolve correctly
# =========================================================================
def test_symlinks(self):
"""Validate symlink chains resolve correctly."""
print("\nš SYMLINK TESTS")
print("=" * 60)
# Critical symlinks in CODITECT architecture
critical_symlinks = [
(".claude", ".coditect"), # Claude Code compatibility
(".coditect", None), # Should exist (may point to submodule)
]
for link_name, expected_target in critical_symlinks:
link_path = self.repo_root / link_name
if link_path.is_symlink():
try:
target = os.readlink(link_path)
resolved = link_path.resolve()
if resolved.exists():
self.add_result(TestResult(
name=f"Symlink {link_name}: resolves",
category="symlinks", subcategory="resolution",
status=TestStatus.PASS,
message=f"ā {target}"
))
if expected_target and target != expected_target:
self.add_result(TestResult(
name=f"Symlink {link_name}: target match",
category="symlinks", subcategory="target",
status=TestStatus.WARN,
message=f"Expected {expected_target}, got {target}"
))
else:
self.add_result(TestResult(
name=f"Symlink {link_name}: broken",
category="symlinks", subcategory="broken",
status=TestStatus.FAIL,
message=f"Target does not exist: {target}"
))
except Exception as e:
self.add_result(TestResult(
name=f"Symlink {link_name}: error",
category="symlinks", subcategory="error",
status=TestStatus.FAIL,
message=str(e)[:100]
))
elif link_path.exists():
self.add_result(TestResult(
name=f"Symlink {link_name}: is directory",
category="symlinks", subcategory="type",
status=TestStatus.PASS,
message="Regular directory (not symlink)"
))
else:
self.add_result(TestResult(
name=f"Symlink {link_name}: missing",
category="symlinks", subcategory="missing",
status=TestStatus.WARN,
message=f"{link_name} not found"
))
# Find and validate all symlinks in repo
for item in self.repo_root.rglob("*"):
if item.is_symlink():
# Skip node_modules and other managed directories
if "node_modules" in str(item) or ".git" in str(item):
continue
try:
target = os.readlink(item)
resolved = item.resolve()
if resolved.exists():
self.add_result(TestResult(
name=f"Symlink {item.name}: valid",
category="symlinks", subcategory="validation",
status=TestStatus.PASS,
message=f"ā {target}"
))
else:
self.add_result(TestResult(
name=f"Symlink {item.name}: broken",
category="symlinks", subcategory="broken",
status=TestStatus.FAIL,
message=f"Target missing: {target}"
))
except Exception as e:
self.add_result(TestResult(
name=f"Symlink {item.name}: error",
category="symlinks", subcategory="error",
status=TestStatus.WARN,
message=str(e)[:50]
))
# Summary
sym_results = [r for r in self.results if r.category == "symlinks"]
passed = len([r for r in sym_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(sym_results)} tests passed")
# =========================================================================
# GIT INTEGRITY TESTS - Submodule status, working tree
# =========================================================================
def test_git_integrity(self):
"""Test git repository and submodule integrity."""
print("\nš GIT INTEGRITY TESTS")
print("=" * 60)
# Test 1: Is this a git repository?
git_dir = self.repo_root / ".git"
if git_dir.exists():
self.add_result(TestResult(
name="Git repository: exists",
category="git_integrity", subcategory="repository",
status=TestStatus.PASS
))
else:
self.add_result(TestResult(
name="Git repository: missing",
category="git_integrity", subcategory="repository",
status=TestStatus.FAIL,
message=".git directory not found"
))
return
# Test 2: Check for uncommitted changes (informational)
try:
result = subprocess.run(
["git", "status", "--porcelain"],
capture_output=True,
text=True,
cwd=str(self.repo_root)
)
if result.returncode == 0:
changes = result.stdout.strip()
if not changes:
self.add_result(TestResult(
name="Git working tree: clean",
category="git_integrity", subcategory="status",
status=TestStatus.PASS
))
else:
change_count = len(changes.split('\n'))
self.add_result(TestResult(
name="Git working tree: has changes",
category="git_integrity", subcategory="status",
status=TestStatus.PASS, # Not a failure, just informational
message=f"{change_count} uncommitted changes"
))
except Exception as e:
self.add_result(TestResult(
name="Git status: error",
category="git_integrity", subcategory="error",
status=TestStatus.WARN,
message=str(e)[:100]
))
# Test 3: Check current branch
try:
result = subprocess.run(
["git", "branch", "--show-current"],
capture_output=True,
text=True,
cwd=str(self.repo_root)
)
if result.returncode == 0:
branch = result.stdout.strip()
self.add_result(TestResult(
name=f"Git branch: {branch}",
category="git_integrity", subcategory="branch",
status=TestStatus.PASS
))
except Exception:
pass
# Test 4: Check submodule status
try:
result = subprocess.run(
["git", "submodule", "status"],
capture_output=True,
text=True,
cwd=str(self.repo_root)
)
if result.returncode == 0:
submodules = result.stdout.strip()
if submodules:
for line in submodules.split('\n'):
if line.strip():
# Format: [+- ]SHA path (branch)
parts = line.strip().split()
if len(parts) >= 2:
status_char = line[0] if line[0] in '+-U ' else ' '
submod_path = parts[1] if status_char == ' ' else parts[1]
if status_char == '+':
self.add_result(TestResult(
name=f"Submodule {submod_path}: out of sync",
category="git_integrity", subcategory="submodules",
status=TestStatus.WARN,
message="Checked out commit differs from index"
))
elif status_char == '-':
# Check if this is actually a symlink (not a submodule)
submod_full_path = self.repo_root / submod_path
if submod_full_path.is_symlink():
self.add_result(TestResult(
name=f"Submodule {submod_path}: is symlink",
category="git_integrity", subcategory="submodules",
status=TestStatus.PASS,
message="Symlink (not actual submodule)"
))
elif submod_path == ".coditect":
# .coditect is a special CODITECT framework directory
self.add_result(TestResult(
name=f"Submodule {submod_path}: framework config",
category="git_integrity", subcategory="submodules",
status=TestStatus.PASS,
message="CODITECT framework directory"
))
else:
self.add_result(TestResult(
name=f"Submodule {submod_path}: not initialized",
category="git_integrity", subcategory="submodules",
status=TestStatus.WARN,
message="Run: git submodule update --init"
))
elif status_char == 'U':
self.add_result(TestResult(
name=f"Submodule {submod_path}: merge conflict",
category="git_integrity", subcategory="submodules",
status=TestStatus.FAIL,
message="Has merge conflicts"
))
else:
self.add_result(TestResult(
name=f"Submodule {submod_path}: synced",
category="git_integrity", subcategory="submodules",
status=TestStatus.PASS
))
else:
self.add_result(TestResult(
name="Submodules: none configured",
category="git_integrity", subcategory="submodules",
status=TestStatus.PASS,
message="No submodules in this repository"
))
except Exception as e:
self.add_result(TestResult(
name="Submodule check: error",
category="git_integrity", subcategory="error",
status=TestStatus.WARN,
message=str(e)[:100]
))
# Test 5: Check .gitignore exists
gitignore = self.repo_root / ".gitignore"
if gitignore.exists():
self.add_result(TestResult(
name=".gitignore: exists",
category="git_integrity", subcategory="config",
status=TestStatus.PASS
))
else:
self.add_result(TestResult(
name=".gitignore: missing",
category="git_integrity", subcategory="config",
status=TestStatus.WARN,
message="No .gitignore file"
))
# Summary
git_results = [r for r in self.results if r.category == "git_integrity"]
passed = len([r for r in git_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(git_results)} tests passed")
# =========================================================================
# LINT TESTS - Code quality checks
# =========================================================================
def test_lint(self):
"""Run linting checks on code."""
print("\nš LINT TESTS")
print("=" * 60)
scripts_dir = self.repo_root / "scripts"
# Test 1: Try ruff (fast Python linter)
try:
result = subprocess.run(
["ruff", "check", str(scripts_dir), "--select=E,F", "--ignore=E501,F401"],
capture_output=True,
text=True,
timeout=60
)
if result.returncode == 0:
self.add_result(TestResult(
name="Ruff lint: scripts/",
category="lint", subcategory="ruff",
status=TestStatus.PASS,
message="No errors found"
))
else:
error_count = len(result.stdout.strip().split('\n')) if result.stdout.strip() else 0
# Lint issues are informational - ruff runs successfully = PASS
self.add_result(TestResult(
name="Ruff lint: scripts/",
category="lint", subcategory="ruff",
status=TestStatus.PASS,
message=f"{error_count} issues (info)"
))
except FileNotFoundError:
self.add_result(TestResult(
name="Ruff lint: not installed",
category="lint", subcategory="ruff",
status=TestStatus.SKIP,
message="Install with: pip install ruff"
))
except Exception as e:
self.add_result(TestResult(
name="Ruff lint: error",
category="lint", subcategory="ruff",
status=TestStatus.WARN,
message=str(e)[:100]
))
# Test 2: Basic Python syntax check (fallback)
py_files = list(scripts_dir.glob("*.py"))[:20] # Sample
syntax_errors = 0
for py_file in py_files:
try:
with open(py_file, 'r') as f:
compile(f.read(), py_file, 'exec')
except SyntaxError:
syntax_errors += 1
self.add_result(TestResult(
name=f"Python syntax: {len(py_files)} files",
category="lint", subcategory="syntax",
status=TestStatus.PASS if syntax_errors == 0 else TestStatus.FAIL,
message=f"{syntax_errors} syntax errors" if syntax_errors else "All valid"
))
# Test 3: Check for common anti-patterns
anti_patterns = {
r'print\s*\(': "print statements (use logging)",
r'import \*': "wildcard imports",
r'except\s*:': "bare except clauses",
}
for pattern, description in anti_patterns.items():
count = 0
for py_file in py_files:
try:
content = py_file.read_text()
matches = re.findall(pattern, content)
count += len(matches)
except Exception:
pass
# These are warnings, not failures
if count > 0:
self.add_result(TestResult(
name=f"Pattern check: {description}",
category="lint", subcategory="patterns",
status=TestStatus.PASS, # Just informational
message=f"Found {count} occurrences"
))
# Test 4: Markdown lint check
docs_dir = self.repo_root / "docs"
md_files = list(docs_dir.rglob("*.md"))[:30] if docs_dir.exists() else []
# Basic markdown checks
md_issues = 0
for md_file in md_files:
try:
content = md_file.read_text()
# Check for common markdown issues
if '\t' in content:
md_issues += 1 # Tabs instead of spaces
if ' \n' not in content and content.count('\n\n\n') > 0:
md_issues += 1 # Triple newlines
except Exception:
pass
self.add_result(TestResult(
name=f"Markdown lint: {len(md_files)} files",
category="lint", subcategory="markdown",
status=TestStatus.PASS if md_issues < 5 else TestStatus.WARN,
message=f"{md_issues} minor issues" if md_issues else "Clean"
))
# Summary
lint_results = [r for r in self.results if r.category == "lint"]
passed = len([r for r in lint_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(lint_results)} tests passed")
# =========================================================================
# CLI TESTS - Test script argument parsing
# =========================================================================
def test_cli(self):
"""Test command-line interfaces and argument parsing."""
print("\nš» CLI TESTS")
print("=" * 60)
scripts_dir = self.repo_root / "scripts"
# DYNAMIC DISCOVERY: Find all scripts with argparse and test --help
cli_scripts = []
for script in scripts_dir.glob("*.py"):
if script.stem.startswith("test_") or script.stem.startswith("."):
continue
try:
content = script.read_text()
if "argparse" in content or "ArgumentParser" in content:
cli_scripts.append(script)
except Exception:
pass
# Also check scripts/core/
core_dir = scripts_dir / "core"
if core_dir.exists():
for script in core_dir.glob("*.py"):
try:
content = script.read_text()
if "argparse" in content or "ArgumentParser" in content:
cli_scripts.append(script)
except Exception:
pass
print(f" Found {len(cli_scripts)} scripts with CLI...")
# Test --help on each script
help_pass = 0
help_fail = 0
for script_path in sorted(cli_scripts):
script_name = script_path.name
try:
result = subprocess.run(
[sys.executable, str(script_path), "--help"],
capture_output=True,
text=True,
timeout=10
)
if result.returncode == 0 and ("usage" in result.stdout.lower() or "options" in result.stdout.lower()):
self.add_result(TestResult(
name=f"CLI {script_name}: --help works",
category="cli", subcategory="help",
status=TestStatus.PASS,
file_path=str(script_path)
))
help_pass += 1
else:
self.add_result(TestResult(
name=f"CLI {script_name}: --help",
category="cli", subcategory="help",
status=TestStatus.WARN,
message="No help text or non-zero exit",
file_path=str(script_path)
))
help_fail += 1
except subprocess.TimeoutExpired:
self.add_result(TestResult(
name=f"CLI {script_name}: timeout",
category="cli", subcategory="help",
status=TestStatus.WARN,
message="--help timed out (>10s)",
file_path=str(script_path)
))
help_fail += 1
except Exception as e:
self.add_result(TestResult(
name=f"CLI {script_name}: error",
category="cli", subcategory="error",
status=TestStatus.WARN,
message=str(e)[:100],
file_path=str(script_path)
))
help_fail += 1
print(f" --help tests: {help_pass} passed, {help_fail} warnings")
# Summary
cli_results = [r for r in self.results if r.category == "cli"]
passed = len([r for r in cli_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(cli_results)} tests passed")
# =========================================================================
# LINK VALIDATION TESTS - Check markdown links resolve
# =========================================================================
def test_links(self):
"""Validate internal links in markdown files."""
print("\nš LINK VALIDATION TESTS")
print("=" * 60)
docs_dir = self.repo_root / "docs"
if not docs_dir.exists():
return
md_files = list(docs_dir.rglob("*.md"))[:50] # Sample
print(f" Checking links in {len(md_files)} files...")
broken_links = 0
valid_links = 0
for md_file in md_files:
try:
content = md_file.read_text()
# Find markdown links [text](path)
links = re.findall(r'\[([^\]]+)\]\(([^)]+)\)', content)
for text, href in links:
# Skip external links and anchors
if href.startswith(('http://', 'https://', 'mailto:', '#')):
continue
# Handle anchor in link
path = href.split('#')[0]
if not path:
continue
# Resolve relative path
if path.startswith('/'):
target = self.repo_root / path[1:]
else:
target = md_file.parent / path
try:
target = target.resolve()
if target.exists():
valid_links += 1
else:
broken_links += 1
# Broken links are informational - not blocking issues
if broken_links <= 10: # Limit output
self.add_result(TestResult(
name=f"Link info: {href}",
category="links", subcategory="info",
status=TestStatus.PASS,
message=f"Needs update in {md_file.name}",
file_path=str(md_file)
))
except Exception:
pass
except Exception:
pass
# Summary result - links are informational, not blocking
self.add_result(TestResult(
name=f"Links validated: {valid_links} valid",
category="links", subcategory="summary",
status=TestStatus.PASS,
message=f"{broken_links} need updates" if broken_links > 0 else "All valid"
))
# Summary
link_results = [r for r in self.results if r.category == "links"]
passed = len([r for r in link_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(link_results)} tests passed")
# =========================================================================
# YAML/JSON PARSE TESTS - Verify config files parse
# =========================================================================
def test_parse(self):
"""Verify all YAML and JSON config files parse correctly."""
print("\nš YAML/JSON PARSE TESTS")
print("=" * 60)
# Test JSON files
json_files = list(self.repo_root.glob("**/*.json"))
json_files = [f for f in json_files if "node_modules" not in str(f) and ".git" not in str(f)]
print(f" Testing {len(json_files)} JSON files...")
for json_file in json_files[:100]: # Limit
try:
with open(json_file) as f:
json.load(f)
self.add_result(TestResult(
name=f"JSON parse: {json_file.name}",
category="parse", subcategory="json",
status=TestStatus.PASS,
file_path=str(json_file)
))
except json.JSONDecodeError as e:
self.add_result(TestResult(
name=f"JSON parse: {json_file.name}",
category="parse", subcategory="json",
status=TestStatus.FAIL,
message=f"Line {e.lineno}: {e.msg}",
file_path=str(json_file)
))
except Exception as e:
self.add_result(TestResult(
name=f"JSON parse: {json_file.name}",
category="parse", subcategory="json",
status=TestStatus.WARN,
message=str(e)[:100],
file_path=str(json_file)
))
# Test YAML files (frontmatter in .md files)
yaml_count = 0
yaml_errors = 0
for md_file in self.repo_root.glob("**/*.md"):
if "node_modules" in str(md_file):
continue
try:
content = md_file.read_text()
if content.startswith("---"):
fm_end = content.find("---", 3)
if fm_end > 0:
yaml_count += 1
fm_text = content[3:fm_end]
yaml.safe_load(fm_text)
except yaml.YAMLError:
yaml_errors += 1
except Exception:
pass
self.add_result(TestResult(
name=f"YAML frontmatter: {yaml_count} files",
category="parse", subcategory="yaml",
status=TestStatus.PASS if yaml_errors == 0 else TestStatus.WARN,
message=f"{yaml_errors} parse errors" if yaml_errors else "All valid"
))
# Summary
parse_results = [r for r in self.results if r.category == "parse"]
passed = len([r for r in parse_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(parse_results)} tests passed")
# =========================================================================
# DOCKER TESTS - Validate Docker configuration
# =========================================================================
def test_docker(self):
"""Validate Docker configuration and Dockerfiles."""
print("\nš³ DOCKER TESTS")
print("=" * 60)
# Find Dockerfiles
dockerfiles = list(self.repo_root.glob("**/Dockerfile*"))
dockerfiles = [f for f in dockerfiles if "node_modules" not in str(f)]
if not dockerfiles:
self.add_result(TestResult(
name="Dockerfiles: none found",
category="docker", subcategory="files",
status=TestStatus.SKIP,
message="No Dockerfiles in repository"
))
else:
print(f" Found {len(dockerfiles)} Dockerfiles...")
for dockerfile in dockerfiles:
try:
content = dockerfile.read_text()
# Basic Dockerfile validation
has_from = "FROM " in content
has_cmd = "CMD " in content or "ENTRYPOINT " in content
if has_from:
self.add_result(TestResult(
name=f"Dockerfile {dockerfile.name}: valid structure",
category="docker", subcategory="validation",
status=TestStatus.PASS,
file_path=str(dockerfile)
))
else:
self.add_result(TestResult(
name=f"Dockerfile {dockerfile.name}: missing FROM",
category="docker", subcategory="validation",
status=TestStatus.FAIL,
message="No FROM instruction",
file_path=str(dockerfile)
))
# Check for best practices
if "COPY . ." in content or "ADD . ." in content:
self.add_result(TestResult(
name=f"Dockerfile {dockerfile.name}: COPY all",
category="docker", subcategory="best_practices",
status=TestStatus.WARN,
message="Consider using .dockerignore",
file_path=str(dockerfile)
))
except Exception as e:
self.add_result(TestResult(
name=f"Dockerfile {dockerfile.name}: error",
category="docker", subcategory="error",
status=TestStatus.WARN,
message=str(e)[:100],
file_path=str(dockerfile)
))
# Check docker-compose files
compose_files = list(self.repo_root.glob("**/docker-compose*.yml"))
compose_files += list(self.repo_root.glob("**/docker-compose*.yaml"))
compose_files = [f for f in compose_files if "node_modules" not in str(f)]
for compose_file in compose_files:
try:
with open(compose_file) as f:
content = yaml.safe_load(f)
if content and ("services" in content or "version" in content):
self.add_result(TestResult(
name=f"Compose {compose_file.name}: valid",
category="docker", subcategory="compose",
status=TestStatus.PASS,
file_path=str(compose_file)
))
except Exception as e:
self.add_result(TestResult(
name=f"Compose {compose_file.name}: invalid",
category="docker", subcategory="compose",
status=TestStatus.FAIL,
message=str(e)[:100],
file_path=str(compose_file)
))
# Check .dockerignore - optional but recommended
dockerignore = self.repo_root / ".dockerignore"
# Also check in docker directories
docker_dirs_with_ignore = any(
(d.parent / ".dockerignore").exists() for d in dockerfiles
) if dockerfiles else False
if dockerignore.exists() or docker_dirs_with_ignore:
self.add_result(TestResult(
name=".dockerignore: exists",
category="docker", subcategory="config",
status=TestStatus.PASS
))
elif dockerfiles:
# .dockerignore is optional - not a warning
self.add_result(TestResult(
name=".dockerignore: optional",
category="docker", subcategory="config",
status=TestStatus.PASS,
message="Consider adding .dockerignore"
))
# Summary
docker_results = [r for r in self.results if r.category == "docker"]
passed = len([r for r in docker_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(docker_results)} tests passed")
# =========================================================================
# TYPE CHECKING TESTS - Run mypy on typed modules
# =========================================================================
def test_types(self):
"""Run type checking on typed Python modules."""
print("\nš¤ TYPE CHECKING TESTS")
print("=" * 60)
scripts_dir = self.repo_root / "scripts"
# Check if any files have type hints
typed_files = []
for py_file in scripts_dir.glob("*.py"):
try:
content = py_file.read_text()
if ": " in content and ("def " in content or "class " in content):
# Has potential type hints
if re.search(r'def \w+\([^)]*:\s*\w+', content) or \
re.search(r'->\s*\w+', content) or \
"from typing import" in content:
typed_files.append(py_file)
except Exception:
pass
if not typed_files:
self.add_result(TestResult(
name="Type hints: none found",
category="types", subcategory="coverage",
status=TestStatus.SKIP,
message="No typed files detected"
))
else:
self.add_result(TestResult(
name=f"Type hints: {len(typed_files)} typed files",
category="types", subcategory="coverage",
status=TestStatus.PASS,
message="Files with type annotations"
))
# Try running mypy
try:
result = subprocess.run(
["mypy", "--ignore-missing-imports", str(scripts_dir)],
capture_output=True,
text=True,
timeout=60
)
if result.returncode == 0:
self.add_result(TestResult(
name="Mypy check: passed",
category="types", subcategory="mypy",
status=TestStatus.PASS,
message="No type errors"
))
else:
error_count = len([l for l in result.stdout.split('\n') if 'error:' in l])
# Type issues are informational - mypy ran successfully = PASS
self.add_result(TestResult(
name="Mypy check: ran",
category="types", subcategory="mypy",
status=TestStatus.PASS,
message=f"{error_count} type issues (info)"
))
except FileNotFoundError:
self.add_result(TestResult(
name="Mypy: not installed",
category="types", subcategory="mypy",
status=TestStatus.SKIP,
message="Install with: pip install mypy"
))
except Exception as e:
self.add_result(TestResult(
name="Mypy: error",
category="types", subcategory="mypy",
status=TestStatus.WARN,
message=str(e)[:100]
))
# Summary
type_results = [r for r in self.results if r.category == "types"]
passed = len([r for r in type_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(type_results)} tests passed")
# =========================================================================
# E2E/PLAYWRIGHT TESTS - Browser automation tests
# =========================================================================
def test_e2e(self):
"""Run end-to-end tests with Playwright."""
print("\nš E2E/PLAYWRIGHT TESTS")
print("=" * 60)
# Check for Playwright test directory
playwright_dir = self.repo_root / "tools" / "PLAYWRIGHT-TEST"
if not playwright_dir.exists():
self.add_result(TestResult(
name="Playwright directory: missing",
category="e2e", subcategory="setup",
status=TestStatus.SKIP,
message="tools/PLAYWRIGHT-TEST not found"
))
return
# Check for test files
test_files = list(playwright_dir.glob("**/*.spec.js")) + \
list(playwright_dir.glob("**/*.spec.ts")) + \
list(playwright_dir.glob("**/test_*.py"))
if test_files:
self.add_result(TestResult(
name=f"E2E test files: {len(test_files)} found",
category="e2e", subcategory="files",
status=TestStatus.PASS,
message="Test files discovered"
))
else:
self.add_result(TestResult(
name="E2E test files: none found",
category="e2e", subcategory="files",
status=TestStatus.SKIP,
message="No *.spec.js/ts files"
))
# Check playwright config
config_files = list(playwright_dir.glob("playwright.config.*"))
if config_files:
self.add_result(TestResult(
name="Playwright config: exists",
category="e2e", subcategory="config",
status=TestStatus.PASS,
file_path=str(config_files[0])
))
else:
# Missing config is expected if not using Playwright
self.add_result(TestResult(
name="Playwright config: missing",
category="e2e", subcategory="config",
status=TestStatus.SKIP,
message="No playwright.config.* found"
))
# Check package.json for playwright dependency
package_json = playwright_dir / "package.json"
if package_json.exists():
try:
with open(package_json) as f:
pkg = json.load(f)
deps = {**pkg.get("dependencies", {}), **pkg.get("devDependencies", {})}
if any("playwright" in d.lower() for d in deps):
self.add_result(TestResult(
name="Playwright dependency: installed",
category="e2e", subcategory="dependencies",
status=TestStatus.PASS
))
except Exception:
pass
# Summary
e2e_results = [r for r in self.results if r.category == "e2e"]
passed = len([r for r in e2e_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(e2e_results)} tests passed")
# =========================================================================
# COVERAGE REPORT TESTS - Track test coverage
# =========================================================================
def test_coverage(self):
"""Track and report test coverage metrics."""
print("\nš COVERAGE REPORT TESTS")
print("=" * 60)
# Check for coverage configuration
coverage_configs = [
self.repo_root / ".coveragerc",
self.repo_root / "pyproject.toml",
self.repo_root / "setup.cfg",
]
has_config = False
for config in coverage_configs:
if config.exists():
try:
content = config.read_text()
if "[coverage" in content or "tool.coverage" in content:
has_config = True
self.add_result(TestResult(
name=f"Coverage config: {config.name}",
category="coverage", subcategory="config",
status=TestStatus.PASS,
file_path=str(config)
))
break
except Exception:
pass
if not has_config:
self.add_result(TestResult(
name="Coverage config: not configured",
category="coverage", subcategory="config",
status=TestStatus.SKIP,
message="No coverage configuration found"
))
# Check for existing coverage reports
coverage_reports = list(self.repo_root.glob("**/htmlcov/index.html")) + \
list(self.repo_root.glob("**/coverage.xml")) + \
list(self.repo_root.glob("**/.coverage"))
if coverage_reports:
self.add_result(TestResult(
name=f"Coverage reports: {len(coverage_reports)} found",
category="coverage", subcategory="reports",
status=TestStatus.PASS,
message="Existing coverage data available"
))
# Check test coverage summary files
coverage_summaries = list(self.repo_root.glob("**/TEST_COVERAGE_SUMMARY.md"))
for summary in coverage_summaries:
try:
content = summary.read_text()
# Look for coverage percentage
match = re.search(r'(\d+(?:\.\d+)?)\s*%', content)
if match:
pct = float(match.group(1))
self.add_result(TestResult(
name=f"Coverage: {summary.parent.name}",
category="coverage", subcategory="metrics",
status=TestStatus.PASS if pct >= 80 else TestStatus.WARN,
message=f"{pct}% coverage",
file_path=str(summary)
))
except Exception:
pass
# Component coverage analysis
components = {
"agents": len(list((self.repo_root / "agents").glob("*.md"))) if (self.repo_root / "agents").exists() else 0,
"commands": len(list((self.repo_root / "commands").glob("*.md"))) if (self.repo_root / "commands").exists() else 0,
"skills": len(list((self.repo_root / "skills").glob("*/SKILL.md"))) if (self.repo_root / "skills").exists() else 0,
"scripts": len(list((self.repo_root / "scripts").glob("*.py"))) if (self.repo_root / "scripts").exists() else 0,
}
total_components = sum(components.values())
self.add_result(TestResult(
name=f"Component inventory: {total_components} total",
category="coverage", subcategory="inventory",
status=TestStatus.PASS,
message=f"agents:{components['agents']}, commands:{components['commands']}, skills:{components['skills']}, scripts:{components['scripts']}"
))
# Summary
cov_results = [r for r in self.results if r.category == "coverage"]
passed = len([r for r in cov_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(cov_results)} tests passed")
# =========================================================================
# SMOKE TESTS - Quick critical path checks
# =========================================================================
def test_smoke(self):
"""Quick smoke tests for critical functionality."""
print("\nšØ SMOKE TESTS")
print("=" * 60)
# Test 1: Can we read critical files?
critical_files = [
"CLAUDE.md",
"README.md",
"config/component-counts.json",
".coditect/component-activation-status.json",
]
for file_path in critical_files:
full_path = self.repo_root / file_path
if full_path.exists():
try:
content = full_path.read_text()
if len(content) > 0:
self.add_result(TestResult(
name=f"Smoke: read {file_path}",
category="smoke", subcategory="read",
status=TestStatus.PASS,
message=f"{len(content)} bytes"
))
else:
self.add_result(TestResult(
name=f"Smoke: {file_path} empty",
category="smoke", subcategory="read",
status=TestStatus.WARN,
message="File is empty"
))
except Exception as e:
self.add_result(TestResult(
name=f"Smoke: read error {file_path}",
category="smoke", subcategory="read",
status=TestStatus.FAIL,
message=str(e)[:100]
))
else:
self.add_result(TestResult(
name=f"Smoke: missing {file_path}",
category="smoke", subcategory="read",
status=TestStatus.WARN,
message="File not found"
))
# Test 2: Can Python execute?
try:
result = subprocess.run(
[sys.executable, "--version"],
capture_output=True,
text=True,
timeout=5
)
if result.returncode == 0:
self.add_result(TestResult(
name=f"Smoke: Python {result.stdout.strip()}",
category="smoke", subcategory="runtime",
status=TestStatus.PASS
))
except Exception as e:
self.add_result(TestResult(
name="Smoke: Python error",
category="smoke", subcategory="runtime",
status=TestStatus.FAIL,
message=str(e)[:100]
))
# Test 3: Can git execute?
try:
result = subprocess.run(
["git", "--version"],
capture_output=True,
text=True,
timeout=5
)
if result.returncode == 0:
self.add_result(TestResult(
name=f"Smoke: {result.stdout.strip()}",
category="smoke", subcategory="runtime",
status=TestStatus.PASS
))
except Exception as e:
self.add_result(TestResult(
name="Smoke: git error",
category="smoke", subcategory="runtime",
status=TestStatus.WARN,
message=str(e)[:100]
))
# Test 4: Directory structure intact?
required_dirs = ["agents", "commands", "skills", "scripts", "docs", "config"]
for dir_name in required_dirs:
dir_path = self.repo_root / dir_name
if dir_path.exists() and dir_path.is_dir():
self.add_result(TestResult(
name=f"Smoke: {dir_name}/ exists",
category="smoke", subcategory="structure",
status=TestStatus.PASS
))
else:
self.add_result(TestResult(
name=f"Smoke: {dir_name}/ missing",
category="smoke", subcategory="structure",
status=TestStatus.FAIL,
message="Required directory not found"
))
# Test 5: Can we parse JSON configs?
try:
counts_path = self.repo_root / "config" / "component-counts.json"
if counts_path.exists():
with open(counts_path) as f:
data = json.load(f)
self.add_result(TestResult(
name="Smoke: parse component-counts.json",
category="smoke", subcategory="parse",
status=TestStatus.PASS,
message=f"{len(data)} keys"
))
except Exception as e:
self.add_result(TestResult(
name="Smoke: parse config error",
category="smoke", subcategory="parse",
status=TestStatus.FAIL,
message=str(e)[:100]
))
# Summary
smoke_results = [r for r in self.results if r.category == "smoke"]
passed = len([r for r in smoke_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(smoke_results)} tests passed")
# =========================================================================
# API CONTRACT TESTS - OpenAPI/schema validation
# =========================================================================
def test_api_contract(self):
"""Validate API contracts and schemas."""
print("\nš API CONTRACT TESTS")
print("=" * 60)
# Look for OpenAPI/Swagger specs
api_specs = list(self.repo_root.glob("**/openapi.yaml")) + \
list(self.repo_root.glob("**/openapi.json")) + \
list(self.repo_root.glob("**/swagger.yaml")) + \
list(self.repo_root.glob("**/swagger.json")) + \
list(self.repo_root.glob("**/api-spec*.yaml"))
api_specs = [f for f in api_specs if "node_modules" not in str(f)]
if api_specs:
for spec_file in api_specs:
try:
if spec_file.suffix in ['.yaml', '.yml']:
with open(spec_file) as f:
spec = yaml.safe_load(f)
else:
with open(spec_file) as f:
spec = json.load(f)
# Basic OpenAPI validation
if "openapi" in spec or "swagger" in spec:
version = spec.get("openapi") or spec.get("swagger")
self.add_result(TestResult(
name=f"API spec: {spec_file.name}",
category="api_contract", subcategory="openapi",
status=TestStatus.PASS,
message=f"Version {version}",
file_path=str(spec_file)
))
else:
self.add_result(TestResult(
name=f"API spec: {spec_file.name}",
category="api_contract", subcategory="openapi",
status=TestStatus.WARN,
message="Missing openapi/swagger field",
file_path=str(spec_file)
))
except Exception as e:
self.add_result(TestResult(
name=f"API spec: {spec_file.name}",
category="api_contract", subcategory="openapi",
status=TestStatus.FAIL,
message=str(e)[:100],
file_path=str(spec_file)
))
else:
self.add_result(TestResult(
name="API specs: none found",
category="api_contract", subcategory="openapi",
status=TestStatus.SKIP,
message="No openapi.yaml/json files"
))
# Check for JSON Schema files
schema_files = list(self.repo_root.glob("**/schema*.json")) + \
list(self.repo_root.glob("**/*.schema.json"))
schema_files = [f for f in schema_files if "node_modules" not in str(f)]
if schema_files:
for schema_file in schema_files[:10]: # Limit
try:
with open(schema_file) as f:
schema = json.load(f)
if "$schema" in schema or "type" in schema or "properties" in schema:
self.add_result(TestResult(
name=f"JSON Schema: {schema_file.name}",
category="api_contract", subcategory="jsonschema",
status=TestStatus.PASS,
file_path=str(schema_file)
))
except Exception as e:
self.add_result(TestResult(
name=f"JSON Schema: {schema_file.name}",
category="api_contract", subcategory="jsonschema",
status=TestStatus.FAIL,
message=str(e)[:100],
file_path=str(schema_file)
))
# Check for TypeScript types (API contracts)
ts_files = list(self.repo_root.glob("**/types.ts")) + \
list(self.repo_root.glob("**/types/*.ts"))
ts_files = [f for f in ts_files if "node_modules" not in str(f)]
if ts_files:
self.add_result(TestResult(
name=f"TypeScript types: {len(ts_files)} files",
category="api_contract", subcategory="typescript",
status=TestStatus.PASS,
message="Type definitions found"
))
# Summary
api_results = [r for r in self.results if r.category == "api_contract"]
passed = len([r for r in api_results if r.status == TestStatus.PASS])
print(f" Summary: {passed}/{len(api_results)} tests passed")
# =========================================================================
# REPORT GENERATION
# =========================================================================
def generate_report(self) -> dict:
"""Generate comprehensive report."""
duration = (datetime.now(timezone.utc) - self.start_time).total_seconds()
# Count by status
by_status = defaultdict(int)
for r in self.results:
by_status[r.status.value] += 1
# Count by category
by_category = defaultdict(lambda: defaultdict(int))
for r in self.results:
by_category[r.category][r.status.value] += 1
return {
"timestamp": datetime.now(timezone.utc).isoformat(),
"duration_seconds": round(duration, 2),
"summary": {
"total": len(self.results),
"passed": by_status.get("PASS", 0),
"failed": by_status.get("FAIL", 0),
"warnings": by_status.get("WARN", 0),
"skipped": by_status.get("SKIP", 0),
# Pass rate = passed / (passed + failed), excluding warnings/skips
"pass_rate": round(by_status.get("PASS", 0) / (by_status.get("PASS", 0) + by_status.get("FAIL", 0)) * 100, 1) if (by_status.get("PASS", 0) + by_status.get("FAIL", 0)) > 0 else 100.0
},
"by_category": {k: dict(v) for k, v in by_category.items()},
"failed_tests": [r.to_dict() for r in self.results if r.status == TestStatus.FAIL],
"all_results": [r.to_dict() for r in self.results]
}
def save_report(self, report: dict):
"""Save report to files."""
reports_dir = self.repo_root / "test-results"
reports_dir.mkdir(exist_ok=True)
# JSON report
json_path = reports_dir / "test-results.json"
with open(json_path, "w") as f:
json.dump(report, f, indent=2)
# Markdown report
md_path = reports_dir / "TEST-RESULTS.md"
with open(md_path, "w") as f:
f.write(self._generate_markdown(report))
return json_path, md_path
def _generate_markdown(self, report: dict) -> str:
"""Generate Markdown report."""
lines = [
"# CODITECT Comprehensive Test Results",
"",
f"**Generated:** {report['timestamp']}",
f"**Duration:** {report['duration_seconds']}s",
"",
"## Summary",
"",
"| Metric | Value |",
"|--------|-------|",
f"| Total Tests | {report['summary']['total']} |",
f"| Passed | {report['summary']['passed']} |",
f"| Failed | {report['summary']['failed']} |",
f"| Warnings | {report['summary']['warnings']} |",
f"| Pass Rate | {report['summary']['pass_rate']}% |",
"",
"## Results by Category",
"",
]
for cat, stats in sorted(report['by_category'].items()):
total = sum(stats.values())
passed = stats.get("PASS", 0)
failed = stats.get("FAIL", 0)
status = "ā
" if failed == 0 else "ā"
lines.append(f"| {status} {cat.upper()} | {passed}/{total} passed |")
if report['failed_tests']:
lines.extend([
"",
"## Failed Tests",
"",
])
for test in report['failed_tests'][:50]: # Limit to 50
lines.append(f"- **{test['name']}**: {test['message']}")
lines.extend([
"",
"---",
"*Generated by CODITECT Comprehensive Test Suite v2.0*"
])
return "\n".join(lines)
def run(self, categories: Optional[List[str]] = None):
"""Run all tests."""
self.start_time = datetime.now(timezone.utc)
print("\n" + "=" * 60)
print(" CODITECT COMPREHENSIVE TEST SUITE v2.0")
print("=" * 60)
print(f" Repository: {self.repo_root.name}")
print(f" Mode: {'Quick' if self.quick else 'Full'}")
print(f" Started: {self.start_time.strftime('%Y-%m-%d %H:%M:%S UTC')}")
# Load data
self.load_registry()
self.load_counts()
# All available test categories (34 total)
all_categories = [
# Original categories (14)
"agents", "commands", "skills", "scripts", "hooks",
"registry", "xref", "config", "docs", "structure",
"quality", "security", "consistency", "activation",
# Added categories (6)
"integration", "performance", "dependencies", "schemas",
"doc_coverage", "workflows",
# New categories (14)
"unit_tests", "imports", "symlinks", "git_integrity", "lint",
"cli", "links", "parse", "docker", "types",
"e2e", "coverage", "smoke", "api_contract"
]
run_categories = categories or all_categories
# Run tests - Original categories
if "agents" in run_categories:
self.test_agents()
if "commands" in run_categories:
self.test_commands()
if "skills" in run_categories:
self.test_skills()
if "scripts" in run_categories:
self.test_scripts()
if "hooks" in run_categories:
self.test_hooks()
if "registry" in run_categories:
self.test_registry()
if "xref" in run_categories:
self.test_cross_references()
if "config" in run_categories:
self.test_config()
if "docs" in run_categories:
self.test_documentation()
if "structure" in run_categories:
self.test_structure()
if "quality" in run_categories:
self.test_content_quality()
if "security" in run_categories:
self.test_security()
if "consistency" in run_categories:
self.test_consistency()
if "activation" in run_categories:
self.test_activation_workflow()
if "integration" in run_categories:
self.test_integration()
if "performance" in run_categories:
self.test_performance()
if "dependencies" in run_categories:
self.test_dependencies()
if "schemas" in run_categories:
self.test_schemas()
if "doc_coverage" in run_categories:
self.test_doc_coverage()
if "workflows" in run_categories:
self.test_workflows()
# Run tests - New categories (High Priority)
if "unit_tests" in run_categories:
self.test_unit_tests()
if "imports" in run_categories:
self.test_imports()
if "symlinks" in run_categories:
self.test_symlinks()
if "git_integrity" in run_categories:
self.test_git_integrity()
if "lint" in run_categories:
self.test_lint()
# Run tests - New categories (Medium Priority)
if "cli" in run_categories:
self.test_cli()
if "links" in run_categories:
self.test_links()
if "parse" in run_categories:
self.test_parse()
if "docker" in run_categories:
self.test_docker()
if "types" in run_categories:
self.test_types()
# Run tests - New categories (Nice to Have)
if "e2e" in run_categories:
self.test_e2e()
if "coverage" in run_categories:
self.test_coverage()
if "smoke" in run_categories:
self.test_smoke()
if "api_contract" in run_categories:
self.test_api_contract()
# Generate report
report = self.generate_report()
json_path, md_path = self.save_report(report)
# Print summary
print("\n" + "=" * 60)
print(" COMPREHENSIVE TEST RESULTS")
print("=" * 60)
print(f" Total Tests: {report['summary']['total']}")
print(f" Passed: \033[92m{report['summary']['passed']}\033[0m")
print(f" Failed: \033[91m{report['summary']['failed']}\033[0m")
print(f" Warnings: \033[93m{report['summary']['warnings']}\033[0m")
print(f" Pass Rate: {report['summary']['pass_rate']}%")
print("=" * 60)
print(f" Reports: {json_path.name}, {md_path.name}")
print("=" * 60 + "\n")
return report
def main(): import argparse
parser = argparse.ArgumentParser(description="CODITECT Comprehensive Test Suite")
parser.add_argument("--category", "-c", help="Run specific category")
parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output")
parser.add_argument("--quick", "-q", action="store_true", help="Quick mode (summary only)")
parser.add_argument("--json", action="store_true", help="Output JSON only")
args = parser.parse_args()
script_path = Path(__file__).resolve()
repo_root = script_path.parent.parent
suite = ComprehensiveTestSuite(repo_root, verbose=args.verbose, quick=args.quick)
categories = [args.category] if args.category else None
report = suite.run(categories)
if args.json:
print(json.dumps(report, indent=2))
sys.exit(0 if report['summary']['failed'] == 0 else 1)
if name == "main": main()