scripts-validate-components

#!/usr/bin/env python3 """

title: "Validate Components" component_type: script version: "1.0.0" audience: contributor status: stable summary: "Component Standards Validation Script" keywords: ['analysis', 'components', 'validate', 'validation'] tokens: ~500 created: 2025-12-22 updated: 2025-12-22 script_name: "validate-components.py" language: python executable: true usage: "python3 scripts/validate-components.py [options]" python_version: "3.10+" dependencies: [] modifies_files: false network_access: false requires_auth: false

Component Standards Validation Script

Validates CODITECT components against standards for gap analysis. Generates compliance reports for agents, skills, commands, hooks, scripts, config.

Usage: python3 scripts/validate-components.py --output gap-analysis-data.json python3 scripts/validate-components.py --type agent --detailed python3 scripts/validate-components.py --summary """

import sys import json import yaml import argparse from pathlib import Path from typing import Dict, List, Any, Optional from collections import defaultdict import re

class ComponentValidator: """Validates CODITECT components against standards"""

def __init__(self, framework_root: Path):
    self.framework_root = framework_root
    self.agents_dir = framework_root / "agents"
    self.skills_dir = framework_root / "skills"
    self.commands_dir = framework_root / "commands"
    self.hooks_dir = framework_root / "hooks"
    self.scripts_dir = framework_root / "scripts"
    self.config_files = [
        framework_root / "settings.local.json",
        framework_root / ".coditect" / "settings.local.json",
        framework_root / "settings.agents-research.json"
    ]

def validate_yaml_frontmatter(self, file_path: Path) -> Dict[str, Any]:
    """Validate YAML frontmatter in a markdown file"""
    try:
        content = file_path.read_text(encoding='utf-8')

        # Check for YAML frontmatter (---\n...\n---)
        if not content.startswith('---\n'):
            return {
                "present": False,
                "error": "No YAML frontmatter found"
            }

        # Extract YAML block
        parts = content.split('---\n', 2)
        if len(parts) < 3:
            return {
                "present": False,
                "error": "Incomplete YAML frontmatter"
            }

        yaml_text = parts[1]
        frontmatter = yaml.safe_load(yaml_text)

        return {
            "present": True,
            "data": frontmatter,
            "error": None
        }
    except Exception as e:
        return {
            "present": False,
            "error": str(e)
        }

def grade_agent(self, agent_path: Path) -> Dict[str, Any]:
    """Grade an agent file against CODITECT-STANDARD-AGENTS.md"""
    result = {
        "file": agent_path.name,
        "path": str(agent_path),
        "checks": {},
        "score": 0,
        "grade": "F"
    }

    points = 0
    max_points = 0

    # Check 1: YAML frontmatter present (40 points)
    max_points += 40
    yaml_check = self.validate_yaml_frontmatter(agent_path)
    result["checks"]["yaml_frontmatter"] = yaml_check["present"]
    if yaml_check["present"]:
        points += 40

        # Check required fields
        frontmatter = yaml_check.get("data", {})
        required_fields = ["name", "description", "tools"]
        for field in required_fields:
            result["checks"][f"has_{field}"] = field in frontmatter
    else:
        result["checks"]["yaml_error"] = yaml_check.get("error")

    # Check 2: Description quality (15 points)
    max_points += 15
    if yaml_check["present"] and "description" in yaml_check.get("data", {}):
        desc = yaml_check["data"]["description"]
        if len(desc) > 50:  # Meaningful description
            points += 15
            result["checks"]["good_description"] = True

    # Check 3: Documentation structure (15 points)
    max_points += 15
    content = agent_path.read_text(encoding='utf-8')
    has_sections = all(
        section in content
        for section in ["##", "###"]  # Has heading structure
    )
    if has_sections:
        points += 15
        result["checks"]["has_sections"] = True

    # Check 4: File size reasonable (10 points)
    max_points += 10
    size_kb = agent_path.stat().st_size / 1024
    if size_kb < 100:  # Under 100KB
        points += 10
        result["checks"]["reasonable_size"] = True
    result["checks"]["size_kb"] = round(size_kb, 1)

    # Check 5: UTF-8 encoding (10 points)
    max_points += 10
    try:
        agent_path.read_text(encoding='utf-8')
        points += 10
        result["checks"]["valid_utf8"] = True
    except UnicodeDecodeError:
        result["checks"]["valid_utf8"] = False

    # Check 6: Model specified (10 points)
    max_points += 10
    if yaml_check["present"] and "model" in yaml_check.get("data", {}):
        points += 10
        result["checks"]["has_model"] = True

    # Calculate score and grade
    result["score"] = round((points / max_points) * 100, 1)
    result["grade"] = self._score_to_grade(result["score"])
    result["points"] = f"{points}/{max_points}"

    return result

def grade_skill(self, skill_path: Path) -> Dict[str, Any]:
    """Grade a skill SKILL.md file against CODITECT-STANDARD-SKILLS.md"""
    result = {
        "file": skill_path.parent.name + "/SKILL.md",
        "path": str(skill_path),
        "checks": {},
        "score": 0,
        "grade": "F"
    }

    points = 0
    max_points = 0

    # Check 1: YAML frontmatter MANDATORY (50 points)
    max_points += 50
    yaml_check = self.validate_yaml_frontmatter(skill_path)
    result["checks"]["yaml_frontmatter"] = yaml_check["present"]
    if yaml_check["present"]:
        points += 50

        # Check required fields
        frontmatter = yaml_check.get("data", {})
        required_fields = ["name", "description"]
        for field in required_fields:
            result["checks"][f"has_{field}"] = field in frontmatter
    else:
        result["checks"]["yaml_error"] = yaml_check.get("error")
        result["checks"]["CRITICAL"] = "YAML frontmatter is MANDATORY for skills"

    # Check 2: Progressive disclosure structure (20 points)
    max_points += 20
    content = skill_path.read_text(encoding='utf-8')
    has_levels = all(
        marker in content
        for marker in ["## When to Use", "## Core Capabilities", "##"]
    )
    if has_levels:
        points += 20
        result["checks"]["progressive_disclosure"] = True

    # Check 3: Token count reasonable (15 points)
    max_points += 15
    token_estimate = len(content) / 4  # Rough estimate
    if token_estimate < 5000:
        points += 15
        result["checks"]["reasonable_tokens"] = True
    result["checks"]["estimated_tokens"] = round(token_estimate)

    # Check 4: allowed-tools specified (15 points)
    max_points += 15
    if yaml_check["present"] and "allowed-tools" in yaml_check.get("data", {}):
        points += 15
        result["checks"]["has_allowed_tools"] = True

    # Calculate score and grade
    result["score"] = round((points / max_points) * 100, 1)
    result["grade"] = self._score_to_grade(result["score"])
    result["points"] = f"{points}/{max_points}"

    return result

def grade_command(self, command_path: Path) -> Dict[str, Any]:
    """Grade a command file against CODITECT-STANDARD-COMMANDS.md"""
    result = {
        "file": command_path.name,
        "path": str(command_path),
        "checks": {},
        "score": 0,
        "grade": "F"
    }

    points = 0
    max_points = 0

    # Check 1: Has structure (30 points)
    max_points += 30
    content = command_path.read_text(encoding='utf-8')
    if len(content) > 50:  # Not empty
        points += 30
        result["checks"]["has_content"] = True

    # Check 2: Uses $ARGUMENTS (25 points)
    max_points += 25
    if "$ARGUMENTS" in content:
        points += 25
        result["checks"]["uses_arguments"] = True

    # Check 3: Has frontmatter (20 points)
    max_points += 20
    yaml_check = self.validate_yaml_frontmatter(command_path)
    if yaml_check["present"]:
        points += 20
        result["checks"]["has_frontmatter"] = True

    # Check 4: Has description (15 points)
    max_points += 15
    if yaml_check["present"] and "description" in yaml_check.get("data", {}):
        points += 15
        result["checks"]["has_description"] = True

    # Check 5: Reasonable size (10 points)
    max_points += 10
    size_kb = command_path.stat().st_size / 1024
    if size_kb < 50:  # Under 50KB
        points += 10
        result["checks"]["reasonable_size"] = True
    result["checks"]["size_kb"] = round(size_kb, 1)

    # Calculate score and grade
    result["score"] = round((points / max_points) * 100, 1)
    result["grade"] = self._score_to_grade(result["score"])
    result["points"] = f"{points}/{max_points}"

    return result

def grade_hook(self, hook_path: Path) -> Dict[str, Any]:
    """Grade a hook file against CODITECT-STANDARD-HOOKS.md"""
    result = {
        "file": hook_path.name,
        "path": str(hook_path),
        "checks": {},
        "score": 0,
        "grade": "F"
    }

    points = 0
    max_points = 0

    content = hook_path.read_text(encoding='utf-8')

    # Check 1: Has shebang (25 points)
    max_points += 25
    if content.startswith("#!"):
        points += 25
        result["checks"]["has_shebang"] = True

    # Check 2: Has error handling (25 points)
    max_points += 25
    if hook_path.suffix == ".sh":
        has_error_handling = "set -e" in content or "trap" in content
    else:  # Python
        has_error_handling = "try:" in content or "except" in content

    if has_error_handling:
        points += 25
        result["checks"]["has_error_handling"] = True

    # Check 3: Reads stdin (20 points)
    max_points += 20
    reads_stdin = "stdin" in content.lower() or "cat" in content or "sys.stdin" in content
    if reads_stdin:
        points += 20
        result["checks"]["reads_stdin"] = True

    # Check 4: Has documentation (15 points)
    max_points += 15
    has_docs = "#" in content and len([line for line in content.split('\n') if line.strip().startswith("#")]) > 5
    if has_docs:
        points += 15
        result["checks"]["has_documentation"] = True

    # Check 5: Is executable (15 points)
    max_points += 15
    if hook_path.stat().st_mode & 0o111:  # Any execute bit set
        points += 15
        result["checks"]["is_executable"] = True

    # Calculate score and grade
    result["score"] = round((points / max_points) * 100, 1)
    result["grade"] = self._score_to_grade(result["score"])
    result["points"] = f"{points}/{max_points}"

    return result

def grade_script(self, script_path: Path) -> Dict[str, Any]:
    """Grade a script file against CODITECT-STANDARD-SCRIPTS.md"""
    result = {
        "file": script_path.name,
        "path": str(script_path),
        "checks": {},
        "score": 0,
        "grade": "F"
    }

    points = 0
    max_points = 0

    content = script_path.read_text(encoding='utf-8')

    # Check 1: Has shebang (30 points)
    max_points += 30
    if content.startswith("#!"):
        points += 30
        result["checks"]["has_shebang"] = True

    # Check 2: Has docstring/header (25 points)
    max_points += 25
    has_docstring = '"""' in content or "'''" in content or (content.count("#") > 10)
    if has_docstring:
        points += 25
        result["checks"]["has_docstring"] = True

    # Check 3: Has error handling (20 points)
    max_points += 20
    if script_path.suffix == ".py":
        has_error_handling = "try:" in content and "except" in content
    else:  # Shell
        has_error_handling = "set -e" in content

    if has_error_handling:
        points += 20
        result["checks"]["has_error_handling"] = True

    # Check 4: Has CLI arguments (15 points)
    max_points += 15
    has_cli = "argparse" in content or "getopts" in content
    if has_cli:
        points += 15
        result["checks"]["has_cli_parsing"] = True

    # Check 5: Is executable (10 points)
    max_points += 10
    if script_path.stat().st_mode & 0o111:
        points += 10
        result["checks"]["is_executable"] = True

    # Calculate score and grade
    result["score"] = round((points / max_points) * 100, 1)
    result["grade"] = self._score_to_grade(result["score"])
    result["points"] = f"{points}/{max_points}"

    return result

def grade_config(self, config_path: Path) -> Dict[str, Any]:
    """Grade a configuration file against CODITECT-STANDARD-CONFIGURATION.md"""
    result = {
        "file": config_path.name,
        "path": str(config_path),
        "checks": {},
        "score": 0,
        "grade": "F"
    }

    if not config_path.exists():
        result["checks"]["exists"] = False
        return result

    result["checks"]["exists"] = True
    points = 0
    max_points = 0

    try:
        config = json.loads(config_path.read_text(encoding='utf-8'))

        # Check 1: Valid JSON (40 points)
        max_points += 40
        points += 40
        result["checks"]["valid_json"] = True

        # Check 2: Has permissions section (20 points)
        max_points += 20
        if "permissions" in config:
            points += 20
            result["checks"]["has_permissions"] = True

        # Check 3: Permissions structure correct (15 points)
        max_points += 15
        if "permissions" in config:
            perms = config["permissions"]
            if isinstance(perms, dict) and ("allow" in perms or "deny" in perms):
                points += 15
                result["checks"]["permissions_structure"] = True

        # Check 4: Has hooks or MCP (15 points)
        max_points += 15
        if "hooks" in config or "mcpServers" in config:
            points += 15
            result["checks"]["has_hooks_or_mcp"] = True

        # Check 5: Reasonable size (10 points)
        max_points += 10
        size_kb = config_path.stat().st_size / 1024
        if size_kb < 100:
            points += 10
            result["checks"]["reasonable_size"] = True
        result["checks"]["size_kb"] = round(size_kb, 1)

    except json.JSONDecodeError as e:
        result["checks"]["valid_json"] = False
        result["checks"]["json_error"] = str(e)
        max_points += 40

    # Calculate score and grade
    result["score"] = round((points / max_points) * 100, 1) if max_points > 0 else 0
    result["grade"] = self._score_to_grade(result["score"])
    result["points"] = f"{points}/{max_points}"

    return result

def _score_to_grade(self, score: float) -> str:
    """Convert numeric score to letter grade"""
    if score >= 90:
        return "A"
    elif score >= 80:
        return "B"
    elif score >= 70:
        return "C"
    elif score >= 60:
        return "D"
    else:
        return "F"

def validate_all(self) -> Dict[str, Any]:
    """Validate all components and generate comprehensive report"""
    report = {
        "summary": {},
        "agents": [],
        "skills": [],
        "commands": [],
        "hooks": [],
        "scripts": [],
        "configs": []
    }

    # Validate agents
    if self.agents_dir.exists():
        agent_files = [f for f in self.agents_dir.glob("*.md") if f.name != "README.md"]
        report["agents"] = [self.grade_agent(f) for f in agent_files]

    # Validate skills
    if self.skills_dir.exists():
        skill_files = list(self.skills_dir.glob("*/SKILL.md"))
        report["skills"] = [self.grade_skill(f) for f in skill_files]

    # Validate commands
    if self.commands_dir.exists():
        command_files = [f for f in self.commands_dir.glob("*.md") if f.name not in ["README.md", "COMMAND-GUIDE.md"]]
        report["commands"] = [self.grade_command(f) for f in command_files]

    # Validate hooks
    if self.hooks_dir.exists():
        hook_files = list(self.hooks_dir.glob("*.sh")) + list(self.hooks_dir.glob("*.py"))
        report["hooks"] = [self.grade_hook(f) for f in hook_files]

    # Validate scripts
    if self.scripts_dir.exists():
        script_files = list(self.scripts_dir.glob("*.py")) + list(self.scripts_dir.glob("*.sh"))
        report["scripts"] = [self.grade_script(f) for f in script_files]

    # Validate configurations
    report["configs"] = [self.grade_config(f) for f in self.config_files if f.exists()]

    # Generate summary
    report["summary"] = self._generate_summary(report)

    return report

def _generate_summary(self, report: Dict[str, Any]) -> Dict[str, Any]:
    """Generate summary statistics from full report"""
    summary = {}

    for component_type in ["agents", "skills", "commands", "hooks", "scripts", "configs"]:
        components = report[component_type]
        total = len(components)

        grade_counts = defaultdict(int)
        total_score = 0

        for component in components:
            grade_counts[component["grade"]] += 1
            total_score += component["score"]

        avg_score = round(total_score / total, 1) if total > 0 else 0

        summary[component_type] = {
            "total": total,
            "average_score": avg_score,
            "average_grade": self._score_to_grade(avg_score),
            "grade_distribution": dict(grade_counts),
            "compliance_rate": round((grade_counts["A"] + grade_counts["B"]) / total * 100, 1) if total > 0 else 0
        }

    return summary

def main(): parser = argparse.ArgumentParser(description="Validate CODITECT components against standards") parser.add_argument("--output", "-o", help="Output JSON file path", default="gap-analysis-data.json") parser.add_argument("--type", "-t", choices=["agent", "skill", "command", "hook", "script", "config"], help="Validate specific type only") parser.add_argument("--summary", "-s", action="store_true", help="Show summary only") parser.add_argument("--detailed", "-d", action="store_true", help="Show detailed results")

args = parser.parse_args()

# Find framework root
script_dir = Path(__file__).parent
framework_root = script_dir.parent

validator = ComponentValidator(framework_root)
report = validator.validate_all()

# Output results
if args.summary:
    print(json.dumps(report["summary"], indent=2))
elif args.detailed:
    print(json.dumps(report, indent=2))
else:
    # Save to file
    output_path = Path(args.output)
    output_path.write_text(json.dumps(report, indent=2), encoding='utf-8')
    print(f"✅ Validation complete. Report saved to: {output_path}")
    print(f"\nSummary:")
    for component_type, stats in report["summary"].items():
        print(f"  {component_type.title()}: {stats['total']} files, avg {stats['average_grade']} ({stats['average_score']}%), {stats['compliance_rate']}% compliant")

if name == "main": main()