#!/usr/bin/env python3 """ CODITECT Skill Validation Test Suite

Comprehensive tests to verify all 186 skills have correct SKILL.md structure, valid configuration, and proper documentation.

Run: python3 scripts/tests/test_skills.py python3 scripts/tests/test_skills.py -v # Verbose python3 scripts/tests/test_skills.py --skill api-design-patterns # Single skill

Author: CODITECT Team Version: 1.0.0 Created: 2025-12-22 """

import os import re import sys import json import argparse from pathlib import Path from typing import Dict, List, Tuple, Optional from dataclasses import dataclass

Shared Colors module (consolidates 36 duplicate definitions)

_script_dir = Path(file).parent.parent # tests/ -> scripts/ sys.path.insert(0, str(_script_dir / "core")) from colors import Colors

class TestResult: passed: bool message: str skill: str test_name: str

@dataclass class SkillValidation: name: str path: str passed: bool tests: List[TestResult] errors: List[str] warnings: List[str]

class SkillTestSuite: """Comprehensive test suite for CODITECT skills"""

REQUIRED_SECTIONS = ['When to Use', 'How to Use']
RECOMMENDED_SECTIONS = ['Examples', 'Best Practices', 'Related']

def __init__(self, framework_root: Path, verbose: bool = False):
    self.framework_root = framework_root
    self.skills_dir = framework_root / "skills"
    self.verbose = verbose
    self.results: List[SkillValidation] = []

def get_all_skills(self) -> List[Path]:
    """Get all SKILL.md files recursively"""
    skills = []
    for skill_file in sorted(self.skills_dir.rglob("SKILL.md")):
        skills.append(skill_file)
    return skills

def parse_frontmatter(self, content: str) -> Tuple[Optional[Dict], str]:
    """Parse YAML frontmatter from markdown content"""
    match = re.match(r'^---\n(.*?)\n---\n(.*)$', content, re.DOTALL)
    if not match:
        return None, content

    frontmatter_text = match.group(1)
    body = match.group(2)

    frontmatter = {}
    current_key = None
    current_value = []

    for line in frontmatter_text.split('\n'):
        if ':' in line and not line.startswith(' ') and not line.startswith('\t'):
            if current_key:
                frontmatter[current_key] = '\n'.join(current_value).strip()
            key, value = line.split(':', 1)
            current_key = key.strip()
            current_value = [value.strip()]
        elif current_key:
            current_value.append(line)

    if current_key:
        frontmatter[current_key] = '\n'.join(current_value).strip()

    return frontmatter, body

def get_skill_name(self, skill_path: Path) -> str:
    """Extract skill name from path"""
    # The skill name is the parent directory of SKILL.md
    return skill_path.parent.name

def test_has_frontmatter(self, content: str, skill_name: str) -> TestResult:
    """Test that skill has YAML frontmatter"""
    has_frontmatter = content.startswith('---\n') and '\n---\n' in content[4:]
    return TestResult(
        passed=has_frontmatter,
        message="Has YAML frontmatter" if has_frontmatter else "Missing YAML frontmatter",
        skill=skill_name,
        test_name="has_frontmatter"
    )

def test_has_title(self, content: str, skill_name: str) -> TestResult:
    """Test that skill has a title (H1 heading)"""
    has_title = bool(re.search(r'^#\s+.+', content, re.MULTILINE))
    return TestResult(
        passed=has_title,
        message="Has title" if has_title else "Missing H1 title",
        skill=skill_name,
        test_name="has_title"
    )

def test_has_description(self, body: str, skill_name: str) -> TestResult:
    """Test that skill has a description after the title"""
    # Look for content after H1 and before first H2
    match = re.search(r'^#\s+.+\n\n(.+?)(?=\n##|\Z)', body, re.MULTILINE | re.DOTALL)
    has_desc = match is not None and len(match.group(1).strip()) > 20
    return TestResult(
        passed=has_desc,
        message="Has description" if has_desc else "Missing or too short description",
        skill=skill_name,
        test_name="has_description"
    )

def test_required_sections(self, body: str, skill_name: str) -> TestResult:
    """Test that skill has required sections"""
    missing = []
    for section in self.REQUIRED_SECTIONS:
        pattern = rf'^##\s+.*{re.escape(section)}'
        if not re.search(pattern, body, re.MULTILINE | re.IGNORECASE):
            missing.append(section)

    if missing:
        return TestResult(
            passed=False,
            message=f"Missing required sections: {', '.join(missing)}",
            skill=skill_name,
            test_name="required_sections"
        )
    return TestResult(True, "All required sections present", skill_name, "required_sections")

def test_has_code_examples(self, body: str, skill_name: str) -> TestResult:
    """Test that skill has code examples"""
    has_code = bool(re.search(r'```', body))
    return TestResult(
        passed=has_code,
        message="Has code examples" if has_code else "Missing code examples",
        skill=skill_name,
        test_name="has_code_examples"
    )

def test_no_placeholders(self, content: str, skill_name: str) -> TestResult:
    """Test that skill has no placeholder text outside code blocks"""
    # Remove code blocks before checking - allows TODO/XXX in example code
    content_no_code = re.sub(r'```[\s\S]*?```', '', content)
    content_no_code = re.sub(r'`[^`]+`', '', content_no_code)

    placeholders = [
        '[INSERT ', '[INSERT]', '[INSERT:',
        '[ADD HERE', '[ADD YOUR', '[ADD THE',
        '[PLACEHOLDER', '[TBD]', '[TBD ',
        'lorem ipsum', '<!-- TODO', '<!-- FIXME'
    ]

    found = []
    for p in placeholders:
        if p.lower() in content_no_code.lower():
            found.append(p)

    if found:
        return TestResult(
            passed=False,
            message=f"Contains placeholders: {', '.join(found)}",
            skill=skill_name,
            test_name="no_placeholders"
        )
    return TestResult(True, "No placeholders found", skill_name, "no_placeholders")

def test_minimum_content(self, content: str, skill_name: str) -> TestResult:
    """Test that skill has minimum content length"""
    min_length = 200  # Characters
    if len(content) < min_length:
        return TestResult(
            passed=False,
            message=f"Content too short ({len(content)} chars, min {min_length})",
            skill=skill_name,
            test_name="minimum_content"
        )
    return TestResult(True, f"Adequate content length ({len(content)} chars)", skill_name, "minimum_content")

def test_valid_markdown(self, content: str, skill_name: str) -> TestResult:
    """Test that markdown is valid (basic checks)"""
    errors = []

    # Check for unclosed code blocks
    code_blocks = content.count('```')
    if code_blocks % 2 != 0:
        errors.append("Unclosed code block")

    # Check for broken links (basic) - exclude code blocks
    # This prevents false positives like [scenario.attribute]() in JavaScript
    content_no_code = re.sub(r'```[\s\S]*?```', '', content)
    content_no_code = re.sub(r'`[^`]+`', '', content_no_code)
    broken_links = re.findall(r'\[([^\]]+)\]\(\)', content_no_code)
    if broken_links:
        errors.append(f"Empty link targets: {broken_links[:3]}")

    if errors:
        return TestResult(
            passed=False,
            message=f"Markdown issues: {'; '.join(errors)}",
            skill=skill_name,
            test_name="valid_markdown"
        )
    return TestResult(True, "Valid markdown structure", skill_name, "valid_markdown")

def test_directory_structure(self, skill_path: Path, skill_name: str) -> TestResult:
    """Test that skill follows directory conventions"""
    parent = skill_path.parent

    # SKILL.md should be in a directory named after the skill
    if parent.name == 'skills':
        return TestResult(
            passed=False,
            message="SKILL.md should be in a subdirectory, not directly in skills/",
            skill=skill_name,
            test_name="directory_structure"
        )

    return TestResult(True, f"Correct directory: {parent.name}/", skill_name, "directory_structure")

def validate_skill(self, skill_path: Path) -> SkillValidation:
    """Run all validation tests on a skill"""
    skill_name = self.get_skill_name(skill_path)
    content = skill_path.read_text(encoding='utf-8')
    frontmatter, body = self.parse_frontmatter(content)

    tests = []
    errors = []
    warnings = []

    # Run all tests
    tests.append(self.test_has_frontmatter(content, skill_name))
    tests.append(self.test_has_title(content, skill_name))
    tests.append(self.test_has_description(body, skill_name))
    tests.append(self.test_required_sections(body, skill_name))
    tests.append(self.test_has_code_examples(body, skill_name))
    tests.append(self.test_no_placeholders(content, skill_name))
    tests.append(self.test_minimum_content(content, skill_name))
    tests.append(self.test_valid_markdown(content, skill_name))
    tests.append(self.test_directory_structure(skill_path, skill_name))

    # Collect errors
    for test in tests:
        if not test.passed:
            errors.append(f"{test.test_name}: {test.message}")

    passed = all(t.passed for t in tests)

    return SkillValidation(
        name=skill_name,
        path=str(skill_path),
        passed=passed,
        tests=tests,
        errors=errors,
        warnings=warnings
    )

def run_all(self, specific_skill: Optional[str] = None) -> bool:
    """Run validation on all skills"""
    skills = self.get_all_skills()

    if specific_skill:
        skills = [s for s in skills if self.get_skill_name(s) == specific_skill]
        if not skills:
            print(f"{Colors.RED}Skill not found: {specific_skill}{Colors.RESET}")
            return False

    print(f"{Colors.BOLD}CODITECT Skill Validation Test Suite{Colors.RESET}")
    print("=" * 50)
    print(f"\nValidating {len(skills)} skills...\n")

    passed_count = 0
    failed_count = 0
    total_tests = 0

    for skill_path in skills:
        validation = self.validate_skill(skill_path)
        self.results.append(validation)

        total_tests += len(validation.tests)
        passed_tests = sum(1 for t in validation.tests if t.passed)

        if validation.passed:
            passed_count += 1
            if self.verbose:
                print(f"{validation.name}: {Colors.GREEN}PASS{Colors.RESET} ({passed_tests}/{len(validation.tests)} tests)")
            else:
                print(f"{validation.name}: {Colors.GREEN}PASS{Colors.RESET}")
        else:
            failed_count += 1
            print(f"{validation.name}: {Colors.RED}FAIL{Colors.RESET}")
            for error in validation.errors:
                print(f"  {Colors.RED}✗{Colors.RESET} {error}")

    # Summary
    print("\n" + "=" * 50)
    print(f"{Colors.BOLD}Test Summary{Colors.RESET}")
    print("=" * 50)
    print(f"\nSkills:  {passed_count}/{len(skills)} passed")
    print(f"Tests:   {sum(1 for r in self.results for t in r.tests if t.passed)}/{total_tests} passed")

    if failed_count == 0:
        print(f"\n{Colors.GREEN}{Colors.BOLD}All skills passed validation!{Colors.RESET}")
        return True
    else:
        print(f"\n{Colors.RED}{Colors.BOLD}{failed_count} skill(s) failed validation{Colors.RESET}")
        return False

def get_framework_root() -> Path: """Get framework root directory""" script_path = Path(file).resolve() return script_path.parent.parent.parent

def main(): parser = argparse.ArgumentParser(description='CODITECT Skill Validation Test Suite') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output') parser.add_argument('--skill', type=str, help='Test specific skill') parser.add_argument('--json', action='store_true', help='Output results as JSON') args = parser.parse_args()

framework_root = get_framework_root()
suite = SkillTestSuite(framework_root, verbose=args.verbose)

success = suite.run_all(specific_skill=args.skill)

if args.json:
    results = {
        'total_skills': len(suite.results),
        'passed': sum(1 for r in suite.results if r.passed),
        'failed': sum(1 for r in suite.results if not r.passed),
        'skills': [
            {
                'name': r.name,
                'passed': r.passed,
                'errors': r.errors,
                'warnings': r.warnings
            }
            for r in suite.results
        ]
    }
    print(json.dumps(results, indent=2))

sys.exit(0 if success else 1)

if name == 'main': main()