Skip to main content

#!/usr/bin/env python3 """ H.1.8: Component Activation Test Suite

Comprehensive testing of component activation status, file integrity, metadata completeness, and registry consistency.

Target: 290+ critical components (agents, commands, prompts, critical skills)

Tests:

  1. File existence and readability
  2. Frontmatter parsing and validation
  3. Required metadata fields present
  4. Activation status consistency
  5. Component indexer registration
  6. Metadata registry integration

Usage: python3 scripts/core/activation_test_suite.py # Run all tests python3 scripts/core/activation_test_suite.py --type agent # Test agents only python3 scripts/core/activation_test_suite.py --report # Generate HTML report python3 scripts/core/activation_test_suite.py --fix # Auto-fix issues """

import argparse import json import re import sqlite3 import sys from dataclasses import dataclass, field from datetime import datetime, timezone from enum import Enum from pathlib import Path from typing import Any, Dict, List, Optional, Set, Tuple

import yaml

=============================================================================

Configuration

=============================================================================

SCRIPT_DIR = Path(file).parent ROOT_DIR = SCRIPT_DIR.parent.parent

ADR-114 & ADR-118: Use centralized path discovery

Component data is in platform.db (Tier 1)

USER_DATA_DIR = Path.home() / "PROJECTS" / ".coditect-data" CONTEXT_STORAGE = USER_DATA_DIR / "context-storage" if USER_DATA_DIR.exists() else ROOT_DIR / "context-storage"

ACTIVATION_STATUS_FILE = ROOT_DIR / "config" / "component-activation-status.json" FRAMEWORK_REGISTRY_FILE = ROOT_DIR / "config" / "framework-registry.json" DB_PATH = CONTEXT_STORAGE / "platform.db" # ADR-118: Components are in platform.db (Tier 1)

Component directories

COMPONENT_DIRS = { "agent": ROOT_DIR / "agents", "command": ROOT_DIR / "commands", "skill": ROOT_DIR / "skills", "script": ROOT_DIR / "scripts", "hook": ROOT_DIR / "hooks", "prompt": ROOT_DIR / "prompts", }

Required frontmatter fields by component type

Note: description OR summary is acceptable (both are used in codebase)

REQUIRED_FIELDS = { "agent": ["title"], # description checked separately (accepts summary) "command": ["title"], "skill": ["title"], "script": ["title"], "hook": ["title"], "prompt": ["title"], }

Fields where either name is acceptable

ALTERNATIVE_FIELDS = { "description": ["description", "summary"], # Accept either }

Recommended fields (warnings if missing)

RECOMMENDED_FIELDS = { "agent": ["keywords", "status", "version"], "command": ["keywords", "status", "usage"], "skill": ["keywords", "status"], "script": ["keywords", "usage", "python_version"], "hook": ["description"], "prompt": ["description"], }

Critical component types (must be 100% activated)

CRITICAL_TYPES = ["agent", "command", "prompt"]

=============================================================================

Enums and Data Classes

=============================================================================

class TestStatus(Enum): """Test result status.""" PASS = "pass" FAIL = "fail" WARN = "warn" SKIP = "skip"

@dataclass class TestResult: """Result of a single test.""" test_name: str status: TestStatus message: str component_id: str = "" details: Dict[str, Any] = field(default_factory=dict)

@dataclass class ComponentTestResult: """Complete test results for a component.""" component_id: str component_type: str name: str path: str activated: bool tests: List[TestResult] = field(default_factory=list)

@property
def passed(self) -> int:
    return sum(1 for t in self.tests if t.status == TestStatus.PASS)

@property
def failed(self) -> int:
    return sum(1 for t in self.tests if t.status == TestStatus.FAIL)

@property
def warnings(self) -> int:
    return sum(1 for t in self.tests if t.status == TestStatus.WARN)

@property
def overall_status(self) -> TestStatus:
    if any(t.status == TestStatus.FAIL for t in self.tests):
        return TestStatus.FAIL
    if any(t.status == TestStatus.WARN for t in self.tests):
        return TestStatus.WARN
    return TestStatus.PASS

@dataclass class TestSuiteResult: """Complete test suite results.""" started_at: str completed_at: str = "" total_components: int = 0 components_tested: int = 0 passed: int = 0 failed: int = 0 warnings: int = 0 skipped: int = 0 results: List[ComponentTestResult] = field(default_factory=list)

def to_dict(self) -> Dict[str, Any]:
    return {
        "started_at": self.started_at,
        "completed_at": self.completed_at,
        "summary": {
            "total_components": self.total_components,
            "components_tested": self.components_tested,
            "passed": self.passed,
            "failed": self.failed,
            "warnings": self.warnings,
            "skipped": self.skipped,
        },
        "results": [
            {
                "component_id": r.component_id,
                "component_type": r.component_type,
                "name": r.name,
                "path": r.path,
                "activated": r.activated,
                "status": r.overall_status.value,
                "passed": r.passed,
                "failed": r.failed,
                "warnings": r.warnings,
                "tests": [
                    {
                        "test": t.test_name,
                        "status": t.status.value,
                        "message": t.message,
                    }
                    for t in r.tests
                ],
            }
            for r in self.results
        ],
    }

=============================================================================

Test Functions

=============================================================================

def parse_frontmatter(content: str) -> Tuple[Dict[str, Any], str]: """Parse YAML frontmatter from markdown content.""" if not content.startswith("---"): return {}, content

match = re.search(r"^---\n(.*?)\n---\n?", content, re.DOTALL)
if not match:
    return {}, content

try:
    frontmatter = yaml.safe_load(match.group(1))
    body = content[match.end():]
    return frontmatter or {}, body
except yaml.YAMLError:
    return {}, content

def test_file_exists(path: Path, component_id: str) -> TestResult: """Test that component file exists.""" if path.exists(): return TestResult( test_name="file_exists", status=TestStatus.PASS, message=f"File exists: {path.name}", component_id=component_id, ) return TestResult( test_name="file_exists", status=TestStatus.FAIL, message=f"File not found: {path}", component_id=component_id, )

def test_file_readable(path: Path, component_id: str) -> Tuple[TestResult, Optional[str]]: """Test that component file is readable.""" try: content = path.read_text(encoding="utf-8") return TestResult( test_name="file_readable", status=TestStatus.PASS, message=f"File readable ({len(content)} chars)", component_id=component_id, ), content except Exception as e: return TestResult( test_name="file_readable", status=TestStatus.FAIL, message=f"Cannot read file: {e}", component_id=component_id, ), None

def test_frontmatter_valid(content: str, component_id: str) -> Tuple[TestResult, Dict[str, Any]]: """Test that frontmatter is valid YAML.""" try: frontmatter, _ = parse_frontmatter(content) if frontmatter: return TestResult( test_name="frontmatter_valid", status=TestStatus.PASS, message=f"Valid frontmatter ({len(frontmatter)} fields)", component_id=component_id, ), frontmatter return TestResult( test_name="frontmatter_valid", status=TestStatus.WARN, message="No frontmatter found", component_id=component_id, ), {} except Exception as e: return TestResult( test_name="frontmatter_valid", status=TestStatus.FAIL, message=f"Invalid frontmatter: {e}", component_id=component_id, ), {}

def test_required_fields( frontmatter: Dict[str, Any], component_type: str, component_id: str ) -> TestResult: """Test that required frontmatter fields are present.""" required = REQUIRED_FIELDS.get(component_type, ["title"]) missing = [f for f in required if f not in frontmatter or not frontmatter[f]]

if not missing:
    return TestResult(
        test_name="required_fields",
        status=TestStatus.PASS,
        message=f"All required fields present: {', '.join(required)}",
        component_id=component_id,
    )
return TestResult(
    test_name="required_fields",
    status=TestStatus.FAIL,
    message=f"Missing required fields: {', '.join(missing)}",
    component_id=component_id,
    details={"missing": missing},
)

def test_recommended_fields( frontmatter: Dict[str, Any], component_type: str, component_id: str ) -> TestResult: """Test that recommended frontmatter fields are present.""" recommended = RECOMMENDED_FIELDS.get(component_type, []) missing = [f for f in recommended if f not in frontmatter]

if not missing:
    return TestResult(
        test_name="recommended_fields",
        status=TestStatus.PASS,
        message=f"All recommended fields present",
        component_id=component_id,
    )
return TestResult(
    test_name="recommended_fields",
    status=TestStatus.WARN,
    message=f"Missing recommended fields: {', '.join(missing)}",
    component_id=component_id,
    details={"missing": missing},
)

def test_has_description( frontmatter: Dict[str, Any], content: str, component_id: str ) -> TestResult: """Test that component has a description (accepts description, summary, or content).""" # Check frontmatter fields desc = frontmatter.get("description", "") or frontmatter.get("summary", "")

if desc and len(desc) >= 10:
    return TestResult(
        test_name="has_description",
        status=TestStatus.PASS,
        message=f"Has description ({len(desc)} chars)",
        component_id=component_id,
    )

# Try to find description in content (first non-header paragraph)
_, body = parse_frontmatter(content)
if body:
    # Look for first substantive paragraph
    first_para = re.search(r"^(?!#|\s*$)(.{20,}?)(?:\n\n|$)", body.strip(), re.MULTILINE)
    if first_para:
        return TestResult(
            test_name="has_description",
            status=TestStatus.PASS,
            message="Description found in content body",
            component_id=component_id,
        )

if desc:
    return TestResult(
        test_name="has_description",
        status=TestStatus.WARN,
        message=f"Description too short ({len(desc)} chars)",
        component_id=component_id,
    )

return TestResult(
    test_name="has_description",
    status=TestStatus.WARN,
    message="No description found in frontmatter or content",
    component_id=component_id,
)

def test_in_database(component_id: str, db_path: Path) -> TestResult: """Test that component is registered in the database.""" if not db_path.exists(): return TestResult( test_name="database_registered", status=TestStatus.SKIP, message="Database not found", component_id=component_id, )

try:
    conn = sqlite3.connect(db_path)
    cursor = conn.execute(
        "SELECT id FROM components WHERE id = ?",
        [component_id]
    )
    row = cursor.fetchone()
    conn.close()

    if row:
        return TestResult(
            test_name="database_registered",
            status=TestStatus.PASS,
            message="Registered in component database",
            component_id=component_id,
        )
    return TestResult(
        test_name="database_registered",
        status=TestStatus.WARN,
        message="Not found in component database",
        component_id=component_id,
    )
except sqlite3.Error as e:
    return TestResult(
        test_name="database_registered",
        status=TestStatus.SKIP,
        message=f"Database error: {e}",
        component_id=component_id,
    )

def test_activation_consistency( component_id: str, activated: bool, file_exists: bool, component_type: str ) -> TestResult: """Test that activation status is consistent.""" if not file_exists and activated: return TestResult( test_name="activation_consistency", status=TestStatus.FAIL, message="Activated but file not found", component_id=component_id, )

if file_exists and not activated and component_type in CRITICAL_TYPES:
    return TestResult(
        test_name="activation_consistency",
        status=TestStatus.WARN,
        message=f"Critical {component_type} not activated",
        component_id=component_id,
    )

return TestResult(
    test_name="activation_consistency",
    status=TestStatus.PASS,
    message=f"Activation status consistent (activated={activated})",
    component_id=component_id,
)

=============================================================================

Test Suite Runner

=============================================================================

class ActivationTestSuite: """Component activation test suite runner."""

def __init__(
    self,
    component_types: Optional[List[str]] = None,
    activated_only: bool = False,
    verbose: bool = False,
):
    self.component_types = component_types or list(CRITICAL_TYPES)
    self.activated_only = activated_only
    self.verbose = verbose
    self.activation_data: Dict[str, Any] = {}
    self.results = TestSuiteResult(
        started_at=datetime.now(timezone.utc).isoformat()
    )

def load_activation_status(self) -> bool:
    """Load activation status file."""
    if not ACTIVATION_STATUS_FILE.exists():
        print(f"Error: Activation status file not found: {ACTIVATION_STATUS_FILE}")
        return False

    try:
        with open(ACTIVATION_STATUS_FILE, 'r') as f:
            self.activation_data = json.load(f)
        return True
    except (json.JSONDecodeError, IOError) as e:
        print(f"Error loading activation status: {e}")
        return False

def get_target_components(self) -> List[Dict[str, Any]]:
    """Get list of components to test."""
    components = self.activation_data.get("components", [])

    # Filter by type
    if self.component_types:
        components = [c for c in components if c.get("type") in self.component_types]

    # Filter by activation status
    if self.activated_only:
        components = [c for c in components if c.get("activated", False)]

    return components

def test_component(self, comp_data: Dict[str, Any]) -> ComponentTestResult:
    """Run all tests on a single component."""
    comp_type = comp_data.get("type", "unknown")
    comp_name = comp_data.get("name", "unknown")
    comp_id = f"{comp_type}/{comp_name}"
    comp_path = comp_data.get("path", "")
    activated = comp_data.get("activated", False)

    # Resolve path
    if comp_path:
        full_path = ROOT_DIR / comp_path
    elif comp_type == "skill":
        full_path = COMPONENT_DIRS.get(comp_type, ROOT_DIR) / comp_name / "SKILL.md"
    else:
        full_path = COMPONENT_DIRS.get(comp_type, ROOT_DIR) / f"{comp_name}.md"

    result = ComponentTestResult(
        component_id=comp_id,
        component_type=comp_type,
        name=comp_name,
        path=str(full_path),
        activated=activated,
    )

    # Test 1: File exists
    file_test = test_file_exists(full_path, comp_id)
    result.tests.append(file_test)
    file_exists = file_test.status == TestStatus.PASS

    content = None
    frontmatter = {}

    if file_exists:
        # Test 2: File readable
        read_test, content = test_file_readable(full_path, comp_id)
        result.tests.append(read_test)

        if content:
            # Test 3: Frontmatter valid
            fm_test, frontmatter = test_frontmatter_valid(content, comp_id)
            result.tests.append(fm_test)

            # Test 4: Required fields
            result.tests.append(test_required_fields(frontmatter, comp_type, comp_id))

            # Test 5: Recommended fields
            result.tests.append(test_recommended_fields(frontmatter, comp_type, comp_id))

            # Test 6: Has description
            result.tests.append(test_has_description(frontmatter, content, comp_id))

    # Test 7: Database registration
    result.tests.append(test_in_database(comp_id, DB_PATH))

    # Test 8: Activation consistency
    result.tests.append(test_activation_consistency(
        comp_id, activated, file_exists, comp_type
    ))

    return result

def run(self) -> TestSuiteResult:
    """Run the complete test suite."""
    if not self.load_activation_status():
        return self.results

    components = self.get_target_components()
    self.results.total_components = len(components)

    print(f"\nRunning activation tests on {len(components)} components...")
    print("=" * 60)

    for i, comp in enumerate(components, 1):
        comp_type = comp.get("type", "unknown")
        comp_name = comp.get("name", "unknown")

        if self.verbose:
            print(f"[{i}/{len(components)}] Testing {comp_type}/{comp_name}...", end=" ")

        result = self.test_component(comp)
        self.results.results.append(result)
        self.results.components_tested += 1

        if result.overall_status == TestStatus.PASS:
            self.results.passed += 1
            if self.verbose:
                print("✓ PASS")
        elif result.overall_status == TestStatus.FAIL:
            self.results.failed += 1
            if self.verbose:
                print("✗ FAIL")
            else:
                # Always show failures
                print(f"✗ FAIL: {comp_type}/{comp_name}")
                for t in result.tests:
                    if t.status == TestStatus.FAIL:
                        print(f"    - {t.test_name}: {t.message}")
        elif result.overall_status == TestStatus.WARN:
            self.results.warnings += 1
            if self.verbose:
                print("⚠ WARN")

    self.results.completed_at = datetime.now(timezone.utc).isoformat()
    return self.results

def print_summary(self) -> None:
    """Print test summary."""
    r = self.results

    print()
    print("=" * 60)
    print("ACTIVATION TEST SUMMARY")
    print("=" * 60)
    print()
    print(f"Components Tested: {r.components_tested}/{r.total_components}")
    print(f"  ✓ Passed:   {r.passed}")
    print(f"  ✗ Failed:   {r.failed}")
    print(f"  ⚠ Warnings: {r.warnings}")
    print()

    # Pass rate
    if r.components_tested > 0:
        pass_rate = (r.passed / r.components_tested) * 100
        print(f"Pass Rate: {pass_rate:.1f}%")

    # Show failed components
    failed = [res for res in r.results if res.overall_status == TestStatus.FAIL]
    if failed:
        print()
        print(f"Failed Components ({len(failed)}):")
        for res in failed[:20]:  # Limit output
            print(f"  - {res.component_id}")
            for t in res.tests:
                if t.status == TestStatus.FAIL:
                    print(f"      {t.test_name}: {t.message}")
        if len(failed) > 20:
            print(f"  ... and {len(failed) - 20} more")

    # Show warning counts by type
    warn_by_test = {}
    for res in r.results:
        for t in res.tests:
            if t.status == TestStatus.WARN:
                warn_by_test[t.test_name] = warn_by_test.get(t.test_name, 0) + 1

    if warn_by_test:
        print()
        print("Warnings by Test:")
        for test_name, count in sorted(warn_by_test.items(), key=lambda x: -x[1]):
            print(f"  - {test_name}: {count}")

def save_report(self, output_path: Path) -> None:
    """Save detailed JSON report."""
    with open(output_path, 'w') as f:
        json.dump(self.results.to_dict(), f, indent=2)
    print(f"\nReport saved to: {output_path}")

def generate_html_report(self, output_path: Path) -> None:
    """Generate HTML report."""
    r = self.results
    pass_rate = (r.passed / r.components_tested * 100) if r.components_tested else 0

    html = f"""<!DOCTYPE html>
Component Activation Test Report

Component Activation Test Report

Generated: {r.completed_at}

<div class="summary">
    <h2>Summary</h2>
    <p><strong>Components Tested:</strong> {r.components_tested}/{r.total_components}</p>
    <p><span class="pass">✓ Passed: {r.passed}</span></p>
    <p><span class="fail">✗ Failed: {r.failed}</span></p>
    <p><span class="warn">⚠ Warnings: {r.warnings}</span></p>
    <p><strong>Pass Rate:</strong> {pass_rate:.1f}%</p>
</div>

<h2>Results by Component</h2>
<table>
    <tr>
        <th>Component</th>
        <th>Type</th>
        <th>Status</th>
        <th>Passed</th>
        <th>Failed</th>
        <th>Warnings</th>
    </tr>

"""

    for res in r.results:
        status_class = f"status-{res.overall_status.value}"
        html += f"""        <tr class="{status_class}">
        <td>{res.component_id}</td>
        <td>{res.component_type}</td>
        <td>{res.overall_status.value.upper()}</td>
        <td>{res.passed}</td>
        <td>{res.failed}</td>
        <td>{res.warnings}</td>
    </tr>

"""

    html += """    </table>
""" 
    with open(output_path, 'w') as f:
        f.write(html)
    print(f"HTML report saved to: {output_path}")

=============================================================================

CLI Interface

=============================================================================

def main(): parser = argparse.ArgumentParser( description="H.1.8: Component Activation Test Suite", formatter_class=argparse.RawDescriptionHelpFormatter )

parser.add_argument(
    "--type", "-t",
    choices=["agent", "command", "skill", "script", "hook", "prompt", "all"],
    default="all",
    help="Component type to test (default: all critical types)"
)

parser.add_argument(
    "--activated-only",
    action="store_true",
    help="Only test activated components"
)

parser.add_argument(
    "--verbose", "-v",
    action="store_true",
    help="Verbose output"
)

parser.add_argument(
    "--report",
    metavar="PATH",
    help="Save JSON report to file"
)

parser.add_argument(
    "--html",
    metavar="PATH",
    help="Generate HTML report"
)

parser.add_argument(
    "--json",
    action="store_true",
    help="Output summary as JSON"
)

args = parser.parse_args()

# Determine component types
if args.type == "all":
    component_types = None  # Will default to critical types
else:
    component_types = [args.type]

# Run tests
suite = ActivationTestSuite(
    component_types=component_types,
    activated_only=args.activated_only,
    verbose=args.verbose,
)

results = suite.run()

# Output
if args.json:
    print(json.dumps(results.to_dict(), indent=2))
else:
    suite.print_summary()

# Save reports
if args.report:
    suite.save_report(Path(args.report))

if args.html:
    suite.generate_html_report(Path(args.html))

# Exit with appropriate code
if results.failed > 0:
    sys.exit(1)
sys.exit(0)

if name == "main": main()