#!/usr/bin/env python3 """ CODITECT Tool QA Grader

Grades tools against CODITECT tool quality criteria.

Weights: Features (20%), Usage Examples (25%), Architecture (20%), Setup (20%), Troubleshooting (15%)

Usage: python3 scripts/qa/grade-tools.py [path] [--json output.json] [--verbose]

ADR-161: Component Quality Assurance Framework """

import os import sys import re import json import argparse from pathlib import Path

sys.path.insert(0, os.path.dirname(file)) from qa_common import ( parse_frontmatter, count_words, grade_from_score, compute_weighted_score, aggregate_results, output_results )

CODITECT_CORE = Path(file).resolve().parents[2] TOOLS_DIR = CODITECT_CORE / "tools"

def grade_tool(tool_dir): """Grade a single tool directory.""" tool_name = tool_dir.name

# Find the primary documentation file
readme = tool_dir / 'README.md'
if not readme.exists():
    # Try other documentation files
    md_files = list(tool_dir.glob('*.md'))
    if md_files:
        readme = md_files[0]
    else:
        return None

with open(readme, 'r', encoding='utf-8', errors='replace') as f:
    content = f.read()

fm, body = parse_frontmatter(content)
body_lower = body.lower()
word_count = count_words(body)
scores = {}

# Inventory tool contents
has_package_json = (tool_dir / 'package.json').exists()
has_pyproject = (tool_dir / 'pyproject.toml').exists()
has_requirements = (tool_dir / 'requirements.txt').exists()
has_dockerfile = (tool_dir / 'Dockerfile').exists()
has_src = (tool_dir / 'src').is_dir()
all_files = [f.name for f in tool_dir.rglob('*') if f.is_file()]

# A. FEATURE DOCUMENTATION (20%)
scores['A1_capabilities'] = 1 if re.search(r'##\s*(features|capabilities|what\s+it\s+does)', body_lower) else 0
# MCP tools documentation (check for MCP-related content)
is_mcp = 'mcp' in tool_name or re.search(r'mcp|model\s+context\s+protocol', body_lower)
if is_mcp:
    scores['A2_mcp_tools'] = 1 if re.search(r'(tool_name|tool.*description|available\s+tools|##\s*tools)', body_lower) else 0
else:
    scores['A2_mcp_tools'] = 1  # N/A for non-MCP tools

# B. USAGE EXAMPLES (25%)
code_blocks = re.findall(r'```[\w]*\n.*?```', body, re.DOTALL)
scores['B1_usage_examples'] = 1 if len(code_blocks) >= 2 else 0
scores['B2_config_examples'] = 1 if re.search(r'(##\s*config|settings|\.env|environment)', body_lower) or re.search(r'```(json|yaml|toml)', body) else 0
scores['B3_integration_patterns'] = 1 if re.search(r'(##\s*integration|##\s*usage|api|endpoint)', body_lower) else 0

# C. ARCHITECTURE (20%)
scores['C1_system_design'] = 1 if re.search(r'(##\s*architecture|##\s*design|##\s*overview|system\s+design)', body_lower) else 0
scores['C2_data_flows'] = 1 if re.search(r'(data\s+flow|component.*interact|sequence|diagram|flowchart|→|->)', body_lower) else 0

# D. SETUP & INSTALLATION (20%)
scores['D1_installation'] = 1 if re.search(r'(##\s*install|##\s*setup|##\s*getting\s+started|npm\s+install|pip\s+install)', body_lower) else 0
scores['D2_dependencies'] = 1 if (has_package_json or has_pyproject or has_requirements or
                                   re.search(r'(##\s*dependencies|##\s*requirements|requires)', body_lower)) else 0

# E. TROUBLESHOOTING (15%)
scores['E1_common_issues'] = 1 if re.search(r'##\s*(troubleshooting|common\s+issues|known\s+issues|faq)', body_lower) else 0
scores['E2_debug_resources'] = 1 if re.search(r'(debug|support|logs?|--verbose|monitoring)', body_lower) else 0

categories = [
    ('A_features', 20, ['A1_capabilities', 'A2_mcp_tools']),
    ('B_usage_examples', 25, ['B1_usage_examples', 'B2_config_examples', 'B3_integration_patterns']),
    ('C_architecture', 20, ['C1_system_design', 'C2_data_flows']),
    ('D_setup', 20, ['D1_installation', 'D2_dependencies']),
    ('E_troubleshooting', 15, ['E1_common_issues', 'E2_debug_resources']),
]
total_base, category_scores = compute_weighted_score(scores, categories)

return {
    'name': tool_name,
    'scores': scores,
    'category_scores': category_scores,
    'total_base': total_base,
    'grade': grade_from_score(total_base),
    'word_count': word_count,
    'has_src': has_src,
    'is_mcp': bool(is_mcp),
    'track': fm.get('track', 'N/A'),
}

def main(): parser = argparse.ArgumentParser(description='Grade CODITECT tools') parser.add_argument('path', nargs='?', default=str(TOOLS_DIR), help='Tool dir or parent') parser.add_argument('--json', dest='json_output', help='Output JSON to file') parser.add_argument('--verbose', action='store_true') args = parser.parse_args()

target = Path(args.path)
# If target itself has a README.md, treat it as a single tool
if (target / 'README.md').exists() and target.parent.name == 'tools':
    tool_dirs = [target]
else:
    tool_dirs = sorted([d for d in target.iterdir() if d.is_dir() and d.name != '__pycache__'])

results = []
errors = []
for d in tool_dirs:
    try:
        r = grade_tool(d)
        if r:
            results.append(r)
    except Exception as e:
        errors.append({'file': d.name, 'error': str(e)})

data = aggregate_results(results, 'tools')
data['errors'] = errors

if args.json_output:
    output_results(data, args.json_output, 'json')
output_results(data, format='summary')

if args.verbose:
    sorted_results = sorted(results, key=lambda x: x['total_base'], reverse=True)
    print(f"\nTOP 10:")
    for r in sorted_results[:10]:
        print(f"  {r['grade']} {r['total_base']:5.1f}% | {r['name']}")
    print(f"\nBOTTOM 10:")
    for r in sorted_results[-10:]:
        print(f"  {r['grade']} {r['total_base']:5.1f}% | {r['name']}")

if name == 'main': main()