scripts-test-circular-detector

#!/usr/bin/env python3 """

title: "Load circular_fix_detector module dynamically" component_type: script version: "1.0.0" audience: contributor status: stable summary: "Test script for Circular Fix Detection System" keywords: ['api', 'circular', 'database', 'detector', 'test'] tokens: ~500 created: 2025-12-22 updated: 2025-12-22 script_name: "test-circular-detector.py" language: python executable: true usage: "python3 scripts/test-circular-detector.py [options]" python_version: "3.10+" dependencies: [] modifies_files: false network_access: false requires_auth: false

Test script for Circular Fix Detection System

Demonstrates:

Jaccard similarity calculation
Circular pattern detection
Alternative strategy suggestion
Visualization
Real-world scenarios """

import sys import importlib.util from pathlib import Path

Load circular_fix_detector module dynamically

script_dir = Path(file).parent detector_path = script_dir / "circular-fix-detector.py"

spec = importlib.util.spec_from_file_location("circular_fix_detector", detector_path) circular_fix_detector = importlib.util.module_from_spec(spec) spec.loader.exec_module(circular_fix_detector)

Import classes from the loaded module

CircularFixDetector = circular_fix_detector.CircularFixDetector FixOutcome = circular_fix_detector.FixOutcome AlternativeStrategy = circular_fix_detector.AlternativeStrategy

def test_jaccard_similarity(): """Test Jaccard similarity calculations.""" print("\n" + "="*70) print("TEST 1: Jaccard Similarity Calculation") print("="*70)

detector = CircularFixDetector()

test_cases = [
    (
        "Fixed the import statement in main.py",
        "Updated import statement in main.py",
        "High similarity (same core action)"
    ),
    (
        "Fixed authentication bug by updating login method",
        "Changed the database connection settings",
        "Low similarity (different actions)"
    ),
    (
        "Added error handling to process_data function",
        "Implemented exception handling in process_data",
        "High similarity (same concept, different words)"
    ),
    (
        "Refactored the entire authentication system",
        "Completely rewrote the auth module from scratch",
        "Medium similarity (similar domain, different approach)"
    ),
]

for text1, text2, description in test_cases:
    similarity = detector.calculate_jaccard_similarity(text1, text2)
    print(f"\n{description}")
    print(f"  Text 1: {text1}")
    print(f"  Text 2: {text2}")
    print(f"  Similarity: {similarity:.2%}")

    # Show tokenization
    tokens1 = detector.normalize_tokens(text1)
    tokens2 = detector.normalize_tokens(text2)
    print(f"  Tokens 1: {tokens1}")
    print(f"  Tokens 2: {tokens2}")
    print(f"  Intersection: {set(tokens1) & set(tokens2)}")

def test_circular_detection(): """Test circular pattern detection.""" print("\n" + "="*70) print("TEST 2: Circular Pattern Detection") print("="*70)

detector = CircularFixDetector()
subtask_id = "test-task-circular"

# Simulate a series of failed attempts with similar approaches
attempts = [
    ("Fixed import error by adding missing import statement", ["src/main.py"]),
    ("Updated the import to include correct module", ["src/main.py"]),
    ("Changed import statement to fix module error", ["src/main.py"]),
]

print(f"\nRecording {len(attempts)} attempts...")

for i, (approach, files) in enumerate(attempts, 1):
    print(f"\n--- Attempt {i} ---")
    attempt = detector.record_attempt(
        subtask_id,
        approach,
        files,
        FixOutcome.FAILED
    )

    if attempt.similarity_to_previous:
        print(f"Similarity to previous: {attempt.similarity_to_previous:.2%}")

# Now check if a new similar approach would be circular
new_approach = "Modified import to resolve import error"
print(f"\n--- Checking New Approach ---")
print(f"Approach: {new_approach}")

result = detector.is_circular_fix(subtask_id, new_approach)

print(f"\nCircular Detection Result:")
print(f"  Is Circular: {result.is_circular}")
print(f"  Similar Attempts: {result.similar_attempts}")
print(f"  Similarity Scores: {[f'{s:.2%}' for s in result.similarity_scores]}")
print(f"  Suggested Strategy: {result.suggested_strategy.value if result.suggested_strategy else 'None'}")
print(f"  Reasoning: {result.reasoning}")

# Show visualization
print(detector.get_visualization(subtask_id))

# Cleanup
detector.cleanup_subtask(subtask_id)

def test_alternative_strategies(): """Test alternative strategy suggestions.""" print("\n" + "="*70) print("TEST 3: Alternative Strategy Suggestions") print("="*70)

detector = CircularFixDetector()

# Scenario 1: Too many attempts -> Escalate
print("\n--- Scenario 1: Too Many Attempts ---")
subtask_id = "test-many-attempts"

for i in range(7):
    detector.record_attempt(
        subtask_id,
        f"Attempt {i+1} to fix the bug",
        ["src/bug.py"],
        FixOutcome.FAILED
    )

result = detector.is_circular_fix(subtask_id, "Another attempt to fix the bug")
print(f"Strategy: {result.suggested_strategy.value if result.suggested_strategy else 'None'}")
print(f"Reasoning: {result.reasoning}")
detector.cleanup_subtask(subtask_id)

# Scenario 2: Very high similarity -> Different method
print("\n--- Scenario 2: Very High Similarity ---")
subtask_id = "test-high-similarity"

for i in range(3):
    detector.record_attempt(
        subtask_id,
        "Fix the authentication error in login module",
        ["src/auth.py"],
        FixOutcome.FAILED
    )

result = detector.is_circular_fix(subtask_id, "Fix authentication error in login")
print(f"Strategy: {result.suggested_strategy.value if result.suggested_strategy else 'None'}")
print(f"Reasoning: {result.reasoning}")
detector.cleanup_subtask(subtask_id)

# Scenario 3: Same files -> Different files
print("\n--- Scenario 3: Same Files Repeatedly ---")
subtask_id = "test-same-files"

for i in range(4):
    detector.record_attempt(
        subtask_id,
        f"Different fix attempt {i+1}",
        ["src/utils.py"],
        FixOutcome.FAILED
    )

result = detector.is_circular_fix(subtask_id, "Yet another fix")
print(f"Strategy: {result.suggested_strategy.value if result.suggested_strategy else 'None'}")
print(f"Reasoning: {result.reasoning}")
detector.cleanup_subtask(subtask_id)

def test_real_world_scenario(): """Test a realistic bug fixing scenario.""" print("\n" + "="*70) print("TEST 4: Real-World Scenario - Import Error Loop") print("="*70)

detector = CircularFixDetector()
subtask_id = "fix-import-error-module-not-found"

print("\nScenario: Agent trying to fix 'ModuleNotFoundError: No module named requests'")
print("-" * 70)

# Attempt 1: Try to add import
print("\n[Attempt 1] Add import statement")
detector.record_attempt(
    subtask_id,
    "Added 'import requests' to the top of the file",
    ["src/api_client.py"],
    FixOutcome.FAILED,
    error_message="ModuleNotFoundError: No module named 'requests'"
)

# Attempt 2: Try different import syntax
print("\n[Attempt 2] Try different import syntax")
result = detector.is_circular_fix(
    subtask_id,
    "Changed import to 'from requests import *'"
)
print(f"  Circular: {result.is_circular}")

detector.record_attempt(
    subtask_id,
    "Changed import to 'from requests import *'",
    ["src/api_client.py"],
    FixOutcome.FAILED,
    error_message="ModuleNotFoundError: No module named 'requests'"
)

# Attempt 3: Another import variation
print("\n[Attempt 3] Try yet another import variation")
result = detector.is_circular_fix(
    subtask_id,
    "Modified import statement to use absolute import path"
)
print(f"  Circular: {result.is_circular}")
if result.is_circular:
    print(f"  🚨 CIRCULAR PATTERN DETECTED!")
    print(f"  Suggested Strategy: {result.suggested_strategy.value}")
    print(f"  Reasoning: {result.reasoning}")
    print("\n  💡 Alternative Approaches:")
    print("     1. Check if 'requests' is in requirements.txt")
    print("     2. Install requests: pip install requests")
    print("     3. Check virtual environment activation")
    print("     4. Verify Python path and package installation")

# Show full history
print(detector.get_visualization(subtask_id))

# Cleanup
detector.cleanup_subtask(subtask_id)

def test_success_cleanup(): """Test automatic cleanup on success.""" print("\n" + "="*70) print("TEST 5: Automatic Cleanup on Success") print("="*70)

detector = CircularFixDetector()
subtask_id = "test-success-cleanup"

# Record some failed attempts
print("\nRecording 2 failed attempts...")
detector.record_attempt(
    subtask_id,
    "First attempt to fix bug",
    ["src/bug.py"],
    FixOutcome.FAILED
)
detector.record_attempt(
    subtask_id,
    "Second attempt to fix bug",
    ["src/bug.py"],
    FixOutcome.FAILED
)

print(f"History before success: {len(detector.history.get(subtask_id, []))} attempts")

# Record successful attempt
print("\nRecording successful attempt...")
detector.record_attempt(
    subtask_id,
    "Final successful fix using different approach",
    ["src/bug.py", "src/utils.py"],
    FixOutcome.SUCCESS
)

print(f"History after success: {len(detector.history.get(subtask_id, []))} attempts (should be 0 due to auto-cleanup)")

def test_summary_stats(): """Test summary statistics.""" print("\n" + "="*70) print("TEST 6: Summary Statistics") print("="*70)

detector = CircularFixDetector()

# Create some test data
for i in range(3):
    subtask_id = f"test-subtask-{i+1}"

    # 2-4 attempts per subtask
    for j in range(2 + i):
        outcome = FixOutcome.FAILED if j < (1 + i) else FixOutcome.SUCCESS
        detector.record_attempt(
            subtask_id,
            f"Attempt {j+1} for subtask {i+1}",
            [f"src/file{i}.py"],
            outcome
        )

# Get summary
summary = detector.get_summary()

print("\nSummary Statistics:")
print(f"  Total Subtasks: {summary['total_subtasks']}")
print(f"  Total Attempts: {summary['total_attempts']}")
print(f"  Average Attempts/Subtask: {summary['average_attempts_per_subtask']}")
print(f"\nOutcome Distribution:")
for outcome, count in summary['outcomes'].items():
    print(f"  {outcome}: {count}")
print(f"\nSubtasks with Most Attempts:")
for subtask_id, count in summary['subtasks_with_most_attempts']:
    print(f"  {subtask_id}: {count} attempts")

def main(): """Run all tests.""" print("\n" + "="*70) print("CIRCULAR FIX DETECTION SYSTEM - COMPREHENSIVE TEST SUITE") print("="*70)

try:
    test_jaccard_similarity()
    test_circular_detection()
    test_alternative_strategies()
    test_real_world_scenario()
    test_success_cleanup()
    test_summary_stats()

    print("\n" + "="*70)
    print("✅ ALL TESTS COMPLETED SUCCESSFULLY")
    print("="*70)
    print("\nThe Circular Fix Detection System is working correctly!")
    print("\nKey Features Demonstrated:")
    print("  ✓ Jaccard similarity calculation with token normalization")
    print("  ✓ Circular pattern detection with configurable thresholds")
    print("  ✓ Alternative strategy suggestions based on failure patterns")
    print("  ✓ Visual similarity timeline")
    print("  ✓ Automatic cleanup on success")
    print("  ✓ Summary statistics and reporting")
    print("\nNext Steps:")
    print("  1. Integrate with recovery manager")
    print("  2. Add to orchestrator workflow")
    print("  3. Configure thresholds for your use case")
    print("  4. Monitor effectiveness and tune parameters")

except Exception as e:
    print(f"\n❌ TEST FAILED: {e}")
    import traceback
    traceback.print_exc()
    return 1

return 0

if name == 'main': sys.exit(main())

#!/usr/bin/env python3 """​

Load circular_fix_detector module dynamically

Import classes from the loaded module

#!/usr/bin/env python3 """