159 lines
6.3 KiB
Python
159 lines
6.3 KiB
Python
"""
|
|
Tests for the log analyzer against historical bot run data.
|
|
These tests verify the analyzer correctly identifies and categorizes
|
|
all failure types from actual bot runs.
|
|
"""
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
import pytest
|
|
|
|
# Add src to path
|
|
SRC = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "src")
|
|
if SRC not in sys.path:
|
|
sys.path.insert(0, SRC)
|
|
|
|
from test.auto.log_analyzer import LogAnalyzer, LogAnalysisResult
|
|
|
|
|
|
REPO_ROOT = str(Path(__file__).resolve().parent.parent.parent)
|
|
LOG_DIR = os.path.join(REPO_ROOT, "log")
|
|
STATS_DIR = os.path.join(LOG_DIR, "stats")
|
|
|
|
|
|
class TestLogAnalyzer:
|
|
"""Test the log analyzer against real historical data."""
|
|
|
|
@pytest.fixture
|
|
def analyzer(self):
|
|
return LogAnalyzer()
|
|
|
|
def test_log_file_analysis(self, analyzer):
|
|
"""Test analyzing the main log.txt file."""
|
|
log_path = os.path.join(LOG_DIR, "log.txt")
|
|
if not os.path.exists(log_path):
|
|
pytest.skip("log.txt not found")
|
|
|
|
result = analyzer.analyze_log_file(log_path)
|
|
assert isinstance(result, LogAnalysisResult)
|
|
# The log file should have been parsed without errors
|
|
assert result.log_file == log_path
|
|
|
|
def test_event_file_analysis(self, analyzer):
|
|
"""Test analyzing event JSONL files."""
|
|
if not os.path.isdir(STATS_DIR):
|
|
pytest.skip("stats directory not found")
|
|
|
|
event_files = [f for f in os.listdir(STATS_DIR) if f.startswith("events_") and f.endswith(".jsonl")]
|
|
if not event_files:
|
|
pytest.skip("no event files found")
|
|
|
|
# Test the most recent event file
|
|
event_files.sort(reverse=True)
|
|
event_path = os.path.join(STATS_DIR, event_files[0])
|
|
result = analyzer.analyze_event_file(event_path)
|
|
|
|
assert isinstance(result, LogAnalysisResult)
|
|
assert result.log_file == event_path
|
|
|
|
def test_all_failures_categorized(self, analyzer):
|
|
"""Verify all failure types are properly categorized."""
|
|
if not os.path.isdir(STATS_DIR):
|
|
pytest.skip("stats directory not found")
|
|
|
|
event_files = [f for f in os.listdir(STATS_DIR) if f.startswith("events_") and f.endswith(".jsonl")]
|
|
if not event_files:
|
|
pytest.skip("no event files found")
|
|
|
|
combined = LogAnalysisResult()
|
|
for fname in event_files[-10:]: # Last 10 event files
|
|
result = analyzer.analyze_event_file(os.path.join(STATS_DIR, fname))
|
|
combined.failures.extend(result.failures)
|
|
combined.approach_failures += result.approach_failures
|
|
combined.maintenance_failures += result.maintenance_failures
|
|
combined.battle_failures += result.battle_failures
|
|
combined.chicken_triggers += result.chicken_triggers
|
|
combined.timeouts += result.timeouts
|
|
|
|
# Verify all failures have required fields
|
|
for f in combined.failures:
|
|
assert f.failure_type in (
|
|
"approach_failed", "maintenance_failed", "battle_failed",
|
|
"chicken", "timeout", "ocr_error", "crash", "unknown_failure"
|
|
), f"Unknown failure type: {f.failure_type}"
|
|
assert f.reason, f"Empty reason for {f.failure_type}"
|
|
|
|
def test_approach_failure_detection(self, analyzer):
|
|
"""Verify approach failures are detected correctly."""
|
|
if not os.path.isdir(STATS_DIR):
|
|
pytest.skip("stats directory not found")
|
|
|
|
# Find event files with approach failures
|
|
found_approach = False
|
|
for fname in os.listdir(STATS_DIR):
|
|
if not fname.startswith("events_") or not fname.endswith(".jsonl"):
|
|
continue
|
|
result = analyzer.analyze_event_file(os.path.join(STATS_DIR, fname))
|
|
if result.approach_failures > 0:
|
|
found_approach = True
|
|
# Verify the failures have correct structure
|
|
for f in result.failures:
|
|
if f.failure_type == "approach_failed":
|
|
assert f.step, f"Approach failure missing step: {f.reason}"
|
|
assert f.run_name, f"Approach failure missing run_name: {f.reason}"
|
|
break
|
|
|
|
if not found_approach:
|
|
pytest.skip("no approach failures in available event data")
|
|
|
|
def test_maintenance_failure_detection(self, analyzer):
|
|
"""Verify maintenance failures are detected correctly."""
|
|
if not os.path.isdir(STATS_DIR):
|
|
pytest.skip("stats directory not found")
|
|
|
|
found_maintenance = False
|
|
for fname in os.listdir(STATS_DIR):
|
|
if not fname.startswith("events_") or not fname.endswith(".jsonl"):
|
|
continue
|
|
result = analyzer.analyze_event_file(os.path.join(STATS_DIR, fname))
|
|
if result.maintenance_failures > 0:
|
|
found_maintenance = True
|
|
for f in result.failures:
|
|
if f.failure_type == "maintenance_failed":
|
|
assert f.step, f"Maintenance failure missing step: {f.reason}"
|
|
break
|
|
|
|
if not found_maintenance:
|
|
pytest.skip("no maintenance failures in available event data")
|
|
|
|
def test_chicken_detection(self, analyzer):
|
|
"""Verify chicken triggers are detected."""
|
|
if not os.path.isdir(STATS_DIR):
|
|
pytest.skip("stats directory not found")
|
|
|
|
found_chicken = False
|
|
for fname in os.listdir(STATS_DIR):
|
|
if not fname.startswith("events_") or not fname.endswith(".jsonl"):
|
|
continue
|
|
result = analyzer.analyze_event_file(os.path.join(STATS_DIR, fname))
|
|
if result.chicken_triggers > 0:
|
|
found_chicken = True
|
|
break
|
|
|
|
if not found_chicken:
|
|
pytest.skip("no chicken triggers in available event data")
|
|
|
|
|
|
def test_import_log_analyzer():
|
|
"""Verify the log analyzer module imports cleanly."""
|
|
from test.auto.log_analyzer import LogAnalyzer, LogAnalysisResult, BotFailure
|
|
assert LogAnalyzer is not None
|
|
assert LogAnalysisResult is not None
|
|
assert BotFailure is not None
|
|
|
|
|
|
def test_import_auto_fixer():
|
|
"""Verify the auto fixer module imports cleanly."""
|
|
from test.auto.auto_fixer import AutoFixer, AppliedFix
|
|
assert AutoFixer is not None
|
|
assert AppliedFix is not None |