From d0a158f23217f73ec3105930b1ea3a0c375c1399 Mon Sep 17 00:00:00 2001
From: alexpolo <alexpolo2850@gmail.com>
Date: Sun, 9 Aug 2026 22:38:17 +0000
Subject: [PATCH] chore: add hermes-autoresearch harness config and scripts

---
 .gitignore                        |  1 +
 .hermes/autoresearch_config.json  | 23 +++++++++++++++++
 .hermes/autoresearch_evaluator.py | 43 +++++++++++++++++++++++++++++++
 .hermes/autoresearch_proposal.py  | 43 +++++++++++++++++++++++++++++++
 4 files changed, 110 insertions(+)
 create mode 100644 .hermes/autoresearch_config.json
 create mode 100644 .hermes/autoresearch_evaluator.py
 create mode 100644 .hermes/autoresearch_proposal.py

diff --git a/.gitignore b/.gitignore
index 7cb9fa4..07e5039 100644
--- a/.gitignore
+++ b/.gitignore
@@ -45,3 +45,4 @@ Thumbs.db
 # Work logs
 arbejdslog.txt
 hermes_driver.sh
+runs/
diff --git a/.hermes/autoresearch_config.json b/.hermes/autoresearch_config.json
new file mode 100644
index 0000000..0ba71f8
--- /dev/null
+++ b/.hermes/autoresearch_config.json
@@ -0,0 +1,23 @@
+{
+  "repo_path": "/home/alex/pony",
+  "proposal_command": [
+    "python3",
+    "/home/alex/pony/.hermes/autoresearch_proposal.py",
+    "Improve game balance: adjust one theme scene difficulty (let/normal/svaert) or stat (krop/sind/charme) in themes.py to bring win rate closer to 50%. Only change one scene per trial."
+  ],
+  "evaluator_command": [
+    "python3",
+    "/home/alex/pony/.hermes/autoresearch_evaluator.py"
+  ],
+  "max_trials": 10,
+  "max_seconds": 3600,
+  "stop_after_no_improve": 3,
+  "min_improvement": 0.0,
+  "keep_on_improve": true,
+  "revert_on_no_improve": true,
+  "revert_on_failure": true,
+  "allowlist_relative_paths": ["web/app/data/themes.py", "web/app/game/dice.py", "web/app/game/pony.py"],
+  "tsv_log_path": "runs/experiments.tsv",
+  "jsonl_log_path": "runs/experiments.jsonl",
+  "command_timeout_seconds": 300
+}
\ No newline at end of file
diff --git a/.hermes/autoresearch_evaluator.py b/.hermes/autoresearch_evaluator.py
new file mode 100644
index 0000000..3227c9a
--- /dev/null
+++ b/.hermes/autoresearch_evaluator.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+"""
+Evaluator for hermes-autoresearch: runs exhaustive evaluation and returns win rate as score.
+Lower deviation from 50% = better score.
+"""
+import os
+import sys
+import json
+import subprocess
+
+repo = os.environ.get("AR_REPO_PATH", "/home/alex/pony")
+
+# Run exhaustive evaluation and capture output
+result = subprocess.run(
+    ["python3", "evaluate_game.py", "--exhaustive"],
+    cwd=repo + "/web",
+    capture_output=True,
+    text=True,
+    timeout=120,
+)
+
+output = result.stdout
+# Parse win rate from output: "Sejr:      16 (50%)"
+import re
+match = re.search(r'Sejr:\s+\d+\s+\((\d+)%\)', output)
+if not match:
+    # Fallback: try to find win rate in other format
+    match = re.search(r'win_rate["\s:]+([\d.]+)', output)
+
+if match:
+    win_rate = float(match.group(1))
+    # Score: 100 - abs deviation from 50%. Closer to 50% = higher score.
+    score = 100 - abs(win_rate - 50)
+else:
+    score = 0.0
+    win_rate = 0
+
+print(json.dumps({
+    "score": score,
+    "accepted": True,
+    "reason": f"win_rate={win_rate}%, deviation={abs(win_rate-50):.0f}pp from 50%",
+    "metrics": {"win_rate": win_rate, "score": score},
+}))
\ No newline at end of file
diff --git a/.hermes/autoresearch_proposal.py b/.hermes/autoresearch_proposal.py
new file mode 100644
index 0000000..882f64f
--- /dev/null
+++ b/.hermes/autoresearch_proposal.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+"""
+Proposal wrapper for hermes-autoresearch: calls `hermes chat -q` with the trial contract.
+The harness sets AR_TRIAL, AR_REPO_PATH, AR_PREVIOUS_SCORE.
+"""
+import os
+import sys
+import json
+import subprocess
+import textwrap
+
+trial = os.environ.get("AR_TRIAL", "0")
+repo = os.environ.get("AR_REPO_PATH", "/home/alex/pony")
+prev_score = os.environ.get("AR_PREVIOUS_SCORE", "")
+
+# Build the prompt
+prompt = textwrap.dedent(f"""\
+You are a game balance engineer for My Little Pony TTRPG.
+
+## Trial {trial}
+Previous best score (win rate %): {prev_score or "none"}
+
+## Objective
+{sys.argv[1] if len(sys.argv) > 1 else "Improve game balance"}
+
+## Rules
+- Edit ONLY /home/alex/pony/web/app/data/themes.py (or dice.py, pony.py)
+- Change ONE scene per trial: adjust 'svaer' (let/normal/svaert) or 'stat' (krop/sind/charme)
+- Do NOT commit or push — leave changes unstaged
+- Goal: overall win rate closer to 50% (target 40-60%)
+- Exit 0 on success, exit 1 if blocked
+
+## Context
+- 8 themes, 5 scenes each, 4 pony types
+- Scenes use dice (svaer field) or non-dice (always pass with correct answer)
+- Only dice scenes affect win/loss
+- 'let' = easy (target 1-2), 'normal' = medium (target 3-4), 'svaert' = hard (target 5-6)
+""")
+
+# Run hermes chat -q with the prompt
+agent_cmd = ["hermes", "chat", "-q"]
+result = subprocess.run(agent_cmd + [prompt], capture_output=True, text=True, timeout=300)
+sys.exit(result.returncode)
\ No newline at end of file