85 lines
3.0 KiB
Python
85 lines
3.0 KiB
Python
#!/usr/bin/env python
|
|
"""Update config/fg_prices.json from the latest scrape results in fg_daily_estimates.json.
|
|
|
|
Reads the most recent scrape output, keeps existing entries that weren't found
|
|
in the latest scrape, and updates/adds entries with new median prices.
|
|
"""
|
|
import json
|
|
from pathlib import Path
|
|
|
|
PROJECT_DIR = Path(__file__).parent.parent
|
|
ESTIMATES_PATH = PROJECT_DIR / "config" / "fg_daily_estimates.json"
|
|
CONFIG_PATH = PROJECT_DIR / "config" / "fg_prices.json"
|
|
|
|
|
|
def main():
|
|
if not ESTIMATES_PATH.exists():
|
|
print(f"ERROR: {ESTIMATES_PATH} not found — run the scraper first")
|
|
return
|
|
|
|
with open(ESTIMATES_PATH) as f:
|
|
data = json.load(f)
|
|
|
|
# Load existing config
|
|
if CONFIG_PATH.exists():
|
|
with open(CONFIG_PATH) as f:
|
|
config = json.load(f)
|
|
else:
|
|
config = {"enabled": True, "notes": "Auto-generated from FG scraper", "rules": []}
|
|
|
|
# Build a map of existing rules by pattern (normalized)
|
|
existing = {}
|
|
for rule in config.get("rules", []):
|
|
pattern = " ".join(rule.get("pattern", "").upper().split())
|
|
existing[pattern] = rule
|
|
|
|
# Collect latest estimates across all days, preferring the most recent day
|
|
estimates = data.get("estimates", {})
|
|
# Sort day keys to get the latest day first
|
|
day_keys = sorted(estimates.keys(), key=lambda k: int(k.replace("day_", "")), reverse=True)
|
|
|
|
# Build updated rules from latest scrape
|
|
updated_items = set()
|
|
for day_key in day_keys:
|
|
items = estimates.get(day_key, {})
|
|
for item, info in items.items():
|
|
median = info.get("median_fg", 0)
|
|
samples = info.get("samples", 0)
|
|
if median <= 0 or samples < 2:
|
|
continue
|
|
# Normalize pattern
|
|
pattern = " ".join(item.upper().split())
|
|
rule = {
|
|
"name": item,
|
|
"match": "contains" if len(item.split()) > 1 else "exact",
|
|
"pattern": pattern,
|
|
"fg": round(median),
|
|
}
|
|
if pattern in existing:
|
|
existing[pattern] = rule
|
|
else:
|
|
existing[pattern] = rule
|
|
updated_items.add(item)
|
|
|
|
# Rebuild rules list preserving order: existing first, then new
|
|
rules = list(existing.values())
|
|
|
|
# Sort by name for readability
|
|
rules.sort(key=lambda r: r.get("name", "").lower())
|
|
|
|
config["rules"] = rules
|
|
config["notes"] = f"Auto-updated from scrape at {data.get('generated_at', 'unknown')} — {len(updated_items)} items updated, {len(rules)} total rules"
|
|
|
|
CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
CONFIG_PATH.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8")
|
|
|
|
print(f"Updated {CONFIG_PATH}: {len(updated_items)} items from scrape, {len(rules)} total rules")
|
|
for item in sorted(updated_items):
|
|
rule = existing.get(" ".join(item.upper().split()))
|
|
if rule:
|
|
print(f" {item}: {rule['fg']} FG")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|