#!/usr/bin/env bash # Jarvis Auto-Healing System # Automatically restart failed services # NOTE: MariaDB runs on Synology NAS - no local restart set -euo pipefail LOG_FILE="/home/alex/clawd/logs/jarvis-auto-heal.log" ALERT_LOG="/home/alex/clawd/logs/jarvis-alerts.log" log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE" } alert() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] ALERT: $*" | tee -a "$ALERT_LOG" } # Create log directory mkdir -p "$(dirname "$LOG_FILE")" log "=== AUTO-HEAL CHECK STARTED ===" # Check and heal Gateway if ! timeout 5 curl -sfk https://192.168.1.220:18789/ >/dev/null 2>&1; then log "⚠️ Gateway down - attempting restart..." if systemctl is-active --quiet openclaw-gateway.service; then sudo systemctl restart openclaw-gateway.service sleep 5 if timeout 5 curl -sfk https://192.168.1.220:18789/ >/dev/null 2>&1; then log "✅ Gateway restarted successfully" else alert "❌ Gateway restart FAILED - manual intervention required" fi else log "⚠️ Gateway service not running - starting..." sudo systemctl start openclaw-gateway.service sleep 5 if timeout 5 curl -sfk https://192.168.1.220:18789/ >/dev/null 2>&1; then log "✅ Gateway started successfully" else alert "❌ Gateway start FAILED" fi fi else log "✓ Gateway healthy" fi # Check and heal Kanban if ! timeout 5 curl -sf http://192.168.1.220:5003/api/tasks?limit=1 >/dev/null 2>&1; then log "⚠️ Kanban down - attempting restart..." if systemctl is-active --quiet kanban-board.service; then sudo systemctl restart kanban-board.service sleep 5 if timeout 5 curl -sf http://192.168.1.220:5003/api/tasks?limit=1 >/dev/null 2>&1; then log "✅ Kanban restarted successfully" else alert "❌ Kanban restart FAILED" fi else log "⚠️ Kanban service not running - starting..." sudo systemctl start kanban-board.service sleep 5 if timeout 5 curl -sf http://192.168.1.220:5003/api/tasks?limit=1 >/dev/null 2>&1; then log "✅ Kanban started successfully" else alert "❌ Kanban start FAILED" fi fi else log "✓ Kanban healthy" fi # MariaDB health check (Synology - read-only check, no restart) if [ -f /etc/openclaw/mariadb.conf ]; then source /etc/openclaw/mariadb.conf if ! mysql -h "$MARIADB_HOST" -P "$MARIADB_PORT" -u "$KANBAN_USER" -p"$KANBAN_PASS" "$KANBAN_DB" -e "$HEALTH_CHECK_QUERY" &>/dev/null; then alert "❌ MariaDB (Synology) unreachable - check Synology NAS" else log "✓ MariaDB (Synology) healthy" fi fi # Check alert log for critical issues if [ -f "$ALERT_LOG" ]; then recent_alerts=$(tail -10 "$ALERT_LOG" 2>/dev/null | grep -c "❌" || echo "0") if [ "$recent_alerts" -gt 0 ]; then log "⚠️ $recent_alerts critical alerts in last 10 entries" fi fi log "=== AUTO-HEAL CHECK COMPLETE ==="