ca9b510922
- Reset master to upstream/main (16,697 commits) - Overlay 2,271 local-only files (skills, tools, workspace, configs, apps) - Restore IDENTITY.md and USER.md templates - Build verified, gateway running, Discord working Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
101 lines
3.1 KiB
Bash
101 lines
3.1 KiB
Bash
#!/usr/bin/env bash
|
|
# Jarvis Auto-Healing System
|
|
# Automatically restart failed services
|
|
# NOTE: MariaDB runs on Synology NAS - no local restart
|
|
|
|
set -euo pipefail
|
|
|
|
LOG_FILE="/home/alex/clawd/logs/jarvis-auto-heal.log"
|
|
ALERT_LOG="/home/alex/clawd/logs/jarvis-alerts.log"
|
|
|
|
log() {
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"
|
|
}
|
|
|
|
alert() {
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] ALERT: $*" | tee -a "$ALERT_LOG"
|
|
}
|
|
|
|
# Create log directory
|
|
mkdir -p "$(dirname "$LOG_FILE")"
|
|
|
|
log "=== AUTO-HEAL CHECK STARTED ==="
|
|
|
|
# Check and heal Gateway
|
|
if ! timeout 5 curl -sfk https://192.168.1.220:18789/ >/dev/null 2>&1; then
|
|
log "⚠️ Gateway down - attempting restart..."
|
|
|
|
if systemctl is-active --quiet openclaw-gateway.service; then
|
|
sudo systemctl restart openclaw-gateway.service
|
|
sleep 5
|
|
|
|
if timeout 5 curl -sfk https://192.168.1.220:18789/ >/dev/null 2>&1; then
|
|
log "✅ Gateway restarted successfully"
|
|
else
|
|
alert "❌ Gateway restart FAILED - manual intervention required"
|
|
fi
|
|
else
|
|
log "⚠️ Gateway service not running - starting..."
|
|
sudo systemctl start openclaw-gateway.service
|
|
sleep 5
|
|
|
|
if timeout 5 curl -sfk https://192.168.1.220:18789/ >/dev/null 2>&1; then
|
|
log "✅ Gateway started successfully"
|
|
else
|
|
alert "❌ Gateway start FAILED"
|
|
fi
|
|
fi
|
|
else
|
|
log "✓ Gateway healthy"
|
|
fi
|
|
|
|
# Check and heal Kanban
|
|
if ! timeout 5 curl -sf http://192.168.1.220:5003/api/tasks?limit=1 >/dev/null 2>&1; then
|
|
log "⚠️ Kanban down - attempting restart..."
|
|
|
|
if systemctl is-active --quiet kanban-board.service; then
|
|
sudo systemctl restart kanban-board.service
|
|
sleep 5
|
|
|
|
if timeout 5 curl -sf http://192.168.1.220:5003/api/tasks?limit=1 >/dev/null 2>&1; then
|
|
log "✅ Kanban restarted successfully"
|
|
else
|
|
alert "❌ Kanban restart FAILED"
|
|
fi
|
|
else
|
|
log "⚠️ Kanban service not running - starting..."
|
|
sudo systemctl start kanban-board.service
|
|
sleep 5
|
|
|
|
if timeout 5 curl -sf http://192.168.1.220:5003/api/tasks?limit=1 >/dev/null 2>&1; then
|
|
log "✅ Kanban started successfully"
|
|
else
|
|
alert "❌ Kanban start FAILED"
|
|
fi
|
|
fi
|
|
else
|
|
log "✓ Kanban healthy"
|
|
fi
|
|
|
|
# MariaDB health check (Synology - read-only check, no restart)
|
|
if [ -f /etc/openclaw/mariadb.conf ]; then
|
|
source /etc/openclaw/mariadb.conf
|
|
|
|
if ! mysql -h "$MARIADB_HOST" -P "$MARIADB_PORT" -u "$KANBAN_USER" -p"$KANBAN_PASS" "$KANBAN_DB" -e "$HEALTH_CHECK_QUERY" &>/dev/null; then
|
|
alert "❌ MariaDB (Synology) unreachable - check Synology NAS"
|
|
else
|
|
log "✓ MariaDB (Synology) healthy"
|
|
fi
|
|
fi
|
|
|
|
# Check alert log for critical issues
|
|
if [ -f "$ALERT_LOG" ]; then
|
|
recent_alerts=$(tail -10 "$ALERT_LOG" 2>/dev/null | grep -c "❌" || echo "0")
|
|
|
|
if [ "$recent_alerts" -gt 0 ]; then
|
|
log "⚠️ $recent_alerts critical alerts in last 10 entries"
|
|
fi
|
|
fi
|
|
|
|
log "=== AUTO-HEAL CHECK COMPLETE ==="
|