#!/bin/sh
# Maestro KDS self-heal — runs on a 30s timer, everything local and idempotent.
#
# A kitchen appliance must recover on its own: nobody is going to SSH in when a
# service crash-loops past its StartLimit, the Wi-Fi drops behind a steel bench,
# or the support VPN dies silently. This is the net under all of that.
set -u
log() { logger -t maestro-kds-watchdog "$*" 2>/dev/null; }

# 1) CRITICAL SERVICES. Each unit has Restart=always, but systemd GIVES UP after
#    StartLimitBurst failures and leaves the unit "failed" — a black board with
#    no further attempts. Clear that and start it again; this loop never gives up.
for svc in maestro-kds.service maestro-touch-pointer.service maestro-printer-bridge.service; do
    systemctl list-unit-files "$svc" >/dev/null 2>&1 || continue
    # A unit whose Condition gate said no is SKIPPED, not down — e.g. the touch
    # shim under gnome-kiosk (mutter delivers real touch; the shim would double
    # every tap). Force-starting it every tick spammed the log 2/min forever.
    [ "$(systemctl show -p ConditionResult --value "$svc" 2>/dev/null)" = "no" ] && continue
    if ! systemctl is-active --quiet "$svc"; then
        log "service $svc is down — reset-failed + start"
        systemctl reset-failed "$svc" 2>/dev/null
        systemctl start "$svc" 2>/dev/null
    fi
done

# 2) WI-FI. NetworkManager retries forever (autoconnect-retries=0), but if the
#    global state has genuinely settled on disconnected, bounce it.
if command -v nmcli >/dev/null 2>&1; then
    state=$(nmcli -t -f STATE g 2>/dev/null)
    case "$state" in
        *connected*) : ;;   # connected / connected (site only) — fine
        *) log "network state '$state' — cycling NetworkManager"
           nmcli networking off 2>/dev/null
           sleep 1
           nmcli networking on 2>/dev/null ;;
    esac
fi

# 3) SUPPORT VPN. The wg tunnel dies with no error — just no traffic — and then
#    the unit is unreachable for support until someone drives out. If wg3 is
#    configured and its newest handshake is missing or older than 200s (a healthy
#    tunnel renews inside ~2min on a 25s keepalive), bounce the tunnel.
if [ -f /etc/wireguard/wg3.conf ] && command -v wg >/dev/null 2>&1; then
    hs=$(wg show wg3 latest-handshakes 2>/dev/null | awk '{print $2}' | sort -rn | head -1)
    now=$(date +%s 2>/dev/null || echo 0)
    if [ -z "$hs" ] || [ "$hs" = "0" ] || { [ "$now" -gt 0 ] && [ $((now - hs)) -gt 200 ]; }; then
        log "wg3 stale (last handshake ${hs:-none}) — restarting tunnel"
        systemctl restart wg-quick@wg3 2>/dev/null
    fi
fi
# 3b) INTERRUPTED INITRAMFS REGEN. A rotate rewrites the plymouth theme and
#     regenerates the initramfs in the background; power-cut that window and
#     the boot intro is stuck on the OLD orientation with nothing left to fix
#     it. The ctl stamps the intent before starting; a surviving stamp with no
#     dracut running means the job needs finishing.
if [ -f /etc/maestro-kds/initramfs-pending ] && ! pgrep -x dracut >/dev/null 2>&1; then
    log "initramfs regen pending — rerunning dracut"
    dracut -f >/dev/null 2>&1 && rm -f /etc/maestro-kds/initramfs-pending
fi

# 4) FROZEN APP. Restart=always only sees a DEAD process — a Flutter app whose
#    render loop wedged (GPU hang, deadlock) stays "active" with a corpse on
#    screen. The app stamps /run/maestro-kds/heartbeat from its FRAME callback
#    (a frame that renders is proof the whole stack is alive: app, compositor,
#    GPU). Stale or missing for long enough while the unit claims active =
#    restart the session.
if systemctl is-active --quiet maestro-kds.service; then
    up_us=$(systemctl show -p ActiveEnterTimestampMonotonic --value maestro-kds.service 2>/dev/null || echo 0)
    now_us=$(awk "{printf \"%d\", \$1 * 1000000}" /proc/uptime 2>/dev/null || echo 0)
    active_s=$(( (now_us - ${up_us:-0}) / 1000000 ))
    hb=/run/maestro-kds/heartbeat
    if [ "$active_s" -gt 180 ]; then
        if [ -f "$hb" ]; then
            age=$(( $(date +%s) - $(stat -c %Y "$hb" 2>/dev/null || echo 0) ))
            if [ "$age" -gt 120 ]; then
                log "app heartbeat stale ${age}s — restarting session"
                systemctl restart maestro-kds.service 2>/dev/null
            fi
        else
            log "no app heartbeat after ${active_s}s active — restarting session"
            systemctl restart maestro-kds.service 2>/dev/null
        fi
    fi
fi

# 5) STUCK INTRO. plymouth-quit is masked so the boot splash can perform until
#    the session's root-level ExecStartPre releases it — which means a session
#    that never starts leaves the intro on screen FOREVER. If plymouthd is
#    still alive this long after boot with no active session, pull the curtain.
up_s=$(awk "{printf \"%d\", \$1}" /proc/uptime 2>/dev/null || echo 0)
if [ "$up_s" -gt 120 ] && pgrep -x plymouthd >/dev/null 2>&1 \
   && ! systemctl is-active --quiet maestro-kds.service; then
    log "plymouth still up ${up_s}s after boot with no session — quitting it"
    plymouth quit --retain-splash 2>/dev/null || true
fi

# 6) CLOCK. Order timestamps, business-day math and TLS all lean on the clock;
#    an appliance with a dead CMOS cell drifts for months unseen. Check the
#    OUTCOME (NTPSynchronized), not one daemon's name — this box may run
#    chrony, timesyncd, or whatever the base image chose.
if command -v timedatectl >/dev/null 2>&1; then
    if [ "$(timedatectl show -p NTPSynchronized --value 2>/dev/null)" != "yes" ]; then
        log "clock not NTP-synchronized — kicking time services"
        timedatectl set-ntp true 2>/dev/null
        for ntp in chrony systemd-timesyncd; do
            systemctl list-unit-files "$ntp.service" >/dev/null 2>&1 \
                && systemctl restart "$ntp.service" 2>/dev/null && break
        done
    fi
fi

# 7) DISK. Journals and cached update debs are the only things that grow. GC
#    old manual debs always (keep the 3 newest); vacuum the journal when the
#    root filesystem crosses 85% — a full disk bricks the next OTA.
for d in /home/*/.cache/maestro-kds/updates/manual; do
    [ -d "$d" ] || continue
    ls -1t "$d"/*.deb 2>/dev/null | tail -n +4 | while read -r f; do
        log "GC old update artifact: $f"
        rm -f "$f"
    done
done
use=$(df --output=pcent / 2>/dev/null | tail -1 | tr -dc 0-9)
if [ "${use:-0}" -gt 85 ]; then
    log "root filesystem at ${use}% — vacuuming journal"
    journalctl --vacuum-size=150M 2>/dev/null
fi

# 8) UPDATE HEALTH GATE. apply-update snapshots the old bundle and leaves a
#    pending marker; the proof a new build works is the same proof of life as
#    section 4 — a frame heartbeat WRITTEN AFTER THE INSTALL. Healthy for two
#    minutes: confirmed. Never healthy within seven: automatic rollback to the
#    snapshot. This is the node's tag-rollback discipline, appliance-shaped.
PENDING=/etc/maestro-kds/update-pending
if [ -f "$PENDING" ]; then
    now=$(date +%s)
    t=$(cat "$PENDING" 2>/dev/null || echo 0)
    case "$t" in ''|*[!0-9]*) t=0 ;; esac
    age=$((now - t))
    hb=/run/maestro-kds/heartbeat
    hb_m=0; hb_age=999999
    if [ -f "$hb" ]; then
        hb_m=$(stat -c %Y "$hb" 2>/dev/null || echo 0)
        hb_age=$((now - hb_m))
    fi
    if [ "$age" -gt 120 ] && [ "$hb_m" -gt "$t" ] && [ "$hb_age" -lt 90 ]; then
        log "update confirmed healthy (live heartbeat ${age}s after install)"
        rm -f "$PENDING"
    elif [ "$age" -gt 420 ]; then
        log "UPDATE FAILED THE HEALTH GATE (${age}s, heartbeat_after_install=$([ "$hb_m" -gt "$t" ] && echo yes || echo no)) — rolling back"
        /usr/local/bin/maestro-kds-ctl rollback 2>/dev/null \
            || log "rollback FAILED — unit needs attention"
    fi
fi

exit 0
