daynivo-kiosk-os/src/daynivo_kiosk/healthcheck.py

"""daynivo-kiosk-healthcheck — runs every minute (timer) as root.

Keeps every kiosk service alive without hiding problems: each recovery is
logged, and a crash loop that hit systemd's start limit is retried only
after a cool-down, so the journal shows the loop instead of an endless
stream of restarts.
"""

from __future__ import annotations

import argparse
import json
import logging
import subprocess
import time
from pathlib import Path
from typing import Any

from . import logutil, services, sysinfo
from .cdp import CDP, CDPError
from .config import load_config
from .paths import Paths, load_paths

log = logging.getLogger("daynivo-kiosk-healthcheck")

ALWAYS_ON = (services.AGENT_UNIT, services.MANAGER_UNIT)
DISPLAY_UNITS = (services.X_UNIT, services.BROWSER_UNIT)
DISPLAY_RETRY_AFTER = 300
CDP_GRACE = 90
CDP_FAILURES_BEFORE_RESTART = 3
WARN_EVERY = 3600


def _systemctl(*args: str) -> None:
    try:
        subprocess.run(["systemctl", *args], capture_output=True, timeout=30)
    except (OSError, subprocess.SubprocessError) as exc:
        log.error("systemctl %s mislukt: %s", " ".join(args), exc)


class HealthCheck:
    def __init__(self, paths: Paths, unit_status=services.unit_status, systemctl=_systemctl, cdp: Any = None,
                 throttled=sysinfo.throttled, disk=sysinfo.disk, now=time.time):
        self.paths = paths
        self.unit_status = unit_status
        self.systemctl = systemctl
        self.cdp = cdp or CDP(timeout=4)
        self.throttled = throttled
        self.disk = disk
        self.now = now

    def load_state(self) -> dict[str, Any]:
        try:
            data = json.loads(self.paths.health_state_file.read_text(encoding="utf-8"))
            return data if isinstance(data, dict) else {}
        except (OSError, ValueError):
            return {}

    def save_state(self, state: dict[str, Any]) -> None:
        try:
            self.paths.health_state_file.parent.mkdir(parents=True, exist_ok=True)
            self.paths.health_state_file.write_text(json.dumps(state), encoding="utf-8")
        except OSError as exc:
            log.warning("Kon healthcheck-status niet opslaan: %s", exc)

    def maintenance(self) -> bool:
        return (self.paths.health_run_dir / "maintenance").exists()

    def run(self, act: bool = True) -> dict[str, Any]:
        now = self.now()
        state = self.load_state()
        failed_since: dict[str, float] = state.setdefault("failed_since", {})
        warned: dict[str, float] = state.setdefault("warned", {})
        config = load_config(self.paths.config_file)
        report: dict[str, Any] = {"units": {}, "actions": [], "maintenance": self.maintenance()}
        act = act and not report["maintenance"]

        for unit in (*DISPLAY_UNITS, *ALWAYS_ON):
            props = self.unit_status(unit)
            active = props.get("ActiveState", "unknown")
            report["units"][unit] = active
            is_display = unit in DISPLAY_UNITS
            if active == "failed":
                if unit not in failed_since:
                    failed_since[unit] = now
                    log.error("%s is in fouttoestand (resultaat: %s, herstarts: %s)", unit, props.get("Result"), props.get("NRestarts"))
                if is_display and not config["browser_restart"]:
                    continue
                wait = DISPLAY_RETRY_AFTER if is_display else 0
                if act and now - failed_since[unit] >= wait:
                    log.warning("%s opnieuw starten na fouttoestand", unit)
                    self.systemctl("reset-failed", unit)
                    self.systemctl("start", "--no-block", unit)
                    report["actions"].append(f"start {unit}")
                    del failed_since[unit]
            elif active == "inactive":
                failed_since.pop(unit, None)
                should_run = not is_display or config["browser_restart"]
                if act and should_run:
                    log.warning("%s draaide niet; wordt gestart", unit)
                    self.systemctl("start", "--no-block", unit)
                    report["actions"].append(f"start {unit}")
            else:
                failed_since.pop(unit, None)

        self._check_devtools(state, report, now, act)
        self._check_hardware(warned, report, now)
        self.save_state(state)
        return report

    def _check_devtools(self, state: dict[str, Any], report: dict[str, Any], now: float, act: bool) -> None:
        if report["units"].get(services.BROWSER_UNIT) != "active":
            state.pop("browser_active_since", None)
            state["cdp_failures"] = 0
            return
        since = state.setdefault("browser_active_since", now)
        if now - since < CDP_GRACE:
            return
        try:
            self.cdp.version()
            state["cdp_failures"] = 0
            report["devtools"] = "ok"
        except CDPError:
            state["cdp_failures"] = state.get("cdp_failures", 0) + 1
            report["devtools"] = "geen antwoord"
            log.warning("Chromium reageert niet op DevTools (%s keer achter elkaar)", state["cdp_failures"])
            if act and state["cdp_failures"] >= CDP_FAILURES_BEFORE_RESTART:
                log.error("Chromium wordt herstart: DevTools reageert niet")
                self.systemctl("restart", "--no-block", services.BROWSER_UNIT)
                report["actions"].append(f"restart {services.BROWSER_UNIT}")
                state["cdp_failures"] = 0
                state.pop("browser_active_since", None)

    def _check_hardware(self, warned: dict[str, float], report: dict[str, Any], now: float) -> None:
        throttle = self.throttled()
        if throttle and throttle["now"]:
            report["throttled"] = throttle["flags"]
            if now - warned.get("throttled", 0) >= WARN_EVERY:
                log.warning("Voeding/temperatuur: %s — controleer de voedingsadapter", ", ".join(throttle["flags"]))
                warned["throttled"] = now
        usage = self.disk("/")
        if usage and usage["percent"] >= 90:
            report["disk_percent"] = usage["percent"]
            if now - warned.get("disk", 0) >= WARN_EVERY:
                log.warning("SD-kaart is voor %s%% vol", usage["percent"])
                warned["disk"] = now


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(prog="daynivo-kiosk-healthcheck")
    parser.add_argument("--check-only", action="store_true", help="alleen rapporteren, niets herstellen")
    parser.add_argument("--json", action="store_true", help="rapport als JSON")
    args = parser.parse_args(argv)
    logutil.setup()
    report = HealthCheck(load_paths()).run(act=not args.check_only)
    if args.json:
        print(json.dumps(report, indent=2, ensure_ascii=False))
    else:
        units = ", ".join(f"{Path(u).stem}={s}" for u, s in report["units"].items())
        log.info("Healthcheck: %s%s", units, f"; acties: {', '.join(report['actions'])}" if report["actions"] else "")
    return 0