"""daynivo-kiosk-healthcheck — runs every minute (timer) as root. Keeps every kiosk service alive without hiding problems: each recovery is logged, and a crash loop that hit systemd's start limit is retried only after a cool-down, so the journal shows the loop instead of an endless stream of restarts. """ from __future__ import annotations import argparse import json import logging import subprocess import time from pathlib import Path from typing import Any from . import logutil, services, sysinfo from .cdp import CDP, CDPError from .config import load_config from .paths import Paths, load_paths log = logging.getLogger("daynivo-kiosk-healthcheck") ALWAYS_ON = (services.AGENT_UNIT, services.MANAGER_UNIT) DISPLAY_UNITS = (services.X_UNIT, services.BROWSER_UNIT) DISPLAY_RETRY_AFTER = 300 CDP_GRACE = 90 CDP_FAILURES_BEFORE_RESTART = 3 WARN_EVERY = 3600 def _systemctl(*args: str) -> None: try: subprocess.run(["systemctl", *args], capture_output=True, timeout=30) except (OSError, subprocess.SubprocessError) as exc: log.error("systemctl %s mislukt: %s", " ".join(args), exc) class HealthCheck: def __init__(self, paths: Paths, unit_status=services.unit_status, systemctl=_systemctl, cdp: Any = None, throttled=sysinfo.throttled, disk=sysinfo.disk, now=time.time): self.paths = paths self.unit_status = unit_status self.systemctl = systemctl self.cdp = cdp or CDP(timeout=4) self.throttled = throttled self.disk = disk self.now = now def load_state(self) -> dict[str, Any]: try: data = json.loads(self.paths.health_state_file.read_text(encoding="utf-8")) return data if isinstance(data, dict) else {} except (OSError, ValueError): return {} def save_state(self, state: dict[str, Any]) -> None: try: self.paths.health_state_file.parent.mkdir(parents=True, exist_ok=True) self.paths.health_state_file.write_text(json.dumps(state), encoding="utf-8") except OSError as exc: log.warning("Kon healthcheck-status niet opslaan: %s", exc) def maintenance(self) -> bool: return (self.paths.health_run_dir / "maintenance").exists() def run(self, act: bool = True) -> dict[str, Any]: now = self.now() state = self.load_state() failed_since: dict[str, float] = state.setdefault("failed_since", {}) warned: dict[str, float] = state.setdefault("warned", {}) config = load_config(self.paths.config_file) report: dict[str, Any] = {"units": {}, "actions": [], "maintenance": self.maintenance()} act = act and not report["maintenance"] for unit in (*DISPLAY_UNITS, *ALWAYS_ON): props = self.unit_status(unit) active = props.get("ActiveState", "unknown") report["units"][unit] = active is_display = unit in DISPLAY_UNITS if active == "failed": if unit not in failed_since: failed_since[unit] = now log.error("%s is in fouttoestand (resultaat: %s, herstarts: %s)", unit, props.get("Result"), props.get("NRestarts")) if is_display and not config["browser_restart"]: continue wait = DISPLAY_RETRY_AFTER if is_display else 0 if act and now - failed_since[unit] >= wait: log.warning("%s opnieuw starten na fouttoestand", unit) self.systemctl("reset-failed", unit) self.systemctl("start", "--no-block", unit) report["actions"].append(f"start {unit}") del failed_since[unit] elif active == "inactive": failed_since.pop(unit, None) should_run = not is_display or config["browser_restart"] if act and should_run: log.warning("%s draaide niet; wordt gestart", unit) self.systemctl("start", "--no-block", unit) report["actions"].append(f"start {unit}") else: failed_since.pop(unit, None) self._check_devtools(state, report, now, act) self._check_hardware(warned, report, now) self.save_state(state) return report def _check_devtools(self, state: dict[str, Any], report: dict[str, Any], now: float, act: bool) -> None: if report["units"].get(services.BROWSER_UNIT) != "active": state.pop("browser_active_since", None) state["cdp_failures"] = 0 return since = state.setdefault("browser_active_since", now) if now - since < CDP_GRACE: return try: self.cdp.version() state["cdp_failures"] = 0 report["devtools"] = "ok" except CDPError: state["cdp_failures"] = state.get("cdp_failures", 0) + 1 report["devtools"] = "geen antwoord" log.warning("Chromium reageert niet op DevTools (%s keer achter elkaar)", state["cdp_failures"]) if act and state["cdp_failures"] >= CDP_FAILURES_BEFORE_RESTART: log.error("Chromium wordt herstart: DevTools reageert niet") self.systemctl("restart", "--no-block", services.BROWSER_UNIT) report["actions"].append(f"restart {services.BROWSER_UNIT}") state["cdp_failures"] = 0 state.pop("browser_active_since", None) def _check_hardware(self, warned: dict[str, float], report: dict[str, Any], now: float) -> None: throttle = self.throttled() if throttle and throttle["now"]: report["throttled"] = throttle["flags"] if now - warned.get("throttled", 0) >= WARN_EVERY: log.warning("Voeding/temperatuur: %s — controleer de voedingsadapter", ", ".join(throttle["flags"])) warned["throttled"] = now usage = self.disk("/") if usage and usage["percent"] >= 90: report["disk_percent"] = usage["percent"] if now - warned.get("disk", 0) >= WARN_EVERY: log.warning("SD-kaart is voor %s%% vol", usage["percent"]) warned["disk"] = now def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="daynivo-kiosk-healthcheck") parser.add_argument("--check-only", action="store_true", help="alleen rapporteren, niets herstellen") parser.add_argument("--json", action="store_true", help="rapport als JSON") args = parser.parse_args(argv) logutil.setup() report = HealthCheck(load_paths()).run(act=not args.check_only) if args.json: print(json.dumps(report, indent=2, ensure_ascii=False)) else: units = ", ".join(f"{Path(u).stem}={s}" for u, s in report["units"].items()) log.info("Healthcheck: %s%s", units, f"; acties: {', '.join(report['actions'])}" if report["actions"] else "") return 0