diff --git a/collectors/backup.py b/collectors/backup.py index 67d777d..9368ce1 100644 --- a/collectors/backup.py +++ b/collectors/backup.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 -import subprocess import json +import logging +import subprocess +from datetime import datetime, timezone RESTIC_ENV = "/etc/restic/edge01.env" +LOGGER = logging.getLogger(__name__) def collect(): @@ -16,17 +19,71 @@ def collect(): ) if result.returncode != 0: + LOGGER.error("Restic snapshots failed: %s", result.stderr.strip()) return { "state": "offline", - "summary": "Restic error", - "detail": result.stderr.strip() + "status": "error", + "summary": "Backup error", + "detail": "Restic snapshot query failed; see collector logs", } - snapshots = json.loads(result.stdout) + try: + snapshots = json.loads(result.stdout) + except json.JSONDecodeError: + LOGGER.error("Restic snapshots returned invalid JSON") + return { + "state": "offline", + "status": "error", + "summary": "Backup error", + "detail": "Restic returned invalid snapshot data; see collector logs", + } + + if not isinstance(snapshots, list): + LOGGER.error("Restic snapshots returned an unexpected JSON shape") + return { + "state": "offline", + "status": "error", + "summary": "Backup error", + "detail": "Restic returned an unexpected snapshot response; see collector logs", + } + + if not snapshots: + return { + "state": "offline", + "status": "error", + "summary": "Backup error", + "detail": "Restic repository contains no snapshots", + "snapshot_count": 0, + } + + try: + latest = max( + snapshots, + key=lambda snapshot: datetime.fromisoformat(snapshot["time"]), + ) + latest_time = datetime.fromisoformat(latest["time"]) + except (KeyError, TypeError, ValueError): + LOGGER.error("Restic snapshots contained an invalid timestamp") + return { + "state": "offline", + "status": "error", + "summary": "Backup error", + "detail": "Restic returned snapshots without valid timestamps; see collector logs", + "snapshot_count": len(snapshots), + } + + age_seconds = max(0, int((datetime.now(timezone.utc) - latest_time).total_seconds())) + age_hours, remainder = divmod(age_seconds, 3600) + age_minutes = remainder // 60 + age = f"{age_hours}h {age_minutes}m" if age_hours else f"{age_minutes}m" return { "state": "online", + "status": "healthy", "summary": "Healthy", - "detail": f"{len(snapshots)} snapshots", - "snapshot_count": len(snapshots) + "detail": f"{len(snapshots)} snapshots · latest {latest_time.isoformat()} · age {age}", + "snapshot_count": len(snapshots), + "latest_snapshot": latest_time.isoformat(), + "latest_snapshot_age": age, + "latest_snapshot_age_seconds": age_seconds, }