#!/bin/bash set -u state_dir=/var/lib/control log_dir=/var/log/control capture_root=$log_dir/boot-captures event_log=$log_dir/boot-events.jsonl latest_event=$log_dir/boot-events.latest.json lock_file=/run/lock/control-boot-capture.lock boot_id=$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || true) if [[ ! "$boot_id" =~ ^[0-9a-f-]{36}$ ]]; then logger -t control-boot-capture "cannot read current boot id" exit 1 fi mkdir -p "$state_dir" "$capture_root" chmod 0750 "$state_dir" "$log_dir" "$capture_root" exec 9>"$lock_file" flock -x 9 last_boot_id=$(cat "$state_dir/last-captured-boot-id" 2>/dev/null || true) if [ "$last_boot_id" = "$boot_id" ] && [ "${CONTROL_BOOT_CAPTURE_REFRESH:-0}" != "1" ]; then exit 0 fi captured_at=$(date --iso-8601=seconds) booted_at=$(uptime -s 2>/dev/null || true) capture_dir=$capture_root/$boot_id mkdir -p "$capture_dir" chmod 0750 "$capture_dir" previous_boot_id=$(journalctl --list-boots --no-pager 2>/dev/null | awk '$1 == -1 {print $2; exit}') previous_last_json=$(journalctl -b -1 -n 1 -o json --no-pager 2>/dev/null || true) previous_last_at=$(printf '%s\n' "$previous_last_json" | python3 -c ' import datetime, json, sys try: row = json.load(sys.stdin) stamp = int(row.get("__REALTIME_TIMESTAMP", 0)) / 1_000_000 print(datetime.datetime.fromtimestamp(stamp).astimezone().isoformat(timespec="seconds") if stamp else "") except Exception: print("") ' 2>/dev/null) journalctl -b -1 -n 5000 -o short-iso-precise --no-pager \ >"$capture_dir/previous-journal-tail.log" 2>/dev/null || true journalctl -b -1 -k -p warning..alert -n 2000 -o short-iso-precise --no-pager \ >"$capture_dir/previous-kernel-warnings.log" 2>/dev/null || true shutdown_lines=$(grep -ciE 'systemd-shutdown|Reached target (System Reboot|Shutdown|Power-Off)|reboot: Restarting system|Powering off' \ "$capture_dir/previous-journal-tail.log" || true) panic_lines=$(grep -ciE 'Kernel panic|not syncing|Oops:|BUG:|soft lockup|hard LOCKUP|hung task' \ "$capture_dir/previous-kernel-warnings.log" || true) oom_lines=$(grep -ciE 'Out of memory|oom-kill|Killed process .* total-vm' \ "$capture_dir/previous-journal-tail.log" || true) storage_error_lines=$(grep -ciE 'I/O error|EXT4-fs error|Buffer I/O error|blk_update_request|uas.*reset|sd .*timing out' \ "$capture_dir/previous-kernel-warnings.log" || true) pstore_source=/var/lib/systemd/pstore pstore_count=0 pstore_panic_lines=0 declare -a pstore_files=() if [ -d "$pstore_source" ]; then while IFS= read -r pstore_name; do pstore_name=$(basename "$pstore_name") [ -f "$pstore_source/$pstore_name" ] && pstore_files+=("$pstore_source/$pstore_name") done < <(journalctl -b 0 -u systemd-pstore.service -o cat --no-pager 2>/dev/null \ | sed -n 's/^PStore \([^ ]*\) moved to .*$/\1/p') if [ "${#pstore_files[@]}" -eq 0 ] && [ -n "$booted_at" ]; then pstore_cutoff=$(date -d "$booted_at 10 minutes ago" --iso-8601=seconds 2>/dev/null || true) if [ -n "$pstore_cutoff" ]; then while IFS= read -r -d '' pstore_file; do pstore_files+=("$pstore_file") done < <(find "$pstore_source" -maxdepth 1 -type f -newermt "$pstore_cutoff" -print0 2>/dev/null) fi fi for pstore_file in "${pstore_files[@]}"; do pstore_count=$((pstore_count + 1)) grep -ciE 'Kernel panic|not syncing|Oops:|BUG:' "$pstore_file" \ >"$capture_dir/.pstore-panic-count" 2>/dev/null || true file_panic_lines=$(cat "$capture_dir/.pstore-panic-count" 2>/dev/null || printf '0') pstore_panic_lines=$((pstore_panic_lines + file_panic_lines)) cp -a "$pstore_file" "$capture_dir/" 2>/dev/null || true done fi rm -f "$capture_dir/.pstore-panic-count" runtime_watchdog=$(systemctl show --property=RuntimeWatchdogUSec --value 2>/dev/null || true) watchdog_bootstatus=$(cat /sys/class/watchdog/watchdog0/bootstatus 2>/dev/null || true) reset_status=$(vcgencmd get_rsts 2>/dev/null | sed 's/^get_rsts=//' || true) unsafe_shutdowns=$(smartctl -a -d sntasmedia /dev/sda 2>/dev/null \ | awk -F: '/Unsafe Shutdowns/ {gsub(/[ ,]/, "", $2); print $2; exit}') classification=unclean_reset if [ "$shutdown_lines" -gt 0 ]; then classification=clean_shutdown elif [ "$pstore_panic_lines" -gt 0 ] || [ "$panic_lines" -gt 0 ]; then classification=kernel_panic elif [ "$pstore_count" -gt 0 ] && [ -n "$runtime_watchdog" ] && [ "$runtime_watchdog" != "0" ]; then classification=watchdog_likely fi gap_seconds='' if [ -n "$booted_at" ] && [ -n "$previous_last_at" ]; then boot_epoch=$(date -d "$booted_at" +%s 2>/dev/null || true) previous_epoch=$(date -d "$previous_last_at" +%s 2>/dev/null || true) if [ -n "$boot_epoch" ] && [ -n "$previous_epoch" ]; then gap_seconds=$((boot_epoch - previous_epoch)) fi fi last_sensor_at='' last_sensor_summary='' if command -v mysql >/dev/null 2>&1 && [ -n "$booted_at" ]; then mysql --batch --skip-column-names fanpanel -e " SELECT recorded_at,cpu_temp_c,fan_rpm,cpu_load_1,cpu_load_5,cpu_load_15, mem_total_mb,mem_used_mb,mem_free_mb,disk_used_kb,uptime_seconds FROM sensor_logs WHERE recorded_at < '$booted_at' ORDER BY id DESC LIMIT 120 " >"$capture_dir/previous-sensors.tsv" 2>/dev/null || true last_sensor_summary=$(head -n 1 "$capture_dir/previous-sensors.tsv" 2>/dev/null || true) last_sensor_at=$(printf '%s' "$last_sensor_summary" | cut -f1) fi { printf 'captured_at=%s\n' "$captured_at" printf 'boot_id=%s\n' "$boot_id" printf 'booted_at=%s\n' "$booted_at" printf 'previous_boot_id=%s\n' "$previous_boot_id" printf 'previous_last_at=%s\n' "$previous_last_at" printf 'gap_seconds=%s\n' "$gap_seconds" printf 'classification=%s\n' "$classification" printf 'shutdown_lines=%s\n' "$shutdown_lines" printf 'panic_lines=%s\n' "$panic_lines" printf 'oom_lines=%s\n' "$oom_lines" printf 'storage_error_lines=%s\n' "$storage_error_lines" printf 'pstore_count=%s\n' "$pstore_count" printf 'pstore_panic_lines=%s\n' "$pstore_panic_lines" printf 'runtime_watchdog=%s\n' "$runtime_watchdog" printf 'watchdog_bootstatus=%s\n' "$watchdog_bootstatus" printf 'reset_status=%s\n' "$reset_status" printf 'unsafe_shutdowns=%s\n' "$unsafe_shutdowns" printf 'last_sensor_at=%s\n' "$last_sensor_at" printf 'last_sensor=%s\n' "$last_sensor_summary" } >"$capture_dir/summary.env" CAPTURED_AT="$captured_at" \ BOOT_ID="$boot_id" \ BOOTED_AT="$booted_at" \ PREVIOUS_BOOT_ID="$previous_boot_id" \ PREVIOUS_LAST_AT="$previous_last_at" \ GAP_SECONDS="$gap_seconds" \ CLASSIFICATION="$classification" \ SHUTDOWN_LINES="$shutdown_lines" \ PANIC_LINES="$panic_lines" \ OOM_LINES="$oom_lines" \ STORAGE_ERROR_LINES="$storage_error_lines" \ PSTORE_COUNT="$pstore_count" \ PSTORE_PANIC_LINES="$pstore_panic_lines" \ RUNTIME_WATCHDOG="$runtime_watchdog" \ WATCHDOG_BOOTSTATUS="$watchdog_bootstatus" \ RESET_STATUS="$reset_status" \ UNSAFE_SHUTDOWNS="$unsafe_shutdowns" \ LAST_SENSOR_AT="$last_sensor_at" \ CAPTURE_DIR="$capture_dir" \ EVENT_LOG="$event_log" \ LATEST_EVENT="$latest_event" \ python3 - <<'PY' import json import os from pathlib import Path def text(name): return os.environ.get(name, "") def integer(name): value = text(name) try: return int(value) except (TypeError, ValueError): return None row = { "captured_at": text("CAPTURED_AT"), "boot_id": text("BOOT_ID"), "booted_at": text("BOOTED_AT"), "previous_boot_id": text("PREVIOUS_BOOT_ID"), "previous_last_at": text("PREVIOUS_LAST_AT"), "gap_seconds": integer("GAP_SECONDS"), "classification": text("CLASSIFICATION"), "shutdown_lines": integer("SHUTDOWN_LINES"), "panic_lines": integer("PANIC_LINES"), "oom_lines": integer("OOM_LINES"), "storage_error_lines": integer("STORAGE_ERROR_LINES"), "pstore_count": integer("PSTORE_COUNT"), "pstore_panic_lines": integer("PSTORE_PANIC_LINES"), "runtime_watchdog": text("RUNTIME_WATCHDOG"), "watchdog_bootstatus": text("WATCHDOG_BOOTSTATUS"), "reset_status": text("RESET_STATUS"), "unsafe_shutdowns": integer("UNSAFE_SHUTDOWNS"), "last_sensor_at": text("LAST_SENSOR_AT"), "capture_dir": text("CAPTURE_DIR"), } encoded = json.dumps(row, ensure_ascii=False, separators=(",", ":")) event_log = Path(text("EVENT_LOG")) latest_event = Path(text("LATEST_EVENT")) existing = event_log.read_text(encoding="utf-8").splitlines() if event_log.exists() else [] updated = [] replaced = False for line in existing: try: old = json.loads(line) except (TypeError, ValueError): updated.append(line) continue if old.get("boot_id") == row["boot_id"]: if not replaced: updated.append(encoded) replaced = True continue updated.append(line) if not replaced: updated.append(encoded) event_log.write_text("\n".join(updated) + "\n", encoding="utf-8") temporary = latest_event.with_suffix(".tmp") temporary.write_text(encoded + "\n", encoding="utf-8") temporary.replace(latest_event) PY chmod 0640 "$event_log" "$latest_event" "$capture_dir"/* 2>/dev/null || true printf '%s\n' "$boot_id" >"$state_dir/last-captured-boot-id" chmod 0640 "$state_dir/last-captured-boot-id" logger -t control-boot-capture "recorded $classification for boot $boot_id"