239 lines
9.2 KiB
Bash
Executable File
239 lines
9.2 KiB
Bash
Executable File
#!/bin/bash
|
|
set -u
|
|
|
|
state_dir=/var/lib/control
|
|
log_dir=/var/log/control
|
|
capture_root=$log_dir/boot-captures
|
|
event_log=$log_dir/boot-events.jsonl
|
|
latest_event=$log_dir/boot-events.latest.json
|
|
lock_file=/run/lock/control-boot-capture.lock
|
|
boot_id=$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || true)
|
|
|
|
if [[ ! "$boot_id" =~ ^[0-9a-f-]{36}$ ]]; then
|
|
logger -t control-boot-capture "cannot read current boot id"
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p "$state_dir" "$capture_root"
|
|
chmod 0750 "$state_dir" "$log_dir" "$capture_root"
|
|
exec 9>"$lock_file"
|
|
flock -x 9
|
|
|
|
last_boot_id=$(cat "$state_dir/last-captured-boot-id" 2>/dev/null || true)
|
|
if [ "$last_boot_id" = "$boot_id" ] && [ "${CONTROL_BOOT_CAPTURE_REFRESH:-0}" != "1" ]; then
|
|
exit 0
|
|
fi
|
|
|
|
captured_at=$(date --iso-8601=seconds)
|
|
booted_at=$(uptime -s 2>/dev/null || true)
|
|
capture_dir=$capture_root/$boot_id
|
|
mkdir -p "$capture_dir"
|
|
chmod 0750 "$capture_dir"
|
|
|
|
previous_boot_id=$(journalctl --list-boots --no-pager 2>/dev/null | awk '$1 == -1 {print $2; exit}')
|
|
previous_last_json=$(journalctl -b -1 -n 1 -o json --no-pager 2>/dev/null || true)
|
|
previous_last_at=$(printf '%s\n' "$previous_last_json" | python3 -c '
|
|
import datetime, json, sys
|
|
try:
|
|
row = json.load(sys.stdin)
|
|
stamp = int(row.get("__REALTIME_TIMESTAMP", 0)) / 1_000_000
|
|
print(datetime.datetime.fromtimestamp(stamp).astimezone().isoformat(timespec="seconds") if stamp else "")
|
|
except Exception:
|
|
print("")
|
|
' 2>/dev/null)
|
|
|
|
journalctl -b -1 -n 5000 -o short-iso-precise --no-pager \
|
|
>"$capture_dir/previous-journal-tail.log" 2>/dev/null || true
|
|
journalctl -b -1 -k -p warning..alert -n 2000 -o short-iso-precise --no-pager \
|
|
>"$capture_dir/previous-kernel-warnings.log" 2>/dev/null || true
|
|
|
|
shutdown_lines=$(grep -ciE 'systemd-shutdown|Reached target (System Reboot|Shutdown|Power-Off)|reboot: Restarting system|Powering off' \
|
|
"$capture_dir/previous-journal-tail.log" || true)
|
|
panic_lines=$(grep -ciE 'Kernel panic|not syncing|Oops:|BUG:|soft lockup|hard LOCKUP|hung task' \
|
|
"$capture_dir/previous-kernel-warnings.log" || true)
|
|
oom_lines=$(grep -ciE 'Out of memory|oom-kill|Killed process .* total-vm' \
|
|
"$capture_dir/previous-journal-tail.log" || true)
|
|
storage_error_lines=$(grep -ciE 'I/O error|EXT4-fs error|Buffer I/O error|blk_update_request|uas.*reset|sd .*timing out' \
|
|
"$capture_dir/previous-kernel-warnings.log" || true)
|
|
|
|
pstore_source=/var/lib/systemd/pstore
|
|
pstore_count=0
|
|
pstore_panic_lines=0
|
|
declare -a pstore_files=()
|
|
if [ -d "$pstore_source" ]; then
|
|
while IFS= read -r pstore_name; do
|
|
pstore_name=$(basename "$pstore_name")
|
|
[ -f "$pstore_source/$pstore_name" ] && pstore_files+=("$pstore_source/$pstore_name")
|
|
done < <(journalctl -b 0 -u systemd-pstore.service -o cat --no-pager 2>/dev/null \
|
|
| sed -n 's/^PStore \([^ ]*\) moved to .*$/\1/p')
|
|
|
|
if [ "${#pstore_files[@]}" -eq 0 ] && [ -n "$booted_at" ]; then
|
|
pstore_cutoff=$(date -d "$booted_at 10 minutes ago" --iso-8601=seconds 2>/dev/null || true)
|
|
if [ -n "$pstore_cutoff" ]; then
|
|
while IFS= read -r -d '' pstore_file; do
|
|
pstore_files+=("$pstore_file")
|
|
done < <(find "$pstore_source" -maxdepth 1 -type f -newermt "$pstore_cutoff" -print0 2>/dev/null)
|
|
fi
|
|
fi
|
|
|
|
for pstore_file in "${pstore_files[@]}"; do
|
|
pstore_count=$((pstore_count + 1))
|
|
grep -ciE 'Kernel panic|not syncing|Oops:|BUG:' "$pstore_file" \
|
|
>"$capture_dir/.pstore-panic-count" 2>/dev/null || true
|
|
file_panic_lines=$(cat "$capture_dir/.pstore-panic-count" 2>/dev/null || printf '0')
|
|
pstore_panic_lines=$((pstore_panic_lines + file_panic_lines))
|
|
cp -a "$pstore_file" "$capture_dir/" 2>/dev/null || true
|
|
done
|
|
fi
|
|
rm -f "$capture_dir/.pstore-panic-count"
|
|
|
|
runtime_watchdog=$(systemctl show --property=RuntimeWatchdogUSec --value 2>/dev/null || true)
|
|
watchdog_bootstatus=$(cat /sys/class/watchdog/watchdog0/bootstatus 2>/dev/null || true)
|
|
reset_status=$(vcgencmd get_rsts 2>/dev/null | sed 's/^get_rsts=//' || true)
|
|
unsafe_shutdowns=$(smartctl -a -d sntasmedia /dev/sda 2>/dev/null \
|
|
| awk -F: '/Unsafe Shutdowns/ {gsub(/[ ,]/, "", $2); print $2; exit}')
|
|
|
|
classification=unclean_reset
|
|
if [ "$shutdown_lines" -gt 0 ]; then
|
|
classification=clean_shutdown
|
|
elif [ "$pstore_panic_lines" -gt 0 ] || [ "$panic_lines" -gt 0 ]; then
|
|
classification=kernel_panic
|
|
elif [ "$pstore_count" -gt 0 ] && [ -n "$runtime_watchdog" ] && [ "$runtime_watchdog" != "0" ]; then
|
|
classification=watchdog_likely
|
|
fi
|
|
|
|
gap_seconds=''
|
|
if [ -n "$booted_at" ] && [ -n "$previous_last_at" ]; then
|
|
boot_epoch=$(date -d "$booted_at" +%s 2>/dev/null || true)
|
|
previous_epoch=$(date -d "$previous_last_at" +%s 2>/dev/null || true)
|
|
if [ -n "$boot_epoch" ] && [ -n "$previous_epoch" ]; then
|
|
gap_seconds=$((boot_epoch - previous_epoch))
|
|
fi
|
|
fi
|
|
|
|
last_sensor_at=''
|
|
last_sensor_summary=''
|
|
if command -v mysql >/dev/null 2>&1 && [ -n "$booted_at" ]; then
|
|
mysql --batch --skip-column-names fanpanel -e "
|
|
SELECT recorded_at,cpu_temp_c,fan_rpm,cpu_load_1,cpu_load_5,cpu_load_15,
|
|
mem_total_mb,mem_used_mb,mem_free_mb,disk_used_kb,uptime_seconds
|
|
FROM sensor_logs
|
|
WHERE recorded_at < '$booted_at'
|
|
ORDER BY id DESC
|
|
LIMIT 120
|
|
" >"$capture_dir/previous-sensors.tsv" 2>/dev/null || true
|
|
last_sensor_summary=$(head -n 1 "$capture_dir/previous-sensors.tsv" 2>/dev/null || true)
|
|
last_sensor_at=$(printf '%s' "$last_sensor_summary" | cut -f1)
|
|
fi
|
|
|
|
{
|
|
printf 'captured_at=%s\n' "$captured_at"
|
|
printf 'boot_id=%s\n' "$boot_id"
|
|
printf 'booted_at=%s\n' "$booted_at"
|
|
printf 'previous_boot_id=%s\n' "$previous_boot_id"
|
|
printf 'previous_last_at=%s\n' "$previous_last_at"
|
|
printf 'gap_seconds=%s\n' "$gap_seconds"
|
|
printf 'classification=%s\n' "$classification"
|
|
printf 'shutdown_lines=%s\n' "$shutdown_lines"
|
|
printf 'panic_lines=%s\n' "$panic_lines"
|
|
printf 'oom_lines=%s\n' "$oom_lines"
|
|
printf 'storage_error_lines=%s\n' "$storage_error_lines"
|
|
printf 'pstore_count=%s\n' "$pstore_count"
|
|
printf 'pstore_panic_lines=%s\n' "$pstore_panic_lines"
|
|
printf 'runtime_watchdog=%s\n' "$runtime_watchdog"
|
|
printf 'watchdog_bootstatus=%s\n' "$watchdog_bootstatus"
|
|
printf 'reset_status=%s\n' "$reset_status"
|
|
printf 'unsafe_shutdowns=%s\n' "$unsafe_shutdowns"
|
|
printf 'last_sensor_at=%s\n' "$last_sensor_at"
|
|
printf 'last_sensor=%s\n' "$last_sensor_summary"
|
|
} >"$capture_dir/summary.env"
|
|
|
|
CAPTURED_AT="$captured_at" \
|
|
BOOT_ID="$boot_id" \
|
|
BOOTED_AT="$booted_at" \
|
|
PREVIOUS_BOOT_ID="$previous_boot_id" \
|
|
PREVIOUS_LAST_AT="$previous_last_at" \
|
|
GAP_SECONDS="$gap_seconds" \
|
|
CLASSIFICATION="$classification" \
|
|
SHUTDOWN_LINES="$shutdown_lines" \
|
|
PANIC_LINES="$panic_lines" \
|
|
OOM_LINES="$oom_lines" \
|
|
STORAGE_ERROR_LINES="$storage_error_lines" \
|
|
PSTORE_COUNT="$pstore_count" \
|
|
PSTORE_PANIC_LINES="$pstore_panic_lines" \
|
|
RUNTIME_WATCHDOG="$runtime_watchdog" \
|
|
WATCHDOG_BOOTSTATUS="$watchdog_bootstatus" \
|
|
RESET_STATUS="$reset_status" \
|
|
UNSAFE_SHUTDOWNS="$unsafe_shutdowns" \
|
|
LAST_SENSOR_AT="$last_sensor_at" \
|
|
CAPTURE_DIR="$capture_dir" \
|
|
EVENT_LOG="$event_log" \
|
|
LATEST_EVENT="$latest_event" \
|
|
python3 - <<'PY'
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
|
|
def text(name):
|
|
return os.environ.get(name, "")
|
|
|
|
def integer(name):
|
|
value = text(name)
|
|
try:
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
row = {
|
|
"captured_at": text("CAPTURED_AT"),
|
|
"boot_id": text("BOOT_ID"),
|
|
"booted_at": text("BOOTED_AT"),
|
|
"previous_boot_id": text("PREVIOUS_BOOT_ID"),
|
|
"previous_last_at": text("PREVIOUS_LAST_AT"),
|
|
"gap_seconds": integer("GAP_SECONDS"),
|
|
"classification": text("CLASSIFICATION"),
|
|
"shutdown_lines": integer("SHUTDOWN_LINES"),
|
|
"panic_lines": integer("PANIC_LINES"),
|
|
"oom_lines": integer("OOM_LINES"),
|
|
"storage_error_lines": integer("STORAGE_ERROR_LINES"),
|
|
"pstore_count": integer("PSTORE_COUNT"),
|
|
"pstore_panic_lines": integer("PSTORE_PANIC_LINES"),
|
|
"runtime_watchdog": text("RUNTIME_WATCHDOG"),
|
|
"watchdog_bootstatus": text("WATCHDOG_BOOTSTATUS"),
|
|
"reset_status": text("RESET_STATUS"),
|
|
"unsafe_shutdowns": integer("UNSAFE_SHUTDOWNS"),
|
|
"last_sensor_at": text("LAST_SENSOR_AT"),
|
|
"capture_dir": text("CAPTURE_DIR"),
|
|
}
|
|
|
|
encoded = json.dumps(row, ensure_ascii=False, separators=(",", ":"))
|
|
event_log = Path(text("EVENT_LOG"))
|
|
latest_event = Path(text("LATEST_EVENT"))
|
|
existing = event_log.read_text(encoding="utf-8").splitlines() if event_log.exists() else []
|
|
updated = []
|
|
replaced = False
|
|
for line in existing:
|
|
try:
|
|
old = json.loads(line)
|
|
except (TypeError, ValueError):
|
|
updated.append(line)
|
|
continue
|
|
if old.get("boot_id") == row["boot_id"]:
|
|
if not replaced:
|
|
updated.append(encoded)
|
|
replaced = True
|
|
continue
|
|
updated.append(line)
|
|
if not replaced:
|
|
updated.append(encoded)
|
|
event_log.write_text("\n".join(updated) + "\n", encoding="utf-8")
|
|
temporary = latest_event.with_suffix(".tmp")
|
|
temporary.write_text(encoded + "\n", encoding="utf-8")
|
|
temporary.replace(latest_event)
|
|
PY
|
|
|
|
chmod 0640 "$event_log" "$latest_event" "$capture_dir"/* 2>/dev/null || true
|
|
printf '%s\n' "$boot_id" >"$state_dir/last-captured-boot-id"
|
|
chmod 0640 "$state_dir/last-captured-boot-id"
|
|
logger -t control-boot-capture "recorded $classification for boot $boot_id"
|