Say when the server fell over, and where to read why

Iris found the backend crash-looping by checking rc-service by hand,
because the card could only say `stopped` -- which reads as a state
somebody chose. dev-updater's contract now has a fourth word, `failed`,
and this script implements it.

The OpenRC detail is the one worth not rederiving: it prints `crashed`
*and* exits non-zero, so the word is read rather than the exit code.
Leaning on the code would report "couldn't check", which is a different
and less useful claim. The `running` check stays on its exit code, which
already worked and does not depend on wording.

The script also arranges the logging rather than only reporting it,
because neither unit wrote a file: systemd went to the journal and
OpenRC's `command_background=true` discarded output entirely, which is why
a crash left nothing to read. Output now goes to
$XDG_DATA_HOME/ai-server/ai-server.log -- generated data, outliving any one
build, and not in a repository shared with a machine that should not read
it. `start` rotates one generation aside, so what is kept is exactly this
run and the one before: the pair worth having after a crash and a restart.
`logs` prints the paths, newest first, and nothing else.

Verified on systemd by causing the failure rather than reasoning about it:
installed, started, confirmed `running` on the wg0 bind, wrote an
unparseable config, restarted, and watched status settle on **failed**
rather than stopped -- with the reason, line and column, in the file `logs`
points at, and the crash preserved in .1 after recovery. Then restored,
confirmed `running` again, and uninstalled.

**The OpenRC branch is written from the documentation and is untested**,
here and in dev-updater, since neither machine that can run it is one
either of us can test on. It is also the branch that actually matters,
since the backend runs under OpenRC on the host. `output_log`/`error_log`
in the openrc-run script are the parts to distrust first.

One thing that bit while writing it: the systemd heredoc is unquoted so
$LOG expands, which makes a backtick in a comment inside it run as command
substitution. A comment saying "`start` rotates" executed `start`, and the
unit was written without ever being valid. There is now a note in the
heredoc saying why it contains no backticks.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017xn8nHw1tw1R6PtiY1eEtw

server/service

@@ -9,9 +9,10 @@
 # reaches ai-server through the tunnel rather than through Dev Updater,
 # Stop leaves this server down until someone starts it again here.
 #
-#   ./service install | uninstall | start | stop | restart | status
+#   ./service install | uninstall | start | stop | restart | status | logs
 #
-# `status` prints exactly one of `running`, `stopped` or `not-installed`
+# `status` prints exactly one of `running`, `stopped`, `failed` or
+# `not-installed`
 # and exits 0. Anything else it prints, or any non-zero exit, means it
 # could not tell -- which the card shows as "couldn't check" rather than
 # as a service that is down.
@@ -32,6 +33,18 @@ SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
 NAME=ai-server
 BINARY="$SCRIPT_DIR/target/release/$NAME"
 
+# Where this service's output goes, and the one generation kept behind it.
+#
+# Under $XDG_DATA_HOME rather than the checkout: a log is generated data,
+# it outlives any one build, and the repository is shared with a machine
+# that should not be able to read it. Rotated on start rather than by size
+# or age, so what is kept is exactly "this run and the one before" -- which
+# is the pair worth having after a crash and a restart, and is the reason
+# the file is not simply appended to forever.
+LOG_DIR="${XDG_DATA_HOME:-$HOME/.local/share}/$NAME"
+LOG="$LOG_DIR/$NAME.log"
+LOG_PREVIOUS="$LOG.1"
+
 # Which init system is here, decided by asking rather than by looking for a
 # binary: a machine can carry both, and an OpenRC older than 0.60 has
 # rc-service but no --user at all.
@@ -74,6 +87,20 @@ installed() {
     esac
 }
 
+# Whether the service fell over, as opposed to being stopped on purpose.
+#
+# OpenRC prints `crashed` *and* exits non-zero for this, so the word is
+# read rather than the exit code -- leaning on the code would report
+# "couldn't check", which is a different and less useful thing to say.
+# The `running` check above stays on its exit code, which already worked
+# and does not depend on wording.
+crashed() {
+    case "$MANAGER" in
+        systemd) systemctl --user --quiet is-failed "$NAME" ;;
+        openrc) rc-service --user "$NAME" status 2>/dev/null | grep -qw crashed ;;
+    esac
+}
+
 require_binary() {
     [ -x "$BINARY" ] && return 0
     echo "No built server at $BINARY -- build it first." >&2
@@ -82,6 +109,7 @@ require_binary() {
 
 do_install() {
     require_binary
+    mkdir -p "$LOG_DIR"
     case "$MANAGER" in
         systemd)
             mkdir -p "$(dirname "$SYSTEMD_UNIT")"
@@ -93,6 +121,16 @@ Description=$NAME (installed by $SCRIPT_DIR/service)
 ExecStart=$BINARY
 Restart=on-failure
 WorkingDirectory=$SCRIPT_DIR
+# Appended rather than truncated: the unit is not the thing that decides
+# when a log starts over, since the start subcommand rotates. Restarting
+# inside one run (Restart=on-failure) then keeps the whole story rather
+# than erasing the reason. Needs systemd 240+.
+#
+# No backticks in this heredoc: the delimiter is unquoted so that \$LOG
+# expands, which means a backtick would run as command substitution while
+# the unit is being written.
+StandardOutput=append:$LOG
+StandardError=append:$LOG
 
 [Install]
 WantedBy=default.target
@@ -110,6 +148,10 @@ command="$BINARY"
 command_background=true
 directory="$SCRIPT_DIR"
 pidfile="\${XDG_RUNTIME_DIR}/$NAME.pid"
+# command_background discards output otherwise, which is why a crash left
+# nothing to read.
+output_log="$LOG"
+error_log="$LOG"
 UNIT
             chmod +x "$OPENRC_UNIT"
             rc-update --user add "$NAME" >/dev/null
@@ -133,6 +175,13 @@ do_uninstall() {
     esac
 }
 
+# Keeps the finished run and starts a fresh file for the next one.
+rotate() {
+    mkdir -p "$LOG_DIR"
+    [ -f "$LOG" ] && mv -f "$LOG" "$LOG_PREVIOUS"
+    : > "$LOG"
+}
+
 control() {
     installed || { echo "$NAME is not installed" >&2; exit 1; }
     case "$MANAGER" in
@@ -144,7 +193,22 @@ control() {
 case "${1:-}" in
     install) do_install ;;
     uninstall) do_uninstall ;;
-    start | stop | restart) control "$1" ;;
+    start | restart)
+        # Rotated before the manager is asked, so the file the service
+        # opens is the new one.
+        rotate
+        control "$1"
+        ;;
+    stop) control stop ;;
+    logs)
+        # One path per line, newest first, and nothing else: the caller
+        # wants somewhere to read from, not a formatted report. Printing
+        # nothing at all is the answer for a service with no log yet,
+        # which reads as "not supported here" and costs nobody anything.
+        [ -f "$LOG" ] && echo "$LOG"
+        [ -f "$LOG_PREVIOUS" ] && echo "$LOG_PREVIOUS"
+        exit 0
+        ;;
     status)
         if ! installed; then
             echo not-installed
@@ -153,12 +217,17 @@ case "${1:-}" in
             openrc) rc-service --user "$NAME" status >/dev/null 2>&1 ;;
         esac then
             echo running
+        elif crashed; then
+            # Before `stopped`, because a crashed service satisfies
+            # neither of the other two and would otherwise be reported as
+            # a state somebody chose.
+            echo failed
         else
             echo stopped
         fi
         ;;
     *)
-        echo "usage: $0 install|uninstall|start|stop|restart|status" >&2
+        echo "usage: $0 install|uninstall|start|stop|restart|status|logs" >&2
         exit 2
         ;;
 esac