ANW-45 Serve: exit nonzero when the supervisor tree fails to start

Hydra's Application::run logs a start failure and returns normally, so
serve exited 0 after the tree never came up. systemd read that as a clean
start and Restart=on-failure never retried; anwesen was unreachable on ap
for 12 hours with NRestarts=0.

Anwesen carries a started flag that start() sets once the supervisor is
up, and main exits 1 when it is still false after run() returns.

Assumed the flag is the only seam available: run() consumes self and
returns (), so the Err is not observable at the call site. Flag if a newer
hydra exposes the result.

src/app.rs

@@ -121,6 +121,12 @@ pub struct Anwesen {
     /// Request-level telemetry handle ([ANW-37]). `None` disables export and
     /// the request middleware entirely.
     pub telemetry: Option<Arc<Telemetry>>,
+    /// Set once the supervisor tree is up. Hydra's `Application::run` logs a
+    /// start failure and returns normally, so `serve` cannot tell a clean
+    /// shutdown from a tree that never came up. `main` reads this after `run`
+    /// and exits nonzero when it is still false, which is what lets systemd
+    /// retry ([ANW-45](https://crvrs.youtrack.cloud/issue/ANW-45)).
+    pub started: Arc<AtomicBool>,
 }
 
 impl Anwesen {
@@ -133,6 +139,7 @@ impl Anwesen {
             store: NoteStore::new(),
             health: HealthState::new(),
             telemetry,
+            started: Arc::new(AtomicBool::new(false)),
         }
     }
 }
@@ -187,10 +194,12 @@ impl Application for Anwesen {
             .child_spec(),
         ];
 
-        Supervisor::with_children(children)
+        let pid = Supervisor::with_children(children)
             .strategy(SupervisionStrategy::OneForOne)
             .start_link(SupervisorOptions::new().name("anwesen_root"))
-            .await
+            .await?;
+        self.started.store(true, Ordering::Release);
+        Ok(pid)
     }
 }
 

src/main.rs

@@ -6,6 +6,7 @@
 mod cli;
 
 use std::sync::Arc;
+use std::sync::atomic::Ordering;
 
 use anwesen::app::Anwesen;
 use anwesen::doctor;
@@ -46,12 +47,22 @@ fn main() -> Result<()> {
                 telemetry = telemetry.is_some(),
                 "anwesen serve: starting supervisor tree"
             );
+            let app = Anwesen::new(args.vault, args.bind, telemetry.clone());
+            let started = app.started.clone();
             // Blocks until the supervisor exits (SIGTERM / SIGINT / crash).
-            Anwesen::new(args.vault, args.bind, telemetry.clone()).run();
+            app.run();
             // Flush and shut down exporters after the server loop returns.
             if let Some(telemetry) = telemetry {
                 telemetry.shutdown();
             }
+            // `run` returns normally whether the tree came up or never
+            // started, so a failed start would otherwise look like a clean
+            // exit and systemd's `Restart=on-failure` would not retry
+            // (ANW-45). Exit nonzero when the tree never came up.
+            if !started.load(Ordering::Acquire) {
+                tracing::error!("anwesen serve: supervisor tree failed to start");
+                std::process::exit(1);
+            }
         }
         Command::Doctor(args) => {
             init_logging(args.log_level);

tests/serve_start_failure.rs

@@ -0,0 +1,33 @@
+//! `anwesen serve` must exit nonzero when the supervisor tree fails to start
+//! ([ANW-45](https://crvrs.youtrack.cloud/issue/ANW-45)). Hydra's
+//! `Application::run` logs the failure and returns normally, so without an
+//! explicit check the process exits 0 and systemd's `Restart=on-failure`
+//! never retries -- the vault stayed unreachable for 12 hours on ap.
+//!
+//! The forced failure is a taken bind address: `http_server` binds eagerly in
+//! its child spec, so the address-in-use error fails the whole start.
+
+use std::net::TcpListener;
+use std::process::Command;
+
+#[test]
+fn serve_exits_nonzero_when_the_tree_fails_to_start() {
+    let vault = tempfile::tempdir().expect("tempdir");
+    // Hold the port for the lifetime of the child so its bind cannot succeed.
+    let held = TcpListener::bind("127.0.0.1:0").expect("bind probe port");
+    let addr = held.local_addr().expect("probe addr");
+
+    let status = Command::new(env!("CARGO_BIN_EXE_anwesen"))
+        .arg("serve")
+        .arg("--vault")
+        .arg(vault.path())
+        .arg("--bind")
+        .arg(addr.to_string())
+        .status()
+        .expect("spawn anwesen serve");
+
+    assert!(
+        !status.success(),
+        "serve exited {status} after a failed supervisor start; systemd reads that as a clean exit"
+    );
+}