ANW-45 Serve: exit nonzero when the supervisor tree fails to start

Hydra's Application::run logs a start failure and returns normally, so
serve exited 0 after the tree never came up. systemd read that as a clean
start and Restart=on-failure never retried; anwesen was unreachable on ap
for 12 hours with NRestarts=0.

Anwesen carries a started flag that start() sets once the supervisor is
up, and main exits 1 when it is still false after run() returns.

Assumed the flag is the only seam available: run() consumes self and
returns (), so the Err is not observable at the call site. Flag if a newer
hydra exposes the result.

src/app.rs

@@ -121,6 +121,12 @@ pub struct Anwesen {
     /// Request-level telemetry handle ([ANW-37]). `None` disables export and
     /// the request middleware entirely.
     pub telemetry: Option<Arc<Telemetry>>,
+    /// Set once the supervisor tree is up. Hydra's `Application::run` logs a
+    /// start failure and returns normally, so `serve` cannot tell a clean
+    /// shutdown from a tree that never came up. `main` reads this after `run`
+    /// and exits nonzero when it is still false, which is what lets systemd
+    /// retry ([ANW-45](https://crvrs.youtrack.cloud/issue/ANW-45)).
+    pub started: Arc<AtomicBool>,
 }
 
 impl Anwesen {
@@ -133,6 +139,7 @@ impl Anwesen {
             store: NoteStore::new(),
             health: HealthState::new(),
             telemetry,
+            started: Arc::new(AtomicBool::new(false)),
         }
     }
 }
@@ -187,10 +194,12 @@ impl Application for Anwesen {
             .child_spec(),
         ];
 
-        Supervisor::with_children(children)
+        let pid = Supervisor::with_children(children)
             .strategy(SupervisionStrategy::OneForOne)
             .start_link(SupervisorOptions::new().name("anwesen_root"))
-            .await
+            .await?;
+        self.started.store(true, Ordering::Release);
+        Ok(pid)
     }
 }
 

src/main.rs

@@ -6,6 +6,7 @@
 mod cli;
 
 use std::sync::Arc;
+use std::sync::atomic::Ordering;
 
 use anwesen::app::Anwesen;
 use anwesen::doctor;
@@ -46,12 +47,22 @@ fn main() -> Result<()> {
                 telemetry = telemetry.is_some(),
                 "anwesen serve: starting supervisor tree"
             );
+            let app = Anwesen::new(args.vault, args.bind, telemetry.clone());
+            let started = app.started.clone();
             // Blocks until the supervisor exits (SIGTERM / SIGINT / crash).
-            Anwesen::new(args.vault, args.bind, telemetry.clone()).run();
+            app.run();
             // Flush and shut down exporters after the server loop returns.
             if let Some(telemetry) = telemetry {
                 telemetry.shutdown();
             }
+            // `run` returns normally whether the tree came up or never
+            // started, so a failed start would otherwise look like a clean
+            // exit and systemd's `Restart=on-failure` would not retry
+            // (ANW-45). Exit nonzero when the tree never came up.
+            if !started.load(Ordering::Acquire) {
+                tracing::error!("anwesen serve: supervisor tree failed to start");
+                std::process::exit(1);
+            }
         }
         Command::Doctor(args) => {
             init_logging(args.log_level);