ANW-11 vault scanner: walk, parse frontmatter, compute ETag

Cargo.lock

@@ -74,6 +74,7 @@ dependencies = [
  "anyhow",
  "axum",
  "blake3",
+ "chrono",
  "clap",
  "hydra",
  "notify",
@@ -81,12 +82,14 @@ dependencies = [
  "serde_json",
  "serde_yaml",
  "tantivy",
+ "tempfile",
  "thiserror",
  "tokio",
  "tower",
  "tower-http",
  "tracing",
  "tracing-subscriber",
+ "walkdir",
 ]
 
 [[package]]
@@ -322,6 +325,15 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
 
+[[package]]
+name = "chrono"
+version = "0.4.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "clap"
 version = "4.6.1"

Cargo.toml

@@ -15,6 +15,7 @@ path = "src/main.rs"
 anyhow = "1"
 axum = "0.8"
 blake3 = "1"
+chrono = { version = "0.4", default-features = false, features = ["std"] }
 clap = { version = "4", features = ["derive", "env"] }
 hydra = "0.1"
 notify = "8"
@@ -28,6 +29,10 @@ tower = "0.5"
 tower-http = { version = "0.6", features = ["trace"] }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+walkdir = "2"
+
+[dev-dependencies]
+tempfile = "3"
 
 [lints.rust]
 unsafe_code = "forbid"

src/lib.rs

@@ -0,0 +1,7 @@
+//! Anwesen: read-only HTTP daemon over a markdown vault.
+//!
+//! The binary in `main.rs` wires this library to a CLI; submodules implement
+//! the vault scanner, the Tantivy index, the filesystem watcher, the
+//! supervisor tree, and the HTTP surface as those issues land.
+
+pub mod vault;

src/vault.rs

@@ -0,0 +1,417 @@
+//! Vault scanner.
+//!
+//! Walks the vault root, parses YAML frontmatter, coerces ISO-8601 / RFC 3339
+//! date strings to typed dates per [[ADR-005 Frontmatter Contract Type
+//! Coercion and Cross-Note Shapes]], retains each file's raw bytes in the
+//! [`Note`] record, and computes the strong `ETag` as `BLAKE3` of those raw
+//! bytes per [[ADR-006 `ETag` Derivation]].
+//!
+//! Files that are not Markdown (extension `.md`) and entries under any
+//! dot-directory (`.obsidian/`, `.git/`, `.trash/`, ...) are skipped per
+//! [[ADR-003 Filesystem Change Tracking]].
+//!
+//! Per-file failures (I/O, non-UTF-8 path, malformed YAML) are surfaced as
+//! [`ScanIssue`]s alongside successful records so callers (the scanner main
+//! loop, `doctor`) can decide whether to skip or report.
+
+use std::collections::BTreeMap;
+use std::io;
+use std::path::{Path, PathBuf};
+use std::time::SystemTime;
+
+use chrono::{DateTime, FixedOffset, NaiveDate};
+use thiserror::Error;
+use walkdir::WalkDir;
+
+/// One scanned note. `raw_bytes` is retained so `Accept: text/markdown` can
+/// return the exact bytes that produced `etag` without re-reading from disk
+/// between watcher events ([ANW-13](https://crvrs.youtrack.cloud/issue/ANW-13)).
+#[derive(Debug, Clone)]
+pub struct Note {
+    /// Path relative to the vault root, forward-slash separated.
+    pub path: String,
+    pub frontmatter: Frontmatter,
+    pub body: String,
+    pub raw_bytes: Vec<u8>,
+    pub last_modified: SystemTime,
+    pub etag: String,
+    pub size: u64,
+}
+
+pub type Frontmatter = BTreeMap<String, Value>;
+
+/// Frontmatter value with the type-coercion contract from
+/// [[ADR-005 Frontmatter Contract Type Coercion and Cross-Note Shapes]] applied.
+#[derive(Debug, Clone, PartialEq)]
+pub enum Value {
+    Null,
+    Bool(bool),
+    Int(i64),
+    Float(f64),
+    String(String),
+    Date(NaiveDate),
+    DateTime(DateTime<FixedOffset>),
+    Sequence(Vec<Value>),
+    Mapping(BTreeMap<String, Value>),
+}
+
+#[derive(Debug)]
+pub struct ScanResult {
+    pub notes: Vec<Note>,
+    pub issues: Vec<ScanIssue>,
+}
+
+#[derive(Debug)]
+pub struct ScanIssue {
+    pub path: PathBuf,
+    pub kind: ScanIssueKind,
+}
+
+#[derive(Debug, Error)]
+pub enum ScanIssueKind {
+    #[error("io: {0}")]
+    Io(#[from] io::Error),
+    #[error("path is not valid UTF-8")]
+    NonUtf8Path,
+    #[error("frontmatter YAML parse failed: {0}")]
+    FrontmatterParse(#[from] serde_yaml::Error),
+    #[error("file body is not valid UTF-8")]
+    NonUtf8Body,
+}
+
+/// Walk the vault and return every readable Markdown note alongside any
+/// per-file issues. The walk never panics on a single broken file.
+pub fn scan(vault_root: &Path) -> ScanResult {
+    let mut notes = Vec::new();
+    let mut issues = Vec::new();
+
+    let walker = WalkDir::new(vault_root)
+        .follow_links(false)
+        .into_iter()
+        // The root entry itself may have a dot-prefixed name (e.g., a
+        // `/tmp/.tmpXXXX` tempdir or a vault under a hidden parent); only
+        // apply the dot-prefix skip to descendants.
+        .filter_entry(|e| e.depth() == 0 || !is_dot_prefixed(e.file_name()));
+
+    for entry in walker {
+        let entry = match entry {
+            Ok(e) => e,
+            Err(err) => {
+                issues.push(ScanIssue {
+                    path: err.path().map_or_else(PathBuf::new, Path::to_path_buf),
+                    kind: ScanIssueKind::Io(io::Error::other(err.to_string())),
+                });
+                continue;
+            }
+        };
+        if !entry.file_type().is_file() {
+            continue;
+        }
+        let abs_path = entry.path();
+        if !is_markdown(abs_path) {
+            continue;
+        }
+        match read_one(vault_root, abs_path) {
+            Ok(note) => notes.push(note),
+            Err(kind) => issues.push(ScanIssue {
+                path: abs_path.to_path_buf(),
+                kind,
+            }),
+        }
+    }
+
+    ScanResult { notes, issues }
+}
+
+fn is_markdown(path: &Path) -> bool {
+    path.extension().is_some_and(|e| e == "md")
+}
+
+fn is_dot_prefixed(name: &std::ffi::OsStr) -> bool {
+    name.to_str().is_some_and(|s| s.starts_with('.'))
+}
+
+fn read_one(vault_root: &Path, abs_path: &Path) -> Result<Note, ScanIssueKind> {
+    let raw_bytes = std::fs::read(abs_path)?;
+    let metadata = std::fs::metadata(abs_path)?;
+    let size = metadata.len();
+    let last_modified = metadata.modified()?;
+    let etag = format!("\"{}\"", blake3::hash(&raw_bytes).to_hex());
+
+    let text = std::str::from_utf8(&raw_bytes).map_err(|_| ScanIssueKind::NonUtf8Body)?;
+    let (frontmatter_yaml, body) = split_frontmatter(text);
+    let frontmatter = parse_frontmatter(frontmatter_yaml)?;
+
+    let rel = abs_path
+        .strip_prefix(vault_root)
+        .unwrap_or(abs_path)
+        .to_path_buf();
+    let path = rel.to_str().ok_or(ScanIssueKind::NonUtf8Path)?.to_string();
+    // Normalize separators for HTTP-facing storage; on Linux this is a no-op.
+    let path = path.replace('\\', "/");
+
+    Ok(Note {
+        path,
+        frontmatter,
+        body: body.to_string(),
+        raw_bytes,
+        last_modified,
+        etag,
+        size,
+    })
+}
+
+/// Split a Markdown source into `(frontmatter_yaml, body)`. The frontmatter
+/// block is the region delimited by a leading `---\n` and a closing line of
+/// exactly `---`. Files without a frontmatter block yield `("", entire body)`.
+fn split_frontmatter(src: &str) -> (&str, &str) {
+    let Some(after_open) = src.strip_prefix("---\n") else {
+        return ("", src);
+    };
+    // Find a line consisting of exactly "---" (followed by \n or EOF).
+    let mut search_start = 0;
+    while let Some(rel_idx) = after_open[search_start..].find("\n---") {
+        let abs = search_start + rel_idx + 1; // position of the "---"
+        let after_close = &after_open[abs + 3..];
+        // Accept either a trailing newline or end-of-file after the closing "---".
+        if after_close.is_empty() || after_close.starts_with('\n') {
+            let yaml = &after_open[..abs];
+            let body = after_close.strip_prefix('\n').unwrap_or(after_close);
+            return (yaml, body);
+        }
+        search_start = abs + 3;
+    }
+    // Open marker but no close: treat whole file as body so the note is still
+    // served. A `doctor` follow-up can surface this; for now, do not lose data.
+    ("", src)
+}
+
+fn parse_frontmatter(yaml: &str) -> Result<Frontmatter, ScanIssueKind> {
+    if yaml.trim().is_empty() {
+        return Ok(BTreeMap::new());
+    }
+    let raw: serde_yaml::Value = serde_yaml::from_str(yaml)?;
+    // A frontmatter that is not a mapping is not what Obsidian writes;
+    // treat as empty to avoid surfacing a contract surprise to consumers.
+    let serde_yaml::Value::Mapping(map) = raw else {
+        return Ok(BTreeMap::new());
+    };
+    let mut out = BTreeMap::new();
+    for (k, v) in map {
+        let key = match k {
+            serde_yaml::Value::String(s) => s,
+            other => yaml_scalar_to_string(&other),
+        };
+        out.insert(key, coerce(v));
+    }
+    Ok(out)
+}
+
+fn coerce(v: serde_yaml::Value) -> Value {
+    match v {
+        serde_yaml::Value::Null => Value::Null,
+        serde_yaml::Value::Bool(b) => Value::Bool(b),
+        serde_yaml::Value::Number(n) => {
+            if let Some(i) = n.as_i64() {
+                Value::Int(i)
+            } else if let Some(f) = n.as_f64() {
+                Value::Float(f)
+            } else {
+                Value::String(n.to_string())
+            }
+        }
+        serde_yaml::Value::String(s) => coerce_string(s),
+        serde_yaml::Value::Sequence(seq) => Value::Sequence(seq.into_iter().map(coerce).collect()),
+        serde_yaml::Value::Mapping(m) => {
+            let mut nested = BTreeMap::new();
+            for (k, v) in m {
+                let key = match k {
+                    serde_yaml::Value::String(s) => s,
+                    other => yaml_scalar_to_string(&other),
+                };
+                nested.insert(key, coerce(v));
+            }
+            Value::Mapping(nested)
+        }
+        serde_yaml::Value::Tagged(t) => coerce(t.value),
+    }
+}
+
+/// Coerce a YAML string to a typed [`Value`]. ISO-8601 date (`YYYY-MM-DD`)
+/// and RFC 3339 datetimes -- the shapes Obsidian writes from its date
+/// property -- become typed dates; anything else stays a string. See
+/// [[ADR-005 Frontmatter Contract Type Coercion and Cross-Note Shapes]].
+fn coerce_string(s: String) -> Value {
+    if let Ok(d) = NaiveDate::parse_from_str(&s, "%Y-%m-%d") {
+        return Value::Date(d);
+    }
+    if let Ok(dt) = DateTime::parse_from_rfc3339(&s) {
+        return Value::DateTime(dt);
+    }
+    Value::String(s)
+}
+
+fn yaml_scalar_to_string(v: &serde_yaml::Value) -> String {
+    match v {
+        serde_yaml::Value::String(s) => s.clone(),
+        serde_yaml::Value::Bool(b) => b.to_string(),
+        serde_yaml::Value::Number(n) => n.to_string(),
+        serde_yaml::Value::Null => "null".to_string(),
+        _ => serde_yaml::to_string(v)
+            .unwrap_or_default()
+            .trim()
+            .to_string(),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs;
+    use tempfile::TempDir;
+
+    fn write_note(root: &Path, rel: &str, body: &str) {
+        let p = root.join(rel);
+        if let Some(parent) = p.parent() {
+            fs::create_dir_all(parent).unwrap();
+        }
+        fs::write(p, body).unwrap();
+    }
+
+    #[test]
+    fn skips_non_markdown_and_dot_directories() {
+        let tmp = TempDir::new().unwrap();
+        let root = tmp.path();
+        write_note(root, "kept.md", "---\ntags: [a]\n---\nbody\n");
+        write_note(root, "skipped.txt", "not markdown");
+        write_note(root, ".obsidian/workspace.json", "{}");
+        write_note(root, ".git/HEAD", "ref: refs/heads/main");
+        write_note(root, ".trash/old.md", "---\n---\n");
+        write_note(root, "nested/.hidden/secret.md", "---\n---\n");
+
+        let result = scan(root);
+        let paths: Vec<&str> = result.notes.iter().map(|n| n.path.as_str()).collect();
+        assert_eq!(paths, vec!["kept.md"]);
+        assert!(result.issues.is_empty(), "issues: {:?}", result.issues);
+    }
+
+    #[test]
+    fn parses_frontmatter_and_separates_body() {
+        let tmp = TempDir::new().unwrap();
+        write_note(
+            tmp.path(),
+            "x.md",
+            "---\ntitle: Hello\ntags: [a, b]\n---\nthe body\n",
+        );
+        let result = scan(tmp.path());
+        assert_eq!(result.notes.len(), 1);
+        let n = &result.notes[0];
+        assert_eq!(n.path, "x.md");
+        assert_eq!(n.body, "the body\n");
+        assert_eq!(
+            n.frontmatter.get("title"),
+            Some(&Value::String("Hello".into()))
+        );
+        let tags = n.frontmatter.get("tags").expect("tags present");
+        match tags {
+            Value::Sequence(s) => {
+                assert_eq!(s.len(), 2);
+                assert_eq!(s[0], Value::String("a".into()));
+            }
+            _ => panic!("tags should be a sequence: {tags:?}"),
+        }
+    }
+
+    #[test]
+    fn coerces_iso_dates_and_rfc3339_datetimes() {
+        let tmp = TempDir::new().unwrap();
+        write_note(
+            tmp.path(),
+            "d.md",
+            "---\ndate: 2026-05-14\nstamp: 2026-05-14T10:14:22Z\nplain: not-a-date\n---\n",
+        );
+        let result = scan(tmp.path());
+        assert_eq!(result.notes.len(), 1);
+        let fm = &result.notes[0].frontmatter;
+        match fm.get("date").unwrap() {
+            Value::Date(d) => {
+                assert_eq!(d.format("%Y-%m-%d").to_string(), "2026-05-14");
+            }
+            other => panic!("expected Date: {other:?}"),
+        }
+        match fm.get("stamp").unwrap() {
+            Value::DateTime(dt) => {
+                assert_eq!(dt.to_rfc3339(), "2026-05-14T10:14:22+00:00");
+            }
+            other => panic!("expected DateTime: {other:?}"),
+        }
+        assert_eq!(
+            fm.get("plain"),
+            Some(&Value::String("not-a-date".into())),
+            "non-date strings stay strings"
+        );
+    }
+
+    #[test]
+    fn etag_is_blake3_of_raw_bytes() {
+        let tmp = TempDir::new().unwrap();
+        let src = "---\ntags: [a]\n---\nbody\n";
+        write_note(tmp.path(), "x.md", src);
+        let result = scan(tmp.path());
+        let n = &result.notes[0];
+        let expected = format!("\"{}\"", blake3::hash(src.as_bytes()).to_hex());
+        assert_eq!(n.etag, expected);
+        assert_eq!(n.raw_bytes, src.as_bytes());
+        assert_eq!(n.size, src.len() as u64);
+    }
+
+    #[test]
+    fn malformed_frontmatter_becomes_scan_issue() {
+        let tmp = TempDir::new().unwrap();
+        // ':' without value and an unterminated list both make this invalid YAML.
+        write_note(tmp.path(), "bad.md", "---\ntags: [a, b\nkey: : :\n---\n");
+        let result = scan(tmp.path());
+        assert!(result.notes.is_empty());
+        assert_eq!(result.issues.len(), 1);
+        assert!(matches!(
+            result.issues[0].kind,
+            ScanIssueKind::FrontmatterParse(_)
+        ));
+    }
+
+    #[test]
+    fn file_without_frontmatter_block_is_kept() {
+        let tmp = TempDir::new().unwrap();
+        write_note(tmp.path(), "plain.md", "just a body, no frontmatter\n");
+        let result = scan(tmp.path());
+        assert_eq!(result.notes.len(), 1);
+        let n = &result.notes[0];
+        assert!(n.frontmatter.is_empty());
+        assert_eq!(n.body, "just a body, no frontmatter\n");
+    }
+
+    #[test]
+    fn relative_paths_are_forward_slash_separated() {
+        let tmp = TempDir::new().unwrap();
+        write_note(tmp.path(), "Projects/anwesen/note.md", "---\n---\n");
+        let result = scan(tmp.path());
+        assert_eq!(result.notes.len(), 1);
+        assert_eq!(result.notes[0].path, "Projects/anwesen/note.md");
+    }
+
+    #[test]
+    fn split_frontmatter_handles_closing_at_eof() {
+        // Closing "---" right at end of file, no trailing newline.
+        let (yaml, body) = split_frontmatter("---\nkey: val\n---");
+        assert_eq!(yaml, "key: val\n");
+        assert_eq!(body, "");
+    }
+
+    #[test]
+    fn split_frontmatter_no_close_falls_back_to_body() {
+        // Open marker but no close means the file is malformed; do not lose data.
+        let (yaml, body) = split_frontmatter("---\nkey: val\nno close here\n");
+        assert!(yaml.is_empty());
+        assert!(body.starts_with("---\n"));
+    }
+}