From 49fc34f8d7599833c4632e2a95add229ce4bdd90 Mon Sep 17 00:00:00 2001 From: Andreas Brenner Date: Thu, 14 May 2026 14:39:32 +0200 Subject: [PATCH] ANW-11 vault scanner: walk, parse frontmatter, compute ETag --- Cargo.lock | 12 ++ Cargo.toml | 5 + src/lib.rs | 7 + src/vault.rs | 417 +++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 441 insertions(+) create mode 100644 src/lib.rs create mode 100644 src/vault.rs diff --git a/Cargo.lock b/Cargo.lock index 9b22dde..9f9af3f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -74,6 +74,7 @@ dependencies = [ "anyhow", "axum", "blake3", + "chrono", "clap", "hydra", "notify", @@ -81,12 +82,14 @@ dependencies = [ "serde_json", "serde_yaml", "tantivy", + "tempfile", "thiserror", "tokio", "tower", "tower-http", "tracing", "tracing-subscriber", + "walkdir", ] [[package]] @@ -322,6 +325,15 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "num-traits", +] + [[package]] name = "clap" version = "4.6.1" diff --git a/Cargo.toml b/Cargo.toml index e427877..f780fc9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ path = "src/main.rs" anyhow = "1" axum = "0.8" blake3 = "1" +chrono = { version = "0.4", default-features = false, features = ["std"] } clap = { version = "4", features = ["derive", "env"] } hydra = "0.1" notify = "8" @@ -28,6 +29,10 @@ tower = "0.5" tower-http = { version = "0.6", features = ["trace"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } +walkdir = "2" + +[dev-dependencies] +tempfile = "3" [lints.rust] unsafe_code = "forbid" diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..21b6858 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,7 @@ +//! Anwesen: read-only HTTP daemon over a markdown vault. +//! +//! The binary in `main.rs` wires this library to a CLI; submodules implement +//! the vault scanner, the Tantivy index, the filesystem watcher, the +//! supervisor tree, and the HTTP surface as those issues land. + +pub mod vault; diff --git a/src/vault.rs b/src/vault.rs new file mode 100644 index 0000000..c09197d --- /dev/null +++ b/src/vault.rs @@ -0,0 +1,417 @@ +//! Vault scanner. +//! +//! Walks the vault root, parses YAML frontmatter, coerces ISO-8601 / RFC 3339 +//! date strings to typed dates per [[ADR-005 Frontmatter Contract Type +//! Coercion and Cross-Note Shapes]], retains each file's raw bytes in the +//! [`Note`] record, and computes the strong `ETag` as `BLAKE3` of those raw +//! bytes per [[ADR-006 `ETag` Derivation]]. +//! +//! Files that are not Markdown (extension `.md`) and entries under any +//! dot-directory (`.obsidian/`, `.git/`, `.trash/`, ...) are skipped per +//! [[ADR-003 Filesystem Change Tracking]]. +//! +//! Per-file failures (I/O, non-UTF-8 path, malformed YAML) are surfaced as +//! [`ScanIssue`]s alongside successful records so callers (the scanner main +//! loop, `doctor`) can decide whether to skip or report. + +use std::collections::BTreeMap; +use std::io; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +use chrono::{DateTime, FixedOffset, NaiveDate}; +use thiserror::Error; +use walkdir::WalkDir; + +/// One scanned note. `raw_bytes` is retained so `Accept: text/markdown` can +/// return the exact bytes that produced `etag` without re-reading from disk +/// between watcher events ([ANW-13](https://crvrs.youtrack.cloud/issue/ANW-13)). +#[derive(Debug, Clone)] +pub struct Note { + /// Path relative to the vault root, forward-slash separated. + pub path: String, + pub frontmatter: Frontmatter, + pub body: String, + pub raw_bytes: Vec, + pub last_modified: SystemTime, + pub etag: String, + pub size: u64, +} + +pub type Frontmatter = BTreeMap; + +/// Frontmatter value with the type-coercion contract from +/// [[ADR-005 Frontmatter Contract Type Coercion and Cross-Note Shapes]] applied. +#[derive(Debug, Clone, PartialEq)] +pub enum Value { + Null, + Bool(bool), + Int(i64), + Float(f64), + String(String), + Date(NaiveDate), + DateTime(DateTime), + Sequence(Vec), + Mapping(BTreeMap), +} + +#[derive(Debug)] +pub struct ScanResult { + pub notes: Vec, + pub issues: Vec, +} + +#[derive(Debug)] +pub struct ScanIssue { + pub path: PathBuf, + pub kind: ScanIssueKind, +} + +#[derive(Debug, Error)] +pub enum ScanIssueKind { + #[error("io: {0}")] + Io(#[from] io::Error), + #[error("path is not valid UTF-8")] + NonUtf8Path, + #[error("frontmatter YAML parse failed: {0}")] + FrontmatterParse(#[from] serde_yaml::Error), + #[error("file body is not valid UTF-8")] + NonUtf8Body, +} + +/// Walk the vault and return every readable Markdown note alongside any +/// per-file issues. The walk never panics on a single broken file. +pub fn scan(vault_root: &Path) -> ScanResult { + let mut notes = Vec::new(); + let mut issues = Vec::new(); + + let walker = WalkDir::new(vault_root) + .follow_links(false) + .into_iter() + // The root entry itself may have a dot-prefixed name (e.g., a + // `/tmp/.tmpXXXX` tempdir or a vault under a hidden parent); only + // apply the dot-prefix skip to descendants. + .filter_entry(|e| e.depth() == 0 || !is_dot_prefixed(e.file_name())); + + for entry in walker { + let entry = match entry { + Ok(e) => e, + Err(err) => { + issues.push(ScanIssue { + path: err.path().map_or_else(PathBuf::new, Path::to_path_buf), + kind: ScanIssueKind::Io(io::Error::other(err.to_string())), + }); + continue; + } + }; + if !entry.file_type().is_file() { + continue; + } + let abs_path = entry.path(); + if !is_markdown(abs_path) { + continue; + } + match read_one(vault_root, abs_path) { + Ok(note) => notes.push(note), + Err(kind) => issues.push(ScanIssue { + path: abs_path.to_path_buf(), + kind, + }), + } + } + + ScanResult { notes, issues } +} + +fn is_markdown(path: &Path) -> bool { + path.extension().is_some_and(|e| e == "md") +} + +fn is_dot_prefixed(name: &std::ffi::OsStr) -> bool { + name.to_str().is_some_and(|s| s.starts_with('.')) +} + +fn read_one(vault_root: &Path, abs_path: &Path) -> Result { + let raw_bytes = std::fs::read(abs_path)?; + let metadata = std::fs::metadata(abs_path)?; + let size = metadata.len(); + let last_modified = metadata.modified()?; + let etag = format!("\"{}\"", blake3::hash(&raw_bytes).to_hex()); + + let text = std::str::from_utf8(&raw_bytes).map_err(|_| ScanIssueKind::NonUtf8Body)?; + let (frontmatter_yaml, body) = split_frontmatter(text); + let frontmatter = parse_frontmatter(frontmatter_yaml)?; + + let rel = abs_path + .strip_prefix(vault_root) + .unwrap_or(abs_path) + .to_path_buf(); + let path = rel.to_str().ok_or(ScanIssueKind::NonUtf8Path)?.to_string(); + // Normalize separators for HTTP-facing storage; on Linux this is a no-op. + let path = path.replace('\\', "/"); + + Ok(Note { + path, + frontmatter, + body: body.to_string(), + raw_bytes, + last_modified, + etag, + size, + }) +} + +/// Split a Markdown source into `(frontmatter_yaml, body)`. The frontmatter +/// block is the region delimited by a leading `---\n` and a closing line of +/// exactly `---`. Files without a frontmatter block yield `("", entire body)`. +fn split_frontmatter(src: &str) -> (&str, &str) { + let Some(after_open) = src.strip_prefix("---\n") else { + return ("", src); + }; + // Find a line consisting of exactly "---" (followed by \n or EOF). + let mut search_start = 0; + while let Some(rel_idx) = after_open[search_start..].find("\n---") { + let abs = search_start + rel_idx + 1; // position of the "---" + let after_close = &after_open[abs + 3..]; + // Accept either a trailing newline or end-of-file after the closing "---". + if after_close.is_empty() || after_close.starts_with('\n') { + let yaml = &after_open[..abs]; + let body = after_close.strip_prefix('\n').unwrap_or(after_close); + return (yaml, body); + } + search_start = abs + 3; + } + // Open marker but no close: treat whole file as body so the note is still + // served. A `doctor` follow-up can surface this; for now, do not lose data. + ("", src) +} + +fn parse_frontmatter(yaml: &str) -> Result { + if yaml.trim().is_empty() { + return Ok(BTreeMap::new()); + } + let raw: serde_yaml::Value = serde_yaml::from_str(yaml)?; + // A frontmatter that is not a mapping is not what Obsidian writes; + // treat as empty to avoid surfacing a contract surprise to consumers. + let serde_yaml::Value::Mapping(map) = raw else { + return Ok(BTreeMap::new()); + }; + let mut out = BTreeMap::new(); + for (k, v) in map { + let key = match k { + serde_yaml::Value::String(s) => s, + other => yaml_scalar_to_string(&other), + }; + out.insert(key, coerce(v)); + } + Ok(out) +} + +fn coerce(v: serde_yaml::Value) -> Value { + match v { + serde_yaml::Value::Null => Value::Null, + serde_yaml::Value::Bool(b) => Value::Bool(b), + serde_yaml::Value::Number(n) => { + if let Some(i) = n.as_i64() { + Value::Int(i) + } else if let Some(f) = n.as_f64() { + Value::Float(f) + } else { + Value::String(n.to_string()) + } + } + serde_yaml::Value::String(s) => coerce_string(s), + serde_yaml::Value::Sequence(seq) => Value::Sequence(seq.into_iter().map(coerce).collect()), + serde_yaml::Value::Mapping(m) => { + let mut nested = BTreeMap::new(); + for (k, v) in m { + let key = match k { + serde_yaml::Value::String(s) => s, + other => yaml_scalar_to_string(&other), + }; + nested.insert(key, coerce(v)); + } + Value::Mapping(nested) + } + serde_yaml::Value::Tagged(t) => coerce(t.value), + } +} + +/// Coerce a YAML string to a typed [`Value`]. ISO-8601 date (`YYYY-MM-DD`) +/// and RFC 3339 datetimes -- the shapes Obsidian writes from its date +/// property -- become typed dates; anything else stays a string. See +/// [[ADR-005 Frontmatter Contract Type Coercion and Cross-Note Shapes]]. +fn coerce_string(s: String) -> Value { + if let Ok(d) = NaiveDate::parse_from_str(&s, "%Y-%m-%d") { + return Value::Date(d); + } + if let Ok(dt) = DateTime::parse_from_rfc3339(&s) { + return Value::DateTime(dt); + } + Value::String(s) +} + +fn yaml_scalar_to_string(v: &serde_yaml::Value) -> String { + match v { + serde_yaml::Value::String(s) => s.clone(), + serde_yaml::Value::Bool(b) => b.to_string(), + serde_yaml::Value::Number(n) => n.to_string(), + serde_yaml::Value::Null => "null".to_string(), + _ => serde_yaml::to_string(v) + .unwrap_or_default() + .trim() + .to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + use tempfile::TempDir; + + fn write_note(root: &Path, rel: &str, body: &str) { + let p = root.join(rel); + if let Some(parent) = p.parent() { + fs::create_dir_all(parent).unwrap(); + } + fs::write(p, body).unwrap(); + } + + #[test] + fn skips_non_markdown_and_dot_directories() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + write_note(root, "kept.md", "---\ntags: [a]\n---\nbody\n"); + write_note(root, "skipped.txt", "not markdown"); + write_note(root, ".obsidian/workspace.json", "{}"); + write_note(root, ".git/HEAD", "ref: refs/heads/main"); + write_note(root, ".trash/old.md", "---\n---\n"); + write_note(root, "nested/.hidden/secret.md", "---\n---\n"); + + let result = scan(root); + let paths: Vec<&str> = result.notes.iter().map(|n| n.path.as_str()).collect(); + assert_eq!(paths, vec!["kept.md"]); + assert!(result.issues.is_empty(), "issues: {:?}", result.issues); + } + + #[test] + fn parses_frontmatter_and_separates_body() { + let tmp = TempDir::new().unwrap(); + write_note( + tmp.path(), + "x.md", + "---\ntitle: Hello\ntags: [a, b]\n---\nthe body\n", + ); + let result = scan(tmp.path()); + assert_eq!(result.notes.len(), 1); + let n = &result.notes[0]; + assert_eq!(n.path, "x.md"); + assert_eq!(n.body, "the body\n"); + assert_eq!( + n.frontmatter.get("title"), + Some(&Value::String("Hello".into())) + ); + let tags = n.frontmatter.get("tags").expect("tags present"); + match tags { + Value::Sequence(s) => { + assert_eq!(s.len(), 2); + assert_eq!(s[0], Value::String("a".into())); + } + _ => panic!("tags should be a sequence: {tags:?}"), + } + } + + #[test] + fn coerces_iso_dates_and_rfc3339_datetimes() { + let tmp = TempDir::new().unwrap(); + write_note( + tmp.path(), + "d.md", + "---\ndate: 2026-05-14\nstamp: 2026-05-14T10:14:22Z\nplain: not-a-date\n---\n", + ); + let result = scan(tmp.path()); + assert_eq!(result.notes.len(), 1); + let fm = &result.notes[0].frontmatter; + match fm.get("date").unwrap() { + Value::Date(d) => { + assert_eq!(d.format("%Y-%m-%d").to_string(), "2026-05-14"); + } + other => panic!("expected Date: {other:?}"), + } + match fm.get("stamp").unwrap() { + Value::DateTime(dt) => { + assert_eq!(dt.to_rfc3339(), "2026-05-14T10:14:22+00:00"); + } + other => panic!("expected DateTime: {other:?}"), + } + assert_eq!( + fm.get("plain"), + Some(&Value::String("not-a-date".into())), + "non-date strings stay strings" + ); + } + + #[test] + fn etag_is_blake3_of_raw_bytes() { + let tmp = TempDir::new().unwrap(); + let src = "---\ntags: [a]\n---\nbody\n"; + write_note(tmp.path(), "x.md", src); + let result = scan(tmp.path()); + let n = &result.notes[0]; + let expected = format!("\"{}\"", blake3::hash(src.as_bytes()).to_hex()); + assert_eq!(n.etag, expected); + assert_eq!(n.raw_bytes, src.as_bytes()); + assert_eq!(n.size, src.len() as u64); + } + + #[test] + fn malformed_frontmatter_becomes_scan_issue() { + let tmp = TempDir::new().unwrap(); + // ':' without value and an unterminated list both make this invalid YAML. + write_note(tmp.path(), "bad.md", "---\ntags: [a, b\nkey: : :\n---\n"); + let result = scan(tmp.path()); + assert!(result.notes.is_empty()); + assert_eq!(result.issues.len(), 1); + assert!(matches!( + result.issues[0].kind, + ScanIssueKind::FrontmatterParse(_) + )); + } + + #[test] + fn file_without_frontmatter_block_is_kept() { + let tmp = TempDir::new().unwrap(); + write_note(tmp.path(), "plain.md", "just a body, no frontmatter\n"); + let result = scan(tmp.path()); + assert_eq!(result.notes.len(), 1); + let n = &result.notes[0]; + assert!(n.frontmatter.is_empty()); + assert_eq!(n.body, "just a body, no frontmatter\n"); + } + + #[test] + fn relative_paths_are_forward_slash_separated() { + let tmp = TempDir::new().unwrap(); + write_note(tmp.path(), "Projects/anwesen/note.md", "---\n---\n"); + let result = scan(tmp.path()); + assert_eq!(result.notes.len(), 1); + assert_eq!(result.notes[0].path, "Projects/anwesen/note.md"); + } + + #[test] + fn split_frontmatter_handles_closing_at_eof() { + // Closing "---" right at end of file, no trailing newline. + let (yaml, body) = split_frontmatter("---\nkey: val\n---"); + assert_eq!(yaml, "key: val\n"); + assert_eq!(body, ""); + } + + #[test] + fn split_frontmatter_no_close_falls_back_to_body() { + // Open marker but no close means the file is malformed; do not lose data. + let (yaml, body) = split_frontmatter("---\nkey: val\nno close here\n"); + assert!(yaml.is_empty()); + assert!(body.starts_with("---\n")); + } +}