carvers/anwesen
5
0
Fork 0
Code Releases 4 Activity Actions

ANW-11 vault scanner: walk, parse frontmatter, compute ETag

Browse source
This commit is contained in:
Andreas Brenner 2026-05-14 14:39:32 +02:00
parent 36492c8058
commit 49fc34f8d7
4 changed files with 441 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

12
Cargo.lock generated
View file

@ -74,6 +74,7 @@ dependencies = [
"anyhow",
"axum",
"blake3",
"chrono",
"clap",
"hydra",
"notify",
@ -81,12 +82,14 @@ dependencies = [
"serde_json",
"serde_yaml",
"tantivy",
"tempfile",
"thiserror",
"tokio",
"tower",
"tower-http",
"tracing",
"tracing-subscriber",
"walkdir",
]
[[package]]
@ -322,6 +325,15 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "chrono"
version = "0.4.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0"
dependencies = [
"num-traits",
]
[[package]]
name = "clap"
version = "4.6.1"

5
Cargo.toml
View file

@ -15,6 +15,7 @@ path = "src/main.rs"
anyhow = "1"
axum = "0.8"
blake3 = "1"
chrono = { version = "0.4", default-features = false, features = ["std"] }
clap = { version = "4", features = ["derive", "env"] }
hydra = "0.1"
notify = "8"
@ -28,6 +29,10 @@ tower = "0.5"
tower-http = { version = "0.6", features = ["trace"] }
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
walkdir = "2"
[dev-dependencies]
tempfile = "3"
[lints.rust]
unsafe_code = "forbid"

7
src/lib.rs Normal file
View file

@ -0,0 +1,7 @@
//! Anwesen: read-only HTTP daemon over a markdown vault.
//!
//! The binary in `main.rs` wires this library to a CLI; submodules implement
//! the vault scanner, the Tantivy index, the filesystem watcher, the
//! supervisor tree, and the HTTP surface as those issues land.
pub mod vault;

417
src/vault.rs Normal file
View file

@ -0,0 +1,417 @@
//! Vault scanner.
//!
//! Walks the vault root, parses YAML frontmatter, coerces ISO-8601 / RFC 3339
//! date strings to typed dates per [[ADR-005 Frontmatter Contract Type
//! Coercion and Cross-Note Shapes]], retains each file's raw bytes in the
//! [`Note`] record, and computes the strong `ETag` as `BLAKE3` of those raw
//! bytes per [[ADR-006 `ETag` Derivation]].
//!
//! Files that are not Markdown (extension `.md`) and entries under any
//! dot-directory (`.obsidian/`, `.git/`, `.trash/`, ...) are skipped per
//! [[ADR-003 Filesystem Change Tracking]].
//!
//! Per-file failures (I/O, non-UTF-8 path, malformed YAML) are surfaced as
//! [`ScanIssue`]s alongside successful records so callers (the scanner main
//! loop, `doctor`) can decide whether to skip or report.
use std::collections::BTreeMap;
use std::io;
use std::path::{Path, PathBuf};
use std::time::SystemTime;
use chrono::{DateTime, FixedOffset, NaiveDate};
use thiserror::Error;
use walkdir::WalkDir;
/// One scanned note. `raw_bytes` is retained so `Accept: text/markdown` can
/// return the exact bytes that produced `etag` without re-reading from disk
/// between watcher events ([ANW-13](https://crvrs.youtrack.cloud/issue/ANW-13)).
#[derive(Debug, Clone)]
pub struct Note {
/// Path relative to the vault root, forward-slash separated.
pub path: String,
pub frontmatter: Frontmatter,
pub body: String,
pub raw_bytes: Vec<u8>,
pub last_modified: SystemTime,
pub etag: String,
pub size: u64,
}
pub type Frontmatter = BTreeMap<String, Value>;
/// Frontmatter value with the type-coercion contract from
/// [[ADR-005 Frontmatter Contract Type Coercion and Cross-Note Shapes]] applied.
#[derive(Debug, Clone, PartialEq)]
pub enum Value {
Null,
Bool(bool),
Int(i64),
Float(f64),
String(String),
Date(NaiveDate),
DateTime(DateTime<FixedOffset>),
Sequence(Vec<Value>),
Mapping(BTreeMap<String, Value>),
}
#[derive(Debug)]
pub struct ScanResult {
pub notes: Vec<Note>,
pub issues: Vec<ScanIssue>,
}
#[derive(Debug)]
pub struct ScanIssue {
pub path: PathBuf,
pub kind: ScanIssueKind,
}
#[derive(Debug, Error)]
pub enum ScanIssueKind {
#[error("io: {0}")]
Io(#[from] io::Error),
#[error("path is not valid UTF-8")]
NonUtf8Path,
#[error("frontmatter YAML parse failed: {0}")]
FrontmatterParse(#[from] serde_yaml::Error),
#[error("file body is not valid UTF-8")]
NonUtf8Body,
}
/// Walk the vault and return every readable Markdown note alongside any
/// per-file issues. The walk never panics on a single broken file.
pub fn scan(vault_root: &Path) -> ScanResult {
let mut notes = Vec::new();
let mut issues = Vec::new();
let walker = WalkDir::new(vault_root)
.follow_links(false)
.into_iter()
// The root entry itself may have a dot-prefixed name (e.g., a
// `/tmp/.tmpXXXX` tempdir or a vault under a hidden parent); only
// apply the dot-prefix skip to descendants.
.filter_entry(|e| e.depth() == 0 || !is_dot_prefixed(e.file_name()));
for entry in walker {
let entry = match entry {
Ok(e) => e,
Err(err) => {
issues.push(ScanIssue {
path: err.path().map_or_else(PathBuf::new, Path::to_path_buf),
kind: ScanIssueKind::Io(io::Error::other(err.to_string())),
});
continue;
}
};
if !entry.file_type().is_file() {
continue;
}
let abs_path = entry.path();
if !is_markdown(abs_path) {
continue;
}
match read_one(vault_root, abs_path) {
Ok(note) => notes.push(note),
Err(kind) => issues.push(ScanIssue {
path: abs_path.to_path_buf(),
kind,
}),
}
}
ScanResult { notes, issues }
}
fn is_markdown(path: &Path) -> bool {
path.extension().is_some_and(|e| e == "md")
}
fn is_dot_prefixed(name: &std::ffi::OsStr) -> bool {
name.to_str().is_some_and(|s| s.starts_with('.'))
}
fn read_one(vault_root: &Path, abs_path: &Path) -> Result<Note, ScanIssueKind> {
let raw_bytes = std::fs::read(abs_path)?;
let metadata = std::fs::metadata(abs_path)?;
let size = metadata.len();
let last_modified = metadata.modified()?;
let etag = format!("\"{}\"", blake3::hash(&raw_bytes).to_hex());
let text = std::str::from_utf8(&raw_bytes).map_err(|_| ScanIssueKind::NonUtf8Body)?;
let (frontmatter_yaml, body) = split_frontmatter(text);
let frontmatter = parse_frontmatter(frontmatter_yaml)?;
let rel = abs_path
.strip_prefix(vault_root)
.unwrap_or(abs_path)
.to_path_buf();
let path = rel.to_str().ok_or(ScanIssueKind::NonUtf8Path)?.to_string();
// Normalize separators for HTTP-facing storage; on Linux this is a no-op.
let path = path.replace('\\', "/");
Ok(Note {
path,
frontmatter,
body: body.to_string(),
raw_bytes,
last_modified,
etag,
size,
})
}
/// Split a Markdown source into `(frontmatter_yaml, body)`. The frontmatter
/// block is the region delimited by a leading `---\n` and a closing line of
/// exactly `---`. Files without a frontmatter block yield `("", entire body)`.
fn split_frontmatter(src: &str) -> (&str, &str) {
let Some(after_open) = src.strip_prefix("---\n") else {
return ("", src);
};
// Find a line consisting of exactly "---" (followed by \n or EOF).
let mut search_start = 0;
while let Some(rel_idx) = after_open[search_start..].find("\n---") {
let abs = search_start + rel_idx + 1; // position of the "---"
let after_close = &after_open[abs + 3..];
// Accept either a trailing newline or end-of-file after the closing "---".
if after_close.is_empty() || after_close.starts_with('\n') {
let yaml = &after_open[..abs];
let body = after_close.strip_prefix('\n').unwrap_or(after_close);
return (yaml, body);
}
search_start = abs + 3;
}
// Open marker but no close: treat whole file as body so the note is still
// served. A `doctor` follow-up can surface this; for now, do not lose data.
("", src)
}
fn parse_frontmatter(yaml: &str) -> Result<Frontmatter, ScanIssueKind> {
if yaml.trim().is_empty() {
return Ok(BTreeMap::new());
}
let raw: serde_yaml::Value = serde_yaml::from_str(yaml)?;
// A frontmatter that is not a mapping is not what Obsidian writes;
// treat as empty to avoid surfacing a contract surprise to consumers.
let serde_yaml::Value::Mapping(map) = raw else {
return Ok(BTreeMap::new());
};
let mut out = BTreeMap::new();
for (k, v) in map {
let key = match k {
serde_yaml::Value::String(s) => s,
other => yaml_scalar_to_string(&other),
};
out.insert(key, coerce(v));
}
Ok(out)
}
fn coerce(v: serde_yaml::Value) -> Value {
match v {
serde_yaml::Value::Null => Value::Null,
serde_yaml::Value::Bool(b) => Value::Bool(b),
serde_yaml::Value::Number(n) => {
if let Some(i) = n.as_i64() {
Value::Int(i)
} else if let Some(f) = n.as_f64() {
Value::Float(f)
} else {
Value::String(n.to_string())
}
}
serde_yaml::Value::String(s) => coerce_string(s),
serde_yaml::Value::Sequence(seq) => Value::Sequence(seq.into_iter().map(coerce).collect()),
serde_yaml::Value::Mapping(m) => {
let mut nested = BTreeMap::new();
for (k, v) in m {
let key = match k {
serde_yaml::Value::String(s) => s,
other => yaml_scalar_to_string(&other),
};
nested.insert(key, coerce(v));
}
Value::Mapping(nested)
}
serde_yaml::Value::Tagged(t) => coerce(t.value),
}
}
/// Coerce a YAML string to a typed [`Value`]. ISO-8601 date (`YYYY-MM-DD`)
/// and RFC 3339 datetimes -- the shapes Obsidian writes from its date
/// property -- become typed dates; anything else stays a string. See
/// [[ADR-005 Frontmatter Contract Type Coercion and Cross-Note Shapes]].
fn coerce_string(s: String) -> Value {
if let Ok(d) = NaiveDate::parse_from_str(&s, "%Y-%m-%d") {
return Value::Date(d);
}
if let Ok(dt) = DateTime::parse_from_rfc3339(&s) {
return Value::DateTime(dt);
}
Value::String(s)
}
fn yaml_scalar_to_string(v: &serde_yaml::Value) -> String {
match v {
serde_yaml::Value::String(s) => s.clone(),
serde_yaml::Value::Bool(b) => b.to_string(),
serde_yaml::Value::Number(n) => n.to_string(),
serde_yaml::Value::Null => "null".to_string(),
_ => serde_yaml::to_string(v)
.unwrap_or_default()
.trim()
.to_string(),
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
use tempfile::TempDir;
fn write_note(root: &Path, rel: &str, body: &str) {
let p = root.join(rel);
if let Some(parent) = p.parent() {
fs::create_dir_all(parent).unwrap();
}
fs::write(p, body).unwrap();
}
#[test]
fn skips_non_markdown_and_dot_directories() {
let tmp = TempDir::new().unwrap();
let root = tmp.path();
write_note(root, "kept.md", "---\ntags: [a]\n---\nbody\n");
write_note(root, "skipped.txt", "not markdown");
write_note(root, ".obsidian/workspace.json", "{}");
write_note(root, ".git/HEAD", "ref: refs/heads/main");
write_note(root, ".trash/old.md", "---\n---\n");
write_note(root, "nested/.hidden/secret.md", "---\n---\n");
let result = scan(root);
let paths: Vec<&str> = result.notes.iter().map(|n| n.path.as_str()).collect();
assert_eq!(paths, vec!["kept.md"]);
assert!(result.issues.is_empty(), "issues: {:?}", result.issues);
}
#[test]
fn parses_frontmatter_and_separates_body() {
let tmp = TempDir::new().unwrap();
write_note(
tmp.path(),
"x.md",
"---\ntitle: Hello\ntags: [a, b]\n---\nthe body\n",
);
let result = scan(tmp.path());
assert_eq!(result.notes.len(), 1);
let n = &result.notes[0];
assert_eq!(n.path, "x.md");
assert_eq!(n.body, "the body\n");
assert_eq!(
n.frontmatter.get("title"),
Some(&Value::String("Hello".into()))
);
let tags = n.frontmatter.get("tags").expect("tags present");
match tags {
Value::Sequence(s) => {
assert_eq!(s.len(), 2);
assert_eq!(s[0], Value::String("a".into()));
}
_ => panic!("tags should be a sequence: {tags:?}"),
}
}
#[test]
fn coerces_iso_dates_and_rfc3339_datetimes() {
let tmp = TempDir::new().unwrap();
write_note(
tmp.path(),
"d.md",
"---\ndate: 2026-05-14\nstamp: 2026-05-14T10:14:22Z\nplain: not-a-date\n---\n",
);
let result = scan(tmp.path());
assert_eq!(result.notes.len(), 1);
let fm = &result.notes[0].frontmatter;
match fm.get("date").unwrap() {
Value::Date(d) => {
assert_eq!(d.format("%Y-%m-%d").to_string(), "2026-05-14");
}
other => panic!("expected Date: {other:?}"),
}
match fm.get("stamp").unwrap() {
Value::DateTime(dt) => {
assert_eq!(dt.to_rfc3339(), "2026-05-14T10:14:22+00:00");
}
other => panic!("expected DateTime: {other:?}"),
}
assert_eq!(
fm.get("plain"),
Some(&Value::String("not-a-date".into())),
"non-date strings stay strings"
);
}
#[test]
fn etag_is_blake3_of_raw_bytes() {
let tmp = TempDir::new().unwrap();
let src = "---\ntags: [a]\n---\nbody\n";
write_note(tmp.path(), "x.md", src);
let result = scan(tmp.path());
let n = &result.notes[0];
let expected = format!("\"{}\"", blake3::hash(src.as_bytes()).to_hex());
assert_eq!(n.etag, expected);
assert_eq!(n.raw_bytes, src.as_bytes());
assert_eq!(n.size, src.len() as u64);
}
#[test]
fn malformed_frontmatter_becomes_scan_issue() {
let tmp = TempDir::new().unwrap();
// ':' without value and an unterminated list both make this invalid YAML.
write_note(tmp.path(), "bad.md", "---\ntags: [a, b\nkey: : :\n---\n");
let result = scan(tmp.path());
assert!(result.notes.is_empty());
assert_eq!(result.issues.len(), 1);
assert!(matches!(
result.issues[0].kind,
ScanIssueKind::FrontmatterParse(_)
));
}
#[test]
fn file_without_frontmatter_block_is_kept() {
let tmp = TempDir::new().unwrap();
write_note(tmp.path(), "plain.md", "just a body, no frontmatter\n");
let result = scan(tmp.path());
assert_eq!(result.notes.len(), 1);
let n = &result.notes[0];
assert!(n.frontmatter.is_empty());
assert_eq!(n.body, "just a body, no frontmatter\n");
}
#[test]
fn relative_paths_are_forward_slash_separated() {
let tmp = TempDir::new().unwrap();
write_note(tmp.path(), "Projects/anwesen/note.md", "---\n---\n");
let result = scan(tmp.path());
assert_eq!(result.notes.len(), 1);
assert_eq!(result.notes[0].path, "Projects/anwesen/note.md");
}
#[test]
fn split_frontmatter_handles_closing_at_eof() {
// Closing "---" right at end of file, no trailing newline.
let (yaml, body) = split_frontmatter("---\nkey: val\n---");
assert_eq!(yaml, "key: val\n");
assert_eq!(body, "");
}
#[test]
fn split_frontmatter_no_close_falls_back_to_body() {
// Open marker but no close means the file is malformed; do not lose data.
let (yaml, body) = split_frontmatter("---\nkey: val\nno close here\n");
assert!(yaml.is_empty());
assert!(body.starts_with("---\n"));
}
}