carvers/anwesen
5
0
Fork 0
Code Releases 4 Activity Actions

ANW-23 remove tantivy from v1; NoteStore is the only authoritative store

Browse source
This commit is contained in:
Andreas Brenner 2026-05-14 18:33:20 +02:00
parent 30c42131b7
commit ec650e6c66
9 changed files with 41 additions and 1169 deletions
Download patch file Download diff file Expand all files Collapse all files

824
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

1
Cargo.toml
View file

@ -23,7 +23,6 @@ regex = "1"
serde = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] }
serde_json = "1" serde_json = "1"
serde_yaml = "0.9" serde_yaml = "0.9"
tantivy = "0.25"
thiserror = "2" thiserror = "2"
tokio = { version = "1", features = ["full"] } tokio = { version = "1", features = ["full"] }
tower = { version = "0.5", features = ["util"] } tower = { version = "0.5", features = ["util"] }

82
src/app.rs
View file

@ -5,7 +5,7 @@
//! RootSupervisor (one_for_one) //! RootSupervisor (one_for_one)
//! -- vault_scanner (permanent; startup walk + overflow recovery) //! -- vault_scanner (permanent; startup walk + overflow recovery)
//! -- filesystem_watcher (permanent; restart on inotify error) //! -- filesystem_watcher (permanent; restart on inotify error)
//! -- index_writer (permanent; owns the Tantivy IndexWriter) //! -- index_writer (permanent; owns the `NoteStore` write path)
//! -- http_server (permanent; restart on bind loss) //! -- http_server (permanent; restart on bind loss)
//! ``` //! ```
//! //!
@ -38,7 +38,6 @@ use tokio::task::JoinHandle;
use crate::health::HealthState; use crate::health::HealthState;
use crate::http::{self as http_layer, HttpState}; use crate::http::{self as http_layer, HttpState};
use crate::index::NoteIndex;
use crate::store::NoteStore; use crate::store::NoteStore;
use crate::vault::{self, Note}; use crate::vault::{self, Note};
use crate::watcher::run_debouncer; use crate::watcher::run_debouncer;
@ -145,17 +144,13 @@ impl Application for Anwesen {
async fn start(&self) -> Result<Pid, ExitReason> { async fn start(&self) -> Result<Pid, ExitReason> {
// Startup order matters: `vault_scanner` casts a `Rebuild` to the // Startup order matters: `vault_scanner` casts a `Rebuild` to the
// `index_writer` name at the end of its init. Hydra start_link is // `index_writer` name at the end of its init. Hydra `start_link` is
// synchronous, so we put `index_writer` first to guarantee it is // synchronous, so we put `index_writer` first to guarantee it is
// registered and in its message loop before `vault_scanner` runs. // registered and in its message loop before `vault_scanner` runs.
// The `one_for_one` strategy makes the order irrelevant for restart // The `one_for_one` strategy makes the order irrelevant for restart
// semantics, only for the cold-start handshake. // semantics, only for the cold-start handshake. After ANW-23 the
// // writer is restart-recoverable (NoteStore is owned by Anwesen and
// The cold-start handshake also relies on this order in the other // outlives the process), so no rescan handshake is needed on init.
// direction: `index_writer.init` skips the `rescan_now` cast on its
// first init (`record_init() == 0`) because `vault_scanner` is
// about to run its own startup walk. Reordering these children
// would silently break that assumption -- so don't.
let children = [ let children = [
IndexWriter { IndexWriter {
counters: self.counters.clone(), counters: self.counters.clone(),
@ -422,10 +417,8 @@ pub enum IndexWriterMessage {
/// Discard the current index and reindex the given notes. Sent by /// Discard the current index and reindex the given notes. Sent by
/// `vault_scanner` at startup and after `rescan_now`. /// `vault_scanner` at startup and after `rescan_now`.
Rebuild(Vec<Note>), Rebuild(Vec<Note>),
/// Apply one debounce-window's worth of upserts and deletes in a single /// Apply one debounce-window's worth of upserts and deletes against
/// Tantivy commit. Sent by `filesystem_watcher`. Picks up sie's ANW-12 /// [`NoteStore`] in a single write. Sent by `filesystem_watcher`.
/// follow-up #3 (commit batching) before ANW-16's watcher can turn
/// per-save events into a hot commit loop.
Batch(IndexBatch), Batch(IndexBatch),
/// Insert-or-replace one note. Retained for direct callers; the watcher /// Insert-or-replace one note. Retained for direct callers; the watcher
/// uses `Batch`. /// uses `Batch`.
@ -461,26 +454,23 @@ impl IndexWriter {
counters: counters.clone(), counters: counters.clone(),
store: store.clone(), store: store.clone(),
health: health.clone(), health: health.clone(),
index: None,
} }
.start_link(GenServerOptions::new().name(INDEX_WRITER_NAME)) .start_link(GenServerOptions::new().name(INDEX_WRITER_NAME))
}) })
} }
} }
/// Runtime state for the [`IndexWriter`] process. Held in a separate struct /// Runtime state for the [`IndexWriter`] process. Post-[ANW-23] the
/// so the `Clone`-friendly child-spec form (which doesn't carry the live /// "index" name survives in the supervisor tree and `/health` per
/// `NoteIndex`) stays simple. Mirrors every write into the shared /// [[ADR-009 Reverse ADR-002 In-Memory Evaluation No Tantivy]]'s naming
/// [`NoteStore`] so the HTTP layer can serve read-one and listing /// call, but the role is now purely a serialized writer into the shared
/// responses without consulting the index. /// [`NoteStore`]. No on-process state survives a restart -- `NoteStore`
/// is owned by `Anwesen` and outlives writer restarts, so no rescan
/// handshake is needed on init.
pub(crate) struct IndexWriterState { pub(crate) struct IndexWriterState {
counters: Arc<RestartCounters>, counters: Arc<RestartCounters>,
store: Arc<NoteStore>, store: Arc<NoteStore>,
health: Arc<HealthState>, health: Arc<HealthState>,
/// Created lazily in `init` so a Tantivy construction failure surfaces
/// as an `ExitReason` and triggers a supervisor restart, rather than
/// poisoning the child spec.
index: Option<NoteIndex>,
} }
impl GenServer for IndexWriterState { impl GenServer for IndexWriterState {
@ -488,70 +478,28 @@ impl GenServer for IndexWriterState {
async fn init(&mut self) -> Result<(), ExitReason> { async fn init(&mut self) -> Result<(), ExitReason> {
let restart = self.counters.index_writer.record_init(); let restart = self.counters.index_writer.record_init();
let index = NoteIndex::new()
.map_err(|e| ExitReason::from(format!("index_writer: NoteIndex::new failed: {e}")))?;
self.index = Some(index);
if restart > 0 {
// Writer-only restart: a fresh empty index has come up while the
// watcher keeps streaming batches at it. Ask `vault_scanner` for
// a full walk via the same `rescan_now` path the inotify-overflow
// recovery uses. Per kaa's pin (ADR-004 amendment 2026-05-14),
// rescan upserts are idempotent on path so in-flight watcher
// batches converge with the rescan.
VaultScanner::cast(
Dest::from(VAULT_SCANNER_NAME),
VaultScannerMessage::RescanNow,
);
tracing::info!(
restart,
"index_writer: init -- rescan_now dispatched to vault_scanner"
);
} else {
tracing::info!(restart, "index_writer: init"); tracing::info!(restart, "index_writer: init");
}
Ok(()) Ok(())
} }
async fn handle_cast(&mut self, message: Self::Message) -> Result<(), ExitReason> { async fn handle_cast(&mut self, message: Self::Message) -> Result<(), ExitReason> {
let Some(index) = self.index.as_mut() else {
return Err(ExitReason::from(
"index_writer: handle_cast invoked before init",
));
};
match message { match message {
IndexWriterMessage::Rebuild(notes) => { IndexWriterMessage::Rebuild(notes) => {
let count = notes.len(); let count = notes.len();
if let Err(e) = index.rebuild(&notes) {
tracing::error!(error = %e, "index_writer: rebuild failed");
return Err(ExitReason::from(format!("rebuild: {e}")));
}
self.store.replace(notes); self.store.replace(notes);
tracing::info!(notes = count, "index_writer: rebuilt"); tracing::info!(notes = count, "index_writer: rebuilt store");
} }
IndexWriterMessage::Batch(batch) => { IndexWriterMessage::Batch(batch) => {
let (u, d) = (batch.upserts.len(), batch.deletes.len()); let (u, d) = (batch.upserts.len(), batch.deletes.len());
if let Err(e) = index.apply_batch(&batch.upserts, &batch.deletes) {
tracing::error!(error = %e, "index_writer: batch apply failed");
return Err(ExitReason::from(format!("batch: {e}")));
}
self.store.apply_batch(batch.upserts, &batch.deletes); self.store.apply_batch(batch.upserts, &batch.deletes);
tracing::info!(upserts = u, deletes = d, "index_writer: batch applied"); tracing::info!(upserts = u, deletes = d, "index_writer: batch applied");
} }
IndexWriterMessage::Upsert(note) => { IndexWriterMessage::Upsert(note) => {
let path = note.path.clone(); let path = note.path.clone();
if let Err(e) = index.upsert(&note) {
tracing::error!(%path, error = %e, "index_writer: upsert failed");
return Err(ExitReason::from(format!("upsert {path}: {e}")));
}
self.store.upsert(*note); self.store.upsert(*note);
tracing::debug!(%path, "index_writer: upserted"); tracing::debug!(%path, "index_writer: upserted");
} }
IndexWriterMessage::Delete(path) => { IndexWriterMessage::Delete(path) => {
if let Err(e) = index.delete(&path) {
tracing::error!(%path, error = %e, "index_writer: delete failed");
return Err(ExitReason::from(format!("delete {path}: {e}")));
}
self.store.delete(&path); self.store.delete(&path);
tracing::debug!(%path, "index_writer: deleted"); tracing::debug!(%path, "index_writer: deleted");
} }

269
src/index.rs
View file

@ -1,269 +0,0 @@
//! In-memory Tantivy index over note frontmatter.
//!
//! Implements [[ADR-002 Tantivy as Frontmatter Index]] + the ANW-12 schema:
//!
//! - one `json` field (`frontmatter`) with the `raw` tokenizer pinned for v1
//! (exact-match semantics on every key, matches the keyword-style operator
//! surface in the User Manual);
//! - one `string` field (`path`) with the `raw` tokenizer (exact lookup +
//! anchored prefix queries).
//!
//! The index lives in a [`RamDirectory`](tantivy::directory::RamDirectory) --
//! rebuilt at startup from the scanner output and maintained incrementally
//! by [`upsert`](NoteIndex::upsert) / [`delete`](NoteIndex::delete) calls
//! driven from the filesystem watcher in [ANW-16].
use anyhow::{Context, Result};
use serde_json::json;
use tantivy::schema::{Field, IndexRecordOption, Schema, TextFieldIndexing, TextOptions};
use tantivy::{Index, IndexWriter, TantivyDocument, Term};
use crate::vault::{Note, frontmatter_to_json};
/// 50 MB heap for the writer. Tantivy's documented minimum is 15 MB; this
/// sits comfortably above that for low-thousands-of-notes vaults without
/// bloating idle memory.
const WRITER_HEAP_BYTES: usize = 50 * 1024 * 1024;
/// Strongly-typed handle to the two schema fields. Carried alongside the
/// [`Index`] so call sites don't restring field names.
#[derive(Debug, Clone, Copy)]
pub struct Fields {
pub path: Field,
pub frontmatter: Field,
}
/// The in-memory note index. Owns the [`Index`] and a long-lived
/// [`IndexWriter`]; one `NoteIndex` per daemon.
pub struct NoteIndex {
fields: Fields,
index: Index,
writer: IndexWriter,
}
impl NoteIndex {
/// Construct an empty in-memory index with the pinned v1 schema.
///
/// # Errors
/// Returns the underlying Tantivy error if writer allocation fails (the
/// only failure mode reachable for a fresh `RamDirectory`).
pub fn new() -> Result<Self> {
let mut schema_builder = Schema::builder();
let raw_indexing = TextFieldIndexing::default()
.set_tokenizer("raw")
.set_index_option(IndexRecordOption::Basic);
let path_options = TextOptions::default()
.set_indexing_options(raw_indexing.clone())
.set_stored();
let path = schema_builder.add_text_field("path", path_options);
// JSON field: open-schema frontmatter. `raw` tokenizer pins exact-match
// semantics on every key, per the open question closed in
// [[ADR-002 Tantivy as Frontmatter Index]].
let json_options = tantivy::schema::JsonObjectOptions::default()
.set_indexing_options(raw_indexing)
.set_stored();
let frontmatter = schema_builder.add_json_field("frontmatter", json_options);
let schema = schema_builder.build();
let index = Index::create_in_ram(schema);
let writer: IndexWriter = index
.writer(WRITER_HEAP_BYTES)
.context("create tantivy writer")?;
Ok(Self {
fields: Fields { path, frontmatter },
index,
writer,
})
}
/// Discard every document and reindex the given notes. Used at startup
/// and during overflow recovery per [[ADR-003 Filesystem Change Tracking]].
///
/// # Errors
/// Returns the underlying Tantivy error if any add/commit fails.
pub fn rebuild(&mut self, notes: &[Note]) -> Result<()> {
self.writer
.delete_all_documents()
.context("delete_all_documents")?;
for note in notes {
self.add_document(note)?;
}
self.writer.commit().context("commit rebuild")?;
Ok(())
}
/// Insert-or-replace one note by path. Subsequent reads through a fresh
/// reader will see the new content after this call returns.
///
/// # Errors
/// Returns the underlying Tantivy error if the add or commit fails.
pub fn upsert(&mut self, note: &Note) -> Result<()> {
self.writer.delete_term(self.path_term(&note.path));
self.add_document(note)?;
self.writer.commit().context("commit upsert")?;
Ok(())
}
/// Drop one note by path. No-op (still commits) if the path was never
/// indexed; callers can call freely on delete-events.
///
/// # Errors
/// Returns the underlying Tantivy error if the commit fails.
pub fn delete(&mut self, path: &str) -> Result<()> {
self.writer.delete_term(self.path_term(path));
self.writer.commit().context("commit delete")?;
Ok(())
}
/// Apply a batch of upserts and deletes in one Tantivy commit. The
/// watcher's debouncer dispatches batches per debounce window so a busy
/// editor save loop does not produce one commit per event.
///
/// # Errors
/// Returns the underlying Tantivy error if any add or the final commit
/// fails.
pub fn apply_batch(&mut self, upserts: &[Note], deletes: &[String]) -> Result<()> {
for path in deletes {
self.writer.delete_term(self.path_term(path));
}
for note in upserts {
self.writer.delete_term(self.path_term(&note.path));
self.add_document(note)?;
}
self.writer.commit().context("commit batch")?;
Ok(())
}
/// Number of indexed documents from a fresh reader. Read-only; intended
/// for tests and the `/health` endpoint ([ANW-8]).
///
/// # Errors
/// Returns the underlying Tantivy error if opening the reader fails.
pub fn document_count(&self) -> Result<u64> {
let reader = self.index.reader().context("open reader")?;
let searcher = reader.searcher();
Ok(searcher.num_docs())
}
/// Expose the [`Index`] so other layers (the future query handler in
/// [ANW-15]) can build their own readers and parsers without re-deriving
/// the schema.
#[must_use]
pub fn index(&self) -> &Index {
&self.index
}
#[must_use]
pub fn fields(&self) -> Fields {
self.fields
}
fn path_term(&self, path: &str) -> Term {
Term::from_field_text(self.fields.path, path)
}
fn add_document(&mut self, note: &Note) -> Result<()> {
// Build the doc as JSON keyed by schema field name and let Tantivy
// parse it -- avoids restating the OwnedValue tree by hand.
let doc_json = json!({
"path": &note.path,
"frontmatter": frontmatter_to_json(&note.frontmatter),
});
let doc = TantivyDocument::parse_json(&self.index.schema(), &doc_json.to_string())
.context("parse_json")?;
self.writer.add_document(doc).context("add_document")?;
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::{TimeZone, Utc};
use std::collections::BTreeMap;
use tantivy::collector::TopDocs;
use tantivy::query::TermQuery;
use crate::vault::Value;
fn sample_note(path: &str, tag: &str) -> Note {
let mut fm: BTreeMap<String, Value> = BTreeMap::new();
fm.insert(
"tags".into(),
Value::Sequence(vec![Value::String(tag.into())]),
);
fm.insert("title".into(), Value::String(format!("note {path}")));
Note {
path: path.into(),
frontmatter: fm,
body: String::new(),
raw_bytes: Vec::new(),
last_modified: Utc.with_ymd_and_hms(2026, 5, 14, 12, 0, 0).unwrap(),
etag: "\"deadbeef\"".into(),
size: 0,
}
}
fn count_path(idx: &NoteIndex, path: &str) -> tantivy::Result<usize> {
let reader = idx.index().reader()?;
let searcher = reader.searcher();
let term = Term::from_field_text(idx.fields().path, path);
let query = TermQuery::new(term, IndexRecordOption::Basic);
let hits = searcher.search(&query, &TopDocs::with_limit(10))?;
Ok(hits.len())
}
#[test]
fn rebuild_populates_index() {
let mut idx = NoteIndex::new().unwrap();
idx.rebuild(&[sample_note("a.md", "x"), sample_note("b.md", "y")])
.unwrap();
assert_eq!(idx.document_count().unwrap(), 2);
assert_eq!(count_path(&idx, "a.md").unwrap(), 1);
assert_eq!(count_path(&idx, "b.md").unwrap(), 1);
}
#[test]
fn rebuild_replaces_previous_contents() {
let mut idx = NoteIndex::new().unwrap();
idx.rebuild(&[sample_note("old.md", "x")]).unwrap();
idx.rebuild(&[sample_note("new.md", "y")]).unwrap();
assert_eq!(idx.document_count().unwrap(), 1);
assert_eq!(count_path(&idx, "old.md").unwrap(), 0);
assert_eq!(count_path(&idx, "new.md").unwrap(), 1);
}
#[test]
fn upsert_replaces_by_path() {
let mut idx = NoteIndex::new().unwrap();
idx.upsert(&sample_note("a.md", "v1")).unwrap();
// Mutate then upsert under the same path; document count must stay 1.
idx.upsert(&sample_note("a.md", "v2")).unwrap();
assert_eq!(idx.document_count().unwrap(), 1);
assert_eq!(count_path(&idx, "a.md").unwrap(), 1);
}
#[test]
fn delete_removes_path() {
let mut idx = NoteIndex::new().unwrap();
idx.upsert(&sample_note("a.md", "x")).unwrap();
idx.upsert(&sample_note("b.md", "y")).unwrap();
idx.delete("a.md").unwrap();
assert_eq!(idx.document_count().unwrap(), 1);
assert_eq!(count_path(&idx, "a.md").unwrap(), 0);
assert_eq!(count_path(&idx, "b.md").unwrap(), 1);
}
#[test]
fn delete_unknown_path_is_noop() {
let mut idx = NoteIndex::new().unwrap();
idx.upsert(&sample_note("a.md", "x")).unwrap();
idx.delete("never-indexed.md").unwrap();
assert_eq!(idx.document_count().unwrap(), 1);
}
}

7
src/lib.rs
View file

@ -1,14 +1,15 @@
//! Anwesen: read-only HTTP daemon over a markdown vault. //! Anwesen: read-only HTTP daemon over a markdown vault.
//! //!
//! The binary in `main.rs` wires this library to a CLI; submodules implement //! The binary in `main.rs` wires this library to a CLI; submodules implement
//! the vault scanner, the Tantivy index, the filesystem watcher, the //! the vault scanner, the in-memory note store, the filesystem watcher, the
//! supervisor tree, and the HTTP surface as those issues land. //! supervisor tree, and the HTTP surface. See
//! [[ADR-009 Reverse ADR-002 In-Memory Evaluation No Tantivy]] for why v1
//! ships without a search index.
pub mod app; pub mod app;
pub mod doctor; pub mod doctor;
pub mod health; pub mod health;
pub mod http; pub mod http;
pub mod index;
pub mod query; pub mod query;
pub mod store; pub mod store;
pub mod vault; pub mod vault;

12
src/query.rs
View file

@ -10,13 +10,11 @@
//! Different field predicates AND together; multiple values under //! Different field predicates AND together; multiple values under
//! `__in` / `__all` are comma-separated; unknown operators are `400`. //! `__in` / `__all` are comma-separated; unknown operators are `400`.
//! //!
//! v1 evaluates predicates by iterating the in-memory [`NoteStore`] and //! Predicates are evaluated by iterating the in-memory [`NoteStore`] and
//! applying each [`Predicate::matches`] in turn. The Tantivy index from //! applying each [`Predicate::matches`] in turn. At the documented scale
//! [[ADR-002 Tantivy as Frontmatter Index]] is still built and maintained, //! (low-thousands-of-notes vaults) this is sub-millisecond; see
//! but query-time filtering doesn't use it yet. At the documented scale //! [[ADR-009 Reverse ADR-002 In-Memory Evaluation No Tantivy]] for the
//! (low-thousands-of-notes vaults) this is sub-millisecond; migration to a //! call to keep evaluation in-memory rather than carrying a Tantivy index.
//! Tantivy-driven candidate set is a future optimization once a real
//! consumer pushes throughput.
use chrono::{DateTime, NaiveDate}; use chrono::{DateTime, NaiveDate};
use regex::Regex; use regex::Regex;

8
src/store.rs
View file

@ -3,10 +3,10 @@
//! HTTP layer ([ANW-13] / [ANW-14] / [ANW-15]) for read-one and listing //! HTTP layer ([ANW-13] / [ANW-14] / [ANW-15]) for read-one and listing
//! responses. //! responses.
//! //!
//! Per [[ADR-002 Tantivy as Frontmatter Index]] / [ANW-11], the daemon //! Per [ANW-11], the daemon rereads from disk only on a watcher event for
//! rereads from disk only on a watcher event for that path. The store is //! that path; this store is that in-memory cache and -- per
//! that in-memory cache; the index in [`crate::index`] sits beside it, //! [[ADR-009 Reverse ADR-002 In-Memory Evaluation No Tantivy]] -- the sole
//! shaped for frontmatter queries rather than full-record retrieval. //! authoritative record set both `/notes` and `/query` read from.
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::sync::{Arc, RwLock}; use std::sync::{Arc, RwLock};

2
src/vault.rs
View file

@ -99,7 +99,7 @@ impl Value {
} }
/// Convert a whole [`Frontmatter`] tree to a [`serde_json::Value`] object /// Convert a whole [`Frontmatter`] tree to a [`serde_json::Value`] object
/// suitable for HTTP responses or for Tantivy ingestion. /// suitable for HTTP responses.
#[must_use] #[must_use]
pub fn frontmatter_to_json(fm: &Frontmatter) -> JsonValue { pub fn frontmatter_to_json(fm: &Frontmatter) -> JsonValue {
let mut map = JsonMap::new(); let mut map = JsonMap::new();

3
src/watcher.rs
View file

@ -5,8 +5,7 @@
//! pushed into a Tokio channel; [`run_debouncer`] drains the channel, //! pushed into a Tokio channel; [`run_debouncer`] drains the channel,
//! classifies events into [`WatchAction`]s, coalesces a 100 ms window's //! classifies events into [`WatchAction`]s, coalesces a 100 ms window's
//! worth into one [`Batch`], and casts the batch to the `index_writer` //! worth into one [`Batch`], and casts the batch to the `index_writer`
//! named process for a single Tantivy commit (folds in sie's ANW-12 //! named process for a single [`crate::store::NoteStore`] write.
//! follow-up #3).
//! //!
//! See [[ADR-003 Filesystem Change Tracking]] for the event model. //! See [[ADR-003 Filesystem Change Tracking]] for the event model.