carvers/anwesen
5
0
Fork 0
Code Releases 4 Activity Actions

ANW-23 remove tantivy from v1; NoteStore is the only authoritative store

Browse source
This commit is contained in:
Andreas Brenner 2026-05-14 18:33:20 +02:00
parent 30c42131b7
commit ec650e6c66
9 changed files with 41 additions and 1169 deletions
Download patch file Download diff file Expand all files Collapse all files

84
src/app.rs
View file

@ -5,7 +5,7 @@
//! RootSupervisor (one_for_one)
//! -- vault_scanner (permanent; startup walk + overflow recovery)
//! -- filesystem_watcher (permanent; restart on inotify error)
//! -- index_writer (permanent; owns the Tantivy IndexWriter)
//! -- index_writer (permanent; owns the `NoteStore` write path)
//! -- http_server (permanent; restart on bind loss)
//! ```
//!
@ -38,7 +38,6 @@ use tokio::task::JoinHandle;
use crate::health::HealthState;
use crate::http::{self as http_layer, HttpState};
use crate::index::NoteIndex;
use crate::store::NoteStore;
use crate::vault::{self, Note};
use crate::watcher::run_debouncer;
@ -145,17 +144,13 @@ impl Application for Anwesen {
async fn start(&self) -> Result<Pid, ExitReason> {
// Startup order matters: `vault_scanner` casts a `Rebuild` to the
// `index_writer` name at the end of its init. Hydra start_link is
// `index_writer` name at the end of its init. Hydra `start_link` is
// synchronous, so we put `index_writer` first to guarantee it is
// registered and in its message loop before `vault_scanner` runs.
// The `one_for_one` strategy makes the order irrelevant for restart
// semantics, only for the cold-start handshake.
//
// The cold-start handshake also relies on this order in the other
// direction: `index_writer.init` skips the `rescan_now` cast on its
// first init (`record_init() == 0`) because `vault_scanner` is
// about to run its own startup walk. Reordering these children
// would silently break that assumption -- so don't.
// semantics, only for the cold-start handshake. After ANW-23 the
// writer is restart-recoverable (NoteStore is owned by Anwesen and
// outlives the process), so no rescan handshake is needed on init.
let children = [
IndexWriter {
counters: self.counters.clone(),
@ -422,10 +417,8 @@ pub enum IndexWriterMessage {
/// Discard the current index and reindex the given notes. Sent by
/// `vault_scanner` at startup and after `rescan_now`.
Rebuild(Vec<Note>),
/// Apply one debounce-window's worth of upserts and deletes in a single
/// Tantivy commit. Sent by `filesystem_watcher`. Picks up sie's ANW-12
/// follow-up #3 (commit batching) before ANW-16's watcher can turn
/// per-save events into a hot commit loop.
/// Apply one debounce-window's worth of upserts and deletes against
/// [`NoteStore`] in a single write. Sent by `filesystem_watcher`.
Batch(IndexBatch),
/// Insert-or-replace one note. Retained for direct callers; the watcher
/// uses `Batch`.
@ -461,26 +454,23 @@ impl IndexWriter {
counters: counters.clone(),
store: store.clone(),
health: health.clone(),
index: None,
}
.start_link(GenServerOptions::new().name(INDEX_WRITER_NAME))
})
}
}
/// Runtime state for the [`IndexWriter`] process. Held in a separate struct
/// so the `Clone`-friendly child-spec form (which doesn't carry the live
/// `NoteIndex`) stays simple. Mirrors every write into the shared
/// [`NoteStore`] so the HTTP layer can serve read-one and listing
/// responses without consulting the index.
/// Runtime state for the [`IndexWriter`] process. Post-[ANW-23] the
/// "index" name survives in the supervisor tree and `/health` per
/// [[ADR-009 Reverse ADR-002 In-Memory Evaluation No Tantivy]]'s naming
/// call, but the role is now purely a serialized writer into the shared
/// [`NoteStore`]. No on-process state survives a restart -- `NoteStore`
/// is owned by `Anwesen` and outlives writer restarts, so no rescan
/// handshake is needed on init.
pub(crate) struct IndexWriterState {
counters: Arc<RestartCounters>,
store: Arc<NoteStore>,
health: Arc<HealthState>,
/// Created lazily in `init` so a Tantivy construction failure surfaces
/// as an `ExitReason` and triggers a supervisor restart, rather than
/// poisoning the child spec.
index: Option<NoteIndex>,
}
impl GenServer for IndexWriterState {
@ -488,70 +478,28 @@ impl GenServer for IndexWriterState {
async fn init(&mut self) -> Result<(), ExitReason> {
let restart = self.counters.index_writer.record_init();
let index = NoteIndex::new()
.map_err(|e| ExitReason::from(format!("index_writer: NoteIndex::new failed: {e}")))?;
self.index = Some(index);
if restart > 0 {
// Writer-only restart: a fresh empty index has come up while the
// watcher keeps streaming batches at it. Ask `vault_scanner` for
// a full walk via the same `rescan_now` path the inotify-overflow
// recovery uses. Per kaa's pin (ADR-004 amendment 2026-05-14),
// rescan upserts are idempotent on path so in-flight watcher
// batches converge with the rescan.
VaultScanner::cast(
Dest::from(VAULT_SCANNER_NAME),
VaultScannerMessage::RescanNow,
);
tracing::info!(
restart,
"index_writer: init -- rescan_now dispatched to vault_scanner"
);
} else {
tracing::info!(restart, "index_writer: init");
}
tracing::info!(restart, "index_writer: init");
Ok(())
}
async fn handle_cast(&mut self, message: Self::Message) -> Result<(), ExitReason> {
let Some(index) = self.index.as_mut() else {
return Err(ExitReason::from(
"index_writer: handle_cast invoked before init",
));
};
match message {
IndexWriterMessage::Rebuild(notes) => {
let count = notes.len();
if let Err(e) = index.rebuild(&notes) {
tracing::error!(error = %e, "index_writer: rebuild failed");
return Err(ExitReason::from(format!("rebuild: {e}")));
}
self.store.replace(notes);
tracing::info!(notes = count, "index_writer: rebuilt");
tracing::info!(notes = count, "index_writer: rebuilt store");
}
IndexWriterMessage::Batch(batch) => {
let (u, d) = (batch.upserts.len(), batch.deletes.len());
if let Err(e) = index.apply_batch(&batch.upserts, &batch.deletes) {
tracing::error!(error = %e, "index_writer: batch apply failed");
return Err(ExitReason::from(format!("batch: {e}")));
}
self.store.apply_batch(batch.upserts, &batch.deletes);
tracing::info!(upserts = u, deletes = d, "index_writer: batch applied");
}
IndexWriterMessage::Upsert(note) => {
let path = note.path.clone();
if let Err(e) = index.upsert(&note) {
tracing::error!(%path, error = %e, "index_writer: upsert failed");
return Err(ExitReason::from(format!("upsert {path}: {e}")));
}
self.store.upsert(*note);
tracing::debug!(%path, "index_writer: upserted");
}
IndexWriterMessage::Delete(path) => {
if let Err(e) = index.delete(&path) {
tracing::error!(%path, error = %e, "index_writer: delete failed");
return Err(ExitReason::from(format!("delete {path}: {e}")));
}
self.store.delete(&path);
tracing::debug!(%path, "index_writer: deleted");
}

269
src/index.rs
View file

@ -1,269 +0,0 @@
//! In-memory Tantivy index over note frontmatter.
//!
//! Implements [[ADR-002 Tantivy as Frontmatter Index]] + the ANW-12 schema:
//!
//! - one `json` field (`frontmatter`) with the `raw` tokenizer pinned for v1
//! (exact-match semantics on every key, matches the keyword-style operator
//! surface in the User Manual);
//! - one `string` field (`path`) with the `raw` tokenizer (exact lookup +
//! anchored prefix queries).
//!
//! The index lives in a [`RamDirectory`](tantivy::directory::RamDirectory) --
//! rebuilt at startup from the scanner output and maintained incrementally
//! by [`upsert`](NoteIndex::upsert) / [`delete`](NoteIndex::delete) calls
//! driven from the filesystem watcher in [ANW-16].
use anyhow::{Context, Result};
use serde_json::json;
use tantivy::schema::{Field, IndexRecordOption, Schema, TextFieldIndexing, TextOptions};
use tantivy::{Index, IndexWriter, TantivyDocument, Term};
use crate::vault::{Note, frontmatter_to_json};
/// 50 MB heap for the writer. Tantivy's documented minimum is 15 MB; this
/// sits comfortably above that for low-thousands-of-notes vaults without
/// bloating idle memory.
const WRITER_HEAP_BYTES: usize = 50 * 1024 * 1024;
/// Strongly-typed handle to the two schema fields. Carried alongside the
/// [`Index`] so call sites don't restring field names.
#[derive(Debug, Clone, Copy)]
pub struct Fields {
pub path: Field,
pub frontmatter: Field,
}
/// The in-memory note index. Owns the [`Index`] and a long-lived
/// [`IndexWriter`]; one `NoteIndex` per daemon.
pub struct NoteIndex {
fields: Fields,
index: Index,
writer: IndexWriter,
}
impl NoteIndex {
/// Construct an empty in-memory index with the pinned v1 schema.
///
/// # Errors
/// Returns the underlying Tantivy error if writer allocation fails (the
/// only failure mode reachable for a fresh `RamDirectory`).
pub fn new() -> Result<Self> {
let mut schema_builder = Schema::builder();
let raw_indexing = TextFieldIndexing::default()
.set_tokenizer("raw")
.set_index_option(IndexRecordOption::Basic);
let path_options = TextOptions::default()
.set_indexing_options(raw_indexing.clone())
.set_stored();
let path = schema_builder.add_text_field("path", path_options);
// JSON field: open-schema frontmatter. `raw` tokenizer pins exact-match
// semantics on every key, per the open question closed in
// [[ADR-002 Tantivy as Frontmatter Index]].
let json_options = tantivy::schema::JsonObjectOptions::default()
.set_indexing_options(raw_indexing)
.set_stored();
let frontmatter = schema_builder.add_json_field("frontmatter", json_options);
let schema = schema_builder.build();
let index = Index::create_in_ram(schema);
let writer: IndexWriter = index
.writer(WRITER_HEAP_BYTES)
.context("create tantivy writer")?;
Ok(Self {
fields: Fields { path, frontmatter },
index,
writer,
})
}
/// Discard every document and reindex the given notes. Used at startup
/// and during overflow recovery per [[ADR-003 Filesystem Change Tracking]].
///
/// # Errors
/// Returns the underlying Tantivy error if any add/commit fails.
pub fn rebuild(&mut self, notes: &[Note]) -> Result<()> {
self.writer
.delete_all_documents()
.context("delete_all_documents")?;
for note in notes {
self.add_document(note)?;
}
self.writer.commit().context("commit rebuild")?;
Ok(())
}
/// Insert-or-replace one note by path. Subsequent reads through a fresh
/// reader will see the new content after this call returns.
///
/// # Errors
/// Returns the underlying Tantivy error if the add or commit fails.
pub fn upsert(&mut self, note: &Note) -> Result<()> {
self.writer.delete_term(self.path_term(&note.path));
self.add_document(note)?;
self.writer.commit().context("commit upsert")?;
Ok(())
}
/// Drop one note by path. No-op (still commits) if the path was never
/// indexed; callers can call freely on delete-events.
///
/// # Errors
/// Returns the underlying Tantivy error if the commit fails.
pub fn delete(&mut self, path: &str) -> Result<()> {
self.writer.delete_term(self.path_term(path));
self.writer.commit().context("commit delete")?;
Ok(())
}
/// Apply a batch of upserts and deletes in one Tantivy commit. The
/// watcher's debouncer dispatches batches per debounce window so a busy
/// editor save loop does not produce one commit per event.
///
/// # Errors
/// Returns the underlying Tantivy error if any add or the final commit
/// fails.
pub fn apply_batch(&mut self, upserts: &[Note], deletes: &[String]) -> Result<()> {
for path in deletes {
self.writer.delete_term(self.path_term(path));
}
for note in upserts {
self.writer.delete_term(self.path_term(&note.path));
self.add_document(note)?;
}
self.writer.commit().context("commit batch")?;
Ok(())
}
/// Number of indexed documents from a fresh reader. Read-only; intended
/// for tests and the `/health` endpoint ([ANW-8]).
///
/// # Errors
/// Returns the underlying Tantivy error if opening the reader fails.
pub fn document_count(&self) -> Result<u64> {
let reader = self.index.reader().context("open reader")?;
let searcher = reader.searcher();
Ok(searcher.num_docs())
}
/// Expose the [`Index`] so other layers (the future query handler in
/// [ANW-15]) can build their own readers and parsers without re-deriving
/// the schema.
#[must_use]
pub fn index(&self) -> &Index {
&self.index
}
#[must_use]
pub fn fields(&self) -> Fields {
self.fields
}
fn path_term(&self, path: &str) -> Term {
Term::from_field_text(self.fields.path, path)
}
fn add_document(&mut self, note: &Note) -> Result<()> {
// Build the doc as JSON keyed by schema field name and let Tantivy
// parse it -- avoids restating the OwnedValue tree by hand.
let doc_json = json!({
"path": &note.path,
"frontmatter": frontmatter_to_json(&note.frontmatter),
});
let doc = TantivyDocument::parse_json(&self.index.schema(), &doc_json.to_string())
.context("parse_json")?;
self.writer.add_document(doc).context("add_document")?;
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::{TimeZone, Utc};
use std::collections::BTreeMap;
use tantivy::collector::TopDocs;
use tantivy::query::TermQuery;
use crate::vault::Value;
fn sample_note(path: &str, tag: &str) -> Note {
let mut fm: BTreeMap<String, Value> = BTreeMap::new();
fm.insert(
"tags".into(),
Value::Sequence(vec![Value::String(tag.into())]),
);
fm.insert("title".into(), Value::String(format!("note {path}")));
Note {
path: path.into(),
frontmatter: fm,
body: String::new(),
raw_bytes: Vec::new(),
last_modified: Utc.with_ymd_and_hms(2026, 5, 14, 12, 0, 0).unwrap(),
etag: "\"deadbeef\"".into(),
size: 0,
}
}
fn count_path(idx: &NoteIndex, path: &str) -> tantivy::Result<usize> {
let reader = idx.index().reader()?;
let searcher = reader.searcher();
let term = Term::from_field_text(idx.fields().path, path);
let query = TermQuery::new(term, IndexRecordOption::Basic);
let hits = searcher.search(&query, &TopDocs::with_limit(10))?;
Ok(hits.len())
}
#[test]
fn rebuild_populates_index() {
let mut idx = NoteIndex::new().unwrap();
idx.rebuild(&[sample_note("a.md", "x"), sample_note("b.md", "y")])
.unwrap();
assert_eq!(idx.document_count().unwrap(), 2);
assert_eq!(count_path(&idx, "a.md").unwrap(), 1);
assert_eq!(count_path(&idx, "b.md").unwrap(), 1);
}
#[test]
fn rebuild_replaces_previous_contents() {
let mut idx = NoteIndex::new().unwrap();
idx.rebuild(&[sample_note("old.md", "x")]).unwrap();
idx.rebuild(&[sample_note("new.md", "y")]).unwrap();
assert_eq!(idx.document_count().unwrap(), 1);
assert_eq!(count_path(&idx, "old.md").unwrap(), 0);
assert_eq!(count_path(&idx, "new.md").unwrap(), 1);
}
#[test]
fn upsert_replaces_by_path() {
let mut idx = NoteIndex::new().unwrap();
idx.upsert(&sample_note("a.md", "v1")).unwrap();
// Mutate then upsert under the same path; document count must stay 1.
idx.upsert(&sample_note("a.md", "v2")).unwrap();
assert_eq!(idx.document_count().unwrap(), 1);
assert_eq!(count_path(&idx, "a.md").unwrap(), 1);
}
#[test]
fn delete_removes_path() {
let mut idx = NoteIndex::new().unwrap();
idx.upsert(&sample_note("a.md", "x")).unwrap();
idx.upsert(&sample_note("b.md", "y")).unwrap();
idx.delete("a.md").unwrap();
assert_eq!(idx.document_count().unwrap(), 1);
assert_eq!(count_path(&idx, "a.md").unwrap(), 0);
assert_eq!(count_path(&idx, "b.md").unwrap(), 1);
}
#[test]
fn delete_unknown_path_is_noop() {
let mut idx = NoteIndex::new().unwrap();
idx.upsert(&sample_note("a.md", "x")).unwrap();
idx.delete("never-indexed.md").unwrap();
assert_eq!(idx.document_count().unwrap(), 1);
}
}

7
src/lib.rs
View file

@ -1,14 +1,15 @@
//! Anwesen: read-only HTTP daemon over a markdown vault.
//!
//! The binary in `main.rs` wires this library to a CLI; submodules implement
//! the vault scanner, the Tantivy index, the filesystem watcher, the
//! supervisor tree, and the HTTP surface as those issues land.
//! the vault scanner, the in-memory note store, the filesystem watcher, the
//! supervisor tree, and the HTTP surface. See
//! [[ADR-009 Reverse ADR-002 In-Memory Evaluation No Tantivy]] for why v1
//! ships without a search index.
pub mod app;
pub mod doctor;
pub mod health;
pub mod http;
pub mod index;
pub mod query;
pub mod store;
pub mod vault;

12
src/query.rs
View file

@ -10,13 +10,11 @@
//! Different field predicates AND together; multiple values under
//! `__in` / `__all` are comma-separated; unknown operators are `400`.
//!
//! v1 evaluates predicates by iterating the in-memory [`NoteStore`] and
//! applying each [`Predicate::matches`] in turn. The Tantivy index from
//! [[ADR-002 Tantivy as Frontmatter Index]] is still built and maintained,
//! but query-time filtering doesn't use it yet. At the documented scale
//! (low-thousands-of-notes vaults) this is sub-millisecond; migration to a
//! Tantivy-driven candidate set is a future optimization once a real
//! consumer pushes throughput.
//! Predicates are evaluated by iterating the in-memory [`NoteStore`] and
//! applying each [`Predicate::matches`] in turn. At the documented scale
//! (low-thousands-of-notes vaults) this is sub-millisecond; see
//! [[ADR-009 Reverse ADR-002 In-Memory Evaluation No Tantivy]] for the
//! call to keep evaluation in-memory rather than carrying a Tantivy index.
use chrono::{DateTime, NaiveDate};
use regex::Regex;

8
src/store.rs
View file

@ -3,10 +3,10 @@
//! HTTP layer ([ANW-13] / [ANW-14] / [ANW-15]) for read-one and listing
//! responses.
//!
//! Per [[ADR-002 Tantivy as Frontmatter Index]] / [ANW-11], the daemon
//! rereads from disk only on a watcher event for that path. The store is
//! that in-memory cache; the index in [`crate::index`] sits beside it,
//! shaped for frontmatter queries rather than full-record retrieval.
//! Per [ANW-11], the daemon rereads from disk only on a watcher event for
//! that path; this store is that in-memory cache and -- per
//! [[ADR-009 Reverse ADR-002 In-Memory Evaluation No Tantivy]] -- the sole
//! authoritative record set both `/notes` and `/query` read from.
use std::collections::BTreeMap;
use std::sync::{Arc, RwLock};

2
src/vault.rs
View file

@ -99,7 +99,7 @@ impl Value {
}
/// Convert a whole [`Frontmatter`] tree to a [`serde_json::Value`] object
/// suitable for HTTP responses or for Tantivy ingestion.
/// suitable for HTTP responses.
#[must_use]
pub fn frontmatter_to_json(fm: &Frontmatter) -> JsonValue {
let mut map = JsonMap::new();

3
src/watcher.rs
View file

@ -5,8 +5,7 @@
//! pushed into a Tokio channel; [`run_debouncer`] drains the channel,
//! classifies events into [`WatchAction`]s, coalesces a 100 ms window's
//! worth into one [`Batch`], and casts the batch to the `index_writer`
//! named process for a single Tantivy commit (folds in sie's ANW-12
//! follow-up #3).
//! named process for a single [`crate::store::NoteStore`] write.
//!
//! See [[ADR-003 Filesystem Change Tracking]] for the event model.