Query it

One JSON shape for every kind of search. A query is a dict (or a plain string, which becomes a substring search over every text field), and the answer carries the score and the exact byte spans that matched.

index.search({"type": "contains", "field": "body", "value": "mutex"}, highlights=True)
index.search({"type": "contains", "field": "body", "value": "mutx", "distance": 1})   # fuzzy
index.search({"type": "fuzzy", "field": "body", "value": "mutx",
              "fuzzy_metric": "jaro_winkler", "min_similarity": 0.9})
index.search({"type": "contains", "field": "body", "value": "lock.*mutex", "regex": True})
index.search({"type": "phrase", "field": "body", "value": "return -ENOMEM"})
index.search({"type": "startsWith", "field": "body", "value": "pthread"})
index.search({"type": "term", "field": "body", "value": "lock"})       # whole words only
index.search({"type": "parse", "fields": ["body", "path"],
              "value": 'mutex AND NOT guard OR "exact phrase"'})

# What the engine will really do, before it does it
index.query_warnings({"type": "contains", "field": "body", "value": "__init"})
# ['separators are ignored (strict_separators=false): "__init" is searched as "init"']

index.search({ type: 'contains', field: 'body', value: 'mutex' }, { highlights: true });
index.search({ type: 'contains', field: 'body', value: 'mutx', distance: 1 });   // fuzzy
index.search({ type: 'fuzzy', field: 'body', value: 'mutx',
               fuzzyMetric: 'jaro_winkler', minSimilarity: 0.9 });
index.search({ type: 'contains', field: 'body', value: 'lock.*mutex', regex: true });
index.search({ type: 'phrase', field: 'body', value: 'return -ENOMEM' });
index.search({ type: 'startsWith', field: 'body', value: 'pthread' });
index.search({ type: 'term', field: 'body', value: 'lock' });          // whole words only
index.search({ type: 'parse', fields: ['body', 'path'],
               value: 'mutex AND NOT guard OR "exact phrase"' });

// What the engine will really do, before it does it
index.queryWarnings({ type: 'contains', field: 'body', value: '__init' });

use lucivy_core::query::QueryConfig;
use serde_json::json;

let q: QueryConfig = serde_json::from_value(json!({
    "type": "contains", "field": "body", "value": "mutex"
}))?;
let sink = std::sync::Arc::new(ld_lucivy::query::HighlightSink::new());
let hits = index.search(&q, 10, Some(sink.clone()))?;    // spans land in the sink

// Every other kind is the same call with a different config
json!({"type": "contains", "field": "body", "value": "mutx", "distance": 1});
json!({"type": "fuzzy", "field": "body", "value": "mutx",
       "fuzzy_metric": "jaro_winkler", "min_similarity": 0.9});
json!({"type": "contains", "field": "body", "value": "lock.*mutex", "regex": true});
json!({"type": "parse", "fields": ["body", "path"], "value": "mutex AND NOT guard"});

// What the engine will really do, before it does it
index.query_warnings(&q);

Filter it

Two kinds: restrict to a set of your own ids, or compose non-text conditions with the text query.

# A pre-filter by your ids — the engine only visits those documents
index.search({"type": "contains", "field": "body", "value": "kmalloc"},
             allowed_ids=[3, 7, 11, 12])

# Non-text conditions, composed with must / should / must_not
index.search({"type": "boolean",
              "must": [{"type": "contains", "field": "body", "value": "error"},
                       {"type": "filter", "op": "gte", "field": "priority", "value": 3}],
              "must_not": [{"type": "term", "field": "state", "value": "closed"}]})

// A pre-filter by your ids — the engine only visits those documents
index.search({ type: 'contains', field: 'body', value: 'kmalloc' },
             { allowedIds: [3, 7, 11, 12] });

// Non-text conditions, composed with must / should / must_not
index.search({ type: 'boolean',
               must: [{ type: 'contains', field: 'body', value: 'error' },
                      { type: 'filter', op: 'gte', field: 'priority', value: 3 }],
               mustNot: [{ type: 'term', field: 'state', value: 'closed' }] });

// A pre-filter by your ids — the engine only visits those documents
let allowed: HashSet<u64> = [3, 7, 11, 12].into_iter().collect();
index.search_filtered(&q, 10, None, allowed)?;

// Non-text conditions, composed with must / should / must_not
let q: QueryConfig = serde_json::from_value(json!({
    "type": "boolean",
    "must": [{"type": "contains", "field": "body", "value": "error"},
             {"type": "filter", "op": "gte", "field": "priority", "value": 3}],
    "must_not": [{"type": "term", "field": "state", "value": "closed"}]
}))?;

Shard it

One number at creation. Indexing and searching then run across shards in parallel, and the scores do not move.

index = lucivy.Index.create("/tmp/idx", fields=[...], shards=4)
index.add(1, body="...")            # the router decides where it lands
index.commit()
index.search({"type": "contains", "field": "body", "value": "mutex"})
index.compact(10_000)               # merge down to segments of at most N documents
index.wait_merges_quiet()           # let the background merges finish

const index = Index.create('/tmp/idx', [{ name: 'body', type: 'text' }], 4);
index.add(1, { body: '...' });      // the router decides where it lands
index.commit();
index.search({ type: 'contains', field: 'body', value: 'mutex' });
index.compact(10000);               // merge down to segments of at most N documents
index.waitMergesQuiet();            // let the background merges finish

use lucivy_core::sharded_handle::ShardedHandle;
use lucivy_core::query::SchemaConfig;

let config: SchemaConfig = serde_json::from_value(json!({
    "fields": [{ "name": "body", "type": "text", "stored": true }],
    "shards": 4,
}))?;
let index = ShardedHandle::create("/tmp/idx", &config)?;
index.add_document_json(1, &json!({ "body": "..." }))?;   // the router places it
index.commit()?;
index.search(&q, 10, None)?;
index.compact(10_000)?;
index.wait_merges_quiet()?;

Distribute it

Several machines, each with its own index, searched as one — without copying or mounting anything. Two round trips of small JSON, no data movement: what crosses the network is statistics, never documents.

#  ── on every node (a process, a machine) ──────────────────────────
# Each node serves its own index and answers two calls over your transport.

def handle_export_stats(query: dict) -> str:
    return index.export_stats(query)          # a JSON string, a few kB

def handle_search(query: dict, merged_stats: str, allowed=None) -> list:
    return index.search_with_global_stats(query, merged_stats, limit=10,
                                          allowed_ids=allowed)

#  ── on the coordinator ────────────────────────────────────────────
# Round 1: ask every node what it knows about this query.
stats = [post(node, "/export_stats", query) for node in NODES]   # parallel in practice
merged = lucivy.merge_stats(stats)            # one JSON string, sent back to all

# Round 2: everyone searches under the same statistics, and returns its top-k.
pages = [post(node, "/search", {"query": query, "stats": merged}) for node in NODES]

# Round 3 is local: merge the top-k lists by score. Nothing else to do —
# the scores are already comparable, which is the whole point.
hits = sorted(sum(pages, []), key=lambda h: -h["score"])[:10]

#  ── restricted to what this caller may see ────────────────────────
# The ids come from your permissions, your tenant, your agent's domain —
# and they are spread over the nodes. Send them along; each node visits
# only those documents, and still scores under the federation's statistics.
allowed = acl.visible_ids(user)               # e.g. [17, 44, 802, ...]
pages = [post(node, "/search", {"query": query, "stats": merged, "allowed": allowed})
         for node in NODES]

# A document keeps the rank it would have had for anyone else: the filter
# says what is *visited*, the statistics say how it *scores*.

//  ── on every node (a process, a machine) ─────────────────────────
app.post('/export_stats', (req, res) =>
  res.send(index.exportStats(JSON.stringify(req.body.query))));   // a JSON string

app.post('/search', (req, res) =>
  res.json(index.searchWithGlobalStats(JSON.stringify(req.body.query),
                                       req.body.stats, 10, false, req.body.allowedIds)));

//  ── on the coordinator ───────────────────────────────────────────
// Round 1: ask every node what it knows about this query.
const stats = await Promise.all(NODES.map(n => post(n, '/export_stats', { query })));
const merged = mergeStats(stats);            // one JSON string, sent back to all

// Round 2: everyone searches under the same statistics.
const pages = await Promise.all(NODES.map(n => post(n, '/search', { query, stats: merged })));

// Round 3 is local: merge the top-k lists by score, and stop there.
const hits = pages.flat().sort((a, b) => b.score - a.score).slice(0, 10);

//  ── restricted to what this caller may see ───────────────────────
// The ids come from your permissions, your tenant, your agent's domain,
// and they are spread over the nodes. Each node visits only those
// documents, and still scores under the federation's statistics.
const allowedIds = acl.visibleIds(user);     // e.g. [17, 44, 802, ...]
const pages2 = await Promise.all(NODES.map(n =>
  post(n, '/search', { query, stats: merged, allowedIds })));

// A document keeps the rank it would have had for anyone else: the filter
// says what is visited, the statistics say how it scores.

use lucivy_core::bm25_global::ExportableStats;

//  ── on every node (a process, a machine) ─────────────────────────
async fn export_stats(index: &ShardedHandle, q: &QueryConfig) -> String {
    serde_json::to_string(&index.export_stats(q).unwrap()).unwrap()
}

async fn search(index: &ShardedHandle, q: &QueryConfig, stats: &str) -> Vec<ShardedSearchResult> {
    let merged: ExportableStats = serde_json::from_str(stats).unwrap();
    index.search_with_global_stats(q, 10, &merged, None).unwrap()
}

//  ── on the coordinator ───────────────────────────────────────────
// Round 1: ask every node, in parallel, what it knows about this query.
let per_node: Vec<ExportableStats> = join_all(nodes.iter().map(|n| n.export_stats(&q))).await;
let merged = ExportableStats::merge(&per_node);

// Round 2: everyone searches under the same statistics.
let pages = join_all(nodes.iter().map(|n| n.search(&q, &merged))).await;

// Round 3 is local: merge the top-k lists by score.
let mut hits: Vec<_> = pages.into_iter().flatten().collect();
hits.sort_by(|a, b| b.score.total_cmp(&a.score));
hits.truncate(10);

//  ── restricted to what this caller may see ───────────────────────
// On the node: the ids come from your permissions, your tenant, your
// agent's domain. Only those documents are visited; the scoring is still
// the federation's.
let allowed: HashSet<u64> = acl.visible_ids(user);
index.search_filtered_with_global_stats(&q, 10, &merged, None, allowed)?;

Bring your own storage

The index does not have to live on a filesystem. Give it an object with five methods and your database becomes the truth — the local files become a disposable cache.

class MyStore:                       # Postgres, S3, anything transactional
    def load(self, index_name, file_name): ...
    def save(self, index_name, file_name, data): ...
    def delete(self, index_name, file_name): ...
    def exists(self, index_name, file_name): ...
    def list(self, index_name): ...
    # optional: blob_len / load_range, which make `lazy=True` worth it

index = lucivy.Index.create_with_blob_store(MyStore(), "products", fields=[...], shards=4)
index = lucivy.Index.open_with_blob_store(MyStore(), "products", lazy=True)

const { BlobIndex } = require('lucivy');

const store = {                      // Postgres, S3, anything transactional
  load: (indexName, fileName) => buffer,
  save: (indexName, fileName, data) => {},
  delete: (indexName, fileName) => {},
  exists: (indexName, fileName) => true,
  list: (indexName) => [names],
  // optional: blobLen / loadRange, which make `lazy` worth it
};

// The blob-backed index is asynchronous: its calls cross into your callbacks
const index = await BlobIndex.create(store, 'products',
                                     [{ name: 'title', type: 'text', stored: true }],
                                     { shards: 4 });
const index = await BlobIndex.open(store, 'products', { lazy: true });

use lucivy_core::blob_store::BlobStore;
use lucivy_core::sharded_handle::{BlobShardStorage, ShardedHandle};

// Your type implements five methods (plus two optional ones for `lazy`)
impl BlobStore for MyStore { /* load, save, delete, exists, list */ }

let storage = BlobShardStorage::new(Arc::new(MyStore), "products", None);
let index = ShardedHandle::create_with_storage(Box::new(storage), &config)?;

// A filesystem index is the same call with the other storage
let storage = lucistore::shard_storage::FsShardStorage::new("/tmp/idx")?;
let index = ShardedHandle::create_with_storage(Box::new(storage), &config)?;

One transaction for your data and its index

The reason the storage is pluggable: if your save writes inside your transaction, then committing your rows and committing the index are the same commit — and rolling back takes the index with it. No dual-write, no reindex queue, no drift between a row and its searchable form.

class PgStore:
    """Every index file is a row. The connection is the caller's, so the
    writes land in whatever transaction is open on it."""
    def __init__(self, conn): self.conn = conn
    def save(self, index_name, file_name, data):
        self.conn.execute(
            "INSERT INTO lucivy_files (idx, name, data) VALUES (%s, %s, %s) "
            "ON CONFLICT (idx, name) DO UPDATE SET data = EXCLUDED.data",
            (index_name, file_name, data))
    def load(self, index_name, file_name):
        return self.conn.execute("SELECT data FROM lucivy_files WHERE idx=%s AND name=%s",
                                 (index_name, file_name)).fetchone()[0]
    def delete(self, index_name, file_name): ...
    def exists(self, index_name, file_name): ...
    def list(self, index_name): ...

index = lucivy.Index.open_with_blob_store(PgStore(conn), "products")

# ── The pattern ───────────────────────────────────────────────────
with conn.transaction():                     # BEGIN
    conn.execute("INSERT INTO products (id, title) VALUES (%s, %s)", (id, title))
    index.add(id, title=title)               # RAM only, nothing written yet
    index.commit()                           # the index's files, through PgStore, INSIDE the transaction
# COMMIT — the row and its searchable form become visible together.
# An exception here rolls both back: no orphan document, no stale index.

index.close()                                # before the connection goes away — see below

const { BlobIndex } = require('lucivy');

// Every index file is a row; the client is the caller's, so the writes land
// in whatever transaction is open on it.
const store = {
  save: (idx, name, data) => client.query(
    `INSERT INTO lucivy_files (idx, name, data) VALUES ($1, $2, $3)
     ON CONFLICT (idx, name) DO UPDATE SET data = EXCLUDED.data`, [idx, name, data]),
  load: (idx, name) => client.query(
    'SELECT data FROM lucivy_files WHERE idx=$1 AND name=$2', [idx, name]).rows[0].data,
  delete: (idx, name) => { /* ... */ },
  exists: (idx, name) => true,
  list: (idx) => [],
};

const index = await BlobIndex.open(store, 'products');

// ── The pattern ──────────────────────────────────────────────────
await client.query('BEGIN');
try {
  await client.query('INSERT INTO products (id, title) VALUES ($1, $2)', [id, title]);
  await index.add(id, { title });          // RAM only, nothing written yet
  await index.commit();                    // the index's files, through the store, inside the transaction
  await client.query('COMMIT');             // row and searchable form become visible together
} catch (e) {
  await client.query('ROLLBACK');           // and the index goes back with it
  throw e;
}
await index.close();                        // before the client goes away — see below

use lucivy_core::blob_store::BlobStore;
use lucivy_core::sharded_handle::{BlobShardStorage, ShardedHandle};

// Every index file is a row. The transaction is the caller's: the store
// holds it, so the writes land wherever it is open.
struct PgStore { tx: Mutex<postgres::Client> }

impl BlobStore for PgStore {
    fn save(&self, index_name: &str, file: &str, data: &[u8]) -> io::Result<()> {
        self.tx.lock().unwrap().execute(
            "INSERT INTO lucivy_files (idx, name, data) VALUES ($1, $2, $3) \
             ON CONFLICT (idx, name) DO UPDATE SET data = EXCLUDED.data",
            &[&index_name, &file, &data])?;
        Ok(())
    }
    // load, delete, exists, list — and optionally blob_len / load_range for `lazy`
}

let storage = BlobShardStorage::new(Arc::new(PgStore { tx }), "products", None);
let index = ShardedHandle::create_with_storage(Box::new(storage), &config)?;

// ── The pattern ──────────────────────────────────────────────────
let mut tx = client.transaction()?;                          // BEGIN
tx.execute("INSERT INTO products (id, title) VALUES ($1, $2)", &[&id, &title])?;
index.add_document_json(id, &json!({ "title": title }))?;    // RAM only
index.commit()?;                                             // files written through PgStore, inside tx
tx.commit()?;                                                // both become visible together

index.close()?;                                              // before the connection goes away

Snapshot and sync

Move an index as one blob, or ship only what changed.

blob = index.export_snapshot()                # LUCE: every shard, one blob
served = lucivy.Index.open_snapshot(blob)     # read-only, nothing extracted

# Incremental: the client says what it has, the server sends the difference
delta = server.export_sharded_delta(client.shard_versions())   # LUCIDS
client.apply_sharded_delta(delta)

const blob = index.exportSnapshot();            // LUCE: every shard, one blob
const served = Index.openSnapshot(blob);        // read-only, nothing extracted

// Incremental: the client says what it has, the server sends the difference
const delta = server.exportShardedDelta(client.shardVersions());   // LUCIDS
client.applyShardedDelta(delta);

use lucivy_core::snapshot::{export_to_snapshot, import_from_snapshot};

let blob = export_to_snapshot(&index, Path::new("/tmp/idx"))?;   // LUCE
let served = ShardedHandle::open_snapshot(OwnedBytes::new(blob))?;  // served in place

// Incremental: the client says what it has, the server sends the difference
let delta = server.export_sharded_delta(&client.shard_versions()?)?;   // LUCIDS
client.apply_sharded_delta("/tmp/client_idx", &delta)?;

In the browser JavaScript only

The same engine, compiled to WebAssembly, in a Web Worker with real threads — so this one has no Python or Rust form. The demo at the top of this page is this code.

import { Lucivy } from 'lucivy-wasm';

const lucivy = new Lucivy('./lucivy-worker.js');   // pthreads + OPFS
await lucivy.ready;
const index = await lucivy.create('/idx', { fields: [{ name: 'body', type: 'text' }], shards: 4 });
await index.add(1, { body: 'pthread_mutex_lock acquires a mutex' });
await index.commit();
await index.preload();                             // hold it in memory, once
await index.search({ type: 'contains', field: 'body', value: 'mutex' }, { highlights: true });

const status = await index.memoryStatus();
// { index_bytes, in_memory, num_docs, warnings, last_search_truncated }

Sparse vectors Rust only

The other half of a hybrid search: an inverted index for the sparse vectors a model produces (SPLADE, BGE-M3), with WAND pruning, sharding and the same storage. It indexes vectors — it does not produce them. Not exposed in the bindings yet: it is a Rust crate, sparse-vector.

use sparse_vector::{handle::SparseHandle, index::SparseVector};

let index = SparseHandle::create("/tmp/sparse")?;
index.insert(1, &SparseVector { indices: vec![7, 42, 1990], values: vec![0.31, 0.12, 0.08] })?;
index.commit_inner()?;

let hits = index.search(&query_vector, 10);            // (id, score), best first
let hits = index.search_filtered(&query_vector, 10, &allowed_ids);
index.compact()?;                                       // merge the segments