WIP 1: implement lancedb vector

This commit is contained in:
Maxime Van Hees
2025-09-29 11:24:31 +02:00
parent 77a53bae86
commit 6a4e2819bf
9 changed files with 5575 additions and 93 deletions

View File

@@ -26,6 +26,9 @@ pub struct Server {
// In-memory registry of Tantivy search indexes for this server
pub search_indexes: Arc<std::sync::RwLock<HashMap<String, Arc<crate::tantivy_search::TantivySearch>>>>,
// Per-DB Lance stores (vector DB), keyed by db_id
pub lance_stores: Arc<std::sync::RwLock<HashMap<u64, Arc<crate::lance_store::LanceStore>>>>,
// BLPOP waiter registry: per (db_index, key) FIFO of waiters
pub list_waiters: Arc<Mutex<HashMap<u64, HashMap<String, Vec<Waiter>>>>>,
pub waiter_seq: Arc<AtomicU64>,
@@ -54,6 +57,7 @@ impl Server {
current_permissions: None,
search_indexes: Arc::new(std::sync::RwLock::new(HashMap::new())),
lance_stores: Arc::new(std::sync::RwLock::new(HashMap::new())),
list_waiters: Arc::new(Mutex::new(HashMap::new())),
waiter_seq: Arc::new(AtomicU64::new(1)),
}
@@ -71,6 +75,18 @@ impl Server {
base
}
// Path where Lance datasets are stored, namespaced per selected DB:
// <base_dir>/lance/<db_id>
pub fn lance_data_path(&self) -> std::path::PathBuf {
let base = std::path::PathBuf::from(&self.option.dir)
.join("lance")
.join(self.selected_db.to_string());
if !base.exists() {
let _ = std::fs::create_dir_all(&base);
}
base
}
pub fn current_storage(&self) -> Result<Arc<dyn StorageBackend>, DBError> {
let mut cache = self.db_cache.write().unwrap();
@@ -99,7 +115,43 @@ impl Server {
cache.insert(self.selected_db, storage.clone());
Ok(storage)
}
/// Get or create the LanceStore for the currently selected DB.
/// Only valid for non-zero DBs and when the backend is Lance.
pub fn lance_store(&self) -> Result<Arc<crate::lance_store::LanceStore>, DBError> {
if self.selected_db == 0 {
return Err(DBError("Lance not available on admin DB 0".to_string()));
}
// Resolve backend for selected_db
let backend_opt = crate::admin_meta::get_database_backend(
&self.option.dir,
self.option.backend.clone(),
&self.option.admin_secret,
self.selected_db,
)
.ok()
.flatten();
if !matches!(backend_opt, Some(crate::options::BackendType::Lance)) {
return Err(DBError("ERR DB backend is not Lance; LANCE.* commands are not allowed".to_string()));
}
// Fast path: read lock
{
let map = self.lance_stores.read().unwrap();
if let Some(store) = map.get(&self.selected_db) {
return Ok(store.clone());
}
}
// Slow path: create and insert
let store = Arc::new(crate::lance_store::LanceStore::new(&self.option.dir, self.selected_db)?);
{
let mut map = self.lance_stores.write().unwrap();
map.insert(self.selected_db, store.clone());
}
Ok(store)
}
/// Check if current permissions allow read operations
pub fn has_read_permission(&self) -> bool {