diff --git a/herodb/Cargo.lock b/herodb/Cargo.lock index 3e329fd..d95245a 100644 --- a/herodb/Cargo.lock +++ b/herodb/Cargo.lock @@ -623,6 +623,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "tst", "uuid", ] @@ -1704,6 +1705,14 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tst" +version = "0.1.0" +dependencies = [ + "ourdb", + "thiserror", +] + [[package]] name = "typenum" version = "1.18.0" diff --git a/herodb/Cargo.toml b/herodb/Cargo.toml index 7a53d9a..d165a75 100644 --- a/herodb/Cargo.toml +++ b/herodb/Cargo.toml @@ -8,6 +8,7 @@ authors = ["HeroCode Team"] [dependencies] ourdb = { path = "../ourdb" } +tst = { path = "../tst" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" thiserror = "1.0" diff --git a/herodb/src/db/db.rs b/herodb/src/db/db.rs index bdf1dfa..9267f04 100644 --- a/herodb/src/db/db.rs +++ b/herodb/src/db/db.rs @@ -1,6 +1,7 @@ use crate::db::error::{DbError, DbResult}; use crate::db::model::Model; use crate::db::store::{DbOperations, OurDbStore}; +use crate::db::tst_index::TSTIndexManager; use std::any::TypeId; use std::collections::HashMap; use std::fmt::Debug; @@ -14,14 +15,18 @@ enum DbOperation { Set { model_type: TypeId, serialized: Vec, + model_prefix: String, // Add model prefix + model_id: u32, // Add model ID }, Delete { model_type: TypeId, id: u32, + model_prefix: String, // Add model prefix }, } /// Transaction state for DB operations +#[derive(Clone)] pub struct TransactionState { operations: Vec, active: bool, @@ -45,6 +50,9 @@ pub struct DB { // Type map for generic operations type_map: HashMap>>, + // TST index manager + tst_index: Arc>, + // Transaction state transaction: Arc>>, } @@ -125,11 +133,17 @@ impl DBBuilder { type_map.insert(type_id, store); } + // Create the TST index manager + let tst_index = TSTIndexManager::new(&base_path).map_err(|e| { + EvalAltResult::ErrorSystem("Could not create TST index manager".to_string(), Box::new(e)) + })?; + let transaction = Arc::new(RwLock::new(None)); Ok(DB { db_path: base_path, type_map, + tst_index: Arc::new(RwLock::new(tst_index)), transaction, }) } @@ -145,11 +159,15 @@ impl DB { std::fs::create_dir_all(&base_path)?; } + // Create the TST index manager + let tst_index = TSTIndexManager::new(&base_path)?; + let transaction = Arc::new(RwLock::new(None)); Ok(Self { db_path: base_path, type_map: HashMap::new(), + tst_index: Arc::new(RwLock::new(tst_index)), transaction, }) } @@ -198,24 +216,52 @@ impl DB { return Err(DbError::TransactionError("Transaction not active".into())); } - // Execute all operations in the transaction - for op in tx_state.operations { - match op { - DbOperation::Set { - model_type, - serialized, - } => { - self.apply_set_operation(model_type, &serialized)?; - } - DbOperation::Delete { model_type, id } => { - let db_ops = self - .type_map - .get(&model_type) - .ok_or_else(|| DbError::TypeError)?; - let mut db_ops_guard = db_ops.write().unwrap(); - db_ops_guard.delete(id)?; + // Create a backup of the transaction state in case we need to rollback + let backup = tx_state.clone(); + + // Try to execute all operations + let result = (|| { + for op in tx_state.operations { + match op { + DbOperation::Set { + model_type, + serialized, + model_prefix, + model_id, + } => { + // Apply to OurDB + self.apply_set_operation(model_type, &serialized)?; + + // Apply to TST index + let mut tst_index = self.tst_index.write().unwrap(); + tst_index.set(&model_prefix, model_id, serialized.clone())?; + } + DbOperation::Delete { + model_type, + id, + model_prefix, + } => { + // Apply to OurDB + let db_ops = self + .type_map + .get(&model_type) + .ok_or_else(|| DbError::TypeError)?; + let mut db_ops_guard = db_ops.write().unwrap(); + db_ops_guard.delete(id)?; + + // Apply to TST index + let mut tst_index = self.tst_index.write().unwrap(); + tst_index.delete(&model_prefix, id)?; + } } } + Ok(()) + })(); + + // If any operation failed, restore the transaction state + if result.is_err() { + *tx_guard = Some(backup); + return result; } Ok(()) @@ -252,10 +298,12 @@ impl DB { // Serialize the model for later use let serialized = model.to_bytes()?; - // Record a Set operation in the transaction + // Record a Set operation in the transaction with prefix and ID tx_state.operations.push(DbOperation::Set { model_type: TypeId::of::(), serialized, + model_prefix: T::db_prefix().to_string(), + model_id: model.get_id(), }); return Ok(()); @@ -270,7 +318,16 @@ impl DB { match self.type_map.get(&TypeId::of::()) { Some(db_ops) => { let mut db_ops_guard = db_ops.write().unwrap(); - db_ops_guard.insert(model) + db_ops_guard.insert(model)?; + + // Also update the TST index + let mut tst_index = self.tst_index.write().unwrap(); + let prefix = T::db_prefix(); + let id = model.get_id(); + let data = model.to_bytes()?; + tst_index.set(prefix, id, data)?; + + Ok(()) }, None => Err(DbError::TypeError), } @@ -295,6 +352,7 @@ impl DB { DbOperation::Delete { model_type, id: op_id, + model_prefix: _, } => { if *model_type == type_id && *op_id == id { // Return NotFound error for deleted records @@ -305,14 +363,14 @@ impl DB { DbOperation::Set { model_type, serialized, + model_prefix: _, + model_id, } => { - if *model_type == type_id { - // Try to deserialize and check the ID + if *model_type == type_id && *model_id == id { + // Try to deserialize match T::from_bytes(serialized) { Ok(model) => { - if model.get_id() == id { - return Some(Ok(Some(model))); - } + return Some(Ok(Some(model))); } Err(_) => continue, // Skip if deserialization fails } @@ -360,10 +418,11 @@ impl DB { // Check if there's an active transaction if let Some(tx_state) = tx_guard.as_mut() { if tx_state.active { - // Record a Delete operation in the transaction + // Record a Delete operation in the transaction with prefix tx_state.operations.push(DbOperation::Delete { model_type: TypeId::of::(), id, + model_prefix: T::db_prefix().to_string(), }); return Ok(()); @@ -378,7 +437,14 @@ impl DB { match self.type_map.get(&TypeId::of::()) { Some(db_ops) => { let mut db_ops_guard = db_ops.write().unwrap(); - db_ops_guard.delete(id) + db_ops_guard.delete(id)?; + + // Also update the TST index + let mut tst_index = self.tst_index.write().unwrap(); + let prefix = T::db_prefix(); + tst_index.delete(prefix, id)?; + + Ok(()) }, None => Err(DbError::TypeError), } @@ -386,7 +452,25 @@ impl DB { /// List all model instances of a specific type pub fn list(&self) -> DbResult> { - // Look up the correct DB operations for type T in our type map + // Get the prefix for this model type + let prefix = T::db_prefix(); + + // Use the TST index to get all objects with this prefix + let mut tst_index = self.tst_index.write().unwrap(); + let items = tst_index.list(prefix)?; + + // Deserialize the objects + let mut result = Vec::with_capacity(items.len()); + for (_, data) in items { + let model = T::from_bytes(&data)?; + result.push(model); + } + + Ok(result) + } + + /// Helper method to list models directly from OurDB (not using TST) + fn list_from_ourdb(&self) -> DbResult> { match self.type_map.get(&TypeId::of::()) { Some(db_ops) => { let db_ops_guard = db_ops.read().unwrap(); @@ -401,6 +485,25 @@ impl DB { } } + /// Synchronize the TST index with OurDB for a specific model type + pub fn synchronize_tst_index(&self) -> DbResult<()> { + // Get all models from OurDB + let models = self.list_from_ourdb::()?; + + // Clear the TST index for this model type + let mut tst_index = self.tst_index.write().unwrap(); + let prefix = T::db_prefix(); + + // Rebuild the TST index + for model in models { + let id = model.get_id(); + let data = model.to_bytes()?; + tst_index.set(prefix, id, data)?; + } + + Ok(()) + } + /// Get the history of a model by its ID pub fn get_history(&self, id: u32, depth: u8) -> DbResult> { // Look up the correct DB operations for type T in our type map @@ -425,8 +528,17 @@ impl DB { // Register a model type with this DB instance pub fn register(&mut self) -> DbResult<()> { + // Create the OurDB store let store = OurDbStore::::open(&self.db_path)?; self.type_map.insert(TypeId::of::(), Arc::new(RwLock::new(store))); + + // Initialize the TST index for this model type + let prefix = T::db_prefix(); + let mut tst_index = self.tst_index.write().unwrap(); + + // Ensure the TST for this prefix exists + tst_index.get_tst(prefix)?; + Ok(()) } } diff --git a/herodb/src/db/mod.rs b/herodb/src/db/mod.rs index 37b8f11..57efa52 100644 --- a/herodb/src/db/mod.rs +++ b/herodb/src/db/mod.rs @@ -14,5 +14,13 @@ pub use store::{DbOperations, OurDbStore}; pub mod db; pub use db::{DB, DBBuilder, ModelRegistration, ModelRegistrar}; +// Export the TST index module +pub mod tst_index; +pub use tst_index::TSTIndexManager; + // Export macros for model methods pub mod macros; + +// Tests +#[cfg(test)] +mod tests; diff --git a/herodb/src/db/model.rs b/herodb/src/db/model.rs index a015b35..2f2de93 100644 --- a/herodb/src/db/model.rs +++ b/herodb/src/db/model.rs @@ -1,4 +1,4 @@ -use crate::db::error::{DbError, DbResult}; +...use crate::db::error::{DbError, DbResult}; use serde::{Deserialize, Serialize}; use std::fmt::Debug; @@ -15,6 +15,16 @@ pub trait Storable: Serialize + for<'de> Deserialize<'de> + Sized { } } +/// Represents an index key for a model +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IndexKey { + /// The name of the index key + pub name: &'static str, + + /// The value of the index key for a specific model instance + pub value: String, +} + /// Trait identifying a model suitable for the database /// The 'static lifetime bound is required for type identification via Any pub trait Model: Storable + Debug + Clone + Send + Sync + 'static { @@ -24,6 +34,14 @@ pub trait Model: Storable + Debug + Clone + Send + Sync + 'static { /// Returns a prefix used for this model type in the database /// Helps to logically separate different model types fn db_prefix() -> &'static str; + + /// Returns a list of index keys for this model instance + /// These keys will be used to create additional indexes in the TST + /// The default implementation returns an empty vector + /// Override this method to provide custom indexes + fn db_keys(&self) -> Vec { + Vec::new() + } } // Note: We don't provide a blanket implementation of Storable diff --git a/herodb/src/db/tests.rs b/herodb/src/db/tests.rs new file mode 100644 index 0000000..924eee6 --- /dev/null +++ b/herodb/src/db/tests.rs @@ -0,0 +1,98 @@ +use super::*; +use crate::db::model::Storable; +use serde::{Deserialize, Serialize}; +use tempfile::tempdir; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +struct TestModel { + id: u32, + name: String, +} + +impl Storable for TestModel {} + +impl Model for TestModel { + fn get_id(&self) -> u32 { + self.id + } + + fn db_prefix() -> &'static str { + "test" + } +} + +#[test] +fn test_tst_integration() { + // Create a temporary directory for the test + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path(); + + // Create a DB instance + let mut db = DB::new(path).unwrap(); + db.register::().unwrap(); + + // Create some test models + let model1 = TestModel { id: 1, name: "Test 1".to_string() }; + let model2 = TestModel { id: 2, name: "Test 2".to_string() }; + let model3 = TestModel { id: 3, name: "Test 3".to_string() }; + + // Insert the models + db.set(&model1).unwrap(); + db.set(&model2).unwrap(); + db.set(&model3).unwrap(); + + // List all models + let models = db.list::().unwrap(); + assert_eq!(models.len(), 3); + + // Verify that all models are in the list + assert!(models.contains(&model1)); + assert!(models.contains(&model2)); + assert!(models.contains(&model3)); + + // Delete a model + db.delete::(2).unwrap(); + + // List again + let models = db.list::().unwrap(); + assert_eq!(models.len(), 2); + assert!(models.contains(&model1)); + assert!(models.contains(&model3)); + assert!(!models.contains(&model2)); + + // Test transaction with commit + db.begin_transaction().unwrap(); + db.set(&model2).unwrap(); // Add back model2 + db.delete::(1).unwrap(); // Delete model1 + db.commit_transaction().unwrap(); + + // List again after transaction + let models = db.list::().unwrap(); + assert_eq!(models.len(), 2); + assert!(!models.contains(&model1)); + assert!(models.contains(&model2)); + assert!(models.contains(&model3)); + + // Test transaction with rollback + db.begin_transaction().unwrap(); + db.delete::(3).unwrap(); // Delete model3 + db.rollback_transaction().unwrap(); + + // List again after rollback + let models = db.list::().unwrap(); + assert_eq!(models.len(), 2); + assert!(!models.contains(&model1)); + assert!(models.contains(&model2)); + assert!(models.contains(&model3)); + + // Test the synchronize_tst_index method + // Since we can't directly access private fields, we'll just verify that + // the method runs without errors + db.synchronize_tst_index::().unwrap(); + + // Verify that our models are still accessible + let models = db.list::().unwrap(); + assert_eq!(models.len(), 2); + assert!(models.contains(&model2)); + assert!(models.contains(&model3)); +} \ No newline at end of file diff --git a/herodb/src/db/tst_index.rs b/herodb/src/db/tst_index.rs new file mode 100644 index 0000000..b8272f1 --- /dev/null +++ b/herodb/src/db/tst_index.rs @@ -0,0 +1,156 @@ +use crate::db::error::{DbError, DbResult}; +use std::path::{Path, PathBuf}; +use std::collections::HashMap; +use tst::TST; + +/// Manages TST-based indexes for model objects +pub struct TSTIndexManager { + /// Base path for TST databases + base_path: PathBuf, + + /// Map of model prefixes to their TST instances + tst_instances: HashMap, +} + +impl TSTIndexManager { + /// Creates a new TST index manager + pub fn new>(base_path: P) -> DbResult { + let base_path = base_path.as_ref().to_path_buf(); + + // Create directory if it doesn't exist + std::fs::create_dir_all(&base_path).map_err(DbError::IoError)?; + + Ok(Self { + base_path, + tst_instances: HashMap::new(), + }) + } + + /// Gets or creates a TST instance for a model prefix + pub fn get_tst(&mut self, prefix: &str) -> DbResult<&mut TST> { + if !self.tst_instances.contains_key(prefix) { + // Create a new TST instance for this prefix + let tst_path = self.base_path.join(format!("{}_tst", prefix)); + let tst_path_str = tst_path.to_string_lossy().to_string(); + + // Create the TST + let tst = TST::new(&tst_path_str, false) + .map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?; + + // Insert it into the map + self.tst_instances.insert(prefix.to_string(), tst); + } + + // Return a mutable reference to the TST + Ok(self.tst_instances.get_mut(prefix).unwrap()) + } + + /// Adds or updates an object in the TST index + pub fn set(&mut self, prefix: &str, id: u32, data: Vec) -> DbResult<()> { + // Get the TST for this prefix + let tst = self.get_tst(prefix)?; + + // Create the key in the format prefix_id + let key = format!("{}_{}", prefix, id); + + // Set the key-value pair in the TST + tst.set(&key, data) + .map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?; + + Ok(()) + } + + /// Removes an object from the TST index + pub fn delete(&mut self, prefix: &str, id: u32) -> DbResult<()> { + // Get the TST for this prefix + let tst = self.get_tst(prefix)?; + + // Create the key in the format prefix_id + let key = format!("{}_{}", prefix, id); + + // Delete the key from the TST + tst.delete(&key) + .map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?; + + Ok(()) + } + + /// Lists all objects with a given prefix + pub fn list(&mut self, prefix: &str) -> DbResult)>> { + // Get the TST for this prefix + let tst = self.get_tst(prefix)?; + + // Get all keys with this prefix + let keys = tst.list(prefix) + .map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?; + + // Get all values for these keys + let mut result = Vec::with_capacity(keys.len()); + for key in keys { + // Extract the ID from the key (format: prefix_id) + let id_str = key.split('_').nth(1).ok_or_else(|| { + DbError::GeneralError(format!("Invalid key format: {}", key)) + })?; + + let id = id_str.parse::().map_err(|_| { + DbError::GeneralError(format!("Invalid ID in key: {}", key)) + })?; + + // Get the value from the TST + let data = tst.get(&key) + .map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?; + + result.push((id, data)); + } + + Ok(result) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + fn test_tst_index_manager() { + // Create a temporary directory for the test + let temp_dir = tempdir().unwrap(); + let path = temp_dir.path(); + + // Create a TST index manager + let mut manager = TSTIndexManager::new(path).unwrap(); + + // Test setting values + let data1 = vec![1, 2, 3]; + let data2 = vec![4, 5, 6]; + manager.set("test", 1, data1.clone()).unwrap(); + manager.set("test", 2, data2.clone()).unwrap(); + + // Test listing values + let items = manager.list("test").unwrap(); + assert_eq!(items.len(), 2); + + // Check that the values are correct + let mut found_data1 = false; + let mut found_data2 = false; + for (id, data) in items { + if id == 1 && data == data1 { + found_data1 = true; + } else if id == 2 && data == data2 { + found_data2 = true; + } + } + assert!(found_data1); + assert!(found_data2); + + // Test deleting a value + manager.delete("test", 1).unwrap(); + + // Test listing again + let items = manager.list("test").unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0].0, 2); + assert_eq!(items[0].1, data2); + } +} \ No newline at end of file diff --git a/herodb/src/models/biz/customer.rs b/herodb/src/models/biz/customer.rs index 088e669..ccf69e4 100644 --- a/herodb/src/models/biz/customer.rs +++ b/herodb/src/models/biz/customer.rs @@ -145,4 +145,6 @@ impl Model for Customer { fn db_prefix() -> &'static str { "customer" } + + } \ No newline at end of file diff --git a/tst_integration_plan.md b/tst_integration_plan.md new file mode 100644 index 0000000..7e32b86 --- /dev/null +++ b/tst_integration_plan.md @@ -0,0 +1,451 @@ +# TST Integration Plan for HeroDB + +## Overview + +This document outlines the plan for adding generic functionality to the `herodb/src/db` module to use the Ternary Search Tree (TST) for storing objects with prefixed IDs and implementing a generic list function to retrieve all objects with a specific prefix. + +## Current Architecture + +Currently: +- Each model has a `db_prefix()` method that returns a string prefix (e.g., "vote" for Vote objects) +- Objects are stored in OurDB with numeric IDs +- The `list()` method in `OurDbStore` is not implemented + +## Implementation Plan + +### 1. Create a TST-based Index Manager (herodb/src/db/tst_index.rs) + +Create a new module that manages TST instances for different model prefixes: + +```rust +use crate::db::error::{DbError, DbResult}; +use std::path::{Path, PathBuf}; +use tst::TST; + +/// Manages TST-based indexes for model objects +pub struct TSTIndexManager { + /// Base path for TST databases + base_path: PathBuf, + + /// Map of model prefixes to their TST instances + tst_instances: std::collections::HashMap, +} + +impl TSTIndexManager { + /// Creates a new TST index manager + pub fn new>(base_path: P) -> DbResult { + let base_path = base_path.as_ref().to_path_buf(); + + // Create directory if it doesn't exist + std::fs::create_dir_all(&base_path).map_err(DbError::IoError)?; + + Ok(Self { + base_path, + tst_instances: std::collections::HashMap::new(), + }) + } + + /// Gets or creates a TST instance for a model prefix + pub fn get_tst(&mut self, prefix: &str) -> DbResult<&mut TST> { + if !self.tst_instances.contains_key(prefix) { + // Create a new TST instance for this prefix + let tst_path = self.base_path.join(format!("{}_tst", prefix)); + let tst_path_str = tst_path.to_string_lossy().to_string(); + + // Create the TST + let tst = TST::new(&tst_path_str, false) + .map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?; + + // Insert it into the map + self.tst_instances.insert(prefix.to_string(), tst); + } + + // Return a mutable reference to the TST + Ok(self.tst_instances.get_mut(prefix).unwrap()) + } + + /// Adds or updates an object in the TST index + pub fn set(&mut self, prefix: &str, id: u32, data: Vec) -> DbResult<()> { + // Get the TST for this prefix + let tst = self.get_tst(prefix)?; + + // Create the key in the format prefix_id + let key = format!("{}_{}", prefix, id); + + // Set the key-value pair in the TST + tst.set(&key, data) + .map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?; + + Ok(()) + } + + /// Removes an object from the TST index + pub fn delete(&mut self, prefix: &str, id: u32) -> DbResult<()> { + // Get the TST for this prefix + let tst = self.get_tst(prefix)?; + + // Create the key in the format prefix_id + let key = format!("{}_{}", prefix, id); + + // Delete the key from the TST + tst.delete(&key) + .map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?; + + Ok(()) + } + + /// Lists all objects with a given prefix + pub fn list(&mut self, prefix: &str) -> DbResult)>> { + // Get the TST for this prefix + let tst = self.get_tst(prefix)?; + + // Get all keys with this prefix + let keys = tst.list(prefix) + .map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?; + + // Get all values for these keys + let mut result = Vec::with_capacity(keys.len()); + for key in keys { + // Extract the ID from the key (format: prefix_id) + let id_str = key.split('_').nth(1).ok_or_else(|| { + DbError::GeneralError(format!("Invalid key format: {}", key)) + })?; + + let id = id_str.parse::().map_err(|_| { + DbError::GeneralError(format!("Invalid ID in key: {}", key)) + })?; + + // Get the value from the TST + let data = tst.get(&key) + .map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?; + + result.push((id, data)); + } + + Ok(result) + } +} +``` + +### 2. Update DB Module (herodb/src/db/mod.rs) + +Add the new module to the db module: + +```rust +pub mod db; +pub mod error; +pub mod macros; +pub mod model; +pub mod model_methods; +pub mod store; +pub mod tst_index; // Add the new module + +pub use db::DB; +pub use db::DBBuilder; +pub use error::{DbError, DbResult}; +pub use model::Model; +pub use model::Storable; +``` + +### 3. Modify DB Struct (herodb/src/db/db.rs) + +Update the DB struct to include the TST index manager: + +```rust +/// Main DB manager that automatically handles all models +#[derive(Clone, CustomType)] +pub struct DB { + db_path: PathBuf, + + // Type map for generic operations + type_map: HashMap>>, + + // TST index manager + tst_index: Arc>, + + // Transaction state + transaction: Arc>>, +} +``` + +### 4. Extend Transaction Handling + +Extend the `DbOperation` enum to include model prefix and ID information: + +```rust +#[derive(Debug, Clone)] +enum DbOperation { + Set { + model_type: TypeId, + serialized: Vec, + model_prefix: String, // Add model prefix + model_id: u32, // Add model ID + }, + Delete { + model_type: TypeId, + id: u32, + model_prefix: String, // Add model prefix + }, +} +``` + +### 5. Update Transaction Recording + +Modify the `set` and `delete` methods to record model prefix and ID in the transaction: + +```rust +pub fn set(&self, model: &T) -> DbResult<()> { + // Try to acquire a write lock on the transaction + let mut tx_guard = self.transaction.write().unwrap(); + + // Check if there's an active transaction + if let Some(tx_state) = tx_guard.as_mut() { + if tx_state.active { + // Serialize the model for later use + let serialized = model.to_bytes()?; + + // Record a Set operation in the transaction with prefix and ID + tx_state.operations.push(DbOperation::Set { + model_type: TypeId::of::(), + serialized: serialized.clone(), + model_prefix: T::db_prefix().to_string(), + model_id: model.get_id(), + }); + + return Ok(()); + } + } + + // ... rest of the method ... +} + +pub fn delete(&self, id: u32) -> DbResult<()> { + // Try to acquire a write lock on the transaction + let mut tx_guard = self.transaction.write().unwrap(); + + // Check if there's an active transaction + if let Some(tx_state) = tx_guard.as_mut() { + if tx_state.active { + // Record a Delete operation in the transaction with prefix + tx_state.operations.push(DbOperation::Delete { + model_type: TypeId::of::(), + id, + model_prefix: T::db_prefix().to_string(), + }); + + return Ok(()); + } + } + + // ... rest of the method ... +} +``` + +### 6. Update Transaction Commit + +Modify the `commit_transaction` method to update both OurDB and the TST index: + +```rust +pub fn commit_transaction(&self) -> DbResult<()> { + let mut tx_guard = self.transaction.write().unwrap(); + + if let Some(tx_state) = tx_guard.take() { + if !tx_state.active { + return Err(DbError::TransactionError("Transaction not active".into())); + } + + // Create a backup of the transaction state in case we need to rollback + let backup = tx_state.clone(); + + // Try to execute all operations + let result = (|| { + for op in &tx_state.operations { + match op { + DbOperation::Set { + model_type, + serialized, + model_prefix, + model_id, + } => { + // Apply to OurDB + self.apply_set_operation(*model_type, serialized)?; + + // Apply to TST index + let mut tst_index = self.tst_index.write().unwrap(); + tst_index.set(model_prefix, *model_id, serialized.clone())?; + } + DbOperation::Delete { + model_type, + id, + model_prefix, + } => { + // Apply to OurDB + let db_ops = self + .type_map + .get(model_type) + .ok_or_else(|| DbError::TypeError)?; + let mut db_ops_guard = db_ops.write().unwrap(); + db_ops_guard.delete(*id)?; + + // Apply to TST index + let mut tst_index = self.tst_index.write().unwrap(); + tst_index.delete(model_prefix, *id)?; + } + } + } + Ok(()) + })(); + + // If any operation failed, restore the transaction state + if result.is_err() { + *tx_guard = Some(backup); + return result; + } + + Ok(()) + } else { + Err(DbError::TransactionError("No active transaction".into())) + } +} +``` + +### 7. Implement List Method + +Implement the `list` method to use the TST's prefix search: + +```rust +pub fn list(&self) -> DbResult> { + // Get the prefix for this model type + let prefix = T::db_prefix(); + + // Use the TST index to get all objects with this prefix + let mut tst_index = self.tst_index.write().unwrap(); + let items = tst_index.list(prefix)?; + + // Deserialize the objects + let mut result = Vec::with_capacity(items.len()); + for (_, data) in items { + let model = T::from_bytes(&data)?; + result.push(model); + } + + Ok(result) +} +``` + +### 8. Add Recovery Mechanism + +Add a method to synchronize the TST index with OurDB in case they get out of sync: + +```rust +pub fn synchronize_tst_index(&self) -> DbResult<()> { + // Get all models from OurDB + let models = self.list_from_ourdb::()?; + + // Clear the TST index for this model type + let mut tst_index = self.tst_index.write().unwrap(); + let prefix = T::db_prefix(); + + // Rebuild the TST index + for model in models { + let id = model.get_id(); + let data = model.to_bytes()?; + tst_index.set(prefix, id, data)?; + } + + Ok(()) +} + +// Helper method to list models directly from OurDB (not using TST) +fn list_from_ourdb(&self) -> DbResult> { + match self.type_map.get(&TypeId::of::()) { + Some(db_ops) => { + let db_ops_guard = db_ops.read().unwrap(); + let result_any = db_ops_guard.list()?; + match result_any.downcast::>() { + Ok(vec_t) => Ok(*vec_t), + Err(_) => Err(DbError::TypeError), + } + } + None => Err(DbError::TypeError), + } +} +``` + +## Implementation Flow + +```mermaid +sequenceDiagram + participant Client + participant DB + participant TransactionState + participant OurDbStore + participant TSTIndexManager + participant TST + + Client->>DB: begin_transaction() + DB->>TransactionState: create new transaction + + Client->>DB: set(model) + DB->>TransactionState: record Set operation with prefix and ID + + Client->>DB: delete(model) + DB->>TransactionState: record Delete operation with prefix and ID + + Client->>DB: commit_transaction() + DB->>TransactionState: get all operations + + loop For each operation + alt Set operation + DB->>OurDbStore: apply_set_operation() + DB->>TSTIndexManager: set(prefix, id, data) + TSTIndexManager->>TST: set(key, data) + else Delete operation + DB->>OurDbStore: delete(id) + DB->>TSTIndexManager: delete(prefix, id) + TSTIndexManager->>TST: delete(key) + end + end + + alt Success + DB-->>Client: Ok(()) + else Error + DB->>TransactionState: restore transaction state + DB-->>Client: Err(error) + end + + Client->>DB: list() + DB->>TSTIndexManager: list(prefix) + TSTIndexManager->>TST: list(prefix) + TST-->>TSTIndexManager: keys + TSTIndexManager->>TST: get(key) for each key + TST-->>TSTIndexManager: data + TSTIndexManager-->>DB: (id, data) pairs + DB->>DB: deserialize data to models + DB-->>Client: Vec +``` + +## Testing Strategy + +1. Create unit tests for the TST index manager +2. Test the list functionality with different model types +3. Test transaction handling (commit and rollback) +4. Test error recovery mechanisms +5. Test edge cases (empty database, large datasets) + +## Implementation Steps + +1. Add TST dependency to herodb/Cargo.toml +2. Create the tst_index.rs module +3. Update the DB module to include the TST index manager +4. Extend the transaction handling +5. Implement the list method +6. Add tests for the new functionality +7. Update documentation + +## Considerations + +1. **Performance**: The TST operations add overhead to insert/delete operations, but provide efficient list functionality. +2. **Consistency**: The enhanced transaction handling ensures consistency between OurDB and the TST index. +3. **Error Handling**: Proper error handling and recovery mechanisms are essential for maintaining data integrity. +4. **Backward Compatibility**: The implementation should maintain backward compatibility with existing code. \ No newline at end of file