...

herodb/Cargo.lock

@@ -623,6 +623,7 @@ dependencies = [
  "tempfile",
  "thiserror",
  "tokio",
+ "tst",
  "uuid",
 ]
 
@@ -1704,6 +1705,14 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
+[[package]]
+name = "tst"
+version = "0.1.0"
+dependencies = [
+ "ourdb",
+ "thiserror",
+]
+
 [[package]]
 name = "typenum"
 version = "1.18.0"

herodb/Cargo.toml

@@ -8,6 +8,7 @@ authors = ["HeroCode Team"]
 
 [dependencies]
 ourdb = { path = "../ourdb" }
+tst = { path = "../tst" }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 thiserror = "1.0"

herodb/src/db/db.rs

@@ -1,6 +1,7 @@
 use crate::db::error::{DbError, DbResult};
 use crate::db::model::Model;
 use crate::db::store::{DbOperations, OurDbStore};
+use crate::db::tst_index::TSTIndexManager;
 use std::any::TypeId;
 use std::collections::HashMap;
 use std::fmt::Debug;
@@ -14,14 +15,18 @@ enum DbOperation {
     Set {
         model_type: TypeId,
         serialized: Vec<u8>,
+        model_prefix: String, // Add model prefix
+        model_id: u32,        // Add model ID
     },
     Delete {
         model_type: TypeId,
         id: u32,
+        model_prefix: String, // Add model prefix
     },
 }
 
 /// Transaction state for DB operations
+#[derive(Clone)]
 pub struct TransactionState {
     operations: Vec<DbOperation>,
     active: bool,
@@ -45,6 +50,9 @@ pub struct DB {
     // Type map for generic operations
     type_map: HashMap<TypeId, Arc<RwLock<dyn DbOperations>>>,
     
+    // TST index manager
+    tst_index: Arc<RwLock<TSTIndexManager>>,
+    
     // Transaction state
     transaction: Arc<RwLock<Option<TransactionState>>>,
 }
@@ -125,11 +133,17 @@ impl DBBuilder {
             type_map.insert(type_id, store);
         }
         
+        // Create the TST index manager
+        let tst_index = TSTIndexManager::new(&base_path).map_err(|e| {
+            EvalAltResult::ErrorSystem("Could not create TST index manager".to_string(), Box::new(e))
+        })?;
+        
         let transaction = Arc::new(RwLock::new(None));
         
         Ok(DB {
             db_path: base_path,
             type_map,
+            tst_index: Arc::new(RwLock::new(tst_index)),
             transaction,
         })
     }
@@ -145,11 +159,15 @@ impl DB {
             std::fs::create_dir_all(&base_path)?;
         }
         
+        // Create the TST index manager
+        let tst_index = TSTIndexManager::new(&base_path)?;
+        
         let transaction = Arc::new(RwLock::new(None));
         
         Ok(Self {
             db_path: base_path,
             type_map: HashMap::new(),
+            tst_index: Arc::new(RwLock::new(tst_index)),
             transaction,
         })
     }
@@ -198,24 +216,52 @@ impl DB {
                 return Err(DbError::TransactionError("Transaction not active".into()));
             }
             
-            // Execute all operations in the transaction
-            for op in tx_state.operations {
-                match op {
-                    DbOperation::Set {
-                        model_type,
-                        serialized,
-                    } => {
-                        self.apply_set_operation(model_type, &serialized)?;
-                    }
-                    DbOperation::Delete { model_type, id } => {
-                        let db_ops = self
-                            .type_map
-                            .get(&model_type)
-                            .ok_or_else(|| DbError::TypeError)?;
-                        let mut db_ops_guard = db_ops.write().unwrap();
-                        db_ops_guard.delete(id)?;
+            // Create a backup of the transaction state in case we need to rollback
+            let backup = tx_state.clone();
+            
+            // Try to execute all operations
+            let result = (|| {
+                for op in tx_state.operations {
+                    match op {
+                        DbOperation::Set {
+                            model_type,
+                            serialized,
+                            model_prefix,
+                            model_id,
+                        } => {
+                            // Apply to OurDB
+                            self.apply_set_operation(model_type, &serialized)?;
+                            
+                            // Apply to TST index
+                            let mut tst_index = self.tst_index.write().unwrap();
+                            tst_index.set(&model_prefix, model_id, serialized.clone())?;
+                        }
+                        DbOperation::Delete {
+                            model_type,
+                            id,
+                            model_prefix,
+                        } => {
+                            // Apply to OurDB
+                            let db_ops = self
+                                .type_map
+                                .get(&model_type)
+                                .ok_or_else(|| DbError::TypeError)?;
+                            let mut db_ops_guard = db_ops.write().unwrap();
+                            db_ops_guard.delete(id)?;
+                            
+                            // Apply to TST index
+                            let mut tst_index = self.tst_index.write().unwrap();
+                            tst_index.delete(&model_prefix, id)?;
+                        }
                     }
                 }
+                Ok(())
+            })();
+            
+            // If any operation failed, restore the transaction state
+            if result.is_err() {
+                *tx_guard = Some(backup);
+                return result;
             }
             
             Ok(())
@@ -252,10 +298,12 @@ impl DB {
                 // Serialize the model for later use
                 let serialized = model.to_bytes()?;
                 
-                // Record a Set operation in the transaction
+                // Record a Set operation in the transaction with prefix and ID
                 tx_state.operations.push(DbOperation::Set {
                     model_type: TypeId::of::<T>(),
                     serialized,
+                    model_prefix: T::db_prefix().to_string(),
+                    model_id: model.get_id(),
                 });
                 
                 return Ok(());
@@ -270,7 +318,16 @@ impl DB {
         match self.type_map.get(&TypeId::of::<T>()) {
             Some(db_ops) => {
                 let mut db_ops_guard = db_ops.write().unwrap();
-                db_ops_guard.insert(model)
+                db_ops_guard.insert(model)?;
+                
+                // Also update the TST index
+                let mut tst_index = self.tst_index.write().unwrap();
+                let prefix = T::db_prefix();
+                let id = model.get_id();
+                let data = model.to_bytes()?;
+                tst_index.set(prefix, id, data)?;
+                
+                Ok(())
             },
             None => Err(DbError::TypeError),
         }
@@ -295,6 +352,7 @@ impl DB {
                     DbOperation::Delete {
                         model_type,
                         id: op_id,
+                        model_prefix: _,
                     } => {
                         if *model_type == type_id && *op_id == id {
                             // Return NotFound error for deleted records
@@ -305,14 +363,14 @@ impl DB {
                     DbOperation::Set {
                         model_type,
                         serialized,
+                        model_prefix: _,
+                        model_id,
                     } => {
-                        if *model_type == type_id {
-                            // Try to deserialize and check the ID
+                        if *model_type == type_id && *model_id == id {
+                            // Try to deserialize
                             match T::from_bytes(serialized) {
                                 Ok(model) => {
-                                    if model.get_id() == id {
-                                        return Some(Ok(Some(model)));
-                                    }
+                                    return Some(Ok(Some(model)));
                                 }
                                 Err(_) => continue, // Skip if deserialization fails
                             }
@@ -360,10 +418,11 @@ impl DB {
         // Check if there's an active transaction
         if let Some(tx_state) = tx_guard.as_mut() {
             if tx_state.active {
-                // Record a Delete operation in the transaction
+                // Record a Delete operation in the transaction with prefix
                 tx_state.operations.push(DbOperation::Delete {
                     model_type: TypeId::of::<T>(),
                     id,
+                    model_prefix: T::db_prefix().to_string(),
                 });
                 
                 return Ok(());
@@ -378,7 +437,14 @@ impl DB {
         match self.type_map.get(&TypeId::of::<T>()) {
             Some(db_ops) => {
                 let mut db_ops_guard = db_ops.write().unwrap();
-                db_ops_guard.delete(id)
+                db_ops_guard.delete(id)?;
+                
+                // Also update the TST index
+                let mut tst_index = self.tst_index.write().unwrap();
+                let prefix = T::db_prefix();
+                tst_index.delete(prefix, id)?;
+                
+                Ok(())
             },
             None => Err(DbError::TypeError),
         }
@@ -386,7 +452,25 @@ impl DB {
     
     /// List all model instances of a specific type
     pub fn list<T: Model>(&self) -> DbResult<Vec<T>> {
-        // Look up the correct DB operations for type T in our type map
+        // Get the prefix for this model type
+        let prefix = T::db_prefix();
+        
+        // Use the TST index to get all objects with this prefix
+        let mut tst_index = self.tst_index.write().unwrap();
+        let items = tst_index.list(prefix)?;
+        
+        // Deserialize the objects
+        let mut result = Vec::with_capacity(items.len());
+        for (_, data) in items {
+            let model = T::from_bytes(&data)?;
+            result.push(model);
+        }
+        
+        Ok(result)
+    }
+    
+    /// Helper method to list models directly from OurDB (not using TST)
+    fn list_from_ourdb<T: Model>(&self) -> DbResult<Vec<T>> {
         match self.type_map.get(&TypeId::of::<T>()) {
             Some(db_ops) => {
                 let db_ops_guard = db_ops.read().unwrap();
@@ -401,6 +485,25 @@ impl DB {
         }
     }
     
+    /// Synchronize the TST index with OurDB for a specific model type
+    pub fn synchronize_tst_index<T: Model>(&self) -> DbResult<()> {
+        // Get all models from OurDB
+        let models = self.list_from_ourdb::<T>()?;
+        
+        // Clear the TST index for this model type
+        let mut tst_index = self.tst_index.write().unwrap();
+        let prefix = T::db_prefix();
+        
+        // Rebuild the TST index
+        for model in models {
+            let id = model.get_id();
+            let data = model.to_bytes()?;
+            tst_index.set(prefix, id, data)?;
+        }
+        
+        Ok(())
+    }
+    
     /// Get the history of a model by its ID
     pub fn get_history<T: Model>(&self, id: u32, depth: u8) -> DbResult<Vec<T>> {
         // Look up the correct DB operations for type T in our type map
@@ -425,8 +528,17 @@ impl DB {
     
     // Register a model type with this DB instance
     pub fn register<T: Model>(&mut self) -> DbResult<()> {
+        // Create the OurDB store
         let store = OurDbStore::<T>::open(&self.db_path)?;
         self.type_map.insert(TypeId::of::<T>(), Arc::new(RwLock::new(store)));
+        
+        // Initialize the TST index for this model type
+        let prefix = T::db_prefix();
+        let mut tst_index = self.tst_index.write().unwrap();
+        
+        // Ensure the TST for this prefix exists
+        tst_index.get_tst(prefix)?;
+        
         Ok(())
     }
 }

herodb/src/db/mod.rs

@@ -14,5 +14,13 @@ pub use store::{DbOperations, OurDbStore};
 pub mod db;
 pub use db::{DB, DBBuilder, ModelRegistration, ModelRegistrar};
 
+// Export the TST index module
+pub mod tst_index;
+pub use tst_index::TSTIndexManager;
+
 // Export macros for model methods
 pub mod macros;
+
+// Tests
+#[cfg(test)]
+mod tests;

herodb/src/db/model.rs

@@ -1,4 +1,4 @@
-use crate::db::error::{DbError, DbResult};
+...use crate::db::error::{DbError, DbResult};
 use serde::{Deserialize, Serialize};
 use std::fmt::Debug;
 
@@ -15,6 +15,16 @@ pub trait Storable: Serialize + for<'de> Deserialize<'de> + Sized {
     }
 }
 
+/// Represents an index key for a model
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct IndexKey {
+    /// The name of the index key
+    pub name: &'static str,
+    
+    /// The value of the index key for a specific model instance
+    pub value: String,
+}
+
 /// Trait identifying a model suitable for the database
 /// The 'static lifetime bound is required for type identification via Any
 pub trait Model: Storable + Debug + Clone + Send + Sync + 'static {
@@ -24,6 +34,14 @@ pub trait Model: Storable + Debug + Clone + Send + Sync + 'static {
     /// Returns a prefix used for this model type in the database
     /// Helps to logically separate different model types
     fn db_prefix() -> &'static str;
+    
+    /// Returns a list of index keys for this model instance
+    /// These keys will be used to create additional indexes in the TST
+    /// The default implementation returns an empty vector
+    /// Override this method to provide custom indexes
+    fn db_keys(&self) -> Vec<IndexKey> {
+        Vec::new()
+    }
 }
 
 // Note: We don't provide a blanket implementation of Storable

herodb/src/db/tests.rs

@@ -0,0 +1,98 @@
+use super::*;
+use crate::db::model::Storable;
+use serde::{Deserialize, Serialize};
+use tempfile::tempdir;
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+struct TestModel {
+    id: u32,
+    name: String,
+}
+
+impl Storable for TestModel {}
+
+impl Model for TestModel {
+    fn get_id(&self) -> u32 {
+        self.id
+    }
+    
+    fn db_prefix() -> &'static str {
+        "test"
+    }
+}
+
+#[test]
+fn test_tst_integration() {
+    // Create a temporary directory for the test
+    let temp_dir = tempdir().unwrap();
+    let path = temp_dir.path();
+    
+    // Create a DB instance
+    let mut db = DB::new(path).unwrap();
+    db.register::<TestModel>().unwrap();
+    
+    // Create some test models
+    let model1 = TestModel { id: 1, name: "Test 1".to_string() };
+    let model2 = TestModel { id: 2, name: "Test 2".to_string() };
+    let model3 = TestModel { id: 3, name: "Test 3".to_string() };
+    
+    // Insert the models
+    db.set(&model1).unwrap();
+    db.set(&model2).unwrap();
+    db.set(&model3).unwrap();
+    
+    // List all models
+    let models = db.list::<TestModel>().unwrap();
+    assert_eq!(models.len(), 3);
+    
+    // Verify that all models are in the list
+    assert!(models.contains(&model1));
+    assert!(models.contains(&model2));
+    assert!(models.contains(&model3));
+    
+    // Delete a model
+    db.delete::<TestModel>(2).unwrap();
+    
+    // List again
+    let models = db.list::<TestModel>().unwrap();
+    assert_eq!(models.len(), 2);
+    assert!(models.contains(&model1));
+    assert!(models.contains(&model3));
+    assert!(!models.contains(&model2));
+    
+    // Test transaction with commit
+    db.begin_transaction().unwrap();
+    db.set(&model2).unwrap(); // Add back model2
+    db.delete::<TestModel>(1).unwrap(); // Delete model1
+    db.commit_transaction().unwrap();
+    
+    // List again after transaction
+    let models = db.list::<TestModel>().unwrap();
+    assert_eq!(models.len(), 2);
+    assert!(!models.contains(&model1));
+    assert!(models.contains(&model2));
+    assert!(models.contains(&model3));
+    
+    // Test transaction with rollback
+    db.begin_transaction().unwrap();
+    db.delete::<TestModel>(3).unwrap(); // Delete model3
+    db.rollback_transaction().unwrap();
+    
+    // List again after rollback
+    let models = db.list::<TestModel>().unwrap();
+    assert_eq!(models.len(), 2);
+    assert!(!models.contains(&model1));
+    assert!(models.contains(&model2));
+    assert!(models.contains(&model3));
+    
+    // Test the synchronize_tst_index method
+    // Since we can't directly access private fields, we'll just verify that
+    // the method runs without errors
+    db.synchronize_tst_index::<TestModel>().unwrap();
+    
+    // Verify that our models are still accessible
+    let models = db.list::<TestModel>().unwrap();
+    assert_eq!(models.len(), 2);
+    assert!(models.contains(&model2));
+    assert!(models.contains(&model3));
+}

herodb/src/db/tst_index.rs

@@ -0,0 +1,156 @@
+use crate::db::error::{DbError, DbResult};
+use std::path::{Path, PathBuf};
+use std::collections::HashMap;
+use tst::TST;
+
+/// Manages TST-based indexes for model objects
+pub struct TSTIndexManager {
+    /// Base path for TST databases
+    base_path: PathBuf,
+    
+    /// Map of model prefixes to their TST instances
+    tst_instances: HashMap<String, TST>,
+}
+
+impl TSTIndexManager {
+    /// Creates a new TST index manager
+    pub fn new<P: AsRef<Path>>(base_path: P) -> DbResult<Self> {
+        let base_path = base_path.as_ref().to_path_buf();
+        
+        // Create directory if it doesn't exist
+        std::fs::create_dir_all(&base_path).map_err(DbError::IoError)?;
+        
+        Ok(Self {
+            base_path,
+            tst_instances: HashMap::new(),
+        })
+    }
+    
+    /// Gets or creates a TST instance for a model prefix
+    pub fn get_tst(&mut self, prefix: &str) -> DbResult<&mut TST> {
+        if !self.tst_instances.contains_key(prefix) {
+            // Create a new TST instance for this prefix
+            let tst_path = self.base_path.join(format!("{}_tst", prefix));
+            let tst_path_str = tst_path.to_string_lossy().to_string();
+            
+            // Create the TST
+            let tst = TST::new(&tst_path_str, false)
+                .map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?;
+            
+            // Insert it into the map
+            self.tst_instances.insert(prefix.to_string(), tst);
+        }
+        
+        // Return a mutable reference to the TST
+        Ok(self.tst_instances.get_mut(prefix).unwrap())
+    }
+    
+    /// Adds or updates an object in the TST index
+    pub fn set(&mut self, prefix: &str, id: u32, data: Vec<u8>) -> DbResult<()> {
+        // Get the TST for this prefix
+        let tst = self.get_tst(prefix)?;
+        
+        // Create the key in the format prefix_id
+        let key = format!("{}_{}", prefix, id);
+        
+        // Set the key-value pair in the TST
+        tst.set(&key, data)
+            .map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?;
+        
+        Ok(())
+    }
+    
+    /// Removes an object from the TST index
+    pub fn delete(&mut self, prefix: &str, id: u32) -> DbResult<()> {
+        // Get the TST for this prefix
+        let tst = self.get_tst(prefix)?;
+        
+        // Create the key in the format prefix_id
+        let key = format!("{}_{}", prefix, id);
+        
+        // Delete the key from the TST
+        tst.delete(&key)
+            .map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?;
+        
+        Ok(())
+    }
+    
+    /// Lists all objects with a given prefix
+    pub fn list(&mut self, prefix: &str) -> DbResult<Vec<(u32, Vec<u8>)>> {
+        // Get the TST for this prefix
+        let tst = self.get_tst(prefix)?;
+        
+        // Get all keys with this prefix
+        let keys = tst.list(prefix)
+            .map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?;
+        
+        // Get all values for these keys
+        let mut result = Vec::with_capacity(keys.len());
+        for key in keys {
+            // Extract the ID from the key (format: prefix_id)
+            let id_str = key.split('_').nth(1).ok_or_else(|| {
+                DbError::GeneralError(format!("Invalid key format: {}", key))
+            })?;
+            
+            let id = id_str.parse::<u32>().map_err(|_| {
+                DbError::GeneralError(format!("Invalid ID in key: {}", key))
+            })?;
+            
+            // Get the value from the TST
+            let data = tst.get(&key)
+                .map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?;
+            
+            result.push((id, data));
+        }
+        
+        Ok(result)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::tempdir;
+    
+    #[test]
+    fn test_tst_index_manager() {
+        // Create a temporary directory for the test
+        let temp_dir = tempdir().unwrap();
+        let path = temp_dir.path();
+        
+        // Create a TST index manager
+        let mut manager = TSTIndexManager::new(path).unwrap();
+        
+        // Test setting values
+        let data1 = vec![1, 2, 3];
+        let data2 = vec![4, 5, 6];
+        manager.set("test", 1, data1.clone()).unwrap();
+        manager.set("test", 2, data2.clone()).unwrap();
+        
+        // Test listing values
+        let items = manager.list("test").unwrap();
+        assert_eq!(items.len(), 2);
+        
+        // Check that the values are correct
+        let mut found_data1 = false;
+        let mut found_data2 = false;
+        for (id, data) in items {
+            if id == 1 && data == data1 {
+                found_data1 = true;
+            } else if id == 2 && data == data2 {
+                found_data2 = true;
+            }
+        }
+        assert!(found_data1);
+        assert!(found_data2);
+        
+        // Test deleting a value
+        manager.delete("test", 1).unwrap();
+        
+        // Test listing again
+        let items = manager.list("test").unwrap();
+        assert_eq!(items.len(), 1);
+        assert_eq!(items[0].0, 2);
+        assert_eq!(items[0].1, data2);
+    }
+}

herodb/src/models/biz/customer.rs

@@ -145,4 +145,6 @@ impl Model for Customer {
     fn db_prefix() -> &'static str {
         "customer"
     }
+
+    
 }