herocode/db
12
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

...

Browse Source
This commit is contained in:
despiegk 2025-04-21 05:02:57 +02:00
parent 98af5a3b02
commit 3b5f9c6012
9 changed files with 882 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
herodb/Cargo.lock generated
View File

@ -623,6 +623,7 @@ dependencies = [
"tempfile",
"thiserror",
"tokio",
"tst",
"uuid",
]
@ -1704,6 +1705,14 @@ version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
[[package]]
name = "tst"
version = "0.1.0"
dependencies = [
"ourdb",
"thiserror",
]
[[package]]
name = "typenum"
version = "1.18.0"

1
herodb/Cargo.toml
View File

@ -8,6 +8,7 @@ authors = ["HeroCode Team"]
[dependencies]
ourdb = { path = "../ourdb" }
tst = { path = "../tst" }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
thiserror = "1.0"

164
herodb/src/db/db.rs
View File

@ -1,6 +1,7 @@
use crate::db::error::{DbError, DbResult};
use crate::db::model::Model;
use crate::db::store::{DbOperations, OurDbStore};
use crate::db::tst_index::TSTIndexManager;
use std::any::TypeId;
use std::collections::HashMap;
use std::fmt::Debug;
@ -14,14 +15,18 @@ enum DbOperation {
Set {
model_type: TypeId,
serialized: Vec<u8>,
model_prefix: String, // Add model prefix
model_id: u32, // Add model ID
},
Delete {
model_type: TypeId,
id: u32,
model_prefix: String, // Add model prefix
},
}
/// Transaction state for DB operations
#[derive(Clone)]
pub struct TransactionState {
operations: Vec<DbOperation>,
active: bool,
@ -45,6 +50,9 @@ pub struct DB {
// Type map for generic operations
type_map: HashMap<TypeId, Arc<RwLock<dyn DbOperations>>>,
// TST index manager
tst_index: Arc<RwLock<TSTIndexManager>>,
// Transaction state
transaction: Arc<RwLock<Option<TransactionState>>>,
}
@ -125,11 +133,17 @@ impl DBBuilder {
type_map.insert(type_id, store);
}
// Create the TST index manager
let tst_index = TSTIndexManager::new(&base_path).map_err(|e| {
EvalAltResult::ErrorSystem("Could not create TST index manager".to_string(), Box::new(e))
})?;
let transaction = Arc::new(RwLock::new(None));
Ok(DB {
db_path: base_path,
type_map,
tst_index: Arc::new(RwLock::new(tst_index)),
transaction,
})
}
@ -145,11 +159,15 @@ impl DB {
std::fs::create_dir_all(&base_path)?;
}
// Create the TST index manager
let tst_index = TSTIndexManager::new(&base_path)?;
let transaction = Arc::new(RwLock::new(None));
Ok(Self {
db_path: base_path,
type_map: HashMap::new(),
tst_index: Arc::new(RwLock::new(tst_index)),
transaction,
})
}
@ -198,24 +216,52 @@ impl DB {
return Err(DbError::TransactionError("Transaction not active".into()));
}
// Execute all operations in the transaction
for op in tx_state.operations {
match op {
DbOperation::Set {
model_type,
serialized,
} => {
self.apply_set_operation(model_type, &serialized)?;
}
DbOperation::Delete { model_type, id } => {
let db_ops = self
.type_map
.get(&model_type)
.ok_or_else(|| DbError::TypeError)?;
let mut db_ops_guard = db_ops.write().unwrap();
db_ops_guard.delete(id)?;
// Create a backup of the transaction state in case we need to rollback
let backup = tx_state.clone();
// Try to execute all operations
let result = (|| {
for op in tx_state.operations {
match op {
DbOperation::Set {
model_type,
serialized,
model_prefix,
model_id,
} => {
// Apply to OurDB
self.apply_set_operation(model_type, &serialized)?;
// Apply to TST index
let mut tst_index = self.tst_index.write().unwrap();
tst_index.set(&model_prefix, model_id, serialized.clone())?;
}
DbOperation::Delete {
model_type,
id,
model_prefix,
} => {
// Apply to OurDB
let db_ops = self
.type_map
.get(&model_type)
.ok_or_else(|| DbError::TypeError)?;
let mut db_ops_guard = db_ops.write().unwrap();
db_ops_guard.delete(id)?;
// Apply to TST index
let mut tst_index = self.tst_index.write().unwrap();
tst_index.delete(&model_prefix, id)?;
}
}
}
Ok(())
})();
// If any operation failed, restore the transaction state
if result.is_err() {
*tx_guard = Some(backup);
return result;
}
Ok(())
@ -252,10 +298,12 @@ impl DB {
// Serialize the model for later use
let serialized = model.to_bytes()?;
// Record a Set operation in the transaction
// Record a Set operation in the transaction with prefix and ID
tx_state.operations.push(DbOperation::Set {
model_type: TypeId::of::<T>(),
serialized,
model_prefix: T::db_prefix().to_string(),
model_id: model.get_id(),
});
return Ok(());
@ -270,7 +318,16 @@ impl DB {
match self.type_map.get(&TypeId::of::<T>()) {
Some(db_ops) => {
let mut db_ops_guard = db_ops.write().unwrap();
db_ops_guard.insert(model)
db_ops_guard.insert(model)?;
// Also update the TST index
let mut tst_index = self.tst_index.write().unwrap();
let prefix = T::db_prefix();
let id = model.get_id();
let data = model.to_bytes()?;
tst_index.set(prefix, id, data)?;
Ok(())
},
None => Err(DbError::TypeError),
}
@ -295,6 +352,7 @@ impl DB {
DbOperation::Delete {
model_type,
id: op_id,
model_prefix: _,
} => {
if *model_type == type_id && *op_id == id {
// Return NotFound error for deleted records
@ -305,14 +363,14 @@ impl DB {
DbOperation::Set {
model_type,
serialized,
model_prefix: _,
model_id,
} => {
if *model_type == type_id {
// Try to deserialize and check the ID
if *model_type == type_id && *model_id == id {
// Try to deserialize
match T::from_bytes(serialized) {
Ok(model) => {
if model.get_id() == id {
return Some(Ok(Some(model)));
}
return Some(Ok(Some(model)));
}
Err(_) => continue, // Skip if deserialization fails
}
@ -360,10 +418,11 @@ impl DB {
// Check if there's an active transaction
if let Some(tx_state) = tx_guard.as_mut() {
if tx_state.active {
// Record a Delete operation in the transaction
// Record a Delete operation in the transaction with prefix
tx_state.operations.push(DbOperation::Delete {
model_type: TypeId::of::<T>(),
id,
model_prefix: T::db_prefix().to_string(),
});
return Ok(());
@ -378,7 +437,14 @@ impl DB {
match self.type_map.get(&TypeId::of::<T>()) {
Some(db_ops) => {
let mut db_ops_guard = db_ops.write().unwrap();
db_ops_guard.delete(id)
db_ops_guard.delete(id)?;
// Also update the TST index
let mut tst_index = self.tst_index.write().unwrap();
let prefix = T::db_prefix();
tst_index.delete(prefix, id)?;
Ok(())
},
None => Err(DbError::TypeError),
}
@ -386,7 +452,25 @@ impl DB {
/// List all model instances of a specific type
pub fn list<T: Model>(&self) -> DbResult<Vec<T>> {
// Look up the correct DB operations for type T in our type map
// Get the prefix for this model type
let prefix = T::db_prefix();
// Use the TST index to get all objects with this prefix
let mut tst_index = self.tst_index.write().unwrap();
let items = tst_index.list(prefix)?;
// Deserialize the objects
let mut result = Vec::with_capacity(items.len());
for (_, data) in items {
let model = T::from_bytes(&data)?;
result.push(model);
}
Ok(result)
}
/// Helper method to list models directly from OurDB (not using TST)
fn list_from_ourdb<T: Model>(&self) -> DbResult<Vec<T>> {
match self.type_map.get(&TypeId::of::<T>()) {
Some(db_ops) => {
let db_ops_guard = db_ops.read().unwrap();
@ -401,6 +485,25 @@ impl DB {
}
}
/// Synchronize the TST index with OurDB for a specific model type
pub fn synchronize_tst_index<T: Model>(&self) -> DbResult<()> {
// Get all models from OurDB
let models = self.list_from_ourdb::<T>()?;
// Clear the TST index for this model type
let mut tst_index = self.tst_index.write().unwrap();
let prefix = T::db_prefix();
// Rebuild the TST index
for model in models {
let id = model.get_id();
let data = model.to_bytes()?;
tst_index.set(prefix, id, data)?;
}
Ok(())
}
/// Get the history of a model by its ID
pub fn get_history<T: Model>(&self, id: u32, depth: u8) -> DbResult<Vec<T>> {
// Look up the correct DB operations for type T in our type map
@ -425,8 +528,17 @@ impl DB {
// Register a model type with this DB instance
pub fn register<T: Model>(&mut self) -> DbResult<()> {
// Create the OurDB store
let store = OurDbStore::<T>::open(&self.db_path)?;
self.type_map.insert(TypeId::of::<T>(), Arc::new(RwLock::new(store)));
// Initialize the TST index for this model type
let prefix = T::db_prefix();
let mut tst_index = self.tst_index.write().unwrap();
// Ensure the TST for this prefix exists
tst_index.get_tst(prefix)?;
Ok(())
}
}

8
herodb/src/db/mod.rs
View File

@ -14,5 +14,13 @@ pub use store::{DbOperations, OurDbStore};
pub mod db;
pub use db::{DB, DBBuilder, ModelRegistration, ModelRegistrar};
// Export the TST index module
pub mod tst_index;
pub use tst_index::TSTIndexManager;
// Export macros for model methods
pub mod macros;
// Tests
#[cfg(test)]
mod tests;

20
herodb/src/db/model.rs
View File

@ -1,4 +1,4 @@
use crate::db::error::{DbError, DbResult};
...use crate::db::error::{DbError, DbResult};
use serde::{Deserialize, Serialize};
use std::fmt::Debug;
@ -15,6 +15,16 @@ pub trait Storable: Serialize + for<'de> Deserialize<'de> + Sized {
}
}
/// Represents an index key for a model
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct IndexKey {
/// The name of the index key
pub name: &'static str,
/// The value of the index key for a specific model instance
pub value: String,
}
/// Trait identifying a model suitable for the database
/// The 'static lifetime bound is required for type identification via Any
pub trait Model: Storable + Debug + Clone + Send + Sync + 'static {
@ -24,6 +34,14 @@ pub trait Model: Storable + Debug + Clone + Send + Sync + 'static {
/// Returns a prefix used for this model type in the database
/// Helps to logically separate different model types
fn db_prefix() -> &'static str;
/// Returns a list of index keys for this model instance
/// These keys will be used to create additional indexes in the TST
/// The default implementation returns an empty vector
/// Override this method to provide custom indexes
fn db_keys(&self) -> Vec<IndexKey> {
Vec::new()
}
}
// Note: We don't provide a blanket implementation of Storable

98
herodb/src/db/tests.rs Normal file
View File

@ -0,0 +1,98 @@
use super::*;
use crate::db::model::Storable;
use serde::{Deserialize, Serialize};
use tempfile::tempdir;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
struct TestModel {
id: u32,
name: String,
}
impl Storable for TestModel {}
impl Model for TestModel {
fn get_id(&self) -> u32 {
self.id
}
fn db_prefix() -> &'static str {
"test"
}
}
#[test]
fn test_tst_integration() {
// Create a temporary directory for the test
let temp_dir = tempdir().unwrap();
let path = temp_dir.path();
// Create a DB instance
let mut db = DB::new(path).unwrap();
db.register::<TestModel>().unwrap();
// Create some test models
let model1 = TestModel { id: 1, name: "Test 1".to_string() };
let model2 = TestModel { id: 2, name: "Test 2".to_string() };
let model3 = TestModel { id: 3, name: "Test 3".to_string() };
// Insert the models
db.set(&model1).unwrap();
db.set(&model2).unwrap();
db.set(&model3).unwrap();
// List all models
let models = db.list::<TestModel>().unwrap();
assert_eq!(models.len(), 3);
// Verify that all models are in the list
assert!(models.contains(&model1));
assert!(models.contains(&model2));
assert!(models.contains(&model3));
// Delete a model
db.delete::<TestModel>(2).unwrap();
// List again
let models = db.list::<TestModel>().unwrap();
assert_eq!(models.len(), 2);
assert!(models.contains(&model1));
assert!(models.contains(&model3));
assert!(!models.contains(&model2));
// Test transaction with commit
db.begin_transaction().unwrap();
db.set(&model2).unwrap(); // Add back model2
db.delete::<TestModel>(1).unwrap(); // Delete model1
db.commit_transaction().unwrap();
// List again after transaction
let models = db.list::<TestModel>().unwrap();
assert_eq!(models.len(), 2);
assert!(!models.contains(&model1));
assert!(models.contains(&model2));
assert!(models.contains(&model3));
// Test transaction with rollback
db.begin_transaction().unwrap();
db.delete::<TestModel>(3).unwrap(); // Delete model3
db.rollback_transaction().unwrap();
// List again after rollback
let models = db.list::<TestModel>().unwrap();
assert_eq!(models.len(), 2);
assert!(!models.contains(&model1));
assert!(models.contains(&model2));
assert!(models.contains(&model3));
// Test the synchronize_tst_index method
// Since we can't directly access private fields, we'll just verify that
// the method runs without errors
db.synchronize_tst_index::<TestModel>().unwrap();
// Verify that our models are still accessible
let models = db.list::<TestModel>().unwrap();
assert_eq!(models.len(), 2);
assert!(models.contains(&model2));
assert!(models.contains(&model3));
}

156
herodb/src/db/tst_index.rs Normal file
View File

@ -0,0 +1,156 @@
use crate::db::error::{DbError, DbResult};
use std::path::{Path, PathBuf};
use std::collections::HashMap;
use tst::TST;
/// Manages TST-based indexes for model objects
pub struct TSTIndexManager {
/// Base path for TST databases
base_path: PathBuf,
/// Map of model prefixes to their TST instances
tst_instances: HashMap<String, TST>,
}
impl TSTIndexManager {
/// Creates a new TST index manager
pub fn new<P: AsRef<Path>>(base_path: P) -> DbResult<Self> {
let base_path = base_path.as_ref().to_path_buf();
// Create directory if it doesn't exist
std::fs::create_dir_all(&base_path).map_err(DbError::IoError)?;
Ok(Self {
base_path,
tst_instances: HashMap::new(),
})
}
/// Gets or creates a TST instance for a model prefix
pub fn get_tst(&mut self, prefix: &str) -> DbResult<&mut TST> {
if !self.tst_instances.contains_key(prefix) {
// Create a new TST instance for this prefix
let tst_path = self.base_path.join(format!("{}_tst", prefix));
let tst_path_str = tst_path.to_string_lossy().to_string();
// Create the TST
let tst = TST::new(&tst_path_str, false)
.map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?;
// Insert it into the map
self.tst_instances.insert(prefix.to_string(), tst);
}
// Return a mutable reference to the TST
Ok(self.tst_instances.get_mut(prefix).unwrap())
}
/// Adds or updates an object in the TST index
pub fn set(&mut self, prefix: &str, id: u32, data: Vec<u8>) -> DbResult<()> {
// Get the TST for this prefix
let tst = self.get_tst(prefix)?;
// Create the key in the format prefix_id
let key = format!("{}_{}", prefix, id);
// Set the key-value pair in the TST
tst.set(&key, data)
.map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?;
Ok(())
}
/// Removes an object from the TST index
pub fn delete(&mut self, prefix: &str, id: u32) -> DbResult<()> {
// Get the TST for this prefix
let tst = self.get_tst(prefix)?;
// Create the key in the format prefix_id
let key = format!("{}_{}", prefix, id);
// Delete the key from the TST
tst.delete(&key)
.map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?;
Ok(())
}
/// Lists all objects with a given prefix
pub fn list(&mut self, prefix: &str) -> DbResult<Vec<(u32, Vec<u8>)>> {
// Get the TST for this prefix
let tst = self.get_tst(prefix)?;
// Get all keys with this prefix
let keys = tst.list(prefix)
.map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?;
// Get all values for these keys
let mut result = Vec::with_capacity(keys.len());
for key in keys {
// Extract the ID from the key (format: prefix_id)
let id_str = key.split('_').nth(1).ok_or_else(|| {
DbError::GeneralError(format!("Invalid key format: {}", key))
})?;
let id = id_str.parse::<u32>().map_err(|_| {
DbError::GeneralError(format!("Invalid ID in key: {}", key))
})?;
// Get the value from the TST
let data = tst.get(&key)
.map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?;
result.push((id, data));
}
Ok(result)
}
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
#[test]
fn test_tst_index_manager() {
// Create a temporary directory for the test
let temp_dir = tempdir().unwrap();
let path = temp_dir.path();
// Create a TST index manager
let mut manager = TSTIndexManager::new(path).unwrap();
// Test setting values
let data1 = vec![1, 2, 3];
let data2 = vec![4, 5, 6];
manager.set("test", 1, data1.clone()).unwrap();
manager.set("test", 2, data2.clone()).unwrap();
// Test listing values
let items = manager.list("test").unwrap();
assert_eq!(items.len(), 2);
// Check that the values are correct
let mut found_data1 = false;
let mut found_data2 = false;
for (id, data) in items {
if id == 1 && data == data1 {
found_data1 = true;
} else if id == 2 && data == data2 {
found_data2 = true;
}
}
assert!(found_data1);
assert!(found_data2);
// Test deleting a value
manager.delete("test", 1).unwrap();
// Test listing again
let items = manager.list("test").unwrap();
assert_eq!(items.len(), 1);
assert_eq!(items[0].0, 2);
assert_eq!(items[0].1, data2);
}
}

2
herodb/src/models/biz/customer.rs
View File

@ -145,4 +145,6 @@ impl Model for Customer {
fn db_prefix() -> &'static str {
"customer"
}
}

451
tst_integration_plan.md Normal file
View File

@ -0,0 +1,451 @@
# TST Integration Plan for HeroDB
## Overview
This document outlines the plan for adding generic functionality to the `herodb/src/db` module to use the Ternary Search Tree (TST) for storing objects with prefixed IDs and implementing a generic list function to retrieve all objects with a specific prefix.
## Current Architecture
Currently:
- Each model has a `db_prefix()` method that returns a string prefix (e.g., "vote" for Vote objects)
- Objects are stored in OurDB with numeric IDs
- The `list()` method in `OurDbStore` is not implemented
## Implementation Plan
### 1. Create a TST-based Index Manager (herodb/src/db/tst_index.rs)
Create a new module that manages TST instances for different model prefixes:
```rust
use crate::db::error::{DbError, DbResult};
use std::path::{Path, PathBuf};
use tst::TST;
/// Manages TST-based indexes for model objects
pub struct TSTIndexManager {
/// Base path for TST databases
base_path: PathBuf,
/// Map of model prefixes to their TST instances
tst_instances: std::collections::HashMap<String, TST>,
}
impl TSTIndexManager {
/// Creates a new TST index manager
pub fn new<P: AsRef<Path>>(base_path: P) -> DbResult<Self> {
let base_path = base_path.as_ref().to_path_buf();
// Create directory if it doesn't exist
std::fs::create_dir_all(&base_path).map_err(DbError::IoError)?;
Ok(Self {
base_path,
tst_instances: std::collections::HashMap::new(),
})
}
/// Gets or creates a TST instance for a model prefix
pub fn get_tst(&mut self, prefix: &str) -> DbResult<&mut TST> {
if !self.tst_instances.contains_key(prefix) {
// Create a new TST instance for this prefix
let tst_path = self.base_path.join(format!("{}_tst", prefix));
let tst_path_str = tst_path.to_string_lossy().to_string();
// Create the TST
let tst = TST::new(&tst_path_str, false)
.map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?;
// Insert it into the map
self.tst_instances.insert(prefix.to_string(), tst);
}
// Return a mutable reference to the TST
Ok(self.tst_instances.get_mut(prefix).unwrap())
}
/// Adds or updates an object in the TST index
pub fn set(&mut self, prefix: &str, id: u32, data: Vec<u8>) -> DbResult<()> {
// Get the TST for this prefix
let tst = self.get_tst(prefix)?;
// Create the key in the format prefix_id
let key = format!("{}_{}", prefix, id);
// Set the key-value pair in the TST
tst.set(&key, data)
.map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?;
Ok(())
}
/// Removes an object from the TST index
pub fn delete(&mut self, prefix: &str, id: u32) -> DbResult<()> {
// Get the TST for this prefix
let tst = self.get_tst(prefix)?;
// Create the key in the format prefix_id
let key = format!("{}_{}", prefix, id);
// Delete the key from the TST
tst.delete(&key)
.map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?;
Ok(())
}
/// Lists all objects with a given prefix
pub fn list(&mut self, prefix: &str) -> DbResult<Vec<(u32, Vec<u8>)>> {
// Get the TST for this prefix
let tst = self.get_tst(prefix)?;
// Get all keys with this prefix
let keys = tst.list(prefix)
.map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?;
// Get all values for these keys
let mut result = Vec::with_capacity(keys.len());
for key in keys {
// Extract the ID from the key (format: prefix_id)
let id_str = key.split('_').nth(1).ok_or_else(|| {
DbError::GeneralError(format!("Invalid key format: {}", key))
})?;
let id = id_str.parse::<u32>().map_err(|_| {
DbError::GeneralError(format!("Invalid ID in key: {}", key))
})?;
// Get the value from the TST
let data = tst.get(&key)
.map_err(|e| DbError::GeneralError(format!("TST error: {:?}", e)))?;
result.push((id, data));
}
Ok(result)
}
}
```
### 2. Update DB Module (herodb/src/db/mod.rs)
Add the new module to the db module:
```rust
pub mod db;
pub mod error;
pub mod macros;
pub mod model;
pub mod model_methods;
pub mod store;
pub mod tst_index; // Add the new module
pub use db::DB;
pub use db::DBBuilder;
pub use error::{DbError, DbResult};
pub use model::Model;
pub use model::Storable;
```
### 3. Modify DB Struct (herodb/src/db/db.rs)
Update the DB struct to include the TST index manager:
```rust
/// Main DB manager that automatically handles all models
#[derive(Clone, CustomType)]
pub struct DB {
db_path: PathBuf,
// Type map for generic operations
type_map: HashMap<TypeId, Arc<RwLock<dyn DbOperations>>>,
// TST index manager
tst_index: Arc<RwLock<TSTIndexManager>>,
// Transaction state
transaction: Arc<RwLock<Option<TransactionState>>>,
}
```
### 4. Extend Transaction Handling
Extend the `DbOperation` enum to include model prefix and ID information:
```rust
#[derive(Debug, Clone)]
enum DbOperation {
Set {
model_type: TypeId,
serialized: Vec<u8>,
model_prefix: String, // Add model prefix
model_id: u32, // Add model ID
},
Delete {
model_type: TypeId,
id: u32,
model_prefix: String, // Add model prefix
},
}
```
### 5. Update Transaction Recording
Modify the `set` and `delete` methods to record model prefix and ID in the transaction:
```rust
pub fn set<T: Model>(&self, model: &T) -> DbResult<()> {
// Try to acquire a write lock on the transaction
let mut tx_guard = self.transaction.write().unwrap();
// Check if there's an active transaction
if let Some(tx_state) = tx_guard.as_mut() {
if tx_state.active {
// Serialize the model for later use
let serialized = model.to_bytes()?;
// Record a Set operation in the transaction with prefix and ID
tx_state.operations.push(DbOperation::Set {
model_type: TypeId::of::<T>(),
serialized: serialized.clone(),
model_prefix: T::db_prefix().to_string(),
model_id: model.get_id(),
});
return Ok(());
}
}
// ... rest of the method ...
}
pub fn delete<T: Model>(&self, id: u32) -> DbResult<()> {
// Try to acquire a write lock on the transaction
let mut tx_guard = self.transaction.write().unwrap();
// Check if there's an active transaction
if let Some(tx_state) = tx_guard.as_mut() {
if tx_state.active {
// Record a Delete operation in the transaction with prefix
tx_state.operations.push(DbOperation::Delete {
model_type: TypeId::of::<T>(),
id,
model_prefix: T::db_prefix().to_string(),
});
return Ok(());
}
}
// ... rest of the method ...
}
```
### 6. Update Transaction Commit
Modify the `commit_transaction` method to update both OurDB and the TST index:
```rust
pub fn commit_transaction(&self) -> DbResult<()> {
let mut tx_guard = self.transaction.write().unwrap();
if let Some(tx_state) = tx_guard.take() {
if !tx_state.active {
return Err(DbError::TransactionError("Transaction not active".into()));
}
// Create a backup of the transaction state in case we need to rollback
let backup = tx_state.clone();
// Try to execute all operations
let result = (|| {
for op in &tx_state.operations {
match op {
DbOperation::Set {
model_type,
serialized,
model_prefix,
model_id,
} => {
// Apply to OurDB
self.apply_set_operation(*model_type, serialized)?;
// Apply to TST index
let mut tst_index = self.tst_index.write().unwrap();
tst_index.set(model_prefix, *model_id, serialized.clone())?;
}
DbOperation::Delete {
model_type,
id,
model_prefix,
} => {
// Apply to OurDB
let db_ops = self
.type_map
.get(model_type)
.ok_or_else(|| DbError::TypeError)?;
let mut db_ops_guard = db_ops.write().unwrap();
db_ops_guard.delete(*id)?;
// Apply to TST index
let mut tst_index = self.tst_index.write().unwrap();
tst_index.delete(model_prefix, *id)?;
}
}
}
Ok(())
})();
// If any operation failed, restore the transaction state
if result.is_err() {
*tx_guard = Some(backup);
return result;
}
Ok(())
} else {
Err(DbError::TransactionError("No active transaction".into()))
}
}
```
### 7. Implement List Method
Implement the `list` method to use the TST's prefix search:
```rust
pub fn list<T: Model>(&self) -> DbResult<Vec<T>> {
// Get the prefix for this model type
let prefix = T::db_prefix();
// Use the TST index to get all objects with this prefix
let mut tst_index = self.tst_index.write().unwrap();
let items = tst_index.list(prefix)?;
// Deserialize the objects
let mut result = Vec::with_capacity(items.len());
for (_, data) in items {
let model = T::from_bytes(&data)?;
result.push(model);
}
Ok(result)
}
```
### 8. Add Recovery Mechanism
Add a method to synchronize the TST index with OurDB in case they get out of sync:
```rust
pub fn synchronize_tst_index<T: Model>(&self) -> DbResult<()> {
// Get all models from OurDB
let models = self.list_from_ourdb::<T>()?;
// Clear the TST index for this model type
let mut tst_index = self.tst_index.write().unwrap();
let prefix = T::db_prefix();
// Rebuild the TST index
for model in models {
let id = model.get_id();
let data = model.to_bytes()?;
tst_index.set(prefix, id, data)?;
}
Ok(())
}
// Helper method to list models directly from OurDB (not using TST)
fn list_from_ourdb<T: Model>(&self) -> DbResult<Vec<T>> {
match self.type_map.get(&TypeId::of::<T>()) {
Some(db_ops) => {
let db_ops_guard = db_ops.read().unwrap();
let result_any = db_ops_guard.list()?;
match result_any.downcast::<Vec<T>>() {
Ok(vec_t) => Ok(*vec_t),
Err(_) => Err(DbError::TypeError),
}
}
None => Err(DbError::TypeError),
}
}
```
## Implementation Flow
```mermaid
sequenceDiagram
participant Client
participant DB
participant TransactionState
participant OurDbStore
participant TSTIndexManager
participant TST
Client->>DB: begin_transaction()
DB->>TransactionState: create new transaction
Client->>DB: set(model)
DB->>TransactionState: record Set operation with prefix and ID
Client->>DB: delete(model)
DB->>TransactionState: record Delete operation with prefix and ID
Client->>DB: commit_transaction()
DB->>TransactionState: get all operations
loop For each operation
alt Set operation
DB->>OurDbStore: apply_set_operation()
DB->>TSTIndexManager: set(prefix, id, data)
TSTIndexManager->>TST: set(key, data)
else Delete operation
DB->>OurDbStore: delete(id)
DB->>TSTIndexManager: delete(prefix, id)
TSTIndexManager->>TST: delete(key)
end
end
alt Success
DB-->>Client: Ok(())
else Error
DB->>TransactionState: restore transaction state
DB-->>Client: Err(error)
end
Client->>DB: list<T>()
DB->>TSTIndexManager: list(prefix)
TSTIndexManager->>TST: list(prefix)
TST-->>TSTIndexManager: keys
TSTIndexManager->>TST: get(key) for each key
TST-->>TSTIndexManager: data
TSTIndexManager-->>DB: (id, data) pairs
DB->>DB: deserialize data to models
DB-->>Client: Vec<T>
```
## Testing Strategy
1. Create unit tests for the TST index manager
2. Test the list functionality with different model types
3. Test transaction handling (commit and rollback)
4. Test error recovery mechanisms
5. Test edge cases (empty database, large datasets)
## Implementation Steps
1. Add TST dependency to herodb/Cargo.toml
2. Create the tst_index.rs module
3. Update the DB module to include the TST index manager
4. Extend the transaction handling
5. Implement the list method
6. Add tests for the new functionality
7. Update documentation
## Considerations
1. **Performance**: The TST operations add overhead to insert/delete operations, but provide efficient list functionality.
2. **Consistency**: The enhanced transaction handling ensures consistency between OurDB and the TST index.
3. **Error Handling**: Proper error handling and recovery mechanisms are essential for maintaining data integrity.
4. **Backward Compatibility**: The implementation should maintain backward compatibility with existing code.