herocode/db
13
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

...

Browse Source
This commit is contained in:
despiegk
2025-04-22 07:50:03 +04:00
parent 6443c6b647
commit d75de1e73c
20 changed files with 1007 additions and 640 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
herodb/src/cmd/dbexample_mcc/main.rs
View File

@@ -1,5 +1,5 @@
use chrono::{Utc, Duration};
use herodb::db::DBBuilder;
use herodb::db::{DBBuilder, GetId};
use herodb::models::mcc::{
Calendar, Event,
Email, Attachment, Envelope,
@@ -23,12 +23,12 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
// Create a database instance with our models registered
let db = DBBuilder::new(&db_path)
.register_model::<Calendar>()
.register_model::<Event>()
.register_model::<Email>()
.register_model::<Contact>()
.register_model::<Message>()
.register_model::<Circle>()
.register_type::<Calendar>("calendar")
.register_type::<Event>("event")
.register_type::<Email>("email")
.register_type::<Contact>("contact")
.register_type::<Message>("message")
.register_model::<Circle>() // Circle still uses the Model trait
.build()?;
println!("\n1. Creating Circles (Groups)");
@@ -48,8 +48,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
);
let friends_circle = Circle::new(
3,
"Friends".to_string(),
3,
"Friends".to_string(),
"Friends communications".to_string()
);
@@ -96,9 +96,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
bob.add_group(work_circle.id); // Bob is both a friend and a work contact
// Insert contacts
db.set::<Contact>(&john)?;
db.set::<Contact>(&alice)?;
db.set::<Contact>(&bob)?;
db.set_any::<Contact>(&john, "contact")?;
db.set_any::<Contact>(&alice, "contact")?;
db.set_any::<Contact>(&bob, "contact")?;
println!("Created contacts:");
println!(" - {}: {} (Groups: {:?})", john.full_name(), john.email, john.groups);
@@ -125,8 +125,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
personal_calendar.add_group(friends_circle.id);
// Insert calendars
db.set::<Calendar>(&work_calendar)?;
db.set::<Calendar>(&personal_calendar)?;
db.set_any::<Calendar>(&work_calendar, "calendar")?;
db.set_any::<Calendar>(&personal_calendar, "calendar")?;
println!("Created calendars:");
println!(" - {}: {} (Groups: {:?})", work_calendar.id, work_calendar.title, work_calendar.groups);
@@ -168,8 +168,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
family_dinner.add_attendee(alice.email.clone());
// Insert events
db.set::<Event>(&work_meeting)?;
db.set::<Event>(&family_dinner)?;
db.set_any::<Event>(&work_meeting, "event")?;
db.set_any::<Event>(&family_dinner, "event")?;
println!("Created events:");
println!(" - {}: {} on {} (Groups: {:?})",
@@ -244,8 +244,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
family_email.set_envelope(family_envelope);
// Insert emails
db.set::<Email>(&work_email)?;
db.set::<Email>(&family_email)?;
db.set_any::<Email>(&work_email, "email")?;
db.set_any::<Email>(&family_email, "email")?;
println!("Created emails:");
println!(" - From: {}, Subject: {} (Groups: {:?})",
@@ -284,8 +284,8 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
friends_chat.add_reaction("👍".to_string());
// Insert messages
db.set::<Message>(&work_chat)?;
db.set::<Message>(&friends_chat)?;
db.set_any::<Message>(&work_chat, "message")?;
db.set_any::<Message>(&friends_chat, "message")?;
println!("Created messages:");
println!(" - From: {}, Content: {} (Groups: {:?})",
@@ -305,7 +305,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
// Filter contacts by group
println!("\nFiltering contacts by work group (ID: {}):", work_circle.id);
let all_contacts = db.list::<Contact>()?;
let all_contacts = db.list_any::<Contact>()?;
for contact in all_contacts {
if contact.filter_by_groups(&[work_circle.id]) {
println!(" - {} ({})", contact.full_name(), contact.email);
@@ -314,7 +314,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
// Search emails by subject
println!("\nSearching emails with subject containing 'Meeting':");
let all_emails = db.list::<Email>()?;
let all_emails = db.list_any::<Email>()?;
for email in all_emails {
if email.search_by_subject("Meeting") {
println!(" - Subject: {}, From: {}",
@@ -326,7 +326,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
// Get events for a calendar
println!("\nGetting events for Work Calendar (ID: {}):", work_calendar.id);
let all_events = db.list::<Event>()?;
let all_events = db.list_any::<Event>()?;
let work_events: Vec<Event> = all_events
.into_iter()
.filter(|event| event.calendar_id == work_calendar.id)
@@ -341,7 +341,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
// Get attendee contacts for an event
println!("\nGetting attendee contacts for Team Meeting (ID: {}):", work_meeting.id);
let all_contacts = db.list::<Contact>()?;
let all_contacts = db.list_any::<Contact>()?;
let attendee_contacts: Vec<Contact> = all_contacts
.into_iter()
.filter(|contact| work_meeting.attendees.contains(&contact.email))
@@ -358,19 +358,19 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
println!(" - Converted Message Groups: {:?}", email_to_message.groups);
// Insert the converted message
db.set::<Message>(&email_to_message)?;
db.set_any::<Message>(&email_to_message, "message")?;
println!("\n8. Relationship Management");
println!("------------------------");
// Get the calendar for an event
println!("\nGetting calendar for Family Dinner event (ID: {}):", family_dinner.id);
let event_calendar = db.get::<Calendar>(family_dinner.calendar_id)?;
let event_calendar = db.get_any::<Calendar>(family_dinner.calendar_id)?;
println!(" - Calendar: {} ({})", event_calendar.title, event_calendar.description);
// Get events for a contact
println!("\nGetting events where John Doe is an attendee:");
let all_events = db.list::<Event>()?;
let all_events = db.list_any::<Event>()?;
let john_events: Vec<Event> = all_events
.into_iter()
.filter(|event| event.attendees.contains(&john.email))
@@ -385,7 +385,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
// Get messages in the same thread
println!("\nGetting all messages in the work chat thread:");
let all_messages = db.list::<Message>()?;
let all_messages = db.list_any::<Message>()?;
let thread_messages: Vec<Message> = all_messages
.into_iter()
.filter(|message| message.thread_id == work_chat.thread_id)

298
herodb/src/db/db.rs
View File

@@ -1,6 +1,7 @@
use crate::db::error::{DbError, DbResult};
use crate::db::model::Model;
use crate::db::model::{Model, IndexKey};
use crate::db::store::{DbOperations, OurDbStore};
use crate::db::generic_store::{GenericStore, GetId};
use crate::db::tst_index::TSTIndexManager;
use std::any::TypeId;
use std::collections::HashMap;
@@ -8,6 +9,7 @@ use std::fmt::Debug;
use std::path::{Path, PathBuf};
use std::sync::{Arc, RwLock};
use rhai::{CustomType, EvalAltResult, TypeBuilder};
use serde::{Serialize, de::DeserializeOwned};
/// Represents a single database operation in a transaction
#[derive(Debug, Clone)]
@@ -82,6 +84,21 @@ impl<T: Model> ModelRegistrar<T> {
}
}
/// Implementation of ModelRegistration for any serializable type that implements GetId
pub struct TypeRegistrar<T: Serialize + DeserializeOwned + GetId + Send + Sync + 'static> {
prefix: &'static str,
phantom: std::marker::PhantomData<T>,
}
impl<T: Serialize + DeserializeOwned + GetId + Send + Sync + 'static> TypeRegistrar<T> {
pub fn new(prefix: &'static str) -> Self {
Self {
prefix,
phantom: std::marker::PhantomData,
}
}
}
impl<T: Model> ModelRegistration for ModelRegistrar<T> {
fn register(&self, path: &Path) -> DbResult<(TypeId, Arc<RwLock<dyn DbOperations>>)> {
let store = OurDbStore::<T>::open(path.join(T::db_prefix()))?;
@@ -89,6 +106,13 @@ impl<T: Model> ModelRegistration for ModelRegistrar<T> {
}
}
impl<T: Serialize + DeserializeOwned + GetId + Send + Sync + 'static> ModelRegistration for TypeRegistrar<T> {
fn register(&self, path: &Path) -> DbResult<(TypeId, Arc<RwLock<dyn DbOperations>>)> {
let store = GenericStore::<T>::open(path, self.prefix)?;
Ok((TypeId::of::<T>(), Arc::new(RwLock::new(store)) as Arc<RwLock<dyn DbOperations>>))
}
}
impl DBBuilder {
/// Create a new DB builder
pub fn new<P: Into<PathBuf>>(base_path: P) -> Self {
@@ -112,6 +136,16 @@ impl DBBuilder {
self
}
/// Register any serializable type with the DB
pub fn register_type<T: Serialize + DeserializeOwned + GetId + Send + Sync + 'static>(
mut self,
prefix: &'static str
) -> Self {
self.model_registrations
.push(Arc::new(TypeRegistrar::<T>::new(prefix)));
self
}
/// Build the DB with the registered models
pub fn build(self) -> Result<DB, Box<EvalAltResult>> {
let base_path = self.base_path;
@@ -342,6 +376,57 @@ impl DB {
}
}
/// Insert any serializable struct that implements GetId
pub fn set_any<T: Serialize + DeserializeOwned + GetId + Send + Sync + 'static>(
&self,
item: &T,
prefix: &str
) -> DbResult<()> {
// Try to acquire a write lock on the transaction
let mut tx_guard = self.transaction.write().unwrap();
// Check if there's an active transaction
if let Some(tx_state) = tx_guard.as_mut() {
if tx_state.active {
// Serialize the item for later use
let serialized = bincode::serialize(item).map_err(DbError::SerializationError)?;
// Record a Set operation in the transaction with prefix and ID
tx_state.operations.push(DbOperation::Set {
model_type: TypeId::of::<T>(),
serialized,
model_prefix: prefix.to_string(),
model_id: item.get_id(),
});
return Ok(());
}
}
// If we got here, either there's no transaction or it's not active
// Drop the write lock before doing a direct database operation
drop(tx_guard);
// Execute directly
match self.type_map.get(&TypeId::of::<T>()) {
Some(db_ops) => {
// Serialize the item
let data = bincode::serialize(item).map_err(DbError::SerializationError)?;
// Insert the raw data
let mut db_ops_guard = db_ops.write().unwrap();
db_ops_guard.insert_raw(&data)?;
// Also update the TST index (primary key only)
let mut tst_index = self.tst_index.write().unwrap();
tst_index.set(prefix, item.get_id(), data)?;
Ok(())
},
None => Err(DbError::TypeError),
}
}
/// Check the transaction state for the given type and id
fn check_transaction<T: Model>(&self, id: u32) -> Option<Result<Option<T>, DbError>> {
// Try to acquire a read lock on the transaction
@@ -399,19 +484,41 @@ impl DB {
if let Some(tx_result) = self.check_transaction::<T>(id) {
match tx_result {
Ok(Some(model)) => return Ok(model),
Ok(None) => return Err(DbError::NotFound(id)),
Err(e) => return Err(e),
Ok(None) => {} // Should never happen
}
}
// If no pending value, look up from the database
// If not found in transaction, get from database
match self.type_map.get(&TypeId::of::<T>()) {
Some(db_ops) => {
let mut db_ops_guard = db_ops.write().unwrap();
let result_any = db_ops_guard.get(id)?;
// We expect the result to be of type T since we looked it up by TypeId
match result_any.downcast::<T>() {
Ok(t) => Ok(*t),
let any_result = db_ops_guard.get(id)?;
// Try to downcast to T
match any_result.downcast::<T>() {
Ok(boxed_t) => Ok(*boxed_t),
Err(_) => Err(DbError::TypeError),
}
}
None => Err(DbError::TypeError),
}
}
/// Get any serializable struct by its ID and type
pub fn get_any<T: Serialize + DeserializeOwned + GetId + Send + Sync + 'static>(
&self,
id: u32
) -> DbResult<T> {
// If not found in transaction, get from database
match self.type_map.get(&TypeId::of::<T>()) {
Some(db_ops) => {
let mut db_ops_guard = db_ops.write().unwrap();
let any_result = db_ops_guard.get(id)?;
// Try to downcast to T
match any_result.downcast::<T>() {
Ok(boxed_t) => Ok(*boxed_t),
Err(_) => Err(DbError::TypeError),
}
}
@@ -421,17 +528,13 @@ impl DB {
/// Delete a model instance by its ID and type
pub fn delete<T: Model>(&self, id: u32) -> DbResult<()> {
// First, get the model to extract its index keys
let model = self.get::<T>(id)?;
let index_keys = model.db_keys();
// Try to acquire a write lock on the transaction
let mut tx_guard = self.transaction.write().unwrap();
// Check if there's an active transaction
if let Some(tx_state) = tx_guard.as_mut() {
if tx_state.active {
// Record a Delete operation in the transaction with prefix
// Record a Delete operation in the transaction
tx_state.operations.push(DbOperation::Delete {
model_type: TypeId::of::<T>(),
id,
@@ -452,45 +555,48 @@ impl DB {
let mut db_ops_guard = db_ops.write().unwrap();
db_ops_guard.delete(id)?;
// Also update the TST index with all index keys
// Also delete from the TST index
let mut tst_index = self.tst_index.write().unwrap();
let prefix = T::db_prefix();
tst_index.delete_with_indexes(prefix, id, &index_keys)?;
tst_index.delete(T::db_prefix(), id)?;
Ok(())
},
}
None => Err(DbError::TypeError),
}
}
/// List all model instances of a specific type
pub fn list<T: Model>(&self) -> DbResult<Vec<T>> {
// Get the prefix for this model type
let prefix = T::db_prefix();
// Use the TST index to get all objects with this prefix
let mut tst_index = self.tst_index.write().unwrap();
let items = tst_index.list(prefix)?;
// Deserialize the objects
let mut result = Vec::with_capacity(items.len());
for (_, data) in items {
let model = T::from_bytes(&data)?;
result.push(model);
/// Delete any serializable struct by its ID and type
pub fn delete_any<T: Serialize + DeserializeOwned + GetId + Send + Sync + 'static>(
&self,
id: u32,
prefix: &str
) -> DbResult<()> {
// Execute directly
match self.type_map.get(&TypeId::of::<T>()) {
Some(db_ops) => {
let mut db_ops_guard = db_ops.write().unwrap();
db_ops_guard.delete(id)?;
// Also delete from the TST index
let mut tst_index = self.tst_index.write().unwrap();
tst_index.delete(prefix, id)?;
Ok(())
}
None => Err(DbError::TypeError),
}
Ok(result)
}
/// Helper method to list models directly from OurDB (not using TST)
fn list_from_ourdb<T: Model>(&self) -> DbResult<Vec<T>> {
/// List all model instances of a given type
pub fn list<T: Model>(&self) -> DbResult<Vec<T>> {
match self.type_map.get(&TypeId::of::<T>()) {
Some(db_ops) => {
let db_ops_guard = db_ops.read().unwrap();
let result_any = db_ops_guard.list()?;
// We expect the result to be of type Vec<T> since we looked it up by TypeId
match result_any.downcast::<Vec<T>>() {
Ok(vec_t) => Ok(*vec_t),
let any_result = db_ops_guard.list()?;
// Try to downcast to Vec<T>
match any_result.downcast::<Vec<T>>() {
Ok(boxed_vec) => Ok(*boxed_vec),
Err(_) => Err(DbError::TypeError),
}
}
@@ -498,105 +604,43 @@ impl DB {
}
}
/// Synchronize the TST index with OurDB for a specific model type
pub fn synchronize_tst_index<T: Model>(&self) -> DbResult<()> {
// Get all models from OurDB
let models = self.list_from_ourdb::<T>()?;
// Clear the TST index for this model type
let mut tst_index = self.tst_index.write().unwrap();
let prefix = T::db_prefix();
// Rebuild the TST index with all index keys
for model in models {
let id = model.get_id();
let data = model.to_bytes()?;
let index_keys = model.db_keys();
tst_index.set_with_indexes(prefix, id, data, &index_keys)?;
}
Ok(())
}
/// Get the history of a model by its ID
pub fn get_history<T: Model>(&self, id: u32, depth: u8) -> DbResult<Vec<T>> {
// Look up the correct DB operations for type T in our type map
/// List all instances of any serializable type
pub fn list_any<T: Serialize + DeserializeOwned + GetId + Send + Sync + 'static>(
&self
) -> DbResult<Vec<T>> {
match self.type_map.get(&TypeId::of::<T>()) {
Some(db_ops) => {
let mut db_ops_guard = db_ops.write().unwrap();
let result_any = db_ops_guard.get_history(id, depth)?;
let mut result = Vec::with_capacity(result_any.len());
let db_ops_guard = db_ops.read().unwrap();
let any_result = db_ops_guard.list()?;
for item in result_any {
match item.downcast::<T>() {
Ok(t) => result.push(*t),
Err(_) => return Err(DbError::TypeError),
}
// Try to downcast to Vec<T>
match any_result.downcast::<Vec<T>>() {
Ok(boxed_vec) => Ok(*boxed_vec),
Err(_) => Err(DbError::TypeError),
}
Ok(result)
}
None => Err(DbError::TypeError),
}
}
// Register a model type with this DB instance
pub fn register<T: Model>(&mut self) -> DbResult<()> {
// Create the OurDB store
let store = OurDbStore::<T>::open(&self.db_path)?;
self.type_map.insert(TypeId::of::<T>(), Arc::new(RwLock::new(store)));
// Initialize the TST index for this model type
let prefix = T::db_prefix();
let mut tst_index = self.tst_index.write().unwrap();
// Ensure the TST for this prefix exists
tst_index.get_tst(prefix)?;
Ok(())
}
/// Find a model by a specific index key
pub fn find_by_index<T: Model>(&self, index_name: &str, index_value: &str) -> DbResult<Vec<T>> {
// Get the prefix for this model type
let prefix = T::db_prefix();
// Use the TST index to find objects with this index key
let mut tst_index = self.tst_index.write().unwrap();
let ids = tst_index.find_by_index(prefix, index_name, index_value)?;
// Get the objects by their IDs
let mut result = Vec::with_capacity(ids.len());
for id in ids {
match self.get::<T>(id) {
Ok(model) => result.push(model),
Err(DbError::NotFound(_)) => continue, // Skip if not found
Err(e) => return Err(e),
/// Get the history of a model instance
pub fn get_history<T: Model>(&self, id: u32, depth: u8) -> DbResult<Vec<T>> {
match self.type_map.get(&TypeId::of::<T>()) {
Some(db_ops) => {
let mut db_ops_guard = db_ops.write().unwrap();
let any_results = db_ops_guard.get_history(id, depth)?;
let mut results = Vec::with_capacity(any_results.len());
for any_result in any_results {
match any_result.downcast::<T>() {
Ok(boxed_t) => results.push(*boxed_t),
Err(_) => return Err(DbError::TypeError),
}
}
Ok(results)
}
None => Err(DbError::TypeError),
}
Ok(result)
}
/// Find models by a prefix of an index key
pub fn find_by_index_prefix<T: Model>(&self, index_name: &str, index_value_prefix: &str) -> DbResult<Vec<T>> {
// Get the prefix for this model type
let prefix = T::db_prefix();
// Use the TST index to find objects with this index key prefix
let mut tst_index = self.tst_index.write().unwrap();
let ids = tst_index.find_by_index_prefix(prefix, index_name, index_value_prefix)?;
// Get the objects by their IDs
let mut result = Vec::with_capacity(ids.len());
for id in ids {
match self.get::<T>(id) {
Ok(model) => result.push(model),
Err(DbError::NotFound(_)) => continue, // Skip if not found
Err(e) => return Err(e),
}
}
Ok(result)
}
}

140
herodb/src/db/generic_store.rs Normal file
View File

@@ -0,0 +1,140 @@
use crate::db::error::{DbError, DbResult};
use crate::db::store::DbOperations;
use ourdb::{OurDB, OurDBConfig, OurDBSetArgs};
use serde::{Serialize, de::DeserializeOwned};
use std::marker::PhantomData;
use std::path::{Path, PathBuf};
use std::any::Any;
// Trait for getting ID from any serializable type
pub trait GetId {
fn get_id(&self) -> u32;
}
/// A store implementation for any serializable type using OurDB as the backend
pub struct GenericStore<T: Serialize + DeserializeOwned + GetId + Send + Sync + 'static> {
db: OurDB,
path: PathBuf,
prefix: String,
_phantom: PhantomData<T>,
}
impl<T: Serialize + DeserializeOwned + GetId + Send + Sync + 'static> GenericStore<T> {
/// Opens or creates an OurDB database at the specified path
pub fn open<P: AsRef<Path>>(path: P, prefix: &str) -> DbResult<Self> {
let path_buf = path.as_ref().to_path_buf();
let db_path = path_buf.join(prefix);
// Create directory if it doesn't exist
std::fs::create_dir_all(&db_path).map_err(DbError::IoError)?;
let config = OurDBConfig {
path: db_path.clone(),
incremental_mode: true, // Always use incremental mode for auto IDs
file_size: None, // Use default (500MB)
keysize: None, // Use default (4 bytes)
reset: None, // Don't reset existing database
};
let db = OurDB::new(config).map_err(DbError::OurDbError)?;
Ok(Self {
db,
path: db_path,
prefix: prefix.to_string(),
_phantom: PhantomData,
})
}
/// Serializes an item to bytes
fn serialize(item: &T) -> DbResult<Vec<u8>> {
bincode::serialize(item).map_err(DbError::SerializationError)
}
/// Deserializes bytes to an item
fn deserialize(data: &[u8]) -> DbResult<T> {
bincode::deserialize(data).map_err(DbError::SerializationError)
}
/// Gets the raw bytes for an item by ID
pub fn get_raw(&self, id: u32) -> DbResult<Vec<u8>> {
self.db.get(id).map_err(DbError::OurDbError)
}
/// Lists all raw items as bytes
pub fn list_raw(&self) -> DbResult<Vec<Vec<u8>>> {
let items = self.db.list().map_err(DbError::OurDbError)?;
Ok(items)
}
/// Get the prefix for this store
pub fn prefix(&self) -> &str {
&self.prefix
}
}
impl<T: Serialize + DeserializeOwned + GetId + Send + Sync + 'static> DbOperations for GenericStore<T> {
fn delete(&mut self, id: u32) -> DbResult<()> {
self.db.delete(id).map_err(DbError::OurDbError)
}
fn get(&mut self, id: u32) -> DbResult<Box<dyn Any>> {
let data = self.db.get(id).map_err(DbError::OurDbError)?;
let item = Self::deserialize(&data)?;
Ok(Box::new(item))
}
fn list(&self) -> DbResult<Box<dyn Any>> {
let items = self.db.list().map_err(DbError::OurDbError)?;
let mut result = Vec::with_capacity(items.len());
for data in items {
let item = Self::deserialize(&data)?;
result.push(item);
}
Ok(Box::new(result))
}
fn insert(&mut self, model: &dyn Any) -> DbResult<()> {
// Try to downcast to T
if let Some(item) = model.downcast_ref::<T>() {
let data = Self::serialize(item)?;
let id = item.get_id();
let args = OurDBSetArgs {
id: Some(id),
data,
};
self.db.set(args).map_err(DbError::OurDbError)
} else {
Err(DbError::TypeError)
}
}
fn insert_raw(&mut self, serialized: &[u8]) -> DbResult<()> {
// Deserialize to get the ID
let item = Self::deserialize(serialized)?;
let id = item.get_id();
let args = OurDBSetArgs {
id: Some(id),
data: serialized.to_vec(),
};
self.db.set(args).map_err(DbError::OurDbError)
}
fn get_history(&mut self, id: u32, depth: u8) -> DbResult<Vec<Box<dyn Any>>> {
let history = self.db.get_history(id, depth).map_err(DbError::OurDbError)?;
let mut result = Vec::with_capacity(history.len());
for data in history {
let item = Self::deserialize(&data)?;
result.push(Box::new(item));
}
Ok(result)
}
}

6
herodb/src/db/mod.rs
View File

@@ -4,12 +4,16 @@ pub use error::{DbError, DbResult};
// Export the model module
pub mod model;
pub use model::{Model, Storable, IndexKey};
pub use model::{Model, Storable, IndexKey, GetId};
// Export the store module
pub mod store;
pub use store::{DbOperations, OurDbStore};
// Export the generic store module
pub mod generic_store;
pub use generic_store::GenericStore;
// Export the db module
pub mod db;
pub use db::{DB, DBBuilder, ModelRegistration, ModelRegistrar};

50
herodb/src/db/model.rs
View File

@@ -1,5 +1,5 @@
use crate::db::error::{DbError, DbResult};
use serde::{Deserialize, Serialize};
use serde::{Deserialize, Serialize, de::DeserializeOwned};
use std::fmt::Debug;
/// Trait for models that can be serialized and deserialized
@@ -44,5 +44,53 @@ pub trait Model: Storable + Debug + Clone + Send + Sync + 'static {
}
}
/// Trait for adapting any serializable struct to work with the database
/// This is a lighter-weight alternative to the Model trait
pub trait ModelAdapter {
/// Returns the unique ID for this model instance
fn get_id(&self) -> u32;
/// Returns a prefix used for this model type in the database
fn db_prefix() -> &'static str;
/// Returns a list of index keys for this model instance
fn db_keys(&self) -> Vec<IndexKey> {
Vec::new()
}
}
/// Trait for getting ID from any serializable type
pub trait GetId {
/// Returns the unique ID for this instance
fn get_id(&self) -> u32;
}
/// Macro to automatically implement GetId for any struct with an id field of type u32
#[macro_export]
macro_rules! impl_get_id {
($type:ty) => {
impl GetId for $type {
fn get_id(&self) -> u32 {
self.id
}
}
};
}
/// Helper functions for serializing and deserializing any type
pub mod serialization {
use super::*;
/// Serialize any serializable type to bytes
pub fn to_bytes<T: Serialize>(value: &T) -> DbResult<Vec<u8>> {
bincode::serialize(value).map_err(DbError::SerializationError)
}
/// Deserialize bytes to any deserializable type
pub fn from_bytes<T: DeserializeOwned>(data: &[u8]) -> DbResult<T> {
bincode::deserialize(data).map_err(DbError::SerializationError)
}
}
// Note: We don't provide a blanket implementation of Storable
// Each model type must implement Storable explicitly

2
herodb/src/lib.rs
View File

@@ -13,7 +13,7 @@ pub mod cmd;
// Re-exports
pub use error::Error;
pub use db::{DB, DBBuilder, Model, Storable, DbError, DbResult};
pub use db::{DB, DBBuilder, Model, Storable, DbError, DbResult, GetId};
/// Re-export ourdb for advanced usage
pub use ourdb;

316
herodb/src/models/mcc/MCC_ENHANCEMENT_PLAN.md
View File

@@ -1,316 +0,0 @@
# MCC Models Enhancement Plan
## 1. Current State Analysis
The current MCC module consists of:
- **Mail**: Email, Attachment, Envelope models
- **Calendar**: Calendar model
- **Event**: Event, EventMeta models
- **Contacts**: Contact model
All models implement the `Storable` and `SledModel` traits for database integration.
## 2. Planned Enhancements
### 2.1 Add Group Support to All Models
Add a `groups: Vec<u32>` field to each model to enable linking to multiple groups defined in the Circle module.
### 2.2 Create New Message Model
Create a new `message.rs` file with a Message model for chat functionality:
- Different structure from Email
- Include thread_id, sender_id, content fields
- Include metadata for chat-specific features
- Implement Storable and SledModel traits
### 2.3 Add Utility Methods
Add utility methods to each model for:
- **Filtering/Searching**: Methods to filter by groups, search by content/subject
- **Format Conversion**: Methods to convert between formats (e.g., Email to Message)
- **Relationship Management**: Methods to manage relationships between models
## 3. Implementation Plan
```mermaid
flowchart TD
A[Review Current Models] --> B[Add groups field to all models]
B --> C[Create Message model]
C --> D[Add utility methods]
D --> E[Update mod.rs and lib.rs]
E --> F[Update README.md]
```
### 3.1 Detailed Changes
#### 3.1.1 Mail Model (`mail.rs`)
- Add `groups: Vec<u32>` field to `Email` struct
- Add utility methods:
- `filter_by_groups(groups: &[u32]) -> bool`
- `search_by_subject(query: &str) -> bool`
- `search_by_content(query: &str) -> bool`
- `to_message(&self) -> Message` (conversion method)
#### 3.1.2 Calendar Model (`calendar.rs`)
- Add `groups: Vec<u32>` field to `Calendar` struct
- Add utility methods:
- `filter_by_groups(groups: &[u32]) -> bool`
- `get_events(&self, db: &SledDB<Event>) -> SledDBResult<Vec<Event>>` (relationship method)
#### 3.1.3 Event Model (`event.rs`)
- Add `groups: Vec<u32>` field to `Event` struct
- Add utility methods:
- `filter_by_groups(groups: &[u32]) -> bool`
- `get_calendar(&self, db: &SledDB<Calendar>) -> SledDBResult<Calendar>` (relationship method)
- `get_attendee_contacts(&self, db: &SledDB<Contact>) -> SledDBResult<Vec<Contact>>` (relationship method)
#### 3.1.4 Contacts Model (`contacts.rs`)
- Add `groups: Vec<u32>` field to `Contact` struct
- Add utility methods:
- `filter_by_groups(groups: &[u32]) -> bool`
- `search_by_name(query: &str) -> bool`
- `search_by_email(query: &str) -> bool`
- `get_events(&self, db: &SledDB<Event>) -> SledDBResult<Vec<Event>>` (relationship method)
#### 3.1.5 New Message Model (`message.rs`)
```rust
use serde::{Deserialize, Serialize};
use crate::core::{SledModel, Storable};
use chrono::{DateTime, Utc};
/// MessageStatus represents the status of a message
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum MessageStatus {
Sent,
Delivered,
Read,
Failed,
}
/// MessageMeta contains metadata for a chat message
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MessageMeta {
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
pub status: MessageStatus,
pub is_edited: bool,
pub reactions: Vec<String>,
}
/// Message represents a chat message
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Message {
pub id: u32, // Unique identifier
pub thread_id: String, // Thread/conversation identifier
pub sender_id: String, // Sender identifier
pub recipients: Vec<String>, // List of recipient identifiers
pub content: String, // Message content
pub attachments: Vec<String>, // References to attachments
pub groups: Vec<u32>, // Groups this message belongs to
pub meta: MessageMeta, // Message metadata
}
impl Message {
/// Create a new message
pub fn new(id: u32, thread_id: String, sender_id: String, content: String) -> Self {
let now = Utc::now();
Self {
id,
thread_id,
sender_id,
recipients: Vec::new(),
content,
attachments: Vec::new(),
groups: Vec::new(),
meta: MessageMeta {
created_at: now,
updated_at: now,
status: MessageStatus::Sent,
is_edited: false,
reactions: Vec::new(),
},
}
}
/// Add a recipient to this message
pub fn add_recipient(&mut self, recipient: String) {
self.recipients.push(recipient);
}
/// Add an attachment to this message
pub fn add_attachment(&mut self, attachment: String) {
self.attachments.push(attachment);
}
/// Add a group to this message
pub fn add_group(&mut self, group_id: u32) {
if !self.groups.contains(&group_id) {
self.groups.push(group_id);
}
}
/// Filter by groups
pub fn filter_by_groups(&self, groups: &[u32]) -> bool {
groups.iter().any(|g| self.groups.contains(g))
}
/// Search by content
pub fn search_by_content(&self, query: &str) -> bool {
self.content.to_lowercase().contains(&query.to_lowercase())
}
/// Update message status
pub fn update_status(&mut self, status: MessageStatus) {
self.meta.status = status;
self.meta.updated_at = Utc::now();
}
/// Edit message content
pub fn edit_content(&mut self, new_content: String) {
self.content = new_content;
self.meta.is_edited = true;
self.meta.updated_at = Utc::now();
}
/// Add a reaction to the message
pub fn add_reaction(&mut self, reaction: String) {
self.meta.reactions.push(reaction);
self.meta.updated_at = Utc::now();
}
}
// Implement Storable trait (provides default dump/load)
impl Storable for Message {}
// Implement SledModel trait
impl SledModel for Message {
fn get_id(&self) -> String {
self.id.to_string()
}
fn db_prefix() -> &'static str {
"message"
}
}
```
#### 3.1.6 Update Module Files
Update `mod.rs` and `lib.rs` to include the new Message model.
#### 3.1.7 Update README.md
Update the README.md to include information about the Message model and the new utility methods.
## 4. Data Model Diagram
```mermaid
classDiagram
class Email {
+u32 id
+u32 uid
+u32 seq_num
+String mailbox
+String message
+Vec~Attachment~ attachments
+Vec~String~ flags
+i64 receivetime
+Option~Envelope~ envelope
+Vec~u32~ groups
+filter_by_groups()
+search_by_subject()
+search_by_content()
+to_message()
}
class Calendar {
+u32 id
+String title
+String description
+Vec~u32~ groups
+filter_by_groups()
+get_events()
}
class Event {
+u32 id
+u32 calendar_id
+String title
+String description
+String location
+DateTime start_time
+DateTime end_time
+bool all_day
+String recurrence
+Vec~String~ attendees
+String organizer
+String status
+EventMeta meta
+Vec~u32~ groups
+filter_by_groups()
+get_calendar()
+get_attendee_contacts()
}
class Contact {
+u32 id
+i64 created_at
+i64 modified_at
+String first_name
+String last_name
+String email
+String group
+Vec~u32~ groups
+filter_by_groups()
+search_by_name()
+search_by_email()
+get_events()
}
class Message {
+u32 id
+String thread_id
+String sender_id
+Vec~String~ recipients
+String content
+Vec~String~ attachments
+Vec~u32~ groups
+MessageMeta meta
+filter_by_groups()
+search_by_content()
+update_status()
+edit_content()
+add_reaction()
}
class Circle {
+u32 id
+String name
+String description
+Vec~Member~ members
}
Calendar "1" -- "many" Event: contains
Contact "many" -- "many" Event: attends
Circle "1" -- "many" Email: groups
Circle "1" -- "many" Calendar: groups
Circle "1" -- "many" Event: groups
Circle "1" -- "many" Contact: groups
Circle "1" -- "many" Message: groups
```
## 5. Testing Strategy
1. Unit tests for each model to verify:
- Group field functionality
- New utility methods
- Serialization/deserialization with the new fields
2. Integration tests to verify:
- Database operations with the updated models
- Relationships between models

33
herodb/src/models/mcc/calendar.rs
View File

@@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize};
use crate::db::{Model, Storable, DB, DbError, DbResult};
use crate::models::mcc::event::Event;
use crate::db::model::impl_get_id;
/// Calendar represents a calendar container for events
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -39,27 +39,18 @@ impl Calendar {
groups.iter().any(|g| self.groups.contains(g))
}
/// Get all events associated with this calendar
pub fn get_events(&self, db: &DB) -> DbResult<Vec<Event>> {
let all_events = db.list::<Event>()?;
let calendar_events = all_events
.into_iter()
/// Filter events by this calendar's ID
pub fn filter_events<'a>(&self, events: &'a [Event]) -> Vec<&'a Event> {
events.iter()
.filter(|event| event.calendar_id == self.id)
.collect();
Ok(calendar_events)
.collect()
}
/// Get the database prefix for this model type
pub fn db_prefix() -> &'static str {
"calendar"
}
}
impl Storable for Calendar{}
// Implement Model trait
impl Model for Calendar {
fn get_id(&self) -> u32 {
self.id
}
fn db_prefix() -> &'static str {
"calendar"
}
}
// Automatically implement GetId trait for Calendar
impl_get_id!(Calendar);

59
herodb/src/models/mcc/contacts.rs
View File

@@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize};
use crate::db::{Model, Storable, DB, DbError, DbResult};
use crate::models::mcc::event::Event;
use crate::db::model::impl_get_id;
use chrono::Utc;
/// Contact represents a contact entry in an address book
@@ -13,14 +13,12 @@ pub struct Contact {
pub modified_at: i64, // Unix epoch timestamp
pub first_name: String,
pub last_name: String,
pub email: String,
pub group: String, // Reference to a dns name, each group has a globally unique dns
pub groups: Vec<u32>, // Groups this contact belongs to (references Circle IDs)
pub emails: Vec<String>, // Changed from []String to Vec<String>
}
impl Contact {
/// Create a new contact
pub fn new(id: u32, first_name: String, last_name: String, email: String, group: String) -> Self {
pub fn new(id: u32, first_name: String, last_name: String, emails: Vec<String>) -> Self {
let now = Utc::now().timestamp();
Self {
id,
@@ -28,29 +26,10 @@ impl Contact {
modified_at: now,
first_name,
last_name,
email,
group,
groups: Vec::new(),
emails : emails,
}
}
/// Add a group to this contact
pub fn add_group(&mut self, group_id: u32) {
if !self.groups.contains(&group_id) {
self.groups.push(group_id);
}
}
/// Remove a group from this contact
pub fn remove_group(&mut self, group_id: u32) {
self.groups.retain(|&id| id != group_id);
}
/// Filter by groups - returns true if this contact belongs to any of the specified groups
pub fn filter_by_groups(&self, groups: &[u32]) -> bool {
groups.iter().any(|g| self.groups.contains(g))
}
/// Search by name - returns true if the name contains the query (case-insensitive)
pub fn search_by_name(&self, query: &str) -> bool {
let full_name = self.full_name().to_lowercase();
@@ -62,15 +41,11 @@ impl Contact {
self.email.to_lowercase().contains(&query.to_lowercase())
}
/// Get events where this contact is an attendee
pub fn get_events(&self, db: &DB) -> DbResult<Vec<Event>> {
let all_events = db.list::<Event>()?;
let contact_events = all_events
.into_iter()
/// Filter events where this contact is an attendee
pub fn filter_events<'a>(&self, events: &'a [Event]) -> Vec<&'a Event> {
events.iter()
.filter(|event| event.attendees.contains(&self.email))
.collect();
Ok(contact_events)
.collect()
}
/// Update the contact's information
@@ -104,18 +79,12 @@ impl Contact {
pub fn full_name(&self) -> String {
format!("{} {}", self.first_name, self.last_name)
}
}
// Implement Storable trait (provides default dump/load)
impl Storable for Contact {}
// Implement Model trait
impl Model for Contact {
fn get_id(&self) -> u32 {
self.id
}
fn db_prefix() -> &'static str {
/// Get the database prefix for this model type
pub fn db_prefix() -> &'static str {
"contact"
}
}
// Automatically implement GetId trait for Contact
impl_get_id!(Contact);

40
herodb/src/models/mcc/event.rs
View File

@@ -1,7 +1,7 @@
use serde::{Deserialize, Serialize};
use crate::db::{Model, Storable, DB, DbError, DbResult};
use crate::models::mcc::calendar::Calendar;
use crate::models::mcc::contacts::Contact;
use crate::db::model::impl_get_id;
use chrono::{DateTime, Utc};
/// EventMeta contains additional metadata for a calendar event
@@ -84,20 +84,16 @@ impl Event {
groups.iter().any(|g| self.groups.contains(g))
}
/// Get the calendar this event belongs to
pub fn get_calendar(&self, db: &DB) -> DbResult<Calendar> {
db.get::<Calendar>(self.calendar_id)
/// Find the calendar this event belongs to
pub fn find_calendar<'a>(&self, calendars: &'a [Calendar]) -> Option<&'a Calendar> {
calendars.iter().find(|cal| cal.id == self.calendar_id)
}
/// Get contacts for all attendees of this event
pub fn get_attendee_contacts(&self, db: &DB) -> DbResult<Vec<Contact>> {
let all_contacts = db.list::<Contact>()?;
let attendee_contacts = all_contacts
.into_iter()
/// Filter contacts that are attendees of this event
pub fn filter_attendee_contacts<'a>(&self, contacts: &'a [Contact]) -> Vec<&'a Contact> {
contacts.iter()
.filter(|contact| self.attendees.contains(&contact.email))
.collect();
Ok(attendee_contacts)
.collect()
}
/// Add an attendee to this event
@@ -124,18 +120,12 @@ impl Event {
pub fn search_by_description(&self, query: &str) -> bool {
self.description.to_lowercase().contains(&query.to_lowercase())
}
}
// Implement Storable trait (provides default dump/load)
impl Storable for Event {}
// Implement Model trait
impl Model for Event {
fn get_id(&self) -> u32 {
self.id
}
fn db_prefix() -> &'static str {
/// Get the database prefix for this model type
pub fn db_prefix() -> &'static str {
"event"
}
}
}
// Automatically implement GetId trait for Event
impl_get_id!(Event);

3
herodb/src/models/mcc/lib.rs
View File

@@ -10,6 +10,3 @@ pub use event::{Event, EventMeta};
pub use mail::{Email, Attachment, Envelope};
pub use contacts::Contact;
pub use message::{Message, MessageMeta, MessageStatus};
// Re-export database components from db module
pub use crate::db::{DB, DBBuilder, Model, Storable, DbError, DbResult};

54
herodb/src/models/mcc/mail.rs
View File

@@ -1,23 +1,17 @@
use serde::{Deserialize, Serialize};
use crate::db::{Model, Storable, DB, DbError, DbResult};
use crate::db::model::impl_get_id;
use chrono::Utc;
/// Email represents an email message with all its metadata and content
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Email {
// Database ID
pub id: u32, // Database ID (assigned by DBHandler)
// Content fields
pub uid: u32, // Unique identifier of the message (in the circle)
pub seq_num: u32, // IMAP sequence number (in the mailbox)
pub mailbox: String, // The mailbox this email belongs to
pub id: u32,
pub message: String, // The email body content
pub attachments: Vec<Attachment>, // Any file attachments
// IMAP specific fields
pub flags: Vec<String>, // IMAP flags like \Seen, \Deleted, etc.
pub receivetime: i64, // Unix timestamp when the email was received
pub envelope: Option<Envelope>, // IMAP envelope information (contains From, To, Subject, etc.)
pub groups: Vec<u32>, // Groups this email belongs to (references Circle IDs)
pub envelope: Option<Envelope>, // IMAP envelope structure
}
/// Attachment represents an email attachment
@@ -41,7 +35,6 @@ pub struct Envelope {
pub cc: Vec<String>,
pub bcc: Vec<String>,
pub in_reply_to: String,
pub message_id: String,
}
impl Email {
@@ -49,15 +42,11 @@ impl Email {
pub fn new(id: u32, uid: u32, seq_num: u32, mailbox: String, message: String) -> Self {
Self {
id,
uid,
seq_num,
mailbox,
message,
attachments: Vec::new(),
flags: Vec::new(),
receivetime: chrono::Utc::now().timestamp(),
envelope: None,
groups: Vec::new(),
}
}
@@ -66,23 +55,6 @@ impl Email {
self.attachments.push(attachment);
}
/// Add a group to this email
pub fn add_group(&mut self, group_id: u32) {
if !self.groups.contains(&group_id) {
self.groups.push(group_id);
}
}
/// Remove a group from this email
pub fn remove_group(&mut self, group_id: u32) {
self.groups.retain(|&id| id != group_id);
}
/// Filter by groups - returns true if this email belongs to any of the specified groups
pub fn filter_by_groups(&self, groups: &[u32]) -> bool {
groups.iter().any(|g| self.groups.contains(g))
}
/// Search by subject - returns true if the subject contains the query (case-insensitive)
pub fn search_by_subject(&self, query: &str) -> bool {
if let Some(env) = &self.envelope {
@@ -146,18 +118,12 @@ impl Email {
message
}
}
// Implement Storable trait (provides default dump/load)
impl Storable for Email {}
// Implement Model trait
impl Model for Email {
fn get_id(&self) -> u32 {
self.id
}
fn db_prefix() -> &'static str {
/// Get the database prefix for this model type
pub fn db_prefix() -> &'static str {
"email"
}
}
}
// Automatically implement GetId trait for Email
impl_get_id!(Email);

36
herodb/src/models/mcc/message.rs
View File

@@ -1,5 +1,5 @@
use serde::{Deserialize, Serialize};
use crate::db::{Model, Storable, DB, DbError, DbResult};
use crate::impl_get_id;
use chrono::{DateTime, Utc};
/// MessageStatus represents the status of a message
@@ -30,7 +30,6 @@ pub struct Message {
pub recipients: Vec<String>, // List of recipient identifiers
pub content: String, // Message content
pub attachments: Vec<String>, // References to attachments
pub groups: Vec<u32>, // Groups this message belongs to (references Circle IDs)
pub meta: MessageMeta, // Message metadata
}
@@ -45,7 +44,6 @@ impl Message {
recipients: Vec::new(),
content,
attachments: Vec::new(),
groups: Vec::new(),
meta: MessageMeta {
created_at: now,
updated_at: now,
@@ -107,28 +105,18 @@ impl Message {
self.meta.updated_at = Utc::now();
}
/// Get all messages in the same thread
pub fn get_thread_messages(&self, db: &DB) -> DbResult<Vec<Message>> {
let all_messages = db.list::<Message>()?;
let thread_messages = all_messages
.into_iter()
/// Filter messages that are in the same thread as this message
pub fn filter_thread_messages<'a>(&self, messages: &'a [Message]) -> Vec<&'a Message> {
messages.iter()
.filter(|msg| msg.thread_id == self.thread_id)
.collect();
Ok(thread_messages)
.collect()
}
/// Get the database prefix for this model type
pub fn db_prefix() -> &'static str {
"message"
}
}
// Implement Storable trait (provides default dump/load)
impl Storable for Message {}
// Implement Model trait
impl Model for Message {
fn get_id(&self) -> u32 {
self.id
}
fn db_prefix() -> &'static str {
"message"
}
}
// Automatically implement GetId trait for Message
impl_get_id!(Message);

5
herodb/src/models/mcc/mod.rs
View File

@@ -9,7 +9,4 @@ pub use calendar::Calendar;
pub use event::{Event, EventMeta};
pub use mail::{Email, Attachment, Envelope};
pub use contacts::Contact;
pub use message::{Message, MessageMeta, MessageStatus};
// Re-export database components from db module
pub use crate::db::{DB, DBBuilder, Model, Storable, DbError, DbResult};
pub use message::{Message, MessageMeta, MessageStatus};