herocode/db
13
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Use incremental ID

Browse Source
This commit is contained in:
Mahmoud Emad
2025-05-17 13:46:16 +03:00
parent a676854251
commit 57f59da43e
10 changed files with 519 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
heromodels/src/db.rs
View File

@@ -32,9 +32,16 @@ where
/// Get an object from its ID. This does not use an index lookup
fn get_by_id(&self, id: u32) -> Result<Option<V>, Error<Self::Error>>;
/// Store an item in the DB and return the assigned ID.
/// This method does not modify the original model.
fn set(&self, value: &V) -> Result<u32, Error<Self::Error>>;
/// Store an item in the DB and return the assigned ID and the updated model.
///
/// # Important Notes
/// - This method does not modify the original model passed as an argument.
/// - For new models (with ID 0), an ID will be auto-generated by OurDB.
/// - The returned model will have the correct ID and should be used instead of the original model.
/// - The original model should not be used after calling this method, as it may have
/// an inconsistent state compared to what's in the database.
/// - ID 0 is reserved for new models and should not be used for existing models.
fn set(&self, value: &V) -> Result<(u32, V), Error<Self::Error>>;
/// Delete all items from the db with a given index.
fn delete<I, Q>(&self, key: &Q) -> Result<(), Error<Self::Error>>
@@ -46,8 +53,11 @@ where
/// Delete an object with a given ID
fn delete_by_id(&self, id: u32) -> Result<(), Error<Self::Error>>;
/// Get all objects from the colelction
/// Get all objects from the collection
fn get_all(&self) -> Result<Vec<V>, Error<Self::Error>>;
/// Begin a transaction for this collection
fn begin_transaction(&self) -> Result<Box<dyn Transaction<Error = Self::Error>>, Error<Self::Error>>;
}
/// Errors returned by the DB implementation
@@ -59,6 +69,14 @@ pub enum Error<E> {
Decode(bincode::error::DecodeError),
/// Error encoding a model for storage
Encode(bincode::error::EncodeError),
/// Invalid ID used (e.g., using ID 0 for an existing model)
InvalidId(String),
/// ID mismatch (e.g., expected ID 5 but got ID 6)
IdMismatch(String),
/// ID collision (e.g., trying to create a model with an ID that already exists)
IdCollision(String),
/// Type error (e.g., trying to get a model of the wrong type)
TypeError,
}
impl<E> From<bincode::error::DecodeError> for Error<E> {
@@ -72,3 +90,21 @@ impl<E> From<bincode::error::EncodeError> for Error<E> {
Error::Encode(value)
}
}
/// A transaction that can be committed or rolled back
pub trait Transaction {
/// Error type for transaction operations
type Error: std::fmt::Debug;
/// Begin a transaction
fn begin(&self) -> Result<(), Error<Self::Error>>;
/// Commit a transaction
fn commit(&self) -> Result<(), Error<Self::Error>>;
/// Roll back a transaction
fn rollback(&self) -> Result<(), Error<Self::Error>>;
/// Check if a transaction is active
fn is_active(&self) -> bool;
}

427
heromodels/src/db/hero.rs
View File

@@ -1,31 +1,97 @@
use heromodels_core::{Index, Model};
use ourdb::OurDBSetArgs;
use serde::Deserialize;
use crate::db::Transaction;
use std::{
borrow::Borrow,
collections::HashSet,
path::PathBuf,
sync::{Arc, Mutex},
sync::{Arc, Mutex, atomic::{AtomicU32, Ordering}},
};
/// Configuration for custom ID sequences
pub struct IdSequence {
/// The starting ID for the sequence
start: u32,
/// The increment for the sequence
increment: u32,
/// The current ID in the sequence
current: AtomicU32,
}
// Implement Clone manually since AtomicU32 doesn't implement Clone
impl Clone for IdSequence {
fn clone(&self) -> Self {
Self {
start: self.start,
increment: self.increment,
current: AtomicU32::new(self.current.load(Ordering::SeqCst)),
}
}
}
impl IdSequence {
/// Create a new ID sequence with default values (start=1, increment=1)
pub fn new() -> Self {
Self {
start: 1,
increment: 1,
current: AtomicU32::new(1),
}
}
/// Create a new ID sequence with custom values
pub fn with_config(start: u32, increment: u32) -> Self {
Self {
start,
increment,
current: AtomicU32::new(start),
}
}
/// Get the next ID in the sequence
pub fn next_id(&self) -> u32 {
self.current.fetch_add(self.increment, Ordering::SeqCst)
}
/// Reset the sequence to its starting value
pub fn reset(&self) {
self.current.store(self.start, Ordering::SeqCst);
}
/// Set the current ID in the sequence
pub fn set_current(&self, id: u32) {
self.current.store(id, Ordering::SeqCst);
}
}
const BINCODE_CONFIG: bincode::config::Configuration = bincode::config::standard();
#[derive(Clone)]
pub struct OurDB {
index: Arc<Mutex<tst::TST>>,
data: Arc<Mutex<ourdb::OurDB>>,
// Mutex for ID generation to prevent race conditions
id_lock: Arc<Mutex<()>>,
// Custom ID sequence configuration
id_sequence: Arc<IdSequence>,
}
impl OurDB {
/// Create a new instance of ourdb
/// Create a new instance of ourdb with default ID sequence (start=1, increment=1)
pub fn new(path: impl Into<PathBuf>, reset: bool) -> Result<Self, tst::Error> {
Self::with_id_sequence(path, reset, IdSequence::new())
}
/// Create a new instance of ourdb with a custom ID sequence
pub fn with_id_sequence(path: impl Into<PathBuf>, reset: bool, id_sequence: IdSequence) -> Result<Self, tst::Error> {
let mut base_path = path.into();
let mut data_path = base_path.clone();
base_path.push("index");
data_path.push("data");
let data_db = ourdb::OurDB::new(ourdb::OurDBConfig {
let mut data_db = ourdb::OurDB::new(ourdb::OurDBConfig {
incremental_mode: true,
path: data_path,
file_size: None,
@@ -33,9 +99,24 @@ impl OurDB {
reset: Some(reset),
})?;
let index_db = tst::TST::new(base_path.to_str().expect("Path is valid UTF-8"), reset)?;
// If we're resetting the database, also reset the ID sequence
if reset {
id_sequence.reset();
} else {
// Otherwise, try to find the highest ID in the database and update the sequence
// Since OurDB doesn't have a get_highest_id method, we'll use get_next_id instead
// This is not ideal, but it's the best we can do with the current API
let highest_id = data_db.get_next_id().unwrap_or(id_sequence.start);
if highest_id >= id_sequence.start {
id_sequence.set_current(highest_id + id_sequence.increment);
}
}
Ok(OurDB {
index: Arc::new(Mutex::new(index_db)),
data: Arc::new(Mutex::new(data_db)),
id_lock: Arc::new(Mutex::new(())),
id_sequence: Arc::new(id_sequence),
})
}
}
@@ -56,6 +137,18 @@ where
{
type Error = tst::Error;
/// Begin a transaction for this collection
fn begin_transaction(&self) -> Result<Box<dyn super::Transaction<Error = Self::Error>>, super::Error<Self::Error>> {
// Create a new transaction
let transaction = OurDBTransaction::new();
// Begin the transaction
transaction.begin()?;
// Return the transaction
Ok(Box::new(transaction))
}
fn get<I, Q>(&self, key: &Q) -> Result<Vec<M>, super::Error<Self::Error>>
where
I: Index<Model = M>,
@@ -87,114 +180,165 @@ where
Self::get_ourdb_value(&mut data_db, id)
}
fn set(&self, value: &M) -> Result<u32, super::Error<Self::Error>> {
// Before inserting the new object, check if an object with this ID already exists. If it does, we potentially need to update indices.
let mut data_db = self.data.lock().expect("can lock data DB");
let old_obj: Option<M> = Self::get_ourdb_value(&mut data_db, value.get_id())?;
let (indices_to_delete, indices_to_add) = if let Some(old_obj) = old_obj {
let mut indices_to_delete = vec![];
let mut indices_to_add = vec![];
let old_indices = old_obj.db_keys();
let new_indices = value.db_keys();
for old_index in old_indices {
for new_index in &new_indices {
if old_index.name == new_index.name {
if old_index.value != new_index.value {
// different value now, remove index
indices_to_delete.push(old_index);
// and later add the new one
indices_to_add.push(new_index.clone());
break;
fn set(&self, value: &M) -> Result<(u32, M), super::Error<Self::Error>> {
// For now, we'll skip using transactions to avoid type inference issues
// In a real implementation, you would use a proper transaction mechanism
// Use a result variable to track success/failure
let result = (|| {
// Before inserting the new object, check if an object with this ID already exists. If it does, we potentially need to update indices.
let mut data_db = self.data.lock().expect("can lock data DB");
let old_obj: Option<M> = Self::get_ourdb_value(&mut data_db, value.get_id())?;
let (indices_to_delete, indices_to_add) = if let Some(ref old_obj) = old_obj {
let mut indices_to_delete = vec![];
let mut indices_to_add = vec![];
let old_indices = old_obj.db_keys();
let new_indices = value.db_keys();
for old_index in old_indices {
for new_index in &new_indices {
if old_index.name == new_index.name {
if old_index.value != new_index.value {
// different value now, remove index
indices_to_delete.push(old_index);
// and later add the new one
indices_to_add.push(new_index.clone());
break;
}
}
}
}
}
// NOTE: we assume here that the index keys are stable, i.e. new index fields don't appear
// and existing ones don't dissapear
(indices_to_delete, indices_to_add)
} else {
(vec![], value.db_keys())
};
let mut index_db = self.index.lock().expect("can lock index db");
// First delete old indices which need to change
for old_index in indices_to_delete {
let key = Self::index_key(M::db_prefix(), old_index.name, &old_index.value);
let raw_ids = index_db.get(&key)?;
let (mut ids, _): (HashSet<u32>, _) =
bincode::serde::decode_from_slice(&raw_ids, BINCODE_CONFIG)?;
ids.remove(&value.get_id());
if ids.is_empty() {
// This was the last ID with this index value, remove index entirely
index_db.delete(&key)?;
// NOTE: we assume here that the index keys are stable, i.e. new index fields don't appear
// and existing ones don't dissapear
(indices_to_delete, indices_to_add)
} else {
// There are still objects left with this index value, write back updated set
let raw_ids = bincode::serde::encode_to_vec(ids, BINCODE_CONFIG)?;
index_db.set(&key, raw_ids)?;
(vec![], value.db_keys())
};
let mut index_db = self.index.lock().expect("can lock index db");
// First delete old indices which need to change
for old_index in indices_to_delete {
let key = Self::index_key(M::db_prefix(), old_index.name, &old_index.value);
let raw_ids = index_db.get(&key)?;
let (mut ids, _): (HashSet<u32>, _) =
bincode::serde::decode_from_slice(&raw_ids, BINCODE_CONFIG)?;
ids.remove(&value.get_id());
if ids.is_empty() {
// This was the last ID with this index value, remove index entirely
index_db.delete(&key)?;
} else {
// There are still objects left with this index value, write back updated set
let raw_ids = bincode::serde::encode_to_vec(ids, BINCODE_CONFIG)?;
index_db.set(&key, raw_ids)?;
}
}
}
// Get the current ID
let id = value.get_id();
// Get the current ID
let id = value.get_id();
// If id is 0, it's a new object, so let OurDB auto-generate an ID
// Otherwise, it's an update to an existing object
let id_param = if id == 0 { None } else { Some(id) };
// Validate that ID 0 is only used for new models
if id == 0 {
// Check if this model already exists in the database
// If it does, it's an error to use ID 0 for an existing model
if let Some(existing) = Self::get_ourdb_value::<M>(&mut data_db, id)? {
return Err(super::Error::InvalidId(format!(
"ID 0 is reserved for new models. Found existing model with ID 0: {:?}",
existing
)));
}
} else {
// Validate that IDs > 0 are only used for existing models
// If the model doesn't exist, it's an error to use a specific ID
if id > 0 && Self::get_ourdb_value::<M>(&mut data_db, id)?.is_none() {
return Err(super::Error::InvalidId(format!(
"ID {} does not exist in the database. Use ID 0 for new models.",
id
)));
}
// For new objects (id == 0), we need to get the assigned ID from OurDB
// and update the model before serializing it
let assigned_id = if id == 0 {
// First, get the next ID that OurDB will assign
let next_id = data_db.get_next_id()?;
// Check for ID collisions when manually setting an ID
if id > 0 && Self::get_ourdb_value::<M>(&mut data_db, id)?.is_some() {
// This is only an error if we're trying to create a new model with this ID
// If we're updating an existing model, this is fine
if old_obj.is_none() {
return Err(super::Error::IdCollision(format!(
"ID {} already exists in the database",
id
)));
}
}
}
// Create a mutable clone of the value and update its ID
// This is a bit of a hack, but we need to update the ID before serializing
let mut value_clone = value.clone();
let base_data = value_clone.base_data_mut();
base_data.update_id(next_id);
// If id is 0, it's a new object, so let OurDB auto-generate an ID
// Otherwise, it's an update to an existing object
let id_param = if id == 0 { None } else { Some(id) };
// Now serialize the updated model
let v = bincode::serde::encode_to_vec(&value_clone, BINCODE_CONFIG)?;
// Thread-safe approach for handling ID assignment
let assigned_id = if id == 0 {
// For new objects, serialize with ID 0
let v = bincode::serde::encode_to_vec(value, BINCODE_CONFIG)?;
// Save to OurDB with the ID parameter set to None to let OurDB auto-generate the ID
let assigned_id = data_db.set(OurDBSetArgs {
id: id_param,
data: &v,
})?;
// Save to OurDB with id_param = None to let OurDB auto-generate the ID
let assigned_id = data_db.set(OurDBSetArgs {
id: id_param,
data: &v,
})?;
// The assigned ID should match the next_id we got earlier
assert_eq!(assigned_id, next_id, "OurDB assigned a different ID than expected");
// Now that we have the actual assigned ID, create a new model with the correct ID
// and save it again to ensure the serialized data contains the correct ID
let mut value_clone = value.clone();
let base_data = value_clone.base_data_mut();
base_data.update_id(assigned_id);
// Return the assigned ID
assigned_id
} else {
// For existing objects, just serialize and save
let v = bincode::serde::encode_to_vec(value, BINCODE_CONFIG)?;
// Serialize the updated model
let v = bincode::serde::encode_to_vec(&value_clone, BINCODE_CONFIG)?;
// Save to OurDB with the existing ID
let assigned_id = data_db.set(OurDBSetArgs {
id: id_param,
data: &v,
})?;
// Save again with the explicit ID
data_db.set(OurDBSetArgs {
id: Some(assigned_id),
data: &v,
})?;
// Return the existing ID
assigned_id
};
// Return the assigned ID
assigned_id
} else {
// For existing objects, just serialize and save
let v = bincode::serde::encode_to_vec(value, BINCODE_CONFIG)?;
// Now add the new indices
for index_key in indices_to_add {
let key = Self::index_key(M::db_prefix(), index_key.name, &index_key.value);
// Load the existing id set for the index or create a new set
let mut existing_ids =
Self::get_tst_value::<HashSet<u32>>(&mut index_db, &key)?.unwrap_or_default();
// Use the assigned ID for new objects
existing_ids.insert(assigned_id);
let encoded_ids = bincode::serde::encode_to_vec(existing_ids, BINCODE_CONFIG)?;
index_db.set(&key, encoded_ids)?;
}
// Save to OurDB with the existing ID
let assigned_id = data_db.set(OurDBSetArgs {
id: id_param,
data: &v,
})?;
// Return the assigned ID
Ok(assigned_id)
// Return the existing ID
assigned_id
};
// Now add the new indices
for index_key in indices_to_add {
let key = Self::index_key(M::db_prefix(), index_key.name, &index_key.value);
// Load the existing id set for the index or create a new set
let mut existing_ids =
Self::get_tst_value::<HashSet<u32>>(&mut index_db, &key)?.unwrap_or_default();
// Use the assigned ID for new objects
existing_ids.insert(assigned_id);
let encoded_ids = bincode::serde::encode_to_vec(existing_ids, BINCODE_CONFIG)?;
index_db.set(&key, encoded_ids)?;
}
// Get the updated model from the database
let updated_model = Self::get_ourdb_value::<M>(&mut data_db, assigned_id)?
.ok_or_else(|| super::Error::InvalidId(format!(
"Failed to retrieve model with ID {} after saving", assigned_id
)))?;
// Return the assigned ID and the updated model
Ok((assigned_id, updated_model))
})();
// Return the result directly
// In a real implementation, you would commit or rollback the transaction here
result
}
fn delete<I, Q>(&self, key: &Q) -> Result<(), super::Error<Self::Error>>
@@ -279,6 +423,29 @@ impl OurDB {
format!("{collection}::{index}::{value}")
}
/// Reserve an ID for future use
pub fn reserve_id(&self) -> u32 {
// Acquire the ID lock to prevent race conditions
let _id_lock = self.id_lock.lock().expect("Failed to acquire ID lock");
// Get the next ID from our custom sequence
self.id_sequence.next_id()
}
/// Reserve multiple IDs for future use
pub fn reserve_ids(&self, count: u32) -> Vec<u32> {
// Acquire the ID lock to prevent race conditions
let _id_lock = self.id_lock.lock().expect("Failed to acquire ID lock");
// Get the next IDs from our custom sequence
let mut ids = Vec::with_capacity(count as usize);
for _ in 0..count {
ids.push(self.id_sequence.next_id());
}
ids
}
/// Wrapper to load values from ourdb and transform a not found error in to Ok(None)
fn get_ourdb_value<V>(
data: &mut ourdb::OurDB,
@@ -326,3 +493,75 @@ impl From<ourdb::Error> for super::Error<tst::Error> {
super::Error::DB(tst::Error::OurDB(value))
}
}
/// A transaction for OurDB
///
/// Note: This is a simplified implementation that doesn't actually provide
/// ACID guarantees. In a real implementation, you would need to use a proper
/// transaction mechanism provided by the underlying database.
///
/// This struct implements Drop to ensure that transactions are properly closed.
/// If a transaction is not explicitly committed or rolled back, it will be
/// rolled back when the transaction is dropped.
struct OurDBTransaction {
active: std::sync::atomic::AtomicBool,
}
impl OurDBTransaction {
/// Create a new transaction
fn new() -> Self {
Self { active: std::sync::atomic::AtomicBool::new(false) }
}
}
impl Drop for OurDBTransaction {
fn drop(&mut self) {
// If the transaction is still active when dropped, roll it back
if self.active.load(std::sync::atomic::Ordering::SeqCst) {
// We can't return an error from drop, so we just log it
eprintln!("Warning: Transaction was dropped without being committed or rolled back. Rolling back automatically.");
self.active.store(false, std::sync::atomic::Ordering::SeqCst);
}
}
}
impl super::Transaction for OurDBTransaction {
type Error = tst::Error;
/// Begin the transaction
fn begin(&self) -> Result<(), super::Error<Self::Error>> {
// In a real implementation, you would start a transaction in the underlying database
// For now, we just set the active flag
self.active.store(true, std::sync::atomic::Ordering::SeqCst);
Ok(())
}
/// Commit the transaction
fn commit(&self) -> Result<(), super::Error<Self::Error>> {
// In a real implementation, you would commit the transaction in the underlying database
// For now, we just check if the transaction is active
if !self.active.load(std::sync::atomic::Ordering::SeqCst) {
return Err(super::Error::InvalidId("Cannot commit an inactive transaction".to_string()));
}
self.active.store(false, std::sync::atomic::Ordering::SeqCst);
Ok(())
}
/// Roll back the transaction
fn rollback(&self) -> Result<(), super::Error<Self::Error>> {
// In a real implementation, you would roll back the transaction in the underlying database
// For now, we just check if the transaction is active
if !self.active.load(std::sync::atomic::Ordering::SeqCst) {
return Err(super::Error::InvalidId("Cannot roll back an inactive transaction".to_string()));
}
self.active.store(false, std::sync::atomic::Ordering::SeqCst);
Ok(())
}
/// Check if the transaction is active
fn is_active(&self) -> bool {
self.active.load(std::sync::atomic::Ordering::SeqCst)
}
}

10
heromodels/src/models/calendar/calendar.rs
View File

@@ -142,10 +142,16 @@ impl Calendar {
/// Creates a new calendar with auto-generated ID
///
/// # Arguments
/// * `id` - Optional ID for the calendar (use None for auto-generated ID)
/// * `name` - Name of the calendar
pub fn new(name: impl ToString) -> Self {
pub fn new(id: Option<u32>, name: impl ToString) -> Self {
let mut base_data = BaseModelData::new();
if let Some(id) = id {
base_data.update_id(id);
}
Self {
base_data: BaseModelData::new(),
base_data,
name: name.to_string(),
description: None,
events: Vec::new(),