Add alternative DB interface and implementation

Signed-off-by: Lee Smet <lee.smet@hotmail.com>

heromodels/src/db.rs

@@ -0,0 +1,69 @@
+use crate::models::{Index, Model};
+use serde::{Deserialize, Serialize};
+
+pub mod fjall;
+pub mod hero;
+
+pub trait Db {
+    /// Error type returned by database operations.
+    type Error: std::fmt::Debug;
+
+    /// Open the collection for a specific model. This method must create the collection if it does not exist.
+    fn collection<M: Model>(&self) -> Result<impl Collection<&str, M>, Error<Self::Error>>;
+}
+
+/// A collection stores a specific model under a specific key
+pub trait Collection<K, V>
+where
+    K: Serialize,
+    V: Serialize + for<'a> Deserialize<'a>,
+{
+    /// Error type for database operations
+    type Error: std::fmt::Debug;
+
+    /// Get all items where the given index field is equal to key.
+    fn get<I>(&self, key: K) -> Result<Vec<V>, Error<Self::Error>>
+    where
+        I: Index<Model = V>;
+
+    /// Get an object from its ID. This does not use an index lookup
+    fn get_by_id(&self, id: u32) -> Result<Option<V>, Error<Self::Error>>;
+
+    /// Store an item in the DB.
+    fn set(&self, value: &V) -> Result<(), Error<Self::Error>>;
+
+    /// Delete all items from the db with a given index.
+    fn delete<I>(&self, key: K) -> Result<(), Error<Self::Error>>
+    where
+        I: Index<Model = V>;
+
+    /// Delete an object with a given ID
+    fn delete_by_id(&self, id: u32) -> Result<(), Error<Self::Error>>;
+
+    /// Get all objects from the colelction
+    fn get_all(&self) -> Result<Vec<V>, Error<Self::Error>>;
+}
+
+/// Errors returned by the DB implementation
+#[derive(Debug)]
+pub enum Error<E> {
+    /// Error in the underlying database
+    DB(E),
+    /// Error decoding a stored model
+    Decode(bincode::error::DecodeError),
+    /// Error encoding a model for storage
+    Encode(bincode::error::EncodeError),
+}
+
+impl<E> From<bincode::error::DecodeError> for Error<E> {
+    fn from(value: bincode::error::DecodeError) -> Self {
+        Error::Decode(value)
+    }
+}
+
+impl<E> From<bincode::error::EncodeError> for Error<E> {
+    fn from(value: bincode::error::EncodeError) -> Self {
+        Error::Encode(value)
+    }
+}
+

heromodels/src/db/hero.rs

@@ -0,0 +1,269 @@
+use ourdb::OurDBSetArgs;
+use serde::Deserialize;
+
+use crate::models::{Index, Model};
+
+use std::{
+    collections::HashSet,
+    sync::{Arc, Mutex},
+};
+
+const BINCODE_CONFIG: bincode::config::Configuration = bincode::config::standard();
+
+#[derive(Clone)]
+pub struct OurDB {
+    index: Arc<Mutex<tst::TST>>,
+    data: Arc<Mutex<ourdb::OurDB>>,
+}
+
+impl OurDB {
+    /// Create a new instance of ourdb
+    pub fn new(index_db: tst::TST, data_db: ourdb::OurDB) -> Self {
+        Self {
+            index: Arc::new(Mutex::new(index_db)),
+            data: Arc::new(Mutex::new(data_db)),
+        }
+    }
+}
+
+impl super::Db for OurDB {
+    type Error = tst::Error;
+
+    fn collection<M: crate::models::Model>(
+        &self,
+    ) -> Result<impl super::Collection<&str, M>, super::Error<Self::Error>> {
+        Ok(self.clone())
+    }
+}
+
+impl<M> super::Collection<&str, M> for OurDB
+where
+    M: Model,
+{
+    type Error = tst::Error;
+
+    fn get<I>(&self, key: &str) -> Result<Vec<M>, super::Error<Self::Error>>
+    where
+        I: Index<Model = M>,
+    {
+        let mut index_db = self.index.lock().expect("can lock index DB");
+        let index_key = Self::index_key(M::db_prefix(), I::key(), key);
+
+        let Some(object_ids) = Self::get_tst_value::<HashSet<u32>>(&mut index_db, &index_key)?
+        else {
+            // If the index is not found just return an empty vector, no items present
+            return Ok(vec![]);
+        };
+
+        let mut data_db = self.data.lock().expect("can lock data DB");
+        object_ids
+            .into_iter()
+            .map(|obj_id| -> Result<M, _> {
+                let raw_obj = data_db.get(obj_id)?;
+                let (obj, _): (M, _) = bincode::serde::decode_from_slice(&raw_obj, BINCODE_CONFIG)?;
+                Ok(obj)
+            })
+            .collect()
+    }
+
+    fn get_by_id(&self, id: u32) -> Result<Option<M>, super::Error<Self::Error>> {
+        let mut data_db = self.data.lock().expect("can lock data db");
+        Self::get_ourdb_value(&mut data_db, id)
+    }
+
+    fn set(&self, value: &M) -> Result<(), super::Error<Self::Error>> {
+        // Before inserting the new object, check if an object with this ID already exists. If it does, we potentially need to update indices.
+        let mut data_db = self.data.lock().expect("can lock data DB");
+        let old_obj: Option<M> = Self::get_ourdb_value(&mut data_db, value.get_id())?;
+        let (indices_to_delete, indices_to_add) = if let Some(old_obj) = old_obj {
+            let mut indices_to_delete = vec![];
+            let mut indices_to_add = vec![];
+            let old_indices = old_obj.db_keys();
+            let new_indices = value.db_keys();
+            for old_index in old_indices {
+                for new_index in &new_indices {
+                    if old_index.name == new_index.name {
+                        if old_index.value != new_index.value {
+                            // different value now, remove index
+                            indices_to_delete.push(old_index);
+                            // and later add the new one
+                            indices_to_add.push(new_index.clone());
+                            break;
+                        }
+                    }
+                }
+            }
+            // NOTE: we assume here that the index keys are stable, i.e. new index fields don't appear
+            // and existing ones don't dissapear
+            (indices_to_delete, indices_to_add)
+        } else {
+            (vec![], value.db_keys())
+        };
+
+        let mut index_db = self.index.lock().expect("can lock index db");
+        // First delete old indices which need to change
+        for old_index in indices_to_delete {
+            let key = Self::index_key(M::db_prefix(), old_index.name, &old_index.value);
+            let raw_ids = index_db.get(&key)?;
+            let (mut ids, _): (HashSet<u32>, _) =
+                bincode::serde::decode_from_slice(&raw_ids, BINCODE_CONFIG)?;
+            ids.remove(&value.get_id());
+            if ids.is_empty() {
+                // This was the last ID with this index value, remove index entirely
+                index_db.delete(&key)?;
+            } else {
+                // There are still objects left with this index value, write back updated set
+                let raw_ids = bincode::serde::encode_to_vec(ids, BINCODE_CONFIG)?;
+                index_db.set(&key, raw_ids)?;
+            }
+        }
+
+        // set or update the object
+        let v = bincode::serde::encode_to_vec(value, BINCODE_CONFIG)?;
+        let id = value.get_id();
+        data_db.set(OurDBSetArgs {
+            id: Some(id),
+            data: &v,
+        })?;
+
+        // Now add the new indices
+        for index_key in indices_to_add {
+            let key = Self::index_key(M::db_prefix(), index_key.name, &index_key.value);
+            // Load the existing id set for the index or create a new set
+            let mut existing_ids =
+                Self::get_tst_value::<HashSet<u32>>(&mut index_db, &key)?.unwrap_or_default();
+            existing_ids.insert(id);
+            let encoded_ids = bincode::serde::encode_to_vec(existing_ids, BINCODE_CONFIG)?;
+            index_db.set(&key, encoded_ids)?;
+        }
+
+        Ok(())
+    }
+
+    fn delete<I>(&self, key: &str) -> Result<(), super::Error<Self::Error>>
+    where
+        I: Index<Model = M>,
+    {
+        let mut index_db = self.index.lock().expect("can lock index db");
+        let key = Self::index_key(M::db_prefix(), I::key(), key);
+        let raw_obj_ids = index_db.get(&key)?;
+        let (obj_ids, _): (HashSet<u32>, _) =
+            bincode::serde::decode_from_slice(&raw_obj_ids, BINCODE_CONFIG)?;
+
+        let mut data_db = self.data.lock().expect("can lock data DB");
+        for obj_id in obj_ids {
+            // Load object
+            let raw = data_db.get(obj_id)?;
+            let (obj, _): (M, _) = bincode::serde::decode_from_slice(&raw, BINCODE_CONFIG)?;
+
+            // Delete indices
+            for index in obj.db_keys() {
+                let key = Self::index_key(M::db_prefix(), index.name, &index.value);
+                let raw_ids = index_db.get(&key)?;
+                let (mut ids, _): (HashSet<u32>, _) =
+                    bincode::serde::decode_from_slice(&raw_ids, BINCODE_CONFIG)?;
+                ids.remove(&obj_id);
+                if ids.is_empty() {
+                    // This was the last ID with this index value, remove index entirely
+                    index_db.delete(&key)?;
+                } else {
+                    // There are still objects left with this index value, write back updated set
+                    let raw_ids = bincode::serde::encode_to_vec(ids, BINCODE_CONFIG)?;
+                    index_db.set(&key, raw_ids)?;
+                }
+            }
+
+            // Delete object
+            data_db.delete(obj_id)?;
+        }
+
+        Ok(())
+    }
+
+    fn delete_by_id(&self, id: u32) -> Result<(), super::Error<Self::Error>> {
+        let mut data_db = self.data.lock().expect("can lock data DB");
+        let raw = data_db.get(id)?;
+
+        // First load the object so we can delete the indices
+        // Delete indices before the actual object, so we don't leave dangling references to a deleted
+        // object ID in case something goes wrong.
+        let (obj, _): (M, _) = bincode::serde::decode_from_slice(&raw, BINCODE_CONFIG)?;
+        let mut index_db = self.index.lock().expect("can lock index DB");
+        for index_key in obj.db_keys() {
+            let key = Self::index_key(M::db_prefix(), index_key.name, &index_key.value);
+            let raw_ids = index_db.get(&key)?;
+            let (mut ids, _): (HashSet<u32>, _) =
+                bincode::serde::decode_from_slice(&raw_ids, BINCODE_CONFIG)?;
+            ids.remove(&obj.get_id());
+            if ids.is_empty() {
+                // This was the last ID with this index value, remove index entirely
+                index_db.delete(&key)?;
+            } else {
+                // There are still objects left with this index value, write back updated set
+                let raw_ids = bincode::serde::encode_to_vec(ids, BINCODE_CONFIG)?;
+                index_db.set(&key, raw_ids)?;
+            }
+        }
+
+        // Finally delete the object itself
+        Ok(data_db.delete(id)?)
+    }
+
+    fn get_all(&self) -> Result<Vec<M>, super::Error<Self::Error>> {
+        todo!("OurDB doesn't have a list all method yet")
+    }
+}
+
+impl OurDB {
+    /// Build the key used for an indexed field of a collection.
+    fn index_key(collection: &str, index: &str, value: &str) -> String {
+        format!("{collection}::{index}::{value}")
+    }
+
+    /// Wrapper to load values from ourdb and transform a not found error in to Ok(None)
+    fn get_ourdb_value<V>(
+        data: &mut ourdb::OurDB,
+        id: u32,
+    ) -> Result<Option<V>, super::Error<tst::Error>>
+    where
+        V: for<'de> Deserialize<'de>,
+    {
+        match data.get(id) {
+            Ok(raw) => {
+                let (obj, _): (V, _) = bincode::serde::decode_from_slice(&raw, BINCODE_CONFIG)?;
+                Ok(Some(obj))
+            }
+            Err(ourdb::Error::NotFound(_)) => Ok(None),
+            Err(e) => Err(e.into()),
+        }
+    }
+
+    fn get_tst_value<V>(
+        index: &mut tst::TST,
+        key: &str,
+    ) -> Result<Option<V>, super::Error<tst::Error>>
+    where
+        V: for<'de> Deserialize<'de>,
+    {
+        match index.get(key) {
+            Ok(raw) => {
+                let (obj, _): (V, _) = bincode::serde::decode_from_slice(&raw, BINCODE_CONFIG)?;
+                Ok(Some(obj))
+            }
+            Err(tst::Error::KeyNotFound(_) | tst::Error::PrefixNotFound(_)) => Ok(None),
+            Err(e) => Err(e.into()),
+        }
+    }
+}
+
+impl From<tst::Error> for super::Error<tst::Error> {
+    fn from(value: tst::Error) -> Self {
+        super::Error::DB(value)
+    }
+}
+
+impl From<ourdb::Error> for super::Error<tst::Error> {
+    fn from(value: ourdb::Error) -> Self {
+        super::Error::DB(tst::Error::OurDB(value))
+    }
+}

heromodels/src/lib.rs

@@ -1,2 +1,5 @@
 // Export the models module
-pub mod models;
+pub mod models;
+
+/// Database implementations
+pub mod db;

heromodels/src/models/core/mod.rs

@@ -3,5 +3,5 @@ pub mod model;
 pub mod comment;
 
 // Re-export key types for convenience
-pub use model::{Model, BaseModelData, IndexKey, IndexKeyBuilder};
+pub use model::{Model, BaseModelData, IndexKey, IndexKeyBuilder, Index};
 pub use comment::Comment;

heromodels/src/models/core/model.rs

@@ -73,6 +73,15 @@ pub trait Model: Debug + Clone + Serialize + for<'de> Deserialize<'de> + Send +
     }
 }
 
+/// An identifier for an index in the DB
+pub trait Index {
+    /// The model for which this is an index in the database
+    type Model: Model;
+
+    /// The key of this index
+    fn key() -> &'static str;
+}
+
 /// Base struct that all models should include
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct BaseModelData {

heromodels/src/models/mod.rs

@@ -3,6 +3,6 @@ pub mod core;
 pub mod userexample;
 
 // Re-export key types for convenience
-pub use core::model::{Model, BaseModelData, IndexKey};
+pub use core::model::{Model, BaseModelData, IndexKey, Index};
 pub use core::Comment;
 pub use userexample::User;

heromodels/src/models/userexample/user.rs

@@ -1,5 +1,5 @@
 use serde::{Deserialize, Serialize};
-use crate::models::core::model::{Model, BaseModelData, IndexKey};
+use crate::models::core::model::{Model, BaseModelData, IndexKey, Index};
 
 /// Represents a user in the system
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -109,3 +109,33 @@ impl Model for User {
         ]
     }
 }
+
+// Marker structs for indexed fields
+
+pub struct UserName;
+pub struct Email;
+pub struct IsActive;
+
+impl Index for UserName {
+    type Model = User;
+
+    fn key() -> &'static str {
+        "username"
+    }
+}
+
+impl Index for Email {
+    type Model = User;
+
+    fn key() -> &'static str {
+        "email"
+    }
+}
+
+impl Index for IsActive {
+    type Model = User;
+
+    fn key() -> &'static str {
+        "is_active"
+    }
+}