herocode/db
12
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Add alternative DB interface and implementation

Browse Source 
Signed-off-by: Lee Smet <lee.smet@hotmail.com>
This commit is contained in:
Lee Smet 2025-04-23 13:53:10 +02:00
parent 1708e6dd06
commit 276cf3c8d8
Signed by untrusted user who does not match committer: lee
GPG Key ID: 72CBFB5FDA7FE025
11 changed files with 1099 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

590
heromodels/Cargo.lock generated
View File

@ -25,19 +25,48 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
[[package]]
name = "bincode"
version = "1.3.3"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740"
dependencies = [
"bincode_derive",
"serde",
"unty",
]
[[package]]
name = "bincode_derive"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09"
dependencies = [
"virtue",
]
[[package]]
name = "bitflags"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd"
[[package]]
name = "bumpalo"
version = "3.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
[[package]]
name = "byteorder"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "byteview"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5"
[[package]]
name = "cc"
version = "1.2.19"
@ -68,19 +97,174 @@ dependencies = [
"windows-link",
]
[[package]]
name = "compare"
version = "0.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7"
[[package]]
name = "core-foundation-sys"
version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "crc32fast"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-skiplist"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
name = "dashmap"
version = "6.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
dependencies = [
"cfg-if",
"crossbeam-utils",
"hashbrown 0.14.5",
"lock_api",
"once_cell",
"parking_lot_core",
]
[[package]]
name = "double-ended-peekable"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57"
[[package]]
name = "enum_dispatch"
version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd"
dependencies = [
"once_cell",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "equivalent"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
[[package]]
name = "errno"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e"
dependencies = [
"libc",
"windows-sys",
]
[[package]]
name = "fastrand"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "fjall"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "958511f67d1f80e6bff9ffac05c626bb340d4602ca6ea5617d9901c218c894f0"
dependencies = [
"byteorder",
"byteview",
"dashmap",
"log",
"lsm-tree",
"path-absolutize",
"std-semaphore",
"tempfile",
"xxhash-rust",
]
[[package]]
name = "getrandom"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [
"cfg-if",
"libc",
"wasi 0.11.0+wasi-snapshot-preview1",
]
[[package]]
name = "getrandom"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0"
dependencies = [
"cfg-if",
"libc",
"r-efi",
"wasi 0.14.2+wasi-0.2.4",
]
[[package]]
name = "guardian"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f"
[[package]]
name = "hashbrown"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
[[package]]
name = "hashbrown"
version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
[[package]]
name = "heromodels"
version = "0.1.0"
dependencies = [
"bincode",
"chrono",
"fjall",
"ourdb",
"serde",
"tst",
]
[[package]]
@ -107,6 +291,15 @@ dependencies = [
"cc",
]
[[package]]
name = "interval-heap"
version = "0.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11274e5e8e89b8607cfedc2910b6626e998779b48a019151c7604d0adcb86ac6"
dependencies = [
"compare",
]
[[package]]
name = "js-sys"
version = "0.3.77"
@ -123,12 +316,58 @@ version = "0.2.172"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
[[package]]
name = "linux-raw-sys"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
[[package]]
name = "lock_api"
version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
dependencies = [
"autocfg",
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
[[package]]
name = "lsm-tree"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87d58bdef2dcbf50fce9f343265bdbd7fb08a458d241eb837ce426be22d674b4"
dependencies = [
"byteorder",
"crossbeam-skiplist",
"double-ended-peekable",
"enum_dispatch",
"guardian",
"interval-heap",
"log",
"lz4_flex",
"path-absolutize",
"quick_cache",
"rustc-hash",
"self_cell",
"tempfile",
"value-log",
"varint-rs",
"xxhash-rust",
]
[[package]]
name = "lz4_flex"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
[[package]]
name = "num-traits"
version = "0.2.19"
@ -144,6 +383,56 @@ version = "1.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
[[package]]
name = "ourdb"
version = "0.1.0"
dependencies = [
"crc32fast",
"log",
"rand",
"thiserror",
]
[[package]]
name = "parking_lot_core"
version = "0.9.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"smallvec",
"windows-targets",
]
[[package]]
name = "path-absolutize"
version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5"
dependencies = [
"path-dedot",
]
[[package]]
name = "path-dedot"
version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397"
dependencies = [
"once_cell",
]
[[package]]
name = "ppv-lite86"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
dependencies = [
"zerocopy",
]
[[package]]
name = "proc-macro2"
version = "1.0.95"
@ -153,6 +442,16 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "quick_cache"
version = "0.6.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "287e56aac5a2b4fb25a6fb050961d157635924c8696305a5c937a76f29841a0f"
dependencies = [
"equivalent",
"hashbrown 0.15.2",
]
[[package]]
name = "quote"
version = "1.0.40"
@ -162,12 +461,88 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "r-efi"
version = "5.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"
[[package]]
name = "rand"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha",
"rand_core",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom 0.2.15",
]
[[package]]
name = "redox_syscall"
version = "0.5.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3"
dependencies = [
"bitflags",
]
[[package]]
name = "rustc-hash"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
[[package]]
name = "rustix"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf"
dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys",
"windows-sys",
]
[[package]]
name = "rustversion"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2"
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "self_cell"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749"
[[package]]
name = "serde"
version = "1.0.219"
@ -194,6 +569,18 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "smallvec"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9"
[[package]]
name = "std-semaphore"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e"
[[package]]
name = "syn"
version = "2.0.100"
@ -205,12 +592,103 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "tempfile"
version = "3.19.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf"
dependencies = [
"fastrand",
"getrandom 0.3.2",
"once_cell",
"rustix",
"windows-sys",
]
[[package]]
name = "thiserror"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "tst"
version = "0.1.0"
dependencies = [
"ourdb",
"thiserror",
]
[[package]]
name = "unicode-ident"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
[[package]]
name = "unty"
version = "0.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae"
[[package]]
name = "value-log"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c"
dependencies = [
"byteorder",
"byteview",
"interval-heap",
"log",
"path-absolutize",
"rustc-hash",
"tempfile",
"varint-rs",
"xxhash-rust",
]
[[package]]
name = "varint-rs"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23"
[[package]]
name = "virtue"
version = "0.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasi"
version = "0.14.2+wasi-0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
dependencies = [
"wit-bindgen-rt",
]
[[package]]
name = "wasm-bindgen"
version = "0.2.100"
@ -327,3 +805,111 @@ checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97"
dependencies = [
"windows-link",
]
[[package]]
name = "windows-sys"
version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-targets"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "wit-bindgen-rt"
version = "0.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
dependencies = [
"bitflags",
]
[[package]]
name = "xxhash-rust"
version = "0.8.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3"
[[package]]
name = "zerocopy"
version = "0.8.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be"
dependencies = [
"proc-macro2",
"quote",
"syn",
]

7
heromodels/Cargo.toml
View File

@ -7,5 +7,8 @@ authors = ["Your Name <your.email@example.com>"]
[dependencies]
serde = { version = "1.0", features = ["derive"] }
bincode = "1.3"
chrono = { version = "0.4", features = ["serde"] }
bincode = { version = "2", features = ["serde"] }
chrono = { version = "0.4", features = ["serde"] }
fjall = "2.9.0"
ourdb = { path = "../ourdb" }
tst = { path = "../tst" }

131
heromodels/examples/basic_user_example.rs
View File

@ -1,28 +1,141 @@
use heromodels::models::{Model, Comment, User};
use heromodels::db::{Collection, Db};
use heromodels::models::userexample::user::{IsActive, UserName};
use heromodels::models::{Comment, Model, User};
fn main() {
let index_db = tst::TST::new("/tmp/ourdb/tst", true).expect("can create index DB");
let data_db = ourdb::OurDB::new(ourdb::OurDBConfig {
incremental_mode: false,
path: "/tmp/ourdb/ourdb".into(),
file_size: None,
keysize: None,
reset: Some(true),
})
.expect("can create data DB");
let db = heromodels::db::hero::OurDB::new(index_db, data_db);
println!("Hero Models - Basic Usage Example");
println!("================================");
// Create a new user using the fluent interface
let user = User::new(1)
.username("johndoe")
.email("john.doe@example.com")
.full_name("John Doe")
.is_active(false)
.build();
let user2 = User::new(2)
.username("janesmith")
.email("jane.smith@example.com")
.full_name("Jane Smith")
.is_active(true)
.build();
let user3 = User::new(3)
.username("willism")
.email("willis.masters@example.com")
.full_name("Willis Masters")
.is_active(true)
.build();
let user4 = User::new(4)
.username("carrols")
.email("carrol.smith@example.com")
.full_name("Carrol Smith")
.is_active(false)
.build();
db.collection()
.expect("can open user collection")
.set(&user)
.expect("can set user");
db.collection()
.expect("can open user collection")
.set(&user2)
.expect("can set user");
db.collection()
.expect("can open user collection")
.set(&user3)
.expect("can set user");
db.collection()
.expect("can open user collection")
.set(&user4)
.expect("can set user");
// Perform an indexed lookup on the Username
let stored_users = db
.collection::<User>()
.expect("can open user collection")
.get::<UserName>("johndoe")
.expect("can load stored user");
assert_eq!(stored_users.len(), 1);
let stored_user = &stored_users[0];
assert_eq!(user.username, stored_user.username);
assert_eq!(user.email, stored_user.email);
assert_eq!(user.is_active, stored_user.is_active);
assert_eq!(user.full_name, stored_user.full_name);
// Load all active users using the IsActive field index
// TODO: expand Index type so it defines the type of the key
let key = true.to_string();
let active_users = db
.collection::<User>()
.expect("can open user collection")
.get::<IsActive>(&key)
.expect("can load stored users");
// We should have 2 active users
assert_eq!(active_users.len(), 2);
// Now remove a user
db.collection::<User>()
.expect("can open user collection")
.delete_by_id(active_users[0].get_id())
.expect("can delete existing user");
// Load the active users again, should be 1 left
let active_users = db
.collection::<User>()
.expect("can open user collection")
.get::<IsActive>(&key)
.expect("can load stored users");
assert_eq!(active_users.len(), 1);
// And verify we still have 2 inactive users
let key = false.to_string();
let inactive_users = db
.collection::<User>()
.expect("can open user collection")
.get::<IsActive>(&key)
.expect("can load stored users");
assert_eq!(inactive_users.len(), 2);
println!("Created user: {:?}", user);
println!("User ID: {}", user.get_id());
println!("User DB Prefix: {}", User::db_prefix());
// Create a comment for the user
let comment = Comment::new(1)
.user_id(2) // commenter's user ID
let comment = Comment::new(5)
.user_id(1) // commenter's user ID
.content("This is a comment on the user")
.build();
db.collection()
.expect("can open commen collection")
.set(&comment)
.expect("can set comment");
let stored_comment = db
.collection::<Comment>()
.expect("can open comment collection")
.get_by_id(5)
.expect("can load stored comment");
assert!(stored_comment.is_some());
let stored_comment = stored_comment.unwrap();
assert_eq!(comment.get_id(), stored_comment.get_id());
assert_eq!(comment.content, stored_comment.content);
println!("\nCreated comment: {:?}", comment);
println!("Comment ID: {}", comment.get_id());
println!("Comment DB Prefix: {}", Comment::db_prefix());
}
}

69
heromodels/src/db.rs Normal file
View File

@ -0,0 +1,69 @@
use crate::models::{Index, Model};
use serde::{Deserialize, Serialize};
pub mod fjall;
pub mod hero;
pub trait Db {
/// Error type returned by database operations.
type Error: std::fmt::Debug;
/// Open the collection for a specific model. This method must create the collection if it does not exist.
fn collection<M: Model>(&self) -> Result<impl Collection<&str, M>, Error<Self::Error>>;
}
/// A collection stores a specific model under a specific key
pub trait Collection<K, V>
where
K: Serialize,
V: Serialize + for<'a> Deserialize<'a>,
{
/// Error type for database operations
type Error: std::fmt::Debug;
/// Get all items where the given index field is equal to key.
fn get<I>(&self, key: K) -> Result<Vec<V>, Error<Self::Error>>
where
I: Index<Model = V>;
/// Get an object from its ID. This does not use an index lookup
fn get_by_id(&self, id: u32) -> Result<Option<V>, Error<Self::Error>>;
/// Store an item in the DB.
fn set(&self, value: &V) -> Result<(), Error<Self::Error>>;
/// Delete all items from the db with a given index.
fn delete<I>(&self, key: K) -> Result<(), Error<Self::Error>>
where
I: Index<Model = V>;
/// Delete an object with a given ID
fn delete_by_id(&self, id: u32) -> Result<(), Error<Self::Error>>;
/// Get all objects from the colelction
fn get_all(&self) -> Result<Vec<V>, Error<Self::Error>>;
}
/// Errors returned by the DB implementation
#[derive(Debug)]
pub enum Error<E> {
/// Error in the underlying database
DB(E),
/// Error decoding a stored model
Decode(bincode::error::DecodeError),
/// Error encoding a model for storage
Encode(bincode::error::EncodeError),
}
impl<E> From<bincode::error::DecodeError> for Error<E> {
fn from(value: bincode::error::DecodeError) -> Self {
Error::Decode(value)
}
}
impl<E> From<bincode::error::EncodeError> for Error<E> {
fn from(value: bincode::error::EncodeError) -> Self {
Error::Encode(value)
}
}

0
heromodels/src/db/fjall.rs Normal file
View File

269
heromodels/src/db/hero.rs Normal file
View File

@ -0,0 +1,269 @@
use ourdb::OurDBSetArgs;
use serde::Deserialize;
use crate::models::{Index, Model};
use std::{
collections::HashSet,
sync::{Arc, Mutex},
};
const BINCODE_CONFIG: bincode::config::Configuration = bincode::config::standard();
#[derive(Clone)]
pub struct OurDB {
index: Arc<Mutex<tst::TST>>,
data: Arc<Mutex<ourdb::OurDB>>,
}
impl OurDB {
/// Create a new instance of ourdb
pub fn new(index_db: tst::TST, data_db: ourdb::OurDB) -> Self {
Self {
index: Arc::new(Mutex::new(index_db)),
data: Arc::new(Mutex::new(data_db)),
}
}
}
impl super::Db for OurDB {
type Error = tst::Error;
fn collection<M: crate::models::Model>(
&self,
) -> Result<impl super::Collection<&str, M>, super::Error<Self::Error>> {
Ok(self.clone())
}
}
impl<M> super::Collection<&str, M> for OurDB
where
M: Model,
{
type Error = tst::Error;
fn get<I>(&self, key: &str) -> Result<Vec<M>, super::Error<Self::Error>>
where
I: Index<Model = M>,
{
let mut index_db = self.index.lock().expect("can lock index DB");
let index_key = Self::index_key(M::db_prefix(), I::key(), key);
let Some(object_ids) = Self::get_tst_value::<HashSet<u32>>(&mut index_db, &index_key)?
else {
// If the index is not found just return an empty vector, no items present
return Ok(vec![]);
};
let mut data_db = self.data.lock().expect("can lock data DB");
object_ids
.into_iter()
.map(|obj_id| -> Result<M, _> {
let raw_obj = data_db.get(obj_id)?;
let (obj, _): (M, _) = bincode::serde::decode_from_slice(&raw_obj, BINCODE_CONFIG)?;
Ok(obj)
})
.collect()
}
fn get_by_id(&self, id: u32) -> Result<Option<M>, super::Error<Self::Error>> {
let mut data_db = self.data.lock().expect("can lock data db");
Self::get_ourdb_value(&mut data_db, id)
}
fn set(&self, value: &M) -> Result<(), super::Error<Self::Error>> {
// Before inserting the new object, check if an object with this ID already exists. If it does, we potentially need to update indices.
let mut data_db = self.data.lock().expect("can lock data DB");
let old_obj: Option<M> = Self::get_ourdb_value(&mut data_db, value.get_id())?;
let (indices_to_delete, indices_to_add) = if let Some(old_obj) = old_obj {
let mut indices_to_delete = vec![];
let mut indices_to_add = vec![];
let old_indices = old_obj.db_keys();
let new_indices = value.db_keys();
for old_index in old_indices {
for new_index in &new_indices {
if old_index.name == new_index.name {
if old_index.value != new_index.value {
// different value now, remove index
indices_to_delete.push(old_index);
// and later add the new one
indices_to_add.push(new_index.clone());
break;
}
}
}
}
// NOTE: we assume here that the index keys are stable, i.e. new index fields don't appear
// and existing ones don't dissapear
(indices_to_delete, indices_to_add)
} else {
(vec![], value.db_keys())
};
let mut index_db = self.index.lock().expect("can lock index db");
// First delete old indices which need to change
for old_index in indices_to_delete {
let key = Self::index_key(M::db_prefix(), old_index.name, &old_index.value);
let raw_ids = index_db.get(&key)?;
let (mut ids, _): (HashSet<u32>, _) =
bincode::serde::decode_from_slice(&raw_ids, BINCODE_CONFIG)?;
ids.remove(&value.get_id());
if ids.is_empty() {
// This was the last ID with this index value, remove index entirely
index_db.delete(&key)?;
} else {
// There are still objects left with this index value, write back updated set
let raw_ids = bincode::serde::encode_to_vec(ids, BINCODE_CONFIG)?;
index_db.set(&key, raw_ids)?;
}
}
// set or update the object
let v = bincode::serde::encode_to_vec(value, BINCODE_CONFIG)?;
let id = value.get_id();
data_db.set(OurDBSetArgs {
id: Some(id),
data: &v,
})?;
// Now add the new indices
for index_key in indices_to_add {
let key = Self::index_key(M::db_prefix(), index_key.name, &index_key.value);
// Load the existing id set for the index or create a new set
let mut existing_ids =
Self::get_tst_value::<HashSet<u32>>(&mut index_db, &key)?.unwrap_or_default();
existing_ids.insert(id);
let encoded_ids = bincode::serde::encode_to_vec(existing_ids, BINCODE_CONFIG)?;
index_db.set(&key, encoded_ids)?;
}
Ok(())
}
fn delete<I>(&self, key: &str) -> Result<(), super::Error<Self::Error>>
where
I: Index<Model = M>,
{
let mut index_db = self.index.lock().expect("can lock index db");
let key = Self::index_key(M::db_prefix(), I::key(), key);
let raw_obj_ids = index_db.get(&key)?;
let (obj_ids, _): (HashSet<u32>, _) =
bincode::serde::decode_from_slice(&raw_obj_ids, BINCODE_CONFIG)?;
let mut data_db = self.data.lock().expect("can lock data DB");
for obj_id in obj_ids {
// Load object
let raw = data_db.get(obj_id)?;
let (obj, _): (M, _) = bincode::serde::decode_from_slice(&raw, BINCODE_CONFIG)?;
// Delete indices
for index in obj.db_keys() {
let key = Self::index_key(M::db_prefix(), index.name, &index.value);
let raw_ids = index_db.get(&key)?;
let (mut ids, _): (HashSet<u32>, _) =
bincode::serde::decode_from_slice(&raw_ids, BINCODE_CONFIG)?;
ids.remove(&obj_id);
if ids.is_empty() {
// This was the last ID with this index value, remove index entirely
index_db.delete(&key)?;
} else {
// There are still objects left with this index value, write back updated set
let raw_ids = bincode::serde::encode_to_vec(ids, BINCODE_CONFIG)?;
index_db.set(&key, raw_ids)?;
}
}
// Delete object
data_db.delete(obj_id)?;
}
Ok(())
}
fn delete_by_id(&self, id: u32) -> Result<(), super::Error<Self::Error>> {
let mut data_db = self.data.lock().expect("can lock data DB");
let raw = data_db.get(id)?;
// First load the object so we can delete the indices
// Delete indices before the actual object, so we don't leave dangling references to a deleted
// object ID in case something goes wrong.
let (obj, _): (M, _) = bincode::serde::decode_from_slice(&raw, BINCODE_CONFIG)?;
let mut index_db = self.index.lock().expect("can lock index DB");
for index_key in obj.db_keys() {
let key = Self::index_key(M::db_prefix(), index_key.name, &index_key.value);
let raw_ids = index_db.get(&key)?;
let (mut ids, _): (HashSet<u32>, _) =
bincode::serde::decode_from_slice(&raw_ids, BINCODE_CONFIG)?;
ids.remove(&obj.get_id());
if ids.is_empty() {
// This was the last ID with this index value, remove index entirely
index_db.delete(&key)?;
} else {
// There are still objects left with this index value, write back updated set
let raw_ids = bincode::serde::encode_to_vec(ids, BINCODE_CONFIG)?;
index_db.set(&key, raw_ids)?;
}
}
// Finally delete the object itself
Ok(data_db.delete(id)?)
}
fn get_all(&self) -> Result<Vec<M>, super::Error<Self::Error>> {
todo!("OurDB doesn't have a list all method yet")
}
}
impl OurDB {
/// Build the key used for an indexed field of a collection.
fn index_key(collection: &str, index: &str, value: &str) -> String {
format!("{collection}::{index}::{value}")
}
/// Wrapper to load values from ourdb and transform a not found error in to Ok(None)
fn get_ourdb_value<V>(
data: &mut ourdb::OurDB,
id: u32,
) -> Result<Option<V>, super::Error<tst::Error>>
where
V: for<'de> Deserialize<'de>,
{
match data.get(id) {
Ok(raw) => {
let (obj, _): (V, _) = bincode::serde::decode_from_slice(&raw, BINCODE_CONFIG)?;
Ok(Some(obj))
}
Err(ourdb::Error::NotFound(_)) => Ok(None),
Err(e) => Err(e.into()),
}
}
fn get_tst_value<V>(
index: &mut tst::TST,
key: &str,
) -> Result<Option<V>, super::Error<tst::Error>>
where
V: for<'de> Deserialize<'de>,
{
match index.get(key) {
Ok(raw) => {
let (obj, _): (V, _) = bincode::serde::decode_from_slice(&raw, BINCODE_CONFIG)?;
Ok(Some(obj))
}
Err(tst::Error::KeyNotFound(_) | tst::Error::PrefixNotFound(_)) => Ok(None),
Err(e) => Err(e.into()),
}
}
}
impl From<tst::Error> for super::Error<tst::Error> {
fn from(value: tst::Error) -> Self {
super::Error::DB(value)
}
}
impl From<ourdb::Error> for super::Error<tst::Error> {
fn from(value: ourdb::Error) -> Self {
super::Error::DB(tst::Error::OurDB(value))
}
}

5
heromodels/src/lib.rs
View File

@ -1,2 +1,5 @@
// Export the models module
pub mod models;
pub mod models;
/// Database implementations
pub mod db;

2
heromodels/src/models/core/mod.rs
View File

@ -3,5 +3,5 @@ pub mod model;
pub mod comment;
// Re-export key types for convenience
pub use model::{Model, BaseModelData, IndexKey, IndexKeyBuilder};
pub use model::{Model, BaseModelData, IndexKey, IndexKeyBuilder, Index};
pub use comment::Comment;

9
heromodels/src/models/core/model.rs
View File

@ -73,6 +73,15 @@ pub trait Model: Debug + Clone + Serialize + for<'de> Deserialize<'de> + Send +
}
}
/// An identifier for an index in the DB
pub trait Index {
/// The model for which this is an index in the database
type Model: Model;
/// The key of this index
fn key() -> &'static str;
}
/// Base struct that all models should include
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BaseModelData {

2
heromodels/src/models/mod.rs
View File

@ -3,6 +3,6 @@ pub mod core;
pub mod userexample;
// Re-export key types for convenience
pub use core::model::{Model, BaseModelData, IndexKey};
pub use core::model::{Model, BaseModelData, IndexKey, Index};
pub use core::Comment;
pub use userexample::User;

32
heromodels/src/models/userexample/user.rs
View File

@ -1,5 +1,5 @@
use serde::{Deserialize, Serialize};
use crate::models::core::model::{Model, BaseModelData, IndexKey};
use crate::models::core::model::{Model, BaseModelData, IndexKey, Index};
/// Represents a user in the system
#[derive(Debug, Clone, Serialize, Deserialize)]
@ -109,3 +109,33 @@ impl Model for User {
]
}
}
// Marker structs for indexed fields
pub struct UserName;
pub struct Email;
pub struct IsActive;
impl Index for UserName {
type Model = User;
fn key() -> &'static str {
"username"
}
}
impl Index for Email {
type Model = User;
fn key() -> &'static str {
"email"
}
}
impl Index for IsActive {
type Model = User;
fn key() -> &'static str {
"is_active"
}
}