Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9177fa4091 | |||
| 51ab90c4ad |
73
Cargo.lock
generated
73
Cargo.lock
generated
@ -221,6 +221,12 @@ dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.10.1"
|
||||
@ -655,33 +661,28 @@ version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "herocrypto"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"libcrypto",
|
||||
"libcryptoa",
|
||||
"redis",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "herodb"
|
||||
version = "0.1.0"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"age",
|
||||
"anyhow",
|
||||
"base64 0.22.1",
|
||||
"bincode",
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"chacha20poly1305",
|
||||
"clap",
|
||||
"ed25519-dalek",
|
||||
"libcryptoa",
|
||||
"libdbstorage",
|
||||
"log",
|
||||
"futures",
|
||||
"rand",
|
||||
"redb",
|
||||
"redis",
|
||||
"secrecy",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
@ -948,41 +949,6 @@ version = "0.2.175"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543"
|
||||
|
||||
[[package]]
|
||||
name = "libcrypto"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"chacha20poly1305",
|
||||
"rand",
|
||||
"sha2",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libcryptoa"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"age",
|
||||
"base64 0.22.1",
|
||||
"ed25519-dalek",
|
||||
"rand",
|
||||
"secrecy",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libdbstorage"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"bincode",
|
||||
"libcrypto",
|
||||
"redb",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litemap"
|
||||
version = "0.8.0"
|
||||
@ -1564,15 +1530,6 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
|
||||
name = "supervisor"
|
||||
version = "0.1.0"
|
||||
|
||||
[[package]]
|
||||
name = "supervisorrpc"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"herocrypto",
|
||||
"redis",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.109"
|
||||
|
||||
42
Cargo.toml
42
Cargo.toml
@ -1,38 +1,12 @@
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
members = [
|
||||
"crates/herodb",
|
||||
"crates/libdbstorage",
|
||||
"crates/libcrypto",
|
||||
"crates/libcryptoa",
|
||||
"crates/herocrypto",
|
||||
"crates/supervisor",
|
||||
"crates/supervisorrpc",
|
||||
"herodb",
|
||||
"supervisor",
|
||||
]
|
||||
resolver = "2"
|
||||
|
||||
[workspace.dependencies]
|
||||
# Common
|
||||
anyhow = "1.0"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
thiserror = "1.0"
|
||||
log = "0.4"
|
||||
bytes = "1.3"
|
||||
|
||||
# Crypto - Asymmetric
|
||||
age = "0.10"
|
||||
secrecy = "0.8"
|
||||
ed25519-dalek = "2"
|
||||
base64 = "0.22"
|
||||
|
||||
# Crypto - Symmetric & Utilities
|
||||
chacha20poly1305 = "0.10"
|
||||
rand = "0.8"
|
||||
sha2 = "0.10"
|
||||
|
||||
# Database
|
||||
redb = "2.1"
|
||||
|
||||
# CLI
|
||||
clap = { version = "4.5", features = ["derive"] }
|
||||
# You can define shared profiles for all workspace members here
|
||||
[profile.release]
|
||||
lto = true
|
||||
codegen-units = 1
|
||||
strip = true
|
||||
@ -1,10 +0,0 @@
|
||||
[package]
|
||||
name = "herocrypto"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
redis = { version = "0.24", features = ["tokio-comp"] }
|
||||
thiserror = { workspace = true }
|
||||
libcrypto = { path = "../libcrypto" }
|
||||
libcryptoa = { path = "../libcryptoa" }
|
||||
@ -1,45 +0,0 @@
|
||||
// In crates/herocrypto/src/lib.rs
|
||||
use redis::{Commands, RedisResult};
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum Error {
|
||||
#[error("Redis connection error: {0}")]
|
||||
Redis(#[from] redis::RedisError),
|
||||
#[error("Asymmetric crypto error: {0}")]
|
||||
Asymmetric(#[from] libcryptoa::AsymmetricCryptoError),
|
||||
#[error("Key not found in database: {0}")]
|
||||
KeyNotFound(String),
|
||||
#[error("Command failed on server: {0}")]
|
||||
CommandError(String),
|
||||
}
|
||||
|
||||
pub struct HeroCrypto {
|
||||
// e.g., using a connection manager from redis-rs
|
||||
client: redis::Client,
|
||||
}
|
||||
|
||||
impl HeroCrypto {
|
||||
pub fn new(redis_url: &str) -> Result<Self, Error> {
|
||||
Ok(Self { client: redis::Client::open(redis_url)? })
|
||||
}
|
||||
|
||||
// --- High-level functions to be implemented ---
|
||||
|
||||
/// Generates a new keypair and stores it in HeroDB under the given name.
|
||||
pub async fn generate_keypair(&self, name: &str) -> Result<(), Error> {
|
||||
let mut con = self.client.get_async_connection().await?;
|
||||
let (_pub, _priv): (String, String) = redis::cmd("AGE")
|
||||
.arg("KEYGEN")
|
||||
.arg(name)
|
||||
.query_async(&mut con)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Encrypts a message using a key stored in HeroDB.
|
||||
pub async fn encrypt_by_name(&self, key_name: &str, plaintext: &str) -> Result<String, Error> {
|
||||
// Implementation will call 'AGE ENCRYPTNAME ...'
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
@ -1,31 +0,0 @@
|
||||
[package]
|
||||
name = "herodb"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
authors = ["Pin Fang <fpfangpin@hotmail.com>"]
|
||||
|
||||
[[bin]]
|
||||
name = "herodb"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
# Workspace dependencies
|
||||
anyhow = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
log = { workspace = true }
|
||||
clap = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
base64 = { workspace = true }
|
||||
age = { workspace = true }
|
||||
secrecy = { workspace = true }
|
||||
ed25519-dalek = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
|
||||
# Local Crate Dependencies
|
||||
libdbstorage = { path = "../libdbstorage" }
|
||||
# We will create these libraries in the next steps
|
||||
libcryptoa = { path = "../libcryptoa" }
|
||||
|
||||
[dev-dependencies]
|
||||
redis = { version = "0.24", features = ["aio", "tokio-comp"] }
|
||||
@ -1,267 +0,0 @@
|
||||
# Build Your Own Redis in Rust
|
||||
|
||||
This project is to build a toy Redis-Server clone that's capable of parsing Redis protocol and handling basic Redis commands, parsing and initializing Redis from RDB file,
|
||||
supporting leader-follower replication, redis streams (queue), redis batch commands in transaction.
|
||||
|
||||
You can find all the source code and commit history in [my github repo](https://github.com/fangpin/redis-rs).
|
||||
|
||||
## Main features
|
||||
+ Parse Redis protocol
|
||||
+ Handle basic Redis commands
|
||||
+ Parse and initialize Redis from RDB file
|
||||
+ Leader-follower Replication
|
||||
|
||||
## Prerequisites
|
||||
install `redis-cli` first (an implementation of redis client for test purpose)
|
||||
```sh
|
||||
cargo install mini-redis
|
||||
```
|
||||
|
||||
Learn about:
|
||||
- [Redis protocoal](https://redis.io/docs/latest/develop/reference/protocol-spec)
|
||||
- [RDB file format](https://rdb.fnordig.de/file_format.html)
|
||||
- [Redis replication](https://redis.io/docs/management/replication/)
|
||||
|
||||
## Start the Redis-rs server
|
||||
```sh
|
||||
# start as master
|
||||
cargo run -- --dir /some/db/path --dbfilename dump.rdb
|
||||
# start as slave
|
||||
cargo run -- --dir /some/db/path --dbfilename dump.rdb --port 6380 --replicaof "localhost 6379"
|
||||
```
|
||||
|
||||
|
||||
## Supported Commands
|
||||
```sh
|
||||
# basic commands
|
||||
redis-cli PING
|
||||
redis-cli ECHO hey
|
||||
redis-cli SET foo bar
|
||||
redis-cli SET foo bar px/ex 100
|
||||
redis-cli GET foo
|
||||
redis-cli SET foo 2
|
||||
redis-cli INCR foo
|
||||
redis-cli INCR missing_key
|
||||
redis-cli TYPE some_key
|
||||
redis-cli KEYS "*"
|
||||
|
||||
# leader-follower replication related commands
|
||||
redis-cli CONFIG GET dbfilename
|
||||
redis-cli INFO replication
|
||||
|
||||
# streams related commands
|
||||
redis-cli XADD stream_key 1526919030474-0 temperature 36 humidity 95
|
||||
redis-cli XADD stream_key 1526919030474-* temperature 37 humidity 94
|
||||
redis-cli XADD stream_key "*" foo bar
|
||||
## read stream
|
||||
redis-cli XRANGE stream_key 0-2 0-3
|
||||
## query with + -
|
||||
redis-cli XRANGE some_key - 1526985054079
|
||||
## query single stream using xread
|
||||
redis-cli XREAD streams some_key 1526985054069-0
|
||||
## query multiple stream using xread
|
||||
redis-cli XREAD streams stream_key other_stream_key 0-0 0-1
|
||||
## blocking reads without timeout
|
||||
redis-cli XREAD block 0 streams some_key 1526985054069-0
|
||||
|
||||
|
||||
# transactions related commands
|
||||
## start a transaction and exec all queued commands in a transaction
|
||||
redis-cli
|
||||
> MULTI
|
||||
> set foo 1
|
||||
> incr foo
|
||||
> exec
|
||||
## start a transaction and queued commands and cancel transaction then
|
||||
redis-cli
|
||||
> MULTI
|
||||
> set foo 1
|
||||
> incr foo
|
||||
> discard
|
||||
|
||||
```
|
||||
|
||||
## RDB Persistence
|
||||
Get Redis-rs server config
|
||||
```sh
|
||||
redis-cli CONFIG GET dbfilename
|
||||
```
|
||||
### RDB file format overview
|
||||
Here are the different sections of the [RDB file](https://rdb.fnordig.de/file_format.html), in order:
|
||||
+ Header section
|
||||
+ Metadata section
|
||||
+ Database section
|
||||
+ End of file section
|
||||
#### Header section
|
||||
start with some magic number
|
||||
```sh
|
||||
52 45 44 49 53 30 30 31 31 // Magic string + version number (ASCII): "REDIS0011".
|
||||
```
|
||||
#### Metadata section
|
||||
contains zero or more "metadata subsections", which each specify a single metadata attribute
|
||||
e.g.
|
||||
```sh
|
||||
FA // Indicates the start of a metadata subsection.
|
||||
09 72 65 64 69 73 2D 76 65 72 // The name of the metadata attribute (string encoded): "redis-ver".
|
||||
06 36 2E 30 2E 31 36 // The value of the metadata attribute (string encoded): "6.0.16".
|
||||
```
|
||||
#### Database section
|
||||
contains zero or more "database subsections," which each describe a single database.
|
||||
e.g.
|
||||
```sh
|
||||
FE // Indicates the start of a database subsection.
|
||||
00 /* The index of the database (size encoded). Here, the index is 0. */
|
||||
|
||||
FB // Indicates that hash table size information follows.
|
||||
03 /* The size of the hash table that stores the keys and values (size encoded). Here, the total key-value hash table size is 3. */
|
||||
02 /* The size of the hash table that stores the expires of the keys (size encoded). Here, the number of keys with an expiry is 2. */
|
||||
```
|
||||
|
||||
```sh
|
||||
00 /* The 1-byte flag that specifies the value’s type and encoding. Here, the flag is 0, which means "string." */
|
||||
06 66 6F 6F 62 61 72 // The name of the key (string encoded). Here, it's "foobar".
|
||||
06 62 61 7A 71 75 78 // The value (string encoded). Here, it's "bazqux".
|
||||
```
|
||||
|
||||
```sh
|
||||
FC /* Indicates that this key ("foo") has an expire, and that the expire timestamp is expressed in milliseconds. */
|
||||
15 72 E7 07 8F 01 00 00 /* The expire timestamp, expressed in Unix time, stored as an 8-byte unsigned long, in little-endian (read right-to-left). Here, the expire timestamp is 1713824559637. */
|
||||
00 // Value type is string.
|
||||
03 66 6F 6F // Key name is "foo".
|
||||
03 62 61 72 // Value is "bar".
|
||||
```
|
||||
|
||||
```sh
|
||||
FD /* Indicates that this key ("baz") has an expire, and that the expire timestamp is expressed in seconds. */
|
||||
52 ED 2A 66 /* The expire timestamp, expressed in Unix time, stored as an 4-byte unsigned integer, in little-endian (read right-to-left). Here, the expire timestamp is 1714089298. */
|
||||
00 // Value type is string.
|
||||
03 62 61 7A // Key name is "baz".
|
||||
03 71 75 78 // Value is "qux".
|
||||
```
|
||||
|
||||
In summary,
|
||||
- Optional expire information (one of the following):
|
||||
- Timestamp in seconds:
|
||||
- FD
|
||||
- Expire timestamp in seconds (4-byte unsigned integer)
|
||||
- Timestamp in milliseconds:
|
||||
- FC
|
||||
- Expire timestamp in milliseconds (8-byte unsigned long)
|
||||
- Value type (1-byte flag)
|
||||
- Key (string encoded)
|
||||
- Value (encoding depends on value type)
|
||||
|
||||
#### End of file section
|
||||
```sh
|
||||
FF /* Indicates that the file is ending, and that the checksum follows. */
|
||||
89 3b b7 4e f8 0f 77 19 // An 8-byte CRC64 checksum of the entire file.
|
||||
```
|
||||
|
||||
#### Size encoding
|
||||
```sh
|
||||
/* If the first two bits are 0b00:
|
||||
The size is the remaining 6 bits of the byte.
|
||||
In this example, the size is 10: */
|
||||
0A
|
||||
00001010
|
||||
|
||||
/* If the first two bits are 0b01:
|
||||
The size is the next 14 bits
|
||||
(remaining 6 bits in the first byte, combined with the next byte),
|
||||
in big-endian (read left-to-right).
|
||||
In this example, the size is 700: */
|
||||
42 BC
|
||||
01000010 10111100
|
||||
|
||||
/* If the first two bits are 0b10:
|
||||
Ignore the remaining 6 bits of the first byte.
|
||||
The size is the next 4 bytes, in big-endian (read left-to-right).
|
||||
In this example, the size is 17000: */
|
||||
80 00 00 42 68
|
||||
10000000 00000000 00000000 01000010 01101000
|
||||
|
||||
/* If the first two bits are 0b11:
|
||||
The remaining 6 bits specify a type of string encoding.
|
||||
See string encoding section. */
|
||||
```
|
||||
|
||||
#### String encoding
|
||||
+ The size of the string (size encoded).
|
||||
+ The string.
|
||||
```sh
|
||||
/* The 0x0D size specifies that the string is 13 characters long. The remaining characters spell out "Hello, World!". */
|
||||
0D 48 65 6C 6C 6F 2C 20 57 6F 72 6C 64 21
|
||||
```
|
||||
For sizes that begin with 0b11, the remaining 6 bits indicate a type of string format:
|
||||
```sh
|
||||
/* The 0xC0 size indicates the string is an 8-bit integer. In this example, the string is "123". */
|
||||
C0 7B
|
||||
|
||||
/* The 0xC1 size indicates the string is a 16-bit integer. The remaining bytes are in little-endian (read right-to-left). In this example, the string is "12345". */
|
||||
C1 39 30
|
||||
|
||||
/* The 0xC2 size indicates the string is a 32-bit integer. The remaining bytes are in little-endian (read right-to-left), In this example, the string is "1234567". */
|
||||
C2 87 D6 12 00
|
||||
|
||||
/* The 0xC3 size indicates that the string is compressed with the LZF algorithm. You will not encounter LZF-compressed strings in this challenge. */
|
||||
C3 ...
|
||||
```
|
||||
|
||||
## Replication
|
||||
Redis server [leader-follower replication](https://redis.io/docs/management/replication/).
|
||||
Run multiple Redis servers with one acting as the "master" and the others as "replicas". Changes made to the master will be automatically replicated to replicas.
|
||||
|
||||
### Send Handshake (follower -> master)
|
||||
1. When the follower starts, it will send a PING command to the master as RESP Array.
|
||||
2. Then 2 REPLCONF (replication config) commands are sent to master from follower to communicate the port and the sync protocol. One is *REPLCONF listening-port <PORT>* and the other is *REPLCONF capa psync2*. psnync2 is an example sync protocol supported in this project.
|
||||
3. The follower sends the *PSYNC* command to master with replication id and offset to start the replication process.
|
||||
|
||||
### Receive Handshake (master -> follower)
|
||||
1. Response a PONG message to follower.
|
||||
2. Response an OK message to follower for both REPLCONF commands.
|
||||
3. Response a *+FULLRESYNC <REPL_ID> 0\r\n* to follower with the replication id and offset.
|
||||
|
||||
### RDB file transfer
|
||||
When the follower starts, it sends a *PSYNC ? -1* command to tell master that it doesn't have any data yet, and needs a full resync.
|
||||
|
||||
Then the master send a *FULLRESYNC* response to the follower as an acknowledgement.
|
||||
|
||||
Finally, the master send the RDB file to represent its current state to the follower. The follower should load the RDB file received to the memory, replacing its current state.
|
||||
|
||||
### Receive write commands (master -> follower)
|
||||
The master sends following write commands to the follower with the offset info.
|
||||
The sending is to reuse the same TCP connection of handshake and RDB file transfer.
|
||||
As the all the commands are encoded as RESP Array just like a normal client command, so the follower could reuse the same logic to handler the replicate commands from master. The only difference is the commands are coming from the master and no need response back.
|
||||
|
||||
|
||||
## Streams
|
||||
A stream is identified by a key, and it contains multiple entries.
|
||||
Each entry consists of one or more key-value pairs, and is assigned a unique ID.
|
||||
[More about redis streams](https://redis.io/docs/latest/develop/data-types/streams/)
|
||||
[Radix tree](https://en.wikipedia.org/wiki/Radix_tree)
|
||||
|
||||
|
||||
It looks like a list of key-value pairs.
|
||||
```sh
|
||||
entries:
|
||||
- id: 1526985054069-0 # (ID of the first entry)
|
||||
temperature: 36 # (A key value pair in the first entry)
|
||||
humidity: 95 # (Another key value pair in the first entry)
|
||||
|
||||
- id: 1526985054079-0 # (ID of the second entry)
|
||||
temperature: 37 # (A key value pair in the first entry)
|
||||
humidity: 94 # (Another key value pair in the first entry)
|
||||
|
||||
# ... (and so on)
|
||||
|
||||
```
|
||||
|
||||
Examples of Redis stream use cases include:
|
||||
- Event sourcing (e.g., tracking user actions, clicks, etc.)
|
||||
- Sensor monitoring (e.g., readings from devices in the field)
|
||||
- Notifications (e.g., storing a record of each user's notifications in a separate stream)
|
||||
|
||||
## Transaction
|
||||
When *MULTI* command is called in a connection, redis just queued all following commands until *EXEC* or *DISCARD* command is called.
|
||||
*EXEC* command will execute all queued commands and return an array representation of all execution result (including), instead the *DISCARD* command just clear all queued commands.
|
||||
The transactions among each client connection are independent.
|
||||
@ -1 +0,0 @@
|
||||
fn main() {}
|
||||
@ -1,10 +0,0 @@
|
||||
[package]
|
||||
name = "libcrypto"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
chacha20poly1305 = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
@ -1,12 +0,0 @@
|
||||
[package]
|
||||
name = "libcryptoa"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
age = { workspace = true }
|
||||
secrecy = { workspace = true }
|
||||
ed25519-dalek = { workspace = true }
|
||||
base64 = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
@ -1,100 +0,0 @@
|
||||
// In crates/libcryptoa/src/lib.rs
|
||||
use std::str::FromStr;
|
||||
use age::{Decryptor, Encryptor, x25519};
|
||||
use base64::{engine::general_purpose::STANDARD as B64, Engine as _};
|
||||
use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey};
|
||||
use secrecy::ExposeSecret;
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum AsymmetricCryptoError {
|
||||
#[error("key parsing failed")]
|
||||
ParseKey,
|
||||
#[error("age crypto error: {0}")]
|
||||
Age(String),
|
||||
#[error("invalid utf-8 in plaintext")]
|
||||
Utf8,
|
||||
#[error("invalid signature length")]
|
||||
SignatureLen,
|
||||
#[error("signature verification failed")]
|
||||
Verify,
|
||||
#[error("base64 decoding failed: {0}")]
|
||||
Base64(#[from] base64::DecodeError),
|
||||
#[error("io error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
}
|
||||
|
||||
fn parse_recipient(s: &str) -> Result<x25519::Recipient, AsymmetricCryptoError> {
|
||||
x25519::Recipient::from_str(s).map_err(|_| AsymmetricCryptoError::ParseKey)
|
||||
}
|
||||
|
||||
fn parse_identity(s: &str) -> Result<x25519::Identity, AsymmetricCryptoError> {
|
||||
x25519::Identity::from_str(s).map_err(|_| AsymmetricCryptoError::ParseKey)
|
||||
}
|
||||
|
||||
fn parse_ed25519_signing_key(s: &str) -> Result<SigningKey, AsymmetricCryptoError> {
|
||||
let bytes = B64.decode(s)?;
|
||||
let key_bytes: [u8; 32] = bytes.try_into().map_err(|_| AsymmetricCryptoError::ParseKey)?;
|
||||
Ok(SigningKey::from_bytes(&key_bytes))
|
||||
}
|
||||
|
||||
fn parse_ed25519_verifying_key(s: &str) -> Result<VerifyingKey, AsymmetricCryptoError> {
|
||||
let bytes = B64.decode(s)?;
|
||||
let key_bytes: [u8; 32] = bytes.try_into().map_err(|_| AsymmetricCryptoError::ParseKey)?;
|
||||
VerifyingKey::from_bytes(&key_bytes).map_err(|_| AsymmetricCryptoError::ParseKey)
|
||||
}
|
||||
|
||||
pub fn gen_enc_keypair() -> (String, String) {
|
||||
let id = x25519::Identity::generate();
|
||||
let pk = id.to_public();
|
||||
(pk.to_string(), id.to_string().expose_secret().to_string())
|
||||
}
|
||||
|
||||
pub fn gen_sign_keypair() -> (String, String) {
|
||||
let signing_key = SigningKey::generate(&mut rand::rngs::OsRng);
|
||||
let verifying_key = signing_key.verifying_key();
|
||||
(B64.encode(verifying_key.to_bytes()), B64.encode(signing_key.to_bytes()))
|
||||
}
|
||||
|
||||
pub fn encrypt_b64(recipient_str: &str, msg: &str) -> Result<String, AsymmetricCryptoError> {
|
||||
let recipient = parse_recipient(recipient_str)?;
|
||||
let encryptor = Encryptor::with_recipients(vec![Box::new(recipient)])
|
||||
.ok_or_else(|| AsymmetricCryptoError::Age("Failed to create encryptor".into()))?;
|
||||
|
||||
let mut encrypted = vec![];
|
||||
let mut writer = encryptor.wrap_output(&mut encrypted)?;
|
||||
std::io::Write::write_all(&mut writer, msg.as_bytes())?;
|
||||
writer.finish()?;
|
||||
|
||||
Ok(B64.encode(encrypted))
|
||||
}
|
||||
|
||||
pub fn decrypt_b64(identity_str: &str, ct_b64: &str) -> Result<String, AsymmetricCryptoError> {
|
||||
let identity = parse_identity(identity_str)?;
|
||||
let ct = B64.decode(ct_b64)?;
|
||||
|
||||
let decryptor = Decryptor::new(&ct[..]).map_err(|e| AsymmetricCryptoError::Age(e.to_string()))?;
|
||||
|
||||
let mut decrypted = vec![];
|
||||
if let Decryptor::Recipients(d) = decryptor {
|
||||
let mut reader = d.decrypt(std::iter::once(&identity as &dyn age::Identity))
|
||||
.map_err(|e| AsymmetricCryptoError::Age(e.to_string()))?;
|
||||
std::io::Read::read_to_end(&mut reader, &mut decrypted)?;
|
||||
String::from_utf8(decrypted).map_err(|_| AsymmetricCryptoError::Utf8)
|
||||
} else {
|
||||
Err(AsymmetricCryptoError::Age("Passphrase decryption not supported".into()))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn sign_b64(signing_secret_str: &str, msg: &str) -> Result<String, AsymmetricCryptoError> {
|
||||
let signing_key = parse_ed25519_signing_key(signing_secret_str)?;
|
||||
let signature = signing_key.sign(msg.as_bytes());
|
||||
Ok(B64.encode(signature.to_bytes()))
|
||||
}
|
||||
|
||||
pub fn verify_b64(verify_pub_str: &str, msg: &str, sig_b64: &str) -> Result<bool, AsymmetricCryptoError> {
|
||||
let verifying_key = parse_ed25519_verifying_key(verify_pub_str)?;
|
||||
let sig_bytes = B64.decode(sig_b64)?;
|
||||
let signature = Signature::from_slice(&sig_bytes).map_err(|_| AsymmetricCryptoError::SignatureLen)?;
|
||||
Ok(verifying_key.verify(msg.as_bytes(), &signature).is_ok())
|
||||
}
|
||||
@ -1,15 +0,0 @@
|
||||
[package]
|
||||
name = "libdbstorage"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
redb = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
# Local Crate Dependencies
|
||||
libcrypto = { path = "../libcrypto" }
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
bincode = "1.3.3"
|
||||
@ -1,4 +0,0 @@
|
||||
pub mod storage_basic;
|
||||
pub mod storage_hset;
|
||||
pub mod storage_lists;
|
||||
pub mod storage_extra;
|
||||
@ -1,18 +0,0 @@
|
||||
[package]
|
||||
name = "supervisorrpc"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[[bin]]
|
||||
name = "supervisorrpc"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
# Example dependencies for an RPC server
|
||||
# axum = "0.7"
|
||||
# jsonrpsee = { version = "0.22", features = ["server"] }
|
||||
# openrpc-types = "0.7"
|
||||
|
||||
tokio = { workspace = true }
|
||||
redis = { version = "0.24", features = ["tokio-comp"] }
|
||||
herocrypto = { path = "../herocrypto" }
|
||||
@ -1,12 +0,0 @@
|
||||
// To be implemented:
|
||||
// 1. Define an OpenRPC schema for supervisor functions (e.g., server status, key rotation).
|
||||
// 2. Implement an HTTP/TCP server (e.g., using Axum or jsonrpsee) that serves the schema
|
||||
// and handles RPC calls.
|
||||
// 3. Implement support for Unix domain sockets in addition to TCP.
|
||||
// 4. Use the `herocrypto` or `redis-rs` crate to interact with the main `herodb` instance.
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
println!("Supervisor RPC server starting... (not implemented)");
|
||||
// Server setup code will go here.
|
||||
}
|
||||
28
herodb/Cargo.toml
Normal file
28
herodb/Cargo.toml
Normal file
@ -0,0 +1,28 @@
|
||||
[package]
|
||||
name = "herodb"
|
||||
version = "0.0.1"
|
||||
authors = ["Pin Fang <fpfangpin@hotmail.com>"]
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0.59"
|
||||
bytes = "1.3.0"
|
||||
thiserror = "1.0.32"
|
||||
tokio = { version = "1.23.0", features = ["full"] }
|
||||
clap = { version = "4.5.20", features = ["derive"] }
|
||||
byteorder = "1.4.3"
|
||||
futures = "0.3"
|
||||
redb = "2.1.3"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = "1.0"
|
||||
bincode = "1.3.3"
|
||||
chacha20poly1305 = "0.10.1"
|
||||
rand = "0.8"
|
||||
sha2 = "0.10"
|
||||
age = "0.10"
|
||||
secrecy = "0.8"
|
||||
ed25519-dalek = "2"
|
||||
base64 = "0.22"
|
||||
|
||||
[dev-dependencies]
|
||||
redis = { version = "0.24", features = ["aio", "tokio-comp"] }
|
||||
0
herodb/README.md
Normal file
0
herodb/README.md
Normal file
9
herodb/build.sh
Executable file
9
herodb/build.sh
Executable file
@ -0,0 +1,9 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
export SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
echo "I am in $SCRIPT_DIR"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
cargo build
|
||||
|
||||
99
herodb/instructions/encrypt.md
Normal file
99
herodb/instructions/encrypt.md
Normal file
@ -0,0 +1,99 @@
|
||||
|
||||
### Cargo.toml
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
chacha20poly1305 = { version = "0.10", features = ["xchacha20"] }
|
||||
rand = "0.8"
|
||||
sha2 = "0.10"
|
||||
```
|
||||
|
||||
### `crypto_factory.rs`
|
||||
|
||||
```rust
|
||||
use chacha20poly1305::{
|
||||
aead::{Aead, KeyInit, OsRng},
|
||||
XChaCha20Poly1305, Key, XNonce,
|
||||
};
|
||||
use rand::RngCore;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
const VERSION: u8 = 1;
|
||||
const NONCE_LEN: usize = 24;
|
||||
const TAG_LEN: usize = 16;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum CryptoError {
|
||||
Format, // wrong length / header
|
||||
Version(u8), // unknown version
|
||||
Decrypt, // wrong key or corrupted data
|
||||
}
|
||||
|
||||
/// Super-simple factory: new(secret) + encrypt(bytes) + decrypt(bytes)
|
||||
pub struct CryptoFactory {
|
||||
key: Key<XChaCha20Poly1305>,
|
||||
}
|
||||
|
||||
impl CryptoFactory {
|
||||
/// Accepts any secret bytes; turns them into a 32-byte key (SHA-256).
|
||||
/// (If your secret is already 32 bytes, this is still fine.)
|
||||
pub fn new<S: AsRef<[u8]>>(secret: S) -> Self {
|
||||
let mut h = Sha256::new();
|
||||
h.update(b"xchacha20poly1305-factory:v1"); // domain separation
|
||||
h.update(secret.as_ref());
|
||||
let digest = h.finalize(); // 32 bytes
|
||||
let key = Key::<XChaCha20Poly1305>::from_slice(&digest).to_owned();
|
||||
Self { key }
|
||||
}
|
||||
|
||||
/// Output layout: [version:1][nonce:24][ciphertext||tag]
|
||||
pub fn encrypt(&self, plaintext: &[u8]) -> Vec<u8> {
|
||||
let cipher = XChaCha20Poly1305::new(&self.key);
|
||||
|
||||
let mut nonce_bytes = [0u8; NONCE_LEN];
|
||||
OsRng.fill_bytes(&mut nonce_bytes);
|
||||
let nonce = XNonce::from_slice(&nonce_bytes);
|
||||
|
||||
let mut out = Vec::with_capacity(1 + NONCE_LEN + plaintext.len() + TAG_LEN);
|
||||
out.push(VERSION);
|
||||
out.extend_from_slice(&nonce_bytes);
|
||||
|
||||
let ct = cipher.encrypt(nonce, plaintext).expect("encrypt");
|
||||
out.extend_from_slice(&ct);
|
||||
out
|
||||
}
|
||||
|
||||
pub fn decrypt(&self, blob: &[u8]) -> Result<Vec<u8>, CryptoError> {
|
||||
if blob.len() < 1 + NONCE_LEN + TAG_LEN {
|
||||
return Err(CryptoError::Format);
|
||||
}
|
||||
let ver = blob[0];
|
||||
if ver != VERSION {
|
||||
return Err(CryptoError::Version(ver));
|
||||
}
|
||||
|
||||
let nonce = XNonce::from_slice(&blob[1..1 + NONCE_LEN]);
|
||||
let ct = &blob[1 + NONCE_LEN..];
|
||||
|
||||
let cipher = XChaCha20Poly1305::new(&self.key);
|
||||
cipher.decrypt(nonce, ct).map_err(|_| CryptoError::Decrypt)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Tiny usage example
|
||||
|
||||
```rust
|
||||
fn main() {
|
||||
let f = CryptoFactory::new(b"super-secret-key-material");
|
||||
let val = b"\x00\xFFbinary\x01\x02\x03";
|
||||
|
||||
let blob = f.encrypt(val);
|
||||
let roundtrip = f.decrypt(&blob).unwrap();
|
||||
|
||||
assert_eq!(roundtrip, val);
|
||||
}
|
||||
```
|
||||
|
||||
That’s it: `new(secret)`, `encrypt(bytes)`, `decrypt(bytes)`.
|
||||
You can stash the returned `blob` directly in your storage layer behind Redis.
|
||||
80
herodb/instructions/redb.md
Normal file
80
herodb/instructions/redb.md
Normal file
@ -0,0 +1,80 @@
|
||||
========================
|
||||
CODE SNIPPETS
|
||||
========================
|
||||
TITLE: 1PC+C Commit Strategy Vulnerability Example
|
||||
DESCRIPTION: Illustrates a scenario where a partially committed transaction might appear complete due to the non-cryptographic checksum (XXH3) used in the 1PC+C commit strategy. This requires controlling page flush order, introducing a crash during fsync, and ensuring valid checksums for partially written data.
|
||||
|
||||
SOURCE: https://github.com/cberner/redb/blob/master/docs/design.md#_snippet_9
|
||||
|
||||
LANGUAGE: rust
|
||||
CODE:
|
||||
```
|
||||
table.insert(malicious_key, malicious_value);
|
||||
table.insert(good_key, good_value);
|
||||
txn.commit();
|
||||
```
|
||||
|
||||
LANGUAGE: rust
|
||||
CODE:
|
||||
```
|
||||
table.insert(malicious_key, malicious_value);
|
||||
txn.commit();
|
||||
```
|
||||
|
||||
----------------------------------------
|
||||
|
||||
TITLE: Basic Key-Value Operations in redb
|
||||
DESCRIPTION: Demonstrates the fundamental usage of redb for creating a database, opening a table, inserting a key-value pair, and retrieving the value within separate read and write transactions.
|
||||
|
||||
SOURCE: https://github.com/cberner/redb/blob/master/README.md#_snippet_0
|
||||
|
||||
LANGUAGE: rust
|
||||
CODE:
|
||||
```
|
||||
use redb::{Database, Error, ReadableTable, TableDefinition};
|
||||
|
||||
const TABLE: TableDefinition<&str, u64> = TableDefinition::new("my_data");
|
||||
|
||||
fn main() -> Result<(), Error> {
|
||||
let db = Database::create("my_db.redb")?;
|
||||
let write_txn = db.begin_write()?;
|
||||
{
|
||||
let mut table = write_txn.open_table(TABLE)?;
|
||||
table.insert("my_key", &123)?;
|
||||
}
|
||||
write_txn.commit()?;
|
||||
|
||||
let read_txn = db.begin_read()?;
|
||||
let table = read_txn.open_table(TABLE)?;
|
||||
assert_eq!(table.get("my_key")?.unwrap().value(), 123);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
|
||||
## What *redb* currently supports:
|
||||
|
||||
* Simple operations like creating databases, inserting key-value pairs, opening and reading tables ([GitHub][1]).
|
||||
* No mention of operations such as:
|
||||
|
||||
* Iterating over keys with a given prefix.
|
||||
* Range queries based on string prefixes.
|
||||
* Specialized prefix‑filtered lookups.
|
||||
|
||||
|
||||
## implement range scans as follows
|
||||
|
||||
You can implement prefix-like functionality using **range scans** combined with manual checks, similar to using a `BTreeSet` in Rust:
|
||||
|
||||
```rust
|
||||
for key in table.range(prefix..).keys() {
|
||||
if !key.starts_with(prefix) {
|
||||
break;
|
||||
}
|
||||
// process key
|
||||
}
|
||||
```
|
||||
|
||||
This pattern iterates keys starting at the prefix, and stops once a key no longer matches the prefix—this works because the keys are sorted ([GitHub][1]).
|
||||
150
herodb/instructions/redis_basic_client.md
Normal file
150
herodb/instructions/redis_basic_client.md
Normal file
@ -0,0 +1,150 @@
|
||||
]
|
||||
# INFO
|
||||
|
||||
**What it does**
|
||||
Returns server stats in a human-readable text block, optionally filtered by sections. Typical sections: `server`, `clients`, `memory`, `persistence`, `stats`, `replication`, `cpu`, `commandstats`, `latencystats`, `cluster`, `modules`, `keyspace`, `errorstats`. Special args: `all`, `default`, `everything`. The reply is a **Bulk String** with `# <Section>` headers and `key:value` lines. ([Redis][1])
|
||||
|
||||
**Syntax**
|
||||
|
||||
```
|
||||
INFO [section [section ...]]
|
||||
```
|
||||
|
||||
**Return (RESP2/RESP3)**: Bulk String. ([Redis][1])
|
||||
|
||||
**RESP request/response**
|
||||
|
||||
```
|
||||
# Request: whole default set
|
||||
*1\r\n$4\r\nINFO\r\n
|
||||
|
||||
# Request: a specific section, e.g., clients
|
||||
*2\r\n$4\r\nINFO\r\n$7\r\nclients\r\n
|
||||
|
||||
# Response (prefix shown; body is long)
|
||||
$1234\r\n# Server\r\nredis_version:7.4.0\r\n...\r\n# Clients\r\nconnected_clients:3\r\n...\r\n
|
||||
```
|
||||
|
||||
(Reply type/format per RESP spec and the INFO page.) ([Redis][2])
|
||||
|
||||
---
|
||||
|
||||
# Connection “name” (there is **no** top-level `NAME` command)
|
||||
|
||||
Redis doesn’t have a standalone `NAME` command. Connection names are handled via `CLIENT SETNAME` and retrieved via `CLIENT GETNAME`. ([Redis][3])
|
||||
|
||||
## CLIENT SETNAME
|
||||
|
||||
Assigns a human label to the current connection (shown in `CLIENT LIST`, logs, etc.). No spaces allowed in the name; empty string clears it. Length is limited by Redis string limits (practically huge). **Reply**: Simple String `OK`. ([Redis][4])
|
||||
|
||||
**Syntax**
|
||||
|
||||
```
|
||||
CLIENT SETNAME connection-name
|
||||
```
|
||||
|
||||
**RESP**
|
||||
|
||||
```
|
||||
# Set the name "myapp"
|
||||
*3\r\n$6\r\nCLIENT\r\n$7\r\nSETNAME\r\n$5\r\nmyapp\r\n
|
||||
|
||||
# Reply
|
||||
+OK\r\n
|
||||
```
|
||||
|
||||
## CLIENT GETNAME
|
||||
|
||||
Returns the current connection’s name or **Null Bulk String** if unset. ([Redis][5])
|
||||
|
||||
**Syntax**
|
||||
|
||||
```
|
||||
CLIENT GETNAME
|
||||
```
|
||||
|
||||
**RESP**
|
||||
|
||||
```
|
||||
# Before SETNAME:
|
||||
*2\r\n$6\r\nCLIENT\r\n$7\r\nGETNAME\r\n
|
||||
$-1\r\n # nil (no name)
|
||||
|
||||
# After SETNAME myapp:
|
||||
*2\r\n$6\r\nCLIENT\r\n$7\r\nGETNAME\r\n
|
||||
$5\r\nmyapp\r\n
|
||||
```
|
||||
|
||||
(Null/Bulk String encoding per RESP spec.) ([Redis][2])
|
||||
|
||||
---
|
||||
|
||||
# CLIENT (container command + key subcommands)
|
||||
|
||||
`CLIENT` is a **container**; use subcommands like `CLIENT LIST`, `CLIENT INFO`, `CLIENT ID`, `CLIENT KILL`, `CLIENT TRACKING`, etc. Call `CLIENT HELP` to enumerate them. ([Redis][3])
|
||||
|
||||
## CLIENT LIST
|
||||
|
||||
Shows all connections as a single **Bulk String**: one line per client with `field=value` pairs (includes `id`, `addr`, `name`, `db`, `user`, `resp`, and more). Filters: `TYPE` and `ID`. **Return**: Bulk String (RESP2/RESP3). ([Redis][6])
|
||||
|
||||
**Syntax**
|
||||
|
||||
```
|
||||
CLIENT LIST [TYPE <NORMAL|MASTER|REPLICA|PUBSUB>] [ID client-id ...]
|
||||
```
|
||||
|
||||
**RESP**
|
||||
|
||||
```
|
||||
*2\r\n$6\r\nCLIENT\r\n$4\r\nLIST\r\n
|
||||
|
||||
# Reply (single Bulk String; example with one line shown)
|
||||
$188\r\nid=7 addr=127.0.0.1:60840 laddr=127.0.0.1:6379 fd=8 name=myapp age=12 idle=3 flags=N db=0 ...\r\n
|
||||
```
|
||||
|
||||
## CLIENT INFO
|
||||
|
||||
Returns info for **this** connection only (same format/fields as a single line of `CLIENT LIST`). **Return**: Bulk String. Available since 6.2.0. ([Redis][7])
|
||||
|
||||
**Syntax**
|
||||
|
||||
```
|
||||
CLIENT INFO
|
||||
```
|
||||
|
||||
**RESP**
|
||||
|
||||
```
|
||||
*2\r\n$6\r\nCLIENT\r\n$4\r\nINFO\r\n
|
||||
|
||||
$160\r\nid=7 addr=127.0.0.1:60840 laddr=127.0.0.1:6379 fd=8 name=myapp db=0 user=default resp=2 ...\r\n
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
# RESP notes you’ll need for your parser
|
||||
|
||||
* **Requests** are Arrays: `*N\r\n` followed by `N` Bulk Strings for verb/args.
|
||||
* **Common replies here**: Simple String (`+OK\r\n`), Bulk String (`$<len>\r\n...\r\n`), and **Null Bulk String** (`$-1\r\n`). (These cover `INFO`, `CLIENT LIST/INFO`, `CLIENT GETNAME`, `CLIENT SETNAME`.) ([Redis][2])
|
||||
|
||||
---
|
||||
|
||||
## Sources (checked)
|
||||
|
||||
* INFO command (syntax, sections, behavior). ([Redis][1])
|
||||
* RESP spec (request/response framing, Bulk/Null Bulk Strings). ([Redis][2])
|
||||
* CLIENT container + subcommands index. ([Redis][3])
|
||||
* CLIENT LIST (fields, bulk-string return, filters). ([Redis][6])
|
||||
* CLIENT INFO (exists since 6.2, reply format). ([Redis][7])
|
||||
* CLIENT SETNAME (no spaces; clears with empty string; huge length OK). ([Redis][4])
|
||||
* CLIENT GETNAME (nil if unset). ([Redis][5])
|
||||
|
||||
If you want, I can fold this into a tiny Rust “command + RESP” test harness that exercises `INFO`, `CLIENT SETNAME/GETNAME`, `CLIENT LIST`, and `CLIENT INFO` against your in-mem RESP parser.
|
||||
|
||||
[1]: https://redis.io/docs/latest/commands/info/ "INFO | Docs"
|
||||
[2]: https://redis.io/docs/latest/develop/reference/protocol-spec/?utm_source=chatgpt.com "Redis serialization protocol specification | Docs"
|
||||
[3]: https://redis.io/docs/latest/commands/client/ "CLIENT | Docs"
|
||||
[4]: https://redis.io/docs/latest/commands/client-setname/?utm_source=chatgpt.com "CLIENT SETNAME | Docs"
|
||||
[5]: https://redis.io/docs/latest/commands/client-getname/?utm_source=chatgpt.com "CLIENT GETNAME | Docs"
|
||||
[6]: https://redis.io/docs/latest/commands/client-list/ "CLIENT LIST | Docs"
|
||||
[7]: https://redis.io/docs/latest/commands/client-info/?utm_source=chatgpt.com "CLIENT INFO | Docs"
|
||||
251
herodb/instructions/redis_basics.md
Normal file
251
herodb/instructions/redis_basics.md
Normal file
@ -0,0 +1,251 @@
|
||||
Got it 👍 — let’s break this down properly.
|
||||
|
||||
Redis has two broad classes you’re asking about:
|
||||
|
||||
1. **Basic key-space functions** (SET, GET, DEL, EXISTS, etc.)
|
||||
2. **Iteration commands** (`SCAN`, `SSCAN`, `HSCAN`, `ZSCAN`)
|
||||
|
||||
And for each I’ll show:
|
||||
|
||||
* What it does
|
||||
* How it works at a high level
|
||||
* Its **RESP protocol implementation** (the actual wire format).
|
||||
|
||||
---
|
||||
|
||||
# 1. Basic Key-Space Commands
|
||||
|
||||
### `SET key value`
|
||||
|
||||
* Stores a string value at a key.
|
||||
* Overwrites if the key already exists.
|
||||
|
||||
**Protocol (RESP2):**
|
||||
|
||||
```
|
||||
*3
|
||||
$3
|
||||
SET
|
||||
$3
|
||||
foo
|
||||
$3
|
||||
bar
|
||||
```
|
||||
|
||||
(client sends: array of 3 bulk strings: `["SET", "foo", "bar"]`)
|
||||
|
||||
**Reply:**
|
||||
|
||||
```
|
||||
+OK
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `GET key`
|
||||
|
||||
* Retrieves the string value stored at the key.
|
||||
* Returns `nil` if key doesn’t exist.
|
||||
|
||||
**Protocol:**
|
||||
|
||||
```
|
||||
*2
|
||||
$3
|
||||
GET
|
||||
$3
|
||||
foo
|
||||
```
|
||||
|
||||
**Reply:**
|
||||
|
||||
```
|
||||
$3
|
||||
bar
|
||||
```
|
||||
|
||||
(or `$-1` for nil)
|
||||
|
||||
---
|
||||
|
||||
### `DEL key [key ...]`
|
||||
|
||||
* Removes one or more keys.
|
||||
* Returns number of keys actually removed.
|
||||
|
||||
**Protocol:**
|
||||
|
||||
```
|
||||
*2
|
||||
$3
|
||||
DEL
|
||||
$3
|
||||
foo
|
||||
```
|
||||
|
||||
**Reply:**
|
||||
|
||||
```
|
||||
:1
|
||||
```
|
||||
|
||||
(integer reply = number of deleted keys)
|
||||
|
||||
---
|
||||
|
||||
### `EXISTS key [key ...]`
|
||||
|
||||
* Checks if one or more keys exist.
|
||||
* Returns count of existing keys.
|
||||
|
||||
**Protocol:**
|
||||
|
||||
```
|
||||
*2
|
||||
$6
|
||||
EXISTS
|
||||
$3
|
||||
foo
|
||||
```
|
||||
|
||||
**Reply:**
|
||||
|
||||
```
|
||||
:1
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### `KEYS pattern`
|
||||
|
||||
* Returns all keys matching a glob-style pattern.
|
||||
⚠️ Not efficient in production (O(N)), better to use `SCAN`.
|
||||
|
||||
**Protocol:**
|
||||
|
||||
```
|
||||
*2
|
||||
$4
|
||||
KEYS
|
||||
$1
|
||||
*
|
||||
```
|
||||
|
||||
**Reply:**
|
||||
|
||||
```
|
||||
*2
|
||||
$3
|
||||
foo
|
||||
$3
|
||||
bar
|
||||
```
|
||||
|
||||
(array of bulk strings with key names)
|
||||
|
||||
---
|
||||
|
||||
# 2. Iteration Commands (`SCAN` family)
|
||||
|
||||
### `SCAN cursor [MATCH pattern] [COUNT n]`
|
||||
|
||||
* Iterates the keyspace incrementally.
|
||||
* Client keeps sending back the cursor from previous call until it returns `0`.
|
||||
|
||||
**Protocol example:**
|
||||
|
||||
```
|
||||
*2
|
||||
$4
|
||||
SCAN
|
||||
$1
|
||||
0
|
||||
```
|
||||
|
||||
**Reply:**
|
||||
|
||||
```
|
||||
*2
|
||||
$1
|
||||
0
|
||||
*2
|
||||
$3
|
||||
foo
|
||||
$3
|
||||
bar
|
||||
```
|
||||
|
||||
Explanation:
|
||||
|
||||
* First element = new cursor (`"0"` means iteration finished).
|
||||
* Second element = array of keys returned in this batch.
|
||||
|
||||
---
|
||||
|
||||
### `HSCAN key cursor [MATCH pattern] [COUNT n]`
|
||||
|
||||
* Like `SCAN`, but iterates fields of a hash.
|
||||
|
||||
**Protocol:**
|
||||
|
||||
```
|
||||
*3
|
||||
$5
|
||||
HSCAN
|
||||
$3
|
||||
myh
|
||||
$1
|
||||
0
|
||||
```
|
||||
|
||||
**Reply:**
|
||||
|
||||
```
|
||||
*2
|
||||
$1
|
||||
0
|
||||
*4
|
||||
$5
|
||||
field
|
||||
$5
|
||||
value
|
||||
$5
|
||||
age
|
||||
$2
|
||||
42
|
||||
```
|
||||
|
||||
(Array of alternating field/value pairs)
|
||||
|
||||
---
|
||||
|
||||
### `SSCAN key cursor [MATCH pattern] [COUNT n]`
|
||||
|
||||
* Iterates members of a set.
|
||||
|
||||
Protocol and reply structure same as SCAN.
|
||||
|
||||
---
|
||||
|
||||
### `ZSCAN key cursor [MATCH pattern] [COUNT n]`
|
||||
|
||||
* Iterates members of a sorted set with scores.
|
||||
* Returns alternating `member`, `score`.
|
||||
|
||||
---
|
||||
|
||||
# Quick Comparison
|
||||
|
||||
| Command | Purpose | Return Type |
|
||||
| -------- | ----------------------------- | --------------------- |
|
||||
| `SET` | Store a string value | Simple string `+OK` |
|
||||
| `GET` | Retrieve a string value | Bulk string / nil |
|
||||
| `DEL` | Delete keys | Integer (count) |
|
||||
| `EXISTS` | Check existence | Integer (count) |
|
||||
| `KEYS` | List all matching keys (slow) | Array of bulk strings |
|
||||
| `SCAN` | Iterate over keys (safe) | `[cursor, array]` |
|
||||
| `HSCAN` | Iterate over hash fields | `[cursor, array]` |
|
||||
| `SSCAN` | Iterate over set members | `[cursor, array]` |
|
||||
| `ZSCAN` | Iterate over sorted set | `[cursor, array]` |
|
||||
|
||||
##
|
||||
307
herodb/instructions/redis_hset_functions.md
Normal file
307
herodb/instructions/redis_hset_functions.md
Normal file
@ -0,0 +1,307 @@
|
||||
|
||||
# 🔑 Redis `HSET` and Related Hash Commands
|
||||
|
||||
## 1. `HSET`
|
||||
|
||||
* **Purpose**: Set the value of one or more fields in a hash.
|
||||
* **Syntax**:
|
||||
|
||||
```bash
|
||||
HSET key field value [field value ...]
|
||||
```
|
||||
* **Return**:
|
||||
|
||||
* Integer: number of fields that were newly added.
|
||||
* **RESP Protocol**:
|
||||
|
||||
```
|
||||
*4
|
||||
$4
|
||||
HSET
|
||||
$3
|
||||
key
|
||||
$5
|
||||
field
|
||||
$5
|
||||
value
|
||||
```
|
||||
|
||||
(If multiple field-value pairs: `*6`, `*8`, etc.)
|
||||
|
||||
---
|
||||
|
||||
## 2. `HSETNX`
|
||||
|
||||
* **Purpose**: Set the value of a hash field only if it does **not** exist.
|
||||
* **Syntax**:
|
||||
|
||||
```bash
|
||||
HSETNX key field value
|
||||
```
|
||||
* **Return**:
|
||||
|
||||
* `1` if field was set.
|
||||
* `0` if field already exists.
|
||||
* **RESP Protocol**:
|
||||
|
||||
```
|
||||
*4
|
||||
$6
|
||||
HSETNX
|
||||
$3
|
||||
key
|
||||
$5
|
||||
field
|
||||
$5
|
||||
value
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. `HGET`
|
||||
|
||||
* **Purpose**: Get the value of a hash field.
|
||||
* **Syntax**:
|
||||
|
||||
```bash
|
||||
HGET key field
|
||||
```
|
||||
* **Return**:
|
||||
|
||||
* Bulk string (value) or `nil` if field does not exist.
|
||||
* **RESP Protocol**:
|
||||
|
||||
```
|
||||
*3
|
||||
$4
|
||||
HGET
|
||||
$3
|
||||
key
|
||||
$5
|
||||
field
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. `HGETALL`
|
||||
|
||||
* **Purpose**: Get all fields and values in a hash.
|
||||
* **Syntax**:
|
||||
|
||||
```bash
|
||||
HGETALL key
|
||||
```
|
||||
* **Return**:
|
||||
|
||||
* Array of `[field1, value1, field2, value2, ...]`.
|
||||
* **RESP Protocol**:
|
||||
|
||||
```
|
||||
*2
|
||||
$7
|
||||
HGETALL
|
||||
$3
|
||||
key
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. `HMSET` (⚠️ Deprecated, use `HSET`)
|
||||
|
||||
* **Purpose**: Set multiple field-value pairs.
|
||||
* **Syntax**:
|
||||
|
||||
```bash
|
||||
HMSET key field value [field value ...]
|
||||
```
|
||||
* **Return**:
|
||||
|
||||
* Always `OK`.
|
||||
* **RESP Protocol**:
|
||||
|
||||
```
|
||||
*6
|
||||
$5
|
||||
HMSET
|
||||
$3
|
||||
key
|
||||
$5
|
||||
field
|
||||
$5
|
||||
value
|
||||
$5
|
||||
field2
|
||||
$5
|
||||
value2
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. `HMGET`
|
||||
|
||||
* **Purpose**: Get values of multiple fields.
|
||||
* **Syntax**:
|
||||
|
||||
```bash
|
||||
HMGET key field [field ...]
|
||||
```
|
||||
* **Return**:
|
||||
|
||||
* Array of values (bulk strings or nils).
|
||||
* **RESP Protocol**:
|
||||
|
||||
```
|
||||
*4
|
||||
$5
|
||||
HMGET
|
||||
$3
|
||||
key
|
||||
$5
|
||||
field1
|
||||
$5
|
||||
field2
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. `HDEL`
|
||||
|
||||
* **Purpose**: Delete one or more fields from a hash.
|
||||
* **Syntax**:
|
||||
|
||||
```bash
|
||||
HDEL key field [field ...]
|
||||
```
|
||||
* **Return**:
|
||||
|
||||
* Integer: number of fields removed.
|
||||
* **RESP Protocol**:
|
||||
|
||||
```
|
||||
*3
|
||||
$4
|
||||
HDEL
|
||||
$3
|
||||
key
|
||||
$5
|
||||
field
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 8. `HEXISTS`
|
||||
|
||||
* **Purpose**: Check if a field exists.
|
||||
* **Syntax**:
|
||||
|
||||
```bash
|
||||
HEXISTS key field
|
||||
```
|
||||
* **Return**:
|
||||
|
||||
* `1` if exists, `0` if not.
|
||||
* **RESP Protocol**:
|
||||
|
||||
```
|
||||
*3
|
||||
$7
|
||||
HEXISTS
|
||||
$3
|
||||
key
|
||||
$5
|
||||
field
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 9. `HKEYS`
|
||||
|
||||
* **Purpose**: Get all field names in a hash.
|
||||
* **Syntax**:
|
||||
|
||||
```bash
|
||||
HKEYS key
|
||||
```
|
||||
* **Return**:
|
||||
|
||||
* Array of field names.
|
||||
* **RESP Protocol**:
|
||||
|
||||
```
|
||||
*2
|
||||
$5
|
||||
HKEYS
|
||||
$3
|
||||
key
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 10. `HVALS`
|
||||
|
||||
* **Purpose**: Get all values in a hash.
|
||||
* **Syntax**:
|
||||
|
||||
```bash
|
||||
HVALS key
|
||||
```
|
||||
* **Return**:
|
||||
|
||||
* Array of values.
|
||||
* **RESP Protocol**:
|
||||
|
||||
```
|
||||
*2
|
||||
$5
|
||||
HVALS
|
||||
$3
|
||||
key
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 11. `HLEN`
|
||||
|
||||
* **Purpose**: Get number of fields in a hash.
|
||||
* **Syntax**:
|
||||
|
||||
```bash
|
||||
HLEN key
|
||||
```
|
||||
* **Return**:
|
||||
|
||||
* Integer: number of fields.
|
||||
* **RESP Protocol**:
|
||||
|
||||
```
|
||||
*2
|
||||
$4
|
||||
HLEN
|
||||
$3
|
||||
key
|
||||
```
|
||||
|
||||
|
||||
|
||||
## 12. `HSCAN`
|
||||
|
||||
* **Purpose**: Iterate fields/values of a hash (cursor-based scan).
|
||||
* **Syntax**:
|
||||
|
||||
```bash
|
||||
HSCAN key cursor [MATCH pattern] [COUNT count]
|
||||
```
|
||||
* **Return**:
|
||||
|
||||
* Array: `[new-cursor, [field1, value1, ...]]`
|
||||
* **RESP Protocol**:
|
||||
|
||||
```
|
||||
*3
|
||||
$5
|
||||
HSCAN
|
||||
$3
|
||||
key
|
||||
$1
|
||||
0
|
||||
```
|
||||
259
herodb/instructions/redis_lists.md
Normal file
259
herodb/instructions/redis_lists.md
Normal file
@ -0,0 +1,259 @@
|
||||
|
||||
# 1) Data model & basics
|
||||
|
||||
* A **queue** is a List at key `queue:<name>`.
|
||||
* Common patterns:
|
||||
|
||||
* **Producer**: `LPUSH queue item` (or `RPUSH`)
|
||||
* **Consumer (non-blocking)**: `RPOP queue` (or `LPOP`)
|
||||
* **Consumer (blocking)**: `BRPOP queue timeout` (or `BLPOP`)
|
||||
* If a key doesn’t exist, it’s treated as an **empty list**; push **creates** the list; when the **last element is popped, the key is deleted**. ([Redis][1])
|
||||
|
||||
---
|
||||
|
||||
# 2) Commands to implement (queues via Lists)
|
||||
|
||||
## LPUSH / RPUSH
|
||||
|
||||
Prepend/append one or more elements. Create the list if it doesn’t exist.
|
||||
**Return**: Integer = new length of the list.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```
|
||||
LPUSH key element [element ...]
|
||||
RPUSH key element [element ...]
|
||||
```
|
||||
|
||||
**RESP (example)**
|
||||
|
||||
```
|
||||
*3\r\n$5\r\nLPUSH\r\n$5\r\nqueue\r\n$5\r\njob-1\r\n
|
||||
:1\r\n
|
||||
```
|
||||
|
||||
Refs: semantics & multi-arg ordering. ([Redis][1])
|
||||
|
||||
### LPUSHX / RPUSHX (optional but useful)
|
||||
|
||||
Like LPUSH/RPUSH, **but only if the list exists**.
|
||||
**Return**: Integer = new length (0 if key didn’t exist).
|
||||
|
||||
```
|
||||
LPUSHX key element [element ...]
|
||||
RPUSHX key element [element ...]
|
||||
```
|
||||
|
||||
Refs: command index. ([Redis][2])
|
||||
|
||||
---
|
||||
|
||||
## LPOP / RPOP
|
||||
|
||||
Remove & return one (default) or **up to COUNT** elements since Redis 6.2.
|
||||
If the list is empty or missing, **Null** is returned (Null Bulk or Null Array if COUNT>1).
|
||||
**Return**:
|
||||
|
||||
* No COUNT: Bulk String or Null Bulk.
|
||||
* With COUNT: Array of Bulk Strings (possibly empty) or Null Array if key missing.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```
|
||||
LPOP key [count]
|
||||
RPOP key [count]
|
||||
```
|
||||
|
||||
**RESP (no COUNT)**
|
||||
|
||||
```
|
||||
*2\r\n$4\r\nRPOP\r\n$5\r\nqueue\r\n
|
||||
$5\r\njob-1\r\n # or $-1\r\n if empty
|
||||
```
|
||||
|
||||
**RESP (COUNT=2)**
|
||||
|
||||
```
|
||||
*3\r\n$4\r\nLPOP\r\n$5\r\nqueue\r\n$1\r\n2\r\n
|
||||
*2\r\n$5\r\njob-2\r\n$5\r\njob-3\r\n # or *-1\r\n if key missing
|
||||
```
|
||||
|
||||
Refs: LPOP w/ COUNT; general pop semantics. ([Redis][3])
|
||||
|
||||
---
|
||||
|
||||
## BLPOP / BRPOP (blocking consumers)
|
||||
|
||||
Block until an element is available in any of the given lists or until `timeout` (seconds, **double**, `0` = forever).
|
||||
**Return** on success: **Array \[key, element]**.
|
||||
**Return** on timeout: **Null Array**.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```
|
||||
BLPOP key [key ...] timeout
|
||||
BRPOP key [key ...] timeout
|
||||
```
|
||||
|
||||
**RESP**
|
||||
|
||||
```
|
||||
*3\r\n$5\r\nBRPOP\r\n$5\r\nqueue\r\n$1\r\n0\r\n # block forever
|
||||
|
||||
# Success reply
|
||||
*2\r\n$5\r\nqueue\r\n$5\r\njob-4\r\n
|
||||
|
||||
# Timeout reply
|
||||
*-1\r\n
|
||||
```
|
||||
|
||||
**Implementation notes**
|
||||
|
||||
* If any listed key is non-empty at call time, reply **immediately** from the first non-empty key **by the command’s key order**.
|
||||
* Otherwise, put the client into a **blocked state** (register per-key waiters). On any `LPUSH/RPUSH` to those keys, **wake the earliest waiter** and serve it atomically.
|
||||
* If timeout expires, return **Null Array** and clear the blocked state.
|
||||
Refs: timeout semantics and return shape. ([Redis][4])
|
||||
|
||||
---
|
||||
|
||||
## LMOVE / BLMOVE (atomic move; replaces RPOPLPUSH/BRPOPLPUSH)
|
||||
|
||||
Atomically **pop from one side** of `source` and **push to one side** of `destination`.
|
||||
|
||||
* Use for **reliable queues** (move to a *processing* list).
|
||||
* `BLMOVE` blocks like `BLPOP` when `source` is empty.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```
|
||||
LMOVE source destination LEFT|RIGHT LEFT|RIGHT
|
||||
BLMOVE source destination LEFT|RIGHT LEFT|RIGHT timeout
|
||||
```
|
||||
|
||||
**Return**: Bulk String element moved, or Null if `source` empty (LMOVE); `BLMOVE` blocks/Null on timeout.
|
||||
|
||||
**RESP (LMOVE RIGHT->LEFT)**
|
||||
|
||||
```
|
||||
*5\r\n$5\r\nLMOVE\r\n$6\r\nsource\r\n$3\r\ndst\r\n$5\r\nRIGHT\r\n$4\r\nLEFT\r\n
|
||||
$5\r\njob-5\r\n
|
||||
```
|
||||
|
||||
**Notes**
|
||||
|
||||
* Prefer `LMOVE/BLMOVE` over deprecated `RPOPLPUSH/BRPOPLPUSH`.
|
||||
* Pattern: consumer `LMOVE queue processing RIGHT LEFT` → work → `LREM processing 1 <elem>` to ACK; a reaper can requeue stale items.
|
||||
Refs: LMOVE/BLMOVE behavior and reliable-queue pattern; deprecation of RPOPLPUSH. ([Redis][5])
|
||||
|
||||
*(Compat: you can still implement `RPOPLPUSH source dest` and `BRPOPLPUSH source dest timeout`, but mark them deprecated and map to LMOVE/BLMOVE.)* ([Redis][6])
|
||||
|
||||
---
|
||||
|
||||
## LLEN (length)
|
||||
|
||||
Useful for metrics/backpressure.
|
||||
|
||||
```
|
||||
LLEN key
|
||||
```
|
||||
|
||||
**RESP**
|
||||
|
||||
```
|
||||
*2\r\n$4\r\nLLEN\r\n$5\r\nqueue\r\n
|
||||
:3\r\n
|
||||
```
|
||||
|
||||
Refs: list overview mentioning LLEN. ([Redis][7])
|
||||
|
||||
---
|
||||
|
||||
## LREM (ack for “reliable” processing)
|
||||
|
||||
Remove occurrences of `element` from the list (head→tail scan).
|
||||
Use `count=1` to ACK a single processed item from `processing`.
|
||||
|
||||
```
|
||||
LREM key count element
|
||||
```
|
||||
|
||||
**RESP**
|
||||
|
||||
```
|
||||
*4\r\n$4\r\nLREM\r\n$9\r\nprocessing\r\n$1\r\n1\r\n$5\r\njob-5\r\n
|
||||
:1\r\n
|
||||
```
|
||||
|
||||
Refs: reliable pattern mentions LREM to ACK. ([Redis][5])
|
||||
|
||||
---
|
||||
|
||||
## LTRIM (bounded queues / retention)
|
||||
|
||||
Keep only `[start, stop]` range; everything else is dropped.
|
||||
Use to cap queue length after pushes.
|
||||
|
||||
```
|
||||
LTRIM key start stop
|
||||
```
|
||||
|
||||
**RESP**
|
||||
|
||||
```
|
||||
*4\r\n$5\r\nLTRIM\r\n$5\r\nqueue\r\n$2\r\n0\r\n$3\r\n999\r\n
|
||||
+OK\r\n
|
||||
```
|
||||
|
||||
Refs: list overview includes LTRIM for retention. ([Redis][7])
|
||||
|
||||
---
|
||||
|
||||
## LRANGE / LINDEX (debugging / peeking)
|
||||
|
||||
* `LRANGE key start stop` → Array of elements (non-destructive).
|
||||
* `LINDEX key index` → one element or Null.
|
||||
|
||||
These aren’t required for queue semantics, but handy. ([Redis][7])
|
||||
|
||||
---
|
||||
|
||||
# 3) Errors & types
|
||||
|
||||
* Wrong type: `-WRONGTYPE Operation against a key holding the wrong kind of value\r\n`
|
||||
* Non-existing key:
|
||||
|
||||
* Push: creates the list (returns new length).
|
||||
* Pop (non-blocking): returns **Null**.
|
||||
* Blocking pop: **Null Array** on timeout. ([Redis][1])
|
||||
|
||||
---
|
||||
|
||||
# 4) Blocking engine (implementation sketch)
|
||||
|
||||
1. **Call time**: scan keys in user order. If a non-empty list is found, pop & reply immediately.
|
||||
2. **Otherwise**: register the client as **blocked** on those keys with `deadline = now + timeout` (or infinite).
|
||||
3. **On push to any key**: if waiters exist, **wake one** (FIFO) and serve its pop **atomically** with the push result.
|
||||
4. **On timer**: for each blocked client whose deadline passed, reply `Null Array` and clear state.
|
||||
5. **Connection close**: remove from any wait queues.
|
||||
|
||||
Refs for timeout/block semantics. ([Redis][4])
|
||||
|
||||
---
|
||||
|
||||
# 5) Reliable queue pattern (recommended)
|
||||
|
||||
* **Consume**: `LMOVE queue processing RIGHT LEFT` (or `BLMOVE ... 0`).
|
||||
* **Process** the job.
|
||||
* **ACK**: `LREM processing 1 <job>` when done.
|
||||
* **Reaper**: auxiliary task that detects stale jobs (e.g., track job IDs + timestamps in a ZSET) and requeues them. (Lists don’t include timestamps; pairing with a ZSET is standard practice.)
|
||||
Refs: LMOVE doc’s pattern. ([Redis][5])
|
||||
|
||||
---
|
||||
|
||||
# 6) Minimal test matrix
|
||||
|
||||
* Push/pop happy path (both ends), with/without COUNT.
|
||||
* Blocking pop: immediate availability, block + timeout, wake on push, multiple keys order, FIFO across multiple waiters.
|
||||
* LMOVE/BLMOVE: RIGHT→LEFT pipeline, block + wake, cross-list atomicity, ACK via LREM.
|
||||
* Type errors and key deletion on last pop.
|
||||
|
||||
@ -12,7 +12,6 @@ echo ""
|
||||
echo "2️⃣ Running Comprehensive Redis Integration Tests (13 tests)..."
|
||||
echo "----------------------------------------------------------------"
|
||||
cargo test -p herodb --test redis_integration_tests -- --nocapture
|
||||
cargo test -p herodb --test redis_basic_client -- --nocapture
|
||||
cargo test -p herodb --test debug_hset -- --nocapture
|
||||
cargo test -p herodb --test debug_hset_simple -- --nocapture
|
||||
|
||||
@ -10,164 +10,146 @@
|
||||
// age:signpriv:{name} -> Ed25519 signing secret key (private, used to sign)
|
||||
// - Base64 wrapping for ciphertext/signature binary blobs.
|
||||
|
||||
use std::str::FromStr;
|
||||
|
||||
use secrecy::ExposeSecret;
|
||||
use age::{Decryptor, Encryptor};
|
||||
use age::x25519;
|
||||
|
||||
use ed25519_dalek::{Signature, Signer, Verifier, SigningKey, VerifyingKey};
|
||||
|
||||
use base64::{engine::general_purpose::STANDARD as B64, Engine as _};
|
||||
|
||||
use crate::protocol::Protocol;
|
||||
use crate::server::Server;
|
||||
use libdbstorage::DBError;
|
||||
use libcryptoa::AsymmetricCryptoError;
|
||||
use crate::error::DBError;
|
||||
|
||||
// ---------- Storage helpers ----------
|
||||
// ---------- Internal helpers ----------
|
||||
|
||||
fn sget(server: &Server, key: &str) -> Result<Option<String>, DBError> {
|
||||
let st = server.current_storage()?;
|
||||
st.get(key)
|
||||
}
|
||||
fn sset(server: &Server, key: &str, val: &str) -> Result<(), DBError> {
|
||||
let st = server.current_storage()?;
|
||||
st.set(key.to_string(), val.to_string())
|
||||
#[derive(Debug)]
|
||||
pub enum AgeWireError {
|
||||
ParseKey,
|
||||
Crypto(String),
|
||||
Utf8,
|
||||
SignatureLen,
|
||||
NotFound(&'static str), // which kind of key was missing
|
||||
Storage(String),
|
||||
}
|
||||
|
||||
fn enc_pub_key_key(name: &str) -> String { format!("age:key:{name}") }
|
||||
fn enc_priv_key_key(name: &str) -> String { format!("age:privkey:{name}") }
|
||||
fn sign_pub_key_key(name: &str) -> String { format!("age:signpub:{name}") }
|
||||
fn sign_priv_key_key(name: &str) -> String { format!("age:signpriv:{name}") }
|
||||
|
||||
// ---------- Command handlers (RESP Protocol) ----------
|
||||
// Basic (stateless) ones kept for completeness
|
||||
|
||||
pub async fn cmd_age_genenc() -> Protocol {
|
||||
let (recip, ident) = libcryptoa::gen_enc_keypair();
|
||||
Protocol::Array(vec![Protocol::BulkString(recip), Protocol::BulkString(ident)])
|
||||
impl AgeWireError {
|
||||
fn to_protocol(self) -> Protocol {
|
||||
match self {
|
||||
AgeWireError::ParseKey => Protocol::err("ERR age: invalid key"),
|
||||
AgeWireError::Crypto(e) => Protocol::err(&format!("ERR age: {e}")),
|
||||
AgeWireError::Utf8 => Protocol::err("ERR age: invalid UTF-8 plaintext"),
|
||||
AgeWireError::SignatureLen => Protocol::err("ERR age: bad signature length"),
|
||||
AgeWireError::NotFound(w) => Protocol::err(&format!("ERR age: missing {w}")),
|
||||
AgeWireError::Storage(e) => Protocol::err(&format!("ERR storage: {e}")),
|
||||
}
|
||||
|
||||
pub async fn cmd_age_gensign() -> Protocol {
|
||||
let (verify, secret) = libcryptoa::gen_sign_keypair();
|
||||
Protocol::Array(vec![Protocol::BulkString(verify), Protocol::BulkString(secret)])
|
||||
}
|
||||
|
||||
pub async fn cmd_age_encrypt(recipient: &str, message: &str) -> Protocol {
|
||||
match libcryptoa::encrypt_b64(recipient, message) {
|
||||
Ok(b64) => Protocol::BulkString(b64),
|
||||
Err(e) => Protocol::err(&format!("ERR age: {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn cmd_age_decrypt(identity: &str, ct_b64: &str) -> Protocol {
|
||||
match libcryptoa::decrypt_b64(identity, ct_b64) {
|
||||
Ok(pt) => Protocol::BulkString(pt),
|
||||
Err(e) => Protocol::err(&format!("ERR age: {e}")),
|
||||
fn parse_recipient(s: &str) -> Result<x25519::Recipient, AgeWireError> {
|
||||
x25519::Recipient::from_str(s).map_err(|_| AgeWireError::ParseKey)
|
||||
}
|
||||
fn parse_identity(s: &str) -> Result<x25519::Identity, AgeWireError> {
|
||||
x25519::Identity::from_str(s).map_err(|_| AgeWireError::ParseKey)
|
||||
}
|
||||
fn parse_ed25519_signing_key(s: &str) -> Result<SigningKey, AgeWireError> {
|
||||
// Parse base64-encoded signing key
|
||||
let bytes = B64.decode(s).map_err(|_| AgeWireError::ParseKey)?;
|
||||
if bytes.len() != 32 {
|
||||
return Err(AgeWireError::ParseKey);
|
||||
}
|
||||
let key_bytes: [u8; 32] = bytes.try_into().map_err(|_| AgeWireError::ParseKey)?;
|
||||
Ok(SigningKey::from_bytes(&key_bytes))
|
||||
}
|
||||
fn parse_ed25519_verifying_key(s: &str) -> Result<VerifyingKey, AgeWireError> {
|
||||
// Parse base64-encoded verifying key
|
||||
let bytes = B64.decode(s).map_err(|_| AgeWireError::ParseKey)?;
|
||||
if bytes.len() != 32 {
|
||||
return Err(AgeWireError::ParseKey);
|
||||
}
|
||||
let key_bytes: [u8; 32] = bytes.try_into().map_err(|_| AgeWireError::ParseKey)?;
|
||||
VerifyingKey::from_bytes(&key_bytes).map_err(|_| AgeWireError::ParseKey)
|
||||
}
|
||||
|
||||
pub async fn cmd_age_sign(secret: &str, message: &str) -> Protocol {
|
||||
match libcryptoa::sign_b64(secret, message) {
|
||||
Ok(b64sig) => Protocol::BulkString(b64sig),
|
||||
Err(e) => Protocol::err(&format!("ERR age: {e}")),
|
||||
}
|
||||
// ---------- Stateless crypto helpers (string in/out) ----------
|
||||
|
||||
pub fn gen_enc_keypair() -> (String, String) {
|
||||
let id = x25519::Identity::generate();
|
||||
let pk = id.to_public();
|
||||
(pk.to_string(), id.to_string().expose_secret().to_string()) // (recipient, identity)
|
||||
}
|
||||
|
||||
pub async fn cmd_age_verify(verify_pub: &str, message: &str, sig_b64: &str) -> Protocol {
|
||||
match libcryptoa::verify_b64(verify_pub, message, sig_b64) {
|
||||
Ok(true) => Protocol::SimpleString("1".to_string()),
|
||||
Ok(false) => Protocol::SimpleString("0".to_string()),
|
||||
Err(e) => Protocol::err(&format!("ERR age: {e}")),
|
||||
}
|
||||
pub fn gen_sign_keypair() -> (String, String) {
|
||||
use rand::RngCore;
|
||||
use rand::rngs::OsRng;
|
||||
|
||||
// Generate random 32 bytes for the signing key
|
||||
let mut secret_bytes = [0u8; 32];
|
||||
OsRng.fill_bytes(&mut secret_bytes);
|
||||
|
||||
let signing_key = SigningKey::from_bytes(&secret_bytes);
|
||||
let verifying_key = signing_key.verifying_key();
|
||||
|
||||
// Encode as base64 for storage
|
||||
let signing_key_b64 = B64.encode(signing_key.to_bytes());
|
||||
let verifying_key_b64 = B64.encode(verifying_key.to_bytes());
|
||||
|
||||
(verifying_key_b64, signing_key_b64) // (verify_pub, signing_secret)
|
||||
}
|
||||
|
||||
// ---------- NEW: Persistent, named-key commands ----------
|
||||
|
||||
pub async fn cmd_age_keygen(server: &Server, name: &str) -> Protocol {
|
||||
let (recip, ident) = libcryptoa::gen_enc_keypair();
|
||||
if let Err(e) = sset(server, &enc_pub_key_key(name), &recip) { return Protocol::err(&e.0); }
|
||||
if let Err(e) = sset(server, &enc_priv_key_key(name), &ident) { return Protocol::err(&e.0); }
|
||||
Protocol::Array(vec![Protocol::BulkString(recip), Protocol::BulkString(ident)])
|
||||
/// Encrypt `msg` for `recipient_str` (X25519). Returns base64(ciphertext).
|
||||
pub fn encrypt_b64(recipient_str: &str, msg: &str) -> Result<String, AgeWireError> {
|
||||
let recipient = parse_recipient(recipient_str)?;
|
||||
let enc = Encryptor::with_recipients(vec![Box::new(recipient)])
|
||||
.expect("failed to create encryptor"); // Handle Option<Encryptor>
|
||||
let mut out = Vec::new();
|
||||
{
|
||||
use std::io::Write;
|
||||
let mut w = enc.wrap_output(&mut out).map_err(|e| AgeWireError::Crypto(e.to_string()))?;
|
||||
w.write_all(msg.as_bytes()).map_err(|e| AgeWireError::Crypto(e.to_string()))?;
|
||||
w.finish().map_err(|e| AgeWireError::Crypto(e.to_string()))?;
|
||||
}
|
||||
Ok(B64.encode(out))
|
||||
}
|
||||
|
||||
pub async fn cmd_age_signkeygen(server: &Server, name: &str) -> Protocol {
|
||||
let (verify, secret) = libcryptoa::gen_sign_keypair();
|
||||
if let Err(e) = sset(server, &sign_pub_key_key(name), &verify) { return Protocol::err(&e.0); }
|
||||
if let Err(e) = sset(server, &sign_priv_key_key(name), &secret) { return Protocol::err(&e.0); }
|
||||
Protocol::Array(vec![Protocol::BulkString(verify), Protocol::BulkString(secret)])
|
||||
}
|
||||
/// Decrypt base64(ciphertext) with `identity_str`. Returns plaintext String.
|
||||
pub fn decrypt_b64(identity_str: &str, ct_b64: &str) -> Result<String, AgeWireError> {
|
||||
let id = parse_identity(identity_str)?;
|
||||
let ct = B64.decode(ct_b64.as_bytes()).map_err(|e| AgeWireError::Crypto(e.to_string()))?;
|
||||
let dec = Decryptor::new(&ct[..]).map_err(|e| AgeWireError::Crypto(e.to_string()))?;
|
||||
|
||||
pub async fn cmd_age_encrypt_name(server: &Server, name: &str, message: &str) -> Protocol {
|
||||
let recip = match sget(server, &enc_pub_key_key(name)) {
|
||||
Ok(Some(v)) => v,
|
||||
Ok(None) => return Protocol::err(&format!("ERR age: missing recipient (age:key:{name})")),
|
||||
Err(e) => return Protocol::err(&e.0),
|
||||
};
|
||||
match libcryptoa::encrypt_b64(&recip, message) {
|
||||
Ok(ct) => Protocol::BulkString(ct),
|
||||
Err(e) => Protocol::err(&format!("ERR age: {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn cmd_age_decrypt_name(server: &Server, name: &str, ct_b64: &str) -> Protocol {
|
||||
let ident = match sget(server, &enc_priv_key_key(name)) {
|
||||
Ok(Some(v)) => v,
|
||||
Ok(None) => return Protocol::err(&format!("ERR age: missing identity (age:privkey:{name})")),
|
||||
Err(e) => return Protocol::err(&e.0),
|
||||
};
|
||||
match libcryptoa::decrypt_b64(&ident, ct_b64) {
|
||||
Ok(pt) => Protocol::BulkString(pt),
|
||||
Err(e) => Protocol::err(&format!("ERR age: {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn cmd_age_sign_name(server: &Server, name: &str, message: &str) -> Protocol {
|
||||
let sec = match sget(server, &sign_priv_key_key(name)) {
|
||||
Ok(Some(v)) => v,
|
||||
Ok(None) => return Protocol::err(&format!("ERR age: missing signing secret (age:signpriv:{name})")),
|
||||
Err(e) => return Protocol::err(&e.0),
|
||||
};
|
||||
match libcryptoa::sign_b64(&sec, message) {
|
||||
Ok(sig) => Protocol::BulkString(sig),
|
||||
Err(e) => Protocol::err(&format!("ERR age: {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn cmd_age_verify_name(server: &Server, name: &str, message: &str, sig_b64: &str) -> Protocol {
|
||||
let pubk = match sget(server, &sign_pub_key_key(name)) {
|
||||
Ok(Some(v)) => v,
|
||||
Ok(None) => return Protocol::err(&format!("ERR age: missing verify pubkey (age:signpub:{name})")),
|
||||
Err(e) => return Protocol::err(&e.0),
|
||||
};
|
||||
match libcryptoa::verify_b64(&pubk, message, sig_b64) {
|
||||
Ok(true) => Protocol::SimpleString("1".to_string()),
|
||||
Ok(false) => Protocol::SimpleString("0".to_string()),
|
||||
Err(e) => Protocol::err(&format!("ERR age: {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn cmd_age_list(server: &Server) -> Protocol {
|
||||
// Returns 4 arrays: ["encpub", <names...>], ["encpriv", ...], ["signpub", ...], ["signpriv", ...]
|
||||
let st = match server.current_storage() { Ok(s) => s, Err(e) => return Protocol::err(&e.0) };
|
||||
|
||||
let pull = |pat: &str, prefix: &str| -> Result<Vec<String>, DBError> {
|
||||
let keys = st.keys(pat)?;
|
||||
let mut names: Vec<String> = keys.into_iter()
|
||||
.filter_map(|k| k.strip_prefix(prefix).map(|x| x.to_string()))
|
||||
.collect();
|
||||
names.sort();
|
||||
Ok(names)
|
||||
// The decrypt method returns a Result<StreamReader, DecryptError>
|
||||
let mut r = match dec {
|
||||
Decryptor::Recipients(d) => d.decrypt(std::iter::once(&id as &dyn age::Identity))
|
||||
.map_err(|e| AgeWireError::Crypto(e.to_string()))?,
|
||||
Decryptor::Passphrase(_) => return Err(AgeWireError::Crypto("Expected recipients, got passphrase".to_string())),
|
||||
};
|
||||
|
||||
let encpub = match pull("age:key:*", "age:key:") { Ok(v) => v, Err(e)=> return Protocol::err(&e.0) };
|
||||
let encpriv = match pull("age:privkey:*", "age:privkey:") { Ok(v) => v, Err(e)=> return Protocol::err(&e.0) };
|
||||
let signpub = match pull("age:signpub:*", "age:signpub:") { Ok(v) => v, Err(e)=> return Protocol::err(&e.0) };
|
||||
let signpriv= match pull("age:signpriv:*", "age:signpriv:") { Ok(v) => v, Err(e)=> return Protocol::err(&e.0) };
|
||||
let mut pt = Vec::new();
|
||||
use std::io::Read;
|
||||
r.read_to_end(&mut pt).map_err(|e| AgeWireError::Crypto(e.to_string()))?;
|
||||
String::from_utf8(pt).map_err(|_| AgeWireError::Utf8)
|
||||
}
|
||||
|
||||
let to_arr = |label: &str, v: Vec<String>| {
|
||||
let mut out = vec![Protocol::BulkString(label.to_string())];
|
||||
out.push(Protocol::Array(v.into_iter().map(Protocol::BulkString).collect()));
|
||||
Protocol::Array(out)
|
||||
};
|
||||
/// Sign bytes of `msg` (detached). Returns base64(signature bytes, 64 bytes).
|
||||
pub fn sign_b64(signing_secret_str: &str, msg: &str) -> Result<String, AgeWireError> {
|
||||
let signing_key = parse_ed25519_signing_key(signing_secret_str)?;
|
||||
let sig = signing_key.sign(msg.as_bytes());
|
||||
Ok(B64.encode(sig.to_bytes()))
|
||||
}
|
||||
|
||||
Protocol::Array(vec![
|
||||
to_arr("encpub", encpub),
|
||||
to_arr("encpriv", encpriv),
|
||||
to_arr("signpub", signpub),
|
||||
to_arr("signpriv", signpriv),
|
||||
])
|
||||
/// Verify detached signature (base64) for `msg` with pubkey.
|
||||
pub fn verify_b64(verify_pub_str: &str, msg: &str, sig_b64: &str) -> Result<bool, AgeWireError> {
|
||||
let verifying_key = parse_ed25519_verifying_key(verify_pub_str)?;
|
||||
let sig_bytes = B64.decode(sig_b64.as_bytes()).map_err(|e| AgeWireError::Crypto(e.to_string()))?;
|
||||
if sig_bytes.len() != 64 {
|
||||
return Err(AgeWireError::SignatureLen);
|
||||
}
|
||||
let sig = Signature::from_bytes(sig_bytes[..].try_into().unwrap());
|
||||
Ok(verifying_key.verify(msg.as_bytes(), &sig).is_ok())
|
||||
}
|
||||
|
||||
// ---------- Storage helpers ----------
|
||||
@ -1,7 +1,4 @@
|
||||
use crate::protocol::Protocol;
|
||||
use crate::server::Server;
|
||||
use libdbstorage::DBError;
|
||||
use libcryptoa;
|
||||
use crate::{error::DBError, protocol::Protocol, server::Server};
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@ -541,12 +538,12 @@ impl Cmd {
|
||||
Cmd::LRange(key, start, stop) => lrange_cmd(server, &key, start, stop).await,
|
||||
Cmd::FlushDb => flushdb_cmd(server).await,
|
||||
// AGE (rage): stateless
|
||||
Cmd::AgeGenEnc => Ok(libcryptoa::gen_enc_keypair().await),
|
||||
Cmd::AgeGenSign => Ok(libcryptoa::gen_sign_keypair().await),
|
||||
Cmd::AgeEncrypt(recipient, message) => Ok(libcryptoa::encrypt_b64(&recipient, &message).await),
|
||||
Cmd::AgeDecrypt(identity, ct_b64) => Ok(libcryptoa::decrypt_b64(&identity, &ct_b64).await),
|
||||
Cmd::AgeSign(secret, message) => Ok(libcryptoa::sign_b64(&secret, &message).await),
|
||||
Cmd::AgeVerify(vpub, msg, sig_b64) => Ok(libcryptoa::verify_b64(&vpub, &msg, &sig_b64).await),
|
||||
Cmd::AgeGenEnc => Ok(crate::age::cmd_age_genenc().await),
|
||||
Cmd::AgeGenSign => Ok(crate::age::cmd_age_gensign().await),
|
||||
Cmd::AgeEncrypt(recipient, message) => Ok(crate::age::cmd_age_encrypt(&recipient, &message).await),
|
||||
Cmd::AgeDecrypt(identity, ct_b64) => Ok(crate::age::cmd_age_decrypt(&identity, &ct_b64).await),
|
||||
Cmd::AgeSign(secret, message) => Ok(crate::age::cmd_age_sign(&secret, &message).await),
|
||||
Cmd::AgeVerify(vpub, msg, sig_b64) => Ok(crate::age::cmd_age_verify(&vpub, &msg, &sig_b64).await),
|
||||
|
||||
// AGE (rage): persistent named keys
|
||||
Cmd::AgeKeygen(name) => Ok(crate::age::cmd_age_keygen(server, &name).await),
|
||||
@ -1,24 +1,25 @@
|
||||
// In crates/libcrypto/src/lib.rs
|
||||
use chacha20poly1305::{
|
||||
aead::{Aead, KeyInit, OsRng},
|
||||
XChaCha20Poly1305, XNonce,
|
||||
};
|
||||
use rand::RngCore;
|
||||
use sha2::{Digest, Sha256};
|
||||
use thiserror::Error;
|
||||
|
||||
const VERSION: u8 = 1;
|
||||
const NONCE_LEN: usize = 24;
|
||||
const TAG_LEN: usize = 16;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
#[derive(Debug)]
|
||||
pub enum CryptoError {
|
||||
#[error("invalid format: data too short")]
|
||||
Format,
|
||||
#[error("unknown version: {0}")]
|
||||
Version(u8),
|
||||
#[error("decryption failed: wrong key or corrupted data")]
|
||||
Decrypt,
|
||||
Format, // wrong length / header
|
||||
Version(u8), // unknown version
|
||||
Decrypt, // wrong key or corrupted data
|
||||
}
|
||||
|
||||
impl From<CryptoError> for crate::error::DBError {
|
||||
fn from(e: CryptoError) -> Self {
|
||||
crate::error::DBError(format!("Crypto error: {:?}", e))
|
||||
}
|
||||
}
|
||||
|
||||
/// Super-simple factory: new(secret) + encrypt(bytes) + decrypt(bytes)
|
||||
@ -87,3 +87,8 @@ impl From<serde_json::Error> for DBError {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<chacha20poly1305::Error> for DBError {
|
||||
fn from(item: chacha20poly1305::Error) -> Self {
|
||||
DBError(item.to_string())
|
||||
}
|
||||
}
|
||||
@ -1,4 +1,8 @@
|
||||
pub mod age; // NEW
|
||||
pub mod cmd;
|
||||
pub mod crypto;
|
||||
pub mod error;
|
||||
pub mod options;
|
||||
pub mod protocol;
|
||||
pub mod server;
|
||||
pub mod storage;
|
||||
@ -30,7 +30,7 @@ impl Server {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn current_storage(&self) -> Result<Arc<libdbstorage::Storage>, libdbstorage::DBError> {
|
||||
pub fn current_storage(&self) -> Result<Arc<Storage>, DBError> {
|
||||
let mut cache = self.db_cache.write().unwrap();
|
||||
|
||||
if let Some(storage) = cache.get(&self.selected_db) {
|
||||
@ -1,18 +1,25 @@
|
||||
// In crates/libdbstorage/src/lib.rs
|
||||
use std::{
|
||||
path::Path,
|
||||
time::{SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use libcrypto::CryptoFactory; // Correct import
|
||||
use redb::{Database, TableDefinition};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub mod error; // Declare the error module
|
||||
pub use error::DBError; // Re-export for users of this crate
|
||||
use crate::crypto::CryptoFactory;
|
||||
use crate::error::DBError;
|
||||
|
||||
// Declare storage module
|
||||
pub mod storage;
|
||||
// Re-export modules
|
||||
mod storage_basic;
|
||||
mod storage_hset;
|
||||
mod storage_lists;
|
||||
mod storage_extra;
|
||||
|
||||
// Re-export implementations
|
||||
// Note: These imports are used by the impl blocks in the submodules
|
||||
// The compiler shows them as unused because they're not directly used in this file
|
||||
// but they're needed for the Storage struct methods to be available
|
||||
pub use storage_extra::*;
|
||||
|
||||
// Table definitions for different Redis data types
|
||||
const TYPES_TABLE: TableDefinition<&str, &str> = TableDefinition::new("types");
|
||||
@ -111,7 +118,7 @@ impl Storage {
|
||||
|
||||
fn decrypt_if_needed(&self, data: &[u8]) -> Result<Vec<u8>, DBError> {
|
||||
if let Some(crypto) = &self.crypto {
|
||||
Ok(crypto.decrypt(data).map_err(|e| DBError(e.to_string()))?)
|
||||
Ok(crypto.decrypt(data)?)
|
||||
} else {
|
||||
Ok(data.to_vec())
|
||||
}
|
||||
@ -1,6 +1,6 @@
|
||||
use redb::{ReadableTable};
|
||||
use crate::error::DBError;
|
||||
use crate::{Storage, TYPES_TABLE, STRINGS_TABLE, HASHES_TABLE, LISTS_TABLE, STREAMS_META_TABLE, STREAMS_DATA_TABLE, EXPIRATION_TABLE, now_in_millis};
|
||||
use super::*;
|
||||
|
||||
impl Storage {
|
||||
pub fn flushdb(&self) -> Result<(), DBError> {
|
||||
@ -1,6 +1,6 @@
|
||||
use redb::{ReadableTable};
|
||||
use crate::error::DBError;
|
||||
use crate::{Storage, TYPES_TABLE, STRINGS_TABLE, EXPIRATION_TABLE, now_in_millis};
|
||||
use super::*;
|
||||
|
||||
impl Storage {
|
||||
// ✅ ENCRYPTION APPLIED: Values are decrypted after retrieval
|
||||
@ -1,6 +1,6 @@
|
||||
use redb::{ReadableTable};
|
||||
use crate::error::DBError;
|
||||
use crate::{Storage, TYPES_TABLE, HASHES_TABLE};
|
||||
use super::*;
|
||||
|
||||
impl Storage {
|
||||
// ✅ ENCRYPTION APPLIED: Values are encrypted before storage
|
||||
@ -12,7 +12,14 @@ impl Storage {
|
||||
let mut types_table = write_txn.open_table(TYPES_TABLE)?;
|
||||
let mut hashes_table = write_txn.open_table(HASHES_TABLE)?;
|
||||
|
||||
// Set the type to hash
|
||||
let key_type = {
|
||||
let access_guard = types_table.get(key)?;
|
||||
access_guard.map(|v| v.value().to_string())
|
||||
};
|
||||
|
||||
match key_type.as_deref() {
|
||||
Some("hash") | None => { // Proceed if hash or new key
|
||||
// Set the type to hash (only if new key or existing hash)
|
||||
types_table.insert(key, "hash")?;
|
||||
|
||||
for (field, value) in pairs {
|
||||
@ -28,6 +35,9 @@ impl Storage {
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(_) => return Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
write_txn.commit()?;
|
||||
Ok(new_fields)
|
||||
@ -38,8 +48,10 @@ impl Storage {
|
||||
let read_txn = self.db.begin_read()?;
|
||||
let types_table = read_txn.open_table(TYPES_TABLE)?;
|
||||
|
||||
match types_table.get(key)? {
|
||||
Some(type_val) if type_val.value() == "hash" => {
|
||||
let key_type = types_table.get(key)?.map(|v| v.value().to_string());
|
||||
|
||||
match key_type.as_deref() {
|
||||
Some("hash") => {
|
||||
let hashes_table = read_txn.open_table(HASHES_TABLE)?;
|
||||
match hashes_table.get((key, field))? {
|
||||
Some(data) => {
|
||||
@ -50,7 +62,8 @@ impl Storage {
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
_ => Ok(None),
|
||||
Some(_) => Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
@ -58,9 +71,13 @@ impl Storage {
|
||||
pub fn hgetall(&self, key: &str) -> Result<Vec<(String, String)>, DBError> {
|
||||
let read_txn = self.db.begin_read()?;
|
||||
let types_table = read_txn.open_table(TYPES_TABLE)?;
|
||||
let key_type = {
|
||||
let access_guard = types_table.get(key)?;
|
||||
access_guard.map(|v| v.value().to_string())
|
||||
};
|
||||
|
||||
match types_table.get(key)? {
|
||||
Some(type_val) if type_val.value() == "hash" => {
|
||||
match key_type.as_deref() {
|
||||
Some("hash") => {
|
||||
let hashes_table = read_txn.open_table(HASHES_TABLE)?;
|
||||
let mut result = Vec::new();
|
||||
|
||||
@ -77,7 +94,8 @@ impl Storage {
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
_ => Ok(Vec::new()),
|
||||
Some(_) => Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())),
|
||||
None => Ok(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@ -86,16 +104,14 @@ impl Storage {
|
||||
let mut deleted = 0i64;
|
||||
|
||||
// First check if key exists and is a hash
|
||||
let is_hash = {
|
||||
let key_type = {
|
||||
let types_table = write_txn.open_table(TYPES_TABLE)?;
|
||||
let result = match types_table.get(key)? {
|
||||
Some(type_val) => type_val.value() == "hash",
|
||||
None => false,
|
||||
};
|
||||
result
|
||||
let access_guard = types_table.get(key)?;
|
||||
access_guard.map(|v| v.value().to_string())
|
||||
};
|
||||
|
||||
if is_hash {
|
||||
match key_type.as_deref() {
|
||||
Some("hash") => {
|
||||
let mut hashes_table = write_txn.open_table(HASHES_TABLE)?;
|
||||
|
||||
for field in fields {
|
||||
@ -122,6 +138,9 @@ impl Storage {
|
||||
types_table.remove(key)?;
|
||||
}
|
||||
}
|
||||
Some(_) => return Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())),
|
||||
None => {} // Key does not exist, nothing to delete, return 0 deleted
|
||||
}
|
||||
|
||||
write_txn.commit()?;
|
||||
Ok(deleted)
|
||||
@ -131,12 +150,19 @@ impl Storage {
|
||||
let read_txn = self.db.begin_read()?;
|
||||
let types_table = read_txn.open_table(TYPES_TABLE)?;
|
||||
|
||||
match types_table.get(key)? {
|
||||
Some(type_val) if type_val.value() == "hash" => {
|
||||
let types_table = read_txn.open_table(TYPES_TABLE)?;
|
||||
let key_type = {
|
||||
let access_guard = types_table.get(key)?;
|
||||
access_guard.map(|v| v.value().to_string())
|
||||
};
|
||||
|
||||
match key_type.as_deref() {
|
||||
Some("hash") => {
|
||||
let hashes_table = read_txn.open_table(HASHES_TABLE)?;
|
||||
Ok(hashes_table.get((key, field))?.is_some())
|
||||
}
|
||||
_ => Ok(false),
|
||||
Some(_) => Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())),
|
||||
None => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
@ -144,8 +170,14 @@ impl Storage {
|
||||
let read_txn = self.db.begin_read()?;
|
||||
let types_table = read_txn.open_table(TYPES_TABLE)?;
|
||||
|
||||
match types_table.get(key)? {
|
||||
Some(type_val) if type_val.value() == "hash" => {
|
||||
let types_table = read_txn.open_table(TYPES_TABLE)?;
|
||||
let key_type = {
|
||||
let access_guard = types_table.get(key)?;
|
||||
access_guard.map(|v| v.value().to_string())
|
||||
};
|
||||
|
||||
match key_type.as_deref() {
|
||||
Some("hash") => {
|
||||
let hashes_table = read_txn.open_table(HASHES_TABLE)?;
|
||||
let mut result = Vec::new();
|
||||
|
||||
@ -160,7 +192,8 @@ impl Storage {
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
_ => Ok(Vec::new()),
|
||||
Some(_) => Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())),
|
||||
None => Ok(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@ -169,8 +202,14 @@ impl Storage {
|
||||
let read_txn = self.db.begin_read()?;
|
||||
let types_table = read_txn.open_table(TYPES_TABLE)?;
|
||||
|
||||
match types_table.get(key)? {
|
||||
Some(type_val) if type_val.value() == "hash" => {
|
||||
let types_table = read_txn.open_table(TYPES_TABLE)?;
|
||||
let key_type = {
|
||||
let access_guard = types_table.get(key)?;
|
||||
access_guard.map(|v| v.value().to_string())
|
||||
};
|
||||
|
||||
match key_type.as_deref() {
|
||||
Some("hash") => {
|
||||
let hashes_table = read_txn.open_table(HASHES_TABLE)?;
|
||||
let mut result = Vec::new();
|
||||
|
||||
@ -187,7 +226,8 @@ impl Storage {
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
_ => Ok(Vec::new()),
|
||||
Some(_) => Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())),
|
||||
None => Ok(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@ -195,8 +235,14 @@ impl Storage {
|
||||
let read_txn = self.db.begin_read()?;
|
||||
let types_table = read_txn.open_table(TYPES_TABLE)?;
|
||||
|
||||
match types_table.get(key)? {
|
||||
Some(type_val) if type_val.value() == "hash" => {
|
||||
let types_table = read_txn.open_table(TYPES_TABLE)?;
|
||||
let key_type = {
|
||||
let access_guard = types_table.get(key)?;
|
||||
access_guard.map(|v| v.value().to_string())
|
||||
};
|
||||
|
||||
match key_type.as_deref() {
|
||||
Some("hash") => {
|
||||
let hashes_table = read_txn.open_table(HASHES_TABLE)?;
|
||||
let mut count = 0i64;
|
||||
|
||||
@ -211,7 +257,8 @@ impl Storage {
|
||||
|
||||
Ok(count)
|
||||
}
|
||||
_ => Ok(0),
|
||||
Some(_) => Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())),
|
||||
None => Ok(0),
|
||||
}
|
||||
}
|
||||
|
||||
@ -220,8 +267,14 @@ impl Storage {
|
||||
let read_txn = self.db.begin_read()?;
|
||||
let types_table = read_txn.open_table(TYPES_TABLE)?;
|
||||
|
||||
match types_table.get(key)? {
|
||||
Some(type_val) if type_val.value() == "hash" => {
|
||||
let types_table = read_txn.open_table(TYPES_TABLE)?;
|
||||
let key_type = {
|
||||
let access_guard = types_table.get(key)?;
|
||||
access_guard.map(|v| v.value().to_string())
|
||||
};
|
||||
|
||||
match key_type.as_deref() {
|
||||
Some("hash") => {
|
||||
let hashes_table = read_txn.open_table(HASHES_TABLE)?;
|
||||
let mut result = Vec::new();
|
||||
|
||||
@ -238,7 +291,8 @@ impl Storage {
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
_ => Ok(fields.into_iter().map(|_| None).collect()),
|
||||
Some(_) => Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())),
|
||||
None => Ok(fields.into_iter().map(|_| None).collect()),
|
||||
}
|
||||
}
|
||||
|
||||
@ -251,9 +305,16 @@ impl Storage {
|
||||
let mut types_table = write_txn.open_table(TYPES_TABLE)?;
|
||||
let mut hashes_table = write_txn.open_table(HASHES_TABLE)?;
|
||||
|
||||
let key_type = {
|
||||
let access_guard = types_table.get(key)?;
|
||||
access_guard.map(|v| v.value().to_string())
|
||||
};
|
||||
|
||||
match key_type.as_deref() {
|
||||
Some("hash") | None => { // Proceed if hash or new key
|
||||
// Check if field already exists
|
||||
if hashes_table.get((key, field))?.is_none() {
|
||||
// Set the type to hash
|
||||
// Set the type to hash (only if new key or existing hash)
|
||||
types_table.insert(key, "hash")?;
|
||||
|
||||
// Encrypt the value before storing
|
||||
@ -262,6 +323,9 @@ impl Storage {
|
||||
result = true;
|
||||
}
|
||||
}
|
||||
Some(_) => return Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
write_txn.commit()?;
|
||||
Ok(result)
|
||||
@ -272,8 +336,14 @@ impl Storage {
|
||||
let read_txn = self.db.begin_read()?;
|
||||
let types_table = read_txn.open_table(TYPES_TABLE)?;
|
||||
|
||||
match types_table.get(key)? {
|
||||
Some(type_val) if type_val.value() == "hash" => {
|
||||
let types_table = read_txn.open_table(TYPES_TABLE)?;
|
||||
let key_type = {
|
||||
let access_guard = types_table.get(key)?;
|
||||
access_guard.map(|v| v.value().to_string())
|
||||
};
|
||||
|
||||
match key_type.as_deref() {
|
||||
Some("hash") => {
|
||||
let hashes_table = read_txn.open_table(HASHES_TABLE)?;
|
||||
let mut result = Vec::new();
|
||||
let mut current_cursor = 0u64;
|
||||
@ -312,7 +382,8 @@ impl Storage {
|
||||
let next_cursor = if result.len() < limit { 0 } else { current_cursor };
|
||||
Ok((next_cursor, result))
|
||||
}
|
||||
_ => Ok((0, Vec::new())),
|
||||
Some(_) => Err(DBError("WRONGTYPE Operation against a key holding the wrong kind of value".to_string())),
|
||||
None => Ok((0, Vec::new())),
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,6 +1,6 @@
|
||||
use redb::{ReadableTable};
|
||||
use crate::error::DBError;
|
||||
use crate::{Storage, TYPES_TABLE, LISTS_TABLE};
|
||||
use super::*;
|
||||
|
||||
impl Storage {
|
||||
// ✅ ENCRYPTION APPLIED: Elements are encrypted before storage
|
||||
@ -25,7 +25,7 @@ impl Storage {
|
||||
};
|
||||
|
||||
// Add elements to the front (left)
|
||||
for element in elements.into_iter().rev() {
|
||||
for element in elements.into_iter() {
|
||||
list.insert(0, element);
|
||||
}
|
||||
|
||||
@ -93,9 +93,16 @@ async fn test_basic_redis_functionality() {
|
||||
assert!(response.contains("string"));
|
||||
|
||||
// Test QUIT to close connection gracefully
|
||||
let response = send_redis_command(port, "*1\r\n$4\r\nQUIT\r\n").await;
|
||||
let mut stream = TcpStream::connect(format!("127.0.0.1:{}", port)).await.unwrap();
|
||||
stream.write_all("*1\r\n$4\r\nQUIT\r\n".as_bytes()).await.unwrap();
|
||||
let mut buffer = [0; 1024];
|
||||
let n = stream.read(&mut buffer).await.unwrap();
|
||||
let response = String::from_utf8_lossy(&buffer[..n]);
|
||||
assert!(response.contains("OK"));
|
||||
|
||||
// Ensure the stream is closed
|
||||
stream.shutdown().await.unwrap();
|
||||
|
||||
// Stop the server
|
||||
server_handle.abort();
|
||||
|
||||
@ -149,6 +156,8 @@ async fn test_hash_operations() {
|
||||
assert!(response.contains("value2"));
|
||||
|
||||
// Stop the server
|
||||
// For hash operations, we don't have a persistent stream, so we'll just abort the server.
|
||||
// The server should handle closing its connections.
|
||||
server_handle.abort();
|
||||
|
||||
println!("✅ All hash operations tests passed!");
|
||||
@ -202,9 +211,16 @@ async fn test_transaction_operations() {
|
||||
assert!(response.contains("OK")); // Should contain array of OK responses
|
||||
|
||||
// Verify commands were executed
|
||||
let response = send_redis_command(port, "*2\r\n$3\r\nGET\r\n$4\r\nkey1\r\n").await;
|
||||
stream.write_all("*2\r\n$3\r\nGET\r\n$4\r\nkey1\r\n".as_bytes()).await.unwrap();
|
||||
let n = stream.read(&mut buffer).await.unwrap();
|
||||
let response = String::from_utf8_lossy(&buffer[..n]);
|
||||
assert!(response.contains("value1"));
|
||||
|
||||
stream.write_all("*2\r\n$3\r\nGET\r\n$4\r\nkey2\r\n".as_bytes()).await.unwrap();
|
||||
let n = stream.read(&mut buffer).await.unwrap();
|
||||
let response = String::from_utf8_lossy(&buffer[..n]);
|
||||
assert!(response.contains("value2"));
|
||||
|
||||
// Stop the server
|
||||
server_handle.abort();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user