add data packages and remove empty submodule

packages/data/tst/examples/basic_usage.rs

@@ -0,0 +1,75 @@
+use std::time::Instant;
+use tst::TST;
+
+fn main() -> Result<(), tst::Error> {
+    // Create a temporary directory for the database
+    let db_path = std::env::temp_dir().join("tst_example");
+    std::fs::create_dir_all(&db_path)?;
+
+    println!("Creating ternary search tree at: {}", db_path.display());
+
+    // Create a new TST
+    let mut tree = TST::new(db_path.to_str().unwrap(), true)?;
+
+    // Store some data
+    println!("Inserting data...");
+    tree.set("hello", b"world".to_vec())?;
+    tree.set("help", b"me".to_vec())?;
+    tree.set("helicopter", b"flying".to_vec())?;
+    tree.set("apple", b"fruit".to_vec())?;
+    tree.set("application", b"software".to_vec())?;
+    tree.set("banana", b"yellow".to_vec())?;
+
+    // Retrieve and print the data
+    let value = tree.get("hello")?;
+    println!("hello: {}", String::from_utf8_lossy(&value));
+
+    // List keys with prefix
+    println!("\nListing keys with prefix 'hel':");
+    let start = Instant::now();
+    let keys = tree.list("hel")?;
+    let duration = start.elapsed();
+
+    for key in &keys {
+        println!("  {}", key);
+    }
+    println!("Found {} keys in {:?}", keys.len(), duration);
+
+    // Get all values with prefix
+    println!("\nGetting all values with prefix 'app':");
+    let start = Instant::now();
+    let values = tree.getall("app")?;
+    let duration = start.elapsed();
+
+    for (i, value) in values.iter().enumerate() {
+        println!("  Value {}: {}", i + 1, String::from_utf8_lossy(value));
+    }
+    println!("Found {} values in {:?}", values.len(), duration);
+
+    // Delete a key
+    println!("\nDeleting 'help'...");
+    tree.delete("help")?;
+
+    // Verify deletion
+    println!("Listing keys with prefix 'hel' after deletion:");
+    let keys_after = tree.list("hel")?;
+    for key in &keys_after {
+        println!("  {}", key);
+    }
+
+    // Try to get a deleted key
+    match tree.get("help") {
+        Ok(_) => println!("Unexpectedly found 'help' after deletion!"),
+        Err(e) => println!("As expected, 'help' was not found: {}", e),
+    }
+
+    // Clean up (optional)
+    if std::env::var("KEEP_DB").is_err() {
+        std::fs::remove_dir_all(&db_path)?;
+        println!("\nCleaned up database directory");
+    } else {
+        println!("\nDatabase kept at: {}", db_path.display());
+    }
+
+    Ok(())
+}

packages/data/tst/examples/performance.rs

@@ -0,0 +1,167 @@
+use std::io::{self, Write};
+use std::time::{Duration, Instant};
+use tst::TST;
+
+// Function to generate a test value of specified size
+fn generate_test_value(index: usize, size: usize) -> Vec<u8> {
+    let base_value = format!("val{:08}", index);
+    let mut value = Vec::with_capacity(size);
+
+    // Fill with repeating pattern to reach desired size
+    while value.len() < size {
+        value.extend_from_slice(base_value.as_bytes());
+    }
+
+    // Truncate to exact size
+    value.truncate(size);
+
+    value
+}
+
+// Number of records to insert
+const TOTAL_RECORDS: usize = 100_000;
+// How often to report progress (every X records)
+const PROGRESS_INTERVAL: usize = 1_000;
+// How many records to use for performance sampling
+const PERFORMANCE_SAMPLE_SIZE: usize = 100;
+
+fn main() -> Result<(), tst::Error> {
+    // Create a temporary directory for the database
+    let db_path = std::env::temp_dir().join("tst_performance_test");
+
+    // Completely remove and recreate the directory to ensure a clean start
+    if db_path.exists() {
+        std::fs::remove_dir_all(&db_path)?;
+    }
+    std::fs::create_dir_all(&db_path)?;
+
+    println!("Creating ternary search tree at: {}", db_path.display());
+    println!("Will insert {} records and show progress...", TOTAL_RECORDS);
+
+    // Create a new TST
+    let mut tree = TST::new(db_path.to_str().unwrap(), true)?;
+
+    // Track overall time
+    let start_time = Instant::now();
+
+    // Track performance metrics
+    let mut insertion_times = Vec::with_capacity(TOTAL_RECORDS / PROGRESS_INTERVAL);
+    let mut last_batch_time = Instant::now();
+    let mut last_batch_records = 0;
+
+    // Insert records and track progress
+    for i in 0..TOTAL_RECORDS {
+        let key = format!("key:{:08}", i);
+        // Generate a 100-byte value
+        let value = generate_test_value(i, 100);
+
+        // Time the insertion of every Nth record for performance sampling
+        if i % PERFORMANCE_SAMPLE_SIZE == 0 {
+            let insert_start = Instant::now();
+            tree.set(&key, value)?;
+            let insert_duration = insert_start.elapsed();
+
+            // Only print detailed timing for specific samples to avoid flooding output
+            if i % (PERFORMANCE_SAMPLE_SIZE * 10) == 0 {
+                println!("Record {}: Insertion took {:?}", i, insert_duration);
+            }
+        } else {
+            tree.set(&key, value)?;
+        }
+
+        // Show progress at intervals
+        if (i + 1) % PROGRESS_INTERVAL == 0 || i == TOTAL_RECORDS - 1 {
+            let records_in_batch = i + 1 - last_batch_records;
+            let batch_duration = last_batch_time.elapsed();
+            let records_per_second = records_in_batch as f64 / batch_duration.as_secs_f64();
+
+            insertion_times.push((i + 1, batch_duration));
+
+            print!(
+                "\rProgress: {}/{} records ({:.2}%) - {:.2} records/sec",
+                i + 1,
+                TOTAL_RECORDS,
+                (i + 1) as f64 / TOTAL_RECORDS as f64 * 100.0,
+                records_per_second
+            );
+            io::stdout().flush().unwrap();
+
+            last_batch_time = Instant::now();
+            last_batch_records = i + 1;
+        }
+    }
+
+    let total_duration = start_time.elapsed();
+    println!("\n\nPerformance Summary:");
+    println!(
+        "Total time to insert {} records: {:?}",
+        TOTAL_RECORDS, total_duration
+    );
+    println!(
+        "Average insertion rate: {:.2} records/second",
+        TOTAL_RECORDS as f64 / total_duration.as_secs_f64()
+    );
+
+    // Show performance trend
+    println!("\nPerformance Trend (records inserted vs. time per batch):");
+    for (i, (record_count, duration)) in insertion_times.iter().enumerate() {
+        if i % 10 == 0 || i == insertion_times.len() - 1 {
+            // Only show every 10th point to avoid too much output
+            println!(
+                "  After {} records: {:?} for {} records ({:.2} records/sec)",
+                record_count,
+                duration,
+                PROGRESS_INTERVAL,
+                PROGRESS_INTERVAL as f64 / duration.as_secs_f64()
+            );
+        }
+    }
+
+    // Test access performance with distributed samples
+    println!("\nTesting access performance with distributed samples...");
+    let mut total_get_time = Duration::new(0, 0);
+    let num_samples = 1000;
+
+    // Use a simple distribution pattern instead of random
+    for i in 0..num_samples {
+        // Distribute samples across the entire range
+        let sample_id = (i * (TOTAL_RECORDS / num_samples)) % TOTAL_RECORDS;
+        let key = format!("key:{:08}", sample_id);
+
+        let get_start = Instant::now();
+        let _ = tree.get(&key)?;
+        total_get_time += get_start.elapsed();
+    }
+
+    println!(
+        "Average time to retrieve a record: {:?}",
+        total_get_time / num_samples as u32
+    );
+
+    // Test prefix search performance
+    println!("\nTesting prefix search performance...");
+    let prefixes = ["key:0", "key:1", "key:5", "key:9"];
+
+    for prefix in &prefixes {
+        let list_start = Instant::now();
+        let keys = tree.list(prefix)?;
+        let list_duration = list_start.elapsed();
+
+        println!(
+            "Found {} keys with prefix '{}' in {:?}",
+            keys.len(),
+            prefix,
+            list_duration
+        );
+    }
+
+    // Clean up (optional)
+    if std::env::var("KEEP_DB").is_err() {
+        std::fs::remove_dir_all(&db_path)?;
+        println!("\nCleaned up database directory");
+    } else {
+        println!("\nDatabase kept at: {}", db_path.display());
+    }
+
+    Ok(())
+}

packages/data/tst/examples/prefix_ops.rs

@@ -0,0 +1,184 @@
+use std::time::Instant;
+use tst::TST;
+
+fn main() -> Result<(), tst::Error> {
+    // Create a temporary directory for the database
+    let db_path = std::env::temp_dir().join("tst_prefix_example");
+    std::fs::create_dir_all(&db_path)?;
+
+    println!("Creating ternary search tree at: {}", db_path.display());
+
+    // Create a new TST
+    let mut tree = TST::new(db_path.to_str().unwrap(), true)?;
+
+    // Insert a variety of keys with different prefixes
+    println!("Inserting data with various prefixes...");
+
+    // Names
+    let names = [
+        "Alice",
+        "Alexander",
+        "Amanda",
+        "Andrew",
+        "Amy",
+        "Bob",
+        "Barbara",
+        "Benjamin",
+        "Brenda",
+        "Brian",
+        "Charlie",
+        "Catherine",
+        "Christopher",
+        "Cynthia",
+        "Carl",
+        "David",
+        "Diana",
+        "Daniel",
+        "Deborah",
+        "Donald",
+        "Edward",
+        "Elizabeth",
+        "Eric",
+        "Emily",
+        "Ethan",
+    ];
+
+    for (i, name) in names.iter().enumerate() {
+        let value = format!("person-{}", i).into_bytes();
+        tree.set(name, value)?;
+    }
+
+    // Cities
+    let cities = [
+        "New York",
+        "Los Angeles",
+        "Chicago",
+        "Houston",
+        "Phoenix",
+        "Philadelphia",
+        "San Antonio",
+        "San Diego",
+        "Dallas",
+        "San Jose",
+        "Austin",
+        "Jacksonville",
+        "Fort Worth",
+        "Columbus",
+        "San Francisco",
+        "Charlotte",
+        "Indianapolis",
+        "Seattle",
+        "Denver",
+        "Washington",
+    ];
+
+    for (i, city) in cities.iter().enumerate() {
+        let value = format!("city-{}", i).into_bytes();
+        tree.set(city, value)?;
+    }
+
+    // Countries
+    let countries = [
+        "United States",
+        "Canada",
+        "Mexico",
+        "Brazil",
+        "Argentina",
+        "United Kingdom",
+        "France",
+        "Germany",
+        "Italy",
+        "Spain",
+        "China",
+        "Japan",
+        "India",
+        "Australia",
+        "Russia",
+    ];
+
+    for (i, country) in countries.iter().enumerate() {
+        let value = format!("country-{}", i).into_bytes();
+        tree.set(country, value)?;
+    }
+
+    println!(
+        "Total items inserted: {}",
+        names.len() + cities.len() + countries.len()
+    );
+
+    // Test prefix operations
+    test_prefix(&mut tree, "A")?;
+    test_prefix(&mut tree, "B")?;
+    test_prefix(&mut tree, "C")?;
+    test_prefix(&mut tree, "San")?;
+    test_prefix(&mut tree, "United")?;
+
+    // Test non-existent prefix
+    test_prefix(&mut tree, "Z")?;
+
+    // Test empty prefix (should return all keys)
+    println!("\nTesting empty prefix (should return all keys):");
+    let start = Instant::now();
+    let all_keys = tree.list("")?;
+    let duration = start.elapsed();
+
+    println!(
+        "Found {} keys with empty prefix in {:?}",
+        all_keys.len(),
+        duration
+    );
+    println!("First 5 keys (alphabetically):");
+    for key in all_keys.iter().take(5) {
+        println!("  {}", key);
+    }
+
+    // Clean up (optional)
+    if std::env::var("KEEP_DB").is_err() {
+        std::fs::remove_dir_all(&db_path)?;
+        println!("\nCleaned up database directory");
+    } else {
+        println!("\nDatabase kept at: {}", db_path.display());
+    }
+
+    Ok(())
+}
+
+fn test_prefix(tree: &mut TST, prefix: &str) -> Result<(), tst::Error> {
+    println!("\nTesting prefix '{}':", prefix);
+
+    // Test list operation
+    let start = Instant::now();
+    let keys = tree.list(prefix)?;
+    let list_duration = start.elapsed();
+
+    println!(
+        "Found {} keys with prefix '{}' in {:?}",
+        keys.len(),
+        prefix,
+        list_duration
+    );
+
+    if !keys.is_empty() {
+        println!("Keys:");
+        for key in &keys {
+            println!("  {}", key);
+        }
+
+        // Test getall operation
+        let start = Instant::now();
+        let values = tree.getall(prefix)?;
+        let getall_duration = start.elapsed();
+
+        println!("Retrieved {} values in {:?}", values.len(), getall_duration);
+        println!(
+            "First value: {}",
+            if !values.is_empty() {
+                String::from_utf8_lossy(&values[0])
+            } else {
+                "None".into()
+            }
+        );
+    }
+
+    Ok(())
+}