...

radixtree/README.md

@@ -38,7 +38,7 @@ use radixtree::RadixTree;
 
 fn main() -> Result<(), radixtree::Error> {
     // Create a new radix tree
-    let mut tree = RadixTree::new("/path/to/storage", false)?;
+    let mut tree = RadixTree::new("/tmp/radix", false)?;
     
     // Set key-value pairs
     tree.set("hello", b"world".to_vec())?;
@@ -68,10 +68,10 @@ fn main() -> Result<(), radixtree::Error> {
 
 ```rust
 // Create a new radix tree
-let mut tree = RadixTree::new("/path/to/storage", false)?;
+let mut tree = RadixTree::new("/tmp/radix", false)?;
 
 // Create a new radix tree and reset if it exists
-let mut tree = RadixTree::new("/path/to/storage", true)?;
+let mut tree = RadixTree::new("/tmp/radix", true)?;
 ```
 
 ### Setting Values

radixtree/examples/large_scale_test.rs

@@ -0,0 +1,121 @@
+use radixtree::RadixTree;
+use std::time::{Duration, Instant};
+use std::io::{self, Write};
+
+// Use much smaller batches to avoid hitting OurDB's size limit
+const BATCH_SIZE: usize = 1_000;
+const NUM_BATCHES: usize = 1_000;  // Total records: 1,000,000
+const PROGRESS_INTERVAL: usize = 100;
+
+fn main() -> Result<(), radixtree::Error> {
+    // Overall metrics
+    let total_start_time = Instant::now();
+    let mut total_records_inserted = 0;
+    let mut batch_times = Vec::with_capacity(NUM_BATCHES);
+    
+    println!("Will insert up to {} records in batches of {}", 
+             BATCH_SIZE * NUM_BATCHES, BATCH_SIZE);
+    
+    // Process in batches to avoid OurDB size limits
+    for batch in 0..NUM_BATCHES {
+        // Create a new database for each batch
+        let batch_path = std::env::temp_dir().join(format!("radixtree_batch_{}", batch));
+        
+        // Clean up any existing database
+        if batch_path.exists() {
+            std::fs::remove_dir_all(&batch_path)?;
+        }
+        std::fs::create_dir_all(&batch_path)?;
+        
+        println!("\nBatch {}/{}: Creating new radix tree...", batch + 1, NUM_BATCHES);
+        let mut tree = RadixTree::new(batch_path.to_str().unwrap(), true)?;
+        
+        let batch_start_time = Instant::now();
+        let mut last_progress_time = Instant::now();
+        let mut last_progress_count = 0;
+        
+        // Insert records for this batch
+        for i in 0..BATCH_SIZE {
+            let global_index = batch * BATCH_SIZE + i;
+            let key = format!("key:{:08}", global_index);
+            let value = format!("val{}", global_index).into_bytes();
+            
+            tree.set(&key, value)?;
+            
+            // Show progress at intervals
+            if (i + 1) % PROGRESS_INTERVAL == 0 || i == BATCH_SIZE - 1 {
+                let records_since_last = i + 1 - last_progress_count;
+                let time_since_last = last_progress_time.elapsed();
+                let records_per_second = records_since_last as f64 / time_since_last.as_secs_f64();
+                
+                print!("\rProgress: {}/{} records ({:.2}%) - {:.2} records/sec", 
+                       i + 1, BATCH_SIZE, 
+                       (i + 1) as f64 / BATCH_SIZE as f64 * 100.0,
+                       records_per_second);
+                io::stdout().flush().unwrap();
+                
+                last_progress_time = Instant::now();
+                last_progress_count = i + 1;
+            }
+        }
+        
+        let batch_duration = batch_start_time.elapsed();
+        batch_times.push(batch_duration);
+        total_records_inserted += BATCH_SIZE;
+        
+        println!("\nBatch {}/{} completed in {:?} ({:.2} records/sec)", 
+                 batch + 1, NUM_BATCHES, 
+                 batch_duration,
+                 BATCH_SIZE as f64 / batch_duration.as_secs_f64());
+        
+        // Test random access performance for this batch
+        println!("Testing access performance for batch {}...", batch + 1);
+        let mut total_get_time = Duration::new(0, 0);
+        let num_samples = 100;
+        
+        // Use a simple distribution pattern
+        for i in 0..num_samples {
+            // Distribute samples across the batch
+            let sample_id = batch * BATCH_SIZE + (i * (BATCH_SIZE / num_samples));
+            let key = format!("key:{:08}", sample_id);
+            
+            let get_start = Instant::now();
+            let _ = tree.get(&key)?;
+            total_get_time += get_start.elapsed();
+        }
+        
+        println!("Average time to retrieve a record: {:?}", 
+                 total_get_time / num_samples as u32);
+                 
+        // Test prefix search performance
+        println!("Testing prefix search performance...");
+        let prefix = format!("key:{:02}", batch % 100);
+        
+        let list_start = Instant::now();
+        let keys = tree.list(&prefix)?;
+        let list_duration = list_start.elapsed();
+        
+        println!("Found {} keys with prefix '{}' in {:?}", 
+                 keys.len(), prefix, list_duration);
+    }
+    
+    // Overall performance summary
+    let total_duration = total_start_time.elapsed();
+    println!("\n\nPerformance Summary:");
+    println!("Total time to insert {} records: {:?}", total_records_inserted, total_duration);
+    println!("Average insertion rate: {:.2} records/second", 
+             total_records_inserted as f64 / total_duration.as_secs_f64());
+    
+    // Show performance trend
+    println!("\nPerformance Trend (batch number vs. time):");
+    for (i, duration) in batch_times.iter().enumerate() {
+        if i % 10 == 0 || i == batch_times.len() - 1 {  // Only show every 10th point
+            println!("  Batch {}: {:?} ({:.2} records/sec)", 
+                     i + 1, 
+                     duration,
+                     BATCH_SIZE as f64 / duration.as_secs_f64());
+        }
+    }
+    
+    Ok(())
+}

radixtree/examples/performance_test.rs

@@ -0,0 +1,134 @@
+use radixtree::RadixTree;
+use std::time::{Duration, Instant};
+use std::io::{self, Write};
+
+// Number of records to insert
+const TOTAL_RECORDS: usize = 1_000_000;
+// How often to report progress (every X records)
+const PROGRESS_INTERVAL: usize = 10_000;
+// How many records to use for performance sampling
+const PERFORMANCE_SAMPLE_SIZE: usize = 1000;
+
+fn main() -> Result<(), radixtree::Error> {
+    // Create a temporary directory for the database
+    let db_path = std::env::temp_dir().join("radixtree_performance_test");
+    
+    // Completely remove and recreate the directory to ensure a clean start
+    if db_path.exists() {
+        std::fs::remove_dir_all(&db_path)?;
+    }
+    std::fs::create_dir_all(&db_path)?;
+    
+    println!("Creating radix tree at: {}", db_path.display());
+    println!("Will insert {} records and show progress...", TOTAL_RECORDS);
+    
+    // Create a new radix tree
+    let mut tree = RadixTree::new(db_path.to_str().unwrap(), true)?;
+    
+    // Track overall time
+    let start_time = Instant::now();
+    
+    // Track performance metrics
+    let mut insertion_times = Vec::with_capacity(TOTAL_RECORDS / PROGRESS_INTERVAL);
+    let mut last_batch_time = Instant::now();
+    let mut last_batch_records = 0;
+    
+    // Insert records and track progress
+    for i in 0..TOTAL_RECORDS {
+        let key = format!("key:{:08}", i);
+        // Use smaller values to avoid exceeding OurDB's size limit
+        let value = format!("val{}", i).into_bytes();
+        
+        // Time the insertion of every Nth record for performance sampling
+        if i % PERFORMANCE_SAMPLE_SIZE == 0 {
+            let insert_start = Instant::now();
+            tree.set(&key, value)?;
+            let insert_duration = insert_start.elapsed();
+            
+            // Only print detailed timing for specific samples to avoid flooding output
+            if i % (PERFORMANCE_SAMPLE_SIZE * 10) == 0 {
+                println!("Record {}: Insertion took {:?}", i, insert_duration);
+            }
+        } else {
+            tree.set(&key, value)?;
+        }
+        
+        // Show progress at intervals
+        if (i + 1) % PROGRESS_INTERVAL == 0 || i == TOTAL_RECORDS - 1 {
+            let records_in_batch = i + 1 - last_batch_records;
+            let batch_duration = last_batch_time.elapsed();
+            let records_per_second = records_in_batch as f64 / batch_duration.as_secs_f64();
+            
+            insertion_times.push((i + 1, batch_duration));
+            
+            print!("\rProgress: {}/{} records ({:.2}%) - {:.2} records/sec", 
+                   i + 1, TOTAL_RECORDS, 
+                   (i + 1) as f64 / TOTAL_RECORDS as f64 * 100.0,
+                   records_per_second);
+            io::stdout().flush().unwrap();
+            
+            last_batch_time = Instant::now();
+            last_batch_records = i + 1;
+        }
+    }
+    
+    let total_duration = start_time.elapsed();
+    println!("\n\nPerformance Summary:");
+    println!("Total time to insert {} records: {:?}", TOTAL_RECORDS, total_duration);
+    println!("Average insertion rate: {:.2} records/second", 
+             TOTAL_RECORDS as f64 / total_duration.as_secs_f64());
+    
+    // Show performance trend
+    println!("\nPerformance Trend (records inserted vs. time per batch):");
+    for (i, (record_count, duration)) in insertion_times.iter().enumerate() {
+        if i % 10 == 0 || i == insertion_times.len() - 1 {  // Only show every 10th point to avoid too much output
+            println!("  After {} records: {:?} for {} records ({:.2} records/sec)", 
+                     record_count, 
+                     duration,
+                     PROGRESS_INTERVAL,
+                     PROGRESS_INTERVAL as f64 / duration.as_secs_f64());
+        }
+    }
+    
+    // Test access performance with distributed samples
+    println!("\nTesting access performance with distributed samples...");
+    let mut total_get_time = Duration::new(0, 0);
+    let num_samples = 1000;
+    
+    // Use a simple distribution pattern instead of random
+    for i in 0..num_samples {
+        // Distribute samples across the entire range
+        let sample_id = (i * (TOTAL_RECORDS / num_samples)) % TOTAL_RECORDS;
+        let key = format!("key:{:08}", sample_id);
+        
+        let get_start = Instant::now();
+        let _ = tree.get(&key)?;
+        total_get_time += get_start.elapsed();
+    }
+    
+    println!("Average time to retrieve a record: {:?}",
+             total_get_time / num_samples as u32);
+    
+    // Test prefix search performance
+    println!("\nTesting prefix search performance...");
+    let prefixes = ["key:0", "key:1", "key:5", "key:9"];
+    
+    for prefix in &prefixes {
+        let list_start = Instant::now();
+        let keys = tree.list(prefix)?;
+        let list_duration = list_start.elapsed();
+        
+        println!("Found {} keys with prefix '{}' in {:?}", 
+                 keys.len(), prefix, list_duration);
+    }
+    
+    // Clean up (optional)
+    if std::env::var("KEEP_DB").is_err() {
+        std::fs::remove_dir_all(&db_path)?;
+        println!("\nCleaned up database directory");
+    } else {
+        println!("\nDatabase kept at: {}", db_path.display());
+    }
+    
+    Ok(())
+}

radixtree/src/operations.rs

@@ -13,8 +13,8 @@ pub fn new_radix_tree(path: &str, reset: bool) -> Result<RadixTree, Error> {
     let config = OurDBConfig {
         path: PathBuf::from(path),
         incremental_mode: true,
-        file_size: Some(1024 * 1024), // 1MB file size
-        keysize: Some(4), // Default key size
+        file_size: Some(1024 * 1024 * 10), // 10MB file size for better performance with large datasets
+        keysize: Some(6), // Use keysize=6 to support multiple files (file_nr + position)
     };
     
     let mut db = OurDB::new(config)?;