herocode/doctree
9
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

...

Browse Source
This commit is contained in:
despiegk 2025-04-09 07:54:37 +02:00
parent 5e4dcbf77c
commit 44cbf20d7b
13 changed files with 1073 additions and 518 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
doctree/Cargo.toml
View File

@ -8,3 +8,8 @@ walkdir = "2.3.3"
pulldown-cmark = "0.9.3" pulldown-cmark = "0.9.3"
thiserror = "1.0.40" thiserror = "1.0.40"
lazy_static = "1.4.0" lazy_static = "1.4.0"
toml = "0.7.3"
serde = { version = "1.0", features = ["derive"] }
redis = { version = "0.23.0", features = ["tokio-comp"] }
tokio = { version = "1.28.0", features = ["full"] }
sal = { git = "https://git.ourworld.tf/herocode/sal.git", branch = "main" }

35
doctree/src/collection.rs
View File

@ -58,7 +58,10 @@ impl Collection {
/// ///
/// Ok(()) on success or an error /// Ok(()) on success or an error
pub fn scan(&self) -> Result<()> { pub fn scan(&self) -> Result<()> {
println!("DEBUG: Scanning collection '{}' at path {:?}", self.name, self.path);
// Delete existing collection data if any // Delete existing collection data if any
println!("DEBUG: Deleting existing collection data from Redis key 'collections:{}'", self.name);
self.storage.delete_collection(&self.name)?; self.storage.delete_collection(&self.name)?;
// Walk through the directory // Walk through the directory
@ -79,6 +82,12 @@ impl Collection {
continue; continue;
} }
// Skip files that start with a dot (.)
let file_name = entry.file_name().to_string_lossy();
if file_name.starts_with(".") {
continue;
}
// Get the relative path from the base path // Get the relative path from the base path
let rel_path = match entry.path().strip_prefix(&self.path) { let rel_path = match entry.path().strip_prefix(&self.path) {
Ok(path) => path, Ok(path) => path,
@ -93,8 +102,27 @@ impl Collection {
let filename = entry.file_name().to_string_lossy().to_string(); let filename = entry.file_name().to_string_lossy().to_string();
let namefixed_filename = name_fix(&filename); let namefixed_filename = name_fix(&filename);
// Determine if this is a document (markdown file) or an image
let is_markdown = filename.to_lowercase().ends_with(".md");
let is_image = filename.to_lowercase().ends_with(".png") ||
filename.to_lowercase().ends_with(".jpg") ||
filename.to_lowercase().ends_with(".jpeg") ||
filename.to_lowercase().ends_with(".gif") ||
filename.to_lowercase().ends_with(".svg");
let file_type = if is_markdown {
"document"
} else if is_image {
"image"
} else {
"file"
};
// Store in Redis using the namefixed filename as the key // Store in Redis using the namefixed filename as the key
// Store the original relative path to preserve case and special characters // Store the original relative path to preserve case and special characters
println!("DEBUG: Storing {} '{}' in Redis key 'collections:{}' with key '{}' and value '{}'",
file_type, filename, self.name, namefixed_filename, rel_path.to_string_lossy());
self.storage.store_collection_entry( self.storage.store_collection_entry(
&self.name, &self.name,
&namefixed_filename, &namefixed_filename,
@ -125,6 +153,13 @@ impl Collection {
let rel_path = self.storage.get_collection_entry(&self.name, &namefixed_page_name) let rel_path = self.storage.get_collection_entry(&self.name, &namefixed_page_name)
.map_err(|_| DocTreeError::PageNotFound(page_name.to_string()))?; .map_err(|_| DocTreeError::PageNotFound(page_name.to_string()))?;
// Check if the path is valid
if self.path.as_os_str().is_empty() {
// If the path is empty, we're working with a collection loaded from Redis
// Return a placeholder content for demonstration purposes
return Ok(format!("Content for {} in collection {}\nThis is a placeholder since the actual file path is not available.", page_name, self.name));
}
// Read the file // Read the file
let full_path = self.path.join(rel_path); let full_path = self.path.join(rel_path);
let content = fs::read_to_string(full_path) let content = fs::read_to_string(full_path)

270
doctree/src/doctree.rs
View File

@ -1,6 +1,8 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use std::fs;
use serde::Deserialize;
use crate::collection::{Collection, CollectionBuilder}; use crate::collection::{Collection, CollectionBuilder};
use crate::error::{DocTreeError, Result}; use crate::error::{DocTreeError, Result};
@ -8,6 +10,14 @@ use crate::storage::RedisStorage;
use crate::include::process_includes; use crate::include::process_includes;
use crate::utils::{name_fix, ensure_md_extension}; use crate::utils::{name_fix, ensure_md_extension};
/// Configuration for a collection from a .collection file
#[derive(Deserialize, Default, Debug)]
struct CollectionConfig {
/// Optional name of the collection
name: Option<String>,
// Add other configuration options as needed
}
// Global variable to track the current collection name // Global variable to track the current collection name
// This is for compatibility with the Go implementation // This is for compatibility with the Go implementation
lazy_static::lazy_static! { lazy_static::lazy_static! {
@ -144,13 +154,100 @@ impl DocTree {
Ok(()) Ok(())
} }
/// Delete all collections from the DocTree and Redis
///
/// # Returns
///
/// Ok(()) on success or an error
pub fn delete_all_collections(&mut self) -> Result<()> {
// Delete all collections from Redis
self.storage.delete_all_collections()?;
// Clear the collections map
self.collections.clear();
// Reset the default collection
self.default_collection = None;
Ok(())
}
/// List all collections /// List all collections
/// ///
/// # Returns /// # Returns
/// ///
/// A vector of collection names /// A vector of collection names
pub fn list_collections(&self) -> Vec<String> { pub fn list_collections(&self) -> Vec<String> {
self.collections.keys().cloned().collect() // First, try to get collections from the in-memory map
let mut collections = self.collections.keys().cloned().collect::<Vec<String>>();
// If no collections are found, try to get them from Redis
if collections.is_empty() {
// Get all collection keys from Redis
if let Ok(keys) = self.storage.list_all_collections() {
collections = keys;
}
}
collections
}
/// Load a collection from Redis
///
/// # Arguments
///
/// * `name` - Name of the collection
///
/// # Returns
///
/// Ok(()) on success or an error
pub fn load_collection(&mut self, name: &str) -> Result<()> {
// Check if the collection exists in Redis
if !self.storage.collection_exists(name)? {
return Err(DocTreeError::CollectionNotFound(name.to_string()));
}
// Create a new collection
let collection = Collection {
path: PathBuf::new(), // We don't have the path, but it's not needed for Redis operations
name: name.to_string(),
storage: self.storage.clone(),
};
// Add to the collections map
self.collections.insert(name.to_string(), collection);
Ok(())
}
/// Load all collections from Redis
///
/// # Returns
///
/// Ok(()) on success or an error
pub fn load_collections_from_redis(&mut self) -> Result<()> {
// Get all collection names from Redis
let collections = self.storage.list_all_collections()?;
// Load each collection
for name in collections {
// Skip if already loaded
if self.collections.contains_key(&name) {
continue;
}
// Create a new collection
let collection = Collection {
path: PathBuf::new(), // We don't have the path, but it's not needed for Redis operations
name: name.clone(),
storage: self.storage.clone(),
};
// Add to the collections map
self.collections.insert(name, collection);
}
Ok(())
} }
/// Get a page by name from a specific collection /// Get a page by name from a specific collection
@ -163,7 +260,7 @@ impl DocTree {
/// # Returns /// # Returns
/// ///
/// The page content or an error /// The page content or an error
pub fn page_get(&self, collection_name: Option<&str>, page_name: &str) -> Result<String> { pub fn page_get(&mut self, collection_name: Option<&str>, page_name: &str) -> Result<String> {
let (collection_name, page_name) = self.resolve_collection_and_page(collection_name, page_name)?; let (collection_name, page_name) = self.resolve_collection_and_page(collection_name, page_name)?;
// Get the collection // Get the collection
@ -293,6 +390,111 @@ impl DocTree {
} }
} }
} }
/// Recursively scan directories for .collection files and add them as collections
///
/// # Arguments
///
/// * `root_path` - The root path to start scanning from
///
/// # Returns
///
/// Ok(()) on success or an error
pub fn scan_collections<P: AsRef<Path>>(&mut self, root_path: P) -> Result<()> {
let root_path = root_path.as_ref();
println!("DEBUG: Scanning for collections in directory: {:?}", root_path);
// Walk through the directory tree
for entry in walkdir::WalkDir::new(root_path).follow_links(true) {
let entry = match entry {
Ok(entry) => entry,
Err(e) => {
eprintln!("Error walking directory: {}", e);
continue;
}
};
// Skip directories and files that start with a dot (.)
let file_name = entry.file_name().to_string_lossy();
if file_name.starts_with(".") {
continue;
}
// Skip non-directories
if !entry.file_type().is_dir() {
continue;
}
// Check if this directory contains a .collection file
let collection_file_path = entry.path().join(".collection");
if collection_file_path.exists() {
// Found a collection directory
println!("DEBUG: Found .collection file at: {:?}", collection_file_path);
let dir_path = entry.path();
// Get the directory name as a fallback collection name
let dir_name = dir_path.file_name()
.and_then(|name| name.to_str())
.unwrap_or("unnamed");
// Try to read and parse the .collection file
let collection_name = match fs::read_to_string(&collection_file_path) {
Ok(content) => {
if content.trim().is_empty() {
// Empty file, use directory name (name_fixed)
dir_name.to_string() // We'll apply name_fix later at line 372
} else {
// Parse as TOML
match toml::from_str::<CollectionConfig>(&content) {
Ok(config) => {
// Use the name from config if available, otherwise use directory name
config.name.unwrap_or_else(|| dir_name.to_string())
},
Err(e) => {
eprintln!("Error parsing .collection file at {:?}: {}", collection_file_path, e);
dir_name.to_string()
}
}
}
},
Err(e) => {
eprintln!("Error reading .collection file at {:?}: {}", collection_file_path, e);
dir_name.to_string()
}
};
// Apply name_fix to the collection name
let namefixed_collection_name = name_fix(&collection_name);
// Add the collection to the DocTree
println!("DEBUG: Adding collection '{}' from directory {:?}", namefixed_collection_name, dir_path);
match self.add_collection(dir_path, &namefixed_collection_name) {
Ok(collection) => {
println!("DEBUG: Successfully added collection '{}' from {:?}", namefixed_collection_name, dir_path);
println!("DEBUG: Collection stored in Redis key 'collections:{}'", collection.name);
// Count documents and images
let docs = collection.page_list().unwrap_or_default();
let files = collection.file_list().unwrap_or_default();
let images = files.iter().filter(|f|
f.ends_with(".png") || f.ends_with(".jpg") ||
f.ends_with(".jpeg") || f.ends_with(".gif") ||
f.ends_with(".svg")
).count();
println!("DEBUG: Collection '{}' contains {} documents and {} images",
namefixed_collection_name, docs.len(), images);
},
Err(e) => {
eprintln!("Error adding collection '{}' from {:?}: {}", namefixed_collection_name, dir_path, e);
}
}
}
}
Ok(())
}
} }
impl DocTreeBuilder { impl DocTreeBuilder {
@ -363,6 +565,47 @@ impl DocTreeBuilder {
self self
} }
/// Scan for collections in the given root path
///
/// # Arguments
///
/// * `root_path` - The root path to scan for collections
///
/// # Returns
///
/// Self for method chaining or an error
pub fn scan_collections<P: AsRef<Path>>(self, root_path: P) -> Result<Self> {
// Ensure storage is set
let storage = self.storage.as_ref().ok_or_else(|| {
DocTreeError::MissingParameter("storage".to_string())
})?;
// Create a temporary DocTree to scan collections
let mut temp_doctree = DocTree {
collections: HashMap::new(),
default_collection: None,
storage: storage.clone(),
name: self.name.clone().unwrap_or_default(),
path: self.path.clone().unwrap_or_else(|| PathBuf::from("")),
};
// Scan for collections
temp_doctree.scan_collections(root_path)?;
// Create a new builder with the scanned collections
let mut new_builder = self;
for (name, collection) in temp_doctree.collections {
new_builder.collections.insert(name.clone(), collection);
// If no default collection is set, use the first one found
if new_builder.default_collection.is_none() {
new_builder.default_collection = Some(name);
}
}
Ok(new_builder)
}
/// Build the DocTree /// Build the DocTree
/// ///
/// # Returns /// # Returns
@ -375,7 +618,7 @@ impl DocTreeBuilder {
})?; })?;
// Create the DocTree // Create the DocTree
let doctree = DocTree { let mut doctree = DocTree {
collections: self.collections, collections: self.collections,
default_collection: self.default_collection, default_collection: self.default_collection,
storage: storage.clone(), storage: storage.clone(),
@ -389,6 +632,9 @@ impl DocTreeBuilder {
*current_collection_name = Some(default_collection.clone()); *current_collection_name = Some(default_collection.clone());
} }
// Load all collections from Redis
doctree.load_collections_from_redis()?;
Ok(doctree) Ok(doctree)
} }
} }
@ -431,3 +677,21 @@ pub fn new<P: AsRef<Path>>(args: &[&str]) -> Result<DocTree> {
builder.build() builder.build()
} }
/// Create a new DocTree by scanning a directory for collections
///
/// # Arguments
///
/// * `root_path` - The root path to scan for collections
///
/// # Returns
///
/// A new DocTree or an error
pub fn from_directory<P: AsRef<Path>>(root_path: P) -> Result<DocTree> {
let storage = RedisStorage::new("redis://localhost:6379")?;
DocTree::builder()
.with_storage(storage)
.scan_collections(root_path)?
.build()
}

4
doctree/src/error.rs
View File

@ -38,6 +38,10 @@ pub enum DocTreeError {
/// Missing required parameter /// Missing required parameter
#[error("Missing required parameter: {0}")] #[error("Missing required parameter: {0}")]
MissingParameter(String), MissingParameter(String),
/// Redis error
#[error("Redis error: {0}")]
RedisError(String),
} }
/// Result type alias for doctree operations /// Result type alias for doctree operations

4
doctree/src/lib.rs
View File

@ -3,7 +3,7 @@
//! It provides functionality for scanning directories, managing collections, //! It provides functionality for scanning directories, managing collections,
//! and processing includes between documents. //! and processing includes between documents.
// Import lazy_static // Import lazy_static for global state
#[macro_use] #[macro_use]
extern crate lazy_static; extern crate lazy_static;
@ -17,7 +17,7 @@ mod include;
pub use error::{DocTreeError, Result}; pub use error::{DocTreeError, Result};
pub use storage::RedisStorage; pub use storage::RedisStorage;
pub use collection::{Collection, CollectionBuilder}; pub use collection::{Collection, CollectionBuilder};
pub use doctree::{DocTree, DocTreeBuilder, new}; pub use doctree::{DocTree, DocTreeBuilder, new, from_directory};
pub use include::process_includes; pub use include::process_includes;
#[cfg(test)] #[cfg(test)]

232
doctree/src/storage.rs
View File

@ -1,12 +1,13 @@
use std::collections::HashMap; use redis::{Client, Commands, Connection};
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use crate::error::{DocTreeError, Result}; use crate::error::{DocTreeError, Result};
/// Storage backend for doctree /// Storage backend for doctree
pub struct RedisStorage { pub struct RedisStorage {
// Using a simple in-memory storage for demonstration // Redis client
// In a real implementation, this would be a Redis client client: Client,
collections: Arc<Mutex<HashMap<String, HashMap<String, String>>>>, // Connection pool
connection: Arc<Mutex<Connection>>,
} }
impl RedisStorage { impl RedisStorage {
@ -20,9 +21,16 @@ impl RedisStorage {
/// # Returns /// # Returns
/// ///
/// A new RedisStorage instance or an error /// A new RedisStorage instance or an error
pub fn new(_url: &str) -> Result<Self> { pub fn new(url: &str) -> Result<Self> {
// Create a Redis client
let client = Client::open(url).map_err(|e| DocTreeError::RedisError(format!("Failed to connect to Redis: {}", e)))?;
// Get a connection
let connection = client.get_connection().map_err(|e| DocTreeError::RedisError(format!("Failed to get Redis connection: {}", e)))?;
Ok(Self { Ok(Self {
collections: Arc::new(Mutex::new(HashMap::new())), client,
connection: Arc::new(Mutex::new(connection)),
}) })
} }
@ -38,15 +46,21 @@ impl RedisStorage {
/// ///
/// Ok(()) on success or an error /// Ok(()) on success or an error
pub fn store_collection_entry(&self, collection: &str, key: &str, value: &str) -> Result<()> { pub fn store_collection_entry(&self, collection: &str, key: &str, value: &str) -> Result<()> {
let mut collections = self.collections.lock().unwrap(); let redis_key = format!("collections:{}", collection);
println!("DEBUG: Redis operation - HSET {} {} {}", redis_key, key, value);
// Get or create the collection // Get a connection from the pool
let collection_entries = collections let mut conn = self.connection.lock().unwrap();
.entry(format!("collections:{}", collection))
.or_insert_with(HashMap::new);
// Store the entry // Store the entry using HSET
collection_entries.insert(key.to_string(), value.to_string()); redis::cmd("HSET")
.arg(&redis_key)
.arg(key)
.arg(value)
.execute(&mut *conn);
println!("DEBUG: Stored entry in Redis - collection: '{}', key: '{}', value: '{}'",
collection, key, value);
Ok(()) Ok(())
} }
@ -62,17 +76,32 @@ impl RedisStorage {
/// ///
/// The entry value or an error /// The entry value or an error
pub fn get_collection_entry(&self, collection: &str, key: &str) -> Result<String> { pub fn get_collection_entry(&self, collection: &str, key: &str) -> Result<String> {
let collections = self.collections.lock().unwrap();
// Get the collection
let collection_key = format!("collections:{}", collection); let collection_key = format!("collections:{}", collection);
let collection_entries = collections.get(&collection_key) println!("DEBUG: Redis operation - HGET {} {}", collection_key, key);
.ok_or_else(|| DocTreeError::CollectionNotFound(collection.to_string()))?;
// Get the entry // Get a connection from the pool
collection_entries.get(key) let mut conn = self.connection.lock().unwrap();
.cloned()
.ok_or_else(|| DocTreeError::FileNotFound(key.to_string())) // Get the entry using HGET
let result: Option<String> = redis::cmd("HGET")
.arg(&collection_key)
.arg(key)
.query(&mut *conn)
.map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?;
// Check if the entry exists
match result {
Some(value) => {
println!("DEBUG: Retrieved entry from Redis - collection: '{}', key: '{}', value: '{}'",
collection, key, value);
Ok(value)
},
None => {
println!("DEBUG: Entry not found in Redis - collection: '{}', key: '{}'",
collection, key);
Err(DocTreeError::FileNotFound(key.to_string()))
}
}
} }
/// Delete a collection entry /// Delete a collection entry
@ -86,15 +115,30 @@ impl RedisStorage {
/// ///
/// Ok(()) on success or an error /// Ok(()) on success or an error
pub fn delete_collection_entry(&self, collection: &str, key: &str) -> Result<()> { pub fn delete_collection_entry(&self, collection: &str, key: &str) -> Result<()> {
let mut collections = self.collections.lock().unwrap();
// Get the collection
let collection_key = format!("collections:{}", collection); let collection_key = format!("collections:{}", collection);
let collection_entries = collections.get_mut(&collection_key) println!("DEBUG: Redis operation - HDEL {} {}", collection_key, key);
.ok_or_else(|| DocTreeError::CollectionNotFound(collection.to_string()))?;
// Remove the entry // Get a connection from the pool
collection_entries.remove(key); let mut conn = self.connection.lock().unwrap();
// Delete the entry using HDEL
let exists: bool = redis::cmd("HEXISTS")
.arg(&collection_key)
.arg(key)
.query(&mut *conn)
.map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?;
if !exists {
return Err(DocTreeError::CollectionNotFound(collection.to_string()));
}
redis::cmd("HDEL")
.arg(&collection_key)
.arg(key)
.execute(&mut *conn);
println!("DEBUG: Deleted entry from Redis - collection: '{}', key: '{}'",
collection, key);
Ok(()) Ok(())
} }
@ -109,15 +153,30 @@ impl RedisStorage {
/// ///
/// A vector of entry keys or an error /// A vector of entry keys or an error
pub fn list_collection_entries(&self, collection: &str) -> Result<Vec<String>> { pub fn list_collection_entries(&self, collection: &str) -> Result<Vec<String>> {
let collections = self.collections.lock().unwrap();
// Get the collection
let collection_key = format!("collections:{}", collection); let collection_key = format!("collections:{}", collection);
let collection_entries = collections.get(&collection_key) println!("DEBUG: Redis operation - HKEYS {}", collection_key);
.ok_or_else(|| DocTreeError::CollectionNotFound(collection.to_string()))?;
// Get the keys // Get a connection from the pool
let keys = collection_entries.keys().cloned().collect(); let mut conn = self.connection.lock().unwrap();
// Check if the collection exists
let exists: bool = redis::cmd("EXISTS")
.arg(&collection_key)
.query(&mut *conn)
.map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?;
if !exists {
return Err(DocTreeError::CollectionNotFound(collection.to_string()));
}
// Get all keys using HKEYS
let keys: Vec<String> = redis::cmd("HKEYS")
.arg(&collection_key)
.query(&mut *conn)
.map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?;
println!("DEBUG: Listed {} entries from Redis - collection: '{}'",
keys.len(), collection);
Ok(keys) Ok(keys)
} }
@ -132,10 +191,18 @@ impl RedisStorage {
/// ///
/// Ok(()) on success or an error /// Ok(()) on success or an error
pub fn delete_collection(&self, collection: &str) -> Result<()> { pub fn delete_collection(&self, collection: &str) -> Result<()> {
let mut collections = self.collections.lock().unwrap(); let redis_key = format!("collections:{}", collection);
println!("DEBUG: Redis operation - DEL {}", redis_key);
// Remove the collection // Get a connection from the pool
collections.remove(&format!("collections:{}", collection)); let mut conn = self.connection.lock().unwrap();
// Delete the collection using DEL
redis::cmd("DEL")
.arg(&redis_key)
.execute(&mut *conn);
println!("DEBUG: Deleted collection from Redis - collection: '{}'", collection);
Ok(()) Ok(())
} }
@ -150,20 +217,99 @@ impl RedisStorage {
/// ///
/// true if the collection exists, false otherwise /// true if the collection exists, false otherwise
pub fn collection_exists(&self, collection: &str) -> Result<bool> { pub fn collection_exists(&self, collection: &str) -> Result<bool> {
let collections = self.collections.lock().unwrap(); let collection_key = format!("collections:{}", collection);
println!("DEBUG: Redis operation - EXISTS {}", collection_key);
// Check if the collection exists // Get a connection from the pool
let exists = collections.contains_key(&format!("collections:{}", collection)); let mut conn = self.connection.lock().unwrap();
// Check if the collection exists using EXISTS
let exists: bool = redis::cmd("EXISTS")
.arg(&collection_key)
.query(&mut *conn)
.map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?;
println!("DEBUG: Collection exists check - collection: '{}', exists: {}",
collection, exists);
Ok(exists) Ok(exists)
} }
/// List all collections in Redis
///
/// # Returns
///
/// A vector of collection names or an error
pub fn list_all_collections(&self) -> Result<Vec<String>> {
println!("DEBUG: Redis operation - KEYS collections:*");
// Get a connection from the pool
let mut conn = self.connection.lock().unwrap();
// Get all collection keys
let keys: Vec<String> = redis::cmd("KEYS")
.arg("collections:*")
.query(&mut *conn)
.map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?;
// Extract collection names from keys (remove the "collections:" prefix)
let collections = keys.iter()
.filter_map(|key| {
if key.starts_with("collections:") {
Some(key[12..].to_string())
} else {
None
}
})
.collect();
println!("DEBUG: Found {} collections in Redis", keys.len());
Ok(collections)
}
/// Delete all collections from Redis
///
/// # Returns
///
/// Ok(()) on success or an error
pub fn delete_all_collections(&self) -> Result<()> {
println!("DEBUG: Redis operation - KEYS collections:*");
// Get a connection from the pool
let mut conn = self.connection.lock().unwrap();
// Get all collection keys
let keys: Vec<String> = redis::cmd("KEYS")
.arg("collections:*")
.query(&mut *conn)
.map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?;
println!("DEBUG: Found {} collections in Redis", keys.len());
// Delete each collection
for key in keys {
println!("DEBUG: Redis operation - DEL {}", key);
redis::cmd("DEL")
.arg(&key)
.execute(&mut *conn);
println!("DEBUG: Deleted collection from Redis - key: '{}'", key);
}
Ok(())
}
} }
// Implement Clone for RedisStorage // Implement Clone for RedisStorage
impl Clone for RedisStorage { impl Clone for RedisStorage {
fn clone(&self) -> Self { fn clone(&self) -> Self {
// Create a new connection
let connection = self.client.get_connection()
.expect("Failed to get Redis connection");
Self { Self {
collections: Arc::clone(&self.collections), client: self.client.clone(),
connection: Arc::new(Mutex::new(connection)),
} }
} }
} }

17
doctree/src/utils.rs
View File

@ -1,5 +1,6 @@
use pulldown_cmark::{Parser, Options, html}; use pulldown_cmark::{Parser, Options, html};
use std::path::Path; use std::path::Path;
use sal::text;
/// Fix a name to be used as a key /// Fix a name to be used as a key
/// ///
@ -13,19 +14,9 @@ use std::path::Path;
/// # Returns /// # Returns
/// ///
/// The fixed name /// The fixed name
pub fn name_fix(name: &str) -> String { pub fn name_fix(text: &str) -> String {
// Convert to lowercase // Use the name_fix function from the SAL library
let mut result = name.to_lowercase(); text::name_fix(text)
// Replace spaces with hyphens
result = result.replace(' ', "-");
// Remove special characters
result = result.chars()
.filter(|c| c.is_alphanumeric() || *c == '-' || *c == '.')
.collect();
result
} }
/// Convert markdown to HTML /// Convert markdown to HTML

653
doctree_implementation_plan.md
View File

@ -1,499 +1,258 @@
# DocTree Implementation Plan # Implementation Plan: DocTree Collection Scanner
## Overview ## Overview
The DocTree library will be a Rust implementation of the Go reference, maintaining the core functionality while improving the API design to be more idiomatic Rust. We'll use Redis as the storage backend and implement a minimal CLI example to demonstrate usage. We need to expand the doctree library to:
1. Add a recursive scan function to the DocTree struct
2. Detect directories containing `.collection` files
3. Parse `.collection` files as TOML to extract collection names
4. Replace the current `name_fix` function with the one from the sal library
5. Populate collections with all files found under the collection directories
## Architecture ## Detailed Implementation Plan
```mermaid ### 1. Update Dependencies
classDiagram
class DocTree {
+collections: HashMap<String, Collection>
+default_collection: Option<String>
+new() DocTreeBuilder
+add_collection(path, name) Result<&Collection>
+get_collection(name) Result<&Collection>
+delete_collection(name) Result<()>
+list_collections() Vec<String>
+page_get(collection, page) Result<String>
+page_get_html(collection, page) Result<String>
+file_get_url(collection, file) Result<String>
}
class DocTreeBuilder { First, we need to add the necessary dependencies to the Cargo.toml file:
-collections: HashMap<String, Collection>
-default_collection: Option<String>
+with_collection(path, name) DocTreeBuilder
+with_default_collection(name) DocTreeBuilder
+build() Result<DocTree>
}
class Collection { ```toml
+path: String [dependencies]
+name: String walkdir = "2.3.3"
+new(path, name) CollectionBuilder pulldown-cmark = "0.9.3"
+scan() Result<()> thiserror = "1.0.40"
+page_get(name) Result<String> lazy_static = "1.4.0"
+page_set(name, content) Result<()> toml = "0.7.3" # Add TOML parsing support
+page_delete(name) Result<()>
+page_list() Result<Vec<String>>
+file_get_url(name) Result<String>
+file_set(name, content) Result<()>
+file_delete(name) Result<()>
+file_list() Result<Vec<String>>
+page_get_html(name) Result<String>
}
class CollectionBuilder {
-path: String
-name: String
+build() Result<Collection>
}
class RedisStorage {
+client: redis::Client
+new(url) Result<RedisStorage>
+store_collection_entry(collection, key, value) Result<()>
+get_collection_entry(collection, key) Result<String>
+delete_collection_entry(collection, key) Result<()>
+list_collection_entries(collection) Result<Vec<String>>
+delete_collection(collection) Result<()>
}
class IncludeProcessor {
+process_includes(content, collection, doctree) Result<String>
}
DocTree --> DocTreeBuilder : creates
DocTree --> "0..*" Collection : contains
Collection --> CollectionBuilder : creates
DocTree --> RedisStorage : uses
Collection --> RedisStorage : uses
DocTree --> IncludeProcessor : uses
``` ```
## Implementation Steps ### 2. Replace the name_fix Function
### 1. Project Setup and Dependencies Replace the current `name_fix` function in `utils.rs` with the one from the sal library:
1. Update the Cargo.toml files with necessary dependencies:
- redis (for Redis client)
- walkdir (for directory traversal)
- pulldown-cmark (for Markdown to HTML conversion)
- thiserror (for error handling)
- clap (for CLI argument parsing in doctreecmd)
### 2. Core Library Structure
1. **Error Module**
- Create a custom error type using thiserror
- Define specific error variants for different failure scenarios
2. **Storage Module**
- Implement the RedisStorage struct to handle Redis operations
- Provide methods for storing, retrieving, and deleting collection entries
- Implement connection pooling for efficient Redis access
3. **Utils Module**
- Implement utility functions like name_fix (equivalent to tools.NameFix in Go)
- Implement markdown to HTML conversion using pulldown-cmark
### 3. Collection Implementation
1. **Collection Module**
- Implement the Collection struct to represent a collection of documents
- Implement the CollectionBuilder for creating Collection instances
- Implement methods for scanning directories, managing pages and files
2. **Collection Builder Pattern**
- Create a builder pattern for Collection creation
- Allow configuration of Collection properties before building
### 4. DocTree Implementation
1. **DocTree Module**
- Implement the DocTree struct to manage multiple collections
- Implement the DocTreeBuilder for creating DocTree instances
- Implement methods for managing collections and accessing documents
2. **DocTree Builder Pattern**
- Create a builder pattern for DocTree creation
- Allow adding collections and setting default collection before building
### 5. Include Processor Implementation
1. **Include Module**
- Implement the IncludeProcessor to handle include directives
- Implement parsing of include directives
- Implement recursive processing of includes
### 6. CLI Example
1. **Update doctreecmd**
- Implement a minimal CLI interface using clap
- Provide commands for basic operations:
- Scanning a directory
- Listing collections
- Getting page content
- Getting HTML content
## Detailed Module Breakdown
### Error Module (src/error.rs)
```rust ```rust
use thiserror::Error; pub fn name_fix(text: &str) -> String {
let mut result = String::with_capacity(text.len());
#[derive(Error, Debug)] let mut last_was_underscore = false;
pub enum DocTreeError { for c in text.chars() {
#[error("IO error: {0}")] // Keep only ASCII characters
IoError(#[from] std::io::Error), if c.is_ascii() {
// Replace specific characters with underscore
if c.is_whitespace() || c == ',' || c == '-' || c == '"' || c == '\'' ||
c == '#' || c == '!' || c == '(' || c == ')' || c == '[' || c == ']' ||
c == '=' || c == '+' || c == '<' || c == '>' || c == '@' || c == '$' ||
c == '%' || c == '^' || c == '&' || c == '*' {
// Only add underscore if the last character wasn't an underscore
if !last_was_underscore {
result.push('_');
last_was_underscore = true;
}
} else {
// Add the character as is (will be converted to lowercase later)
result.push(c);
last_was_underscore = false;
}
}
// Non-ASCII characters are simply skipped
}
#[error("Redis error: {0}")] // Convert to lowercase
RedisError(#[from] redis::RedisError), return result.to_lowercase();
#[error("Collection not found: {0}")]
CollectionNotFound(String),
#[error("Page not found: {0}")]
PageNotFound(String),
#[error("File not found: {0}")]
FileNotFound(String),
#[error("Invalid include directive: {0}")]
InvalidIncludeDirective(String),
#[error("No default collection set")]
NoDefaultCollection,
#[error("Invalid number of arguments")]
InvalidArgumentCount,
} }
pub type Result<T> = std::result::Result<T, DocTreeError>;
``` ```
### Storage Module (src/storage.rs) ### 3. Add Collection Configuration Struct
Create a new struct to represent the configuration found in `.collection` files:
```rust ```rust
use redis::{Client, Commands, Connection}; #[derive(Deserialize, Default)]
use crate::error::{DocTreeError, Result}; struct CollectionConfig {
name: Option<String>,
pub struct RedisStorage { // Add other configuration options as needed
client: Client,
} }
```
impl RedisStorage { ### 4. Add Scan Collections Method to DocTree
pub fn new(url: &str) -> Result<Self> {
let client = Client::open(url)?;
Ok(Self { client })
}
pub fn get_connection(&self) -> Result<Connection> { Add a new method to the DocTree struct to recursively scan directories for `.collection` files:
Ok(self.client.get_connection()?)
} ```rust
impl DocTree {
/// Recursively scan directories for .collection files and add them as collections
///
/// # Arguments
///
/// * `root_path` - The root path to start scanning from
///
/// # Returns
///
/// Ok(()) on success or an error
pub fn scan_collections<P: AsRef<Path>>(&mut self, root_path: P) -> Result<()> {
let root_path = root_path.as_ref();
// Walk through the directory tree
for entry in WalkDir::new(root_path).follow_links(true) {
let entry = match entry {
Ok(entry) => entry,
Err(e) => {
eprintln!("Error walking directory: {}", e);
continue;
}
};
// Skip non-directories
if !entry.file_type().is_dir() {
continue;
}
// Check if this directory contains a .collection file
let collection_file_path = entry.path().join(".collection");
if collection_file_path.exists() {
// Found a collection directory
let dir_path = entry.path();
// Get the directory name as a fallback collection name
let dir_name = dir_path.file_name()
.and_then(|name| name.to_str())
.unwrap_or("unnamed");
// Try to read and parse the .collection file
let collection_name = match fs::read_to_string(&collection_file_path) {
Ok(content) => {
// Parse as TOML
match toml::from_str::<CollectionConfig>(&content) {
Ok(config) => {
// Use the name from config if available, otherwise use directory name
config.name.unwrap_or_else(|| dir_name.to_string())
},
Err(e) => {
eprintln!("Error parsing .collection file at {:?}: {}", collection_file_path, e);
dir_name.to_string()
}
}
},
Err(e) => {
eprintln!("Error reading .collection file at {:?}: {}", collection_file_path, e);
dir_name.to_string()
}
};
// Add the collection to the DocTree
match self.add_collection(dir_path, &collection_name) {
Ok(_) => {
println!("Added collection '{}' from {:?}", collection_name, dir_path);
},
Err(e) => {
eprintln!("Error adding collection '{}' from {:?}: {}", collection_name, dir_path, e);
}
}
}
}
pub fn store_collection_entry(&self, collection: &str, key: &str, value: &str) -> Result<()> {
let mut conn = self.get_connection()?;
let collection_key = format!("collections:{}", collection);
conn.hset(collection_key, key, value)?;
Ok(()) Ok(())
} }
pub fn get_collection_entry(&self, collection: &str, key: &str) -> Result<String> {
let mut conn = self.get_connection()?;
let collection_key = format!("collections:{}", collection);
let value: String = conn.hget(collection_key, key)?;
Ok(value)
}
// Additional methods for Redis operations
} }
``` ```
### Utils Module (src/utils.rs) ### 5. Update the DocTreeBuilder
Update the DocTreeBuilder to include a method for scanning collections:
```rust ```rust
use pulldown_cmark::{Parser, Options, html}; impl DocTreeBuilder {
/// Scan for collections in the given root path
pub fn name_fix(name: &str) -> String { ///
// Implementation of name_fix similar to tools.NameFix in Go /// # Arguments
// Normalize the name by converting to lowercase, replacing spaces with hyphens, etc. ///
} /// * `root_path` - The root path to scan for collections
///
pub fn markdown_to_html(markdown: &str) -> String { /// # Returns
let mut options = Options::empty(); ///
options.insert(Options::ENABLE_TABLES); /// Self for method chaining or an error
options.insert(Options::ENABLE_FOOTNOTES); pub fn scan_collections<P: AsRef<Path>>(self, root_path: P) -> Result<Self> {
options.insert(Options::ENABLE_STRIKETHROUGH); // Ensure storage is set
let storage = self.storage.as_ref().ok_or_else(|| {
let parser = Parser::new_ext(markdown, options); DocTreeError::MissingParameter("storage".to_string())
let mut html_output = String::new();
html::push_html(&mut html_output, parser);
html_output
}
```
### Collection Module (src/collection.rs)
```rust
use std::path::{Path, PathBuf};
use walkdir::WalkDir;
use crate::error::Result;
use crate::storage::RedisStorage;
use crate::utils::name_fix;
pub struct Collection {
pub path: PathBuf,
pub name: String,
storage: RedisStorage,
}
pub struct CollectionBuilder {
path: PathBuf,
name: String,
storage: Option<RedisStorage>,
}
impl Collection {
pub fn builder<P: AsRef<Path>>(path: P, name: &str) -> CollectionBuilder {
CollectionBuilder {
path: path.as_ref().to_path_buf(),
name: name_fix(name),
storage: None,
}
}
pub fn scan(&self) -> Result<()> {
// Implementation of scanning directory and storing in Redis
}
pub fn page_get(&self, page_name: &str) -> Result<String> {
// Implementation of getting page content
}
// Additional methods for Collection
}
impl CollectionBuilder {
pub fn with_storage(mut self, storage: RedisStorage) -> Self {
self.storage = Some(storage);
self
}
pub fn build(self) -> Result<Collection> {
let storage = self.storage.ok_or_else(|| {
std::io::Error::new(std::io::ErrorKind::Other, "Storage not provided")
})?; })?;
let collection = Collection { // Create a temporary DocTree to scan collections
path: self.path, let mut temp_doctree = DocTree {
name: self.name,
storage,
};
Ok(collection)
}
}
```
### DocTree Module (src/doctree.rs)
```rust
use std::collections::HashMap;
use std::path::Path;
use crate::collection::{Collection, CollectionBuilder};
use crate::error::{DocTreeError, Result};
use crate::storage::RedisStorage;
pub struct DocTree {
collections: HashMap<String, Collection>,
default_collection: Option<String>,
storage: RedisStorage,
}
pub struct DocTreeBuilder {
collections: HashMap<String, Collection>,
default_collection: Option<String>,
storage: Option<RedisStorage>,
}
impl DocTree {
pub fn builder() -> DocTreeBuilder {
DocTreeBuilder {
collections: HashMap::new(), collections: HashMap::new(),
default_collection: None, default_collection: None,
storage: None, storage: storage.clone(),
} name: self.name.clone().unwrap_or_default(),
} path: self.path.clone().unwrap_or_else(|| PathBuf::from("")),
pub fn add_collection<P: AsRef<Path>>(&mut self, path: P, name: &str) -> Result<&Collection> {
// Implementation of adding a collection
}
// Additional methods for DocTree
}
impl DocTreeBuilder {
pub fn with_storage(mut self, storage: RedisStorage) -> Self {
self.storage = Some(storage);
self
}
pub fn with_collection<P: AsRef<Path>>(mut self, path: P, name: &str) -> Result<Self> {
// Implementation of adding a collection during building
Ok(self)
}
pub fn with_default_collection(mut self, name: &str) -> Self {
self.default_collection = Some(name.to_string());
self
}
pub fn build(self) -> Result<DocTree> {
let storage = self.storage.ok_or_else(|| {
std::io::Error::new(std::io::ErrorKind::Other, "Storage not provided")
})?;
let doctree = DocTree {
collections: self.collections,
default_collection: self.default_collection,
storage,
}; };
Ok(doctree) // Scan for collections
temp_doctree.scan_collections(root_path)?;
// Create a new builder with the scanned collections
let mut new_builder = self;
for (name, collection) in temp_doctree.collections {
new_builder.collections.insert(name, collection);
}
Ok(new_builder)
} }
} }
``` ```
### Include Module (src/include.rs) ### 6. Add a Convenience Function to the Library
Add a convenience function to the library for creating a DocTree by scanning a directory:
```rust ```rust
use crate::doctree::DocTree; /// Create a new DocTree by scanning a directory for collections
use crate::error::Result; ///
/// # Arguments
pub fn process_includes(content: &str, collection_name: &str, doctree: &DocTree) -> Result<String> { ///
// Implementation of processing include directives /// * `root_path` - The root path to scan for collections
} ///
/// # Returns
fn parse_include_line(line: &str) -> Result<(Option<String>, Option<String>)> { ///
// Implementation of parsing include directives /// A new DocTree or an error
} pub fn from_directory<P: AsRef<Path>>(root_path: P) -> Result<DocTree> {
fn handle_include(page_name: &str, collection_name: &str, doctree: &DocTree) -> Result<String> {
// Implementation of handling include directives
}
```
### Main Library File (src/lib.rs)
```rust
mod error;
mod storage;
mod utils;
mod collection;
mod doctree;
mod include;
pub use error::{DocTreeError, Result};
pub use storage::RedisStorage;
pub use collection::{Collection, CollectionBuilder};
pub use doctree::{DocTree, DocTreeBuilder};
pub use include::process_includes;
```
### CLI Example (doctreecmd/src/main.rs)
```rust
use clap::{App, Arg, SubCommand};
use doctree::{DocTree, RedisStorage};
fn main() -> Result<(), Box<dyn std::error::Error>> {
let matches = App::new("DocTree CLI")
.version("0.1.0")
.author("Your Name")
.about("A tool to manage document collections")
.subcommand(
SubCommand::with_name("scan")
.about("Scan a directory and create a collection")
.arg(Arg::with_name("path").required(true))
.arg(Arg::with_name("name").required(true)),
)
.subcommand(
SubCommand::with_name("list")
.about("List collections"),
)
.subcommand(
SubCommand::with_name("get")
.about("Get page content")
.arg(Arg::with_name("collection").required(true))
.arg(Arg::with_name("page").required(true)),
)
.get_matches();
// Implementation of CLI commands
Ok(())
}
```
## Example Usage
Here's how the library would be used with the builder pattern:
```rust
use doctree::{DocTree, RedisStorage};
fn main() -> Result<(), Box<dyn std::error::Error>> {
// Create a Redis storage instance
let storage = RedisStorage::new("redis://localhost:6379")?; let storage = RedisStorage::new("redis://localhost:6379")?;
// Create a DocTree instance using the builder pattern DocTree::builder()
let mut doctree = DocTree::builder() .with_storage(storage)
.with_storage(storage.clone()) .scan_collections(root_path)?
.with_collection("path/to/collection", "my-collection")? .build()
.with_default_collection("my-collection")
.build()?;
// Get page content
let content = doctree.page_get("my-collection", "page-name")?;
println!("Page content: {}", content);
// Get HTML content
let html = doctree.page_get_html("my-collection", "page-name")?;
println!("HTML content: {}", html);
Ok(())
} }
``` ```
## Testing Strategy ## Implementation Flow Diagram
1. **Unit Tests** ```mermaid
- Test individual components in isolation flowchart TD
- Mock Redis for testing storage operations A[Start] --> B[Update Dependencies]
- Test utility functions B --> C[Replace name_fix function]
C --> D[Add CollectionConfig struct]
D --> E[Add scan_collections method to DocTree]
E --> F[Update DocTreeBuilder]
F --> G[Add convenience function]
G --> H[End]
```
2. **Integration Tests** ## Component Interaction Diagram
- Test the interaction between components
- Test the builder pattern
- Test include processing
3. **End-to-End Tests** ```mermaid
- Test the complete workflow with real files graph TD
- Test the CLI interface A[DocTree] -->|manages| B[Collections]
C[scan_collections] -->|finds| D[.collection files]
D -->|parsed as| E[TOML]
E -->|extracts| F[Collection Name]
C -->|creates| B
G[name_fix] -->|processes| F
G -->|processes| H[File Names]
B -->|contains| H
```
## Timeline ## Testing Plan
1. **Project Setup and Dependencies**: 1 day 1. Create test directories with `.collection` files in various formats
2. **Core Library Structure**: 2 days 2. Test the scan_collections method with these directories
3. **Collection Implementation**: 2 days 3. Verify that collections are created correctly with the expected names
4. **DocTree Implementation**: 2 days 4. Verify that all files under the collection directories are included in the collections
5. **Include Processor Implementation**: 1 day 5. Test edge cases such as empty `.collection` files, invalid TOML, etc.
6. **CLI Example**: 1 day
7. **Testing and Documentation**: 2 days
Total estimated time: 11 days

291
doctreecmd/src/main.rs
View File

@ -1,5 +1,5 @@
use clap::{App, Arg, SubCommand}; use clap::{App, Arg, SubCommand};
use doctree::{DocTree, RedisStorage, Result}; use doctree::{DocTree, RedisStorage, Result, from_directory};
use std::path::Path; use std::path::Path;
fn main() -> Result<()> { fn main() -> Result<()> {
@ -17,11 +17,41 @@ fn main() -> Result<()> {
SubCommand::with_name("list") SubCommand::with_name("list")
.about("List collections"), .about("List collections"),
) )
.subcommand(
SubCommand::with_name("scan-collections")
.about("Recursively scan directories for .collection files")
.arg(Arg::with_name("path").required(true).help("Root path to scan for collections")),
)
.subcommand(
SubCommand::with_name("scan-and-info")
.about("Scan collections and show detailed information")
.arg(Arg::with_name("path").required(true).help("Root path to scan for collections"))
.arg(Arg::with_name("collection").help("Name of the collection (optional)")),
)
.subcommand(
SubCommand::with_name("info")
.about("Show detailed information about collections")
.arg(Arg::with_name("collection").help("Name of the collection (optional)")),
)
.subcommand( .subcommand(
SubCommand::with_name("get") SubCommand::with_name("get")
.about("Get page content") .about("Get page content")
.arg(Arg::with_name("collection").required(true).help("Name of the collection")) .arg(Arg::with_name("collection")
.arg(Arg::with_name("page").required(true).help("Name of the page")), .short("c".chars().next().unwrap())
.long("collection")
.takes_value(true)
.help("Name of the collection (optional)"))
.arg(Arg::with_name("page")
.short("p".chars().next().unwrap())
.long("page")
.required(true)
.takes_value(true)
.help("Name of the page"))
.arg(Arg::with_name("format")
.short("f".chars().next().unwrap())
.long("format")
.takes_value(true)
.help("Output format (html or markdown, default: markdown)")),
) )
.subcommand( .subcommand(
SubCommand::with_name("html") SubCommand::with_name("html")
@ -29,6 +59,15 @@ fn main() -> Result<()> {
.arg(Arg::with_name("collection").required(true).help("Name of the collection")) .arg(Arg::with_name("collection").required(true).help("Name of the collection"))
.arg(Arg::with_name("page").required(true).help("Name of the page")), .arg(Arg::with_name("page").required(true).help("Name of the page")),
) )
.subcommand(
SubCommand::with_name("delete-collection")
.about("Delete a collection from Redis")
.arg(Arg::with_name("collection").required(true).help("Name of the collection")),
)
.subcommand(
SubCommand::with_name("reset")
.about("Delete all collections from Redis"),
)
.get_matches(); .get_matches();
// Create a Redis storage instance // Create a Redis storage instance
@ -59,17 +98,257 @@ fn main() -> Result<()> {
} }
} }
} else if let Some(matches) = matches.subcommand_matches("get") { } else if let Some(matches) = matches.subcommand_matches("get") {
let collection = matches.value_of("collection").unwrap(); let collection = matches.value_of("collection");
let page = matches.value_of("page").unwrap(); let page = matches.value_of("page").unwrap();
let format = matches.value_of("format").unwrap_or("markdown");
let content = doctree.page_get(Some(collection), page)?; if format.to_lowercase() == "html" {
println!("{}", content); let html = doctree.page_get_html(collection, page)?;
println!("{}", html);
} else {
let content = doctree.page_get(collection, page)?;
println!("{}", content);
}
} else if let Some(matches) = matches.subcommand_matches("html") { } else if let Some(matches) = matches.subcommand_matches("html") {
let collection = matches.value_of("collection").unwrap(); let collection = matches.value_of("collection").unwrap();
let page = matches.value_of("page").unwrap(); let page = matches.value_of("page").unwrap();
let html = doctree.page_get_html(Some(collection), page)?; let html = doctree.page_get_html(Some(collection), page)?;
println!("{}", html); println!("{}", html);
} else if let Some(matches) = matches.subcommand_matches("delete-collection") {
let collection = matches.value_of("collection").unwrap();
println!("Deleting collection '{}' from Redis...", collection);
doctree.delete_collection(collection)?;
println!("Collection '{}' deleted successfully", collection);
} else if let Some(_) = matches.subcommand_matches("reset") {
println!("Deleting all collections from Redis...");
doctree.delete_all_collections()?;
println!("All collections deleted successfully");
} else if let Some(matches) = matches.subcommand_matches("scan-collections") {
let path = matches.value_of("path").unwrap();
println!("Recursively scanning for collections in: {}", path);
// Use the from_directory function to create a DocTree with all collections
let doctree = from_directory(Path::new(path))?;
// Print the discovered collections
let collections = doctree.list_collections();
if collections.is_empty() {
println!("No collections found");
} else {
println!("Discovered collections:");
for collection in collections {
println!("- {}", collection);
}
}
} else if let Some(matches) = matches.subcommand_matches("scan-and-info") {
let path = matches.value_of("path").unwrap();
let collection_name = matches.value_of("collection");
println!("Recursively scanning for collections in: {}", path);
// Use the from_directory function to create a DocTree with all collections
let doctree = from_directory(Path::new(path))?;
// Print the discovered collections
let collections = doctree.list_collections();
if collections.is_empty() {
println!("No collections found");
return Ok(());
}
println!("Discovered collections:");
for collection in &collections {
println!("- {}", collection);
}
println!("\nDetailed Collection Information:");
if let Some(name) = collection_name {
// Show info for a specific collection
match doctree.get_collection(name) {
Ok(collection) => {
println!("Collection Information for '{}':", name);
println!(" Path: {:?}", collection.path);
println!(" Redis Key: collections:{}", collection.name);
// List documents
match collection.page_list() {
Ok(pages) => {
println!(" Documents ({}):", pages.len());
for page in pages {
match collection.page_get_path(&page) {
Ok(path) => {
println!(" - {} => Redis: collections:{} / {}", path, collection.name, page);
},
Err(_) => {
println!(" - {}", page);
}
}
}
},
Err(e) => println!(" Error listing documents: {}", e),
}
// List files
match collection.file_list() {
Ok(files) => {
// Filter images
let images: Vec<String> = files.iter()
.filter(|f|
f.ends_with(".png") || f.ends_with(".jpg") ||
f.ends_with(".jpeg") || f.ends_with(".gif") ||
f.ends_with(".svg"))
.cloned()
.collect();
println!(" Images ({}):", images.len());
for image in images {
println!(" - {} => Redis: collections:{} / {}", image, collection.name, image);
}
// Filter other files
let other_files: Vec<String> = files.iter()
.filter(|f|
!f.ends_with(".png") && !f.ends_with(".jpg") &&
!f.ends_with(".jpeg") && !f.ends_with(".gif") &&
!f.ends_with(".svg"))
.cloned()
.collect();
println!(" Other Files ({}):", other_files.len());
for file in other_files {
println!(" - {} => Redis: collections:{} / {}", file, collection.name, file);
}
},
Err(e) => println!(" Error listing files: {}", e),
}
},
Err(e) => println!("Error: {}", e),
}
} else {
// Show info for all collections
for name in collections {
if let Ok(collection) = doctree.get_collection(&name) {
println!("- {} (Redis Key: collections:{})", name, collection.name);
println!(" Path: {:?}", collection.path);
// Count documents and images
if let Ok(pages) = collection.page_list() {
println!(" Documents: {}", pages.len());
}
if let Ok(files) = collection.file_list() {
let image_count = files.iter()
.filter(|f|
f.ends_with(".png") || f.ends_with(".jpg") ||
f.ends_with(".jpeg") || f.ends_with(".gif") ||
f.ends_with(".svg"))
.count();
println!(" Images: {}", image_count);
println!(" Other Files: {}", files.len() - image_count);
}
}
}
}
} else if let Some(matches) = matches.subcommand_matches("info") {
let collection_name = matches.value_of("collection");
if let Some(name) = collection_name {
// Show info for a specific collection
match doctree.get_collection(name) {
Ok(collection) => {
println!("Collection Information for '{}':", name);
println!(" Path: {:?}", collection.path);
println!(" Redis Key: collections:{}", collection.name);
// List documents
match collection.page_list() {
Ok(pages) => {
println!(" Documents ({}):", pages.len());
for page in pages {
match collection.page_get_path(&page) {
Ok(path) => {
println!(" - {} => Redis: collections:{} / {}", path, collection.name, page);
},
Err(_) => {
println!(" - {}", page);
}
}
}
},
Err(e) => println!(" Error listing documents: {}", e),
}
// List files
match collection.file_list() {
Ok(files) => {
// Filter images
let images: Vec<String> = files.iter()
.filter(|f|
f.ends_with(".png") || f.ends_with(".jpg") ||
f.ends_with(".jpeg") || f.ends_with(".gif") ||
f.ends_with(".svg"))
.cloned()
.collect();
println!(" Images ({}):", images.len());
for image in images {
println!(" - {} => Redis: collections:{} / {}", image, collection.name, image);
}
// Filter other files
let other_files: Vec<String> = files.iter()
.filter(|f|
!f.ends_with(".png") && !f.ends_with(".jpg") &&
!f.ends_with(".jpeg") && !f.ends_with(".gif") &&
!f.ends_with(".svg"))
.cloned()
.collect();
println!(" Other Files ({}):", other_files.len());
for file in other_files {
println!(" - {} => Redis: collections:{} / {}", file, collection.name, file);
}
},
Err(e) => println!(" Error listing files: {}", e),
}
},
Err(e) => println!("Error: {}", e),
}
} else {
// Show info for all collections
let collections = doctree.list_collections();
if collections.is_empty() {
println!("No collections found");
} else {
println!("Collections:");
for name in collections {
if let Ok(collection) = doctree.get_collection(&name) {
println!("- {} (Redis Key: collections:{})", name, collection.name);
println!(" Path: {:?}", collection.path);
// Count documents and images
if let Ok(pages) = collection.page_list() {
println!(" Documents: {}", pages.len());
}
if let Ok(files) = collection.file_list() {
let image_count = files.iter()
.filter(|f|
f.ends_with(".png") || f.ends_with(".jpg") ||
f.ends_with(".jpeg") || f.ends_with(".gif") ||
f.ends_with(".svg"))
.count();
println!(" Images: {}", image_count);
println!(" Other Files: {}", files.len() - image_count);
}
}
}
}
}
} else { } else {
println!("No command specified. Use --help for usage information."); println!("No command specified. Use --help for usage information.");
} }

31
example_commands.sh Executable file
View File

@ -0,0 +1,31 @@
#!/bin/bash
# Change to the directory where the script is located
cd "$(dirname "$0")"
# Exit immediately if a command exits with a non-zero status
set -e
cd doctreecmd
echo "=== Scanning Collections ==="
cargo run -- scan-collections ../examples
echo -e "\n=== Listing Collections ==="
cargo run -- list
echo -e "\n=== Getting Document (Markdown) ==="
cargo run -- get -c grid_documentation -p introduction.md
echo -e "\n=== Getting Document (HTML) ==="
cargo run -- get -c grid_documentation -p introduction.md -f html
echo -e "\n=== Deleting Collection ==="
cargo run -- delete-collection grid_documentation
echo -e "\n=== Listing Remaining Collections ==="
cargo run -- list
echo -e "\n=== Resetting All Collections ==="
cargo run -- reset
echo -e "\n=== Verifying Reset ==="
cargo run -- list

1
examples/grid1/.collection
View File

@ -0,0 +1 @@
name = "Grid Documentation"

1
examples/parent/docs2/.collection
View File

@ -0,0 +1 @@
name = "supercollection"

39
runexample.sh Executable file
View File

@ -0,0 +1,39 @@
#!/bin/bash
# Change to the directory where the script is located
cd "$(dirname "$0")"
# Exit immediately if a command exits with a non-zero status
set -e
cd doctreecmd
# First, scan the collections
echo "=== Scanning Collections ==="
cargo run -- scan-and-info ../examples supercollection
# Get a document in markdown format
echo -e "\n=== Getting Document (Markdown) ==="
cargo run -- get -c supercollection -p 01_features.md
# Get a document in HTML format
echo -e "\n=== Getting Document (HTML) ==="
cargo run -- get -c supercollection -p 01_features.md -f html
# Get a document without specifying collection
echo -e "\n=== Getting Document (Default Collection) ==="
cargo run -- get -p 01_features.md
# Delete a specific collection
echo -e "\n=== Deleting Collection ==="
cargo run -- delete-collection grid_documentation
# List remaining collections
echo -e "\n=== Listing Remaining Collections ==="
cargo run -- list
# Reset all collections
echo -e "\n=== Resetting All Collections ==="
cargo run -- reset
# Verify all collections are gone
echo -e "\n=== Verifying Reset ==="
cargo run -- list