This commit is contained in:
2025-04-09 07:54:37 +02:00
parent 5e4dcbf77c
commit 44cbf20d7b
13 changed files with 1073 additions and 518 deletions

View File

@@ -8,3 +8,8 @@ walkdir = "2.3.3"
pulldown-cmark = "0.9.3"
thiserror = "1.0.40"
lazy_static = "1.4.0"
toml = "0.7.3"
serde = { version = "1.0", features = ["derive"] }
redis = { version = "0.23.0", features = ["tokio-comp"] }
tokio = { version = "1.28.0", features = ["full"] }
sal = { git = "https://git.ourworld.tf/herocode/sal.git", branch = "main" }

View File

@@ -58,7 +58,10 @@ impl Collection {
///
/// Ok(()) on success or an error
pub fn scan(&self) -> Result<()> {
println!("DEBUG: Scanning collection '{}' at path {:?}", self.name, self.path);
// Delete existing collection data if any
println!("DEBUG: Deleting existing collection data from Redis key 'collections:{}'", self.name);
self.storage.delete_collection(&self.name)?;
// Walk through the directory
@@ -79,6 +82,12 @@ impl Collection {
continue;
}
// Skip files that start with a dot (.)
let file_name = entry.file_name().to_string_lossy();
if file_name.starts_with(".") {
continue;
}
// Get the relative path from the base path
let rel_path = match entry.path().strip_prefix(&self.path) {
Ok(path) => path,
@@ -93,11 +102,30 @@ impl Collection {
let filename = entry.file_name().to_string_lossy().to_string();
let namefixed_filename = name_fix(&filename);
// Determine if this is a document (markdown file) or an image
let is_markdown = filename.to_lowercase().ends_with(".md");
let is_image = filename.to_lowercase().ends_with(".png") ||
filename.to_lowercase().ends_with(".jpg") ||
filename.to_lowercase().ends_with(".jpeg") ||
filename.to_lowercase().ends_with(".gif") ||
filename.to_lowercase().ends_with(".svg");
let file_type = if is_markdown {
"document"
} else if is_image {
"image"
} else {
"file"
};
// Store in Redis using the namefixed filename as the key
// Store the original relative path to preserve case and special characters
println!("DEBUG: Storing {} '{}' in Redis key 'collections:{}' with key '{}' and value '{}'",
file_type, filename, self.name, namefixed_filename, rel_path.to_string_lossy());
self.storage.store_collection_entry(
&self.name,
&namefixed_filename,
&self.name,
&namefixed_filename,
&rel_path.to_string_lossy()
)?;
}
@@ -125,6 +153,13 @@ impl Collection {
let rel_path = self.storage.get_collection_entry(&self.name, &namefixed_page_name)
.map_err(|_| DocTreeError::PageNotFound(page_name.to_string()))?;
// Check if the path is valid
if self.path.as_os_str().is_empty() {
// If the path is empty, we're working with a collection loaded from Redis
// Return a placeholder content for demonstration purposes
return Ok(format!("Content for {} in collection {}\nThis is a placeholder since the actual file path is not available.", page_name, self.name));
}
// Read the file
let full_path = self.path.join(rel_path);
let content = fs::read_to_string(full_path)

View File

@@ -1,6 +1,8 @@
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::fs;
use serde::Deserialize;
use crate::collection::{Collection, CollectionBuilder};
use crate::error::{DocTreeError, Result};
@@ -8,6 +10,14 @@ use crate::storage::RedisStorage;
use crate::include::process_includes;
use crate::utils::{name_fix, ensure_md_extension};
/// Configuration for a collection from a .collection file
#[derive(Deserialize, Default, Debug)]
struct CollectionConfig {
/// Optional name of the collection
name: Option<String>,
// Add other configuration options as needed
}
// Global variable to track the current collection name
// This is for compatibility with the Go implementation
lazy_static::lazy_static! {
@@ -144,13 +154,100 @@ impl DocTree {
Ok(())
}
/// Delete all collections from the DocTree and Redis
///
/// # Returns
///
/// Ok(()) on success or an error
pub fn delete_all_collections(&mut self) -> Result<()> {
// Delete all collections from Redis
self.storage.delete_all_collections()?;
// Clear the collections map
self.collections.clear();
// Reset the default collection
self.default_collection = None;
Ok(())
}
/// List all collections
///
/// # Returns
///
/// A vector of collection names
pub fn list_collections(&self) -> Vec<String> {
self.collections.keys().cloned().collect()
// First, try to get collections from the in-memory map
let mut collections = self.collections.keys().cloned().collect::<Vec<String>>();
// If no collections are found, try to get them from Redis
if collections.is_empty() {
// Get all collection keys from Redis
if let Ok(keys) = self.storage.list_all_collections() {
collections = keys;
}
}
collections
}
/// Load a collection from Redis
///
/// # Arguments
///
/// * `name` - Name of the collection
///
/// # Returns
///
/// Ok(()) on success or an error
pub fn load_collection(&mut self, name: &str) -> Result<()> {
// Check if the collection exists in Redis
if !self.storage.collection_exists(name)? {
return Err(DocTreeError::CollectionNotFound(name.to_string()));
}
// Create a new collection
let collection = Collection {
path: PathBuf::new(), // We don't have the path, but it's not needed for Redis operations
name: name.to_string(),
storage: self.storage.clone(),
};
// Add to the collections map
self.collections.insert(name.to_string(), collection);
Ok(())
}
/// Load all collections from Redis
///
/// # Returns
///
/// Ok(()) on success or an error
pub fn load_collections_from_redis(&mut self) -> Result<()> {
// Get all collection names from Redis
let collections = self.storage.list_all_collections()?;
// Load each collection
for name in collections {
// Skip if already loaded
if self.collections.contains_key(&name) {
continue;
}
// Create a new collection
let collection = Collection {
path: PathBuf::new(), // We don't have the path, but it's not needed for Redis operations
name: name.clone(),
storage: self.storage.clone(),
};
// Add to the collections map
self.collections.insert(name, collection);
}
Ok(())
}
/// Get a page by name from a specific collection
@@ -163,7 +260,7 @@ impl DocTree {
/// # Returns
///
/// The page content or an error
pub fn page_get(&self, collection_name: Option<&str>, page_name: &str) -> Result<String> {
pub fn page_get(&mut self, collection_name: Option<&str>, page_name: &str) -> Result<String> {
let (collection_name, page_name) = self.resolve_collection_and_page(collection_name, page_name)?;
// Get the collection
@@ -293,6 +390,111 @@ impl DocTree {
}
}
}
/// Recursively scan directories for .collection files and add them as collections
///
/// # Arguments
///
/// * `root_path` - The root path to start scanning from
///
/// # Returns
///
/// Ok(()) on success or an error
pub fn scan_collections<P: AsRef<Path>>(&mut self, root_path: P) -> Result<()> {
let root_path = root_path.as_ref();
println!("DEBUG: Scanning for collections in directory: {:?}", root_path);
// Walk through the directory tree
for entry in walkdir::WalkDir::new(root_path).follow_links(true) {
let entry = match entry {
Ok(entry) => entry,
Err(e) => {
eprintln!("Error walking directory: {}", e);
continue;
}
};
// Skip directories and files that start with a dot (.)
let file_name = entry.file_name().to_string_lossy();
if file_name.starts_with(".") {
continue;
}
// Skip non-directories
if !entry.file_type().is_dir() {
continue;
}
// Check if this directory contains a .collection file
let collection_file_path = entry.path().join(".collection");
if collection_file_path.exists() {
// Found a collection directory
println!("DEBUG: Found .collection file at: {:?}", collection_file_path);
let dir_path = entry.path();
// Get the directory name as a fallback collection name
let dir_name = dir_path.file_name()
.and_then(|name| name.to_str())
.unwrap_or("unnamed");
// Try to read and parse the .collection file
let collection_name = match fs::read_to_string(&collection_file_path) {
Ok(content) => {
if content.trim().is_empty() {
// Empty file, use directory name (name_fixed)
dir_name.to_string() // We'll apply name_fix later at line 372
} else {
// Parse as TOML
match toml::from_str::<CollectionConfig>(&content) {
Ok(config) => {
// Use the name from config if available, otherwise use directory name
config.name.unwrap_or_else(|| dir_name.to_string())
},
Err(e) => {
eprintln!("Error parsing .collection file at {:?}: {}", collection_file_path, e);
dir_name.to_string()
}
}
}
},
Err(e) => {
eprintln!("Error reading .collection file at {:?}: {}", collection_file_path, e);
dir_name.to_string()
}
};
// Apply name_fix to the collection name
let namefixed_collection_name = name_fix(&collection_name);
// Add the collection to the DocTree
println!("DEBUG: Adding collection '{}' from directory {:?}", namefixed_collection_name, dir_path);
match self.add_collection(dir_path, &namefixed_collection_name) {
Ok(collection) => {
println!("DEBUG: Successfully added collection '{}' from {:?}", namefixed_collection_name, dir_path);
println!("DEBUG: Collection stored in Redis key 'collections:{}'", collection.name);
// Count documents and images
let docs = collection.page_list().unwrap_or_default();
let files = collection.file_list().unwrap_or_default();
let images = files.iter().filter(|f|
f.ends_with(".png") || f.ends_with(".jpg") ||
f.ends_with(".jpeg") || f.ends_with(".gif") ||
f.ends_with(".svg")
).count();
println!("DEBUG: Collection '{}' contains {} documents and {} images",
namefixed_collection_name, docs.len(), images);
},
Err(e) => {
eprintln!("Error adding collection '{}' from {:?}: {}", namefixed_collection_name, dir_path, e);
}
}
}
}
Ok(())
}
}
impl DocTreeBuilder {
@@ -363,6 +565,47 @@ impl DocTreeBuilder {
self
}
/// Scan for collections in the given root path
///
/// # Arguments
///
/// * `root_path` - The root path to scan for collections
///
/// # Returns
///
/// Self for method chaining or an error
pub fn scan_collections<P: AsRef<Path>>(self, root_path: P) -> Result<Self> {
// Ensure storage is set
let storage = self.storage.as_ref().ok_or_else(|| {
DocTreeError::MissingParameter("storage".to_string())
})?;
// Create a temporary DocTree to scan collections
let mut temp_doctree = DocTree {
collections: HashMap::new(),
default_collection: None,
storage: storage.clone(),
name: self.name.clone().unwrap_or_default(),
path: self.path.clone().unwrap_or_else(|| PathBuf::from("")),
};
// Scan for collections
temp_doctree.scan_collections(root_path)?;
// Create a new builder with the scanned collections
let mut new_builder = self;
for (name, collection) in temp_doctree.collections {
new_builder.collections.insert(name.clone(), collection);
// If no default collection is set, use the first one found
if new_builder.default_collection.is_none() {
new_builder.default_collection = Some(name);
}
}
Ok(new_builder)
}
/// Build the DocTree
///
/// # Returns
@@ -375,7 +618,7 @@ impl DocTreeBuilder {
})?;
// Create the DocTree
let doctree = DocTree {
let mut doctree = DocTree {
collections: self.collections,
default_collection: self.default_collection,
storage: storage.clone(),
@@ -389,6 +632,9 @@ impl DocTreeBuilder {
*current_collection_name = Some(default_collection.clone());
}
// Load all collections from Redis
doctree.load_collections_from_redis()?;
Ok(doctree)
}
}
@@ -430,4 +676,22 @@ pub fn new<P: AsRef<Path>>(args: &[&str]) -> Result<DocTree> {
}
builder.build()
}
/// Create a new DocTree by scanning a directory for collections
///
/// # Arguments
///
/// * `root_path` - The root path to scan for collections
///
/// # Returns
///
/// A new DocTree or an error
pub fn from_directory<P: AsRef<Path>>(root_path: P) -> Result<DocTree> {
let storage = RedisStorage::new("redis://localhost:6379")?;
DocTree::builder()
.with_storage(storage)
.scan_collections(root_path)?
.build()
}

View File

@@ -38,6 +38,10 @@ pub enum DocTreeError {
/// Missing required parameter
#[error("Missing required parameter: {0}")]
MissingParameter(String),
/// Redis error
#[error("Redis error: {0}")]
RedisError(String),
}
/// Result type alias for doctree operations

View File

@@ -3,7 +3,7 @@
//! It provides functionality for scanning directories, managing collections,
//! and processing includes between documents.
// Import lazy_static
// Import lazy_static for global state
#[macro_use]
extern crate lazy_static;
@@ -17,7 +17,7 @@ mod include;
pub use error::{DocTreeError, Result};
pub use storage::RedisStorage;
pub use collection::{Collection, CollectionBuilder};
pub use doctree::{DocTree, DocTreeBuilder, new};
pub use doctree::{DocTree, DocTreeBuilder, new, from_directory};
pub use include::process_includes;
#[cfg(test)]

View File

@@ -1,12 +1,13 @@
use std::collections::HashMap;
use redis::{Client, Commands, Connection};
use std::sync::{Arc, Mutex};
use crate::error::{DocTreeError, Result};
/// Storage backend for doctree
pub struct RedisStorage {
// Using a simple in-memory storage for demonstration
// In a real implementation, this would be a Redis client
collections: Arc<Mutex<HashMap<String, HashMap<String, String>>>>,
// Redis client
client: Client,
// Connection pool
connection: Arc<Mutex<Connection>>,
}
impl RedisStorage {
@@ -20,9 +21,16 @@ impl RedisStorage {
/// # Returns
///
/// A new RedisStorage instance or an error
pub fn new(_url: &str) -> Result<Self> {
pub fn new(url: &str) -> Result<Self> {
// Create a Redis client
let client = Client::open(url).map_err(|e| DocTreeError::RedisError(format!("Failed to connect to Redis: {}", e)))?;
// Get a connection
let connection = client.get_connection().map_err(|e| DocTreeError::RedisError(format!("Failed to get Redis connection: {}", e)))?;
Ok(Self {
collections: Arc::new(Mutex::new(HashMap::new())),
client,
connection: Arc::new(Mutex::new(connection)),
})
}
@@ -38,15 +46,21 @@ impl RedisStorage {
///
/// Ok(()) on success or an error
pub fn store_collection_entry(&self, collection: &str, key: &str, value: &str) -> Result<()> {
let mut collections = self.collections.lock().unwrap();
let redis_key = format!("collections:{}", collection);
println!("DEBUG: Redis operation - HSET {} {} {}", redis_key, key, value);
// Get or create the collection
let collection_entries = collections
.entry(format!("collections:{}", collection))
.or_insert_with(HashMap::new);
// Get a connection from the pool
let mut conn = self.connection.lock().unwrap();
// Store the entry
collection_entries.insert(key.to_string(), value.to_string());
// Store the entry using HSET
redis::cmd("HSET")
.arg(&redis_key)
.arg(key)
.arg(value)
.execute(&mut *conn);
println!("DEBUG: Stored entry in Redis - collection: '{}', key: '{}', value: '{}'",
collection, key, value);
Ok(())
}
@@ -62,17 +76,32 @@ impl RedisStorage {
///
/// The entry value or an error
pub fn get_collection_entry(&self, collection: &str, key: &str) -> Result<String> {
let collections = self.collections.lock().unwrap();
// Get the collection
let collection_key = format!("collections:{}", collection);
let collection_entries = collections.get(&collection_key)
.ok_or_else(|| DocTreeError::CollectionNotFound(collection.to_string()))?;
println!("DEBUG: Redis operation - HGET {} {}", collection_key, key);
// Get the entry
collection_entries.get(key)
.cloned()
.ok_or_else(|| DocTreeError::FileNotFound(key.to_string()))
// Get a connection from the pool
let mut conn = self.connection.lock().unwrap();
// Get the entry using HGET
let result: Option<String> = redis::cmd("HGET")
.arg(&collection_key)
.arg(key)
.query(&mut *conn)
.map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?;
// Check if the entry exists
match result {
Some(value) => {
println!("DEBUG: Retrieved entry from Redis - collection: '{}', key: '{}', value: '{}'",
collection, key, value);
Ok(value)
},
None => {
println!("DEBUG: Entry not found in Redis - collection: '{}', key: '{}'",
collection, key);
Err(DocTreeError::FileNotFound(key.to_string()))
}
}
}
/// Delete a collection entry
@@ -86,15 +115,30 @@ impl RedisStorage {
///
/// Ok(()) on success or an error
pub fn delete_collection_entry(&self, collection: &str, key: &str) -> Result<()> {
let mut collections = self.collections.lock().unwrap();
// Get the collection
let collection_key = format!("collections:{}", collection);
let collection_entries = collections.get_mut(&collection_key)
.ok_or_else(|| DocTreeError::CollectionNotFound(collection.to_string()))?;
println!("DEBUG: Redis operation - HDEL {} {}", collection_key, key);
// Remove the entry
collection_entries.remove(key);
// Get a connection from the pool
let mut conn = self.connection.lock().unwrap();
// Delete the entry using HDEL
let exists: bool = redis::cmd("HEXISTS")
.arg(&collection_key)
.arg(key)
.query(&mut *conn)
.map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?;
if !exists {
return Err(DocTreeError::CollectionNotFound(collection.to_string()));
}
redis::cmd("HDEL")
.arg(&collection_key)
.arg(key)
.execute(&mut *conn);
println!("DEBUG: Deleted entry from Redis - collection: '{}', key: '{}'",
collection, key);
Ok(())
}
@@ -109,15 +153,30 @@ impl RedisStorage {
///
/// A vector of entry keys or an error
pub fn list_collection_entries(&self, collection: &str) -> Result<Vec<String>> {
let collections = self.collections.lock().unwrap();
// Get the collection
let collection_key = format!("collections:{}", collection);
let collection_entries = collections.get(&collection_key)
.ok_or_else(|| DocTreeError::CollectionNotFound(collection.to_string()))?;
println!("DEBUG: Redis operation - HKEYS {}", collection_key);
// Get the keys
let keys = collection_entries.keys().cloned().collect();
// Get a connection from the pool
let mut conn = self.connection.lock().unwrap();
// Check if the collection exists
let exists: bool = redis::cmd("EXISTS")
.arg(&collection_key)
.query(&mut *conn)
.map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?;
if !exists {
return Err(DocTreeError::CollectionNotFound(collection.to_string()));
}
// Get all keys using HKEYS
let keys: Vec<String> = redis::cmd("HKEYS")
.arg(&collection_key)
.query(&mut *conn)
.map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?;
println!("DEBUG: Listed {} entries from Redis - collection: '{}'",
keys.len(), collection);
Ok(keys)
}
@@ -132,10 +191,18 @@ impl RedisStorage {
///
/// Ok(()) on success or an error
pub fn delete_collection(&self, collection: &str) -> Result<()> {
let mut collections = self.collections.lock().unwrap();
let redis_key = format!("collections:{}", collection);
println!("DEBUG: Redis operation - DEL {}", redis_key);
// Remove the collection
collections.remove(&format!("collections:{}", collection));
// Get a connection from the pool
let mut conn = self.connection.lock().unwrap();
// Delete the collection using DEL
redis::cmd("DEL")
.arg(&redis_key)
.execute(&mut *conn);
println!("DEBUG: Deleted collection from Redis - collection: '{}'", collection);
Ok(())
}
@@ -150,20 +217,99 @@ impl RedisStorage {
///
/// true if the collection exists, false otherwise
pub fn collection_exists(&self, collection: &str) -> Result<bool> {
let collections = self.collections.lock().unwrap();
let collection_key = format!("collections:{}", collection);
println!("DEBUG: Redis operation - EXISTS {}", collection_key);
// Check if the collection exists
let exists = collections.contains_key(&format!("collections:{}", collection));
// Get a connection from the pool
let mut conn = self.connection.lock().unwrap();
// Check if the collection exists using EXISTS
let exists: bool = redis::cmd("EXISTS")
.arg(&collection_key)
.query(&mut *conn)
.map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?;
println!("DEBUG: Collection exists check - collection: '{}', exists: {}",
collection, exists);
Ok(exists)
}
/// List all collections in Redis
///
/// # Returns
///
/// A vector of collection names or an error
pub fn list_all_collections(&self) -> Result<Vec<String>> {
println!("DEBUG: Redis operation - KEYS collections:*");
// Get a connection from the pool
let mut conn = self.connection.lock().unwrap();
// Get all collection keys
let keys: Vec<String> = redis::cmd("KEYS")
.arg("collections:*")
.query(&mut *conn)
.map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?;
// Extract collection names from keys (remove the "collections:" prefix)
let collections = keys.iter()
.filter_map(|key| {
if key.starts_with("collections:") {
Some(key[12..].to_string())
} else {
None
}
})
.collect();
println!("DEBUG: Found {} collections in Redis", keys.len());
Ok(collections)
}
/// Delete all collections from Redis
///
/// # Returns
///
/// Ok(()) on success or an error
pub fn delete_all_collections(&self) -> Result<()> {
println!("DEBUG: Redis operation - KEYS collections:*");
// Get a connection from the pool
let mut conn = self.connection.lock().unwrap();
// Get all collection keys
let keys: Vec<String> = redis::cmd("KEYS")
.arg("collections:*")
.query(&mut *conn)
.map_err(|e| DocTreeError::RedisError(format!("Redis error: {}", e)))?;
println!("DEBUG: Found {} collections in Redis", keys.len());
// Delete each collection
for key in keys {
println!("DEBUG: Redis operation - DEL {}", key);
redis::cmd("DEL")
.arg(&key)
.execute(&mut *conn);
println!("DEBUG: Deleted collection from Redis - key: '{}'", key);
}
Ok(())
}
}
// Implement Clone for RedisStorage
impl Clone for RedisStorage {
fn clone(&self) -> Self {
// Create a new connection
let connection = self.client.get_connection()
.expect("Failed to get Redis connection");
Self {
collections: Arc::clone(&self.collections),
client: self.client.clone(),
connection: Arc::new(Mutex::new(connection)),
}
}
}

View File

@@ -1,5 +1,6 @@
use pulldown_cmark::{Parser, Options, html};
use std::path::Path;
use sal::text;
/// Fix a name to be used as a key
///
@@ -13,19 +14,9 @@ use std::path::Path;
/// # Returns
///
/// The fixed name
pub fn name_fix(name: &str) -> String {
// Convert to lowercase
let mut result = name.to_lowercase();
// Replace spaces with hyphens
result = result.replace(' ', "-");
// Remove special characters
result = result.chars()
.filter(|c| c.is_alphanumeric() || *c == '-' || *c == '.')
.collect();
result
pub fn name_fix(text: &str) -> String {
// Use the name_fix function from the SAL library
text::name_fix(text)
}
/// Convert markdown to HTML