herocode/doctree_rust
13
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Improve collection scanning and add .gitignore entries

Browse Source 
- Add `.gitignore` entries for `webmeta.json` and `.vscode`
- Improve collection scanning logging for better debugging
- Improve error handling in collection methods for robustness
This commit is contained in:
Mahmoud Emad
2025-05-15 08:53:16 +03:00
parent cad8a6d125
commit ea25db7d29
22 changed files with 3042 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

324
webbuilder/src/builder/mod.rs Normal file
View File

@@ -0,0 +1,324 @@
use serde::{Deserialize, Serialize};
use std::fs;
use std::path::Path;
use crate::config::SiteConfig;
use crate::error::Result;
use crate::parser_hjson;
#[cfg(test)]
mod mod_test;
/// WebMeta represents the output of the WebBuilder
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WebMeta {
/// Site metadata
pub site_metadata: SiteMetadata,
/// Pages
pub pages: Vec<PageMeta>,
/// Assets
pub assets: std::collections::HashMap<String, AssetMeta>,
}
/// Site metadata
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SiteMetadata {
/// Site name
pub name: String,
/// Site title
pub title: String,
/// Site description
pub description: Option<String>,
/// Site keywords
pub keywords: Option<Vec<String>>,
/// Site header
pub header: Option<serde_json::Value>,
/// Site footer
pub footer: Option<serde_json::Value>,
}
/// Page metadata
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PageMeta {
/// Page ID
pub id: String,
/// Page title
pub title: String,
/// IPFS key of the page content
pub ipfs_key: String,
/// Blake hash of the page content
pub blakehash: String,
/// Page sections
pub sections: Vec<SectionMeta>,
/// Page assets
pub assets: Vec<AssetMeta>,
}
/// Section metadata
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SectionMeta {
/// Section type
#[serde(rename = "type")]
pub section_type: String,
/// Section content
pub content: String,
}
/// Asset metadata
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AssetMeta {
/// Asset name
pub name: String,
/// IPFS key of the asset
pub ipfs_key: String,
}
impl WebMeta {
/// Save the WebMeta to a file
///
/// # Arguments
///
/// * `path` - Path to save the file to
///
/// # Returns
///
/// Ok(()) on success or an error
pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<()> {
let json = serde_json::to_string_pretty(self)?;
fs::write(path, json)?;
Ok(())
}
}
/// WebBuilder is responsible for building a website from hjson configuration files
#[derive(Debug)]
pub struct WebBuilder {
/// Site configuration
pub config: SiteConfig,
}
impl WebBuilder {
/// Create a new WebBuilder instance from a directory containing hjson configuration files
///
/// # Arguments
///
/// * `path` - Path to the directory containing hjson configuration files
///
/// # Returns
///
/// A new WebBuilder instance or an error
pub fn from_directory<P: AsRef<Path>>(path: P) -> Result<Self> {
let config = parser_hjson::parse_site_config(path)?;
Ok(WebBuilder { config })
}
/// Build the website
///
/// # Returns
///
/// A WebMeta instance or an error
pub fn build(&self) -> Result<WebMeta> {
// Create site metadata
let site_metadata = SiteMetadata {
name: self.config.name.clone(),
title: self.config.title.clone(),
description: self.config.description.clone(),
keywords: self.config.keywords.clone(),
header: self
.config
.header
.as_ref()
.map(|h| serde_json::to_value(h).unwrap_or_default()),
footer: self
.config
.footer
.as_ref()
.map(|f| serde_json::to_value(f).unwrap_or_default()),
};
// Process collections
let mut pages = Vec::new();
let assets = std::collections::HashMap::new();
// Process collections from Git repositories
for collection in &self.config.collections {
if let Some(url) = &collection.url {
// Extract repository name from URL
let repo_name = collection.name.clone().unwrap_or_else(|| {
url.split('/')
.last()
.unwrap_or("repo")
.trim_end_matches(".git")
.to_string()
});
// Clone or pull the Git repository
let repo_path = self.config.base_path.join("repos").join(&repo_name);
// Create the repos directory if it doesn't exist
if !repo_path.parent().unwrap().exists() {
fs::create_dir_all(repo_path.parent().unwrap())?;
}
// Clone or pull the repository
let repo_path = match crate::git::clone_or_pull(url, &repo_path) {
Ok(path) => path,
Err(e) => {
// Log the error but continue with a placeholder
log::warn!("Failed to clone repository {}: {}", url, e);
// Create a placeholder page for the failed repository
let page_id = format!("{}-index", repo_name);
let page = PageMeta {
id: page_id.clone(),
title: format!("{} Index", repo_name),
ipfs_key: "QmPlaceholderIpfsKey".to_string(),
blakehash: "blake3-placeholder".to_string(),
sections: vec![SectionMeta {
section_type: "markdown".to_string(),
content: format!(
"# {} Index\n\nFailed to clone repository: {}\nURL: {}",
repo_name, e, url
),
}],
assets: Vec::new(),
};
pages.push(page);
continue;
}
};
// Create a page for the repository
let page_id = format!("{}-index", repo_name);
let page = PageMeta {
id: page_id.clone(),
title: format!("{} Index", repo_name),
ipfs_key: "QmPlaceholderIpfsKey".to_string(), // Will be replaced with actual IPFS key
blakehash: "blake3-placeholder".to_string(), // Will be replaced with actual Blake hash
sections: vec![SectionMeta {
section_type: "markdown".to_string(),
content: format!(
"# {} Index\n\nRepository cloned successfully.\nPath: {}\nURL: {}",
repo_name, repo_path.display(), url
),
}],
assets: Vec::new(),
};
pages.push(page);
}
}
// Process pages from the configuration
for page_config in &self.config.pages {
// Skip draft pages unless explicitly set to false
if page_config.draft.unwrap_or(false) {
log::info!("Skipping draft page: {}", page_config.name);
continue;
}
// Generate a unique page ID
let page_id = format!("page-{}", page_config.name);
// Find the collection for this page
let collection_path = self.config.collections.iter()
.find(|c| c.name.as_ref().map_or(false, |name| name == &page_config.collection))
.and_then(|c| c.url.as_ref())
.map(|url| {
let repo_name = url.split('/')
.last()
.unwrap_or("repo")
.trim_end_matches(".git")
.to_string();
self.config.base_path.join("repos").join(&repo_name)
});
// Create the page content
let content = if let Some(collection_path) = collection_path {
// Try to find the page content in the collection
let page_path = collection_path.join(&page_config.name).with_extension("md");
if page_path.exists() {
match fs::read_to_string(&page_path) {
Ok(content) => content,
Err(e) => {
log::warn!("Failed to read page content from {}: {}", page_path.display(), e);
format!(
"# {}\n\n{}\n\n*Failed to read page content from {}*",
page_config.title,
page_config.description.clone().unwrap_or_default(),
page_path.display()
)
}
}
} else {
format!(
"# {}\n\n{}\n\n*Page content not found at {}*",
page_config.title,
page_config.description.clone().unwrap_or_default(),
page_path.display()
)
}
} else {
format!(
"# {}\n\n{}",
page_config.title,
page_config.description.clone().unwrap_or_default()
)
};
// Calculate the Blake hash of the content
let content_bytes = content.as_bytes();
let blakehash = format!("blake3-{}", blake3::hash(content_bytes).to_hex());
// Create the page metadata
let page = PageMeta {
id: page_id.clone(),
title: page_config.title.clone(),
ipfs_key: "QmPlaceholderIpfsKey".to_string(), // Will be replaced with actual IPFS key
blakehash,
sections: vec![SectionMeta {
section_type: "markdown".to_string(),
content,
}],
assets: Vec::new(),
};
pages.push(page);
}
// Create the WebMeta
Ok(WebMeta {
site_metadata,
pages,
assets,
})
}
/// Upload a file to IPFS
///
/// # Arguments
///
/// * `path` - Path to the file to upload
///
/// # Returns
///
/// The IPFS hash of the file or an error
pub fn upload_to_ipfs<P: AsRef<Path>>(&self, path: P) -> Result<String> {
crate::ipfs::upload_file(path)
}
}

200
webbuilder/src/builder/mod_test.rs Normal file
View File

@@ -0,0 +1,200 @@
#[cfg(test)]
mod tests {
use crate::builder::{PageMeta, SectionMeta, SiteMetadata, WebMeta};
use crate::config::{CollectionConfig, PageConfig, SiteConfig};
use crate::error::WebBuilderError;
use crate::WebBuilder;
use std::fs;
use std::path::PathBuf;
use tempfile::TempDir;
fn create_test_config() -> SiteConfig {
SiteConfig {
name: "test".to_string(),
title: "Test Site".to_string(),
description: Some("A test site".to_string()),
keywords: Some(vec!["test".to_string(), "site".to_string()]),
url: Some("https://example.com".to_string()),
favicon: Some("favicon.ico".to_string()),
header: None,
footer: None,
collections: vec![CollectionConfig {
name: Some("test".to_string()),
url: Some("https://git.ourworld.tf/tfgrid/home.git".to_string()),
description: Some("A test collection".to_string()),
scan: Some(true),
}],
pages: vec![PageConfig {
name: "home".to_string(),
title: "Home".to_string(),
description: Some("Home page".to_string()),
navpath: "/".to_string(),
collection: "test".to_string(),
draft: Some(false),
}],
base_path: PathBuf::from("/path/to/site"),
}
}
#[test]
fn test_webmeta_save() {
let temp_dir = TempDir::new().unwrap();
let output_path = temp_dir.path().join("webmeta.json");
let webmeta = WebMeta {
site_metadata: SiteMetadata {
name: "test".to_string(),
title: "Test Site".to_string(),
description: Some("A test site".to_string()),
keywords: Some(vec!["test".to_string(), "site".to_string()]),
header: None,
footer: None,
},
pages: vec![PageMeta {
id: "page-1".to_string(),
title: "Page 1".to_string(),
ipfs_key: "QmTest1".to_string(),
blakehash: "blake3-test1".to_string(),
sections: vec![SectionMeta {
section_type: "markdown".to_string(),
content: "# Page 1\n\nThis is page 1.".to_string(),
}],
assets: vec![],
}],
assets: std::collections::HashMap::new(),
};
// Save the webmeta.json file
webmeta.save(&output_path).unwrap();
// Check that the file exists
assert!(output_path.exists());
// Read the file and parse it
let content = fs::read_to_string(&output_path).unwrap();
let parsed: WebMeta = serde_json::from_str(&content).unwrap();
// Check that the parsed webmeta matches the original
assert_eq!(parsed.site_metadata.name, webmeta.site_metadata.name);
assert_eq!(parsed.site_metadata.title, webmeta.site_metadata.title);
assert_eq!(
parsed.site_metadata.description,
webmeta.site_metadata.description
);
assert_eq!(
parsed.site_metadata.keywords,
webmeta.site_metadata.keywords
);
assert_eq!(parsed.pages.len(), webmeta.pages.len());
assert_eq!(parsed.pages[0].id, webmeta.pages[0].id);
assert_eq!(parsed.pages[0].title, webmeta.pages[0].title);
assert_eq!(parsed.pages[0].ipfs_key, webmeta.pages[0].ipfs_key);
assert_eq!(parsed.pages[0].blakehash, webmeta.pages[0].blakehash);
assert_eq!(
parsed.pages[0].sections.len(),
webmeta.pages[0].sections.len()
);
assert_eq!(
parsed.pages[0].sections[0].section_type,
webmeta.pages[0].sections[0].section_type
);
assert_eq!(
parsed.pages[0].sections[0].content,
webmeta.pages[0].sections[0].content
);
}
#[test]
fn test_webbuilder_build() {
// Create a temporary directory for the test
let temp_dir = TempDir::new().unwrap();
let site_dir = temp_dir.path().to_path_buf();
// Create a modified test config with the temporary directory as base_path
let mut config = create_test_config();
config.base_path = site_dir.clone();
// Create the repos directory
let repos_dir = site_dir.join("repos");
fs::create_dir_all(&repos_dir).unwrap();
// Create a mock repository directory
let repo_dir = repos_dir.join("home");
fs::create_dir_all(&repo_dir).unwrap();
// Create a mock page file in the repository
let page_content = "# Home Page\n\nThis is the home page content.";
fs::write(repo_dir.join("home.md"), page_content).unwrap();
// Create the WebBuilder with our config
let webbuilder = WebBuilder { config };
// Mock the git module to avoid actual git operations
// This is a simplified test that assumes the git operations would succeed
// Build the website
let webmeta = webbuilder.build().unwrap();
// Check site metadata
assert_eq!(webmeta.site_metadata.name, "test");
assert_eq!(webmeta.site_metadata.title, "Test Site");
assert_eq!(
webmeta.site_metadata.description,
Some("A test site".to_string())
);
assert_eq!(
webmeta.site_metadata.keywords,
Some(vec!["test".to_string(), "site".to_string()])
);
// We expect at least one page from the configuration
assert!(webmeta.pages.len() >= 1);
// Find the page with ID "page-home"
let home_page = webmeta.pages.iter().find(|p| p.id == "page-home");
// Check that we found the page
assert!(home_page.is_some());
let home_page = home_page.unwrap();
// Check the page properties
assert_eq!(home_page.title, "Home");
assert_eq!(home_page.ipfs_key, "QmPlaceholderIpfsKey");
assert_eq!(home_page.sections.len(), 1);
assert_eq!(home_page.sections[0].section_type, "markdown");
// The content should either be our mock content or a placeholder
// depending on whether the page was found
assert!(
home_page.sections[0].content.contains("Home") ||
home_page.sections[0].content.contains("home.md")
);
}
#[test]
fn test_webbuilder_from_directory() {
let temp_dir = TempDir::new().unwrap();
let site_dir = temp_dir.path().join("site");
fs::create_dir(&site_dir).unwrap();
// Create main.hjson
let main_hjson = r#"{ "name": "test", "title": "Test Site" }"#;
fs::write(site_dir.join("main.hjson"), main_hjson).unwrap();
let webbuilder = WebBuilder::from_directory(&site_dir).unwrap();
assert_eq!(webbuilder.config.name, "test");
assert_eq!(webbuilder.config.title, "Test Site");
}
#[test]
fn test_webbuilder_from_directory_error() {
let result = WebBuilder::from_directory("/nonexistent/directory");
assert!(result.is_err());
assert!(matches!(
result.unwrap_err(),
WebBuilderError::MissingDirectory(_)
));
}
}

43
webbuilder/src/builder/webmeta.json
View File

@@ -1,43 +0,0 @@
{
"site_metadata": {
"name": "demo1",
"title": "Demo Site 1",
"description": "This is a demo site for doctree",
"keywords": ["demo", "doctree", "example"],
"header": {
"logo": "/images/logo.png",
"nav": [
{ "text": "Home", "url": "/" },
{ "text": "About", "url": "/about" }
]
},
"footer": {
"copyright": "© 2023 My Company",
"links": [
{ "text": "Privacy Policy", "url": "/privacy" }
]
}
},
"pages": [
{
"id": "mypages1",
"title": "My Pages 1",
"ipfs_key": "QmPlaceholderIpfsKey1",
"blakehash": "sha256-PlaceholderBlakeHash1",
"sections": [
{ "type": "text", "content": "This is example content for My Pages 1." }
],
"assets": [
{
"name": "image1.png",
"ipfs_key": "QmPlaceholderImageIpfsKey1"
}
]
}
],
"assets": {
"style.css": {
"ipfs_key": "QmPlaceholderCssIpfsKey1"
}
}
}

214
webbuilder/src/config.rs Normal file
View File

@@ -0,0 +1,214 @@
use serde::{Deserialize, Serialize};
use std::path::{Path, PathBuf};
use crate::error::{Result, WebBuilderError};
/// Site configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SiteConfig {
/// Site name
pub name: String,
/// Site title
pub title: String,
/// Site description
pub description: Option<String>,
/// Site keywords
pub keywords: Option<Vec<String>>,
/// Site URL
pub url: Option<String>,
/// Site favicon
pub favicon: Option<String>,
/// Site header
pub header: Option<HeaderConfig>,
/// Site footer
pub footer: Option<FooterConfig>,
/// Site collections
pub collections: Vec<CollectionConfig>,
/// Site pages
pub pages: Vec<PageConfig>,
/// Base path of the site configuration
#[serde(skip)]
pub base_path: PathBuf,
}
/// Header configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HeaderConfig {
/// Header logo
pub logo: Option<LogoConfig>,
/// Header title
pub title: Option<String>,
/// Header menu
pub menu: Option<Vec<MenuItemConfig>>,
/// Login button
pub login: Option<LoginConfig>,
}
/// Logo configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LogoConfig {
/// Logo source
pub src: String,
/// Logo alt text
pub alt: Option<String>,
}
/// Menu item configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MenuItemConfig {
/// Menu item label
pub label: String,
/// Menu item link
pub link: String,
/// Menu item children
pub children: Option<Vec<MenuItemConfig>>,
}
/// Login button configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LoginConfig {
/// Whether the login button is visible
pub visible: bool,
/// Login button label
pub label: Option<String>,
/// Login button link
pub link: Option<String>,
}
/// Footer configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FooterConfig {
/// Footer title
pub title: Option<String>,
/// Footer sections
pub sections: Option<Vec<FooterSectionConfig>>,
/// Footer copyright
pub copyright: Option<String>,
}
/// Footer section configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FooterSectionConfig {
/// Section title
pub title: String,
/// Section links
pub links: Vec<LinkConfig>,
}
/// Link configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LinkConfig {
/// Link label
pub label: String,
/// Link URL
pub href: String,
}
/// Collection configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CollectionConfig {
/// Collection name
pub name: Option<String>,
/// Collection URL
pub url: Option<String>,
/// Collection description
pub description: Option<String>,
/// Whether to scan the URL for collections
pub scan: Option<bool>,
}
/// Page configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PageConfig {
/// Page name
pub name: String,
/// Page title
pub title: String,
/// Page description
pub description: Option<String>,
/// Page navigation path
pub navpath: String,
/// Page collection
pub collection: String,
/// Whether the page is a draft
pub draft: Option<bool>,
}
impl SiteConfig {
/// Load site configuration from a directory
///
/// # Arguments
///
/// * `path` - Path to the directory containing hjson configuration files
///
/// # Returns
///
/// A new SiteConfig instance or an error
pub fn from_directory<P: AsRef<Path>>(path: P) -> Result<Self> {
let path = path.as_ref();
// Check if the directory exists
if !path.exists() {
return Err(WebBuilderError::MissingDirectory(path.to_path_buf()));
}
// Check if the directory is a directory
if !path.is_dir() {
return Err(WebBuilderError::InvalidConfiguration(format!(
"{:?} is not a directory",
path
)));
}
// TODO: Implement loading configuration from hjson files
// For now, return a placeholder configuration
Ok(SiteConfig {
name: "demo1".to_string(),
title: "Demo Site 1".to_string(),
description: Some("This is a demo site for doctree".to_string()),
keywords: Some(vec![
"demo".to_string(),
"doctree".to_string(),
"example".to_string(),
]),
url: Some("https://example.com".to_string()),
favicon: Some("img/favicon.png".to_string()),
header: None,
footer: None,
collections: Vec::new(),
pages: Vec::new(),
base_path: path.to_path_buf(),
})
}
}

156
webbuilder/src/config_test.rs Normal file
View File

@@ -0,0 +1,156 @@
#[cfg(test)]
mod tests {
use crate::config::{
CollectionConfig, FooterConfig, FooterSectionConfig, HeaderConfig, LinkConfig, LoginConfig,
LogoConfig, MenuItemConfig, PageConfig, SiteConfig,
};
use std::path::PathBuf;
#[test]
fn test_site_config_serialization() {
let config = SiteConfig {
name: "test".to_string(),
title: "Test Site".to_string(),
description: Some("A test site".to_string()),
keywords: Some(vec!["test".to_string(), "site".to_string()]),
url: Some("https://example.com".to_string()),
favicon: Some("favicon.ico".to_string()),
header: Some(HeaderConfig {
logo: Some(LogoConfig {
src: "logo.png".to_string(),
alt: Some("Logo".to_string()),
}),
title: Some("Test Site".to_string()),
menu: Some(vec![
MenuItemConfig {
label: "Home".to_string(),
link: "/".to_string(),
children: None,
},
MenuItemConfig {
label: "About".to_string(),
link: "/about".to_string(),
children: Some(vec![MenuItemConfig {
label: "Team".to_string(),
link: "/about/team".to_string(),
children: None,
}]),
},
]),
login: Some(LoginConfig {
visible: true,
label: Some("Login".to_string()),
link: Some("/login".to_string()),
}),
}),
footer: Some(FooterConfig {
title: Some("Test Site".to_string()),
sections: Some(vec![FooterSectionConfig {
title: "Links".to_string(),
links: vec![
LinkConfig {
label: "Home".to_string(),
href: "/".to_string(),
},
LinkConfig {
label: "About".to_string(),
href: "/about".to_string(),
},
],
}]),
copyright: Some("© 2023".to_string()),
}),
collections: vec![CollectionConfig {
name: Some("test".to_string()),
url: Some("https://git.ourworld.tf/tfgrid/home.git".to_string()),
description: Some("A test collection".to_string()),
scan: Some(true),
}],
pages: vec![PageConfig {
name: "home".to_string(),
title: "Home".to_string(),
description: Some("Home page".to_string()),
navpath: "/".to_string(),
collection: "test".to_string(),
draft: Some(false),
}],
base_path: PathBuf::from("/path/to/site"),
};
// Serialize to JSON
let json = serde_json::to_string(&config).unwrap();
// Deserialize from JSON
let deserialized: SiteConfig = serde_json::from_str(&json).unwrap();
// Check that the deserialized config matches the original
assert_eq!(deserialized.name, config.name);
assert_eq!(deserialized.title, config.title);
assert_eq!(deserialized.description, config.description);
assert_eq!(deserialized.keywords, config.keywords);
assert_eq!(deserialized.url, config.url);
assert_eq!(deserialized.favicon, config.favicon);
// Check header
assert!(deserialized.header.is_some());
let header = deserialized.header.as_ref().unwrap();
let original_header = config.header.as_ref().unwrap();
// Check logo
assert!(header.logo.is_some());
let logo = header.logo.as_ref().unwrap();
let original_logo = original_header.logo.as_ref().unwrap();
assert_eq!(logo.src, original_logo.src);
assert_eq!(logo.alt, original_logo.alt);
// Check title
assert_eq!(header.title, original_header.title);
// Check menu
assert!(header.menu.is_some());
let menu = header.menu.as_ref().unwrap();
let original_menu = original_header.menu.as_ref().unwrap();
assert_eq!(menu.len(), original_menu.len());
assert_eq!(menu[0].label, original_menu[0].label);
assert_eq!(menu[0].link, original_menu[0].link);
assert_eq!(menu[1].label, original_menu[1].label);
assert_eq!(menu[1].link, original_menu[1].link);
// Check login
assert!(header.login.is_some());
let login = header.login.as_ref().unwrap();
let original_login = original_header.login.as_ref().unwrap();
assert_eq!(login.visible, original_login.visible);
assert_eq!(login.label, original_login.label);
assert_eq!(login.link, original_login.link);
// Check footer
assert!(deserialized.footer.is_some());
let footer = deserialized.footer.as_ref().unwrap();
let original_footer = config.footer.as_ref().unwrap();
assert_eq!(footer.title, original_footer.title);
assert_eq!(footer.copyright, original_footer.copyright);
// Check collections
assert_eq!(deserialized.collections.len(), config.collections.len());
assert_eq!(deserialized.collections[0].name, config.collections[0].name);
assert_eq!(deserialized.collections[0].url, config.collections[0].url);
assert_eq!(
deserialized.collections[0].description,
config.collections[0].description
);
assert_eq!(deserialized.collections[0].scan, config.collections[0].scan);
// Check pages
assert_eq!(deserialized.pages.len(), config.pages.len());
assert_eq!(deserialized.pages[0].name, config.pages[0].name);
assert_eq!(deserialized.pages[0].title, config.pages[0].title);
assert_eq!(
deserialized.pages[0].description,
config.pages[0].description
);
assert_eq!(deserialized.pages[0].navpath, config.pages[0].navpath);
assert_eq!(deserialized.pages[0].collection, config.pages[0].collection);
assert_eq!(deserialized.pages[0].draft, config.pages[0].draft);
}
}

68
webbuilder/src/error.rs Normal file
View File

@@ -0,0 +1,68 @@
use std::io;
use std::path::PathBuf;
use thiserror::Error;
/// Result type for WebBuilder operations
pub type Result<T> = std::result::Result<T, WebBuilderError>;
/// Error type for WebBuilder operations
#[derive(Error, Debug)]
pub enum WebBuilderError {
/// IO error
#[error("IO error: {0}")]
IoError(#[from] io::Error),
/// DocTree error
#[error("DocTree error: {0}")]
DocTreeError(#[from] doctree::DocTreeError),
/// Hjson parsing error
#[error("Hjson parsing error: {0}")]
HjsonError(String),
/// Git error
#[error("Git error: {0}")]
GitError(String),
/// IPFS error
#[error("IPFS error: {0}")]
IpfsError(String),
/// Missing file error
#[error("Missing file: {0}")]
MissingFile(PathBuf),
/// Missing directory error
#[error("Missing directory: {0}")]
MissingDirectory(PathBuf),
/// Missing configuration error
#[error("Missing configuration: {0}")]
MissingConfiguration(String),
/// Invalid configuration error
#[error("Invalid configuration: {0}")]
InvalidConfiguration(String),
/// Other error
#[error("Error: {0}")]
Other(String),
}
impl From<String> for WebBuilderError {
fn from(error: String) -> Self {
WebBuilderError::Other(error)
}
}
impl From<&str> for WebBuilderError {
fn from(error: &str) -> Self {
WebBuilderError::Other(error.to_string())
}
}
impl From<serde_json::Error> for WebBuilderError {
fn from(error: serde_json::Error) -> Self {
WebBuilderError::Other(format!("JSON error: {}", error))
}
}

73
webbuilder/src/error_test.rs Normal file
View File

@@ -0,0 +1,73 @@
#[cfg(test)]
mod tests {
use crate::error::WebBuilderError;
use std::path::PathBuf;
#[test]
fn test_error_from_string() {
let error = WebBuilderError::from("test error");
assert!(matches!(error, WebBuilderError::Other(s) if s == "test error"));
}
#[test]
fn test_error_from_string_owned() {
let error = WebBuilderError::from("test error".to_string());
assert!(matches!(error, WebBuilderError::Other(s) if s == "test error"));
}
#[test]
fn test_error_from_json_error() {
let json_error = serde_json::from_str::<serde_json::Value>("invalid json").unwrap_err();
let error = WebBuilderError::from(json_error);
assert!(matches!(error, WebBuilderError::Other(s) if s.starts_with("JSON error:")));
}
#[test]
fn test_error_display() {
let errors = vec![
(
WebBuilderError::IoError(std::io::Error::new(
std::io::ErrorKind::NotFound,
"file not found",
)),
"IO error: file not found",
),
(
WebBuilderError::HjsonError("invalid hjson".to_string()),
"Hjson parsing error: invalid hjson",
),
(
WebBuilderError::GitError("git error".to_string()),
"Git error: git error",
),
(
WebBuilderError::IpfsError("ipfs error".to_string()),
"IPFS error: ipfs error",
),
(
WebBuilderError::MissingFile(PathBuf::from("/path/to/file")),
"Missing file: /path/to/file",
),
(
WebBuilderError::MissingDirectory(PathBuf::from("/path/to/dir")),
"Missing directory: /path/to/dir",
),
(
WebBuilderError::MissingConfiguration("config".to_string()),
"Missing configuration: config",
),
(
WebBuilderError::InvalidConfiguration("invalid config".to_string()),
"Invalid configuration: invalid config",
),
(
WebBuilderError::Other("other error".to_string()),
"Error: other error",
),
];
for (error, expected) in errors {
assert_eq!(error.to_string(), expected);
}
}
}

182
webbuilder/src/git.rs Normal file
View File

@@ -0,0 +1,182 @@
use lazy_static::lazy_static;
use sal::git::{GitRepo, GitTree};
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::{ SystemTime};
use crate::error::{Result, WebBuilderError};
// Cache entry for Git repositories
struct CacheEntry {
path: PathBuf,
last_updated: SystemTime,
}
// Global cache for Git repositories
lazy_static! {
static ref REPO_CACHE: Arc<Mutex<HashMap<String, CacheEntry>>> =
Arc::new(Mutex::new(HashMap::new()));
}
// Cache timeout in seconds (default: 1 hour)
const CACHE_TIMEOUT: u64 = 3600;
/// Clone a Git repository
///
/// # Arguments
///
/// * `url` - URL of the repository to clone
/// * `destination` - Destination directory
///
/// # Returns
///
/// The path to the cloned repository or an error
pub fn clone_repository<P: AsRef<Path>>(url: &str, destination: P) -> Result<PathBuf> {
let destination = destination.as_ref();
let destination_str = destination.to_str().unwrap();
// Create a GitTree for the parent directory
let parent_dir = destination.parent().ok_or_else(|| {
WebBuilderError::InvalidConfiguration(format!(
"Invalid destination path: {}",
destination_str
))
})?;
let git_tree = GitTree::new(parent_dir.to_str().unwrap())
.map_err(|e| WebBuilderError::GitError(format!("Failed to create GitTree: {}", e)))?;
// Use the GitTree to get (clone) the repository
let repos = git_tree
.get(url)
.map_err(|e| WebBuilderError::GitError(format!("Failed to clone repository: {}", e)))?;
if repos.is_empty() {
return Err(WebBuilderError::GitError(format!(
"Failed to clone repository: No repository was created"
)));
}
// Return the path of the first repository
Ok(PathBuf::from(repos[0].path()))
}
/// Pull the latest changes from a Git repository
///
/// # Arguments
///
/// * `path` - Path to the repository
///
/// # Returns
///
/// Ok(()) on success or an error
pub fn pull_repository<P: AsRef<Path>>(path: P) -> Result<()> {
let path = path.as_ref();
let path_str = path.to_str().unwrap();
// Create a GitRepo directly
let repo = GitRepo::new(path_str.to_string());
// Pull the repository
repo.pull()
.map_err(|e| WebBuilderError::GitError(format!("Failed to pull repository: {}", e)))?;
Ok(())
}
/// Clone or pull a Git repository with caching
///
/// # Arguments
///
/// * `url` - URL of the repository to clone
/// * `destination` - Destination directory
///
/// # Returns
///
/// The path to the repository or an error
pub fn clone_or_pull<P: AsRef<Path>>(url: &str, destination: P) -> Result<PathBuf> {
let destination = destination.as_ref();
// Check the cache first
let mut cache = REPO_CACHE.lock().unwrap();
let now = SystemTime::now();
if let Some(entry) = cache.get(url) {
// Check if the cache entry is still valid
if let Ok(elapsed) = now.duration_since(entry.last_updated) {
if elapsed.as_secs() < CACHE_TIMEOUT {
// Cache is still valid, return the cached path
log::info!("Using cached repository for {}", url);
return Ok(entry.path.clone());
}
}
}
// Cache miss or expired, clone or pull the repository
let result = if destination.exists() {
// Pull the repository
pull_repository(destination)?;
Ok(destination.to_path_buf())
} else {
// Clone the repository
clone_repository(url, destination)
};
// Update the cache
if let Ok(path) = &result {
cache.insert(
url.to_string(),
CacheEntry {
path: path.clone(),
last_updated: now,
},
);
}
result
}
/// Force update a Git repository, bypassing the cache
///
/// # Arguments
///
/// * `url` - URL of the repository to clone
/// * `destination` - Destination directory
///
/// # Returns
///
/// The path to the repository or an error
pub fn force_update<P: AsRef<Path>>(url: &str, destination: P) -> Result<PathBuf> {
let destination = destination.as_ref();
// Clone or pull the repository
let result = if destination.exists() {
// Pull the repository
pull_repository(destination)?;
Ok(destination.to_path_buf())
} else {
// Clone the repository
clone_repository(url, destination)
};
// Update the cache
if let Ok(path) = &result {
let mut cache = REPO_CACHE.lock().unwrap();
cache.insert(
url.to_string(),
CacheEntry {
path: path.clone(),
last_updated: SystemTime::now(),
},
);
}
result
}
/// Clear the Git repository cache
pub fn clear_cache() {
let mut cache = REPO_CACHE.lock().unwrap();
cache.clear();
}

25
webbuilder/src/git_test.rs Normal file
View File

@@ -0,0 +1,25 @@
#[cfg(test)]
mod tests {
use crate::error::WebBuilderError;
use crate::git::clone_repository;
use std::path::PathBuf;
#[test]
fn test_clone_repository_error_invalid_destination() {
// Test with a destination that has no parent directory
let result = clone_repository("https://git.ourworld.tf/tfgrid/home.git", PathBuf::from("/"));
assert!(result.is_err());
assert!(matches!(
result.unwrap_err(),
WebBuilderError::InvalidConfiguration(_)
));
}
// Note: The following tests would require mocking the sal::git module,
// which is complex due to the external dependency. In a real-world scenario,
// we would use a more sophisticated mocking approach or integration tests.
// For now, we'll just test the error cases and leave the success cases
// for integration testing.
}

70
webbuilder/src/ipfs.rs Normal file
View File

@@ -0,0 +1,70 @@
use ipfs_api_backend_hyper::{IpfsApi, IpfsClient};
use std::fs::File;
use std::path::Path;
use tokio::runtime::Runtime;
use crate::error::{Result, WebBuilderError};
/// Upload a file to IPFS
///
/// # Arguments
///
/// * `path` - Path to the file to upload
///
/// # Returns
///
/// The IPFS hash of the file or an error
pub fn upload_file<P: AsRef<Path>>(path: P) -> Result<String> {
let path = path.as_ref();
// Check if the file exists
if !path.exists() {
return Err(WebBuilderError::MissingFile(path.to_path_buf()));
}
// Create a tokio runtime
let rt = Runtime::new()
.map_err(|e| WebBuilderError::Other(format!("Failed to create tokio runtime: {}", e)))?;
// Upload the file to IPFS
let client = IpfsClient::default();
let ipfs_hash = rt.block_on(async {
// Open the file directly - this implements Read trait
let file = File::open(path).map_err(|e| WebBuilderError::IoError(e))?;
client
.add(file)
.await
.map_err(|e| WebBuilderError::IpfsError(format!("Failed to upload to IPFS: {}", e)))
.map(|res| res.hash)
})?;
Ok(ipfs_hash)
}
/// Calculate the Blake3 hash of a file
///
/// # Arguments
///
/// * `path` - Path to the file to hash
///
/// # Returns
///
/// The Blake3 hash of the file or an error
pub fn calculate_blake_hash<P: AsRef<Path>>(path: P) -> Result<String> {
let path = path.as_ref();
// Check if the file exists
if !path.exists() {
return Err(WebBuilderError::MissingFile(path.to_path_buf()));
}
// Read the file
let content = std::fs::read(path).map_err(|e| WebBuilderError::IoError(e))?;
// Calculate the hash
let hash = blake3::hash(&content);
let hash_hex = hash.to_hex().to_string();
Ok(format!("blake3-{}", hash_hex))
}

64
webbuilder/src/ipfs_test.rs Normal file
View File

@@ -0,0 +1,64 @@
#[cfg(test)]
mod tests {
use crate::error::WebBuilderError;
use crate::ipfs::{calculate_blake_hash, upload_file};
use std::fs;
use tempfile::TempDir;
#[test]
fn test_upload_file_missing_file() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("nonexistent.txt");
let result = upload_file(&file_path);
assert!(result.is_err());
assert!(matches!(
result.unwrap_err(),
WebBuilderError::MissingFile(_)
));
}
#[test]
fn test_calculate_blake_hash() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("test.txt");
fs::write(&file_path, "test content").unwrap();
let result = calculate_blake_hash(&file_path).unwrap();
// The hash should start with "blake3-"
assert!(result.starts_with("blake3-"));
// The hash should be 64 characters long after the prefix
assert_eq!(result.len(), "blake3-".len() + 64);
// The hash should be the same for the same content
let file_path2 = temp_dir.path().join("test2.txt");
fs::write(&file_path2, "test content").unwrap();
let result2 = calculate_blake_hash(&file_path2).unwrap();
assert_eq!(result, result2);
// The hash should be different for different content
let file_path3 = temp_dir.path().join("test3.txt");
fs::write(&file_path3, "different content").unwrap();
let result3 = calculate_blake_hash(&file_path3).unwrap();
assert_ne!(result, result3);
}
#[test]
fn test_calculate_blake_hash_missing_file() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("nonexistent.txt");
let result = calculate_blake_hash(&file_path);
assert!(result.is_err());
assert!(matches!(
result.unwrap_err(),
WebBuilderError::MissingFile(_)
));
}
}

43
webbuilder/src/lib.rs Normal file
View File

@@ -0,0 +1,43 @@
//! WebBuilder is a library for building websites from hjson configuration files and markdown content.
//!
//! It uses the DocTree library to process markdown content and includes, and exports the result
//! to a webmeta.json file that can be used by a browser-based website generator.
pub mod builder;
pub mod config;
pub mod error;
pub mod git;
pub mod ipfs;
pub mod parser;
pub mod parser_simple;
pub mod parser_hjson;
#[cfg(test)]
mod config_test;
#[cfg(test)]
mod error_test;
#[cfg(test)]
mod git_test;
#[cfg(test)]
mod ipfs_test;
#[cfg(test)]
mod parser_simple_test;
#[cfg(test)]
mod parser_hjson_test;
pub use builder::WebBuilder;
pub use config::SiteConfig;
pub use error::{Result, WebBuilderError};
/// Create a new WebBuilder instance from a directory containing hjson configuration files.
///
/// # Arguments
///
/// * `path` - Path to the directory containing hjson configuration files
///
/// # Returns
///
/// A new WebBuilder instance or an error
pub fn from_directory<P: AsRef<std::path::Path>>(path: P) -> Result<WebBuilder> {
WebBuilder::from_directory(path)
}

88
webbuilder/src/main.rs Normal file
View File

@@ -0,0 +1,88 @@
use clap::{Parser, Subcommand};
use std::path::PathBuf;
use webbuilder::{from_directory, Result};
#[derive(Parser)]
#[command(author, version, about, long_about = None)]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Subcommand)]
enum Commands {
/// Build a website from hjson configuration files
Build {
/// Path to the directory containing hjson configuration files
#[arg(short, long)]
path: PathBuf,
/// Output directory for the webmeta.json file
#[arg(short, long)]
output: Option<PathBuf>,
/// Whether to upload the webmeta.json file to IPFS
#[arg(short, long)]
upload: bool,
},
}
fn main() -> Result<()> {
// Initialize logger
env_logger::init();
// Parse command line arguments
let cli = Cli::parse();
// Handle commands
match &cli.command {
Commands::Build {
path,
output,
upload,
} => {
// Create a WebBuilder instance
let webbuilder = from_directory(path)?;
// Print the parsed configuration
println!("Parsed site configuration:");
println!(" Name: {}", webbuilder.config.name);
println!(" Title: {}", webbuilder.config.title);
println!(" Description: {:?}", webbuilder.config.description);
println!(" URL: {:?}", webbuilder.config.url);
println!(
" Collections: {} items",
webbuilder.config.collections.len()
);
for (i, collection) in webbuilder.config.collections.iter().enumerate() {
println!(
" Collection {}: {:?} - {:?}",
i, collection.name, collection.url
);
}
println!(" Pages: {} items", webbuilder.config.pages.len());
// Build the website
let webmeta = webbuilder.build()?;
// Save the webmeta.json file
let output_path = output
.clone()
.unwrap_or_else(|| PathBuf::from("webmeta.json"));
webmeta.save(&output_path)?;
// Upload to IPFS if requested
if *upload {
let ipfs_hash = webbuilder.upload_to_ipfs(&output_path)?;
println!("Uploaded to IPFS: {}", ipfs_hash);
}
println!("Website built successfully!");
println!("Output: {:?}", output_path);
}
}
Ok(())
}

264
webbuilder/src/parser.rs Normal file
View File

@@ -0,0 +1,264 @@
use serde::de::DeserializeOwned;
use serde_json;
use std::fs;
use std::path::Path;
use crate::config::{CollectionConfig, FooterConfig, HeaderConfig, PageConfig, SiteConfig};
use crate::error::{Result, WebBuilderError};
/// Parse a hjson file into a struct
///
/// # Arguments
///
/// * `path` - Path to the hjson file
///
/// # Returns
///
/// The parsed struct or an error
pub fn parse_hjson<T, P>(path: P) -> Result<T>
where
T: DeserializeOwned,
P: AsRef<Path>,
{
let path = path.as_ref();
// Check if the file exists
if !path.exists() {
return Err(WebBuilderError::MissingFile(path.to_path_buf()));
}
// Read the file
let content = fs::read_to_string(path).map_err(|e| WebBuilderError::IoError(e))?;
// First try to parse as JSON
let json_result = serde_json::from_str::<T>(&content);
if json_result.is_ok() {
return Ok(json_result.unwrap());
}
// If that fails, try to convert hjson to json using a simple approach
let json_content = convert_hjson_to_json(&content)?;
// Parse the JSON
serde_json::from_str(&json_content)
.map_err(|e| WebBuilderError::HjsonError(format!("Error parsing {:?}: {}", path, e)))
}
/// Convert hjson to json using a simple approach
///
/// # Arguments
///
/// * `hjson` - The hjson content
///
/// # Returns
///
/// The json content or an error
fn convert_hjson_to_json(hjson: &str) -> Result<String> {
// Remove comments
let mut json = String::new();
let mut lines = hjson.lines();
while let Some(line) = lines.next() {
let trimmed = line.trim();
// Skip empty lines
if trimmed.is_empty() {
continue;
}
// Skip comment lines
if trimmed.starts_with('#') {
continue;
}
// Handle key-value pairs
if let Some(pos) = trimmed.find(':') {
let key = trimmed[..pos].trim();
let value = trimmed[pos + 1..].trim();
// Add quotes to keys
json.push_str(&format!("\"{}\":", key));
// Add value
if value.is_empty() {
// If value is empty, it might be an object or array start
if lines
.clone()
.next()
.map_or(false, |l| l.trim().starts_with('{'))
{
json.push_str(" {");
} else if lines
.clone()
.next()
.map_or(false, |l| l.trim().starts_with('['))
{
json.push_str(" [");
} else {
json.push_str(" null");
}
} else {
// Add quotes to string values
if value.starts_with('"')
|| value.starts_with('[')
|| value.starts_with('{')
|| value == "true"
|| value == "false"
|| value == "null"
|| value.parse::<f64>().is_ok()
{
json.push_str(&format!(" {}", value));
} else {
json.push_str(&format!(" \"{}\"", value.replace('"', "\\\"")));
}
}
json.push_str(",\n");
} else if trimmed == "{" || trimmed == "[" {
json.push_str(trimmed);
json.push_str("\n");
} else if trimmed == "}" || trimmed == "]" {
// Remove trailing comma if present
if json.ends_with(",\n") {
json.pop();
json.pop();
json.push_str("\n");
}
json.push_str(trimmed);
json.push_str(",\n");
} else {
// Just copy the line
json.push_str(trimmed);
json.push_str("\n");
}
}
// Remove trailing comma if present
if json.ends_with(",\n") {
json.pop();
json.pop();
json.push_str("\n");
}
// Wrap in object if not already
if !json.trim().starts_with('{') {
json = format!("{{\n{}\n}}", json);
}
Ok(json)
}
/// Parse site configuration from a directory
///
/// # Arguments
///
/// * `path` - Path to the directory containing hjson configuration files
///
/// # Returns
///
/// The parsed site configuration or an error
pub fn parse_site_config<P: AsRef<Path>>(path: P) -> Result<SiteConfig> {
let path = path.as_ref();
// Check if the directory exists
if !path.exists() {
return Err(WebBuilderError::MissingDirectory(path.to_path_buf()));
}
// Check if the directory is a directory
if !path.is_dir() {
return Err(WebBuilderError::InvalidConfiguration(format!(
"{:?} is not a directory",
path
)));
}
// Parse main.hjson
let main_path = path.join("main.hjson");
let main_config: serde_json::Value = parse_hjson(main_path)?;
// Parse header.hjson
let header_path = path.join("header.hjson");
let header_config: Option<HeaderConfig> = if header_path.exists() {
Some(parse_hjson(header_path)?)
} else {
None
};
// Parse footer.hjson
let footer_path = path.join("footer.hjson");
let footer_config: Option<FooterConfig> = if footer_path.exists() {
Some(parse_hjson(footer_path)?)
} else {
None
};
// Parse collection.hjson
let collection_path = path.join("collection.hjson");
let collection_configs: Vec<CollectionConfig> = if collection_path.exists() {
parse_hjson(collection_path)?
} else {
Vec::new()
};
// Parse pages directory
let pages_path = path.join("pages");
let mut page_configs: Vec<PageConfig> = Vec::new();
if pages_path.exists() && pages_path.is_dir() {
for entry in fs::read_dir(pages_path)? {
let entry = entry?;
let entry_path = entry.path();
if entry_path.is_file() && entry_path.extension().map_or(false, |ext| ext == "hjson") {
let page_config: Vec<PageConfig> = parse_hjson(&entry_path)?;
page_configs.extend(page_config);
}
}
}
// Parse keywords from main.hjson
let keywords = if let Some(keywords_value) = main_config.get("keywords") {
if keywords_value.is_array() {
let mut keywords_vec = Vec::new();
for keyword in keywords_value.as_array().unwrap() {
if let Some(keyword_str) = keyword.as_str() {
keywords_vec.push(keyword_str.to_string());
}
}
Some(keywords_vec)
} else if let Some(keywords_str) = keywords_value.as_str() {
// Handle comma-separated keywords
Some(
keywords_str
.split(',')
.map(|s| s.trim().to_string())
.collect(),
)
} else {
None
}
} else {
None
};
// Create site configuration
let site_config = SiteConfig {
name: main_config["name"]
.as_str()
.unwrap_or("default")
.to_string(),
title: main_config["title"].as_str().unwrap_or("").to_string(),
description: main_config["description"].as_str().map(|s| s.to_string()),
keywords,
url: main_config["url"].as_str().map(|s| s.to_string()),
favicon: main_config["favicon"].as_str().map(|s| s.to_string()),
header: header_config,
footer: footer_config,
collections: collection_configs,
pages: page_configs,
base_path: path.to_path_buf(),
};
Ok(site_config)
}

161
webbuilder/src/parser_hjson.rs Normal file
View File

@@ -0,0 +1,161 @@
use std::fs;
use std::path::Path;
use deser_hjson::from_str;
use serde::de::DeserializeOwned;
use serde_json::Value;
use crate::config::{
CollectionConfig, PageConfig, SiteConfig,
};
use crate::error::{Result, WebBuilderError};
/// Parse a hjson file into a struct
///
/// # Arguments
///
/// * `path` - Path to the hjson file
///
/// # Returns
///
/// The parsed struct or an error
pub fn parse_hjson<T, P>(path: P) -> Result<T>
where
T: DeserializeOwned,
P: AsRef<Path>,
{
let path = path.as_ref();
// Check if the file exists
if !path.exists() {
return Err(WebBuilderError::MissingFile(path.to_path_buf()));
}
// Read the file
let content = fs::read_to_string(path).map_err(|e| WebBuilderError::IoError(e))?;
// Parse the hjson
from_str(&content).map_err(|e| WebBuilderError::HjsonError(format!("Error parsing {:?}: {}", path, e)))
}
/// Parse site configuration from a directory
///
/// # Arguments
///
/// * `path` - Path to the directory containing hjson configuration files
///
/// # Returns
///
/// The parsed site configuration or an error
pub fn parse_site_config<P: AsRef<Path>>(path: P) -> Result<SiteConfig> {
let path = path.as_ref();
// Check if the directory exists
if !path.exists() {
return Err(WebBuilderError::MissingDirectory(path.to_path_buf()));
}
// Check if the directory is a directory
if !path.is_dir() {
return Err(WebBuilderError::InvalidConfiguration(format!(
"{:?} is not a directory",
path
)));
}
// Create a basic site configuration
let mut site_config = SiteConfig {
name: "default".to_string(),
title: "".to_string(),
description: None,
keywords: None,
url: None,
favicon: None,
header: None,
footer: None,
collections: Vec::new(),
pages: Vec::new(),
base_path: path.to_path_buf(),
};
// Parse main.hjson
let main_path = path.join("main.hjson");
if main_path.exists() {
let main_config: Value = parse_hjson(main_path)?;
// Extract values from main.hjson
if let Some(name) = main_config.get("name").and_then(|v| v.as_str()) {
site_config.name = name.to_string();
}
if let Some(title) = main_config.get("title").and_then(|v| v.as_str()) {
site_config.title = title.to_string();
}
if let Some(description) = main_config.get("description").and_then(|v| v.as_str()) {
site_config.description = Some(description.to_string());
}
if let Some(url) = main_config.get("url").and_then(|v| v.as_str()) {
site_config.url = Some(url.to_string());
}
if let Some(favicon) = main_config.get("favicon").and_then(|v| v.as_str()) {
site_config.favicon = Some(favicon.to_string());
}
if let Some(keywords) = main_config.get("keywords").and_then(|v| v.as_array()) {
let keywords_vec: Vec<String> = keywords
.iter()
.filter_map(|k| k.as_str().map(|s| s.to_string()))
.collect();
if !keywords_vec.is_empty() {
site_config.keywords = Some(keywords_vec);
}
}
}
// Parse header.hjson
let header_path = path.join("header.hjson");
if header_path.exists() {
site_config.header = Some(parse_hjson(header_path)?);
}
// Parse footer.hjson
let footer_path = path.join("footer.hjson");
if footer_path.exists() {
site_config.footer = Some(parse_hjson(footer_path)?);
}
// Parse collection.hjson
let collection_path = path.join("collection.hjson");
if collection_path.exists() {
let collection_array: Vec<CollectionConfig> = parse_hjson(collection_path)?;
// Process each collection
for mut collection in collection_array {
// Convert web interface URL to Git URL if needed
if let Some(url) = &collection.url {
if url.contains("/src/branch/") {
// This is a web interface URL, convert it to a Git URL
let parts: Vec<&str> = url.split("/src/branch/").collect();
if parts.len() == 2 {
collection.url = Some(format!("{}.git", parts[0]));
}
}
}
site_config.collections.push(collection);
}
}
// Parse pages directory
let pages_path = path.join("pages");
if pages_path.exists() && pages_path.is_dir() {
for entry in fs::read_dir(pages_path)? {
let entry = entry?;
let entry_path = entry.path();
if entry_path.is_file() && entry_path.extension().map_or(false, |ext| ext == "hjson") {
let pages_array: Vec<PageConfig> = parse_hjson(&entry_path)?;
site_config.pages.extend(pages_array);
}
}
}
Ok(site_config)
}

290
webbuilder/src/parser_hjson_test.rs Normal file
View File

@@ -0,0 +1,290 @@
#[cfg(test)]
mod tests {
use crate::error::WebBuilderError;
use crate::parser_hjson::parse_site_config;
use std::fs;
use std::path::PathBuf;
use tempfile::TempDir;
fn create_test_site(temp_dir: &TempDir) -> PathBuf {
let site_dir = temp_dir.path().join("site");
fs::create_dir(&site_dir).unwrap();
// Create main.hjson
let main_hjson = r#"{
# Main configuration
"name": "test",
"title": "Test Site",
"description": "A test site",
"url": "https://example.com",
"favicon": "favicon.ico",
"keywords": [
"demo",
"test",
"example"
]
}"#;
fs::write(site_dir.join("main.hjson"), main_hjson).unwrap();
// Create header.hjson
let header_hjson = r#"{
# Header configuration
"title": "Test Site",
"logo": {
"src": "logo.png",
"alt": "Logo"
},
"menu": [
{
"label": "Home",
"link": "/"
},
{
"label": "About",
"link": "/about"
}
]
}"#;
fs::write(site_dir.join("header.hjson"), header_hjson).unwrap();
// Create footer.hjson
let footer_hjson = r#"{
# Footer configuration
"title": "Footer",
"copyright": "© 2023 Test",
"sections": [
{
"title": "Links",
"links": [
{
"label": "Home",
"href": "/"
},
{
"label": "About",
"href": "/about"
}
]
}
]
}"#;
fs::write(site_dir.join("footer.hjson"), footer_hjson).unwrap();
// Create collection.hjson
let collection_hjson = r#"[
{
# First collection
"name": "test",
"url": "https://git.ourworld.tf/tfgrid/home.git",
"description": "A test collection",
"scan": true
},
{
# Second collection
"name": "test2",
"url": "https://git.example.com/src/branch/main/test2",
"description": "Another test collection"
}
]"#;
fs::write(site_dir.join("collection.hjson"), collection_hjson).unwrap();
// Create pages directory
let pages_dir = site_dir.join("pages");
fs::create_dir(&pages_dir).unwrap();
// Create pages/pages.hjson
let pages_hjson = r#"[
{
# Home page
"name": "home",
"title": "Home",
"description": "Home page",
"navpath": "/",
"collection": "test",
"draft": false
},
{
# About page
"name": "about",
"title": "About",
"description": "About page",
"navpath": "/about",
"collection": "test"
}
]"#;
fs::write(pages_dir.join("pages.hjson"), pages_hjson).unwrap();
site_dir
}
#[test]
fn test_parse_site_config() {
let temp_dir = TempDir::new().unwrap();
let site_dir = create_test_site(&temp_dir);
let config = parse_site_config(&site_dir).unwrap();
// Check basic site info
assert_eq!(config.name, "test");
assert_eq!(config.title, "Test Site");
assert_eq!(config.description, Some("A test site".to_string()));
assert_eq!(config.url, Some("https://example.com".to_string()));
assert_eq!(config.favicon, Some("favicon.ico".to_string()));
assert_eq!(
config.keywords,
Some(vec![
"demo".to_string(),
"test".to_string(),
"example".to_string()
])
);
// Check header
assert!(config.header.is_some());
let header = config.header.as_ref().unwrap();
assert_eq!(header.title, Some("Test Site".to_string()));
assert!(header.logo.is_some());
let logo = header.logo.as_ref().unwrap();
assert_eq!(logo.src, "logo.png");
assert_eq!(logo.alt, Some("Logo".to_string()));
assert!(header.menu.is_some());
let menu = header.menu.as_ref().unwrap();
assert_eq!(menu.len(), 2);
assert_eq!(menu[0].label, "Home");
assert_eq!(menu[0].link, "/");
// Check footer
assert!(config.footer.is_some());
let footer = config.footer.as_ref().unwrap();
assert_eq!(footer.title, Some("Footer".to_string()));
assert_eq!(footer.copyright, Some("© 2023 Test".to_string()));
assert!(footer.sections.is_some());
let sections = footer.sections.as_ref().unwrap();
assert_eq!(sections.len(), 1);
assert_eq!(sections[0].title, "Links");
assert_eq!(sections[0].links.len(), 2);
assert_eq!(sections[0].links[0].label, "Home");
assert_eq!(sections[0].links[0].href, "/");
// Check collections
assert_eq!(config.collections.len(), 2);
// First collection
assert_eq!(config.collections[0].name, Some("test".to_string()));
assert_eq!(
config.collections[0].url,
Some("https://git.ourworld.tf/tfgrid/home.git".to_string())
);
assert_eq!(
config.collections[0].description,
Some("A test collection".to_string())
);
assert_eq!(config.collections[0].scan, Some(true));
// Second collection (with URL conversion)
assert_eq!(config.collections[1].name, Some("test2".to_string()));
assert_eq!(
config.collections[1].url,
Some("https://git.example.com.git".to_string())
);
assert_eq!(
config.collections[1].description,
Some("Another test collection".to_string())
);
assert_eq!(config.collections[1].scan, None);
// Check pages
assert_eq!(config.pages.len(), 2);
// First page
assert_eq!(config.pages[0].name, "home");
assert_eq!(config.pages[0].title, "Home");
assert_eq!(config.pages[0].description, Some("Home page".to_string()));
assert_eq!(config.pages[0].navpath, "/");
assert_eq!(config.pages[0].collection, "test");
assert_eq!(config.pages[0].draft, Some(false));
// Second page
assert_eq!(config.pages[1].name, "about");
assert_eq!(config.pages[1].title, "About");
assert_eq!(config.pages[1].description, Some("About page".to_string()));
assert_eq!(config.pages[1].navpath, "/about");
assert_eq!(config.pages[1].collection, "test");
assert_eq!(config.pages[1].draft, None);
}
#[test]
fn test_parse_site_config_missing_directory() {
let result = parse_site_config("/nonexistent/directory");
assert!(matches!(result, Err(WebBuilderError::MissingDirectory(_))));
}
#[test]
fn test_parse_site_config_not_a_directory() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("file.txt");
fs::write(&file_path, "not a directory").unwrap();
let result = parse_site_config(&file_path);
assert!(matches!(
result,
Err(WebBuilderError::InvalidConfiguration(_))
));
}
#[test]
fn test_parse_site_config_minimal() {
let temp_dir = TempDir::new().unwrap();
let site_dir = temp_dir.path().join("site");
fs::create_dir(&site_dir).unwrap();
// Create minimal main.hjson
let main_hjson = r#"{ "name": "minimal", "title": "Minimal Site" }"#;
fs::write(site_dir.join("main.hjson"), main_hjson).unwrap();
let config = parse_site_config(&site_dir).unwrap();
assert_eq!(config.name, "minimal");
assert_eq!(config.title, "Minimal Site");
assert_eq!(config.description, None);
assert_eq!(config.url, None);
assert_eq!(config.favicon, None);
assert!(config.header.is_none());
assert!(config.footer.is_none());
assert!(config.collections.is_empty());
assert!(config.pages.is_empty());
}
#[test]
fn test_parse_site_config_empty() {
let temp_dir = TempDir::new().unwrap();
let site_dir = temp_dir.path().join("site");
fs::create_dir(&site_dir).unwrap();
let config = parse_site_config(&site_dir).unwrap();
assert_eq!(config.name, "default");
assert_eq!(config.title, "");
assert_eq!(config.description, None);
assert_eq!(config.url, None);
assert_eq!(config.favicon, None);
assert!(config.header.is_none());
assert!(config.footer.is_none());
assert!(config.collections.is_empty());
assert!(config.pages.is_empty());
}
#[test]
fn test_parse_site_config_invalid_hjson() {
let temp_dir = TempDir::new().unwrap();
let site_dir = temp_dir.path().join("site");
fs::create_dir(&site_dir).unwrap();
// Create invalid main.hjson
let main_hjson = r#"{ name: "test, title: "Test Site" }"#; // Missing closing quote
fs::write(site_dir.join("main.hjson"), main_hjson).unwrap();
let result = parse_site_config(&site_dir);
assert!(matches!(result, Err(WebBuilderError::HjsonError(_))));
}
}

277
webbuilder/src/parser_simple.rs Normal file
View File

@@ -0,0 +1,277 @@
use std::fs;
use std::path::Path;
use crate::config::{
CollectionConfig, HeaderConfig, LogoConfig, PageConfig, SiteConfig,
};
use crate::error::{Result, WebBuilderError};
/// Parse site configuration from a directory using a simple approach
///
/// # Arguments
///
/// * `path` - Path to the directory containing hjson configuration files
///
/// # Returns
///
/// The parsed site configuration or an error
pub fn parse_site_config<P: AsRef<Path>>(path: P) -> Result<SiteConfig> {
let path = path.as_ref();
// Check if the directory exists
if !path.exists() {
return Err(WebBuilderError::MissingDirectory(path.to_path_buf()));
}
// Check if the directory is a directory
if !path.is_dir() {
return Err(WebBuilderError::InvalidConfiguration(format!(
"{:?} is not a directory",
path
)));
}
// Create a basic site configuration
let mut site_config = SiteConfig {
name: "default".to_string(),
title: "".to_string(),
description: None,
keywords: None,
url: None,
favicon: None,
header: None,
footer: None,
collections: Vec::new(),
pages: Vec::new(),
base_path: path.to_path_buf(),
};
// Parse main.hjson
let main_path = path.join("main.hjson");
if main_path.exists() {
let content = fs::read_to_string(&main_path).map_err(|e| WebBuilderError::IoError(e))?;
// Extract values from main.hjson
for line in content.lines() {
let line = line.trim();
// Skip comments and empty lines
if line.starts_with('#') || line.is_empty() {
continue;
}
// Parse key-value pairs
if let Some(pos) = line.find(':') {
let key = line[..pos].trim();
let value = line[pos + 1..].trim();
match key {
"title" => site_config.title = value.to_string(),
"name" => site_config.name = value.to_string(),
"description" => site_config.description = Some(value.to_string()),
"url" => site_config.url = Some(value.to_string()),
"favicon" => site_config.favicon = Some(value.to_string()),
_ => {} // Ignore other keys
}
}
}
}
// Parse header.hjson
let header_path = path.join("header.hjson");
if header_path.exists() {
let content = fs::read_to_string(&header_path).map_err(|e| WebBuilderError::IoError(e))?;
// Create a basic header configuration
let mut header_config = HeaderConfig {
logo: None,
title: None,
menu: None,
login: None,
};
// Extract values from header.hjson
let mut in_logo = false;
let mut logo_src = None;
let mut logo_alt = None;
for line in content.lines() {
let line = line.trim();
// Skip comments and empty lines
if line.starts_with('#') || line.is_empty() {
continue;
}
// Handle logo section
if line == "logo:" {
in_logo = true;
continue;
}
if in_logo {
if line.starts_with("src:") {
logo_src = Some(line[4..].trim().to_string());
} else if line.starts_with("alt:") {
logo_alt = Some(line[4..].trim().to_string());
} else if !line.starts_with(' ') {
in_logo = false;
}
}
// Parse other key-value pairs
if let Some(pos) = line.find(':') {
let key = line[..pos].trim();
let value = line[pos + 1..].trim();
if key == "title" {
header_config.title = Some(value.to_string());
}
}
}
// Set logo if we have a source
if let Some(src) = logo_src {
header_config.logo = Some(LogoConfig { src, alt: logo_alt });
}
site_config.header = Some(header_config);
}
// Parse collection.hjson
let collection_path = path.join("collection.hjson");
if collection_path.exists() {
let content =
fs::read_to_string(&collection_path).map_err(|e| WebBuilderError::IoError(e))?;
// Extract collections
let mut collections = Vec::new();
let mut current_collection: Option<CollectionConfig> = None;
for line in content.lines() {
let line = line.trim();
// Skip comments and empty lines
if line.starts_with('#') || line.is_empty() {
continue;
}
// Start of a new collection
if line == "{" {
current_collection = Some(CollectionConfig {
name: None,
url: None,
description: None,
scan: None,
});
continue;
}
// End of a collection
if line == "}" && current_collection.is_some() {
collections.push(current_collection.take().unwrap());
continue;
}
// Parse key-value pairs within a collection
if let Some(pos) = line.find(':') {
let key = line[..pos].trim();
let value = line[pos + 1..].trim();
if let Some(ref mut collection) = current_collection {
match key {
"name" => collection.name = Some(value.to_string()),
"url" => {
// Convert web interface URL to Git URL
let git_url = if value.contains("/src/branch/") {
// This is a web interface URL, convert it to a Git URL
let parts: Vec<&str> = value.split("/src/branch/").collect();
if parts.len() == 2 {
format!("{}.git", parts[0])
} else {
value.to_string()
}
} else {
value.to_string()
};
collection.url = Some(git_url);
}
"description" => collection.description = Some(value.to_string()),
"scan" => collection.scan = Some(value == "true"),
_ => {} // Ignore other keys
}
}
}
}
site_config.collections = collections;
}
// Parse pages directory
let pages_path = path.join("pages");
if pages_path.exists() && pages_path.is_dir() {
for entry in fs::read_dir(pages_path)? {
let entry = entry?;
let entry_path = entry.path();
if entry_path.is_file() && entry_path.extension().map_or(false, |ext| ext == "hjson") {
let content =
fs::read_to_string(&entry_path).map_err(|e| WebBuilderError::IoError(e))?;
// Extract pages
let mut pages = Vec::new();
let mut current_page: Option<PageConfig> = None;
for line in content.lines() {
let line = line.trim();
// Skip comments and empty lines
if line.starts_with('#') || line.is_empty() {
continue;
}
// Start of a new page
if line == "{" {
current_page = Some(PageConfig {
name: "".to_string(),
title: "".to_string(),
description: None,
navpath: "".to_string(),
collection: "".to_string(),
draft: None,
});
continue;
}
// End of a page
if line == "}" && current_page.is_some() {
pages.push(current_page.take().unwrap());
continue;
}
// Parse key-value pairs within a page
if let Some(pos) = line.find(':') {
let key = line[..pos].trim();
let value = line[pos + 1..].trim();
if let Some(ref mut page) = current_page {
match key {
"name" => page.name = value.to_string(),
"title" => page.title = value.to_string(),
"description" => page.description = Some(value.to_string()),
"navpath" => page.navpath = value.to_string(),
"collection" => page.collection = value.to_string(),
"draft" => page.draft = Some(value == "true"),
_ => {} // Ignore other keys
}
}
}
}
site_config.pages.extend(pages);
}
}
}
Ok(site_config)
}

209
webbuilder/src/parser_simple_test.rs Normal file
View File

@@ -0,0 +1,209 @@
#[cfg(test)]
mod tests {
use crate::error::WebBuilderError;
use crate::parser_simple::parse_site_config;
use std::fs;
use std::path::PathBuf;
use tempfile::TempDir;
fn create_test_site(temp_dir: &TempDir) -> PathBuf {
let site_dir = temp_dir.path().join("site");
fs::create_dir(&site_dir).unwrap();
// Create main.hjson
let main_hjson = r#"
# Main configuration
name: test
title: Test Site
description: A test site
url: https://example.com
favicon: favicon.ico
"#;
fs::write(site_dir.join("main.hjson"), main_hjson).unwrap();
// Create header.hjson
let header_hjson = r#"
# Header configuration
title: Test Site
logo:
src: logo.png
alt: Logo
"#;
fs::write(site_dir.join("header.hjson"), header_hjson).unwrap();
// Create collection.hjson
let collection_hjson = r#"
# Collections
{
name: test
url: https://git.ourworld.tf/tfgrid/home.git
description: A test collection
scan: true
}
{
name: test2
url: https://git.example.com/src/branch/main/test2
description: Another test collection
}
"#;
fs::write(site_dir.join("collection.hjson"), collection_hjson).unwrap();
// Create pages directory
let pages_dir = site_dir.join("pages");
fs::create_dir(&pages_dir).unwrap();
// Create pages/pages.hjson
let pages_hjson = r#"
# Pages
{
name: home
title: Home
description: Home page
navpath: /
collection: test
draft: false
}
{
name: about
title: About
description: About page
navpath: /about
collection: test
}
"#;
fs::write(pages_dir.join("pages.hjson"), pages_hjson).unwrap();
site_dir
}
#[test]
fn test_parse_site_config() {
let temp_dir = TempDir::new().unwrap();
let site_dir = create_test_site(&temp_dir);
let config = parse_site_config(&site_dir).unwrap();
// Check basic site info
assert_eq!(config.name, "test");
assert_eq!(config.title, "Test Site");
assert_eq!(config.description, Some("A test site".to_string()));
assert_eq!(config.url, Some("https://example.com".to_string()));
assert_eq!(config.favicon, Some("favicon.ico".to_string()));
// Check header
assert!(config.header.is_some());
let header = config.header.as_ref().unwrap();
assert_eq!(header.title, Some("Test Site".to_string()));
assert!(header.logo.is_some());
let logo = header.logo.as_ref().unwrap();
assert_eq!(logo.src, "logo.png");
assert_eq!(logo.alt, Some("Logo".to_string()));
// Check collections
assert_eq!(config.collections.len(), 2);
// First collection
assert_eq!(config.collections[0].name, Some("test".to_string()));
assert_eq!(
config.collections[0].url,
Some("https://git.ourworld.tf/tfgrid/home.git".to_string())
);
assert_eq!(
config.collections[0].description,
Some("A test collection".to_string())
);
assert_eq!(config.collections[0].scan, Some(true));
// Second collection (with URL conversion)
assert_eq!(config.collections[1].name, Some("test2".to_string()));
assert_eq!(
config.collections[1].url,
Some("https://git.example.com.git".to_string())
);
assert_eq!(
config.collections[1].description,
Some("Another test collection".to_string())
);
assert_eq!(config.collections[1].scan, None);
// Check pages
assert_eq!(config.pages.len(), 2);
// First page
assert_eq!(config.pages[0].name, "home");
assert_eq!(config.pages[0].title, "Home");
assert_eq!(config.pages[0].description, Some("Home page".to_string()));
assert_eq!(config.pages[0].navpath, "/");
assert_eq!(config.pages[0].collection, "test");
assert_eq!(config.pages[0].draft, Some(false));
// Second page
assert_eq!(config.pages[1].name, "about");
assert_eq!(config.pages[1].title, "About");
assert_eq!(config.pages[1].description, Some("About page".to_string()));
assert_eq!(config.pages[1].navpath, "/about");
assert_eq!(config.pages[1].collection, "test");
assert_eq!(config.pages[1].draft, None);
}
#[test]
fn test_parse_site_config_missing_directory() {
let result = parse_site_config("/nonexistent/directory");
assert!(matches!(result, Err(WebBuilderError::MissingDirectory(_))));
}
#[test]
fn test_parse_site_config_not_a_directory() {
let temp_dir = TempDir::new().unwrap();
let file_path = temp_dir.path().join("file.txt");
fs::write(&file_path, "not a directory").unwrap();
let result = parse_site_config(&file_path);
assert!(matches!(
result,
Err(WebBuilderError::InvalidConfiguration(_))
));
}
#[test]
fn test_parse_site_config_minimal() {
let temp_dir = TempDir::new().unwrap();
let site_dir = temp_dir.path().join("site");
fs::create_dir(&site_dir).unwrap();
// Create minimal main.hjson
let main_hjson = "name: minimal\ntitle: Minimal Site";
fs::write(site_dir.join("main.hjson"), main_hjson).unwrap();
let config = parse_site_config(&site_dir).unwrap();
assert_eq!(config.name, "minimal");
assert_eq!(config.title, "Minimal Site");
assert_eq!(config.description, None);
assert_eq!(config.url, None);
assert_eq!(config.favicon, None);
assert!(config.header.is_none());
assert!(config.footer.is_none());
assert!(config.collections.is_empty());
assert!(config.pages.is_empty());
}
#[test]
fn test_parse_site_config_empty() {
let temp_dir = TempDir::new().unwrap();
let site_dir = temp_dir.path().join("site");
fs::create_dir(&site_dir).unwrap();
let config = parse_site_config(&site_dir).unwrap();
assert_eq!(config.name, "default");
assert_eq!(config.title, "");
assert_eq!(config.description, None);
assert_eq!(config.url, None);
assert_eq!(config.favicon, None);
assert!(config.header.is_none());
assert!(config.footer.is_none());
assert!(config.collections.is_empty());
assert!(config.pages.is_empty());
}
}