Compare commits

..

18 Commits

Author SHA1 Message Date
121eee3ccd ... 2025-08-25 07:07:59 +02:00
Maxime Van Hees
0749a423bd fixed overlapping workspace roots 2025-08-21 16:19:42 +02:00
Timur Gordon
9a509f95cc Merge branch 'main' of https://git.ourworld.tf/herocode/hero 2025-08-20 11:26:55 +02:00
Timur Gordon
c8fbc6680b remove preserve tasks from actor trait 2025-08-20 11:25:32 +02:00
Maxime Van Hees
54b1b0adf5 generate keys and test rpc functions with python script 2025-08-14 16:33:41 +02:00
Maxime Van Hees
0ebda7c1aa Updates 2025-08-14 14:14:34 +02:00
Timur Gordon
04a1af2423 terminal ui better job refreshing 2025-08-07 16:07:49 +02:00
Timur Gordon
337ec2f660 terminal ui fixes 2025-08-07 15:49:35 +02:00
Timur Gordon
69e612e521 clean up debug logging and restore normal tree behavior 2025-08-07 15:45:09 +02:00
Timur Gordon
0df79e78c6 update terminal ui to show nested examples 2025-08-07 15:36:55 +02:00
Timur Gordon
b31651cfeb make func pub 2025-08-07 13:41:19 +02:00
Timur Gordon
831b25dbfa implement unix and ws using jsonrpsee 2025-08-07 11:56:49 +02:00
Timur Gordon
ce76f0a2f7 implement actor terminal ui 2025-08-07 10:26:11 +02:00
Timur Gordon
6c5c97e647 actor execute job fix 2025-08-06 12:56:25 +02:00
Timur Gordon
dcf0f41bb8 actor trait improvements and ui implementation 2025-08-06 12:48:32 +02:00
Timur Gordon
9f9149a950 baobab bin wip 2025-08-06 10:17:16 +02:00
Timur Gordon
2f15387f9f Merge branch 'main' of https://git.ourworld.tf/herocode/hero 2025-08-05 15:45:38 +02:00
Timur Gordon
89e953ca1d rename worker to actor 2025-08-05 15:44:33 +02:00
131 changed files with 14773 additions and 2196 deletions

1505
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -53,13 +53,14 @@ uuid = { version = "1.6", features = ["v4", "serde"] }
[workspace]
members = [
"interfaces/unix/client",
"interfaces/unix/server",
"interfaces/websocket/client",
"interfaces/websocket/server",
"core/supervisor",
"core/worker",
"core/actor",
"core/job", "interfaces/websocket/examples",
"proxies/http",
"interfaces/openrpc/client",
"interfaces/openrpc/server",
"frontend/baobap-frontend"
]
resolver = "2" # Recommended for new workspaces

View File

@@ -12,29 +12,26 @@ Hero is a program that runs scripts in contexts on behalf of a peer. Hero aims t
## Core
In its core, a [supervisor](#supervisor) dispatches jobs to execute scripts to [workers](#worker) over redis. Workers spawn appropriate engine instances to execute scripts within the defined [confines]() of the job.
In its core, a [supervisor](#supervisor) dispatches jobs to execute scripts to [actors](#actor) over redis. Actors spawn appropriate engine instances to execute scripts within the defined [confines]() of the job.
### Components
### [Supervisor](./core/supervisor)
#### [Supervisor](./core/supervisor)
Component responsible for distributing jobs to actors over Redis.
Component responsible for distributing jobs to workers over Redis.
#### [Engine](./core/engine)
A process that runs a script in a confined environment.
#### [Job](./core/job)
### [Job](./core/job)
A unit of work that executes a Rhai or Hero script.
#### [Worker](./core/worker)
### [Actor](./core/actor)
An entity that processes jobs dispatched by the supervisor.
## Interfaces
The backend supports an OpenRPC interface over Websocket and Unix sockets, and a wasm app interface for simple debugging logging etc.
### Websocket
### Unix
### WASM

View File

@@ -1,20 +1,20 @@
# Minimal Rhailib Benchmark
# Minimal baobab Benchmark
A simplified, minimal benchmarking tool for rhailib performance testing.
A simplified, minimal benchmarking tool for baobab performance testing.
## Overview
This benchmark focuses on simplicity and direct timing measurements:
- Creates a single task (n=1) using Lua script
- Measures latency using Redis timestamps
- Uses existing worker binary
- Uses existing actor binary
- ~85 lines of code total
## Usage
### Prerequisites
- Redis running on `127.0.0.1:6379`
- Worker binary built: `cd src/worker && cargo build --release`
- Actor binary built: `cd src/actor && cargo build --release`
### Run Benchmark
```bash
@@ -25,7 +25,7 @@ cargo bench
### Expected Output
```
🧹 Cleaning up Redis...
🚀 Starting worker...
🚀 Starting actor...
📝 Creating single task...
⏱️ Waiting for completion...
✅ Task completed in 23.45ms
@@ -42,10 +42,10 @@ cargo bench
## How It Works
1. **Cleanup**: Clear Redis queues and task details
2. **Start Worker**: Spawn single worker process
2. **Start Actor**: Spawn single actor process
3. **Create Task**: Use Lua script to create one task with timestamp
4. **Wait & Measure**: Poll task until complete, calculate latency
5. **Cleanup**: Kill worker and clear Redis
5. **Cleanup**: Kill actor and clear Redis
## Latency Calculation
@@ -55,7 +55,7 @@ latency_ms = updated_at - created_at
Where:
- `created_at`: Timestamp when task was created (Lua script)
- `updated_at`: Timestamp when worker completed task
- `updated_at`: Timestamp when actor completed task
## Future Iterations

View File

@@ -15,7 +15,7 @@ if task_count <= 0 or task_count > 10000 then
return redis.error_reply("task_count must be a positive integer between 1 and 10000")
end
-- Get current timestamp in Unix seconds (to match worker expectations)
-- Get current timestamp in Unix seconds (to match actor expectations)
local rhai_task_queue = 'rhai_tasks:' .. circle_name
local task_keys = {}
local current_time = redis.call('TIME')[1]
@@ -35,7 +35,7 @@ for i = 1, task_count do
'task_sequence', tostring(i)
)
-- Queue the task for workers
-- Queue the task for actors
redis.call('LPUSH', rhai_task_queue, task_id)
-- Add key to return array

View File

@@ -23,23 +23,23 @@ fn cleanup_redis() -> Result<(), redis::RedisError> {
Ok(())
}
fn start_worker() -> Result<Child, std::io::Error> {
fn start_actor() -> Result<Child, std::io::Error> {
Command::new("cargo")
.args(&[
"run",
"--release",
"--bin",
"worker",
"actor",
"--",
"--circle",
CIRCLE_NAME,
"--redis-url",
REDIS_URL,
"--worker-id",
"bench_worker",
"--actor-id",
"bench_actor",
"--preserve-tasks",
])
.current_dir("src/worker")
.current_dir("src/actor")
.stdout(Stdio::null())
.stderr(Stdio::null())
.spawn()
@@ -126,26 +126,26 @@ fn wait_for_batch_completion(task_keys: &[String]) -> Result<f64, Box<dyn std::e
}
}
fn cleanup_worker(mut worker: Child) -> Result<(), std::io::Error> {
worker.kill()?;
worker.wait()?;
fn cleanup_actor(mut actor: Child) -> Result<(), std::io::Error> {
actor.kill()?;
actor.wait()?;
Ok(())
}
fn bench_single_rhai_task(c: &mut Criterion) {
// Setup: ensure worker is built
// Setup: ensure actor is built
let _ = Command::new("cargo")
.args(&["build", "--release", "--bin", "worker"])
.current_dir("src/worker")
.args(&["build", "--release", "--bin", "actor"])
.current_dir("src/actor")
.output()
.expect("Failed to build worker");
.expect("Failed to build actor");
// Clean up before starting
cleanup_redis().expect("Failed to cleanup Redis");
// Start worker once and reuse it
let worker = start_worker().expect("Failed to start worker");
thread::sleep(Duration::from_millis(1000)); // Give worker time to start
// Start actor once and reuse it
let actor = start_actor().expect("Failed to start actor");
thread::sleep(Duration::from_millis(1000)); // Give actor time to start
let mut group = c.benchmark_group("rhai_task_execution");
group.sample_size(10); // Reduce sample size
@@ -174,8 +174,8 @@ fn bench_single_rhai_task(c: &mut Criterion) {
group.finish();
// Cleanup worker
cleanup_worker(worker).expect("Failed to cleanup worker");
// Cleanup actor
cleanup_actor(actor).expect("Failed to cleanup actor");
cleanup_redis().expect("Failed to cleanup Redis");
}

View File

@@ -1,38 +1,38 @@
# Rhai Worker Binary
# Rhai Actor Binary
A command-line worker for executing Rhai scripts from Redis task queues.
A command-line actor for executing Rhai scripts from Redis task queues.
## Binary: `worker`
## Binary: `actor`
### Installation
Build the binary:
```bash
cargo build --bin worker --release
cargo build --bin actor --release
```
### Usage
```bash
# Basic usage - requires circle public key
worker --circle-public-key <CIRCLE_PUBLIC_KEY>
actor --circle-public-key <CIRCLE_PUBLIC_KEY>
# Custom Redis URL
worker -c <CIRCLE_PUBLIC_KEY> --redis-url redis://localhost:6379/1
actor -c <CIRCLE_PUBLIC_KEY> --redis-url redis://localhost:6379/1
# Custom worker ID and database path
worker -c <CIRCLE_PUBLIC_KEY> --worker-id my_worker --db-path /tmp/worker_db
# Custom actor ID and database path
actor -c <CIRCLE_PUBLIC_KEY> --actor-id my_actor --db-path /tmp/actor_db
# Preserve tasks for debugging/benchmarking
worker -c <CIRCLE_PUBLIC_KEY> --preserve-tasks
actor -c <CIRCLE_PUBLIC_KEY> --preserve-tasks
# Remove timestamps from logs
worker -c <CIRCLE_PUBLIC_KEY> --no-timestamp
actor -c <CIRCLE_PUBLIC_KEY> --no-timestamp
# Increase verbosity
worker -c <CIRCLE_PUBLIC_KEY> -v # Debug logging
worker -c <CIRCLE_PUBLIC_KEY> -vv # Full debug
worker -c <CIRCLE_PUBLIC_KEY> -vvv # Trace logging
actor -c <CIRCLE_PUBLIC_KEY> -v # Debug logging
actor -c <CIRCLE_PUBLIC_KEY> -vv # Full debug
actor -c <CIRCLE_PUBLIC_KEY> -vvv # Trace logging
```
### Command-Line Options
@@ -41,9 +41,9 @@ worker -c <CIRCLE_PUBLIC_KEY> -vvv # Trace logging
|--------|-------|---------|-------------|
| `--circle-public-key` | `-c` | **Required** | Circle public key to listen for tasks |
| `--redis-url` | `-r` | `redis://localhost:6379` | Redis connection URL |
| `--worker-id` | `-w` | `worker_1` | Unique worker identifier |
| `--actor-id` | `-w` | `actor_1` | Unique actor identifier |
| `--preserve-tasks` | | `false` | Preserve task details after completion |
| `--db-path` | | `worker_rhai_temp_db` | Database path for Rhai engine |
| `--db-path` | | `actor_rhai_temp_db` | Database path for Rhai engine |
| `--no-timestamp` | | `false` | Remove timestamps from log output |
| `--verbose` | `-v` | | Increase verbosity (stackable) |
@@ -58,7 +58,7 @@ worker -c <CIRCLE_PUBLIC_KEY> -vvv # Trace logging
### How It Works
1. **Queue Listening**: Worker listens on Redis queue `rhailib:{circle_public_key}`
1. **Queue Listening**: Actor listens on Redis queue `baobab:{circle_public_key}`
2. **Task Processing**: Receives task IDs, fetches task details from Redis
3. **Script Execution**: Executes Rhai scripts with configured engine
4. **Result Handling**: Updates task status and sends results to reply queues
@@ -66,30 +66,30 @@ worker -c <CIRCLE_PUBLIC_KEY> -vvv # Trace logging
### Configuration Examples
#### Development Worker
#### Development Actor
```bash
# Simple development worker
worker -c dev_circle_123
# Simple development actor
actor -c dev_circle_123
# Development with verbose logging (no timestamps)
worker -c dev_circle_123 -v --no-timestamp
actor -c dev_circle_123 -v --no-timestamp
```
#### Production Worker
#### Production Actor
```bash
# Production worker with custom configuration
worker \
# Production actor with custom configuration
actor \
--circle-public-key prod_circle_456 \
--redis-url redis://redis-server:6379/0 \
--worker-id prod_worker_1 \
--db-path /var/lib/worker/db \
--actor-id prod_actor_1 \
--db-path /var/lib/actor/db \
--preserve-tasks
```
#### Benchmarking Worker
#### Benchmarking Actor
```bash
# Worker optimized for benchmarking
worker \
# Actor optimized for benchmarking
actor \
--circle-public-key bench_circle_789 \
--preserve-tasks \
--no-timestamp \
@@ -98,7 +98,7 @@ worker \
### Error Handling
The worker provides clear error messages for:
The actor provides clear error messages for:
- Missing or invalid circle public key
- Redis connection failures
- Script execution errors
@@ -106,7 +106,7 @@ The worker provides clear error messages for:
### Dependencies
- `rhailib_engine`: Rhai engine with heromodels integration
- `baobab_engine`: Rhai engine with heromodels integration
- `redis`: Redis client for task queue management
- `rhai`: Script execution engine
- `clap`: Command-line argument parsing

View File

@@ -1,11 +1,11 @@
//! OSIS Worker Binary - Synchronous worker for system-level operations
//! OSIS Actor Binary - Synchronous actor for system-level operations
use clap::Parser;
use log::{error, info};
use rhailib_worker::config::{ConfigError, WorkerConfig};
use rhailib_worker::engine::create_heromodels_engine;
use rhailib_worker::sync_worker::SyncWorker;
use rhailib_worker::worker_trait::{spawn_worker, WorkerConfig as TraitWorkerConfig};
use baobab_actor::config::{ConfigError, ActorConfig};
use baobab_actor::engine::create_heromodels_engine;
use baobab_actor::sync_actor::SyncActor;
use baobab_actor::actor_trait::{spawn_actor, ActorConfig as TraitActorConfig};
use std::path::PathBuf;
use std::sync::Arc;
use tokio::signal;
@@ -15,8 +15,8 @@ use tokio::sync::mpsc;
#[command(
name = "osis",
version = "0.1.0",
about = "OSIS (Operating System Integration Service) - Synchronous Worker",
long_about = "A synchronous worker for Hero framework that processes jobs sequentially. \
about = "OSIS (Operating System Integration Service) - Synchronous Actor",
long_about = "A synchronous actor for Hero framework that processes jobs sequentially. \
Ideal for system-level operations that require careful resource management."
)]
struct Args {
@@ -24,9 +24,9 @@ struct Args {
#[arg(short, long, help = "Path to TOML configuration file")]
config: PathBuf,
/// Override worker ID from config
#[arg(long, help = "Override worker ID from configuration file")]
worker_id: Option<String>,
/// Override actor ID from config
#[arg(long, help = "Override actor ID from configuration file")]
actor_id: Option<String>,
/// Override Redis URL from config
#[arg(long, help = "Override Redis URL from configuration file")]
@@ -50,7 +50,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let args = Args::parse();
// Load configuration from TOML file
let mut config = match WorkerConfig::from_file(&args.config) {
let mut config = match ActorConfig::from_file(&args.config) {
Ok(config) => config,
Err(e) => {
eprintln!("Failed to load configuration from {:?}: {}", args.config, e);
@@ -58,17 +58,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
}
};
// Validate that this is a sync worker configuration
// Validate that this is a sync actor configuration
if !config.is_sync() {
eprintln!("Error: OSIS worker requires a sync worker configuration");
eprintln!("Expected: [worker_type] type = \"sync\"");
eprintln!("Found: {:?}", config.worker_type);
eprintln!("Error: OSIS actor requires a sync actor configuration");
eprintln!("Expected: [actor_type] type = \"sync\"");
eprintln!("Found: {:?}", config.actor_type);
std::process::exit(1);
}
// Apply command line overrides
if let Some(worker_id) = args.worker_id {
config.worker_id = worker_id;
if let Some(actor_id) = args.actor_id {
config.actor_id = actor_id;
}
if let Some(redis_url) = args.redis_url {
config.redis_url = redis_url;
@@ -80,8 +80,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// Configure logging
setup_logging(&config, args.verbose, args.no_timestamp)?;
info!("🚀 OSIS Worker starting...");
info!("Worker ID: {}", config.worker_id);
info!("🚀 OSIS Actor starting...");
info!("Actor ID: {}", config.actor_id);
info!("Redis URL: {}", config.redis_url);
info!("Database Path: {}", config.db_path);
info!("Preserve Tasks: {}", config.preserve_tasks);
@@ -90,17 +90,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let engine = create_heromodels_engine();
info!("✅ Rhai engine initialized");
// Create worker configuration for the trait-based interface
let worker_config = TraitWorkerConfig::new(
config.worker_id.clone(),
// Create actor configuration for the trait-based interface
let actor_config = TraitActorConfig::new(
config.actor_id.clone(),
config.db_path.clone(),
config.redis_url.clone(),
config.preserve_tasks,
);
// Create sync worker instance
let worker = Arc::new(SyncWorker::default());
info!("✅ Sync worker instance created");
// Create sync actor instance
let actor = Arc::new(SyncActor::default());
info!("✅ Sync actor instance created");
// Setup shutdown signal handling
let (shutdown_tx, shutdown_rx) = mpsc::channel(1);
@@ -118,21 +118,21 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
}
});
// Spawn the worker
info!("🔄 Starting worker loop...");
let worker_handle = spawn_worker(worker, engine, shutdown_rx);
// Spawn the actor
info!("🔄 Starting actor loop...");
let actor_handle = spawn_actor(actor, engine, shutdown_rx);
// Wait for the worker to complete
match worker_handle.await {
// Wait for the actor to complete
match actor_handle.await {
Ok(Ok(())) => {
info!("✅ OSIS Worker shut down gracefully");
info!("✅ OSIS Actor shut down gracefully");
}
Ok(Err(e)) => {
error!("❌ OSIS Worker encountered an error: {}", e);
error!("❌ OSIS Actor encountered an error: {}", e);
std::process::exit(1);
}
Err(e) => {
error!("❌ Failed to join worker task: {}", e);
error!("❌ Failed to join actor task: {}", e);
std::process::exit(1);
}
}
@@ -142,7 +142,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
/// Setup logging based on configuration and command line arguments
fn setup_logging(
config: &WorkerConfig,
config: &ActorConfig,
verbose: bool,
no_timestamp: bool,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
@@ -187,11 +187,11 @@ mod tests {
#[test]
fn test_config_validation() {
let config_toml = r#"
worker_id = "test_osis"
actor_id = "test_osis"
redis_url = "redis://localhost:6379"
db_path = "/tmp/test_db"
[worker_type]
[actor_type]
type = "sync"
[logging]
@@ -201,20 +201,20 @@ level = "info"
let mut temp_file = NamedTempFile::new().unwrap();
temp_file.write_all(config_toml.as_bytes()).unwrap();
let config = WorkerConfig::from_file(temp_file.path()).unwrap();
let config = ActorConfig::from_file(temp_file.path()).unwrap();
assert!(config.is_sync());
assert!(!config.is_async());
assert_eq!(config.worker_id, "test_osis");
assert_eq!(config.actor_id, "test_osis");
}
#[test]
fn test_async_config_rejection() {
let config_toml = r#"
worker_id = "test_osis"
actor_id = "test_osis"
redis_url = "redis://localhost:6379"
db_path = "/tmp/test_db"
[worker_type]
[actor_type]
type = "async"
default_timeout_seconds = 300
@@ -225,7 +225,7 @@ level = "info"
let mut temp_file = NamedTempFile::new().unwrap();
temp_file.write_all(config_toml.as_bytes()).unwrap();
let config = WorkerConfig::from_file(temp_file.path()).unwrap();
let config = ActorConfig::from_file(temp_file.path()).unwrap();
assert!(!config.is_sync());
assert!(config.is_async());
// This would be rejected in main() function

View File

@@ -1,11 +1,11 @@
//! System Worker Binary - Asynchronous worker for high-throughput concurrent processing
//! System Actor Binary - Asynchronous actor for high-throughput concurrent processing
use clap::Parser;
use log::{error, info, warn};
use rhailib_worker::async_worker_impl::AsyncWorker;
use rhailib_worker::config::{ConfigError, WorkerConfig};
use rhailib_worker::engine::create_heromodels_engine;
use rhailib_worker::worker_trait::{spawn_worker, WorkerConfig as TraitWorkerConfig};
use baobab_actor::async_actor_impl::AsyncActor;
use baobab_actor::config::{ConfigError, ActorConfig};
use baobab_actor::engine::create_heromodels_engine;
use baobab_actor::actor_trait::{spawn_actor, ActorConfig as TraitActorConfig};
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
@@ -16,8 +16,8 @@ use tokio::sync::mpsc;
#[command(
name = "system",
version = "0.1.0",
about = "System Worker - Asynchronous Worker with Concurrent Job Processing",
long_about = "An asynchronous worker for Hero framework that processes multiple jobs \
about = "System Actor - Asynchronous Actor with Concurrent Job Processing",
long_about = "An asynchronous actor for Hero framework that processes multiple jobs \
concurrently with timeout support. Ideal for high-throughput scenarios \
where jobs can be executed in parallel."
)]
@@ -26,9 +26,9 @@ struct Args {
#[arg(short, long, help = "Path to TOML configuration file")]
config: PathBuf,
/// Override worker ID from config
#[arg(long, help = "Override worker ID from configuration file")]
worker_id: Option<String>,
/// Override actor ID from config
#[arg(long, help = "Override actor ID from configuration file")]
actor_id: Option<String>,
/// Override Redis URL from config
#[arg(long, help = "Override Redis URL from configuration file")]
@@ -50,8 +50,8 @@ struct Args {
#[arg(long, help = "Remove timestamps from log output")]
no_timestamp: bool,
/// Show worker statistics periodically
#[arg(long, help = "Show periodic worker statistics")]
/// Show actor statistics periodically
#[arg(long, help = "Show periodic actor statistics")]
show_stats: bool,
}
@@ -60,7 +60,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let args = Args::parse();
// Load configuration from TOML file
let mut config = match WorkerConfig::from_file(&args.config) {
let mut config = match ActorConfig::from_file(&args.config) {
Ok(config) => config,
Err(e) => {
eprintln!("Failed to load configuration from {:?}: {}", args.config, e);
@@ -68,17 +68,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
}
};
// Validate that this is an async worker configuration
// Validate that this is an async actor configuration
if !config.is_async() {
eprintln!("Error: System worker requires an async worker configuration");
eprintln!("Expected: [worker_type] type = \"async\"");
eprintln!("Found: {:?}", config.worker_type);
eprintln!("Error: System actor requires an async actor configuration");
eprintln!("Expected: [actor_type] type = \"async\"");
eprintln!("Found: {:?}", config.actor_type);
std::process::exit(1);
}
// Apply command line overrides
if let Some(worker_id) = args.worker_id {
config.worker_id = worker_id;
if let Some(actor_id) = args.actor_id {
config.actor_id = actor_id;
}
if let Some(redis_url) = args.redis_url {
config.redis_url = redis_url;
@@ -89,7 +89,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// Override timeout if specified
if let Some(timeout_secs) = args.timeout {
if let rhailib_worker::config::WorkerType::Async { ref mut default_timeout_seconds } = config.worker_type {
if let baobab_actor::config::ActorType::Async { ref mut default_timeout_seconds } = config.actor_type {
*default_timeout_seconds = timeout_secs;
}
}
@@ -97,8 +97,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// Configure logging
setup_logging(&config, args.verbose, args.no_timestamp)?;
info!("🚀 System Worker starting...");
info!("Worker ID: {}", config.worker_id);
info!("🚀 System Actor starting...");
info!("Actor ID: {}", config.actor_id);
info!("Redis URL: {}", config.redis_url);
info!("Database Path: {}", config.db_path);
info!("Preserve Tasks: {}", config.preserve_tasks);
@@ -111,22 +111,22 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let engine = create_heromodels_engine();
info!("✅ Rhai engine initialized");
// Create worker configuration for the trait-based interface
let mut worker_config = TraitWorkerConfig::new(
config.worker_id.clone(),
// Create actor configuration for the trait-based interface
let mut actor_config = TraitActorConfig::new(
config.actor_id.clone(),
config.db_path.clone(),
config.redis_url.clone(),
config.preserve_tasks,
);
// Add timeout configuration for async worker
// Add timeout configuration for async actor
if let Some(timeout) = config.get_default_timeout() {
worker_config = worker_config.with_default_timeout(timeout);
actor_config = actor_config.with_default_timeout(timeout);
}
// Create async worker instance
let worker = Arc::new(AsyncWorker::default());
info!("✅ Async worker instance created");
// Create async actor instance
let actor = Arc::new(AsyncActor::default());
info!("✅ Async actor instance created");
// Setup shutdown signal handling
let (shutdown_tx, shutdown_rx) = mpsc::channel(1);
@@ -146,36 +146,36 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// Spawn statistics reporter if requested
if args.show_stats {
let worker_stats = Arc::clone(&worker);
let actor_stats = Arc::clone(&actor);
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(30));
loop {
interval.tick().await;
let running_count = worker_stats.running_job_count().await;
let running_count = actor_stats.running_job_count().await;
if running_count > 0 {
info!("📊 Worker Stats: {} jobs currently running", running_count);
info!("📊 Actor Stats: {} jobs currently running", running_count);
} else {
info!("📊 Worker Stats: No jobs currently running");
info!("📊 Actor Stats: No jobs currently running");
}
}
});
}
// Spawn the worker
info!("🔄 Starting worker loop...");
let worker_handle = spawn_worker(worker, engine, shutdown_rx);
// Spawn the actor
info!("🔄 Starting actor loop...");
let actor_handle = spawn_actor(actor, engine, shutdown_rx);
// Wait for the worker to complete
match worker_handle.await {
// Wait for the actor to complete
match actor_handle.await {
Ok(Ok(())) => {
info!("✅ System Worker shut down gracefully");
info!("✅ System Actor shut down gracefully");
}
Ok(Err(e)) => {
error!("❌ System Worker encountered an error: {}", e);
error!("❌ System Actor encountered an error: {}", e);
std::process::exit(1);
}
Err(e) => {
error!("❌ Failed to join worker task: {}", e);
error!("❌ Failed to join actor task: {}", e);
std::process::exit(1);
}
}
@@ -185,7 +185,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
/// Setup logging based on configuration and command line arguments
fn setup_logging(
config: &WorkerConfig,
config: &ActorConfig,
verbose: bool,
no_timestamp: bool,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
@@ -230,11 +230,11 @@ mod tests {
#[test]
fn test_config_validation() {
let config_toml = r#"
worker_id = "test_system"
actor_id = "test_system"
redis_url = "redis://localhost:6379"
db_path = "/tmp/test_db"
[worker_type]
[actor_type]
type = "async"
default_timeout_seconds = 600
@@ -245,21 +245,21 @@ level = "info"
let mut temp_file = NamedTempFile::new().unwrap();
temp_file.write_all(config_toml.as_bytes()).unwrap();
let config = WorkerConfig::from_file(temp_file.path()).unwrap();
let config = ActorConfig::from_file(temp_file.path()).unwrap();
assert!(!config.is_sync());
assert!(config.is_async());
assert_eq!(config.worker_id, "test_system");
assert_eq!(config.actor_id, "test_system");
assert_eq!(config.get_default_timeout(), Some(Duration::from_secs(600)));
}
#[test]
fn test_sync_config_rejection() {
let config_toml = r#"
worker_id = "test_system"
actor_id = "test_system"
redis_url = "redis://localhost:6379"
db_path = "/tmp/test_db"
[worker_type]
[actor_type]
type = "sync"
[logging]
@@ -269,7 +269,7 @@ level = "info"
let mut temp_file = NamedTempFile::new().unwrap();
temp_file.write_all(config_toml.as_bytes()).unwrap();
let config = WorkerConfig::from_file(temp_file.path()).unwrap();
let config = ActorConfig::from_file(temp_file.path()).unwrap();
assert!(config.is_sync());
assert!(!config.is_async());
// This would be rejected in main() function
@@ -278,11 +278,11 @@ level = "info"
#[test]
fn test_timeout_override() {
let config_toml = r#"
worker_id = "test_system"
actor_id = "test_system"
redis_url = "redis://localhost:6379"
db_path = "/tmp/test_db"
[worker_type]
[actor_type]
type = "async"
default_timeout_seconds = 300
"#;
@@ -290,11 +290,11 @@ default_timeout_seconds = 300
let mut temp_file = NamedTempFile::new().unwrap();
temp_file.write_all(config_toml.as_bytes()).unwrap();
let mut config = WorkerConfig::from_file(temp_file.path()).unwrap();
let mut config = ActorConfig::from_file(temp_file.path()).unwrap();
assert_eq!(config.get_default_timeout(), Some(Duration::from_secs(300)));
// Test timeout override
if let rhailib_worker::config::WorkerType::Async { ref mut default_timeout_seconds } = config.worker_type {
if let baobab_actor::config::ActorType::Async { ref mut default_timeout_seconds } = config.actor_type {
*default_timeout_seconds = 600;
}
assert_eq!(config.get_default_timeout(), Some(Duration::from_secs(600)));

View File

@@ -1,14 +1,14 @@
use clap::Parser;
use rhailib_worker::engine::create_heromodels_engine;
use rhailib_worker::spawn_rhai_worker;
use baobab_actor::engine::create_heromodels_engine;
use baobab_actor::spawn_rhai_actor;
use tokio::sync::mpsc;
#[derive(Parser, Debug)]
#[command(author, version, about, long_about = None)]
struct Args {
/// Worker ID for identification
/// Actor ID for identification
#[arg(short, long)]
worker_id: String,
actor_id: String,
/// Redis URL
#[arg(short, long, default_value = "redis://localhost:6379")]
@@ -19,7 +19,7 @@ struct Args {
preserve_tasks: bool,
/// Root directory for engine database
#[arg(long, default_value = "worker_rhai_temp_db")]
#[arg(long, default_value = "actor_rhai_temp_db")]
db_path: String,
/// Disable timestamps in log output
@@ -41,10 +41,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
}
log::info!("Rhai Worker (binary) starting with performance-optimized engine.");
log::info!("Rhai Actor (binary) starting with performance-optimized engine.");
log::info!(
"Worker ID: {}, Redis: {}",
args.worker_id,
"Actor ID: {}, Redis: {}",
args.actor_id,
args.redis_url
);
@@ -65,9 +65,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
// Create shutdown channel (for graceful shutdown, though not used in benchmarks)
let (_shutdown_tx, shutdown_rx) = mpsc::channel::<()>(1);
// Spawn the worker
let worker_handle = spawn_rhai_worker(
args.worker_id,
// Spawn the actor
let actor_handle = spawn_rhai_actor(
args.actor_id,
args.db_path,
engine,
args.redis_url,
@@ -75,20 +75,20 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
args.preserve_tasks,
);
// Wait for the worker to complete
match worker_handle.await {
// Wait for the actor to complete
match actor_handle.await {
Ok(result) => match result {
Ok(_) => {
log::info!("Worker completed successfully");
log::info!("Actor completed successfully");
Ok(())
}
Err(e) => {
log::error!("Worker failed: {}", e);
log::error!("Actor failed: {}", e);
Err(e)
}
},
Err(e) => {
log::error!("Worker task panicked: {}", e);
log::error!("Actor task panicked: {}", e);
Err(Box::new(e) as Box<dyn std::error::Error + Send + Sync>)
}
}

View File

@@ -1,11 +1,11 @@
# Worker Examples
# Actor Examples
This directory contains example configurations and test scripts for both OSIS and System worker binaries.
This directory contains example configurations and test scripts for both OSIS and System actor binaries.
## Overview
Both examples demonstrate the ping/pong functionality built into the Hero workers:
- Workers automatically detect jobs with script content "ping"
Both examples demonstrate the ping/pong functionality built into the Hero actors:
- Actors automatically detect jobs with script content "ping"
- They respond immediately with "pong" without executing the Rhai engine
- This provides a fast health check and connectivity test mechanism
@@ -20,20 +20,20 @@ Both examples demonstrate the ping/pong functionality built into the Hero worker
redis-server
```
2. **Rust Environment**: Make sure you can build the worker binaries
2. **Rust Environment**: Make sure you can build the actor binaries
```bash
cd /path/to/herocode/hero/core/worker
cd /path/to/herocode/baobab/core/actor
cargo build --bin osis --bin system
```
## OSIS Worker Example
## OSIS Actor Example
**Location**: `examples/osis/`
The OSIS (Operating System Integration Service) worker processes jobs synchronously, one at a time.
The OSIS (Operating System Integration Service) actor processes jobs synchronously, one at a time.
### Files
- `config.toml` - Configuration for the OSIS worker
- `config.toml` - Configuration for the OSIS actor
- `example.sh` - Test script that demonstrates ping/pong functionality
### Usage
@@ -45,31 +45,31 @@ cd examples/osis
### What the script does:
1. Checks Redis connectivity
2. Cleans up any existing jobs
3. Starts the OSIS worker in the background
3. Starts the OSIS actor in the background
4. Sends 3 ping jobs sequentially
5. Verifies each job receives a "pong" response
6. Reports success/failure statistics
7. Cleans up worker and Redis data
7. Cleans up actor and Redis data
### Expected Output
```
=== OSIS Worker Example ===
=== OSIS Actor Example ===
✅ Redis is running
✅ OSIS worker started (PID: 12345)
✅ OSIS actor started (PID: 12345)
📤 Sending ping job: ping_job_1_1234567890
✅ Job ping_job_1_1234567890 completed successfully with result: pong
...
🎉 All tests passed! OSIS worker is working correctly.
🎉 All tests passed! OSIS actor is working correctly.
```
## System Worker Example
## System Actor Example
**Location**: `examples/system/`
The System worker processes jobs asynchronously, handling multiple jobs concurrently.
The System actor processes jobs asynchronously, handling multiple jobs concurrently.
### Files
- `config.toml` - Configuration for the System worker (includes async settings)
- `config.toml` - Configuration for the System actor (includes async settings)
- `example.sh` - Test script that demonstrates concurrent ping/pong functionality
### Usage
@@ -81,22 +81,22 @@ cd examples/system
### What the script does:
1. Checks Redis connectivity
2. Cleans up any existing jobs
3. Starts the System worker with stats reporting
3. Starts the System actor with stats reporting
4. Sends 5 concurrent ping jobs
5. Sends 10 rapid-fire ping jobs to test async capabilities
6. Verifies all jobs receive "pong" responses
7. Reports comprehensive success/failure statistics
8. Cleans up worker and Redis data
8. Cleans up actor and Redis data
### Expected Output
```
=== System Worker Example ===
=== System Actor Example ===
✅ Redis is running
✅ System worker started (PID: 12345)
✅ System actor started (PID: 12345)
📤 Sending ping job: ping_job_1_1234567890123
✅ Job ping_job_1_1234567890123 completed successfully with result: pong
...
🎉 All tests passed! System worker is handling concurrent jobs correctly.
🎉 All tests passed! System actor is handling concurrent jobs correctly.
Overall success rate: 15/15
```
@@ -104,12 +104,12 @@ Overall success rate: 15/15
### OSIS Configuration (`examples/osis/config.toml`)
```toml
worker_id = "osis_example_worker"
actor_id = "osis_example_actor"
redis_url = "redis://localhost:6379"
db_path = "/tmp/osis_example_db"
preserve_tasks = false
[worker_type]
[actor_type]
type = "sync"
[logging]
@@ -119,12 +119,12 @@ level = "info"
### System Configuration (`examples/system/config.toml`)
```toml
worker_id = "system_example_worker"
actor_id = "system_example_actor"
redis_url = "redis://localhost:6379"
db_path = "/tmp/system_example_db"
preserve_tasks = false
[worker_type]
[actor_type]
type = "async"
default_timeout_seconds = 30
@@ -135,7 +135,7 @@ level = "info"
## Key Differences
| Feature | OSIS Worker | System Worker |
| Feature | OSIS Actor | System Actor |
|---------|-------------|---------------|
| **Processing** | Sequential (one job at a time) | Concurrent (multiple jobs simultaneously) |
| **Use Case** | System-level operations requiring resource management | High-throughput job processing |
@@ -154,7 +154,7 @@ redis-cli ping
redis-server --loglevel verbose
```
### Worker Compilation Issues
### Actor Compilation Issues
```bash
# Clean and rebuild
cargo clean
@@ -164,7 +164,7 @@ cargo build --bin osis --bin system
### Job Processing Issues
- Check Redis for stuck jobs: `redis-cli keys "hero:*"`
- Clear all Hero jobs: `redis-cli eval "return redis.call('del', unpack(redis.call('keys', 'hero:*')))" 0`
- Check worker logs for detailed error messages
- Check actor logs for detailed error messages
## Extending the Examples
@@ -183,15 +183,15 @@ To test with custom Rhai scripts instead of ping jobs:
### Testing Different Configurations
- Modify `config.toml` files to test different Redis URLs, database paths, or logging levels
- Test with `preserve_tasks = true` to inspect job details after completion
- Adjust timeout values in the System worker configuration
- Adjust timeout values in the System actor configuration
## Architecture Notes
Both examples demonstrate the unified Worker trait architecture:
- **Common Interface**: Both workers implement the same `Worker` trait
Both examples demonstrate the unified Actor trait architecture:
- **Common Interface**: Both actors implement the same `Actor` trait
- **Ping/Pong Handling**: Built into the trait's `spawn` method before job delegation
- **Redis Integration**: Uses the shared Job struct from `hero_job` crate
- **Configuration**: TOML-based configuration with CLI overrides
- **Graceful Shutdown**: Both workers handle SIGTERM/SIGINT properly
- **Graceful Shutdown**: Both actors handle SIGTERM/SIGINT properly
This architecture allows for easy extension with new worker types while maintaining consistent behavior and configuration patterns.
This architecture allows for easy extension with new actor types while maintaining consistent behavior and configuration patterns.

View File

@@ -1,9 +1,9 @@
worker_id = "osis_example_worker"
actor_id = "osis_example_actor"
redis_url = "redis://localhost:6379"
db_path = "/tmp/osis_example_db"
preserve_tasks = false
[worker_type]
[actor_type]
type = "sync"
[logging]

View File

@@ -1,8 +1,8 @@
#!/bin/bash
# OSIS Worker Example Script
# This script demonstrates the OSIS worker by:
# 1. Starting the worker with the config.toml
# OSIS Actor Example Script
# This script demonstrates the OSIS actor by:
# 1. Starting the actor with the config.toml
# 2. Sending ping jobs to Redis
# 3. Verifying pong responses
@@ -10,13 +10,13 @@ set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CONFIG_FILE="$SCRIPT_DIR/config.toml"
WORKER_ID="osis_example_worker"
ACTOR_ID="osis_example_actor"
REDIS_URL="redis://localhost:6379"
echo "=== OSIS Worker Example ==="
echo "=== OSIS Actor Example ==="
echo "Script directory: $SCRIPT_DIR"
echo "Config file: $CONFIG_FILE"
echo "Worker ID: $WORKER_ID"
echo "Actor ID: $ACTOR_ID"
echo "Redis URL: $REDIS_URL"
echo
@@ -32,21 +32,21 @@ echo
# Clean up any existing jobs in the queue
echo "Cleaning up existing jobs in Redis..."
redis-cli -u "$REDIS_URL" del "hero:jobs:$WORKER_ID" > /dev/null 2>&1 || true
redis-cli -u "$REDIS_URL" del "hero:jobs:$ACTOR_ID" > /dev/null 2>&1 || true
redis-cli -u "$REDIS_URL" eval "return redis.call('del', unpack(redis.call('keys', 'hero:job:*')))" 0 > /dev/null 2>&1 || true
echo "✅ Redis queues cleaned"
echo
# Start the OSIS worker in the background
echo "Starting OSIS worker..."
# Start the OSIS actor in the background
echo "Starting OSIS actor..."
cd "$SCRIPT_DIR/../.."
cargo run --bin osis -- --config "$CONFIG_FILE" &
WORKER_PID=$!
echo "✅ OSIS worker started (PID: $WORKER_PID)"
ACTOR_PID=$!
echo "✅ OSIS actor started (PID: $ACTOR_PID)"
echo
# Wait a moment for the worker to initialize
echo "Waiting for worker to initialize..."
# Wait a moment for the actor to initialize
echo "Waiting for actor to initialize..."
sleep 3
# Function to send a ping job and check for pong response
@@ -62,10 +62,10 @@ send_ping_job() {
script "ping" \
status "Queued" \
created_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
worker_id "$WORKER_ID" > /dev/null
actor_id "$ACTOR_ID" > /dev/null
# Add job to worker queue
redis-cli -u "$REDIS_URL" lpush "hero:jobs:$WORKER_ID" "$job_id" > /dev/null
# Add job to actor queue
redis-cli -u "$REDIS_URL" lpush "hero:jobs:$ACTOR_ID" "$job_id" > /dev/null
# Wait for job completion and check result
local timeout=10
@@ -94,7 +94,7 @@ send_ping_job() {
return 1
}
# Send multiple ping jobs to test the worker
# Send multiple ping jobs to test the actor
echo "Testing ping/pong functionality..."
success_count=0
total_jobs=3
@@ -113,26 +113,26 @@ echo "=== Test Results ==="
echo "Successful ping/pong tests: $success_count/$total_jobs"
if [ $success_count -eq $total_jobs ]; then
echo "🎉 All tests passed! OSIS worker is working correctly."
echo "🎉 All tests passed! OSIS actor is working correctly."
exit_code=0
else
echo "⚠️ Some tests failed. Check the worker logs for details."
echo "⚠️ Some tests failed. Check the actor logs for details."
exit_code=1
fi
# Clean up
echo
echo "Cleaning up..."
echo "Stopping OSIS worker (PID: $WORKER_PID)..."
kill $WORKER_PID 2>/dev/null || true
wait $WORKER_PID 2>/dev/null || true
echo "✅ Worker stopped"
echo "Stopping OSIS actor (PID: $ACTOR_PID)..."
kill $ACTOR_PID 2>/dev/null || true
wait $ACTOR_PID 2>/dev/null || true
echo "✅ Actor stopped"
echo "Cleaning up Redis jobs..."
redis-cli -u "$REDIS_URL" del "hero:jobs:$WORKER_ID" > /dev/null 2>&1 || true
redis-cli -u "$REDIS_URL" del "hero:jobs:$ACTOR_ID" > /dev/null 2>&1 || true
redis-cli -u "$REDIS_URL" eval "return redis.call('del', unpack(redis.call('keys', 'hero:job:*')))" 0 > /dev/null 2>&1 || true
echo "✅ Redis cleaned up"
echo
echo "=== OSIS Worker Example Complete ==="
echo "=== OSIS Actor Example Complete ==="
exit $exit_code

View File

@@ -0,0 +1,14 @@
# OSIS Actor Configuration
# Synchronous actor for system-level operations
actor_id = "osis_actor_1"
redis_url = "redis://localhost:6379"
db_path = "/tmp/osis_actor_db"
preserve_tasks = false
[actor_type]
type = "sync"
[logging]
timestamps = true
level = "info"

View File

@@ -3,12 +3,12 @@ use std::path::Path;
use std::env;
use std::io::{self, Write};
/// OSIS Worker Demo Runner
/// OSIS Actor Demo Runner
///
/// This Rust wrapper executes the OSIS worker bash script example.
/// This Rust wrapper executes the OSIS actor bash script example.
/// It provides a way to run shell-based examples through Cargo.
fn main() -> Result<(), Box<dyn std::error::Error>> {
println!("🚀 OSIS Worker Demo");
println!("🚀 OSIS Actor Demo");
println!("==================");
println!();
@@ -19,12 +19,12 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
// Check if the script exists
if !script_path.exists() {
eprintln!("❌ Error: Script not found at {:?}", script_path);
eprintln!(" Make sure you're running this from the worker crate root directory.");
eprintln!(" Make sure you're running this from the actor crate root directory.");
std::process::exit(1);
}
println!("📁 Script location: {:?}", script_path);
println!("🔧 Executing OSIS worker example...");
println!("🔧 Executing OSIS actor example...");
println!();
// Make sure the script is executable
@@ -50,9 +50,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
println!();
if status.success() {
println!("✅ OSIS worker demo completed successfully!");
println!("✅ OSIS actor demo completed successfully!");
} else {
println!("❌ OSIS worker demo failed with exit code: {:?}", status.code());
println!("❌ OSIS actor demo failed with exit code: {:?}", status.code());
std::process::exit(status.code().unwrap_or(1));
}

View File

@@ -1,9 +1,9 @@
worker_id = "system_example_worker"
actor_id = "system_example_actor"
redis_url = "redis://localhost:6379"
db_path = "/tmp/system_example_db"
preserve_tasks = false
[worker_type]
[actor_type]
type = "async"
default_timeout_seconds = 30

View File

@@ -1,8 +1,8 @@
#!/bin/bash
# System Worker Example Script
# This script demonstrates the System worker by:
# 1. Starting the worker with the config.toml
# System Actor Example Script
# This script demonstrates the System actor by:
# 1. Starting the actor with the config.toml
# 2. Sending multiple concurrent ping jobs to Redis
# 3. Verifying pong responses
@@ -10,13 +10,13 @@ set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CONFIG_FILE="$SCRIPT_DIR/config.toml"
WORKER_ID="system_example_worker"
ACTOR_ID="system_example_actor"
REDIS_URL="redis://localhost:6379"
echo "=== System Worker Example ==="
echo "=== System Actor Example ==="
echo "Script directory: $SCRIPT_DIR"
echo "Config file: $CONFIG_FILE"
echo "Worker ID: $WORKER_ID"
echo "Actor ID: $ACTOR_ID"
echo "Redis URL: $REDIS_URL"
echo
@@ -32,21 +32,21 @@ echo
# Clean up any existing jobs in the queue
echo "Cleaning up existing jobs in Redis..."
redis-cli -u "$REDIS_URL" del "hero:jobs:$WORKER_ID" > /dev/null 2>&1 || true
redis-cli -u "$REDIS_URL" del "hero:jobs:$ACTOR_ID" > /dev/null 2>&1 || true
redis-cli -u "$REDIS_URL" eval "return redis.call('del', unpack(redis.call('keys', 'hero:job:*')))" 0 > /dev/null 2>&1 || true
echo "✅ Redis queues cleaned"
echo
# Start the System worker in the background
echo "Starting System worker..."
# Start the System actor in the background
echo "Starting System actor..."
cd "$SCRIPT_DIR/../.."
cargo run --bin system -- --config "$CONFIG_FILE" --show-stats &
WORKER_PID=$!
echo "✅ System worker started (PID: $WORKER_PID)"
ACTOR_PID=$!
echo "✅ System actor started (PID: $ACTOR_PID)"
echo
# Wait a moment for the worker to initialize
echo "Waiting for worker to initialize..."
# Wait a moment for the actor to initialize
echo "Waiting for actor to initialize..."
sleep 3
# Function to send a ping job (non-blocking)
@@ -62,10 +62,10 @@ send_ping_job() {
script "ping" \
status "Queued" \
created_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
worker_id "$WORKER_ID" > /dev/null
actor_id "$ACTOR_ID" > /dev/null
# Add job to worker queue
redis-cli -u "$REDIS_URL" lpush "hero:jobs:$WORKER_ID" "$job_id" > /dev/null
# Add job to actor queue
redis-cli -u "$REDIS_URL" lpush "hero:jobs:$ACTOR_ID" "$job_id" > /dev/null
echo "$job_id"
}
@@ -129,10 +129,10 @@ echo "=== Test Results ==="
echo "Successful concurrent ping/pong tests: $success_count/$total_jobs"
if [ $success_count -eq $total_jobs ]; then
echo "🎉 All tests passed! System worker is handling concurrent jobs correctly."
echo "🎉 All tests passed! System actor is handling concurrent jobs correctly."
exit_code=0
else
echo "⚠️ Some tests failed. Check the worker logs for details."
echo "⚠️ Some tests failed. Check the actor logs for details."
exit_code=1
fi
@@ -160,18 +160,18 @@ echo "Rapid submission test: $rapid_success/$rapid_jobs successful"
# Clean up
echo
echo "Cleaning up..."
echo "Stopping System worker (PID: $WORKER_PID)..."
kill $WORKER_PID 2>/dev/null || true
wait $WORKER_PID 2>/dev/null || true
echo "✅ Worker stopped"
echo "Stopping System actor (PID: $ACTOR_PID)..."
kill $ACTOR_PID 2>/dev/null || true
wait $ACTOR_PID 2>/dev/null || true
echo "✅ Actor stopped"
echo "Cleaning up Redis jobs..."
redis-cli -u "$REDIS_URL" del "hero:jobs:$WORKER_ID" > /dev/null 2>&1 || true
redis-cli -u "$REDIS_URL" del "hero:jobs:$ACTOR_ID" > /dev/null 2>&1 || true
redis-cli -u "$REDIS_URL" eval "return redis.call('del', unpack(redis.call('keys', 'hero:job:*')))" 0 > /dev/null 2>&1 || true
echo "✅ Redis cleaned up"
echo
echo "=== System Worker Example Complete ==="
echo "=== System Actor Example Complete ==="
total_success=$((success_count + rapid_success))
total_tests=$((total_jobs + rapid_jobs))
echo "Overall success rate: $total_success/$total_tests"

View File

@@ -0,0 +1,15 @@
# System Actor Configuration
# Asynchronous actor for high-throughput concurrent processing
actor_id = "system_actor_1"
redis_url = "redis://localhost:6379"
db_path = "/tmp/system_actor_db"
preserve_tasks = false
[actor_type]
type = "async"
default_timeout_seconds = 300 # 5 minutes
[logging]
timestamps = true
level = "info"

View File

@@ -3,12 +3,12 @@ use std::path::Path;
use std::env;
use std::io::{self, Write};
/// System Worker Demo Runner
/// System Actor Demo Runner
///
/// This Rust wrapper executes the System worker bash script example.
/// This Rust wrapper executes the System actor bash script example.
/// It provides a way to run shell-based examples through Cargo.
fn main() -> Result<(), Box<dyn std::error::Error>> {
println!("🚀 System Worker Demo");
println!("🚀 System Actor Demo");
println!("====================");
println!();
@@ -19,12 +19,12 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
// Check if the script exists
if !script_path.exists() {
eprintln!("❌ Error: Script not found at {:?}", script_path);
eprintln!(" Make sure you're running this from the worker crate root directory.");
eprintln!(" Make sure you're running this from the actor crate root directory.");
std::process::exit(1);
}
println!("📁 Script location: {:?}", script_path);
println!("🔧 Executing System worker example...");
println!("🔧 Executing System actor example...");
println!();
// Make sure the script is executable
@@ -50,9 +50,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
println!();
if status.success() {
println!("✅ System worker demo completed successfully!");
println!("✅ System actor demo completed successfully!");
} else {
println!("❌ System worker demo failed with exit code: {:?}", status.code());
println!("❌ System actor demo failed with exit code: {:?}", status.code());
std::process::exit(status.code().unwrap_or(1));
}

View File

@@ -1,13 +1,13 @@
//! # Trait-Based Worker Demo
//! # Trait-Based Actor Demo
//!
//! This example demonstrates the new unified worker interface using the Worker trait.
//! It shows how both synchronous and asynchronous workers can be used with the same
//! This example demonstrates the new unified actor interface using the Actor trait.
//! It shows how both synchronous and asynchronous actors can be used with the same
//! API, eliminating code duplication and providing a clean, consistent interface.
//!
//! ## Features Demonstrated
//!
//! - Unified worker interface using the Worker trait
//! - Both sync and async worker implementations
//! - Unified actor interface using the Actor trait
//! - Both sync and async actor implementations
//! - Shared configuration and spawn logic
//! - Clean shutdown handling
//! - Job processing with different strategies
@@ -16,16 +16,16 @@
//!
//! Make sure Redis is running on localhost:6379, then run:
//! ```bash
//! cargo run --example trait_based_worker_demo
//! cargo run --example trait_based_actor_demo
//! ```
use hero_job::{Job, JobStatus, ScriptType};
use log::{info, warn, error};
use rhailib_worker::{
SyncWorker, AsyncWorker,
spawn_sync_worker, spawn_async_worker,
use baobab_actor::{
SyncActor, AsyncActor,
spawn_sync_actor, spawn_async_actor,
engine::create_heromodels_engine,
worker_trait::{spawn_worker, Worker}
actor_trait::{spawn_actor, Actor}
};
use redis::AsyncCommands;
use std::sync::Arc;
@@ -40,7 +40,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Initialize logging
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
info!("Starting Trait-Based Worker Demo");
info!("Starting Trait-Based Actor Demo");
// Create Redis connection for job creation
let redis_client = redis::Client::open(REDIS_URL)?;
@@ -49,83 +49,83 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Demo 1: Using the unified trait-based interface
info!("=== Demo 1: Unified Trait-Based Interface ===");
// Create shutdown channels for both workers
// Create shutdown channels for both actors
let (sync_shutdown_tx, sync_shutdown_rx) = mpsc::channel::<()>(1);
let (async_shutdown_tx, async_shutdown_rx) = mpsc::channel::<()>(1);
// Workers are now configured using builder pattern directly
// Actors are now configured using builder pattern directly
// Create worker instances using builder pattern
let sync_worker = Arc::new(
SyncWorker::builder()
.worker_id("demo_sync_worker")
// Create actor instances using builder pattern
let sync_actor = Arc::new(
SyncActor::builder()
.actor_id("demo_sync_actor")
.db_path("/tmp")
.redis_url("redis://localhost:6379")
.preserve_tasks(false)
.build()
.expect("Failed to build SyncWorker")
.expect("Failed to build SyncActor")
);
let async_worker = Arc::new(
AsyncWorker::builder()
.worker_id("demo_async_worker")
let async_actor = Arc::new(
AsyncActor::builder()
.actor_id("demo_async_actor")
.db_path("/tmp")
.redis_url("redis://localhost:6379")
.default_timeout(Duration::from_secs(300))
.build()
.expect("Failed to build AsyncWorker")
.expect("Failed to build AsyncActor")
);
let sync_engine = create_heromodels_engine();
let async_engine = create_heromodels_engine();
info!("Spawning {} worker: {}", sync_worker.worker_type(), sync_worker.worker_id());
let sync_handle = spawn_worker(sync_worker.clone(), sync_engine, sync_shutdown_rx);
info!("Spawning {} actor: {}", sync_actor.actor_type(), sync_actor.actor_id());
let sync_handle = spawn_actor(sync_actor.clone(), sync_engine, sync_shutdown_rx);
info!("Spawning {} worker: {}", async_worker.worker_type(), async_worker.worker_id());
let async_handle = spawn_worker(async_worker.clone(), async_engine, async_shutdown_rx);
info!("Spawning {} actor: {}", async_actor.actor_type(), async_actor.actor_id());
let async_handle = spawn_actor(async_actor.clone(), async_engine, async_shutdown_rx);
// Give workers time to start
// Give actors time to start
sleep(Duration::from_secs(1)).await;
// Create and dispatch jobs to both workers
info!("Creating demo jobs for both workers...");
// Create and dispatch jobs to both actors
info!("Creating demo jobs for both actors...");
// Job for sync worker - simple calculation
// Job for sync actor - simple calculation
let sync_job = create_demo_job(
"sync_calculation",
r#"
print("Sync worker: Starting calculation...");
print("Sync actor: Starting calculation...");
let result = 0;
for i in 1..=100 {
result += i;
}
print("Sync worker: Sum of 1-100 = " + result);
print("Sync actor: Sum of 1-100 = " + result);
result
"#,
None,
).await?;
dispatch_job(&mut redis_conn, &sync_job, sync_worker.worker_id()).await?;
info!("Dispatched job to sync worker: {}", sync_job.id);
dispatch_job(&mut redis_conn, &sync_job, sync_actor.actor_id()).await?;
info!("Dispatched job to sync actor: {}", sync_job.id);
// Job for async worker - with timeout demonstration
// Job for async actor - with timeout demonstration
let async_job = create_demo_job(
"async_calculation",
r#"
print("Async worker: Starting calculation...");
print("Async actor: Starting calculation...");
let result = 1;
for i in 1..=10 {
result *= i;
}
print("Async worker: 10! = " + result);
print("Async actor: 10! = " + result);
result
"#,
Some(15), // 15 second timeout
).await?;
dispatch_job(&mut redis_conn, &async_job, async_worker.worker_id()).await?;
info!("Dispatched job to async worker: {}", async_job.id);
dispatch_job(&mut redis_conn, &async_job, async_actor.actor_id()).await?;
info!("Dispatched job to async actor: {}", async_job.id);
// Monitor job execution
info!("Monitoring job execution for 10 seconds...");
@@ -188,13 +188,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let (conv_sync_shutdown_tx, conv_sync_shutdown_rx) = mpsc::channel::<()>(1);
let (conv_async_shutdown_tx, conv_async_shutdown_rx) = mpsc::channel::<()>(1);
// Spawn workers using convenience functions
// Spawn actors using convenience functions
let conv_sync_engine = create_heromodels_engine();
let conv_async_engine = create_heromodels_engine();
info!("Spawning sync worker using convenience function...");
let conv_sync_handle = spawn_sync_worker(
"convenience_sync_worker".to_string(),
info!("Spawning sync actor using convenience function...");
let conv_sync_handle = spawn_sync_actor(
"convenience_sync_actor".to_string(),
"/tmp".to_string(),
conv_sync_engine,
REDIS_URL.to_string(),
@@ -202,9 +202,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
false,
);
info!("Spawning async worker using convenience function...");
let conv_async_handle = spawn_async_worker(
"convenience_async_worker".to_string(),
info!("Spawning async actor using convenience function...");
let conv_async_handle = spawn_async_actor(
"convenience_async_actor".to_string(),
"/tmp".to_string(),
conv_async_engine,
REDIS_URL.to_string(),
@@ -212,15 +212,15 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
Duration::from_secs(20), // 20 second timeout
);
// Give convenience workers time to start
// Give convenience actors time to start
sleep(Duration::from_secs(1)).await;
// Create jobs for convenience workers
// Create jobs for convenience actors
let conv_sync_job = create_demo_job(
"convenience_sync",
r#"
print("Convenience sync worker: Hello World!");
"Hello from convenience sync worker"
print("Convenience sync actor: Hello World!");
"Hello from convenience sync actor"
"#,
None,
).await?;
@@ -228,22 +228,22 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let conv_async_job = create_demo_job(
"convenience_async",
r#"
print("Convenience async worker: Hello World!");
"Hello from convenience async worker"
print("Convenience async actor: Hello World!");
"Hello from convenience async actor"
"#,
Some(10),
).await?;
dispatch_job(&mut redis_conn, &conv_sync_job, "convenience_sync_worker").await?;
dispatch_job(&mut redis_conn, &conv_async_job, "convenience_async_worker").await?;
dispatch_job(&mut redis_conn, &conv_sync_job, "convenience_sync_actor").await?;
dispatch_job(&mut redis_conn, &conv_async_job, "convenience_async_actor").await?;
info!("Dispatched jobs to convenience workers");
info!("Dispatched jobs to convenience actors");
// Wait a bit for jobs to complete
sleep(Duration::from_secs(5)).await;
// Shutdown all workers gracefully
info!("\n=== Shutting Down All Workers ===");
// Shutdown all actors gracefully
info!("\n=== Shutting Down All Actors ===");
info!("Sending shutdown signals...");
let _ = sync_shutdown_tx.send(()).await;
@@ -251,9 +251,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let _ = conv_sync_shutdown_tx.send(()).await;
let _ = conv_async_shutdown_tx.send(()).await;
info!("Waiting for workers to shutdown...");
info!("Waiting for actors to shutdown...");
// Wait for all workers to shutdown
// Wait for all actors to shutdown
let results = tokio::join!(
sync_handle,
async_handle,
@@ -263,23 +263,23 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
match results {
(Ok(Ok(())), Ok(Ok(())), Ok(Ok(())), Ok(Ok(()))) => {
info!("All workers shut down successfully!");
info!("All actors shut down successfully!");
}
_ => {
error!("Some workers encountered errors during shutdown");
error!("Some actors encountered errors during shutdown");
}
}
info!("Trait-Based Worker Demo completed successfully!");
info!("Trait-Based Actor Demo completed successfully!");
// Summary
info!("\n=== Summary ===");
info!("✅ Demonstrated unified Worker trait interface");
info!("✅ Showed both sync and async worker implementations");
info!("✅ Demonstrated unified Actor trait interface");
info!("✅ Showed both sync and async actor implementations");
info!("✅ Used shared configuration and spawn logic");
info!("✅ Maintained backward compatibility with convenience functions");
info!("✅ Eliminated code duplication between worker types");
info!("✅ Provided clean, consistent API for all worker operations");
info!("✅ Eliminated code duplication between actor types");
info!("✅ Provided clean, consistent API for all actor operations");
Ok(())
}
@@ -305,17 +305,17 @@ async fn create_demo_job(
Ok(job)
}
/// Dispatch a job to the worker queue
/// Dispatch a job to the actor queue
async fn dispatch_job(
redis_conn: &mut redis::aio::MultiplexedConnection,
job: &Job,
worker_queue: &str,
actor_queue: &str,
) -> Result<(), Box<dyn std::error::Error>> {
// Store job in Redis
job.store_in_redis(redis_conn).await?;
// Add job to worker queue
let queue_key = format!("hero:job:{}", worker_queue);
// Add job to actor queue
let queue_key = format!("hero:job:{}", actor_queue);
let _: () = redis_conn.rpush(&queue_key, &job.id).await?;
Ok(())

View File

@@ -1,6 +1,6 @@
//! # Asynchronous Worker Implementation
//! # Asynchronous Actor Implementation
//!
//! This module provides an asynchronous worker implementation that can process
//! This module provides an asynchronous actor implementation that can process
//! multiple jobs concurrently with timeout support. Each job is spawned as a
//! separate Tokio task, allowing for parallel execution and proper timeout handling.
//!
@@ -9,7 +9,7 @@
//! - **Concurrent Processing**: Multiple jobs can run simultaneously
//! - **Timeout Support**: Jobs that exceed their timeout are automatically cancelled
//! - **Resource Cleanup**: Proper cleanup of aborted/cancelled jobs
//! - **Non-blocking**: Worker continues processing new jobs while others are running
//! - **Non-blocking**: Actor continues processing new jobs while others are running
//! - **Scalable**: Can handle high job throughput with parallel execution
//!
//! ## Usage
@@ -17,25 +17,25 @@
//! ```rust
//! use std::sync::Arc;
//! use std::time::Duration;
//! use rhailib_worker::async_worker_impl::AsyncWorker;
//! use rhailib_worker::worker_trait::{spawn_worker, WorkerConfig};
//! use rhailib_worker::engine::create_heromodels_engine;
//! use baobab_actor::async_actor_impl::AsyncActor;
//! use baobab_actor::actor_trait::{spawn_actor, ActorConfig};
//! use baobab_actor::engine::create_heromodels_engine;
//! use tokio::sync::mpsc;
//!
//! let config = WorkerConfig::new(
//! "async_worker_1".to_string(),
//! let config = ActorConfig::new(
//! "async_actor_1".to_string(),
//! "/path/to/db".to_string(),
//! "redis://localhost:6379".to_string(),
//! false, // preserve_tasks
//! ).with_default_timeout(Duration::from_secs(300));
//!
//! let worker = Arc::new(AsyncWorker::new());
//! let actor = Arc::new(AsyncActor::new());
//! let engine = create_heromodels_engine();
//! let (shutdown_tx, shutdown_rx) = mpsc::channel(1);
//!
//! let handle = spawn_worker(worker, config, engine, shutdown_rx);
//! let handle = spawn_actor(actor, config, engine, shutdown_rx);
//!
//! // Later, shutdown the worker
//! // Later, shutdown the actor
//! shutdown_tx.send(()).await.unwrap();
//! handle.await.unwrap().unwrap();
//! ```
@@ -52,7 +52,7 @@ use tokio::task::JoinHandle;
use tokio::time::timeout;
use crate::engine::eval_script;
use crate::worker_trait::{Worker, WorkerConfig};
use crate::actor_trait::{Actor, ActorConfig};
use crate::initialize_redis_connection;
/// Represents a running job with its handle and metadata
@@ -63,22 +63,22 @@ struct RunningJob {
started_at: std::time::Instant,
}
/// Builder for AsyncWorker
/// Builder for AsyncActor
#[derive(Debug, Default)]
pub struct AsyncWorkerBuilder {
worker_id: Option<String>,
pub struct AsyncActorBuilder {
actor_id: Option<String>,
db_path: Option<String>,
redis_url: Option<String>,
default_timeout: Option<Duration>,
}
impl AsyncWorkerBuilder {
impl AsyncActorBuilder {
pub fn new() -> Self {
Self::default()
}
pub fn worker_id<S: Into<String>>(mut self, worker_id: S) -> Self {
self.worker_id = Some(worker_id.into());
pub fn actor_id<S: Into<String>>(mut self, actor_id: S) -> Self {
self.actor_id = Some(actor_id.into());
self
}
@@ -97,9 +97,9 @@ impl AsyncWorkerBuilder {
self
}
pub fn build(self) -> Result<AsyncWorker, String> {
Ok(AsyncWorker {
worker_id: self.worker_id.ok_or("worker_id is required")?,
pub fn build(self) -> Result<AsyncActor, String> {
Ok(AsyncActor {
actor_id: self.actor_id.ok_or("actor_id is required")?,
db_path: self.db_path.ok_or("db_path is required")?,
redis_url: self.redis_url.ok_or("redis_url is required")?,
default_timeout: self.default_timeout.unwrap_or(Duration::from_secs(300)),
@@ -108,20 +108,20 @@ impl AsyncWorkerBuilder {
}
}
/// Asynchronous worker that processes jobs concurrently
/// Asynchronous actor that processes jobs concurrently
#[derive(Debug, Clone)]
pub struct AsyncWorker {
pub worker_id: String,
pub struct AsyncActor {
pub actor_id: String,
pub db_path: String,
pub redis_url: String,
pub default_timeout: Duration,
running_jobs: Arc<Mutex<HashMap<String, RunningJob>>>,
}
impl AsyncWorker {
/// Create a new AsyncWorkerBuilder
pub fn builder() -> AsyncWorkerBuilder {
AsyncWorkerBuilder::new()
impl AsyncActor {
/// Create a new AsyncActorBuilder
pub fn builder() -> AsyncActorBuilder {
AsyncActorBuilder::new()
}
/// Add a running job to the tracking map
@@ -134,7 +134,7 @@ impl AsyncWorker {
let mut jobs = self.running_jobs.lock().await;
jobs.insert(job_id.clone(), running_job);
debug!("Async Worker: Added running job '{}'. Total running: {}",
debug!("Async Actor: Added running job '{}'. Total running: {}",
job_id, jobs.len());
}
@@ -143,7 +143,7 @@ impl AsyncWorker {
let mut jobs = self.running_jobs.lock().await;
if let Some(job) = jobs.remove(job_id) {
let duration = job.started_at.elapsed();
debug!("Async Worker: Removed completed job '{}' after {:?}. Remaining: {}",
debug!("Async Actor: Removed completed job '{}' after {:?}. Remaining: {}",
job_id, duration, jobs.len());
}
}
@@ -168,7 +168,7 @@ impl AsyncWorker {
for job_id in to_remove {
if let Some(job) = jobs.remove(&job_id) {
let duration = job.started_at.elapsed();
debug!("Async Worker: Cleaned up finished job '{}' after {:?}",
debug!("Async Actor: Cleaned up finished job '{}' after {:?}",
job_id, duration);
}
}
@@ -178,28 +178,28 @@ impl AsyncWorker {
async fn execute_job_with_timeout(
job: Job,
engine: Engine,
worker_id: String,
actor_id: String,
redis_url: String,
job_timeout: Duration,
) {
let job_id = job.id.clone();
info!("Async Worker '{}', Job {}: Starting execution with timeout {:?}",
worker_id, job_id, job_timeout);
info!("Async Actor '{}', Job {}: Starting execution with timeout {:?}",
actor_id, job_id, job_timeout);
// Create a new Redis connection for this job
let mut redis_conn = match initialize_redis_connection(&worker_id, &redis_url).await {
let mut redis_conn = match initialize_redis_connection(&actor_id, &redis_url).await {
Ok(conn) => conn,
Err(e) => {
error!("Async Worker '{}', Job {}: Failed to initialize Redis connection: {}",
worker_id, job_id, e);
error!("Async Actor '{}', Job {}: Failed to initialize Redis connection: {}",
actor_id, job_id, e);
return;
}
};
// Update job status to Started
if let Err(e) = Job::update_status(&mut redis_conn, &job_id, JobStatus::Started).await {
error!("Async Worker '{}', Job {}: Failed to update status to Started: {}",
worker_id, job_id, e);
error!("Async Actor '{}', Job {}: Failed to update status to Started: {}",
actor_id, job_id, e);
return;
}
@@ -209,35 +209,35 @@ impl AsyncWorker {
match eval_script(&engine, &job.script) {
Ok(result) => {
let result_str = format!("{:?}", result);
info!("Async Worker '{}', Job {}: Script executed successfully. Result: {}",
worker_id, job_id, result_str);
info!("Async Actor '{}', Job {}: Script executed successfully. Result: {}",
actor_id, job_id, result_str);
// Update job with success result
if let Err(e) = Job::set_result(&mut redis_conn, &job_id, &result_str).await {
error!("Async Worker '{}', Job {}: Failed to set result: {}",
worker_id, job_id, e);
error!("Async Actor '{}', Job {}: Failed to set result: {}",
actor_id, job_id, e);
return;
}
if let Err(e) = Job::update_status(&mut redis_conn, &job_id, JobStatus::Finished).await {
error!("Async Worker '{}', Job {}: Failed to update status to Finished: {}",
worker_id, job_id, e);
error!("Async Actor '{}', Job {}: Failed to update status to Finished: {}",
actor_id, job_id, e);
}
}
Err(e) => {
let error_msg = format!("Script execution error: {}", e);
error!("Async Worker '{}', Job {}: {}", worker_id, job_id, error_msg);
error!("Async Actor '{}', Job {}: {}", actor_id, job_id, error_msg);
// Update job with error
if let Err(e) = Job::set_error(&mut redis_conn, &job_id, &error_msg).await {
error!("Async Worker '{}', Job {}: Failed to set error: {}",
worker_id, job_id, e);
error!("Async Actor '{}', Job {}: Failed to set error: {}",
actor_id, job_id, e);
return;
}
if let Err(e) = Job::update_status(&mut redis_conn, &job_id, JobStatus::Error).await {
error!("Async Worker '{}', Job {}: Failed to update status to Error: {}",
worker_id, job_id, e);
error!("Async Actor '{}', Job {}: Failed to update status to Error: {}",
actor_id, job_id, e);
}
}
}
@@ -246,35 +246,35 @@ impl AsyncWorker {
// Execute the script with timeout
match timeout(job_timeout, script_task).await {
Ok(()) => {
info!("Async Worker '{}', Job {}: Completed within timeout", worker_id, job_id);
info!("Async Actor '{}', Job {}: Completed within timeout", actor_id, job_id);
}
Err(_) => {
warn!("Async Worker '{}', Job {}: Timed out after {:?}, marking as error",
worker_id, job_id, job_timeout);
warn!("Async Actor '{}', Job {}: Timed out after {:?}, marking as error",
actor_id, job_id, job_timeout);
let timeout_msg = format!("Job timed out after {:?}", job_timeout);
if let Err(e) = Job::set_error(&mut redis_conn, &job_id, &timeout_msg).await {
error!("Async Worker '{}', Job {}: Failed to set timeout error: {}",
worker_id, job_id, e);
error!("Async Actor '{}', Job {}: Failed to set timeout error: {}",
actor_id, job_id, e);
}
if let Err(e) = Job::update_status(&mut redis_conn, &job_id, JobStatus::Error).await {
error!("Async Worker '{}', Job {}: Failed to update status to Error after timeout: {}",
worker_id, job_id, e);
error!("Async Actor '{}', Job {}: Failed to update status to Error after timeout: {}",
actor_id, job_id, e);
}
}
}
info!("Async Worker '{}', Job {}: Job processing completed", worker_id, job_id);
info!("Async Actor '{}', Job {}: Job processing completed", actor_id, job_id);
}
}
impl Default for AsyncWorker {
impl Default for AsyncActor {
fn default() -> Self {
// Default AsyncWorker with placeholder values
// Default AsyncActor with placeholder values
// In practice, use the builder pattern instead
Self {
worker_id: "default_async_worker".to_string(),
actor_id: "default_async_actor".to_string(),
db_path: "/tmp".to_string(),
redis_url: "redis://localhost:6379".to_string(),
default_timeout: Duration::from_secs(300),
@@ -284,7 +284,7 @@ impl Default for AsyncWorker {
}
#[async_trait]
impl Worker for AsyncWorker {
impl Actor for AsyncActor {
async fn process_job(
&self,
job: Job,
@@ -292,22 +292,22 @@ impl Worker for AsyncWorker {
_redis_conn: &mut redis::aio::MultiplexedConnection,
) {
let job_id = job.id.clone();
let worker_id = &self.worker_id.clone();
let actor_id = &self.actor_id.clone();
// Determine timeout (use job-specific timeout if available, otherwise default)
let job_timeout = if job.timeout.as_secs() > 0 {
job.timeout
} else {
self.default_timeout // Use worker's default timeout
self.default_timeout // Use actor's default timeout
};
info!("Async Worker '{}', Job {}: Spawning job execution task with timeout {:?}",
worker_id, job_id, job_timeout);
info!("Async Actor '{}', Job {}: Spawning job execution task with timeout {:?}",
actor_id, job_id, job_timeout);
// Clone necessary data for the spawned task
let job_id_clone = job_id.clone();
let worker_id_clone = worker_id.clone();
let worker_id_debug = worker_id.clone(); // Additional clone for debug statement
let actor_id_clone = actor_id.clone();
let actor_id_debug = actor_id.clone(); // Additional clone for debug statement
let job_id_debug = job_id.clone(); // Additional clone for debug statement
let redis_url_clone = self.redis_url.clone();
let running_jobs_clone = Arc::clone(&self.running_jobs);
@@ -317,7 +317,7 @@ impl Worker for AsyncWorker {
Self::execute_job_with_timeout(
job,
engine,
worker_id_clone,
actor_id_clone,
redis_url_clone,
job_timeout,
).await;
@@ -326,8 +326,8 @@ impl Worker for AsyncWorker {
let mut jobs = running_jobs_clone.lock().await;
if let Some(running_job) = jobs.remove(&job_id_clone) {
let duration = running_job.started_at.elapsed();
debug!("Async Worker '{}': Removed completed job '{}' after {:?}",
worker_id_debug, job_id_debug, duration);
debug!("Async Actor '{}': Removed completed job '{}' after {:?}",
actor_id_debug, job_id_debug, duration);
}
});
@@ -338,12 +338,12 @@ impl Worker for AsyncWorker {
self.cleanup_finished_jobs().await;
}
fn worker_type(&self) -> &'static str {
fn actor_type(&self) -> &'static str {
"Async"
}
fn worker_id(&self) -> &str {
&self.worker_id
fn actor_id(&self) -> &str {
&self.actor_id
}
fn redis_url(&self) -> &str {
@@ -358,51 +358,51 @@ mod tests {
use hero_job::ScriptType;
#[tokio::test]
async fn test_async_worker_creation() {
let worker = AsyncWorker::new();
assert_eq!(worker.worker_type(), "Async");
assert_eq!(worker.running_job_count().await, 0);
async fn test_async_actor_creation() {
let actor = AsyncActor::new();
assert_eq!(actor.actor_type(), "Async");
assert_eq!(actor.running_job_count().await, 0);
}
#[tokio::test]
async fn test_async_worker_default() {
let worker = AsyncWorker::default();
assert_eq!(worker.worker_type(), "Async");
async fn test_async_actor_default() {
let actor = AsyncActor::default();
assert_eq!(actor.actor_type(), "Async");
}
#[tokio::test]
async fn test_async_worker_job_tracking() {
let worker = AsyncWorker::new();
async fn test_async_actor_job_tracking() {
let actor = AsyncActor::new();
// Simulate adding a job
let handle = tokio::spawn(async {
tokio::time::sleep(Duration::from_millis(100)).await;
});
worker.add_running_job("job_1".to_string(), handle).await;
assert_eq!(worker.running_job_count().await, 1);
actor.add_running_job("job_1".to_string(), handle).await;
assert_eq!(actor.running_job_count().await, 1);
// Wait for job to complete
tokio::time::sleep(Duration::from_millis(200)).await;
worker.cleanup_finished_jobs().await;
assert_eq!(worker.running_job_count().await, 0);
actor.cleanup_finished_jobs().await;
assert_eq!(actor.running_job_count().await, 0);
}
#[tokio::test]
async fn test_async_worker_process_job_interface() {
let worker = AsyncWorker::new();
async fn test_async_actor_process_job_interface() {
let actor = AsyncActor::new();
let engine = create_heromodels_engine();
// Create a simple test job
let job = Job::new(
"test_caller".to_string(),
"test_context".to_string(),
r#"print("Hello from async worker test!"); 42"#.to_string(),
r#"print("Hello from async actor test!"); 42"#.to_string(),
ScriptType::OSIS,
);
let config = WorkerConfig::new(
"test_async_worker".to_string(),
let config = ActorConfig::new(
"test_async_actor".to_string(),
"/tmp".to_string(),
"redis://localhost:6379".to_string(),
false,
@@ -412,9 +412,9 @@ mod tests {
// In a real test environment, you'd need a Redis instance or mock
// The process_job method should be callable (interface test)
// worker.process_job(job, engine, &mut redis_conn, &config).await;
// actor.process_job(job, engine, &mut redis_conn, &config).await;
// For now, just verify the worker was created successfully
assert_eq!(worker.worker_type(), "Async");
// For now, just verify the actor was created successfully
assert_eq!(actor.actor_type(), "Async");
}
}

View File

@@ -1,15 +1,15 @@
//! Worker Configuration Module - TOML-based configuration for Hero workers
//! Actor Configuration Module - TOML-based configuration for Hero actors
use serde::{Deserialize, Serialize};
use std::fs;
use std::path::Path;
use std::time::Duration;
/// Worker configuration loaded from TOML file
/// Actor configuration loaded from TOML file
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WorkerConfig {
/// Worker identification
pub worker_id: String,
pub struct ActorConfig {
/// Actor identification
pub actor_id: String,
/// Redis connection URL
pub redis_url: String,
@@ -21,23 +21,23 @@ pub struct WorkerConfig {
#[serde(default = "default_preserve_tasks")]
pub preserve_tasks: bool,
/// Worker type configuration
pub worker_type: WorkerType,
/// Actor type configuration
pub actor_type: ActorType,
/// Logging configuration
#[serde(default)]
pub logging: LoggingConfig,
}
/// Worker type configuration
/// Actor type configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type")]
pub enum WorkerType {
/// Synchronous worker configuration
pub enum ActorType {
/// Synchronous actor configuration
#[serde(rename = "sync")]
Sync,
/// Asynchronous worker configuration
/// Asynchronous actor configuration
#[serde(rename = "async")]
Async {
/// Default timeout for jobs in seconds
@@ -67,13 +67,13 @@ impl Default for LoggingConfig {
}
}
impl WorkerConfig {
impl ActorConfig {
/// Load configuration from TOML file
pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self, ConfigError> {
let content = fs::read_to_string(&path)
.map_err(|e| ConfigError::IoError(format!("Failed to read config file: {}", e)))?;
let config: WorkerConfig = toml::from_str(&content)
let config: ActorConfig = toml::from_str(&content)
.map_err(|e| ConfigError::ParseError(format!("Failed to parse TOML: {}", e)))?;
config.validate()?;
@@ -82,8 +82,8 @@ impl WorkerConfig {
/// Validate the configuration
fn validate(&self) -> Result<(), ConfigError> {
if self.worker_id.is_empty() {
return Err(ConfigError::ValidationError("worker_id cannot be empty".to_string()));
if self.actor_id.is_empty() {
return Err(ConfigError::ValidationError("actor_id cannot be empty".to_string()));
}
if self.redis_url.is_empty() {
@@ -105,24 +105,24 @@ impl WorkerConfig {
Ok(())
}
/// Get the default timeout duration for async workers
/// Get the default timeout duration for async actors
pub fn get_default_timeout(&self) -> Option<Duration> {
match &self.worker_type {
WorkerType::Sync => None,
WorkerType::Async { default_timeout_seconds } => {
match &self.actor_type {
ActorType::Sync => None,
ActorType::Async { default_timeout_seconds } => {
Some(Duration::from_secs(*default_timeout_seconds))
}
}
}
/// Check if this is a sync worker configuration
/// Check if this is a sync actor configuration
pub fn is_sync(&self) -> bool {
matches!(self.worker_type, WorkerType::Sync)
matches!(self.actor_type, ActorType::Sync)
}
/// Check if this is an async worker configuration
/// Check if this is an async actor configuration
pub fn is_async(&self) -> bool {
matches!(self.worker_type, WorkerType::Async { .. })
matches!(self.actor_type, ActorType::Async { .. })
}
}
@@ -163,13 +163,13 @@ mod tests {
use tempfile::NamedTempFile;
#[test]
fn test_sync_worker_config() {
fn test_sync_actor_config() {
let config_toml = r#"
worker_id = "sync_worker_1"
actor_id = "sync_actor_1"
redis_url = "redis://localhost:6379"
db_path = "/tmp/worker_db"
db_path = "/tmp/actor_db"
[worker_type]
[actor_type]
type = "sync"
[logging]
@@ -177,8 +177,8 @@ timestamps = false
level = "debug"
"#;
let config: WorkerConfig = toml::from_str(config_toml).unwrap();
assert_eq!(config.worker_id, "sync_worker_1");
let config: ActorConfig = toml::from_str(config_toml).unwrap();
assert_eq!(config.actor_id, "sync_actor_1");
assert!(config.is_sync());
assert!(!config.is_async());
assert_eq!(config.get_default_timeout(), None);
@@ -187,13 +187,13 @@ level = "debug"
}
#[test]
fn test_async_worker_config() {
fn test_async_actor_config() {
let config_toml = r#"
worker_id = "async_worker_1"
actor_id = "async_actor_1"
redis_url = "redis://localhost:6379"
db_path = "/tmp/worker_db"
db_path = "/tmp/actor_db"
[worker_type]
[actor_type]
type = "async"
default_timeout_seconds = 600
@@ -202,8 +202,8 @@ timestamps = true
level = "info"
"#;
let config: WorkerConfig = toml::from_str(config_toml).unwrap();
assert_eq!(config.worker_id, "async_worker_1");
let config: ActorConfig = toml::from_str(config_toml).unwrap();
assert_eq!(config.actor_id, "async_actor_1");
assert!(!config.is_sync());
assert!(config.is_async());
assert_eq!(config.get_default_timeout(), Some(Duration::from_secs(600)));
@@ -214,34 +214,34 @@ level = "info"
#[test]
fn test_config_from_file() {
let config_toml = r#"
worker_id = "test_worker"
actor_id = "test_actor"
redis_url = "redis://localhost:6379"
db_path = "/tmp/test_db"
[worker_type]
[actor_type]
type = "sync"
"#;
let mut temp_file = NamedTempFile::new().unwrap();
temp_file.write_all(config_toml.as_bytes()).unwrap();
let config = WorkerConfig::from_file(temp_file.path()).unwrap();
assert_eq!(config.worker_id, "test_worker");
let config = ActorConfig::from_file(temp_file.path()).unwrap();
assert_eq!(config.actor_id, "test_actor");
assert!(config.is_sync());
}
#[test]
fn test_config_validation() {
let config_toml = r#"
worker_id = ""
actor_id = ""
redis_url = "redis://localhost:6379"
db_path = "/tmp/test_db"
[worker_type]
[actor_type]
type = "sync"
"#;
let result: Result<WorkerConfig, _> = toml::from_str(config_toml);
let result: Result<ActorConfig, _> = toml::from_str(config_toml);
assert!(result.is_ok());
let config = result.unwrap();

View File

@@ -14,7 +14,7 @@
//! ## Quick Start
//!
//! ```rust
//! use rhailib_worker::engine::{create_heromodels_engine, eval_script};
//! use baobab_actor::engine::{create_heromodels_engine, eval_script};
//!
//! // Create a fully configured engine
//! let engine = create_heromodels_engine();
@@ -40,7 +40,6 @@
//! - `biz`: Business operations and entities
use rhai::{Engine, EvalAltResult, Scope, AST};
use rhailib_dsl;
use std::fs;
use std::path::Path;
@@ -67,7 +66,7 @@ use std::path::Path;
/// # Example
///
/// ```rust
/// use rhailib_worker::engine::create_heromodels_engine;
/// use baobab_actor::engine::create_heromodels_engine;
///
/// let engine = create_heromodels_engine();
///
@@ -82,14 +81,14 @@ use std::path::Path;
/// The engine is optimized for production use with reasonable defaults for
/// operation limits, expression depth, and memory usage. For benchmarking
/// or special use cases, you may want to adjust these limits after creation.
pub fn create_heromodels_engine() -> Engine {
let mut engine = Engine::new();
// pub fn create_heromodels_engine() -> Engine {
// let mut engine = Engine::new();
// Register all heromodels Rhai modules
rhailib_dsl::register_dsl_modules(&mut engine);
// // Register all heromodels Rhai modules
// baobab_dsl::register_dsl_modules(&mut engine);
engine
}
// engine
// }
/// Evaluates a Rhai script string and returns the result.
///
@@ -110,7 +109,7 @@ pub fn create_heromodels_engine() -> Engine {
/// # Example
///
/// ```rust
/// use rhailib_worker::engine::{create_heromodels_engine, eval_script};
/// use baobab_actor::engine::{create_heromodels_engine, eval_script};
///
/// let engine = create_heromodels_engine();
/// let result = eval_script(&engine, r#"
@@ -146,7 +145,7 @@ pub fn eval_script(
/// # Example
///
/// ```rust
/// use rhailib_worker::engine::{create_heromodels_engine, eval_file};
/// use baobab_actor::engine::{create_heromodels_engine, eval_file};
/// use std::path::Path;
///
/// let engine = create_heromodels_engine();
@@ -191,7 +190,7 @@ pub fn eval_file(
/// # Example
///
/// ```rust
/// use rhailib_worker::engine::{create_heromodels_engine, compile_script, run_ast};
/// use baobab_actor::engine::{create_heromodels_engine, compile_script, run_ast};
/// use rhai::Scope;
///
/// let engine = create_heromodels_engine();
@@ -233,7 +232,7 @@ pub fn compile_script(engine: &Engine, script: &str) -> Result<AST, Box<rhai::Ev
/// # Example
///
/// ```rust
/// use rhailib_worker::engine::{create_heromodels_engine, compile_script, run_ast};
/// use baobab_actor::engine::{create_heromodels_engine, compile_script, run_ast};
/// use rhai::Scope;
///
/// let engine = create_heromodels_engine();

View File

@@ -1,7 +1,7 @@
//! # Synchronous Worker Implementation
//! # Synchronous Actor Implementation
//!
//! This module provides a synchronous worker implementation that processes jobs
//! one at a time in sequence. This is the original worker behavior that's suitable
//! This module provides a synchronous actor implementation that processes jobs
//! one at a time in sequence. This is the original actor behavior that's suitable
//! for scenarios where job execution should not overlap or when resource constraints
//! require sequential processing.
//!
@@ -16,25 +16,25 @@
//!
//! ```rust
//! use std::sync::Arc;
//! use rhailib_worker::sync_worker::SyncWorker;
//! use rhailib_worker::worker_trait::{spawn_worker, WorkerConfig};
//! use rhailib_worker::engine::create_heromodels_engine;
//! use baobab_actor::sync_actor::SyncActor;
//! use baobab_actor::actor_trait::{spawn_actor, ActorConfig};
//! use baobab_actor::engine::create_heromodels_engine;
//! use tokio::sync::mpsc;
//!
//! let config = WorkerConfig::new(
//! "sync_worker_1".to_string(),
//! let config = ActorConfig::new(
//! "sync_actor_1".to_string(),
//! "/path/to/db".to_string(),
//! "redis://localhost:6379".to_string(),
//! false, // preserve_tasks
//! );
//!
//! let worker = Arc::new(SyncWorker::new());
//! let actor = Arc::new(SyncActor::new());
//! let engine = create_heromodels_engine();
//! let (shutdown_tx, shutdown_rx) = mpsc::channel(1);
//!
//! let handle = spawn_worker(worker, config, engine, shutdown_rx);
//! let handle = spawn_actor(actor, config, engine, shutdown_rx);
//!
//! // Later, shutdown the worker
//! // Later, shutdown the actor
//! shutdown_tx.send(()).await.unwrap();
//! handle.await.unwrap().unwrap();
//! ```
@@ -45,24 +45,24 @@ use log::{debug, error, info};
use rhai::Engine;
use crate::engine::eval_script;
use crate::worker_trait::{Worker, WorkerConfig};
use crate::actor_trait::{Actor, ActorConfig};
/// Builder for SyncWorker
/// Builder for SyncActor
#[derive(Debug, Default)]
pub struct SyncWorkerBuilder {
worker_id: Option<String>,
pub struct SyncActorBuilder {
actor_id: Option<String>,
db_path: Option<String>,
redis_url: Option<String>,
preserve_tasks: bool,
}
impl SyncWorkerBuilder {
impl SyncActorBuilder {
pub fn new() -> Self {
Self::default()
}
pub fn worker_id<S: Into<String>>(mut self, worker_id: S) -> Self {
self.worker_id = Some(worker_id.into());
pub fn actor_id<S: Into<String>>(mut self, actor_id: S) -> Self {
self.actor_id = Some(actor_id.into());
self
}
@@ -81,9 +81,9 @@ impl SyncWorkerBuilder {
self
}
pub fn build(self) -> Result<SyncWorker, String> {
Ok(SyncWorker {
worker_id: self.worker_id.ok_or("worker_id is required")?,
pub fn build(self) -> Result<SyncActor, String> {
Ok(SyncActor {
actor_id: self.actor_id.ok_or("actor_id is required")?,
db_path: self.db_path.ok_or("db_path is required")?,
redis_url: self.redis_url.ok_or("redis_url is required")?,
preserve_tasks: self.preserve_tasks,
@@ -91,28 +91,28 @@ impl SyncWorkerBuilder {
}
}
/// Synchronous worker that processes jobs sequentially
/// Synchronous actor that processes jobs sequentially
#[derive(Debug, Clone)]
pub struct SyncWorker {
pub worker_id: String,
pub struct SyncActor {
pub actor_id: String,
pub db_path: String,
pub redis_url: String,
pub preserve_tasks: bool,
}
impl SyncWorker {
/// Create a new SyncWorkerBuilder
pub fn builder() -> SyncWorkerBuilder {
SyncWorkerBuilder::new()
impl SyncActor {
/// Create a new SyncActorBuilder
pub fn builder() -> SyncActorBuilder {
SyncActorBuilder::new()
}
}
impl Default for SyncWorker {
impl Default for SyncActor {
fn default() -> Self {
// Default SyncWorker with placeholder values
// Default SyncActor with placeholder values
// In practice, use the builder pattern instead
Self {
worker_id: "default_sync_worker".to_string(),
actor_id: "default_sync_actor".to_string(),
db_path: "/tmp".to_string(),
redis_url: "redis://localhost:6379".to_string(),
preserve_tasks: false,
@@ -121,7 +121,7 @@ impl Default for SyncWorker {
}
#[async_trait]
impl Worker for SyncWorker {
impl Actor for SyncActor {
async fn process_job(
&self,
job: Job,
@@ -129,15 +129,15 @@ impl Worker for SyncWorker {
redis_conn: &mut redis::aio::MultiplexedConnection,
) {
let job_id = &job.id;
let worker_id = &self.worker_id;
let actor_id = &self.actor_id;
let db_path = &self.db_path;
info!("Sync Worker '{}', Job {}: Starting sequential processing", worker_id, job_id);
info!("Sync Actor '{}', Job {}: Starting sequential processing", actor_id, job_id);
// Update job status to Started
if let Err(e) = Job::update_status(redis_conn, job_id, JobStatus::Started).await {
error!("Sync Worker '{}', Job {}: Failed to update status to Started: {}",
worker_id, job_id, e);
error!("Sync Actor '{}', Job {}: Failed to update status to Started: {}",
actor_id, job_id, e);
return;
}
@@ -145,35 +145,35 @@ impl Worker for SyncWorker {
match eval_script(&engine, &job.script) {
Ok(result) => {
let result_str = format!("{:?}", result);
info!("Sync Worker '{}', Job {}: Script executed successfully. Result: {}",
worker_id, job_id, result_str);
info!("Sync Actor '{}', Job {}: Script executed successfully. Result: {}",
actor_id, job_id, result_str);
// Update job with success result
if let Err(e) = Job::set_result(redis_conn, job_id, &result_str).await {
error!("Sync Worker '{}', Job {}: Failed to set result: {}",
worker_id, job_id, e);
error!("Sync Actor '{}', Job {}: Failed to set result: {}",
actor_id, job_id, e);
return;
}
if let Err(e) = Job::update_status(redis_conn, job_id, JobStatus::Finished).await {
error!("Sync Worker '{}', Job {}: Failed to update status to Finished: {}",
worker_id, job_id, e);
error!("Sync Actor '{}', Job {}: Failed to update status to Finished: {}",
actor_id, job_id, e);
}
}
Err(e) => {
let error_msg = format!("Script execution error: {}", e);
error!("Sync Worker '{}', Job {}: {}", worker_id, job_id, error_msg);
error!("Sync Actor '{}', Job {}: {}", actor_id, job_id, error_msg);
// Update job with error
if let Err(e) = Job::set_error(redis_conn, job_id, &error_msg).await {
error!("Sync Worker '{}', Job {}: Failed to set error: {}",
worker_id, job_id, e);
error!("Sync Actor '{}', Job {}: Failed to set error: {}",
actor_id, job_id, e);
return;
}
if let Err(e) = Job::update_status(redis_conn, job_id, JobStatus::Error).await {
error!("Sync Worker '{}', Job {}: Failed to update status to Error: {}",
worker_id, job_id, e);
error!("Sync Actor '{}', Job {}: Failed to update status to Error: {}",
actor_id, job_id, e);
}
}
}
@@ -181,22 +181,22 @@ impl Worker for SyncWorker {
// Cleanup job if preserve_tasks is false
if !self.preserve_tasks {
if let Err(e) = Job::delete_from_redis(redis_conn, job_id).await {
error!("Sync Worker '{}', Job {}: Failed to cleanup job: {}",
worker_id, job_id, e);
error!("Sync Actor '{}', Job {}: Failed to cleanup job: {}",
actor_id, job_id, e);
} else {
debug!("Sync Worker '{}', Job {}: Job cleaned up from Redis", worker_id, job_id);
debug!("Sync Actor '{}', Job {}: Job cleaned up from Redis", actor_id, job_id);
}
}
info!("Sync Worker '{}', Job {}: Sequential processing completed", worker_id, job_id);
info!("Sync Actor '{}', Job {}: Sequential processing completed", actor_id, job_id);
}
fn worker_type(&self) -> &'static str {
fn actor_type(&self) -> &'static str {
"Sync"
}
fn worker_id(&self) -> &str {
&self.worker_id
fn actor_id(&self) -> &str {
&self.actor_id
}
fn redis_url(&self) -> &str {
@@ -212,32 +212,32 @@ mod tests {
use std::time::Duration;
#[tokio::test]
async fn test_sync_worker_creation() {
let worker = SyncWorker::new();
assert_eq!(worker.worker_type(), "Sync");
async fn test_sync_actor_creation() {
let actor = SyncActor::new();
assert_eq!(actor.actor_type(), "Sync");
}
#[tokio::test]
async fn test_sync_worker_default() {
let worker = SyncWorker::default();
assert_eq!(worker.worker_type(), "Sync");
async fn test_sync_actor_default() {
let actor = SyncActor::default();
assert_eq!(actor.actor_type(), "Sync");
}
#[tokio::test]
async fn test_sync_worker_process_job_interface() {
let worker = SyncWorker::new();
async fn test_sync_actor_process_job_interface() {
let actor = SyncActor::new();
let engine = create_heromodels_engine();
// Create a simple test job
let job = Job::new(
"test_caller".to_string(),
"test_context".to_string(),
r#"print("Hello from sync worker test!"); 42"#.to_string(),
r#"print("Hello from sync actor test!"); 42"#.to_string(),
ScriptType::OSIS,
);
let config = WorkerConfig::new(
"test_sync_worker".to_string(),
let config = ActorConfig::new(
"test_sync_actor".to_string(),
"/tmp".to_string(),
"redis://localhost:6379".to_string(),
false,
@@ -247,9 +247,9 @@ mod tests {
// In a real test environment, you'd need a Redis instance or mock
// The process_job method should be callable (interface test)
// worker.process_job(job, engine, &mut redis_conn, &config).await;
// actor.process_job(job, engine, &mut redis_conn, &config).await;
// For now, just verify the worker was created successfully
assert_eq!(worker.worker_type(), "Sync");
// For now, just verify the actor was created successfully
assert_eq!(actor.actor_type(), "Sync");
}
}

View File

@@ -4,8 +4,8 @@ version = "0.1.0"
edition = "2021"
[[bin]]
name = "supervisor_worker_demo"
path = "supervisor_worker_demo.rs"
name = "supervisor_actor_demo"
path = "supervisor_actor_demo.rs"
[dependencies]
hero_supervisor = { path = "../supervisor" }

View File

@@ -1,8 +1,8 @@
//! Hero Supervisor Worker Demo
//! Hero Supervisor Actor Demo
//!
//! This example demonstrates the new Hero Supervisor API with:
//! - Synchronous build() method
//! - Asynchronous start_workers() method
//! - Asynchronous start_actors() method
//! - Proper cleanup on program exit
//! - Signal handling for graceful shutdown
@@ -18,21 +18,21 @@ async fn run_supervisor_demo() -> Result<(), Box<dyn std::error::Error>> {
// Build supervisor synchronously (no .await needed)
let supervisor = SupervisorBuilder::new()
.redis_url("redis://127.0.0.1:6379")
.osis_worker("/usr/local/bin/osis_worker")
.sal_worker("/usr/local/bin/sal_worker")
.v_worker("/usr/local/bin/v_worker")
.python_worker("/usr/local/bin/python_worker")
.worker_env_var("REDIS_URL", "redis://127.0.0.1:6379")
.worker_env_var("LOG_LEVEL", "info")
.osis_actor("/usr/local/bin/osis_actor")
.sal_actor("/usr/local/bin/sal_actor")
.v_actor("/usr/local/bin/v_actor")
.python_actor("/usr/local/bin/python_actor")
.actor_env_var("REDIS_URL", "redis://127.0.0.1:6379")
.actor_env_var("LOG_LEVEL", "info")
.build()?;
println!("{}", "✅ Supervisor built successfully!".green());
println!("{}", "Starting workers asynchronously...".yellow());
println!("{}", "Starting actors asynchronously...".yellow());
// Start workers asynchronously
supervisor.start_workers().await?;
// Start actors asynchronously
supervisor.start_actors().await?;
println!("{}", "✅ All workers started successfully!".green());
println!("{}", "✅ All actors started successfully!".green());
// Demonstrate job creation and execution
println!("{}", "\n📋 Creating and running test jobs...".cyan().bold());
@@ -43,7 +43,7 @@ async fn run_supervisor_demo() -> Result<(), Box<dyn std::error::Error>> {
// Submit and run the job
match supervisor.new_job()
.script_type(ScriptType::OSIS)
.script("println('Hello from OSIS worker!')")
.script("println('Hello from OSIS actor!')")
.timeout(Duration::from_secs(30))
.await_response().await {
Ok(result) => {
@@ -60,7 +60,7 @@ async fn run_supervisor_demo() -> Result<(), Box<dyn std::error::Error>> {
println!("{}", "\n🛑 Shutdown signal received, cleaning up...".yellow().bold());
// Cleanup workers before exit
// Cleanup actors before exit
supervisor.cleanup_and_shutdown().await?;
println!("{}", "✅ Cleanup completed. Goodbye!".green().bold());

View File

@@ -25,20 +25,20 @@ Where config is toml file with the following structure:
[global]
redis_url = "redis://localhost:6379"
[osis_worker]
binary_path = "/path/to/osis_worker"
[osis_actor]
binary_path = "/path/to/osis_actor"
env_vars = { "VAR1" = "value1", "VAR2" = "value2" }
[sal_worker]
binary_path = "/path/to/sal_worker"
[sal_actor]
binary_path = "/path/to/sal_actor"
env_vars = { "VAR1" = "value1", "VAR2" = "value2" }
[v_worker]
binary_path = "/path/to/v_worker"
[v_actor]
binary_path = "/path/to/v_actor"
env_vars = { "VAR1" = "value1", "VAR2" = "value2" }
[python_worker]
binary_path = "/path/to/python_worker"
[python_actor]
binary_path = "/path/to/python_actor"
env_vars = { "VAR1" = "value1", "VAR2" = "value2" }
```
@@ -46,7 +46,7 @@ env_vars = { "VAR1" = "value1", "VAR2" = "value2" }
Lets have verbosity settings etc.
CLI Offers a few commands:
workers:
actors:
start
stop
restart
@@ -63,4 +63,4 @@ jobs:
logs
list
repl: you can enter interactive mode to run scripts, however predefine caller_id, context_id and worker type so supervisor dispathces jobs accordingly
repl: you can enter interactive mode to run scripts, however predefine caller_id, context_id and actor type so supervisor dispathces jobs accordingly

View File

@@ -43,7 +43,7 @@ struct Args {
struct Config {
global: GlobalConfig,
#[serde(flatten)]
workers: std::collections::HashMap<String, WorkerConfigToml>,
actors: std::collections::HashMap<String, ActorConfigToml>,
}
#[derive(Debug, Deserialize)]
@@ -52,7 +52,7 @@ struct GlobalConfig {
}
#[derive(Debug, Deserialize)]
struct WorkerConfigToml {
struct ActorConfigToml {
binary_path: String,
env_vars: Option<std::collections::HashMap<String, String>>,
}
@@ -60,20 +60,20 @@ struct WorkerConfigToml {
#[derive(Debug, Clone, PartialEq)]
enum TabId {
Dashboard,
Workers,
Actors,
Jobs,
Logs,
}
impl TabId {
fn all() -> Vec<TabId> {
vec![TabId::Dashboard, TabId::Workers, TabId::Jobs, TabId::Logs]
vec![TabId::Dashboard, TabId::Actors, TabId::Jobs, TabId::Logs]
}
fn title(&self) -> &str {
match self {
TabId::Dashboard => "Dashboard",
TabId::Workers => "Workers",
TabId::Actors => "Actors",
TabId::Jobs => "Jobs",
TabId::Logs => "Logs",
}
@@ -167,7 +167,7 @@ fn render_ui(f: &mut Frame, app: &mut App) {
// Render content based on selected tab
match app.current_tab {
TabId::Dashboard => render_dashboard(f, chunks[1], app),
TabId::Workers => render_workers(f, chunks[1], app),
TabId::Actors => render_actors(f, chunks[1], app),
TabId::Jobs => render_jobs(f, chunks[1], app),
TabId::Logs => render_logs(f, chunks[1], app),
}
@@ -180,7 +180,7 @@ fn render_dashboard(f: &mut Frame, area: Rect, app: &App) {
.split(area);
// Status overview - supervisor is already running if we get here
let status_text = "Status: ✓ Running\nWorkers: Started successfully\nJobs: Ready for processing\n\nPress 'q' to quit, Tab to navigate";
let status_text = "Status: ✓ Running\nActors: Started successfully\nJobs: Ready for processing\n\nPress 'q' to quit, Tab to navigate";
let status_paragraph = Paragraph::new(status_text)
.block(Block::default().borders(Borders::ALL).title("System Status"))
@@ -202,9 +202,9 @@ fn render_dashboard(f: &mut Frame, area: Rect, app: &App) {
f.render_widget(logs_list, chunks[1]);
}
fn render_workers(f: &mut Frame, area: Rect, _app: &App) {
let paragraph = Paragraph::new("Workers tab - Status checking not implemented yet to avoid system issues")
.block(Block::default().borders(Borders::ALL).title("Workers"))
fn render_actors(f: &mut Frame, area: Rect, _app: &App) {
let paragraph = Paragraph::new("Actors tab - Status checking not implemented yet to avoid system issues")
.block(Block::default().borders(Borders::ALL).title("Actors"))
.wrap(Wrap { trim: true });
f.render_widget(paragraph, area);
@@ -305,18 +305,18 @@ async fn main() -> Result<()> {
let mut builder = SupervisorBuilder::new()
.redis_url(&config.global.redis_url);
for (worker_name, worker_config) in &config.workers {
match worker_name.as_str() {
"osis_worker" => builder = builder.osis_worker(&worker_config.binary_path),
"sal_worker" => builder = builder.sal_worker(&worker_config.binary_path),
"v_worker" => builder = builder.v_worker(&worker_config.binary_path),
"python_worker" => builder = builder.python_worker(&worker_config.binary_path),
_ => log::warn!("Unknown worker type: {}", worker_name),
for (actor_name, actor_config) in &config.actors {
match actor_name.as_str() {
"osis_actor" => builder = builder.osis_actor(&actor_config.binary_path),
"sal_actor" => builder = builder.sal_actor(&actor_config.binary_path),
"v_actor" => builder = builder.v_actor(&actor_config.binary_path),
"python_actor" => builder = builder.python_actor(&actor_config.binary_path),
_ => log::warn!("Unknown actor type: {}", actor_name),
}
if let Some(env_vars) = &worker_config.env_vars {
if let Some(env_vars) = &actor_config.env_vars {
for (key, value) in env_vars {
builder = builder.worker_env_var(key, value);
builder = builder.actor_env_var(key, value);
}
}
}
@@ -325,11 +325,11 @@ async fn main() -> Result<()> {
.map_err(|e| anyhow::anyhow!("Failed to build supervisor: {}", e))?);
info!("✓ Supervisor built successfully");
// Step 4: Start supervisor and workers
info!("Step 4/4: Starting supervisor and workers...");
supervisor.start_workers().await
.map_err(|e| anyhow::anyhow!("Failed to start workers: {}", e))?;
info!("✓ All workers started successfully");
// Step 4: Start supervisor and actors
info!("Step 4/4: Starting supervisor and actors...");
supervisor.start_actors().await
.map_err(|e| anyhow::anyhow!("Failed to start actors: {}", e))?;
info!("✓ All actors started successfully");
// All initialization successful - now start TUI
info!("Initialization complete - starting TUI...");

View File

@@ -73,7 +73,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Validate script type
match args.script_type.to_lowercase().as_str() {
"osis" | "sal" | "v" | "python" => {
// Valid script types - no worker validation needed since we use hardcoded queues
// Valid script types - no actor validation needed since we use hardcoded queues
}
_ => {
error!("❌ Invalid script type: {}. Valid types: osis, sal, v, python", args.script_type);
@@ -89,7 +89,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
info!(" Script Type: {}", args.script_type);
info!(" Redis URL: {}", args.redis_url);
info!(" Timeout: {}s", args.timeout);
info!(" Using hardcoded worker queues for script type: {}", args.script_type);
info!(" Using hardcoded actor queues for script type: {}", args.script_type);
info!("");
}

View File

@@ -1,14 +0,0 @@
# OSIS Worker Configuration
# Synchronous worker for system-level operations
worker_id = "osis_worker_1"
redis_url = "redis://localhost:6379"
db_path = "/tmp/osis_worker_db"
preserve_tasks = false
[worker_type]
type = "sync"
[logging]
timestamps = true
level = "info"

View File

@@ -1,15 +0,0 @@
# System Worker Configuration
# Asynchronous worker for high-throughput concurrent processing
worker_id = "system_worker_1"
redis_url = "redis://localhost:6379"
db_path = "/tmp/system_worker_db"
preserve_tasks = false
[worker_type]
type = "async"
default_timeout_seconds = 300 # 5 minutes
[logging]
timestamps = true
level = "info"

View File

@@ -0,0 +1,87 @@
1. Generate a keypair locally (public key is safe to share)
- `python tools/gen_auth.py --nonce init`
- Copy PUBLIC_HEX (compressed 33-byte hex, 66 chars). PRIVATE_HEX is your secret—keep it safe.
- Example output:
```
PRIVATE_HEX=5d38d57c83ef1845032fdee1c954958b66912218744ea31d0bc61a07115b6b93
PUBLIC_HEX=0270c0fe3599e82f7142d349fc88e47b07077a43fa00b0fe218ee7bdef4b42d316
NONCE=init
SIGNATURE_HEX=1b109a464c8a6326e66e7bd2caf4c537611f24c6e5e74b0003dc2d5025b6cd6ed180417eacf540938fb306d46d8ebeeed1e6e6c6b69f536d62144baf4a13a139
```
2. Fetch a real nonce from the server
- In hero-openrpc-client menu, choose fetch_nonce
- Paste PUBLIC_HEX when prompted
- Copy the returned nonce string (the exact ASCII hex string)
- Example output:
```
7428f639c215b5ab655283632a39fbd8dc713805cc3b7b0a84c99a5f0e7d5465
```
3. Sign the nonce locally
- python tools/gen_auth.py --nonce "PASTE_NONCE" --priv "YOUR_PRIVATE_HEX"
- Copy SIGNATURE_HEX
- Example output:
```
PRIVATE_HEX=5d38d57c83ef1845032fdee1c954958b66912218744ea31d0bc61a07115b6b93
PUBLIC_HEX=0270c0fe3599e82f7142d349fc88e47b07077a43fa00b0fe218ee7bdef4b42d316
NONCE=7428f639c215b5ab655283632a39fbd8dc713805cc3b7b0a84c99a5f0e7d5465
SIGNATURE_HEX=47dca63f191f328ca9404843a1b3229e4e2affb85ff41dad8125320be3ee07507222c809876d5faa93bfafebdff9e9aef9e17d0b7792d7fcac4d19c92a4b303f
```
4. Authenticate
- In hero-openrpc-client menu, choose authenticate
- Public key (hex): PUBLIC_HEX
- Signature (hex): SIGNATURE_HEX
- Nonce (hex): PASTE_NONCE
After success, whoami should return an authenticated state (basic placeholder in this phase) rust.interfaces/openrpc/server/src/lib.rs.
5. Run `python tools/rpc_smoke_test.py`
- Example output:
```
[rpc] URL: http://127.0.0.1:9944
[rpc] fetch_nonce(pubkey=03fc656cda...): OK
nonce: 4317af6ef04605c7e61ec4759611345f7288497564784cc08afc158553e5ecf1
[rpc] whoami(): OK
whoami: {"authenticated":true,"user_id":"anonymous"}
[rpc] list_jobs(): OK
total: 3
[0] 5f8b4951-35de-4568-8906-a5e9598729e1
[1] 8a0ee6ea-c053-4b72-807a-568c959f5188
[2] 1f929972-3aa5-40c6-af46-6cb81f5a0bae
[rpc] get_job_status(5f8b4951-35de-4568-8906-a5e9598729e1): OK
status: Finished
[rpc] get_job_output(5f8b4951-35de-4568-8906-a5e9598729e1): OK
output: 17
[rpc] get_job_logs(5f8b4951-35de-4568-8906-a5e9598729e1): OK
logs: (no logs)
[rpc] get_job_status(8a0ee6ea-c053-4b72-807a-568c959f5188): OK
status: Finished
[rpc] get_job_output(8a0ee6ea-c053-4b72-807a-568c959f5188): OK
output: 43
[rpc] get_job_logs(8a0ee6ea-c053-4b72-807a-568c959f5188): OK
logs: (no logs)
[rpc] get_job_status(1f929972-3aa5-40c6-af46-6cb81f5a0bae): OK
status: Finished
[rpc] get_job_output(1f929972-3aa5-40c6-af46-6cb81f5a0bae): OK
output: 43
[rpc] get_job_logs(1f929972-3aa5-40c6-af46-6cb81f5a0bae): OK
logs: (no logs)
Smoke tests complete.
Summary:
whoami tested
fetch_nonce tested (pubkey provided/generated)
list_jobs tested (count printed)
detailed queries for up to 3 job(s) (status/output/logs)
```

View File

@@ -21,8 +21,7 @@ tls = false
# "users" = ["04ghi789...", "04jkl012..."]
# "ws" = [] # Public circle (no auth required)
# OSIS Worker Configuration
# OSIS Actor Configuration
# Handles OSIS (HeroScript) execution
[osis_worker]
binary_path = "../target/debug/osis"
env_vars = { "RUST_LOG" = "info", "WORKER_TYPE" = "osis", "MAX_CONCURRENT_JOBS" = "5" }
[osis_actor]
binary_path = "/home/maxime/actor_osis/target/debug/actor_osis"

View File

@@ -37,17 +37,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Wrap supervisor in Arc for sharing across tasks
let supervisor = Arc::new(supervisor);
// Extract worker configurations from TOML config
let worker_configs = supervisor.get_worker_configs()?;
info!("Loaded {} worker configurations from TOML", worker_configs.len());
// Extract actor configurations from TOML config
let actor_configs = supervisor.get_actor_configs()?;
info!("Loaded {} actor configurations from TOML", actor_configs.len());
// Spawn the background lifecycle manager with 5-minute health check interval
let health_check_interval = Duration::from_secs(5 * 60); // 5 minutes
let mut lifecycle_handle = supervisor.clone().spawn_lifecycle_manager(worker_configs, health_check_interval);
let mut lifecycle_handle = supervisor.clone().spawn_lifecycle_manager(actor_configs, health_check_interval);
info!("Hero Supervisor started successfully!");
info!("Background lifecycle manager is running with 5-minute health checks.");
info!("Workers are being monitored and will be automatically restarted if they fail.");
info!("Actors are being monitored and will be automatically restarted if they fail.");
// Start WebSocket server for job dispatching
info!("Starting WebSocket server for job dispatching...");

2
core/actor/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
/target
actor_rhai_temp_db

View File

@@ -1302,7 +1302,7 @@ dependencies = [
]
[[package]]
name = "worker"
name = "actor"
version = "0.1.0"
dependencies = [
"chrono",

View File

@@ -1,11 +1,16 @@
[package]
name = "rhailib_worker"
name = "baobab_actor"
version = "0.1.0"
edition = "2021"
[lib]
name = "rhailib_worker" # Can be different from package name, or same
name = "baobab_actor" # Can be different from package name, or same
path = "src/lib.rs"
crate-type = ["cdylib", "rlib"]
[[bin]]
name = "baobab-actor-tui"
path = "cmd/terminal_ui_main.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -18,17 +23,23 @@ tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] }
log = "0.4"
env_logger = "0.10"
clap = { version = "4.4", features = ["derive"] }
uuid = { version = "1.6", features = ["v4", "serde"] } # Though task_id is string, uuid might be useful
uuid = { version = "1.6", features = ["v4", "serde"] }
chrono = { version = "0.4", features = ["serde"] }
toml = "0.8"
thiserror = "1.0"
async-trait = "0.1"
# TUI dependencies
anyhow = "1.0"
crossterm = "0.28"
ratatui = "0.28"
hero_supervisor = { path = "../supervisor" }
hero_job = { path = "../job" }
heromodels = { git = "https://git.ourworld.tf/herocode/db.git" }
heromodels_core = { git = "https://git.ourworld.tf/herocode/db.git" }
heromodels-derive = { git = "https://git.ourworld.tf/herocode/db.git" }
[features]
default = ["calendar", "finance"]
calendar = []
@@ -37,3 +48,4 @@ flow = []
legal = []
projects = []
biz = []

View File

@@ -1,6 +1,8 @@
# Rhai Worker
# Actor
The `rhai_worker` crate implements a standalone worker service that listens for Rhai script execution tasks from a Redis queue, executes them, and posts results back to Redis. It is designed to be spawned as a separate OS process by an orchestrator like the `launcher` crate.
The `actor` crate defines the trait `Actor` is
and implements a standalone actor service that listens for Rhai script execution tasks from a Redis queue, executes them, and posts results back to Redis. It is designed to be spawned as a separate OS process by an orchestrator like the `launcher` crate.
## Features
@@ -8,61 +10,59 @@ The `rhai_worker` crate implements a standalone worker service that listens for
- **Rhai Script Execution**: Executes Rhai scripts retrieved from Redis based on task IDs.
- **Task State Management**: Updates task status (`processing`, `completed`, `error`) and stores results in Redis hashes.
- **Script Scope Injection**: Automatically injects two important constants into the Rhai script's scope:
- `CONTEXT_ID`: The public key of the worker's own circle.
- `CONTEXT_ID`: The public key of the actor's own circle.
- `CALLER_ID`: The public key of the entity that requested the script execution.
- **Asynchronous Operations**: Built with `tokio` for non-blocking Redis communication.
- **Graceful Error Handling**: Captures errors during script execution and stores them for the client.
## Core Components
- **`worker_lib` (Library Crate)**:
- **`actor_lib` (Library Crate)**:
- **`Args`**: A struct (using `clap`) for parsing command-line arguments: `--redis-url` and `--circle-public-key`.
- **`run_worker_loop(engine: Engine, args: Args)`**: The main asynchronous function that:
- **`run_actor_loop(engine: Engine, args: Args)`**: The main asynchronous function that:
- Connects to Redis.
- Continuously polls the designated Redis queue (`rhai_tasks:<circle_public_key>`) using `BLPOP`.
- Upon receiving a `task_id`, it fetches the task details from a Redis hash.
- It injects `CALLER_ID` and `CONTEXT_ID` into the script's scope.
- It executes the script and updates the task status in Redis with the output or error.
- **`worker` (Binary Crate - `cmd/worker.rs`)**:
- The main executable entry point. It parses command-line arguments, initializes a Rhai engine, and invokes `run_worker_loop`.
## How It Works
1. The worker executable is launched by an external process (e.g., `launcher`), which passes the required command-line arguments.
1. The actor executable is launched by an external process (e.g., `launcher`), which passes the required command-line arguments.
```bash
# This is typically done programmatically by a parent process.
/path/to/worker --redis-url redis://127.0.0.1/ --circle-public-key 02...abc
/path/to/actor --redis-url redis://127.0.0.1/ --circle-public-key 02...abc
```
2. The `run_worker_loop` connects to Redis and starts listening to its designated task queue (e.g., `rhai_tasks:02...abc`).
2. The `run_actor_loop` connects to Redis and starts listening to its designated task queue (e.g., `rhai_tasks:02...abc`).
3. A `rhai_supervisor` submits a task by pushing a `task_id` to this queue and storing the script and other details in a Redis hash.
4. The worker's `BLPOP` command picks up the `task_id`.
5. The worker retrieves the script from the corresponding `rhai_task_details:<task_id>` hash.
4. The actor's `BLPOP` command picks up the `task_id`.
5. The actor retrieves the script from the corresponding `rhai_task_details:<task_id>` hash.
6. It updates the task's status to "processing".
7. The Rhai script is executed within a scope that contains both `CONTEXT_ID` and `CALLER_ID`.
8. After execution, the status is updated to "completed" (with output) or "error" (with an error message).
9. The worker then goes back to listening for the next task.
9. The actor then goes back to listening for the next task.
## Prerequisites
- A running Redis instance accessible by the worker.
- An orchestrator process (like `launcher`) to spawn the worker.
- A running Redis instance accessible by the actor.
- An orchestrator process (like `launcher`) to spawn the actor.
- A `rhai_supervisor` (or another system) to populate the Redis queues.
## Building and Running
The worker is intended to be built as a dependency and run by another program.
The actor is intended to be built as a dependency and run by another program.
1. **Build the worker:**
1. **Build the actor:**
```bash
# From the root of the rhailib project
cargo build --package worker
# From the root of the baobab project
cargo build --package actor
```
The binary will be located at `target/debug/worker`.
The binary will be located at `target/debug/actor`.
2. **Running the worker:**
The worker is not typically run manually. The `launcher` crate is responsible for spawning it with the correct arguments. If you need to run it manually for testing, you must provide the required arguments:
2. **Running the actor:**
The actor is not typically run manually. The `launcher` crate is responsible for spawning it with the correct arguments. If you need to run it manually for testing, you must provide the required arguments:
```bash
./target/debug/worker --redis-url redis://127.0.0.1/ --circle-public-key <a_valid_hex_public_key>
./target/debug/actor --redis-url redis://127.0.0.1/ --circle-public-key <a_valid_hex_public_key>
```
## Dependencies
@@ -73,3 +73,12 @@ Key dependencies include:
- `clap`: For command-line argument parsing.
- `tokio`: For the asynchronous runtime.
- `log`, `env_logger`: For logging.
## TUI Example
```bash
cargo run --example baobab-actor-tui -- --id osis --path /Users/timurgordon/code/git.ourworld.tf/herocode/actor_osis/target/debug/actor_osis --example-dir /Users/timurgordon/code/git.ourworld.tf/herocode/actor_osis/examples/scripts
```
The TUI will allow you to monitor the actor's job queue and dispatch new jobs to it.

File diff suppressed because it is too large Load Diff

View File

@@ -1,12 +1,12 @@
# Architecture of the `rhailib_worker` Crate
# Architecture of the `baobab_actor` Crate
The `rhailib_worker` crate implements a distributed task execution system for Rhai scripts, providing scalable, reliable script processing through Redis-based task queues. Workers are decoupled from contexts, allowing a single worker to process tasks for multiple contexts (circles).
The `baobab_actor` crate implements a distributed task execution system for Rhai scripts, providing scalable, reliable script processing through Redis-based task queues. Actors are decoupled from contexts, allowing a single actor to process tasks for multiple contexts (circles).
## Core Architecture
```mermaid
graph TD
A[Worker Process] --> B[Task Queue Processing]
A[Actor Process] --> B[Task Queue Processing]
A --> C[Script Execution Engine]
A --> D[Result Management]
@@ -31,12 +31,12 @@ graph TD
- **Result Handling**: Comprehensive result and error management
### Engine Integration
- **Rhailib Engine**: Full integration with rhailib_engine for DSL access
- **baobab Engine**: Full integration with baobab_engine for DSL access
- **Context Injection**: Proper authentication and database context setup
- **Security**: Isolated execution environment with access controls
### Scalability Features
- **Horizontal Scaling**: Multiple worker instances for load distribution
- **Horizontal Scaling**: Multiple actor instances for load distribution
- **Queue-based Architecture**: Reliable task distribution via Redis
- **Fault Tolerance**: Robust error handling and recovery mechanisms
@@ -50,4 +50,4 @@ graph TD
## Deployment Patterns
Workers can be deployed as standalone processes, containerized services, or embedded components, providing flexibility for various deployment scenarios from development to production.
Actors can be deployed as standalone processes, containerized services, or embedded components, providing flexibility for various deployment scenarios from development to production.

View File

@@ -0,0 +1,146 @@
//! Simplified main function for Baobab Actor TUI
//!
//! This binary provides a clean entry point for the actor monitoring and job dispatch interface.
use anyhow::{Result, Context};
use baobab_actor::terminal_ui::{App, setup_and_run_tui};
use clap::Parser;
use log::{info, warn, error};
use std::path::PathBuf;
use std::process::{Child, Command};
use tokio::signal;
#[derive(Parser)]
#[command(name = "baobab-actor-tui")]
#[command(about = "Terminal UI for Baobab Actor - Monitor and dispatch jobs to a single actor")]
struct Args {
/// Actor ID to monitor
#[arg(short, long)]
id: String,
/// Path to actor binary
#[arg(short, long)]
path: PathBuf,
/// Directory containing example .rhai scripts
#[arg(short, long)]
example_dir: Option<PathBuf>,
/// Redis URL for job queue
#[arg(short, long, default_value = "redis://localhost:6379")]
redis_url: String,
/// Enable verbose logging
#[arg(short, long)]
verbose: bool,
}
/// Initialize logging based on verbosity level
fn init_logging(verbose: bool) {
if verbose {
env_logger::Builder::from_default_env()
.filter_level(log::LevelFilter::Debug)
.init();
} else {
env_logger::Builder::from_default_env()
.filter_level(log::LevelFilter::Info)
.init();
}
}
/// Create and configure the TUI application
fn create_app(args: &Args) -> Result<App> {
App::new(
args.id.clone(),
args.path.clone(),
args.redis_url.clone(),
args.example_dir.clone(),
)
}
/// Spawn the actor binary as a background process
fn spawn_actor_process(args: &Args) -> Result<Child> {
info!("🎬 Spawning actor process: {}", args.path.display());
let mut cmd = Command::new(&args.path);
// Redirect stdout and stderr to null to prevent logs from interfering with TUI
cmd.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null());
// Spawn the process
let child = cmd
.spawn()
.with_context(|| format!("Failed to spawn actor process: {}", args.path.display()))?;
info!("✅ Actor process spawned with PID: {}", child.id());
Ok(child)
}
/// Cleanup function to terminate actor process
fn cleanup_actor_process(mut actor_process: Child) {
info!("🧹 Cleaning up actor process...");
match actor_process.try_wait() {
Ok(Some(status)) => {
info!("Actor process already exited with status: {}", status);
}
Ok(None) => {
info!("Terminating actor process...");
if let Err(e) = actor_process.kill() {
error!("Failed to kill actor process: {}", e);
} else {
match actor_process.wait() {
Ok(status) => info!("Actor process terminated with status: {}", status),
Err(e) => error!("Failed to wait for actor process: {}", e),
}
}
}
Err(e) => {
error!("Failed to check actor process status: {}", e);
}
}
}
#[tokio::main]
async fn main() -> Result<()> {
let args = Args::parse();
// Initialize logging
init_logging(args.verbose);
info!("🚀 Starting Baobab Actor TUI...");
info!("Actor ID: {}", args.id);
info!("Actor Path: {}", args.path.display());
info!("Redis URL: {}", args.redis_url);
if let Some(ref example_dir) = args.example_dir {
info!("Example Directory: {}", example_dir.display());
}
// Spawn the actor process first
let actor_process = spawn_actor_process(&args)?;
// Give the actor a moment to start up
tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
// Create app and run TUI
let app = create_app(&args)?;
// Set up signal handling for graceful shutdown
let result = tokio::select! {
tui_result = setup_and_run_tui(app) => {
info!("TUI exited");
tui_result
}
_ = signal::ctrl_c() => {
info!("Received Ctrl+C, shutting down...");
Ok(())
}
};
// Clean up the actor process
cleanup_actor_process(actor_process);
result
}

View File

@@ -0,0 +1,272 @@
//! # Actor Trait Abstraction
//!
//! This module provides a trait-based abstraction for Rhai actors that eliminates
//! code duplication between synchronous and asynchronous actor implementations.
//!
//! The `Actor` trait defines the common interface and behavior, while specific
//! implementations handle job processing differently (sync vs async).
//!
//! ## Architecture
//!
//! ```text
//! ┌─────────────────┐ ┌─────────────────┐
//! │ SyncActor │ │ AsyncActor │
//! │ │ │ │
//! │ process_job() │ │ process_job() │
//! │ (sequential) │ │ (concurrent) │
//! └─────────────────┘ └─────────────────┘
//! │ │
//! └───────┬───────────────┘
//! │
//! ┌───────▼───────┐
//! │ Actor Trait │
//! │ │
//! │ spawn() │
//! │ config │
//! │ common loop │
//! └───────────────┘
//! ```
use hero_job::{Job, ScriptType};
use hero_job::keys;
use log::{debug, error, info};
use redis::AsyncCommands;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::mpsc;
use tokio::task::JoinHandle;
use crate::{initialize_redis_connection, BLPOP_TIMEOUT_SECONDS};
/// Configuration for actor instances
#[derive(Debug, Clone)]
pub struct ActorConfig {
pub actor_id: String,
pub db_path: String,
pub redis_url: String,
pub default_timeout: Option<Duration>, // Only used by async actors
}
impl ActorConfig {
/// Create a new actor configuration
pub fn new(
actor_id: String,
db_path: String,
redis_url: String,
) -> Self {
Self {
actor_id,
db_path,
redis_url,
default_timeout: None,
}
}
/// Set default timeout for async actors
pub fn with_default_timeout(mut self, timeout: Duration) -> Self {
self.default_timeout = Some(timeout);
self
}
}
/// Trait defining the common interface for Rhai actors
///
/// This trait abstracts the common functionality between synchronous and
/// asynchronous actors, allowing them to share the same spawn logic and
/// Redis polling loop while implementing different job processing strategies.
#[async_trait::async_trait]
pub trait Actor: Send + Sync + 'static {
/// Process a single job
///
/// This is the core method that differentiates actor implementations:
/// - Sync actors process jobs sequentially, one at a time
/// - Async actors spawn concurrent tasks for each job
///
/// # Arguments
///
/// * `job` - The job to process
/// * `redis_conn` - Redis connection for status updates
///
/// Note: The engine is now owned by the actor implementation as a field
async fn process_job(
&self,
job: Job,
redis_conn: &mut redis::aio::MultiplexedConnection,
);
/// Get the actor type name for logging
fn actor_type(&self) -> &'static str;
/// Get actor ID for this actor instance
fn actor_id(&self) -> &str;
/// Get Redis URL for this actor instance
fn redis_url(&self) -> &str;
/// Spawn the actor
///
/// This method provides the common actor loop implementation that both
/// sync and async actors can use. It handles:
/// - Redis connection setup
/// - Job polling from Redis queue
/// - Shutdown signal handling
/// - Delegating job processing to the implementation
///
/// Note: The engine is now owned by the actor implementation as a field
fn spawn(
self: Arc<Self>,
mut shutdown_rx: mpsc::Receiver<()>,
) -> JoinHandle<Result<(), Box<dyn std::error::Error + Send + Sync>>> {
tokio::spawn(async move {
let actor_id = self.actor_id();
let redis_url = self.redis_url();
// Canonical work queue based on script type (instance/group selection can be added later)
let script_type = derive_script_type_from_actor_id(actor_id);
let queue_key = keys::work_type(&script_type);
info!(
"{} Actor '{}' starting. Type {:?}. Connecting to Redis at {}. Listening on queue: {}",
self.actor_type(),
actor_id,
script_type,
redis_url,
queue_key
);
let mut redis_conn = initialize_redis_connection(actor_id, redis_url).await?;
loop {
let blpop_keys = vec![queue_key.clone()];
tokio::select! {
// Listen for shutdown signal
_ = shutdown_rx.recv() => {
info!("{} Actor '{}': Shutdown signal received. Terminating loop.",
self.actor_type(), actor_id);
break;
}
// Listen for tasks from Redis
blpop_result = redis_conn.blpop(&blpop_keys, BLPOP_TIMEOUT_SECONDS as f64) => {
debug!("{} Actor '{}': Attempting BLPOP on queue: {}",
self.actor_type(), actor_id, queue_key);
let response: Option<(String, String)> = match blpop_result {
Ok(resp) => resp,
Err(e) => {
error!("{} Actor '{}': Redis BLPOP error on queue {}: {}. Actor for this circle might stop.",
self.actor_type(), actor_id, queue_key, e);
return Err(Box::new(e) as Box<dyn std::error::Error + Send + Sync>);
}
};
if let Some((_queue_name_recv, job_id)) = response {
info!("{} Actor '{}' received job_id: {} from queue: {}",
self.actor_type(), actor_id, job_id, _queue_name_recv);
// Load the job from Redis
match crate::load_job_from_redis(&mut redis_conn, &job_id, actor_id).await {
Ok(job) => {
// Check for ping job and handle it directly
if job.script.trim() == "ping" {
info!("{} Actor '{}': Received ping job '{}', responding with pong",
self.actor_type(), actor_id, job_id);
// Update job status to started
if let Err(e) = hero_job::Job::update_status(&mut redis_conn, &job_id, hero_job::JobStatus::Started).await {
error!("{} Actor '{}': Failed to update ping job '{}' status to Started: {}",
self.actor_type(), actor_id, job_id, e);
}
// Set result to "pong" and mark as finished
if let Err(e) = hero_job::Job::set_result(&mut redis_conn, &job_id, "pong").await {
error!("{} Actor '{}': Failed to set ping job '{}' result: {}",
self.actor_type(), actor_id, job_id, e);
}
info!("{} Actor '{}': Successfully responded to ping job '{}' with pong",
self.actor_type(), actor_id, job_id);
} else {
// Delegate job processing to the implementation
// The engine is now owned by the actor implementation
self.process_job(job, &mut redis_conn).await;
}
}
Err(e) => {
error!("{} Actor '{}': Failed to load job '{}': {}",
self.actor_type(), actor_id, job_id, e);
}
}
} else {
debug!("{} Actor '{}': BLPOP timed out on queue {}. No new tasks.",
self.actor_type(), actor_id, queue_key);
}
}
}
}
info!("{} Actor '{}' has shut down.", self.actor_type(), actor_id);
Ok(())
})
}
}
/// Convenience function to spawn a actor with the trait-based interface
///
/// This function provides a unified interface for spawning any actor implementation
/// that implements the Actor trait.
///
/// # Arguments
///
/// * `actor` - The actor implementation to spawn
/// * `config` - Actor configuration
/// * `engine` - Rhai engine for script execution
/// * `shutdown_rx` - Channel receiver for shutdown signals
///
/// # Returns
///
/// Returns a `JoinHandle` that can be awaited to wait for actor shutdown.
///
/// # Example
///
/// ```rust
/// use std::sync::Arc;
/// use std::time::Duration;
///
/// let config = ActorConfig::new(
/// "actor_1".to_string(),
/// "/path/to/db".to_string(),
/// "redis://localhost:6379".to_string(),
/// false,
/// );
///
/// let actor = Arc::new(SyncActor::new());
/// let engine = create_heromodels_engine();
/// let (shutdown_tx, shutdown_rx) = mpsc::channel(1);
///
/// let handle = spawn_actor(actor, config, engine, shutdown_rx);
///
/// // Later, shutdown the actor
/// shutdown_tx.send(()).await.unwrap();
/// handle.await.unwrap().unwrap();
/// ```
pub fn spawn_actor<W: Actor>(
actor: Arc<W>,
shutdown_rx: mpsc::Receiver<()>,
) -> JoinHandle<Result<(), Box<dyn std::error::Error + Send + Sync>>> {
actor.spawn(shutdown_rx)
}
fn derive_script_type_from_actor_id(actor_id: &str) -> ScriptType {
let lower = actor_id.to_lowercase();
if lower.contains("sal") {
ScriptType::SAL
} else if lower.contains("osis") {
ScriptType::OSIS
} else if lower.contains("python") {
ScriptType::Python
} else if lower.contains("v") {
ScriptType::V
} else {
// Default to OSIS when uncertain
ScriptType::OSIS
}
}

0
core/actor/src/config.rs Normal file
View File

283
core/actor/src/lib.rs Normal file
View File

@@ -0,0 +1,283 @@
use hero_job::{Job, JobStatus, ScriptType};
use hero_job::keys;
use log::{debug, error, info};
use redis::AsyncCommands;
use rhai::{Dynamic, Engine};
use tokio::sync::mpsc; // For shutdown signal
use tokio::task::JoinHandle;
/// Actor trait abstraction for unified actor interface
pub mod actor_trait;
/// Terminal UI module for actor monitoring and job dispatch
pub mod terminal_ui;
const NAMESPACE_PREFIX: &str = "hero:job:";
const BLPOP_TIMEOUT_SECONDS: usize = 5;
/// Initialize Redis connection for the actor
pub async fn initialize_redis_connection(
actor_id: &str,
redis_url: &str,
) -> Result<redis::aio::MultiplexedConnection, Box<dyn std::error::Error + Send + Sync>> {
let redis_client = redis::Client::open(redis_url)
.map_err(|e| {
error!("Actor for Actor ID '{}': Failed to open Redis client: {}", actor_id, e);
e
})?;
let redis_conn = redis_client.get_multiplexed_async_connection().await
.map_err(|e| {
error!("Actor for Actor ID '{}': Failed to get Redis connection: {}", actor_id, e);
e
})?;
info!("Actor for Actor ID '{}' successfully connected to Redis.", actor_id);
Ok(redis_conn)
}
/// Load job from Redis using Job struct
pub(crate) async fn load_job_from_redis(
redis_conn: &mut redis::aio::MultiplexedConnection,
job_id: &str,
actor_id: &str,
) -> Result<Job, Box<dyn std::error::Error + Send + Sync>> {
debug!("Actor '{}', Job {}: Loading job from Redis", actor_id, job_id);
match Job::load_from_redis(redis_conn, job_id).await {
Ok(job) => {
debug!("Actor '{}', Job {}: Successfully loaded job", actor_id, job_id);
Ok(job)
}
Err(e) => {
error!("Actor '{}', Job {}: Failed to load job from Redis: {}", actor_id, job_id, e);
Err(Box::new(e))
}
}
}
/// Execute the Rhai script and update job status in Redis
async fn execute_script_and_update_status(
redis_conn: &mut redis::aio::MultiplexedConnection,
engine: &mut Engine,
job: &Job,
db_path: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let mut db_config = rhai::Map::new();
db_config.insert("DB_PATH".into(), db_path.to_string().into());
db_config.insert("CALLER_ID".into(), job.caller_id.clone().into());
db_config.insert("CONTEXT_ID".into(), job.context_id.clone().into());
engine.set_default_tag(Dynamic::from(db_config));
debug!("Actor for Context ID '{}': Evaluating script with Rhai engine.", job.context_id);
match engine.eval::<rhai::Dynamic>(&job.script) {
Ok(result) => {
let output_str = if result.is::<String>() {
result.into_string().unwrap()
} else {
result.to_string()
};
info!("Actor for Context ID '{}' job {} completed. Output: {}", job.context_id, job.id, output_str);
// Update job status to finished and set result
Job::update_status(redis_conn, &job.id, JobStatus::Finished).await
.map_err(|e| {
error!("Failed to update job {} status to finished: {}", job.id, e);
e
})?;
Job::set_result(redis_conn, &job.id, &output_str).await
.map_err(|e| {
error!("Failed to set job {} result: {}", job.id, e);
e
})?;
Ok(())
}
Err(e) => {
let error_str = format!("{:?}", *e);
error!("Actor for Context ID '{}' job {} script evaluation failed. Error: {}", job.context_id, job.id, error_str);
// Update job status to error and set error message
Job::update_status(redis_conn, &job.id, JobStatus::Error).await
.map_err(|e| {
error!("Failed to update job {} status to error: {}", job.id, e);
e
})?;
Job::set_error(redis_conn, &job.id, &error_str).await
.map_err(|e| {
error!("Failed to set job {} error: {}", job.id, e);
e
})?;
Ok(())
}
}
}
/// Execute a job with the given engine, setting proper job context
///
/// This function sets up the engine with job context (DB_PATH, CALLER_ID, CONTEXT_ID)
/// and evaluates the script. It returns the result or error without updating Redis.
/// This allows actors to handle Redis updates according to their own patterns.
pub async fn execute_job_with_engine(
engine: &mut Engine,
job: &Job,
db_path: &str,
) -> Result<Dynamic, Box<rhai::EvalAltResult>> {
// Set up job context in the engine
let mut db_config = rhai::Map::new();
db_config.insert("DB_PATH".into(), db_path.to_string().into());
db_config.insert("CALLER_ID".into(), job.caller_id.clone().into());
db_config.insert("CONTEXT_ID".into(), job.context_id.clone().into());
engine.set_default_tag(Dynamic::from(db_config));
debug!("Actor for Context ID '{}': Evaluating script with Rhai engine (job context set).", job.context_id);
// Execute the script with the configured engine
engine.eval::<Dynamic>(&job.script)
}
/// Clean up job from Redis if preserve_tasks is false
async fn cleanup_job(
redis_conn: &mut redis::aio::MultiplexedConnection,
job_id: &str,
context_id: &str,
preserve_tasks: bool,
) {
if !preserve_tasks {
if let Err(e) = Job::delete_from_redis(redis_conn, job_id).await {
error!("Actor for Context ID '{}', Job {}: Failed to delete job: {}", context_id, job_id, e);
} else {
debug!("Actor for Context ID '{}', Job {}: Cleaned up job.", context_id, job_id);
}
} else {
debug!("Actor for Context ID '{}', Job {}: Preserving job (preserve_tasks=true)", context_id, job_id);
}
}
/// Process a single job from the queue
async fn process_job(
redis_conn: &mut redis::aio::MultiplexedConnection,
job_id: &str,
actor_id: &str,
db_path: &str,
engine: &mut Engine,
preserve_tasks: bool,
) {
debug!("Actor '{}', Job {}: Processing started.", actor_id, job_id);
// Load job from Redis
match load_job_from_redis(redis_conn, job_id, actor_id).await {
Ok(job) => {
info!("Actor '{}' processing job_id: {}. Script: {:.50}...", job.context_id, job_id, job.script);
// Update status to started
debug!("Actor for Context ID '{}', Job {}: Attempting to update status to 'started'.", job.context_id, job_id);
if let Err(e) = Job::update_status(redis_conn, job_id, JobStatus::Started).await {
error!("Actor for Context ID '{}', Job {}: Failed to update status to 'started': {}", job.context_id, job_id, e);
} else {
debug!("Actor for Context ID '{}', Job {}: Status updated to 'started'.", job.context_id, job_id);
}
// Execute the script and update status
if let Err(e) = execute_script_and_update_status(redis_conn, engine, &job, db_path).await {
error!("Actor for Context ID '{}', Job {}: Script execution failed: {}", job.context_id, job_id, e);
// Ensure job status is set to error if execution failed
if let Err(status_err) = Job::update_status(redis_conn, job_id, JobStatus::Error).await {
error!("Actor for Context ID '{}', Job {}: Failed to update status to error after execution failure: {}", job.context_id, job_id, status_err);
}
}
// Clean up job if needed
cleanup_job(redis_conn, job_id, &job.context_id, preserve_tasks).await;
}
Err(e) => {
error!("Actor '{}', Job {}: Failed to load job: {}", actor_id, job_id, e);
// Clean up invalid job if needed
if !preserve_tasks {
if let Err(del_err) = Job::delete_from_redis(redis_conn, job_id).await {
error!("Actor '{}', Job {}: Failed to delete invalid job: {}", actor_id, job_id, del_err);
}
} else {
debug!("Actor '{}', Job {}: Preserving invalid job (preserve_tasks=true)", actor_id, job_id);
}
}
}
}
pub fn spawn_rhai_actor(
actor_id: String,
db_path: String,
mut engine: Engine,
redis_url: String,
mut shutdown_rx: mpsc::Receiver<()>,
preserve_tasks: bool,
) -> JoinHandle<Result<(), Box<dyn std::error::Error + Send + Sync>>> {
tokio::spawn(async move {
let script_type = derive_script_type_from_actor_id(&actor_id);
let queue_key = keys::work_type(&script_type);
info!(
"Rhai Actor '{}' starting. Type {:?}. Connecting to Redis at {}. Listening on queue: {}. Waiting for tasks or shutdown signal.",
actor_id, script_type, redis_url, queue_key
);
let mut redis_conn = initialize_redis_connection(&actor_id, &redis_url).await?;
loop {
let blpop_keys = vec![queue_key.clone()];
tokio::select! {
// Listen for shutdown signal
_ = shutdown_rx.recv() => {
info!("Actor for Actor ID '{}': Shutdown signal received. Terminating loop.", actor_id);
break;
}
// Listen for tasks from Redis
blpop_result = redis_conn.blpop(&blpop_keys, BLPOP_TIMEOUT_SECONDS as f64) => {
debug!("Actor for Actor ID '{}': Attempting BLPOP on queue: {}", actor_id, queue_key);
let response: Option<(String, String)> = match blpop_result {
Ok(resp) => resp,
Err(e) => {
error!("Actor '{}': Redis BLPOP error on queue {}: {}. Actor for this circle might stop.", actor_id, queue_key, e);
return Err(Box::new(e) as Box<dyn std::error::Error + Send + Sync>);
}
};
if let Some((_queue_name_recv, job_id)) = response {
info!("Actor '{}' received job_id: {} from queue: {}", actor_id, job_id, _queue_name_recv);
process_job(&mut redis_conn, &job_id, &actor_id, &db_path, &mut engine, preserve_tasks).await;
} else {
debug!("Actor '{}': BLPOP timed out on queue {}. No new tasks. Checking for shutdown signal again.", actor_id, queue_key);
}
}
}
}
info!("Actor '{}' has shut down.", actor_id);
Ok(())
})
}
// Helper to derive script type from actor_id for canonical queue selection
fn derive_script_type_from_actor_id(actor_id: &str) -> ScriptType {
let lower = actor_id.to_lowercase();
if lower.contains("sal") {
ScriptType::SAL
} else if lower.contains("osis") {
ScriptType::OSIS
} else if lower.contains("python") {
ScriptType::Python
} else if lower == "v" || lower.contains(":v") || lower.contains(" v") {
ScriptType::V
} else {
// Default to OSIS when uncertain
ScriptType::OSIS
}
}
// Re-export the main trait-based interface for convenience
pub use actor_trait::{Actor, ActorConfig, spawn_actor};

38
core/actor/src/main.rs Normal file
View File

@@ -0,0 +1,38 @@
#[cfg(feature = "wasm")]
use baobab_actor::ui::App;
#[cfg(feature = "wasm")]
use yew::prelude::*;
#[cfg(feature = "wasm")]
fn main() {
console_log::init_with_level(log::Level::Debug).expect("Failed to initialize logger");
// Get configuration from URL parameters or local storage
let window = web_sys::window().expect("No global window exists");
let location = window.location();
let search = location.search().unwrap_or_default();
// Parse URL parameters for actor configuration
let url_params = web_sys::UrlSearchParams::new_with_str(&search).unwrap();
let actor_id = url_params.get("id").unwrap_or_else(|| "default_actor".to_string());
let actor_path = url_params.get("path").unwrap_or_else(|| "/path/to/actor".to_string());
let example_dir = url_params.get("example_dir");
let redis_url = url_params.get("redis_url").unwrap_or_else(|| "redis://localhost:6379".to_string());
log::info!("Starting Baobab Actor UI with actor_id: {}", actor_id);
yew::Renderer::<App>::with_props(baobab_actor::ui::app::AppProps {
actor_id,
actor_path,
example_dir,
redis_url,
}).render();
}
#[cfg(not(feature = "wasm"))]
fn main() {
eprintln!("This binary is only available with the 'wasm' feature enabled.");
eprintln!("Please compile with: cargo build --features wasm --target wasm32-unknown-unknown");
std::process::exit(1);
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,5 @@
# Architecture
Supervisor runs actors and manages their lifecycle. Additionally supervisor dispatches jobs to workers and provides an API for job supervision. Jobs are dispatched to workers over a redis protocol. Jobs have a script which is the code that is to be executed by the worker. There are two script formats used: Rhai and HeroScript. Jobs also have params such as timeout and priority for job management, and context variables which are available to the script such as CALLER_ID and CONTEXT_ID. There are four different types of workers: OSIS, SAL, V and Python. OSIS and SAL workers use Rhai scripts, while V and Python workers use HeroScript. Each worker has its own queue and is responsible for processing jobs of its type. Each worker has a unique way of executing the script.
Supervisor runs actors and manages their lifecycle. Additionally supervisor dispatches jobs to actors and provides an API for job supervision. Jobs are dispatched to actors over a redis protocol. Jobs have a script which is the code that is to be executed by the actor. There are two script formats used: Rhai and HeroScript. Jobs also have params such as timeout and priority for job management, and context variables which are available to the script such as CALLER_ID and CONTEXT_ID. There are four different types of actors: OSIS, SAL, V and Python. OSIS and SAL actors use Rhai scripts, while V and Python actors use HeroScript. Each actor has its own queue and is responsible for processing jobs of its type. Each actor has a unique way of executing the script.
The OSIS worker executes non-blocking Rhai scripts one after another using the Rhai engine on a single thread. The SAL worker executes blocking asynchronous Rhai scripts concurrently: it spawns a new thread for each script evaluation. V and Python workers execute HeroScript scripts using a V or Python heroscript engine.
The OSIS actor executes non-blocking Rhai scripts one after another using the Rhai engine on a single thread. The SAL actor executes blocking asynchronous Rhai scripts concurrently: it spawns a new thread for each script evaluation. V and Python actors execute HeroScript scripts using a V or Python heroscript engine.

View File

@@ -1,14 +1,14 @@
### `Job`
Represents a script execution request with:
- Unique ID and timestamps
- Script content and target worker
- Script content and target actor
- Execution settings (timeout, retries, concurrency)
- Logging configuration
### `JobBuilder`
Fluent builder for configuring jobs:
- `script()` - Set the script content
- `worker_id()` - Target specific worker
- `actor_id()` - Target specific actor
- `timeout()` - Set execution timeout
- `build()` - Create the job
- `submit()` - Fire-and-forget submission

View File

@@ -76,6 +76,11 @@ impl JobBuilder {
self
}
pub fn caller_id(mut self, caller_id: &str) -> Self {
self.caller_id = caller_id.to_string();
self
}
pub fn script(mut self, script: &str) -> Self {
self.script = script.to_string();
self

View File

@@ -7,26 +7,27 @@ use redis::AsyncCommands;
use thiserror::Error;
mod builder;
pub use builder::JobBuilder;
/// Redis namespace prefix for all Hero job-related keys
pub const NAMESPACE_PREFIX: &str = "hero:job:";
/// Script type enumeration for different worker types
/// Script type enumeration for different actor types
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub enum ScriptType {
/// OSIS - A worker that executes Rhai/HeroScript
/// OSIS - A actor that executes Rhai/HeroScript
OSIS,
/// SAL - A worker that executes system abstraction layer functionalities in rhai
/// SAL - A actor that executes system abstraction layer functionalities in rhai
SAL,
/// V - A worker that executes heroscript in V
/// V - A actor that executes heroscript in V
V,
/// Python - A worker that executes heroscript in python
/// Python - A actor that executes heroscript in python
Python,
}
impl ScriptType {
/// Get the worker queue suffix for this script type
pub fn worker_queue_suffix(&self) -> &'static str {
/// Get the actor queue suffix for this script type
pub fn actor_queue_suffix(&self) -> &'static str {
match self {
ScriptType::OSIS => "osis",
ScriptType::SAL => "sal",
@@ -81,7 +82,7 @@ impl JobStatus {
/// Representation of a script execution request.
///
/// This structure contains all the information needed to execute a script
/// on a worker service, including the script content, dependencies, and metadata.
/// on a actor service, including the script content, dependencies, and metadata.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Job {
pub id: String,
@@ -386,3 +387,47 @@ impl Job {
Ok(job_ids)
}
}
// Canonical Redis key builders for queues and hashes
pub mod keys {
use super::{NAMESPACE_PREFIX, ScriptType};
// hero:job:{job_id}
pub fn job_hash(job_id: &str) -> String {
format!("{}{}", NAMESPACE_PREFIX, job_id)
}
// hero:q:reply:{job_id}
pub fn reply(job_id: &str) -> String {
format!("hero:q:reply:{}", job_id)
}
// hero:q:work:type:{script_type}
pub fn work_type(script_type: &ScriptType) -> String {
format!("hero:q:work:type:{}", script_type.actor_queue_suffix())
}
// hero:q:work:type:{script_type}:group:{group}
pub fn work_group(script_type: &ScriptType, group: &str) -> String {
format!(
"hero:q:work:type:{}:group:{}",
script_type.actor_queue_suffix(),
group
)
}
// hero:q:work:type:{script_type}:group:{group}:inst:{instance}
pub fn work_instance(script_type: &ScriptType, group: &str, instance: &str) -> String {
format!(
"hero:q:work:type:{}:group:{}:inst:{}",
script_type.actor_queue_suffix(),
group,
instance
)
}
// hero:q:ctl:type:{script_type}
pub fn stop_type(script_type: &ScriptType) -> String {
format!("hero:q:ctl:type:{}", script_type.actor_queue_suffix())
}
}

View File

@@ -3,6 +3,14 @@ name = "hero_supervisor"
version = "0.1.0"
edition = "2021"
[[bin]]
name = "supervisor-cli"
path = "cmd/supervisor_cli.rs"
[[bin]]
name = "supervisor-tui"
path = "cmd/supervisor_tui.rs"
[dependencies]
clap = { version = "4.4", features = ["derive"] }
env_logger = "0.10"

View File

@@ -1,20 +1,20 @@
# Worker Lifecycle Management
# Actor Lifecycle Management
The Hero Supervisor includes comprehensive worker lifecycle management functionality using [Zinit](https://github.com/threefoldtech/zinit) as the process manager. This enables the supervisor to manage worker processes, perform health monitoring, and implement load balancing.
The Hero Supervisor includes comprehensive actor lifecycle management functionality using [Zinit](https://github.com/threefoldtech/zinit) as the process manager. This enables the supervisor to manage actor processes, perform health monitoring, and implement load balancing.
## Overview
The lifecycle management system provides:
- **Worker Process Management**: Start, stop, restart, and monitor worker binaries
- **Health Monitoring**: Automatic ping jobs every 10 minutes for idle workers
- **Graceful Shutdown**: Clean termination of worker processes
- **Actor Process Management**: Start, stop, restart, and monitor actor binaries
- **Health Monitoring**: Automatic ping jobs every 10 minutes for idle actors
- **Graceful Shutdown**: Clean termination of actor processes
## Architecture
```
┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐
│ Supervisor │ │ WorkerLifecycle │ │ Zinit │
│ Supervisor │ │ ActorLifecycle │ │ Zinit │
│ │◄──►│ Manager │◄──►│ (Process │
│ (Job Dispatch) │ │ │ │ Manager) │
└─────────────────┘ └──────────────────┘ └─────────────────┘
@@ -22,49 +22,49 @@ The lifecycle management system provides:
│ │ │
▼ ▼ ▼
┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐
│ Redis │ │ Health Monitor │ │ Worker Binaries │
│ Redis │ │ Health Monitor │ │ Actor Binaries │
│ (Job Queue) │ │ (Ping Jobs) │ │ (OSIS/SAL/V) │
└─────────────────┘ └──────────────────┘ └─────────────────┘
```
## Components
### WorkerConfig
### ActorConfig
Defines configuration for a worker binary:
Defines configuration for a actor binary:
```rust
use hero_supervisor::{WorkerConfig, ScriptType};
use hero_supervisor::{ActorConfig, ScriptType};
use std::path::PathBuf;
use std::collections::HashMap;
let config = WorkerConfig::new(
"osis_worker_0".to_string(),
PathBuf::from("/usr/local/bin/osis_worker"),
let config = ActorConfig::new(
"osis_actor_0".to_string(),
PathBuf::from("/usr/local/bin/osis_actor"),
ScriptType::OSIS,
)
.with_args(vec![
"--redis-url".to_string(),
"redis://localhost:6379".to_string(),
"--worker-id".to_string(),
"osis_worker_0".to_string(),
"--actor-id".to_string(),
"osis_actor_0".to_string(),
])
.with_env({
let mut env = HashMap::new();
env.insert("RUST_LOG".to_string(), "info".to_string());
env.insert("WORKER_TYPE".to_string(), "osis".to_string());
env.insert("ACTOR_TYPE".to_string(), "osis".to_string());
env
})
.with_health_check("/usr/local/bin/osis_worker --health-check".to_string())
.with_health_check("/usr/local/bin/osis_actor --health-check".to_string())
.with_dependencies(vec!["redis".to_string()]);
```
### WorkerLifecycleManager
### ActorLifecycleManager
Main component for managing worker lifecycles:
Main component for managing actor lifecycles:
```rust
use hero_supervisor::{WorkerLifecycleManagerBuilder, Supervisor};
use hero_supervisor::{ActorLifecycleManagerBuilder, Supervisor};
let supervisor = SupervisorBuilder::new()
.redis_url("redis://localhost:6379")
@@ -72,11 +72,11 @@ let supervisor = SupervisorBuilder::new()
.context_id("production")
.build()?;
let mut lifecycle_manager = WorkerLifecycleManagerBuilder::new("/var/run/zinit.sock".to_string())
let mut lifecycle_manager = ActorLifecycleManagerBuilder::new("/var/run/zinit.sock".to_string())
.with_supervisor(supervisor.clone())
.add_worker(osis_worker_config)
.add_worker(sal_worker_config)
.add_worker(v_worker_config)
.add_actor(osis_actor_config)
.add_actor(sal_actor_config)
.add_actor(v_actor_config)
.build();
```
@@ -84,45 +84,45 @@ let mut lifecycle_manager = WorkerLifecycleManagerBuilder::new("/var/run/zinit.s
The lifecycle manager supports all Hero script types:
- **OSIS**: Rhai/HeroScript execution workers
- **SAL**: System Abstraction Layer workers
- **OSIS**: Rhai/HeroScript execution actors
- **SAL**: System Abstraction Layer actors
- **V**: HeroScript execution in V language
- **Python**: HeroScript execution in Python
## Key Features
### 1. Worker Management
### 1. Actor Management
```rust
// Start all configured workers
lifecycle_manager.start_all_workers().await?;
// Start all configured actors
lifecycle_manager.start_all_actors().await?;
// Stop all workers
lifecycle_manager.stop_all_workers().await?;
// Stop all actors
lifecycle_manager.stop_all_actors().await?;
// Restart specific worker
lifecycle_manager.restart_worker("osis_worker_0").await?;
// Restart specific actor
lifecycle_manager.restart_actor("osis_actor_0").await?;
// Get worker status
let status = lifecycle_manager.get_worker_status("osis_worker_0").await?;
println!("Worker state: {:?}, PID: {}", status.state, status.pid);
// Get actor status
let status = lifecycle_manager.get_actor_status("osis_actor_0").await?;
println!("Actor state: {:?}, PID: {}", status.state, status.pid);
```
### 2. Health Monitoring
The system automatically monitors worker health:
The system automatically monitors actor health:
- Tracks last job execution time for each worker
- Sends ping jobs to workers idle for 10+ minutes
- Restarts workers that fail ping checks 3 times
- Updates job times when workers receive tasks
- Tracks last job execution time for each actor
- Sends ping jobs to actors idle for 10+ minutes
- Restarts actors that fail ping checks 3 times
- Updates job times when actors receive tasks
```rust
// Manual health check
lifecycle_manager.monitor_worker_health().await?;
lifecycle_manager.monitor_actor_health().await?;
// Update job time (called automatically by supervisor)
lifecycle_manager.update_worker_job_time("osis_worker_0");
lifecycle_manager.update_actor_job_time("osis_actor_0");
// Start continuous health monitoring
lifecycle_manager.start_health_monitoring().await; // Runs forever
@@ -130,26 +130,26 @@ lifecycle_manager.start_health_monitoring().await; // Runs forever
### 3. Dynamic Scaling
Scale workers up or down based on demand:
Scale actors up or down based on demand:
```rust
// Scale OSIS workers to 5 instances
lifecycle_manager.scale_workers(&ScriptType::OSIS, 5).await?;
// Scale OSIS actors to 5 instances
lifecycle_manager.scale_actors(&ScriptType::OSIS, 5).await?;
// Scale down SAL workers to 1 instance
lifecycle_manager.scale_workers(&ScriptType::SAL, 1).await?;
// Scale down SAL actors to 1 instance
lifecycle_manager.scale_actors(&ScriptType::SAL, 1).await?;
// Check current running count
let count = lifecycle_manager.get_running_worker_count(&ScriptType::V).await;
println!("Running V workers: {}", count);
let count = lifecycle_manager.get_running_actor_count(&ScriptType::V).await;
println!("Running V actors: {}", count);
```
### 4. Service Dependencies
Workers can depend on other services:
Actors can depend on other services:
```rust
let config = WorkerConfig::new(name, binary, script_type)
let config = ActorConfig::new(name, binary, script_type)
.with_dependencies(vec![
"redis".to_string(),
"database".to_string(),
@@ -157,25 +157,25 @@ let config = WorkerConfig::new(name, binary, script_type)
]);
```
Zinit ensures dependencies start before the worker.
Zinit ensures dependencies start before the actor.
## Integration with Supervisor
The lifecycle manager integrates seamlessly with the supervisor:
```rust
use hero_supervisor::{Supervisor, WorkerLifecycleManager};
use hero_supervisor::{Supervisor, ActorLifecycleManager};
// Create supervisor and lifecycle manager
let supervisor = SupervisorBuilder::new().build()?;
let mut lifecycle_manager = WorkerLifecycleManagerBuilder::new(zinit_socket)
let mut lifecycle_manager = ActorLifecycleManagerBuilder::new(zinit_socket)
.with_supervisor(supervisor.clone())
.build();
// Start workers
lifecycle_manager.start_all_workers().await?;
// Start actors
lifecycle_manager.start_all_actors().await?;
// Create and execute jobs (supervisor automatically routes to workers)
// Create and execute jobs (supervisor automatically routes to actors)
let job = supervisor
.new_job()
.script_type(ScriptType::OSIS)
@@ -191,15 +191,15 @@ println!("Job result: {}", result);
The lifecycle manager automatically creates Zinit service configurations:
```yaml
# Generated service config for osis_worker_0
exec: "/usr/local/bin/osis_worker --redis-url redis://localhost:6379 --worker-id osis_worker_0"
test: "/usr/local/bin/osis_worker --health-check"
# Generated service config for osis_actor_0
exec: "/usr/local/bin/osis_actor --redis-url redis://localhost:6379 --actor-id osis_actor_0"
test: "/usr/local/bin/osis_actor --health-check"
oneshot: false # Restart on exit
after:
- redis
env:
RUST_LOG: "info"
WORKER_TYPE: "osis"
ACTOR_TYPE: "osis"
```
## Error Handling
@@ -209,10 +209,10 @@ The system provides comprehensive error handling:
```rust
use hero_supervisor::SupervisorError;
match lifecycle_manager.start_worker(&config).await {
Ok(_) => println!("Worker started successfully"),
Err(SupervisorError::WorkerStartFailed(worker, reason)) => {
eprintln!("Failed to start {}: {}", worker, reason);
match lifecycle_manager.start_actor(&config).await {
Ok(_) => println!("Actor started successfully"),
Err(SupervisorError::ActorStartFailed(actor, reason)) => {
eprintln!("Failed to start {}: {}", actor, reason);
}
Err(e) => eprintln!("Other error: {}", e),
}
@@ -243,11 +243,11 @@ REDIS_URL=redis://localhost:6379 cargo run --example lifecycle_demo
redis-server
```
3. **Worker Binaries**: Compiled worker binaries for each script type
- `/usr/local/bin/osis_worker`
- `/usr/local/bin/sal_worker`
- `/usr/local/bin/v_worker`
- `/usr/local/bin/python_worker`
3. **Actor Binaries**: Compiled actor binaries for each script type
- `/usr/local/bin/osis_actor`
- `/usr/local/bin/sal_actor`
- `/usr/local/bin/v_actor`
- `/usr/local/bin/python_actor`
## Configuration Best Practices
@@ -267,15 +267,15 @@ REDIS_URL=redis://localhost:6379 cargo run --example lifecycle_demo
- Check socket permissions: `ls -la /var/run/zinit.sock`
- Verify socket path in configuration
2. **Worker Start Failed**
2. **Actor Start Failed**
- Check binary exists and is executable
- Verify dependencies are running
- Review Zinit logs: `zinit logs <service-name>`
3. **Health Check Failures**
- Implement proper health check endpoint in workers
- Implement proper health check endpoint in actors
- Verify health check command syntax
- Check worker responsiveness
- Check actor responsiveness
4. **Redis Connection Issues**
- Ensure Redis is running and accessible
@@ -289,10 +289,10 @@ REDIS_URL=redis://localhost:6379 cargo run --example lifecycle_demo
zinit list
# View service logs
zinit logs osis_worker_0
zinit logs osis_actor_0
# Check service status
zinit status osis_worker_0
zinit status osis_actor_0
# Monitor Redis queues
redis-cli keys "hero:job:*"
@@ -300,20 +300,20 @@ redis-cli keys "hero:job:*"
## Performance Considerations
- **Scaling**: Start with minimal workers and scale based on queue depth
- **Scaling**: Start with minimal actors and scale based on queue depth
- **Health Monitoring**: Adjust ping intervals based on workload patterns
- **Resource Usage**: Monitor CPU/memory usage of worker processes
- **Resource Usage**: Monitor CPU/memory usage of actor processes
- **Queue Depth**: Monitor Redis queue lengths for scaling decisions
## Security
- **Process Isolation**: Zinit provides process isolation
- **User Permissions**: Run workers with appropriate user permissions
- **User Permissions**: Run actors with appropriate user permissions
- **Network Security**: Secure Redis and Zinit socket access
- **Binary Validation**: Verify worker binary integrity before deployment
- **Binary Validation**: Verify actor binary integrity before deployment
## Future
- **Load Balancing**: Dynamic scaling of workers based on demand
- **Load Balancing**: Dynamic scaling of actors based on demand
- **Service Dependencies**: Proper startup ordering with dependency management

View File

@@ -1,60 +1,60 @@
# Hero Supervisor
The **Hero Supervisor** is responsible for supervising the lifecycle of workers and dispatching jobs to them via Redis queues.
The **Hero Supervisor** is responsible for supervising the lifecycle of actors and dispatching jobs to them via Redis queues.
## Overview
The system involves four primary actors:
1. **OSIS**: A worker that executes Rhai and HeroScript.
2. **SAL**: A worker that performs system abstraction layer functionalities using Rhai.
3. **V**: A worker that executes HeroScript in the V programming language.
4. **Python**: A worker that executes HeroScript in Python.
1. **OSIS**: A actor that executes Rhai and HeroScript.
2. **SAL**: A actor that performs system abstraction layer functionalities using Rhai.
3. **V**: A actor that executes HeroScript in the V programming language.
4. **Python**: A actor that executes HeroScript in Python.
The Supervisor utilizes **zinit** to start and monitor these workers, ensuring they are running correctly.
The Supervisor utilizes **zinit** to start and monitor these actors, ensuring they are running correctly.
### Key Features
- **Worker Lifecycle Supervision**: Oversee the lifecycle of workers, including starting, stopping, restarting, and load balancing based on job demand.
- **Job Supervision**: API for efficiently managing jobs dispatched to workers over Redis queues.
- **Actor Lifecycle Supervision**: Oversee the lifecycle of actors, including starting, stopping, restarting, and load balancing based on job demand.
- **Job Supervision**: API for efficiently managing jobs dispatched to actors over Redis queues.
## Worker Lifecycle Supervision
## Actor Lifecycle Supervision
The Supervisor oversees the lifecycle of the workers, ensuring they are operational and efficiently allocated. Load balancing is implemented to dynamically adjust the number of active workers based on job demand.
The Supervisor oversees the lifecycle of the actors, ensuring they are operational and efficiently allocated. Load balancing is implemented to dynamically adjust the number of active actors based on job demand.
Additionally, the Supervisor implements health monitoring for worker engines: if a worker engine does not receive a job within 10 minutes, the Supervisor sends a ping job. The engine must respond immediately; if it fails to do so, the Supervisor restarts the requested job engine.
Additionally, the Supervisor implements health monitoring for actor engines: if a actor engine does not receive a job within 10 minutes, the Supervisor sends a ping job. The engine must respond immediately; if it fails to do so, the Supervisor restarts the requested job engine.
### Prerequisites
**Important**: Before running any lifecycle examples or using worker management features, you must start the Zinit daemon:
**Important**: Before running any lifecycle examples or using actor management features, you must start the Zinit daemon:
```bash
# Start Zinit daemon (required for worker lifecycle management)
# Start Zinit daemon (required for actor lifecycle management)
sudo zinit init
# Or start Zinit with a custom socket path
sudo zinit --socket /var/run/zinit.sock init
```
**Note**: The Supervisor uses Zinit as the process manager for worker lifecycle operations. The default socket path is `/var/run/zinit.sock`, but you can configure a custom path using the `SupervisorBuilder::zinit_socket_path()` method.
**Note**: The Supervisor uses Zinit as the process manager for actor lifecycle operations. The default socket path is `/var/run/zinit.sock`, but you can configure a custom path using the `SupervisorBuilder::zinit_socket_path()` method.
**Troubleshooting**: If you get connection errors when running examples, ensure:
1. Zinit daemon is running (`zinit list` should work)
2. The socket path matches between Zinit and your Supervisor configuration
3. You have appropriate permissions to access the Zinit socket
### Supervisor API for Worker Lifecycle
### Supervisor API for Actor Lifecycle
The Supervisor provides the following methods for supervising the worker lifecycle:
The Supervisor provides the following methods for supervising the actor lifecycle:
- **`start_worker()`**: Initializes and starts a specified worker.
- **`stop_worker()`**: Gracefully stops a specified worker.
- **`restart_worker()`**: Restarts a specified worker to ensure it operates correctly.
- **`get_worker_status()`**: Checks the status of a specific worker.
- **`start_actor()`**: Initializes and starts a specified actor.
- **`stop_actor()`**: Gracefully stops a specified actor.
- **`restart_actor()`**: Restarts a specified actor to ensure it operates correctly.
- **`get_actor_status()`**: Checks the status of a specific actor.
## Job Supervision
Jobs are dispatched to workers through their designated Redis queues, and the Supervisor provides an API for comprehensive job supervision.
Jobs are dispatched to actors through their designated Redis queues, and the Supervisor provides an API for comprehensive job supervision.
### Supervisor API for Job Supervision
@@ -95,9 +95,9 @@ You can modify these in the example source code if your setup differs.
Jobs are managed within the `hero:` namespace in Redis:
- **`hero:job:{job_id}`**: Stores job parameters as a Redis hash.
- **`hero:work_queue:{worker_id}`**: Contains worker-specific job queues for dispatching jobs.
- **`hero:work_queue:{actor_id}`**: Contains actor-specific job queues for dispatching jobs.
- **`hero:reply:{job_id}`**: Dedicated queues for job results.
## Prerequisites
- A Redis server must be accessible to both the Supervisor and the workers.
- A Redis server must be accessible to both the Supervisor and the actors.

View File

@@ -0,0 +1,117 @@
# Supervisor CLI
Interactive command-line interface for the Hero Supervisor that allows you to dispatch jobs to actors and manage the job lifecycle.
## Features
- **Interactive Menu**: Easy-to-use menu system for all supervisor operations
- **Job Management**: Create, run, monitor, and manage jobs
- **OSIS Actor Integration**: Dispatch Rhai scripts to the OSIS actor
- **Real-time Results**: Get immediate feedback from job execution
- **Colorized Output**: Clear visual feedback with colored status indicators
## Usage
### 1. Build the OSIS Actor
First, ensure the OSIS actor is built:
```bash
cd /Users/timurgordon/code/git.ourworld.tf/herocode/actor_osis
cargo build
```
### 2. Configure the Supervisor
Create or use the example configuration file at `examples/cli_config.toml`:
```toml
[global]
redis_url = "redis://127.0.0.1/"
[actors]
osis_actor = "/Users/timurgordon/code/git.ourworld.tf/herocode/actor_osis/target/debug/actor_osis"
```
### 3. Run the CLI
```bash
cd /Users/timurgordon/code/git.ourworld.tf/herocode/baobab/core/supervisor
cargo run --bin supervisor-cli -- --config examples/cli_config.toml
```
Or with verbose logging:
```bash
cargo run --bin supervisor-cli -- --config examples/cli_config.toml --verbose
```
## Available Commands
1. **list_jobs** - List all jobs in the system
2. **run_job** - Create and run a new job interactively
3. **get_job_status** - Get status of a specific job
4. **get_job_output** - Get output of a completed job
5. **get_job_logs** - Get logs for a specific job
6. **stop_job** - Stop a running job
7. **delete_job** - Delete a specific job
8. **clear_all_jobs** - Clear all jobs from the system
9. **quit** - Exit the CLI
## Example Workflow
1. Start the CLI with your configuration
2. Select option `2` (run_job)
3. Enter job details:
- **Caller**: Your name or identifier
- **Context**: Description of what the job does
- **Script**: Rhai script to execute (end with empty line)
4. The job is automatically dispatched to the OSIS actor
5. View the real-time result
### Example Rhai Script
```rhai
// Simple calculation
let result = 10 + 20 * 3;
print("Calculation result: " + result);
result
```
```rhai
// Working with strings
let message = "Hello from OSIS Actor!";
print(message);
message.to_upper()
```
## Job Status Colors
- **Created** - Cyan
- **Dispatched** - Blue
- **Started** - Yellow
- **Finished** - Green
- **Error** - Red
## Prerequisites
- Redis server running on localhost:6379 (or configured URL)
- OSIS actor binary built and accessible
- Proper permissions to start/stop processes via Zinit
## Troubleshooting
### Actor Not Starting
- Verify the OSIS actor binary path in the TOML config
- Check that the binary exists and is executable
- Ensure Redis is running and accessible
### Connection Issues
- Verify Redis URL in configuration
- Check network connectivity to Redis server
- Ensure no firewall blocking connections
### Job Execution Failures
- Check job logs using `get_job_logs` command
- Verify Rhai script syntax
- Check actor logs for detailed error information

View File

@@ -0,0 +1,178 @@
# Supervisor Terminal UI (TUI)
A modern, interactive Terminal User Interface for the Hero Supervisor that provides intuitive job management with real-time updates and visual navigation.
## Features
### 🎯 **Intuitive Interface**
- **Split-pane Layout**: Job list on the left, details on the right
- **Real-time Updates**: Auto-refreshes every 2 seconds
- **Color-coded Status**: Visual job status indicators
- **Keyboard Navigation**: Vim-style and arrow key support
### 📋 **Job Management**
- **Create Jobs**: Interactive form with tab navigation
- **Monitor Jobs**: Real-time status updates with color coding
- **View Details**: Detailed job information and output
- **View Logs**: Access job execution logs
- **Stop/Delete**: Job lifecycle management
- **Bulk Operations**: Clear all jobs with confirmation
### 🎨 **Visual Design**
- **Status Colors**:
- 🔵 **Blue**: Dispatched
- 🟡 **Yellow**: Started
- 🟢 **Green**: Finished
- 🔴 **Red**: Error
- 🟣 **Magenta**: Waiting for Prerequisites
- **Highlighted Selection**: Clear visual feedback
- **Popup Messages**: Status and error notifications
- **Confirmation Dialogs**: Safe bulk operations
## Usage
### 1. Start the TUI
```bash
cd /Users/timurgordon/code/git.ourworld.tf/herocode/baobab/core/supervisor
cargo run --bin supervisor-tui -- --config examples/cli_config.toml
```
### 2. Navigation
#### Main View
- **↑/↓ or j/k**: Navigate job list
- **Enter/Space**: View job details
- **n/c**: Create new job
- **r**: Manual refresh
- **d**: Delete selected job (with confirmation)
- **s**: Stop selected job
- **C**: Clear all jobs (with confirmation)
- **q**: Quit application
#### Job Creation Form
- **Tab**: Next field
- **Shift+Tab**: Previous field
- **Enter**: Next field (or newline in script field)
- **F5**: Submit job
- **Esc**: Cancel and return to main view
#### Job Details/Logs View
- **Esc/q**: Return to main view
- **l**: Switch to logs view
- **d**: Switch to details view
## Interface Layout
```
┌─────────────────────────────────────────────────────────────┐
│ Hero Supervisor TUI - Job Management │
├─────────────────────┬───────────────────────────────────────┤
│ Jobs │ Job Details │
│ │ │
│ >> 1a2b3c4d - ✅ Fi │ Job ID: 1a2b3c4d5e6f7g8h │
│ 2b3c4d5e - 🟡 St │ Status: Finished │
│ 3c4d5e6f - 🔴 Er │ │
│ 4d5e6f7g - 🔵 Di │ Output: │
│ │ Calculation result: 70 │
│ │ 70 │
├─────────────────────┴───────────────────────────────────────┤
│ q: Quit | n: New Job | ↑↓: Navigate | Enter: Details │
└─────────────────────────────────────────────────────────────┘
```
## Job Creation Workflow
1. **Press 'n'** to create a new job
2. **Fill in the form**:
- **Caller**: Your name or identifier
- **Context**: Job description
- **Script**: Rhai script (supports multi-line)
3. **Press F5** to submit
4. **Watch real-time execution** in the main view
### Example Rhai Scripts
```rhai
// Simple calculation
let result = 10 + 20 * 3;
print("Calculation result: " + result);
result
```
```rhai
// String manipulation
let message = "Hello from OSIS Actor!";
print(message);
message.to_upper()
```
```rhai
// Loop example
let sum = 0;
for i in 1..=10 {
sum += i;
}
print("Sum of 1-10: " + sum);
sum
```
## Key Improvements over CLI
### ✅ **Better UX**
- **Visual Navigation**: No need to remember numbers
- **Real-time Updates**: See job progress immediately
- **Split-pane Design**: View list and details simultaneously
- **Form Validation**: Clear error messages
### ✅ **Enhanced Productivity**
- **Auto-refresh**: Always up-to-date information
- **Keyboard Shortcuts**: Fast navigation and actions
- **Confirmation Dialogs**: Prevent accidental operations
- **Multi-line Script Input**: Better script editing
### ✅ **Professional Interface**
- **Color-coded Status**: Quick visual assessment
- **Consistent Layout**: Predictable interface elements
- **Popup Notifications**: Non-intrusive feedback
- **Graceful Error Handling**: User-friendly error messages
## Prerequisites
- Redis server running (default: localhost:6379)
- OSIS actor binary built and configured
- Terminal with color support
- Minimum terminal size: 80x24
## Troubleshooting
### Display Issues
- Ensure terminal supports colors and Unicode
- Resize terminal if layout appears broken
- Use a modern terminal emulator (iTerm2, Alacritty, etc.)
### Performance
- TUI auto-refreshes every 2 seconds
- Large job lists may impact performance
- Use 'r' for manual refresh if needed
### Navigation Issues
- Use arrow keys if vim keys (j/k) don't work
- Ensure terminal is in focus
- Try Esc to reset state if stuck
## Advanced Features
### Bulk Operations
- **Clear All Jobs**: Press 'C' with confirmation
- **Safe Deletion**: Confirmation required for destructive operations
### Real-time Monitoring
- **Auto-refresh**: Updates every 2 seconds
- **Status Tracking**: Watch job progression
- **Immediate Feedback**: See results as they complete
### Multi-line Scripts
- **Rich Text Input**: Full script editing in TUI
- **Syntax Awareness**: Better than single-line CLI input
- **Preview**: See script before submission

View File

@@ -0,0 +1,398 @@
use clap::Parser;
use colored::*;
use hero_supervisor::{Supervisor, SupervisorBuilder, SupervisorError, Job, JobStatus, ScriptType};
use log::{error, info};
use std::io::{self, Write};
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use tokio::time::sleep;
#[derive(Parser)]
#[command(name = "supervisor-cli")]
#[command(about = "Interactive CLI for Hero Supervisor - Dispatch jobs to actors")]
struct Args {
/// Path to TOML configuration file
#[arg(short, long)]
config: PathBuf,
/// Enable verbose logging
#[arg(short, long)]
verbose: bool,
}
#[derive(Debug, Clone)]
enum CliCommand {
ListJobs,
RunJob,
GetJobStatus,
GetJobOutput,
GetJobLogs,
StopJob,
DeleteJob,
ClearAllJobs,
Quit,
}
impl CliCommand {
fn all_commands() -> Vec<(CliCommand, &'static str, &'static str)> {
vec![
(CliCommand::ListJobs, "list_jobs", "List all jobs in the system"),
(CliCommand::RunJob, "run_job", "Create and run a new job"),
(CliCommand::GetJobStatus, "get_job_status", "Get status of a specific job"),
(CliCommand::GetJobOutput, "get_job_output", "Get output of a completed job"),
(CliCommand::GetJobLogs, "get_job_logs", "Get logs for a specific job"),
(CliCommand::StopJob, "stop_job", "Stop a running job"),
(CliCommand::DeleteJob, "delete_job", "Delete a specific job"),
(CliCommand::ClearAllJobs, "clear_all_jobs", "Clear all jobs from the system"),
(CliCommand::Quit, "quit", "Exit the CLI"),
]
}
fn from_index(index: usize) -> Option<CliCommand> {
Self::all_commands().get(index).map(|(cmd, _, _)| cmd.clone())
}
}
struct SupervisorCli {
supervisor: Arc<Supervisor>,
}
impl SupervisorCli {
fn new(supervisor: Arc<Supervisor>) -> Self {
Self { supervisor }
}
async fn run(&self) -> Result<(), SupervisorError> {
println!("{}", "=== Hero Supervisor CLI ===".bright_blue().bold());
println!("{}", "Interactive job management interface".cyan());
println!();
loop {
self.display_menu();
match self.get_user_choice().await {
Some(command) => {
match command {
CliCommand::Quit => {
println!("{}", "Goodbye!".bright_green());
break;
}
_ => {
if let Err(e) = self.execute_command(command).await {
eprintln!("{} {}", "Error:".bright_red(), e);
}
}
}
}
None => {
println!("{}", "Invalid selection. Please try again.".yellow());
}
}
println!();
}
Ok(())
}
fn display_menu(&self) {
println!("{}", "Available Commands:".bright_yellow().bold());
for (index, (_, name, description)) in CliCommand::all_commands().iter().enumerate() {
println!(" {}. {} - {}",
(index + 1).to_string().bright_white().bold(),
name.bright_cyan(),
description
);
}
print!("\n{} ", "Select a command (1-9):".bright_white());
io::stdout().flush().unwrap();
}
async fn get_user_choice(&self) -> Option<CliCommand> {
let mut input = String::new();
if io::stdin().read_line(&mut input).is_ok() {
if let Ok(choice) = input.trim().parse::<usize>() {
if choice > 0 {
return CliCommand::from_index(choice - 1);
}
}
}
None
}
async fn execute_command(&self, command: CliCommand) -> Result<(), SupervisorError> {
match command {
CliCommand::ListJobs => self.list_jobs().await,
CliCommand::RunJob => self.run_job().await,
CliCommand::GetJobStatus => self.get_job_status().await,
CliCommand::GetJobOutput => self.get_job_output().await,
CliCommand::GetJobLogs => self.get_job_logs().await,
CliCommand::StopJob => self.stop_job().await,
CliCommand::DeleteJob => self.delete_job().await,
CliCommand::ClearAllJobs => self.clear_all_jobs().await,
CliCommand::Quit => Ok(()),
}
}
async fn list_jobs(&self) -> Result<(), SupervisorError> {
println!("{}", "Listing all jobs...".bright_blue());
let jobs = self.supervisor.list_jobs().await?;
if jobs.is_empty() {
println!("{}", "No jobs found.".yellow());
} else {
println!("{} jobs found:", jobs.len().to_string().bright_white().bold());
for job_id in jobs {
let status = self.supervisor.get_job_status(&job_id).await?;
let status_color = match status {
JobStatus::Dispatched => "blue",
JobStatus::Started => "yellow",
JobStatus::Finished => "green",
JobStatus::Error => "red",
JobStatus::WaitingForPrerequisites => "magenta",
};
println!(" {} - {}",
job_id.bright_white(),
format!("{:?}", status).color(status_color)
);
}
}
Ok(())
}
async fn run_job(&self) -> Result<(), SupervisorError> {
println!("{}", "Creating a new job...".bright_blue());
// Get caller
print!("Enter caller name: ");
io::stdout().flush().unwrap();
let mut caller = String::new();
io::stdin().read_line(&mut caller).unwrap();
let caller = caller.trim().to_string();
// Get context
print!("Enter job context: ");
io::stdout().flush().unwrap();
let mut context = String::new();
io::stdin().read_line(&mut context).unwrap();
let context = context.trim().to_string();
// Get script
println!("Enter Rhai script (end with empty line):");
let mut script_lines = Vec::new();
loop {
let mut line = String::new();
io::stdin().read_line(&mut line).unwrap();
let line = line.trim_end_matches('\n');
if line.is_empty() {
break;
}
script_lines.push(line.to_string());
}
let script = script_lines.join("\n");
if script.is_empty() {
println!("{}", "Script cannot be empty!".bright_red());
return Ok(());
}
// For now, default to OSIS actor (ScriptType::OSIS)
let script_type = ScriptType::OSIS;
// Create the job
let job = Job::new(caller, context, script, script_type);
println!("{} Job ID: {}",
"Created job with".bright_green(),
job.id.bright_white().bold()
);
// Run the job and await result
println!("{}", "Dispatching job and waiting for result...".bright_blue());
match self.supervisor.run_job_and_await_result(&job).await {
Ok(result) => {
println!("{}", "Job completed successfully!".bright_green().bold());
println!("{} {}", "Result:".bright_yellow(), result.bright_white());
}
Err(e) => {
println!("{} {}", "Job failed:".bright_red().bold(), e);
}
}
Ok(())
}
async fn get_job_status(&self) -> Result<(), SupervisorError> {
let job_id = self.prompt_for_job_id("Enter job ID to check status: ")?;
let status = self.supervisor.get_job_status(&job_id).await?;
let status_color = match status {
JobStatus::Dispatched => "blue",
JobStatus::Started => "yellow",
JobStatus::Finished => "green",
JobStatus::Error => "red",
JobStatus::WaitingForPrerequisites => "magenta",
};
println!("{} {} - {}",
"Job".bright_white(),
job_id.bright_white().bold(),
format!("{:?}", status).color(status_color).bold()
);
Ok(())
}
async fn get_job_output(&self) -> Result<(), SupervisorError> {
let job_id = self.prompt_for_job_id("Enter job ID to get output: ")?;
match self.supervisor.get_job_output(&job_id).await? {
Some(output) => {
println!("{}", "Job Output:".bright_yellow().bold());
println!("{}", output.bright_white());
}
None => {
println!("{}", "No output available for this job.".yellow());
}
}
Ok(())
}
async fn get_job_logs(&self) -> Result<(), SupervisorError> {
let job_id = self.prompt_for_job_id("Enter job ID to get logs: ")?;
match self.supervisor.get_job_logs(&job_id).await? {
Some(logs) => {
println!("{}", "Job Logs:".bright_yellow().bold());
println!("{}", logs.bright_white());
}
None => {
println!("{}", "No logs available for this job.".yellow());
}
}
Ok(())
}
async fn stop_job(&self) -> Result<(), SupervisorError> {
let job_id = self.prompt_for_job_id("Enter job ID to stop: ")?;
self.supervisor.stop_job(&job_id).await?;
println!("{} {}",
"Stop signal sent for job".bright_green(),
job_id.bright_white().bold()
);
Ok(())
}
async fn delete_job(&self) -> Result<(), SupervisorError> {
let job_id = self.prompt_for_job_id("Enter job ID to delete: ")?;
self.supervisor.delete_job(&job_id).await?;
println!("{} {}",
"Deleted job".bright_green(),
job_id.bright_white().bold()
);
Ok(())
}
async fn clear_all_jobs(&self) -> Result<(), SupervisorError> {
print!("Are you sure you want to clear ALL jobs? (y/N): ");
io::stdout().flush().unwrap();
let mut confirmation = String::new();
io::stdin().read_line(&mut confirmation).unwrap();
if confirmation.trim().to_lowercase() == "y" {
let count = self.supervisor.clear_all_jobs().await?;
println!("{} {} jobs",
"Cleared".bright_green().bold(),
count.to_string().bright_white().bold()
);
} else {
println!("{}", "Operation cancelled.".yellow());
}
Ok(())
}
fn prompt_for_job_id(&self, prompt: &str) -> Result<String, SupervisorError> {
print!("{}", prompt);
io::stdout().flush().unwrap();
let mut job_id = String::new();
io::stdin().read_line(&mut job_id).unwrap();
let job_id = job_id.trim().to_string();
if job_id.is_empty() {
return Err(SupervisorError::ConfigError("Job ID cannot be empty".to_string()));
}
Ok(job_id)
}
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let args = Args::parse();
// Setup logging
if args.verbose {
env_logger::Builder::from_default_env()
.filter_level(log::LevelFilter::Debug)
.init();
} else {
env_logger::Builder::from_default_env()
.filter_level(log::LevelFilter::Info)
.init();
}
info!("Starting Supervisor CLI with config: {:?}", args.config);
// Build supervisor from TOML config
let supervisor = Arc::new(
SupervisorBuilder::from_toml(&args.config)?
.build().await?
);
println!("{}", "Starting actors...".bright_blue());
// Start the actors
supervisor.start_actors().await?;
// Give actors time to start up
sleep(Duration::from_secs(2)).await;
println!("{}", "Actors started successfully!".bright_green());
println!();
// Create and run the CLI
let cli = SupervisorCli::new(supervisor.clone());
// Setup cleanup on exit
let supervisor_cleanup = supervisor.clone();
tokio::spawn(async move {
tokio::signal::ctrl_c().await.expect("Failed to listen for ctrl+c");
println!("\n{}", "Shutting down...".bright_yellow());
if let Err(e) = supervisor_cleanup.cleanup_and_shutdown().await {
eprintln!("Error during cleanup: {}", e);
}
std::process::exit(0);
});
// Run the interactive CLI
cli.run().await?;
// Cleanup on normal exit
supervisor.cleanup_and_shutdown().await?;
Ok(())
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,10 +1,10 @@
# Hero Supervisor Protocol
This document describes the Redis-based protocol used by the Hero Supervisor for job management and worker communication.
This document describes the Redis-based protocol used by the Hero Supervisor for job management and actor communication.
## Overview
The Hero Supervisor uses Redis as a message broker and data store for managing distributed job execution. Jobs are stored as Redis hashes, and communication with workers happens through Redis lists (queues).
The Hero Supervisor uses Redis as a message broker and data store for managing distributed job execution. Jobs are stored as Redis hashes, and communication with actors happens through Redis lists (queues).
## Redis Namespace
@@ -22,7 +22,7 @@ hero:job:{job_id}
**Job Hash Fields:**
- `id`: Unique job identifier (UUID v4)
- `caller_id`: Identifier of the client that created the job
- `worker_id`: Target worker identifier
- `actor_id`: Target actor identifier
- `context_id`: Execution context identifier
- `script`: Script content to execute (Rhai or HeroScript)
- `timeout`: Execution timeout in seconds
@@ -35,8 +35,8 @@ hero:job:{job_id}
- `env_vars`: Environment variables as JSON object (optional)
- `prerequisites`: JSON array of job IDs that must complete before this job (optional)
- `dependents`: JSON array of job IDs that depend on this job completing (optional)
- `output`: Job execution result (set by worker)
- `error`: Error message if job failed (set by worker)
- `output`: Job execution result (set by actor)
- `error`: Error message if job failed (set by actor)
- `dependencies`: List of job IDs that this job depends on
### Job Dependencies
@@ -45,30 +45,58 @@ Jobs can have dependencies on other jobs, which are stored in the `dependencies`
### Work Queues
Jobs are queued for execution using Redis lists:
Jobs are queued for execution using Redis lists with the following naming convention:
```
hero:work_queue:{worker_id}
hero:job:actor_queue:{script_type_suffix}
```
Workers listen on their specific queue using `BLPOP` for job IDs to process.
Where `{script_type_suffix}` corresponds to the script type:
- `osis` for OSIS actors (Rhai/HeroScript execution)
- `sal` for SAL actors (System Abstraction Layer)
- `v` for V actors (V language execution)
- `python` for Python actors
**Examples:**
- OSIS actor queue: `hero:job:actor_queue:osis`
- SAL actor queue: `hero:job:actor_queue:sal`
- V actor queue: `hero:job:actor_queue:v`
- Python actor queue: `hero:job:actor_queue:python`
Actors listen on their specific queue using `BLPOP` for job IDs to process.
**Important:** Actors must use the same queue naming convention in their `actor_id()` method to ensure proper job dispatch. The actor should return `"actor_queue:{script_type_suffix}"` as its actor ID.
### Stop Queues
Job stop requests are sent through dedicated stop queues:
```
hero:stop_queue:{worker_id}
hero:stop_queue:{actor_id}
```
Workers monitor these queues to receive stop requests for running jobs.
Actors monitor these queues to receive stop requests for running jobs.
### Reply Queues
For synchronous job execution, dedicated reply queues are used:
```
hero:reply:{job_id}
```
Reply queues are used for responses to specific requests:
Workers send results to these queues when jobs complete.
- `hero:reply:{request_id}`: Response to a specific request
### Result and Error Queues
When actors process jobs, they store results and errors in two places:
1. **Job Hash Storage**: Results are stored in the job hash fields:
- `hero:job:{job_id}` hash with `output` field for results
- `hero:job:{job_id}` hash with `error` field for errors
2. **Dedicated Queues**: Results and errors are also pushed to dedicated queues for asynchronous retrieval:
- `hero:job:{job_id}:result`: Queue containing job result (use `LPOP` to retrieve)
- `hero:job:{job_id}:error`: Queue containing job error (use `LPOP` to retrieve)
This dual storage approach allows clients to:
- Access results/errors directly from job hash for immediate retrieval
- Listen on result/error queues for asynchronous notification of job completion
- Use `BLPOP` on result/error queues for blocking waits on job completion
## Job Lifecycle
@@ -79,20 +107,20 @@ Client -> Redis: HSET hero:job:{job_id} {job_fields}
### 2. Job Submission
```
Client -> Redis: LPUSH hero:work_queue:{worker_id} {job_id}
Client -> Redis: LPUSH hero:work_queue:{actor_id} {job_id}
```
### 3. Job Processing
```
Worker -> Redis: BLPOP hero:work_queue:{worker_id}
Worker -> Redis: HSET hero:job:{job_id} status "started"
Worker: Execute script
Worker -> Redis: HSET hero:job:{job_id} status "finished" output "{result}"
Actor -> Redis: BLPOP hero:work_queue:{actor_id}
Actor -> Redis: HSET hero:job:{job_id} status "started"
Actor: Execute script
Actor -> Redis: HSET hero:job:{job_id} status "finished" output "{result}"
```
### 4. Job Completion (Async)
```
Worker -> Redis: LPUSH hero:reply:{job_id} {result}
Actor -> Redis: LPUSH hero:reply:{job_id} {result}
```
## API Operations
@@ -110,7 +138,7 @@ supervisor.list_jobs() -> Vec<String>
supervisor.stop_job(job_id) -> Result<(), SupervisorError>
```
**Redis Operations:**
- `LPUSH hero:stop_queue:{worker_id} {job_id}` - Send stop request
- `LPUSH hero:stop_queue:{actor_id} {job_id}` - Send stop request
### Get Job Status
```rust
@@ -131,20 +159,20 @@ supervisor.get_job_logs(job_id) -> Result<Option<String>, SupervisorError>
### Run Job and Await Result
```rust
supervisor.run_job_and_await_result(job, worker_id) -> Result<String, SupervisorError>
supervisor.run_job_and_await_result(job, actor_id) -> Result<String, SupervisorError>
```
**Redis Operations:**
1. `HSET hero:job:{job_id} {job_fields}` - Store job
2. `LPUSH hero:work_queue:{worker_id} {job_id}` - Submit job
2. `LPUSH hero:work_queue:{actor_id} {job_id}` - Submit job
3. `BLPOP hero:reply:{job_id} {timeout}` - Wait for result
## Worker Protocol
## Actor Protocol
### Job Processing Loop
```rust
loop {
// 1. Wait for job
job_id = BLPOP hero:work_queue:{worker_id}
job_id = BLPOP hero:work_queue:{actor_id}
// 2. Get job details
job_data = HGETALL hero:job:{job_id}
@@ -153,8 +181,8 @@ loop {
HSET hero:job:{job_id} status "started"
// 4. Check for stop requests
if LLEN hero:stop_queue:{worker_id} > 0 {
stop_job_id = LPOP hero:stop_queue:{worker_id}
if LLEN hero:stop_queue:{actor_id} > 0 {
stop_job_id = LPOP hero:stop_queue:{actor_id}
if stop_job_id == job_id {
HSET hero:job:{job_id} status "error" error "stopped"
continue
@@ -175,15 +203,15 @@ loop {
```
### Stop Request Handling
Workers should periodically check the stop queue during long-running jobs:
Actors should periodically check the stop queue during long-running jobs:
```rust
if LLEN hero:stop_queue:{worker_id} > 0 {
stop_requests = LRANGE hero:stop_queue:{worker_id} 0 -1
if LLEN hero:stop_queue:{actor_id} > 0 {
stop_requests = LRANGE hero:stop_queue:{actor_id} 0 -1
if stop_requests.contains(current_job_id) {
// Stop current job execution
HSET hero:job:{current_job_id} status "error" error "stopped_by_request"
// Remove stop request
LREM hero:stop_queue:{worker_id} 1 current_job_id
LREM hero:stop_queue:{actor_id} 1 current_job_id
return
}
}
@@ -193,17 +221,17 @@ if LLEN hero:stop_queue:{worker_id} > 0 {
### Job Timeouts
- Client sets timeout when creating job
- Worker should respect timeout and stop execution
- Actor should respect timeout and stop execution
- If timeout exceeded: `HSET hero:job:{job_id} status "error" error "timeout"`
### Worker Failures
- If worker crashes, job remains in "started" status
### Actor Failures
- If actor crashes, job remains in "started" status
- Monitoring systems can detect stale jobs and retry
- Jobs can be requeued: `LPUSH hero:work_queue:{worker_id} {job_id}`
- Jobs can be requeued: `LPUSH hero:work_queue:{actor_id} {job_id}`
### Redis Connection Issues
- Clients should implement retry logic with exponential backoff
- Workers should reconnect and resume processing
- Actors should reconnect and resume processing
- Use Redis persistence to survive Redis restarts
## Monitoring and Observability
@@ -211,10 +239,10 @@ if LLEN hero:stop_queue:{worker_id} > 0 {
### Queue Monitoring
```bash
# Check work queue length
LLEN hero:work_queue:{worker_id}
LLEN hero:work_queue:{actor_id}
# Check stop queue length
LLEN hero:stop_queue:{worker_id}
LLEN hero:stop_queue:{actor_id}
# List all jobs
KEYS hero:job:*
@@ -228,7 +256,7 @@ HGETALL hero:job:{job_id}
- Jobs completed per second
- Average job execution time
- Queue depths
- Worker availability
- Actor availability
- Error rates by job type
## Security Considerations
@@ -237,7 +265,7 @@ HGETALL hero:job:{job_id}
- Use Redis AUTH for authentication
- Enable TLS for Redis connections
- Restrict Redis network access
- Use Redis ACLs to limit worker permissions
- Use Redis ACLs to limit actor permissions
### Job Security
- Validate script content before execution
@@ -265,8 +293,8 @@ HGETALL hero:job:{job_id}
- Batch similar jobs when possible
- Implement job prioritization if needed
### Worker Optimization
- Pool worker connections to Redis
### Actor Optimization
- Pool actor connections to Redis
- Use async I/O for Redis operations
- Implement graceful shutdown handling
- Monitor worker resource usage
- Monitor actor resource usage

View File

@@ -1,6 +1,6 @@
# Hero Supervisor CLI Example
This example demonstrates how to use the `hive-supervisor` CLI tool for managing workers and jobs in the Hero ecosystem.
This example demonstrates how to use the `hive-supervisor` CLI tool for managing actors and jobs in the Hero ecosystem.
## Prerequisites
@@ -19,20 +19,20 @@ This example demonstrates how to use the `hive-supervisor` CLI tool for managing
# Follow Zinit installation instructions for your platform
```
3. **Worker Binaries**: The configuration references worker binaries that need to be available:
- `/usr/local/bin/osis_worker`
- `/usr/local/bin/sal_worker`
- `/usr/local/bin/v_worker`
- `/usr/local/bin/python_worker`
3. **Actor Binaries**: The configuration references actor binaries that need to be available:
- `/usr/local/bin/osis_actor`
- `/usr/local/bin/sal_actor`
- `/usr/local/bin/v_actor`
- `/usr/local/bin/python_actor`
For testing purposes, you can create mock worker binaries or update the paths in `config.toml` to point to existing binaries.
For testing purposes, you can create mock actor binaries or update the paths in `config.toml` to point to existing binaries.
## Configuration
The `config.toml` file contains the supervisor configuration:
- **Global settings**: Redis URL and Zinit socket path
- **Worker configurations**: Binary paths and environment variables for each worker type
- **Actor configurations**: Binary paths and environment variables for each actor type
## Usage Examples
@@ -43,29 +43,29 @@ The `config.toml` file contains the supervisor configuration:
cargo build --bin hive-supervisor --release
```
### 2. Worker Management
### 2. Actor Management
```bash
# Show help
./target/release/hive-supervisor --config examples/cli/config.toml --help
# List all configured workers
./target/release/hive-supervisor --config examples/cli/config.toml workers list
# List all configured actors
./target/release/hive-supervisor --config examples/cli/config.toml actors list
# Start all workers
./target/release/hive-supervisor --config examples/cli/config.toml workers start
# Start all actors
./target/release/hive-supervisor --config examples/cli/config.toml actors start
# Start specific workers
./target/release/hive-supervisor --config examples/cli/config.toml workers start osis_worker sal_worker
# Start specific actors
./target/release/hive-supervisor --config examples/cli/config.toml actors start osis_actor sal_actor
# Check worker status
./target/release/hive-supervisor --config examples/cli/config.toml workers status
# Check actor status
./target/release/hive-supervisor --config examples/cli/config.toml actors status
# Stop all workers
./target/release/hive-supervisor --config examples/cli/config.toml workers stop
# Stop all actors
./target/release/hive-supervisor --config examples/cli/config.toml actors stop
# Restart specific worker
./target/release/hive-supervisor --config examples/cli/config.toml workers restart osis_worker
# Restart specific actor
./target/release/hive-supervisor --config examples/cli/config.toml actors restart osis_actor
```
### 3. Job Management
@@ -73,7 +73,7 @@ cargo build --bin hive-supervisor --release
```bash
# Create a job with inline script
./target/release/hive-supervisor --config examples/cli/config.toml jobs create \
--script 'print("Hello from OSIS worker!");' \
--script 'print("Hello from OSIS actor!");' \
--script-type osis \
--caller-id "user123" \
--context-id "session456"
@@ -118,18 +118,18 @@ cargo build --bin hive-supervisor --release
```bash
# Enable debug logging
./target/release/hive-supervisor --config examples/cli/config.toml -v workers status
./target/release/hive-supervisor --config examples/cli/config.toml -v actors status
# Enable trace logging
./target/release/hive-supervisor --config examples/cli/config.toml -vv workers status
./target/release/hive-supervisor --config examples/cli/config.toml -vv actors status
# Disable timestamps
./target/release/hive-supervisor --config examples/cli/config.toml --no-timestamp workers status
./target/release/hive-supervisor --config examples/cli/config.toml --no-timestamp actors status
```
## Sample Scripts
The `sample_scripts/` directory contains example scripts for different worker types:
The `sample_scripts/` directory contains example scripts for different actor types:
- `hello_osis.rhai` - Simple OSIS/HeroScript example
- `system_sal.rhai` - SAL system operation example
@@ -148,9 +148,9 @@ The `sample_scripts/` directory contains example scripts for different worker ty
- Verify Zinit is running and the socket path is correct
- Check permissions on the socket file
3. **Worker Binary Not Found**
3. **Actor Binary Not Found**
- Update binary paths in `config.toml` to match your system
- Ensure worker binaries are executable
- Ensure actor binaries are executable
4. **Permission Denied**
- Check file permissions on configuration and binary files
@@ -161,7 +161,7 @@ The `sample_scripts/` directory contains example scripts for different worker ty
Run with verbose logging to see detailed operation information:
```bash
RUST_LOG=debug ./target/release/hive-supervisor --config examples/cli/config.toml -vv workers status
RUST_LOG=debug ./target/release/hive-supervisor --config examples/cli/config.toml -vv actors status
```
## Configuration Customization
@@ -170,15 +170,15 @@ You can customize the configuration for your environment:
1. **Update Redis URL**: Change `redis_url` in the `[global]` section
2. **Update Zinit Socket**: Change `zinit_socket_path` for your Zinit installation
3. **Worker Paths**: Update binary paths in worker sections to match your setup
4. **Environment Variables**: Add or modify environment variables for each worker type
3. **Actor Paths**: Update binary paths in actor sections to match your setup
4. **Environment Variables**: Add or modify environment variables for each actor type
## Integration with Hero Ecosystem
This CLI integrates with the broader Hero ecosystem:
- **Job Queue**: Uses Redis for job queuing and status tracking
- **Process Management**: Uses Zinit for worker lifecycle management
- **Process Management**: Uses Zinit for actor lifecycle management
- **Script Execution**: Supports multiple script types (OSIS, SAL, V, Python)
- **Monitoring**: Provides real-time status and logging capabilities

View File

@@ -1,19 +1,19 @@
# Hero Supervisor CLI Configuration Example
# This configuration demonstrates how to set up the hive-supervisor CLI
# with different worker types for script execution.
# with different actor types for script execution.
[global]
# Redis connection URL for job queuing
redis_url = "redis://localhost:6379"
# OSIS Worker Configuration
# OSIS Actor Configuration
# Handles OSIS (HeroScript) execution
[osis_worker]
[osis_actor]
binary_path = "../../../target/debug/osis"
env_vars = { "RUST_LOG" = "info", "WORKER_TYPE" = "osis", "MAX_CONCURRENT_JOBS" = "5" }
env_vars = { "RUST_LOG" = "info", "ACTOR_TYPE" = "osis", "MAX_CONCURRENT_JOBS" = "5" }
# SAL Worker Configuration
# SAL Actor Configuration
# Handles System Abstraction Layer scripts
[sal_worker]
[sal_actor]
binary_path = "../../../target/debug/sal"
env_vars = { "RUST_LOG" = "info", "WORKER_TYPE" = "sal", "MAX_CONCURRENT_JOBS" = "3" }
env_vars = { "RUST_LOG" = "info", "ACTOR_TYPE" = "sal", "MAX_CONCURRENT_JOBS" = "3" }

View File

@@ -58,25 +58,25 @@ fi
echo -e "${BLUE}=== CLI Help and Information ===${NC}"
run_cli "Show main help" --help
echo -e "${BLUE}=== Worker Management Examples ===${NC}"
run_cli "List configured workers" workers list
run_cli "Show worker management help" workers --help
echo -e "${BLUE}=== Actor Management Examples ===${NC}"
run_cli "List configured actors" actors list
run_cli "Show actor management help" actors --help
# Note: These commands would require actual worker binaries and Zinit setup
echo -e "${YELLOW}Note: The following commands require actual worker binaries and Zinit setup${NC}"
# Note: These commands would require actual actor binaries and Zinit setup
echo -e "${YELLOW}Note: The following commands require actual actor binaries and Zinit setup${NC}"
echo -e "${YELLOW}They are shown for demonstration but may fail without proper setup${NC}"
echo
# Uncomment these if you have the proper setup
# run_cli "Check worker status" workers status
# run_cli "Start all workers" workers start
# run_cli "Check worker status after start" workers status
# run_cli "Check actor status" actors status
# run_cli "Start all actors" actors start
# run_cli "Check actor status after start" actors status
echo -e "${BLUE}=== Job Management Examples ===${NC}"
run_cli "Show job management help" jobs --help
# Create sample jobs (these will also require workers to be running)
echo -e "${YELLOW}Sample job creation commands (require running workers):${NC}"
# Create sample jobs (these will also require actors to be running)
echo -e "${YELLOW}Sample job creation commands (require running actors):${NC}"
echo
echo "# Create OSIS job with inline script:"
@@ -123,22 +123,22 @@ echo
echo -e "${BLUE}=== Verbose Logging Examples ===${NC}"
echo "# Debug logging:"
echo "$CLI_BINARY --config $CONFIG_FILE -v workers list"
echo "$CLI_BINARY --config $CONFIG_FILE -v actors list"
echo
echo "# Trace logging:"
echo "$CLI_BINARY --config $CONFIG_FILE -vv workers list"
echo "$CLI_BINARY --config $CONFIG_FILE -vv actors list"
echo
echo "# No timestamps:"
echo "$CLI_BINARY --config $CONFIG_FILE --no-timestamp workers list"
echo "$CLI_BINARY --config $CONFIG_FILE --no-timestamp actors list"
echo
echo -e "${GREEN}=== Example Runner Complete ===${NC}"
echo -e "${YELLOW}To run actual commands, ensure you have:${NC}"
echo "1. Redis server running on localhost:6379"
echo "2. Zinit process manager installed and configured"
echo "3. Worker binaries available at the paths specified in config.toml"
echo "3. Actor binaries available at the paths specified in config.toml"
echo
echo -e "${YELLOW}For testing without full setup, you can:${NC}"
echo "1. Update config.toml with paths to existing binaries"
echo "2. Use the CLI help commands and configuration validation"
echo "3. Test the REPL mode (requires workers to be running)"
echo "3. Test the REPL mode (requires actors to be running)"

View File

@@ -1,7 +1,7 @@
#!/usr/bin/env python3
"""
Sample Python script for demonstration
This script demonstrates Python worker functionality
This script demonstrates Python actor functionality
"""
import json
@@ -9,7 +9,7 @@ import datetime
from typing import List, Dict
def main():
print("=== Python Worker Demo ===")
print("=== Python Actor Demo ===")
print("Python data processing operations")
# Data structures

View File

@@ -1,8 +1,8 @@
// Sample OSIS/HeroScript for demonstration
// This script demonstrates basic OSIS worker functionality
// This script demonstrates basic OSIS actor functionality
print("=== OSIS Worker Demo ===");
print("Hello from the OSIS worker!");
print("=== OSIS Actor Demo ===");
print("Hello from the OSIS actor!");
// Basic variable operations
let name = "Hero";

View File

@@ -1,12 +1,12 @@
// Sample V language script for demonstration
// This script demonstrates V worker functionality
// This script demonstrates V actor functionality
module main
import math
fn main() {
println("=== V Worker Demo ===")
println("=== V Actor Demo ===")
println("V language mathematical operations")
// Basic arithmetic

View File

@@ -1,7 +1,7 @@
// Sample SAL (System Abstraction Layer) script for demonstration
// This script demonstrates system-level operations through SAL worker
// This script demonstrates system-level operations through SAL actor
print("=== SAL Worker Demo ===");
print("=== SAL Actor Demo ===");
print("System Abstraction Layer operations");
// System information gathering

View File

@@ -0,0 +1,20 @@
# Hero Supervisor CLI Configuration
# This configuration sets up the supervisor with an OSIS actor for job processing
[global]
redis_url = "redis://127.0.0.1/"
[actors]
# OSIS Actor configuration - handles Object Storage and Indexing System jobs
osis_actor = "/Users/timurgordon/code/git.ourworld.tf/herocode/actor_osis/target/debug/actor_osis"
# Optional: Other actors can be configured here
# sal_actor = "/path/to/sal_actor"
# v_actor = "/path/to/v_actor"
# python_actor = "/path/to/python_actor"
# Optional: WebSocket server configuration for remote API access
# [websocket]
# host = "127.0.0.1"
# port = 8443
# redis_url = "redis://127.0.0.1/"

View File

@@ -1,6 +1,6 @@
use hero_supervisor::{
Supervisor, SupervisorBuilder, WorkerConfig, WorkerLifecycleManager,
WorkerLifecycleManagerBuilder, ScriptType
Supervisor, SupervisorBuilder, ActorConfig, ActorLifecycleManager,
ActorLifecycleManagerBuilder, ScriptType
};
use log::{info, warn, error};
use std::collections::HashMap;
@@ -13,7 +13,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Initialize logging
env_logger::init();
info!("Starting Worker Lifecycle Management Demo");
info!("Starting Actor Lifecycle Management Demo");
// Configuration
let redis_url = "redis://localhost:6379";
@@ -25,154 +25,154 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.context_id("demo_context")
.build()?;
// Configure workers for different script types
let mut worker_configs = Vec::new();
// Configure actors for different script types
let mut actor_configs = Vec::new();
// OSIS workers (Rhai/HeroScript)
// OSIS actors (Rhai/HeroScript)
for i in 0..2 {
let config = WorkerConfig::new(
format!("osis_worker_{}", i),
PathBuf::from("/usr/local/bin/osis_worker"),
let config = ActorConfig::new(
format!("osis_actor_{}", i),
PathBuf::from("/usr/local/bin/osis_actor"),
ScriptType::OSIS,
)
.with_args(vec![
"--redis-url".to_string(),
redis_url.to_string(),
"--worker-id".to_string(),
format!("osis_worker_{}", i),
"--actor-id".to_string(),
format!("osis_actor_{}", i),
])
.with_env({
let mut env = HashMap::new();
env.insert("RUST_LOG".to_string(), "info".to_string());
env.insert("WORKER_TYPE".to_string(), "osis".to_string());
env.insert("ACTOR_TYPE".to_string(), "osis".to_string());
env
})
.with_health_check("/usr/local/bin/osis_worker --health-check".to_string())
.with_health_check("/usr/local/bin/osis_actor --health-check".to_string())
.with_dependencies(vec!["redis".to_string()]);
worker_configs.push(config);
actor_configs.push(config);
}
// SAL workers (System Abstraction Layer)
// SAL actors (System Abstraction Layer)
for i in 0..3 {
let config = WorkerConfig::new(
format!("sal_worker_{}", i),
PathBuf::from("/usr/local/bin/sal_worker"),
let config = ActorConfig::new(
format!("sal_actor_{}", i),
PathBuf::from("/usr/local/bin/sal_actor"),
ScriptType::SAL,
)
.with_args(vec![
"--redis-url".to_string(),
redis_url.to_string(),
"--worker-id".to_string(),
format!("sal_worker_{}", i),
"--actor-id".to_string(),
format!("sal_actor_{}", i),
])
.with_env({
let mut env = HashMap::new();
env.insert("RUST_LOG".to_string(), "info".to_string());
env.insert("WORKER_TYPE".to_string(), "sal".to_string());
env.insert("ACTOR_TYPE".to_string(), "sal".to_string());
env
})
.with_health_check("/usr/local/bin/sal_worker --health-check".to_string())
.with_health_check("/usr/local/bin/sal_actor --health-check".to_string())
.with_dependencies(vec!["redis".to_string()]);
worker_configs.push(config);
actor_configs.push(config);
}
// V workers (HeroScript in V language)
// V actors (HeroScript in V language)
for i in 0..2 {
let config = WorkerConfig::new(
format!("v_worker_{}", i),
PathBuf::from("/usr/local/bin/v_worker"),
let config = ActorConfig::new(
format!("v_actor_{}", i),
PathBuf::from("/usr/local/bin/v_actor"),
ScriptType::V,
)
.with_args(vec![
"--redis-url".to_string(),
redis_url.to_string(),
"--worker-id".to_string(),
format!("v_worker_{}", i),
"--actor-id".to_string(),
format!("v_actor_{}", i),
])
.with_env({
let mut env = HashMap::new();
env.insert("RUST_LOG".to_string(), "info".to_string());
env.insert("WORKER_TYPE".to_string(), "v".to_string());
env.insert("ACTOR_TYPE".to_string(), "v".to_string());
env
})
.with_health_check("/usr/local/bin/v_worker --health-check".to_string())
.with_health_check("/usr/local/bin/v_actor --health-check".to_string())
.with_dependencies(vec!["redis".to_string()]);
worker_configs.push(config);
actor_configs.push(config);
}
// Create lifecycle manager
let mut lifecycle_manager = WorkerLifecycleManagerBuilder::new(zinit_socket.to_string())
let mut lifecycle_manager = ActorLifecycleManagerBuilder::new(zinit_socket.to_string())
.with_supervisor(supervisor.clone());
// Add all worker configurations
for config in worker_configs {
lifecycle_manager = lifecycle_manager.add_worker(config);
// Add all actor configurations
for config in actor_configs {
lifecycle_manager = lifecycle_manager.add_actor(config);
}
let mut lifecycle_manager = lifecycle_manager.build();
// Demonstrate lifecycle operations
info!("=== Starting Worker Lifecycle Demo ===");
info!("=== Starting Actor Lifecycle Demo ===");
// 1. Start all workers
info!("1. Starting all workers...");
match lifecycle_manager.start_all_workers().await {
Ok(_) => info!("✅ All workers started successfully"),
// 1. Start all actors
info!("1. Starting all actors...");
match lifecycle_manager.start_all_actors().await {
Ok(_) => info!("✅ All actors started successfully"),
Err(e) => {
error!("❌ Failed to start workers: {}", e);
error!("❌ Failed to start actors: {}", e);
return Err(e.into());
}
}
// Wait for workers to initialize
// Wait for actors to initialize
sleep(Duration::from_secs(5)).await;
// 2. Check worker status
info!("2. Checking worker status...");
match lifecycle_manager.get_all_worker_status().await {
// 2. Check actor status
info!("2. Checking actor status...");
match lifecycle_manager.get_all_actor_status().await {
Ok(status_map) => {
for (worker_name, status) in status_map {
info!(" Worker '{}': State={:?}, PID={}", worker_name, status.state, status.pid);
for (actor_name, status) in status_map {
info!(" Actor '{}': State={:?}, PID={}", actor_name, status.state, status.pid);
}
}
Err(e) => warn!("Failed to get worker status: {}", e),
Err(e) => warn!("Failed to get actor status: {}", e),
}
// 3. Demonstrate scaling
info!("3. Demonstrating worker scaling...");
info!("3. Demonstrating actor scaling...");
// Scale up OSIS workers
info!(" Scaling up OSIS workers to 3...");
if let Err(e) = lifecycle_manager.scale_workers(&ScriptType::OSIS, 3).await {
warn!("Failed to scale OSIS workers: {}", e);
// Scale up OSIS actors
info!(" Scaling up OSIS actors to 3...");
if let Err(e) = lifecycle_manager.scale_actors(&ScriptType::OSIS, 3).await {
warn!("Failed to scale OSIS actors: {}", e);
}
sleep(Duration::from_secs(3)).await;
// Scale down SAL workers
info!(" Scaling down SAL workers to 1...");
if let Err(e) = lifecycle_manager.scale_workers(&ScriptType::SAL, 1).await {
warn!("Failed to scale SAL workers: {}", e);
// Scale down SAL actors
info!(" Scaling down SAL actors to 1...");
if let Err(e) = lifecycle_manager.scale_actors(&ScriptType::SAL, 1).await {
warn!("Failed to scale SAL actors: {}", e);
}
sleep(Duration::from_secs(3)).await;
// 4. Check running worker counts
info!("4. Checking running worker counts after scaling...");
// 4. Check running actor counts
info!("4. Checking running actor counts after scaling...");
for script_type in [ScriptType::OSIS, ScriptType::SAL, ScriptType::V] {
let count = lifecycle_manager.get_running_worker_count(&script_type).await;
info!(" {:?}: {} workers running", script_type, count);
let count = lifecycle_manager.get_running_actor_count(&script_type).await;
info!(" {:?}: {} actors running", script_type, count);
}
// 5. Demonstrate restart functionality
info!("5. Demonstrating worker restart...");
if let Err(e) = lifecycle_manager.restart_worker("osis_worker_0").await {
warn!("Failed to restart worker: {}", e);
info!("5. Demonstrating actor restart...");
if let Err(e) = lifecycle_manager.restart_actor("osis_actor_0").await {
warn!("Failed to restart actor: {}", e);
} else {
info!(" ✅ Successfully restarted osis_worker_0");
info!(" ✅ Successfully restarted osis_actor_0");
}
sleep(Duration::from_secs(3)).await;
@@ -180,12 +180,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// 6. Simulate job dispatch and health monitoring
info!("6. Simulating job dispatch and health monitoring...");
// Update job time for a worker (simulating job dispatch)
lifecycle_manager.update_worker_job_time("sal_worker_0");
info!(" Updated job time for sal_worker_0");
// Update job time for a actor (simulating job dispatch)
lifecycle_manager.update_actor_job_time("sal_actor_0");
info!(" Updated job time for sal_actor_0");
// Perform health monitoring check
if let Err(e) = lifecycle_manager.monitor_worker_health().await {
if let Err(e) = lifecycle_manager.monitor_actor_health().await {
warn!("Health monitoring failed: {}", e);
} else {
info!(" ✅ Health monitoring completed");
@@ -196,7 +196,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let test_job = supervisor
.new_job()
.script_type(ScriptType::OSIS)
.script_content("println!(\"Hello from worker!\");".to_string())
.script_content("println!(\"Hello from actor!\");".to_string())
.timeout(Duration::from_secs(30))
.build()?;
@@ -208,27 +208,27 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// 8. Demonstrate graceful shutdown
info!("8. Demonstrating graceful shutdown...");
// Stop specific workers
info!(" Stopping specific workers...");
for worker_name in ["osis_worker_1", "v_worker_0"] {
if let Err(e) = lifecycle_manager.stop_worker(worker_name).await {
warn!("Failed to stop worker {}: {}", worker_name, e);
// Stop specific actors
info!(" Stopping specific actors...");
for actor_name in ["osis_actor_1", "v_actor_0"] {
if let Err(e) = lifecycle_manager.stop_actor(actor_name).await {
warn!("Failed to stop actor {}: {}", actor_name, e);
} else {
info!(" ✅ Stopped worker: {}", worker_name);
info!(" ✅ Stopped actor: {}", actor_name);
}
}
sleep(Duration::from_secs(2)).await;
// Stop all remaining workers
info!(" Stopping all remaining workers...");
if let Err(e) = lifecycle_manager.stop_all_workers().await {
error!("Failed to stop all workers: {}", e);
// Stop all remaining actors
info!(" Stopping all remaining actors...");
if let Err(e) = lifecycle_manager.stop_all_actors().await {
error!("Failed to stop all actors: {}", e);
} else {
info!(" ✅ All workers stopped successfully");
info!(" ✅ All actors stopped successfully");
}
info!("=== Worker Lifecycle Demo Completed ===");
info!("=== Actor Lifecycle Demo Completed ===");
// Optional: Start health monitoring loop (commented out for demo)
// info!("Starting health monitoring loop (Ctrl+C to stop)...");

View File

@@ -0,0 +1,52 @@
use hero_supervisor::{SupervisorBuilder, ScriptType};
use hero_job::JobBuilder as CoreJobBuilder;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
// 1) Build a Supervisor
let supervisor = SupervisorBuilder::new()
.redis_url("redis://127.0.0.1/")
.build()
.await?;
// 2) Build a Job (using core job builder to set caller_id, context_id)
let job = CoreJobBuilder::new()
.caller_id("02abc...caller") // required
.context_id("02def...context") // required
.script_type(ScriptType::OSIS) // select the OSIS actor (matches configured osis_actor_1)
.script("40 + 3") // simple Rhai script
.timeout(std::time::Duration::from_secs(10))
.build()?; // returns hero_job::Job
let job_id = job.id.clone();
// 3a) Store the job in Redis
supervisor.create_job(&job).await?;
// 3b) Start the job (pushes ID to the actors Redis queue)
supervisor.start_job(&job_id).await?;
// 3c) Wait until finished, then fetch output
use tokio::time::sleep;
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);
loop {
let status = supervisor.get_job_status(&job_id).await?;
if status == hero_supervisor::JobStatus::Finished {
break;
}
if std::time::Instant::now() >= deadline {
println!("Job {} timed out waiting for completion (status: {:?})", job_id, status);
break;
}
sleep(std::time::Duration::from_millis(250)).await;
}
if let Some(output) = supervisor.get_job_output(&job_id).await? {
println!("Job {} output: {}", job_id, output);
} else {
println!("Job {} completed with no output field set", job_id);
}
Ok(())
}

View File

@@ -8,44 +8,44 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
info!("Starting Hero Supervisor Lifecycle Demo");
// Build supervisor with simplified worker configuration
// Workers are automatically launched during build
// Build supervisor with simplified actor configuration
// Actors are automatically launched during build
let supervisor = SupervisorBuilder::new()
.redis_url("redis://localhost:6379")
.osis_worker("/usr/local/bin/osis_worker")
.sal_worker("/usr/local/bin/sal_worker")
.v_worker("/usr/local/bin/v_worker")
.worker_env_var("REDIS_URL", "redis://localhost:6379")
.worker_env_var("LOG_LEVEL", "info")
.osis_actor("/usr/local/bin/osis_actor")
.sal_actor("/usr/local/bin/sal_actor")
.v_actor("/usr/local/bin/v_actor")
.actor_env_var("REDIS_URL", "redis://localhost:6379")
.actor_env_var("LOG_LEVEL", "info")
.build().await?;
info!("Supervisor created and workers launched successfully");
info!("Supervisor created and actors launched successfully");
// Wait a moment for workers to start
// Wait a moment for actors to start
sleep(Duration::from_secs(2)).await;
// Check worker status using the simplified API
info!("Checking worker status...");
let workers = supervisor.get_workers(&[]).await;
// Check actor status using the simplified API
info!("Checking actor status...");
let actors = supervisor.get_actors(&[]).await;
for worker in &workers {
let status_info = if worker.is_running {
format!("Running (PID: {})", worker.status.as_ref().map(|s| s.pid).unwrap_or(0))
for actor in &actors {
let status_info = if actor.is_running {
format!("Running (PID: {})", actor.status.as_ref().map(|s| s.pid).unwrap_or(0))
} else {
"Stopped".to_string()
};
info!(" Worker '{}' ({:?}): {}", worker.config.name, worker.config.script_type, status_info);
info!(" Actor '{}' ({:?}): {}", actor.config.name, actor.config.script_type, status_info);
}
// Demonstrate lifecycle operations with simplified API
info!("=== Worker Lifecycle Operations ===");
info!("=== Actor Lifecycle Operations ===");
// 1. Demonstrate restart functionality
info!("1. Demonstrating worker restart...");
if let Err(e) = supervisor.restart_worker("osis_worker_1").await {
error!("Failed to restart worker: {}", e);
info!("1. Demonstrating actor restart...");
if let Err(e) = supervisor.restart_actor("osis_actor_1").await {
error!("Failed to restart actor: {}", e);
} else {
info!(" ✅ Successfully restarted osis_worker_1");
info!(" ✅ Successfully restarted osis_actor_1");
}
sleep(Duration::from_secs(2)).await;
@@ -61,11 +61,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// 3. Demonstrate graceful shutdown
info!("3. Demonstrating graceful shutdown...");
// Stop specific workers
if let Err(e) = supervisor.stop_worker("osis_worker_1").await {
error!("Failed to stop worker: {}", e);
// Stop specific actors
if let Err(e) = supervisor.stop_actor("osis_actor_1").await {
error!("Failed to stop actor: {}", e);
} else {
info!("Worker stopped successfully");
info!("Actor stopped successfully");
}
info!("Demo completed successfully!");

View File

@@ -1,18 +1,18 @@
[global]
redis_url = "redis://localhost:6379"
[osis_worker]
binary_path = "/path/to/osis_worker"
[osis_actor]
binary_path = "/path/to/osis_actor"
env_vars = { "VAR1" = "value1", "VAR2" = "value2" }
[sal_worker]
binary_path = "/path/to/sal_worker"
[sal_actor]
binary_path = "/path/to/sal_actor"
env_vars = { "VAR1" = "value1", "VAR2" = "value2" }
[v_worker]
binary_path = "/path/to/v_worker"
[v_actor]
binary_path = "/path/to/v_actor"
env_vars = { "VAR1" = "value1", "VAR2" = "value2" }
[python_worker]
binary_path = "/path/to/python_worker"
[python_actor]
binary_path = "/path/to/python_actor"
env_vars = { "VAR1" = "value1", "VAR2" = "value2" }

View File

@@ -16,14 +16,14 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
info!("Supervisor created.");
let script_content = r#"
// This script will never be executed by a worker because the recipient does not exist.
// This script will never be executed by a actor because the recipient does not exist.
let x = 10;
let y = x + 32;
y
"#;
// The worker_id points to a worker queue that doesn't have a worker.
let non_existent_recipient = "non_existent_worker_for_timeout_test";
// The actor_id points to a actor queue that doesn't have a actor.
let non_existent_recipient = "non_existent_actor_for_timeout_test";
let very_short_timeout = Duration::from_secs(2);
info!(

View File

@@ -21,12 +21,12 @@ pub enum SupervisorError {
InvalidInput(String),
/// Job operation error
JobError(hero_job::JobError),
/// Worker lifecycle management errors
WorkerStartFailed(String, String),
WorkerStopFailed(String, String),
WorkerRestartFailed(String, String),
WorkerStatusFailed(String, String),
WorkerNotFound(String),
/// Actor lifecycle management errors
ActorStartFailed(String, String),
ActorStopFailed(String, String),
ActorRestartFailed(String, String),
ActorStatusFailed(String, String),
ActorNotFound(String),
PingJobFailed(String, String),
/// Zinit client operation error
ZinitError(String),
@@ -73,23 +73,23 @@ impl std::fmt::Display for SupervisorError {
SupervisorError::JobError(e) => {
write!(f, "Job error: {}", e)
}
SupervisorError::WorkerStartFailed(worker, reason) => {
write!(f, "Failed to start worker '{}': {}", worker, reason)
SupervisorError::ActorStartFailed(actor, reason) => {
write!(f, "Failed to start actor '{}': {}", actor, reason)
}
SupervisorError::WorkerStopFailed(worker, reason) => {
write!(f, "Failed to stop worker '{}': {}", worker, reason)
SupervisorError::ActorStopFailed(actor, reason) => {
write!(f, "Failed to stop actor '{}': {}", actor, reason)
}
SupervisorError::WorkerRestartFailed(worker, reason) => {
write!(f, "Failed to restart worker '{}': {}", worker, reason)
SupervisorError::ActorRestartFailed(actor, reason) => {
write!(f, "Failed to restart actor '{}': {}", actor, reason)
}
SupervisorError::WorkerStatusFailed(worker, reason) => {
write!(f, "Failed to get status for worker '{}': {}", worker, reason)
SupervisorError::ActorStatusFailed(actor, reason) => {
write!(f, "Failed to get status for actor '{}': {}", actor, reason)
}
SupervisorError::WorkerNotFound(worker) => {
write!(f, "Worker '{}' not found", worker)
SupervisorError::ActorNotFound(actor) => {
write!(f, "Actor '{}' not found", actor)
}
SupervisorError::PingJobFailed(worker, reason) => {
write!(f, "Ping job failed for worker '{}': {}", worker, reason)
SupervisorError::PingJobFailed(actor, reason) => {
write!(f, "Ping job failed for actor '{}': {}", actor, reason)
}
SupervisorError::ZinitError(msg) => {
write!(f, "Zinit error: {}", msg)

View File

@@ -16,7 +16,7 @@ mod lifecycle;
pub use crate::error::SupervisorError;
pub use crate::job::JobBuilder;
pub use crate::lifecycle::WorkerConfig;
pub use crate::lifecycle::ActorConfig;
// Re-export types from hero_job for public API
pub use hero_job::{Job, JobStatus, ScriptType};
@@ -28,22 +28,22 @@ pub struct Supervisor {
pub struct SupervisorBuilder {
redis_url: Option<String>,
osis_worker: Option<String>,
sal_worker: Option<String>,
v_worker: Option<String>,
python_worker: Option<String>,
worker_env_vars: HashMap<String, String>,
osis_actor: Option<String>,
sal_actor: Option<String>,
v_actor: Option<String>,
python_actor: Option<String>,
actor_env_vars: HashMap<String, String>,
websocket_config: Option<WebSocketServerConfig>,
}
/// Helper struct to pass builder data to worker launch method
/// Helper struct to pass builder data to actor launch method
#[derive(Clone)]
struct SupervisorBuilderData {
osis_worker: Option<String>,
sal_worker: Option<String>,
v_worker: Option<String>,
python_worker: Option<String>,
worker_env_vars: HashMap<String, String>,
osis_actor: Option<String>,
sal_actor: Option<String>,
v_actor: Option<String>,
python_actor: Option<String>,
actor_env_vars: HashMap<String, String>,
websocket_config: Option<WebSocketServerConfig>,
}
@@ -52,10 +52,10 @@ struct SupervisorBuilderData {
pub struct SupervisorConfig {
pub global: GlobalConfig,
pub websocket_server: Option<WebSocketServerConfig>,
pub osis_worker: Option<WorkerConfigToml>,
pub sal_worker: Option<WorkerConfigToml>,
pub v_worker: Option<WorkerConfigToml>,
pub python_worker: Option<WorkerConfigToml>,
pub osis_actor: Option<ActorConfigToml>,
pub sal_actor: Option<ActorConfigToml>,
pub v_actor: Option<ActorConfigToml>,
pub python_actor: Option<ActorConfigToml>,
}
/// Global configuration section
@@ -64,12 +64,10 @@ pub struct GlobalConfig {
pub redis_url: String,
}
/// Worker configuration section in TOML
/// Actor configuration section in TOML
#[derive(Debug, Deserialize, Serialize)]
pub struct WorkerConfigToml {
pub struct ActorConfigToml {
pub binary_path: String,
#[serde(default)]
pub env_vars: HashMap<String, String>,
}
/// WebSocket server configuration section in TOML
@@ -127,11 +125,11 @@ impl SupervisorBuilder {
pub fn new() -> Self {
Self {
redis_url: None,
osis_worker: None,
sal_worker: None,
v_worker: None,
python_worker: None,
worker_env_vars: HashMap::new(),
osis_actor: None,
sal_actor: None,
v_actor: None,
python_actor: None,
actor_env_vars: HashMap::new(),
websocket_config: None,
}
}
@@ -147,25 +145,21 @@ impl SupervisorBuilder {
let mut builder = Self::new()
.redis_url(&config.global.redis_url);
// Configure workers based on TOML config
if let Some(osis_config) = config.osis_worker {
builder = builder.osis_worker(&osis_config.binary_path)
.worker_env_vars(osis_config.env_vars);
// Configure actors based on TOML config
if let Some(osis_config) = config.osis_actor {
builder = builder.osis_actor(&osis_config.binary_path);
}
if let Some(sal_config) = config.sal_worker {
builder = builder.sal_worker(&sal_config.binary_path)
.worker_env_vars(sal_config.env_vars);
if let Some(sal_config) = config.sal_actor {
builder = builder.sal_actor(&sal_config.binary_path);
}
if let Some(v_config) = config.v_worker {
builder = builder.v_worker(&v_config.binary_path)
.worker_env_vars(v_config.env_vars);
if let Some(v_config) = config.v_actor {
builder = builder.v_actor(&v_config.binary_path);
}
if let Some(python_config) = config.python_worker {
builder = builder.python_worker(&python_config.binary_path)
.worker_env_vars(python_config.env_vars);
if let Some(python_config) = config.python_actor {
builder = builder.python_actor(&python_config.binary_path);
}
// Store WebSocket configuration for later use
@@ -176,28 +170,28 @@ impl SupervisorBuilder {
Ok(builder)
}
/// Validate that all configured worker binaries exist and are executable
fn validate_worker_binaries(&self) -> Result<(), SupervisorError> {
let workers = [
("OSIS", &self.osis_worker),
("SAL", &self.sal_worker),
("V", &self.v_worker),
("Python", &self.python_worker),
/// Validate that all configured actor binaries exist and are executable
fn validate_actor_binaries(&self) -> Result<(), SupervisorError> {
let actors = [
("OSIS", &self.osis_actor),
("SAL", &self.sal_actor),
("V", &self.v_actor),
("Python", &self.python_actor),
];
for (worker_type, binary_path) in workers {
for (actor_type, binary_path) in actors {
if let Some(path) = binary_path {
let path_obj = Path::new(path);
if !path_obj.exists() {
return Err(SupervisorError::ConfigError(
format!("{} worker binary does not exist: {}", worker_type, path)
format!("{} actor binary does not exist: {}", actor_type, path)
));
}
if !path_obj.is_file() {
return Err(SupervisorError::ConfigError(
format!("{} worker path is not a file: {}", worker_type, path)
format!("{} actor path is not a file: {}", actor_type, path)
));
}
@@ -207,19 +201,19 @@ impl SupervisorBuilder {
use std::os::unix::fs::PermissionsExt;
let metadata = path_obj.metadata().map_err(|e| {
SupervisorError::ConfigError(
format!("Failed to read metadata for {} worker binary {}: {}", worker_type, path, e)
format!("Failed to read metadata for {} actor binary {}: {}", actor_type, path, e)
)
})?;
let permissions = metadata.permissions();
if permissions.mode() & 0o111 == 0 {
return Err(SupervisorError::ConfigError(
format!("{} worker binary is not executable: {}", worker_type, path)
format!("{} actor binary is not executable: {}", actor_type, path)
));
}
}
info!("Validated {} worker binary: {}", worker_type, path);
info!("Validated {} actor binary: {}", actor_type, path);
}
}
@@ -231,48 +225,48 @@ impl SupervisorBuilder {
self
}
pub fn osis_worker(mut self, binary_path: &str) -> Self {
self.osis_worker = Some(binary_path.to_string());
pub fn osis_actor(mut self, binary_path: &str) -> Self {
self.osis_actor = Some(binary_path.to_string());
self
}
pub fn sal_worker(mut self, binary_path: &str) -> Self {
self.sal_worker = Some(binary_path.to_string());
pub fn sal_actor(mut self, binary_path: &str) -> Self {
self.sal_actor = Some(binary_path.to_string());
self
}
pub fn v_worker(mut self, binary_path: &str) -> Self {
self.v_worker = Some(binary_path.to_string());
pub fn v_actor(mut self, binary_path: &str) -> Self {
self.v_actor = Some(binary_path.to_string());
self
}
pub fn python_worker(mut self, binary_path: &str) -> Self {
self.python_worker = Some(binary_path.to_string());
pub fn python_actor(mut self, binary_path: &str) -> Self {
self.python_actor = Some(binary_path.to_string());
self
}
pub fn worker_env_var(mut self, key: &str, value: &str) -> Self {
self.worker_env_vars.insert(key.to_string(), value.to_string());
pub fn actor_env_var(mut self, key: &str, value: &str) -> Self {
self.actor_env_vars.insert(key.to_string(), value.to_string());
self
}
pub fn worker_env_vars(mut self, env_vars: HashMap<String, String>) -> Self {
self.worker_env_vars.extend(env_vars);
pub fn actor_env_vars(mut self, env_vars: HashMap<String, String>) -> Self {
self.actor_env_vars.extend(env_vars);
self
}
/// Builds the final `Supervisor` instance synchronously.
///
/// This method validates the configuration, checks worker binary existence,
/// and creates the Redis client. Worker launching is deferred to the `start_workers()` method.
/// This method validates the configuration, checks actor binary existence,
/// and creates the Redis client. Actor launching is deferred to the `start_actors()` method.
///
/// # Returns
///
/// * `Ok(Supervisor)` - Successfully configured client with valid binaries
/// * `Err(SupervisorError)` - Configuration, binary validation, or connection error
pub async fn build(self) -> Result<Supervisor, SupervisorError> {
// Validate that all configured worker binaries exist first
Self::validate_worker_binaries(&self)?;
// Validate that all configured actor binaries exist first
Self::validate_actor_binaries(&self)?;
let url = self.redis_url
.unwrap_or_else(|| "redis://127.0.0.1/".to_string());
@@ -281,13 +275,13 @@ impl SupervisorBuilder {
let zinit_client = ZinitClient::unix_socket("/tmp/zinit.sock").await
.map_err(|e| SupervisorError::ZinitError(format!("Failed to create Zinit client: {}", e)))?;
// Store builder data for later use in start_workers()
// Store builder data for later use in start_actors()
let builder_data = SupervisorBuilderData {
osis_worker: self.osis_worker,
sal_worker: self.sal_worker,
v_worker: self.v_worker,
python_worker: self.python_worker,
worker_env_vars: self.worker_env_vars,
osis_actor: self.osis_actor,
sal_actor: self.sal_actor,
v_actor: self.v_actor,
python_actor: self.python_actor,
actor_env_vars: self.actor_env_vars,
websocket_config: self.websocket_config,
};
@@ -302,10 +296,10 @@ impl SupervisorBuilder {
}
impl Supervisor {
/// Start all configured workers asynchronously.
/// This method should be called after build() to launch the workers.
pub async fn start_workers(&self) -> Result<(), SupervisorError> {
info!("Starting Hero Supervisor workers...");
/// Start all configured actors asynchronously.
/// This method should be called after build() to launch the actors.
pub async fn start_actors(&self) -> Result<(), SupervisorError> {
info!("Starting Hero Supervisor actors...");
// Test Zinit connection first
info!("Testing Zinit connection at /tmp/zinit.sock...");
@@ -319,102 +313,103 @@ impl Supervisor {
}
}
// Clean up any existing worker services first
info!("Cleaning up existing worker services...");
self.cleanup_existing_workers().await?;
// Clean up any existing actor services first
info!("Cleaning up existing actor services...");
self.cleanup_existing_actors().await?;
// Launch configured workers if builder data is available
// Launch configured actors if builder data is available
if let Some(builder_data) = &self.builder_data {
info!("Launching configured workers...");
self.launch_configured_workers(builder_data).await?;
info!("Launching configured actors...");
self.launch_configured_actors(builder_data).await?;
} else {
warn!("No builder data available, no workers to start");
warn!("No builder data available, no actors to start");
}
info!("All workers started successfully!");
info!("All actors started successfully!");
Ok(())
}
/// Clean up all worker services from zinit on program exit
/// Clean up all actor services from zinit on program exit
pub async fn cleanup_and_shutdown(&self) -> Result<(), SupervisorError> {
info!("Cleaning up worker services before shutdown...");
info!("Cleaning up actor services before shutdown...");
let worker_names = vec![
"osis_worker_1",
"sal_worker_1",
"v_worker_1",
"python_worker_1"
let actor_names = vec![
"osis_actor_1",
"sal_actor_1",
"v_actor_1",
"python_actor_1"
];
for worker_name in worker_names {
if let Err(e) = self.stop_and_delete_worker(worker_name).await {
warn!("Failed to cleanup worker {}: {}", worker_name, e);
for actor_name in actor_names {
if let Err(e) = self.stop_and_delete_actor(actor_name).await {
warn!("Failed to cleanup actor {}: {}", actor_name, e);
}
}
info!("Worker cleanup completed");
info!("Actor cleanup completed");
Ok(())
}
/// Clean up any existing worker services on startup
async fn cleanup_existing_workers(&self) -> Result<(), SupervisorError> {
info!("Cleaning up any existing worker services...");
/// Clean up any existing actor services on startup
async fn cleanup_existing_actors(&self) -> Result<(), SupervisorError> {
info!("Cleaning up any existing actor services...");
let worker_names = vec![
"osis_worker_1",
"sal_worker_1",
"v_worker_1",
"python_worker_1"
let actor_names = vec![
"osis_actor_1",
"sal_actor_1",
"v_actor_1",
"python_actor_1"
];
for worker_name in worker_names {
for actor_name in actor_names {
// Try to stop and delete, but don't fail if they don't exist
info!("Attempting to cleanup worker: {}", worker_name);
match self.stop_and_delete_worker(worker_name).await {
Ok(_) => info!("Successfully cleaned up worker: {}", worker_name),
Err(e) => debug!("Failed to cleanup worker {}: {}", worker_name, e),
info!("Attempting to cleanup actor: {}", actor_name);
match self.stop_and_delete_actor(actor_name).await {
Ok(_) => info!("Successfully cleaned up actor: {}", actor_name),
Err(e) => debug!("Failed to cleanup actor {}: {}", actor_name, e),
}
}
info!("Existing worker cleanup completed");
info!("Existing actor cleanup completed");
Ok(())
}
/// Stop and delete a worker service from zinit
async fn stop_and_delete_worker(&self, worker_name: &str) -> Result<(), SupervisorError> {
info!("Starting cleanup for worker: {}", worker_name);
/// Stop and delete a actor service from zinit
async fn stop_and_delete_actor(&self, actor_name: &str) -> Result<(), SupervisorError> {
info!("Starting cleanup for actor: {}", actor_name);
// First try to stop the worker
info!("Attempting to stop worker: {}", worker_name);
if let Err(e) = self.zinit_client.stop(worker_name).await {
debug!("Worker {} was not running or failed to stop: {}", worker_name, e);
// First try to stop the actor
info!("Attempting to stop actor: {}", actor_name);
if let Err(e) = self.zinit_client.stop(actor_name).await {
debug!("Actor {} was not running or failed to stop: {}", actor_name, e);
} else {
info!("Successfully stopped worker: {}", worker_name);
info!("Successfully stopped actor: {}", actor_name);
}
// Then forget the service to stop monitoring it
info!("Attempting to forget worker: {}", worker_name);
if let Err(e) = self.zinit_client.forget(worker_name).await {
info!("Worker {} was not being monitored or failed to forget: {}", worker_name, e);
info!("Attempting to forget actor: {}", actor_name);
if let Err(e) = self.zinit_client.forget(actor_name).await {
info!("Actor {} was not being monitored or failed to forget: {}", actor_name, e);
} else {
info!("Successfully forgot worker service: {}", worker_name);
info!("Successfully forgot actor service: {}", actor_name);
}
// Finally, delete the service configuration
info!("Attempting to delete service for worker: {}", worker_name);
if let Err(e) = self.zinit_client.delete_service(worker_name).await {
debug!("Worker {} service did not exist or failed to delete: {}", worker_name, e);
info!("Attempting to delete service for actor: {}", actor_name);
if let Err(e) = self.zinit_client.delete_service(actor_name).await {
debug!("Actor {} service did not exist or failed to delete: {}", actor_name, e);
} else {
info!("Successfully deleted worker service: {}", worker_name);
info!("Successfully deleted actor service: {}", actor_name);
}
info!("Completed cleanup for worker: {}", worker_name);
info!("Completed cleanup for actor: {}", actor_name);
Ok(())
}
/// Get the hardcoded worker queue key for the script type
fn get_worker_queue_key(&self, script_type: &ScriptType) -> String {
format!("{}worker_queue:{}", NAMESPACE_PREFIX, script_type.worker_queue_suffix())
/// Get the hardcoded actor queue key for the script type
fn get_actor_queue_key(&self, script_type: &ScriptType) -> String {
// Canonical type queue
hero_job::keys::work_type(script_type)
}
pub fn new_job(&self) -> JobBuilder {
@@ -432,63 +427,58 @@ impl Supervisor {
})
}
/// Extract worker configurations from the supervisor's builder data
pub fn get_worker_configs(&self) -> Result<Vec<WorkerConfig>, SupervisorError> {
/// Extract actor configurations from the supervisor's builder data
pub fn get_actor_configs(&self) -> Result<Vec<ActorConfig>, SupervisorError> {
let builder_data = self.builder_data.as_ref().ok_or_else(|| {
SupervisorError::ConfigError("No builder data available for worker configs".to_string())
SupervisorError::ConfigError("No builder data available for actor configs".to_string())
})?;
let mut configs = Vec::new();
let env_vars = builder_data.worker_env_vars.clone();
if let Some(osis_path) = &builder_data.osis_worker {
if let Some(osis_path) = &builder_data.osis_actor {
configs.push(
WorkerConfig::new("osis_worker_1".to_string(), PathBuf::from(osis_path), ScriptType::OSIS)
.with_env(env_vars.clone())
ActorConfig::new("osis_actor_1".to_string(), PathBuf::from(osis_path), ScriptType::OSIS)
);
}
if let Some(sal_path) = &builder_data.sal_worker {
if let Some(sal_path) = &builder_data.sal_actor {
configs.push(
WorkerConfig::new("sal_worker_1".to_string(), PathBuf::from(sal_path), ScriptType::SAL)
.with_env(env_vars.clone())
ActorConfig::new("sal_actor_1".to_string(), PathBuf::from(sal_path), ScriptType::SAL)
);
}
if let Some(v_path) = &builder_data.v_worker {
if let Some(v_path) = &builder_data.v_actor {
configs.push(
WorkerConfig::new("v_worker_1".to_string(), PathBuf::from(v_path), ScriptType::V)
.with_env(env_vars.clone())
ActorConfig::new("v_actor_1".to_string(), PathBuf::from(v_path), ScriptType::V)
);
}
if let Some(python_path) = &builder_data.python_worker {
if let Some(python_path) = &builder_data.python_actor {
configs.push(
WorkerConfig::new("python_worker_1".to_string(), PathBuf::from(python_path), ScriptType::Python)
.with_env(env_vars.clone())
ActorConfig::new("python_actor_1".to_string(), PathBuf::from(python_path), ScriptType::Python)
);
}
Ok(configs)
}
/// Spawn a background lifecycle manager that continuously monitors and maintains worker health
/// Spawn a background lifecycle manager that continuously monitors and maintains actor health
/// Returns a JoinHandle that can be used to stop the lifecycle manager
pub fn spawn_lifecycle_manager(
self: Arc<Self>,
worker_configs: Vec<WorkerConfig>,
actor_configs: Vec<ActorConfig>,
health_check_interval: Duration,
) -> tokio::task::JoinHandle<Result<(), SupervisorError>> {
let supervisor = self;
tokio::spawn(async move {
info!("Starting background lifecycle manager with {} workers", worker_configs.len());
info!("Starting background lifecycle manager with {} actors", actor_configs.len());
info!("Health check interval: {:?}", health_check_interval);
// Initial worker startup
info!("Performing initial worker startup...");
if let Err(e) = supervisor.start_workers().await {
error!("Failed to start workers during initialization: {}", e);
// Initial actor startup
info!("Performing initial actor startup...");
if let Err(e) = supervisor.start_actors().await {
error!("Failed to start actors during initialization: {}", e);
return Err(e);
}
@@ -499,12 +489,12 @@ impl Supervisor {
loop {
interval.tick().await;
info!("Running periodic worker health check...");
info!("Running periodic actor health check...");
// Check each worker's health and restart if needed
for worker_config in &worker_configs {
if let Err(e) = supervisor.check_and_restart_worker(worker_config).await {
error!("Failed to check/restart worker {}: {}", worker_config.name, e);
// Check each actor's health and restart if needed
for actor_config in &actor_configs {
if let Err(e) = supervisor.check_and_restart_actor(actor_config).await {
error!("Failed to check/restart actor {}: {}", actor_config.name, e);
}
}
@@ -513,59 +503,59 @@ impl Supervisor {
})
}
/// Check a single worker's health and restart if needed
async fn check_and_restart_worker(&self, worker_config: &WorkerConfig) -> Result<(), SupervisorError> {
let worker_name = &worker_config.name;
/// Check a single actor's health and restart if needed
async fn check_and_restart_actor(&self, actor_config: &ActorConfig) -> Result<(), SupervisorError> {
let actor_name = &actor_config.name;
// Get worker status
match self.zinit_client.status(worker_name).await {
// Get actor status
match self.zinit_client.status(actor_name).await {
Ok(status) => {
let is_healthy = status.state == "running" && status.pid > 0;
if is_healthy {
debug!("Worker {} is healthy (state: {}, pid: {})", worker_name, status.state, status.pid);
debug!("Actor {} is healthy (state: {}, pid: {})", actor_name, status.state, status.pid);
// Optionally send a ping job for deeper health check
if let Err(e) = self.send_ping_job(worker_config.script_type.clone()).await {
warn!("Ping job failed for worker {}: {}", worker_name, e);
if let Err(e) = self.send_ping_job(actor_config.script_type.clone()).await {
warn!("Ping job failed for actor {}: {}", actor_name, e);
// Note: We don't restart on ping failure as it might be temporary
}
} else {
warn!("Worker {} is unhealthy (state: {}, pid: {}), restarting...",
worker_name, status.state, status.pid);
warn!("Actor {} is unhealthy (state: {}, pid: {}), restarting...",
actor_name, status.state, status.pid);
// Attempt to restart the worker
if let Err(e) = self.restart_worker(worker_name).await {
error!("Failed to restart unhealthy worker {}: {}", worker_name, e);
// Attempt to restart the actor
if let Err(e) = self.restart_actor(actor_name).await {
error!("Failed to restart unhealthy actor {}: {}", actor_name, e);
// If restart fails, try a full stop/start cycle
warn!("Attempting full stop/start cycle for worker: {}", worker_name);
if let Err(e) = self.stop_and_delete_worker(worker_name).await {
error!("Failed to stop worker {} during recovery: {}", worker_name, e);
warn!("Attempting full stop/start cycle for actor: {}", actor_name);
if let Err(e) = self.stop_and_delete_actor(actor_name).await {
error!("Failed to stop actor {} during recovery: {}", actor_name, e);
}
if let Err(e) = self.start_worker(worker_config).await {
error!("Failed to start worker {} during recovery: {}", worker_name, e);
if let Err(e) = self.start_actor(actor_config).await {
error!("Failed to start actor {} during recovery: {}", actor_name, e);
return Err(e);
}
info!("Successfully recovered worker: {}", worker_name);
info!("Successfully recovered actor: {}", actor_name);
} else {
info!("Successfully restarted worker: {}", worker_name);
info!("Successfully restarted actor: {}", actor_name);
}
}
}
Err(e) => {
warn!("Could not get status for worker {} (may not exist): {}", worker_name, e);
warn!("Could not get status for actor {} (may not exist): {}", actor_name, e);
// Worker doesn't exist, try to start it
info!("Attempting to start missing worker: {}", worker_name);
if let Err(e) = self.start_worker(worker_config).await {
error!("Failed to start missing worker {}: {}", worker_name, e);
// Actor doesn't exist, try to start it
info!("Attempting to start missing actor: {}", actor_name);
if let Err(e) = self.start_actor(actor_config).await {
error!("Failed to start missing actor {}: {}", actor_name, e);
return Err(e);
}
info!("Successfully started missing worker: {}", worker_name);
info!("Successfully started missing actor: {}", actor_name);
}
}
@@ -597,18 +587,13 @@ impl Supervisor {
job_id: String,
script_type: &ScriptType
) -> Result<(), SupervisorError> {
let worker_queue_key = self.get_worker_queue_key(script_type);
// lpush also infers its types, RV is typically i64 (length of list) or () depending on exact command variant
// For `redis::AsyncCommands::lpush`, it's `RedisResult<R>` where R: FromRedisValue
// Often this is the length of the list. Let's allow inference or specify if needed.
let _: redis::RedisResult<i64> =
conn.lpush(&worker_queue_key, job_id.clone()).await;
// Canonical dispatch to type queue
let actor_queue_key = hero_job::keys::work_type(script_type);
let _: redis::RedisResult<i64> = conn.lpush(&actor_queue_key, job_id.clone()).await;
Ok(())
}
// Internal helper to await response from worker
// Internal helper to await response from actor
async fn await_response_from_connection(
&self,
conn: &mut redis::aio::MultiplexedConnection,
@@ -679,14 +664,15 @@ impl Supervisor {
Ok(())
}
// New method using dedicated reply queue with automatic worker selection
// New method using dedicated reply queue with automatic actor selection
pub async fn run_job_and_await_result(
&self,
job: &Job
) -> Result<String, SupervisorError> {
let mut conn = self.redis_client.get_multiplexed_async_connection().await?;
let reply_queue_key = format!("{}:reply:{}", NAMESPACE_PREFIX, job.id); // Derived from the passed job_id
// Canonical reply queue
let reply_queue_key = hero_job::keys::reply(&job.id);
self.create_job_using_connection(
&mut conn,
@@ -703,13 +689,48 @@ impl Supervisor {
job.timeout
);
self.await_response_from_connection(
&mut conn,
&job.id,
&reply_queue_key,
job.timeout,
)
.await
// Some actors update the job hash directly and do not use reply queues.
// Poll the job hash for output until timeout to support both models.
let start_time = std::time::Instant::now();
loop {
// If output is present in the job hash, return it immediately
match self.get_job_output(&job.id).await {
Ok(Some(output)) => {
// Optional: cleanup reply queue in case it was created
let _: redis::RedisResult<i32> = conn.del(&reply_queue_key).await;
return Ok(output);
}
Ok(None) => {
// Check for error state
match self.get_job_status(&job.id).await {
Ok(JobStatus::Error) => {
// Try to read the error field for context
let mut conn2 = self.redis_client.get_multiplexed_async_connection().await?;
let job_key = format!("{}{}", NAMESPACE_PREFIX, job.id);
let err: Option<String> = conn2.hget(&job_key, "error").await.ok();
return Err(SupervisorError::InvalidInput(
err.unwrap_or_else(|| "Job failed".to_string())
));
}
_ => {
// keep polling
}
}
}
Err(_) => {
// Ignore transient read errors and continue polling
}
}
if start_time.elapsed() >= job.timeout {
// On timeout, ensure any reply queue is cleaned up and return a Timeout error
let _: redis::RedisResult<i32> = conn.del(&reply_queue_key).await;
return Err(SupervisorError::Timeout(job.id.clone()));
}
tokio::time::sleep(std::time::Duration::from_millis(200)).await;
}
}
// Method to get job status
@@ -782,8 +803,8 @@ impl Supervisor {
pub async fn stop_job(&self, job_id: &str) -> Result<(), SupervisorError> {
let mut conn = self.redis_client.get_multiplexed_async_connection().await?;
// Get job details to determine script type and appropriate worker
let job_key = format!("{}job:{}", NAMESPACE_PREFIX, job_id);
// Get job details to determine script type and appropriate actor
let job_key = format!("{}{}", NAMESPACE_PREFIX, job_id);
let job_data: std::collections::HashMap<String, String> = conn.hgetall(&job_key).await?;
if job_data.is_empty() {
@@ -798,7 +819,8 @@ impl Supervisor {
.map_err(|e| SupervisorError::InvalidInput(format!("Invalid script type: {}", e)))?;
// Use hardcoded stop queue key for this script type
let stop_queue_key = format!("{}stop_queue:{}", NAMESPACE_PREFIX, script_type.worker_queue_suffix());
// Stop queue per protocol: hero:stop_queue:{suffix}
let stop_queue_key = format!("hero:stop_queue:{}", script_type.actor_queue_suffix());
// Push job ID to the stop queue
conn.lpush::<_, _, ()>(&stop_queue_key, job_id).await?;
@@ -810,7 +832,7 @@ impl Supervisor {
/// Get logs for a job by reading from its log file
pub async fn get_job_logs(&self, job_id: &str) -> Result<Option<String>, SupervisorError> {
let mut conn = self.redis_client.get_multiplexed_async_connection().await?;
let job_key = format!("{}job:{}", NAMESPACE_PREFIX, job_id);
let job_key = format!("{}{}", NAMESPACE_PREFIX, job_id);
// Get the job data to find the log path
let result_map: Option<std::collections::HashMap<String, String>> =
@@ -931,9 +953,9 @@ impl Supervisor {
/// Dispatch jobs that are ready (have all prerequisites completed)
pub async fn dispatch_ready_jobs(&self, ready_job_ids: Vec<String>) -> Result<(), SupervisorError> {
for job_id in ready_job_ids {
// Get job data to determine script type and select worker
// Get job data to determine script type and select actor
let mut conn = self.redis_client.get_multiplexed_async_connection().await?;
let job_key = format!("{}job:{}", NAMESPACE_PREFIX, job_id);
let job_key = format!("{}{}", NAMESPACE_PREFIX, job_id);
let job_data: std::collections::HashMap<String, String> = conn.hgetall(&job_key).await?;
if let Some(script_type_str) = job_data.get("script_type") {

View File

@@ -1,6 +1,6 @@
//! Worker lifecycle management functionality for the Hero Supervisor
//! Actor lifecycle management functionality for the Hero Supervisor
//!
//! This module provides worker process lifecycle management using Zinit as the process manager.
//! This module provides actor process lifecycle management using Zinit as the process manager.
//! All functionality is implemented as methods on the Supervisor struct for a clean API.
use log::{debug, error, info, warn};
@@ -12,28 +12,28 @@ use zinit_client::{Client as ZinitClient, Status};
use hero_job::ScriptType;
use crate::{Supervisor, SupervisorError};
/// Information about a worker including its configuration and current status
/// Information about a actor including its configuration and current status
#[derive(Debug, Clone)]
pub struct WorkerInfo {
pub config: WorkerConfig,
pub struct ActorInfo {
pub config: ActorConfig,
pub status: Option<Status>,
pub is_running: bool,
}
/// Configuration for a worker binary
/// Configuration for a actor binary
#[derive(Debug, Clone)]
pub struct WorkerConfig {
/// Name of the worker service
pub struct ActorConfig {
/// Name of the actor service
pub name: String,
/// Path to the worker binary
/// Path to the actor binary
pub binary_path: PathBuf,
/// Script type this worker handles
/// Script type this actor handles
pub script_type: ScriptType,
/// Command line arguments for the worker
/// Command line arguments for the actor
pub args: Vec<String>,
/// Environment variables for the worker
/// Environment variables for the actor
pub env: HashMap<String, String>,
/// Whether this worker should restart on exit
/// Whether this actor should restart on exit
pub restart_on_exit: bool,
/// Health check command (optional)
pub health_check: Option<String>,
@@ -41,7 +41,7 @@ pub struct WorkerConfig {
pub dependencies: Vec<String>,
}
impl WorkerConfig {
impl ActorConfig {
pub fn new(name: String, binary_path: PathBuf, script_type: ScriptType) -> Self {
Self {
name,
@@ -81,122 +81,122 @@ impl WorkerConfig {
}
}
/// Worker lifecycle management methods for Supervisor
/// Actor lifecycle management methods for Supervisor
impl Supervisor {
/// Get all workers with their configuration and status - unified method
pub async fn get_workers(&self, worker_configs: &[WorkerConfig]) -> Vec<WorkerInfo> {
let mut workers = Vec::new();
/// Get all actors with their configuration and status - unified method
pub async fn get_actors(&self, actor_configs: &[ActorConfig]) -> Vec<ActorInfo> {
let mut actors = Vec::new();
for config in worker_configs {
for config in actor_configs {
let status = self.zinit_client.status(&config.name).await.ok();
let is_running = status.as_ref()
.map(|s| s.state == "running" && s.pid > 0)
.unwrap_or(false);
workers.push(WorkerInfo {
actors.push(ActorInfo {
config: config.clone(),
status,
is_running,
});
}
workers
actors
}
/// Start a worker using Zinit
pub async fn start_worker(
/// Start a actor using Zinit
pub async fn start_actor(
&self,
worker_config: &WorkerConfig,
actor_config: &ActorConfig,
) -> Result<(), SupervisorError> {
info!("Starting worker: {}", worker_config.name);
info!("Starting actor: {}", actor_config.name);
// Create service configuration for Zinit
let service_config = self.create_service_config(worker_config);
let service_config = self.create_service_config(actor_config);
// Create the service in Zinit
self.zinit_client.create_service(&worker_config.name, service_config).await
self.zinit_client.create_service(&actor_config.name, service_config).await
.map_err(|e| SupervisorError::ZinitError(format!("Failed to create service: {}", e)))?;
// Monitor the service so Zinit starts managing it
self.zinit_client.monitor(&worker_config.name).await
self.zinit_client.monitor(&actor_config.name).await
.map_err(|e| SupervisorError::ZinitError(format!("Failed to monitor service: {}", e)))?;
// Start the service
self.zinit_client.start(&worker_config.name).await
.map_err(|e| SupervisorError::ZinitError(format!("Failed to start worker: {}", e)))?;
self.zinit_client.start(&actor_config.name).await
.map_err(|e| SupervisorError::ZinitError(format!("Failed to start actor: {}", e)))?;
info!("Successfully started worker: {}", worker_config.name);
info!("Successfully started actor: {}", actor_config.name);
Ok(())
}
/// Stop a worker using Zinit
pub async fn stop_worker(
/// Stop a actor using Zinit
pub async fn stop_actor(
&self,
worker_name: &str,
actor_name: &str,
) -> Result<(), SupervisorError> {
info!("Stopping worker: {}", worker_name);
info!("Stopping actor: {}", actor_name);
match self.zinit_client.stop(worker_name).await {
match self.zinit_client.stop(actor_name).await {
Ok(_) => {
info!("Successfully stopped worker: {}", worker_name);
info!("Successfully stopped actor: {}", actor_name);
Ok(())
}
Err(e) => {
error!("Failed to stop worker {}: {}", worker_name, e);
Err(SupervisorError::WorkerStopFailed(worker_name.to_string(), e.to_string()))
error!("Failed to stop actor {}: {}", actor_name, e);
Err(SupervisorError::ActorStopFailed(actor_name.to_string(), e.to_string()))
}
}
}
/// Restart a worker using Zinit
pub async fn restart_worker(
/// Restart a actor using Zinit
pub async fn restart_actor(
&self,
worker_name: &str,
actor_name: &str,
) -> Result<(), SupervisorError> {
info!("Restarting worker: {}", worker_name);
info!("Restarting actor: {}", actor_name);
match self.zinit_client.restart(worker_name).await {
match self.zinit_client.restart(actor_name).await {
Ok(_) => {
info!("Successfully restarted worker: {}", worker_name);
info!("Successfully restarted actor: {}", actor_name);
Ok(())
}
Err(e) => {
error!("Failed to restart worker {}: {}", worker_name, e);
Err(SupervisorError::WorkerRestartFailed(worker_name.to_string(), e.to_string()))
error!("Failed to restart actor {}: {}", actor_name, e);
Err(SupervisorError::ActorRestartFailed(actor_name.to_string(), e.to_string()))
}
}
}
/// Get status of a worker using Zinit
pub async fn get_worker_status(
/// Get status of a actor using Zinit
pub async fn get_actor_status(
&self,
worker_name: &str,
actor_name: &str,
zinit_client: &ZinitClient,
) -> Result<Status, SupervisorError> {
match zinit_client.status(worker_name).await {
match zinit_client.status(actor_name).await {
Ok(status) => Ok(status),
Err(e) => {
error!("Failed to get status for worker {}: {}", worker_name, e);
Err(SupervisorError::WorkerStatusFailed(worker_name.to_string(), e.to_string()))
error!("Failed to get status for actor {}: {}", actor_name, e);
Err(SupervisorError::ActorStatusFailed(actor_name.to_string(), e.to_string()))
}
}
}
/// Get status of all workers
pub async fn get_all_worker_status(
/// Get status of all actors
pub async fn get_all_actor_status(
&self,
worker_configs: &[WorkerConfig],
actor_configs: &[ActorConfig],
zinit_client: &ZinitClient,
) -> Result<HashMap<String, Status>, SupervisorError> {
let mut status_map = HashMap::new();
for worker in worker_configs {
match zinit_client.status(&worker.name).await {
for actor in actor_configs {
match zinit_client.status(&actor.name).await {
Ok(status) => {
status_map.insert(worker.name.clone(), status);
status_map.insert(actor.name.clone(), status);
}
Err(e) => {
warn!("Failed to get status for worker {}: {}", worker.name, e);
warn!("Failed to get status for actor {}: {}", actor.name, e);
}
}
}
@@ -206,32 +206,32 @@ impl Supervisor {
/// Stop multiple workers
pub async fn stop_workers(
/// Stop multiple actors
pub async fn stop_actors(
&self,
worker_names: &[String],
actor_names: &[String],
) -> Result<(), SupervisorError> {
info!("Stopping {} workers", worker_names.len());
info!("Stopping {} actors", actor_names.len());
for worker_name in worker_names {
self.stop_worker(worker_name).await?;
for actor_name in actor_names {
self.stop_actor(actor_name).await?;
}
Ok(())
}
/// Get count of running workers for a script type
pub async fn get_running_worker_count(
/// Get count of running actors for a script type
pub async fn get_running_actor_count(
&self,
worker_configs: &[WorkerConfig],
actor_configs: &[ActorConfig],
script_type: &ScriptType,
zinit_client: &ZinitClient,
) -> usize {
let mut running_count = 0;
for worker in worker_configs {
if worker.script_type == *script_type {
if let Ok(status) = zinit_client.status(&worker.name).await {
for actor in actor_configs {
if actor.script_type == *script_type {
if let Ok(status) = zinit_client.status(&actor.name).await {
if status.state == "running" {
running_count += 1;
}
@@ -242,7 +242,7 @@ impl Supervisor {
running_count
}
/// Send a ping job to a worker for health checking
/// Send a ping job to a actor for health checking
pub async fn send_ping_job(
&self,
script_type: ScriptType,
@@ -268,8 +268,8 @@ impl Supervisor {
}
}
/// Create Zinit service configuration from worker config
fn create_service_config(&self, worker: &WorkerConfig) -> serde_json::Map<String, serde_json::Value> {
/// Create Zinit service configuration from actor config
fn create_service_config(&self, actor: &ActorConfig) -> serde_json::Map<String, serde_json::Value> {
use serde_json::{Map, Value};
let mut config = Map::new();
@@ -277,117 +277,117 @@ impl Supervisor {
config.insert(
"exec".to_string(),
Value::String(format!("{} {}",
worker.binary_path.display(),
worker.args.join(" ")
actor.binary_path.display(),
actor.args.join(" ")
))
);
config.insert(
"oneshot".to_string(),
Value::Bool(!worker.restart_on_exit)
Value::Bool(!actor.restart_on_exit)
);
if let Some(health_check) = &worker.health_check {
if let Some(health_check) = &actor.health_check {
config.insert("test".to_string(), Value::String(health_check.clone()));
}
if !worker.dependencies.is_empty() {
config.insert("after".to_string(), json!(worker.dependencies));
if !actor.dependencies.is_empty() {
config.insert("after".to_string(), json!(actor.dependencies));
}
// Add environment variables if any
if !worker.env.is_empty() {
config.insert("env".to_string(), json!(worker.env));
if !actor.env.is_empty() {
config.insert("env".to_string(), json!(actor.env));
}
config
}
/// Launch workers based on SupervisorBuilder configuration
pub(crate) async fn launch_configured_workers(&self, builder: &crate::SupervisorBuilderData) -> Result<(), SupervisorError> {
/// Launch actors based on SupervisorBuilder configuration
pub(crate) async fn launch_configured_actors(&self, builder: &crate::SupervisorBuilderData) -> Result<(), SupervisorError> {
use hero_job::ScriptType;
use std::path::PathBuf;
let mut errors = Vec::new();
// Launch OSIS worker if configured
if let Some(binary_path) = &builder.osis_worker {
let worker_id = "osis_worker_1";
let mut config = WorkerConfig::new(
worker_id.to_string(),
// Launch OSIS actor if configured
if let Some(binary_path) = &builder.osis_actor {
let actor_id = "osis_actor_1";
let mut config = ActorConfig::new(
actor_id.to_string(),
PathBuf::from(binary_path),
ScriptType::OSIS
);
config.env.extend(builder.worker_env_vars.clone());
config.env.extend(builder.actor_env_vars.clone());
info!("Launching OSIS worker: {}", worker_id);
if let Err(e) = self.start_worker(&config).await {
let error_msg = format!("Failed to start OSIS worker: {}", e);
info!("Launching OSIS actor: {}", actor_id);
if let Err(e) = self.start_actor(&config).await {
let error_msg = format!("Failed to start OSIS actor: {}", e);
warn!("{}", error_msg);
errors.push(error_msg);
}
}
// Launch SAL worker if configured
if let Some(binary_path) = &builder.sal_worker {
let worker_id = "sal_worker_1";
let mut config = WorkerConfig::new(
worker_id.to_string(),
// Launch SAL actor if configured
if let Some(binary_path) = &builder.sal_actor {
let actor_id = "sal_actor_1";
let mut config = ActorConfig::new(
actor_id.to_string(),
PathBuf::from(binary_path),
ScriptType::SAL
);
config.env.extend(builder.worker_env_vars.clone());
config.env.extend(builder.actor_env_vars.clone());
info!("Launching SAL worker: {}", worker_id);
if let Err(e) = self.start_worker(&config).await {
let error_msg = format!("Failed to start SAL worker: {}", e);
info!("Launching SAL actor: {}", actor_id);
if let Err(e) = self.start_actor(&config).await {
let error_msg = format!("Failed to start SAL actor: {}", e);
warn!("{}", error_msg);
errors.push(error_msg);
}
}
// Launch V worker if configured
if let Some(binary_path) = &builder.v_worker {
let worker_id = "v_worker_1";
let mut config = WorkerConfig::new(
worker_id.to_string(),
// Launch V actor if configured
if let Some(binary_path) = &builder.v_actor {
let actor_id = "v_actor_1";
let mut config = ActorConfig::new(
actor_id.to_string(),
PathBuf::from(binary_path),
ScriptType::V
);
config.env.extend(builder.worker_env_vars.clone());
config.env.extend(builder.actor_env_vars.clone());
info!("Launching V worker: {}", worker_id);
if let Err(e) = self.start_worker(&config).await {
let error_msg = format!("Failed to start V worker: {}", e);
info!("Launching V actor: {}", actor_id);
if let Err(e) = self.start_actor(&config).await {
let error_msg = format!("Failed to start V actor: {}", e);
warn!("{}", error_msg);
errors.push(error_msg);
}
}
// Launch Python worker if configured
if let Some(binary_path) = &builder.python_worker {
let worker_id = "python_worker_1";
let mut config = WorkerConfig::new(
worker_id.to_string(),
// Launch Python actor if configured
if let Some(binary_path) = &builder.python_actor {
let actor_id = "python_actor_1";
let mut config = ActorConfig::new(
actor_id.to_string(),
PathBuf::from(binary_path),
ScriptType::Python
);
config.env.extend(builder.worker_env_vars.clone());
config.env.extend(builder.actor_env_vars.clone());
info!("Launching Python worker: {}", worker_id);
if let Err(e) = self.start_worker(&config).await {
let error_msg = format!("Failed to start Python worker: {}", e);
info!("Launching Python actor: {}", actor_id);
if let Err(e) = self.start_actor(&config).await {
let error_msg = format!("Failed to start Python actor: {}", e);
warn!("{}", error_msg);
errors.push(error_msg);
}
}
// Return result based on whether any workers started successfully
// Return result based on whether any actors started successfully
if errors.is_empty() {
info!("All configured workers started successfully");
info!("All configured actors started successfully");
Ok(())
} else {
let combined_error = format!("Some workers failed to start: {}", errors.join("; "));
let combined_error = format!("Some actors failed to start: {}", errors.join("; "));
warn!("{}", combined_error);
Err(SupervisorError::ZinitError(combined_error))
}

View File

@@ -1,2 +0,0 @@
/target
worker_rhai_temp_db

View File

@@ -1,303 +0,0 @@
use hero_job::{Job, JobStatus};
use log::{debug, error, info};
use redis::AsyncCommands;
use rhai::{Dynamic, Engine};
use tokio::sync::mpsc; // For shutdown signal
use tokio::task::JoinHandle;
/// Engine module containing Rhai engine creation and script execution utilities
pub mod engine;
/// Worker trait abstraction for unified worker interface
pub mod worker_trait;
/// Synchronous worker implementation
pub mod sync_worker;
/// Asynchronous worker implementation with trait-based interface
pub mod async_worker_impl;
/// Configuration module for TOML-based worker configuration
pub mod config;
const NAMESPACE_PREFIX: &str = "hero:job:";
const BLPOP_TIMEOUT_SECONDS: usize = 5;
/// Initialize Redis connection for the worker
pub(crate) async fn initialize_redis_connection(
worker_id: &str,
redis_url: &str,
) -> Result<redis::aio::MultiplexedConnection, Box<dyn std::error::Error + Send + Sync>> {
let redis_client = redis::Client::open(redis_url)
.map_err(|e| {
error!("Worker for Worker ID '{}': Failed to open Redis client: {}", worker_id, e);
e
})?;
let redis_conn = redis_client.get_multiplexed_async_connection().await
.map_err(|e| {
error!("Worker for Worker ID '{}': Failed to get Redis connection: {}", worker_id, e);
e
})?;
info!("Worker for Worker ID '{}' successfully connected to Redis.", worker_id);
Ok(redis_conn)
}
/// Load job from Redis using Job struct
pub(crate) async fn load_job_from_redis(
redis_conn: &mut redis::aio::MultiplexedConnection,
job_id: &str,
worker_id: &str,
) -> Result<Job, Box<dyn std::error::Error + Send + Sync>> {
debug!("Worker '{}', Job {}: Loading job from Redis", worker_id, job_id);
match Job::load_from_redis(redis_conn, job_id).await {
Ok(job) => {
debug!("Worker '{}', Job {}: Successfully loaded job", worker_id, job_id);
Ok(job)
}
Err(e) => {
error!("Worker '{}', Job {}: Failed to load job from Redis: {}", worker_id, job_id, e);
Err(Box::new(e))
}
}
}
/// Execute the Rhai script and update job status in Redis
async fn execute_script_and_update_status(
redis_conn: &mut redis::aio::MultiplexedConnection,
engine: &mut Engine,
job: &Job,
db_path: &str,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let mut db_config = rhai::Map::new();
db_config.insert("DB_PATH".into(), db_path.to_string().into());
db_config.insert("CALLER_ID".into(), job.caller_id.clone().into());
db_config.insert("CONTEXT_ID".into(), job.context_id.clone().into());
engine.set_default_tag(Dynamic::from(db_config));
debug!("Worker for Context ID '{}': Evaluating script with Rhai engine.", job.context_id);
match engine.eval::<rhai::Dynamic>(&job.script) {
Ok(result) => {
let output_str = if result.is::<String>() {
result.into_string().unwrap()
} else {
result.to_string()
};
info!("Worker for Context ID '{}' job {} completed. Output: {}", job.context_id, job.id, output_str);
// Update job status to finished and set result
Job::update_status(redis_conn, &job.id, JobStatus::Finished).await
.map_err(|e| {
error!("Failed to update job {} status to finished: {}", job.id, e);
e
})?;
Job::set_result(redis_conn, &job.id, &output_str).await
.map_err(|e| {
error!("Failed to set job {} result: {}", job.id, e);
e
})?;
Ok(())
}
Err(e) => {
let error_str = format!("{:?}", *e);
error!("Worker for Context ID '{}' job {} script evaluation failed. Error: {}", job.context_id, job.id, error_str);
// Update job status to error and set error message
Job::update_status(redis_conn, &job.id, JobStatus::Error).await
.map_err(|e| {
error!("Failed to update job {} status to error: {}", job.id, e);
e
})?;
Job::set_error(redis_conn, &job.id, &error_str).await
.map_err(|e| {
error!("Failed to set job {} error: {}", job.id, e);
e
})?;
Ok(())
}
}
}
/// Clean up job from Redis if preserve_tasks is false
async fn cleanup_job(
redis_conn: &mut redis::aio::MultiplexedConnection,
job_id: &str,
context_id: &str,
preserve_tasks: bool,
) {
if !preserve_tasks {
if let Err(e) = Job::delete_from_redis(redis_conn, job_id).await {
error!("Worker for Context ID '{}', Job {}: Failed to delete job: {}", context_id, job_id, e);
} else {
debug!("Worker for Context ID '{}', Job {}: Cleaned up job.", context_id, job_id);
}
} else {
debug!("Worker for Context ID '{}', Job {}: Preserving job (preserve_tasks=true)", context_id, job_id);
}
}
/// Process a single job from the queue
async fn process_job(
redis_conn: &mut redis::aio::MultiplexedConnection,
job_id: &str,
worker_id: &str,
db_path: &str,
engine: &mut Engine,
preserve_tasks: bool,
) {
debug!("Worker '{}', Job {}: Processing started.", worker_id, job_id);
// Load job from Redis
match load_job_from_redis(redis_conn, job_id, worker_id).await {
Ok(job) => {
info!("Worker '{}' processing job_id: {}. Script: {:.50}...", job.context_id, job_id, job.script);
// Update status to started
debug!("Worker for Context ID '{}', Job {}: Attempting to update status to 'started'.", job.context_id, job_id);
if let Err(e) = Job::update_status(redis_conn, job_id, JobStatus::Started).await {
error!("Worker for Context ID '{}', Job {}: Failed to update status to 'started': {}", job.context_id, job_id, e);
} else {
debug!("Worker for Context ID '{}', Job {}: Status updated to 'started'.", job.context_id, job_id);
}
// Execute the script and update status
if let Err(e) = execute_script_and_update_status(redis_conn, engine, &job, db_path).await {
error!("Worker for Context ID '{}', Job {}: Script execution failed: {}", job.context_id, job_id, e);
// Ensure job status is set to error if execution failed
if let Err(status_err) = Job::update_status(redis_conn, job_id, JobStatus::Error).await {
error!("Worker for Context ID '{}', Job {}: Failed to update status to error after execution failure: {}", job.context_id, job_id, status_err);
}
}
// Clean up job if needed
cleanup_job(redis_conn, job_id, &job.context_id, preserve_tasks).await;
}
Err(e) => {
error!("Worker '{}', Job {}: Failed to load job: {}", worker_id, job_id, e);
// Clean up invalid job if needed
if !preserve_tasks {
if let Err(del_err) = Job::delete_from_redis(redis_conn, job_id).await {
error!("Worker '{}', Job {}: Failed to delete invalid job: {}", worker_id, job_id, del_err);
}
} else {
debug!("Worker '{}', Job {}: Preserving invalid job (preserve_tasks=true)", worker_id, job_id);
}
}
}
}
pub fn spawn_rhai_worker(
worker_id: String,
db_path: String,
mut engine: Engine,
redis_url: String,
mut shutdown_rx: mpsc::Receiver<()>,
preserve_tasks: bool,
) -> JoinHandle<Result<(), Box<dyn std::error::Error + Send + Sync>>> {
tokio::spawn(async move {
let queue_key = format!("{}{}", NAMESPACE_PREFIX, worker_id);
info!(
"Rhai Worker for Worker ID '{}' starting. Connecting to Redis at {}. Listening on queue: {}. Waiting for tasks or shutdown signal.",
worker_id, redis_url, queue_key
);
let mut redis_conn = initialize_redis_connection(&worker_id, &redis_url).await?;
loop {
let blpop_keys = vec![queue_key.clone()];
tokio::select! {
// Listen for shutdown signal
_ = shutdown_rx.recv() => {
info!("Worker for Worker ID '{}': Shutdown signal received. Terminating loop.", worker_id);
break;
}
// Listen for tasks from Redis
blpop_result = redis_conn.blpop(&blpop_keys, BLPOP_TIMEOUT_SECONDS as f64) => {
debug!("Worker for Worker ID '{}': Attempting BLPOP on queue: {}", worker_id, queue_key);
let response: Option<(String, String)> = match blpop_result {
Ok(resp) => resp,
Err(e) => {
error!("Worker '{}': Redis BLPOP error on queue {}: {}. Worker for this circle might stop.", worker_id, queue_key, e);
return Err(Box::new(e) as Box<dyn std::error::Error + Send + Sync>);
}
};
if let Some((_queue_name_recv, job_id)) = response {
info!("Worker '{}' received job_id: {} from queue: {}", worker_id, job_id, _queue_name_recv);
process_job(&mut redis_conn, &job_id, &worker_id, &db_path, &mut engine, preserve_tasks).await;
} else {
debug!("Worker '{}': BLPOP timed out on queue {}. No new tasks. Checking for shutdown signal again.", worker_id, queue_key);
}
}
}
}
info!("Worker '{}' has shut down.", worker_id);
Ok(())
})
}
// Re-export the main trait-based interface for convenience
pub use worker_trait::{Worker, WorkerConfig, spawn_worker};
pub use sync_worker::SyncWorker;
pub use async_worker_impl::AsyncWorker;
/// Convenience function to spawn a synchronous worker using the trait interface
///
/// This function provides backward compatibility with the original sync worker API
/// while using the new trait-based implementation.
pub fn spawn_sync_worker(
worker_id: String,
db_path: String,
engine: rhai::Engine,
redis_url: String,
shutdown_rx: mpsc::Receiver<()>,
preserve_tasks: bool,
) -> JoinHandle<Result<(), Box<dyn std::error::Error + Send + Sync>>> {
use std::sync::Arc;
let worker = Arc::new(
SyncWorker::builder()
.worker_id(worker_id)
.db_path(db_path)
.redis_url(redis_url)
.preserve_tasks(preserve_tasks)
.build()
.expect("Failed to build SyncWorker")
);
spawn_worker(worker, engine, shutdown_rx)
}
/// Convenience function to spawn an asynchronous worker using the trait interface
///
/// This function provides a clean interface for the new async worker implementation
/// with timeout support.
pub fn spawn_async_worker(
worker_id: String,
db_path: String,
engine: rhai::Engine,
redis_url: String,
shutdown_rx: mpsc::Receiver<()>,
default_timeout: std::time::Duration,
) -> JoinHandle<Result<(), Box<dyn std::error::Error + Send + Sync>>> {
use std::sync::Arc;
let worker = Arc::new(
AsyncWorker::builder()
.worker_id(worker_id)
.db_path(db_path)
.redis_url(redis_url)
.default_timeout(default_timeout)
.build()
.expect("Failed to build AsyncWorker")
);
spawn_worker(worker, engine, shutdown_rx)
}

View File

@@ -1,339 +0,0 @@
//! # Worker Trait Abstraction
//!
//! This module provides a trait-based abstraction for Rhai workers that eliminates
//! code duplication between synchronous and asynchronous worker implementations.
//!
//! The `Worker` trait defines the common interface and behavior, while specific
//! implementations handle job processing differently (sync vs async).
//!
//! ## Architecture
//!
//! ```text
//! ┌─────────────────┐ ┌─────────────────┐
//! │ SyncWorker │ │ AsyncWorker │
//! │ │ │ │
//! │ process_job() │ │ process_job() │
//! │ (sequential) │ │ (concurrent) │
//! └─────────────────┘ └─────────────────┘
//! │ │
//! └───────┬───────────────┘
//! │
//! ┌───────▼───────┐
//! │ Worker Trait │
//! │ │
//! │ spawn() │
//! │ config │
//! │ common loop │
//! └───────────────┘
//! ```
use hero_job::Job;
use log::{debug, error, info};
use redis::AsyncCommands;
use rhai::Engine;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::mpsc;
use tokio::task::JoinHandle;
use crate::{initialize_redis_connection, NAMESPACE_PREFIX, BLPOP_TIMEOUT_SECONDS};
/// Configuration for worker instances
#[derive(Debug, Clone)]
pub struct WorkerConfig {
pub worker_id: String,
pub db_path: String,
pub redis_url: String,
pub preserve_tasks: bool,
pub default_timeout: Option<Duration>, // Only used by async workers
}
impl WorkerConfig {
/// Create a new worker configuration
pub fn new(
worker_id: String,
db_path: String,
redis_url: String,
preserve_tasks: bool,
) -> Self {
Self {
worker_id,
db_path,
redis_url,
preserve_tasks,
default_timeout: None,
}
}
/// Set default timeout for async workers
pub fn with_default_timeout(mut self, timeout: Duration) -> Self {
self.default_timeout = Some(timeout);
self
}
}
/// Trait defining the common interface for Rhai workers
///
/// This trait abstracts the common functionality between synchronous and
/// asynchronous workers, allowing them to share the same spawn logic and
/// Redis polling loop while implementing different job processing strategies.
#[async_trait::async_trait]
pub trait Worker: Send + Sync + 'static {
/// Process a single job
///
/// This is the core method that differentiates worker implementations:
/// - Sync workers process jobs sequentially, one at a time
/// - Async workers spawn concurrent tasks for each job
///
/// # Arguments
///
/// * `job` - The job to process
/// * `engine` - Rhai engine for script execution
/// * `redis_conn` - Redis connection for status updates
async fn process_job(
&self,
job: Job,
engine: Engine,
redis_conn: &mut redis::aio::MultiplexedConnection,
);
/// Get the worker type name for logging
fn worker_type(&self) -> &'static str;
/// Get worker ID for this worker instance
fn worker_id(&self) -> &str;
/// Get Redis URL for this worker instance
fn redis_url(&self) -> &str;
/// Spawn the worker
///
/// This method provides the common worker loop implementation that both
/// sync and async workers can use. It handles:
/// - Redis connection setup
/// - Job polling from Redis queue
/// - Shutdown signal handling
/// - Delegating job processing to the implementation
fn spawn(
self: Arc<Self>,
engine: Engine,
mut shutdown_rx: mpsc::Receiver<()>,
) -> JoinHandle<Result<(), Box<dyn std::error::Error + Send + Sync>>> {
tokio::spawn(async move {
let worker_id = self.worker_id();
let redis_url = self.redis_url();
let queue_key = format!("{}{}", NAMESPACE_PREFIX, worker_id);
info!(
"{} Worker '{}' starting. Connecting to Redis at {}. Listening on queue: {}",
self.worker_type(),
worker_id,
redis_url,
queue_key
);
let mut redis_conn = initialize_redis_connection(worker_id, redis_url).await?;
loop {
let blpop_keys = vec![queue_key.clone()];
tokio::select! {
// Listen for shutdown signal
_ = shutdown_rx.recv() => {
info!("{} Worker '{}': Shutdown signal received. Terminating loop.",
self.worker_type(), worker_id);
break;
}
// Listen for tasks from Redis
blpop_result = redis_conn.blpop(&blpop_keys, BLPOP_TIMEOUT_SECONDS as f64) => {
debug!("{} Worker '{}': Attempting BLPOP on queue: {}",
self.worker_type(), worker_id, queue_key);
let response: Option<(String, String)> = match blpop_result {
Ok(resp) => resp,
Err(e) => {
error!("{} Worker '{}': Redis BLPOP error on queue {}: {}. Worker for this circle might stop.",
self.worker_type(), worker_id, queue_key, e);
return Err(Box::new(e) as Box<dyn std::error::Error + Send + Sync>);
}
};
if let Some((_queue_name_recv, job_id)) = response {
info!("{} Worker '{}' received job_id: {} from queue: {}",
self.worker_type(), worker_id, job_id, _queue_name_recv);
// Load the job from Redis
match crate::load_job_from_redis(&mut redis_conn, &job_id, worker_id).await {
Ok(mut job) => {
// Check for ping job and handle it directly
if job.script.trim() == "ping" {
info!("{} Worker '{}': Received ping job '{}', responding with pong",
self.worker_type(), worker_id, job_id);
// Update job status to started
if let Err(e) = hero_job::Job::update_status(&mut redis_conn, &job_id, hero_job::JobStatus::Started).await {
error!("{} Worker '{}': Failed to update ping job '{}' status to Started: {}",
self.worker_type(), worker_id, job_id, e);
}
// Set result to "pong" and mark as finished
if let Err(e) = hero_job::Job::set_result(&mut redis_conn, &job_id, "pong").await {
error!("{} Worker '{}': Failed to set ping job '{}' result: {}",
self.worker_type(), worker_id, job_id, e);
}
info!("{} Worker '{}': Successfully responded to ping job '{}' with pong",
self.worker_type(), worker_id, job_id);
} else {
// Create a new engine for each job to avoid sharing state
let job_engine = crate::engine::create_heromodels_engine();
// Delegate job processing to the implementation
self.process_job(job, job_engine, &mut redis_conn).await;
}
}
Err(e) => {
error!("{} Worker '{}': Failed to load job '{}': {}",
self.worker_type(), worker_id, job_id, e);
}
}
} else {
debug!("{} Worker '{}': BLPOP timed out on queue {}. No new tasks.",
self.worker_type(), worker_id, queue_key);
}
}
}
}
info!("{} Worker '{}' has shut down.", self.worker_type(), worker_id);
Ok(())
})
}
}
/// Convenience function to spawn a worker with the trait-based interface
///
/// This function provides a unified interface for spawning any worker implementation
/// that implements the Worker trait.
///
/// # Arguments
///
/// * `worker` - The worker implementation to spawn
/// * `config` - Worker configuration
/// * `engine` - Rhai engine for script execution
/// * `shutdown_rx` - Channel receiver for shutdown signals
///
/// # Returns
///
/// Returns a `JoinHandle` that can be awaited to wait for worker shutdown.
///
/// # Example
///
/// ```rust
/// use std::sync::Arc;
/// use std::time::Duration;
///
/// let config = WorkerConfig::new(
/// "worker_1".to_string(),
/// "/path/to/db".to_string(),
/// "redis://localhost:6379".to_string(),
/// false,
/// );
///
/// let worker = Arc::new(SyncWorker::new());
/// let engine = create_heromodels_engine();
/// let (shutdown_tx, shutdown_rx) = mpsc::channel(1);
///
/// let handle = spawn_worker(worker, config, engine, shutdown_rx);
///
/// // Later, shutdown the worker
/// shutdown_tx.send(()).await.unwrap();
/// handle.await.unwrap().unwrap();
/// ```
pub fn spawn_worker<W: Worker>(
worker: Arc<W>,
engine: Engine,
shutdown_rx: mpsc::Receiver<()>,
) -> JoinHandle<Result<(), Box<dyn std::error::Error + Send + Sync>>> {
worker.spawn(engine, shutdown_rx)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::engine::create_heromodels_engine;
// Mock worker for testing
struct MockWorker;
#[async_trait::async_trait]
impl Worker for MockWorker {
async fn process_job(
&self,
_job: Job,
_engine: Engine,
_redis_conn: &mut redis::aio::MultiplexedConnection,
) {
// Mock implementation - do nothing
}
fn worker_type(&self) -> &'static str {
"Mock"
}
fn worker_id(&self) -> &str {
"mock_worker"
}
fn redis_url(&self) -> &str {
"redis://localhost:6379"
}
}
#[tokio::test]
async fn test_worker_config_creation() {
let config = WorkerConfig::new(
"test_worker".to_string(),
"/tmp".to_string(),
"redis://localhost:6379".to_string(),
false,
);
assert_eq!(config.worker_id, "test_worker");
assert_eq!(config.db_path, "/tmp");
assert_eq!(config.redis_url, "redis://localhost:6379");
assert!(!config.preserve_tasks);
assert!(config.default_timeout.is_none());
}
#[tokio::test]
async fn test_worker_config_with_timeout() {
let timeout = Duration::from_secs(300);
let config = WorkerConfig::new(
"test_worker".to_string(),
"/tmp".to_string(),
"redis://localhost:6379".to_string(),
false,
).with_default_timeout(timeout);
assert_eq!(config.default_timeout, Some(timeout));
}
#[tokio::test]
async fn test_spawn_worker_function() {
let (_shutdown_tx, shutdown_rx) = mpsc::channel(1);
let config = WorkerConfig::new(
"test_worker".to_string(),
"/tmp".to_string(),
"redis://localhost:6379".to_string(),
false,
);
let engine = create_heromodels_engine();
let worker = Arc::new(MockWorker);
let handle = spawn_worker(worker, config, engine, shutdown_rx);
// The worker should be created successfully
assert!(!handle.is_finished());
// Abort the worker for cleanup
handle.abort();
}
}

View File

@@ -265,11 +265,11 @@
"params": [],
"result": {
"name": "jobList",
"description": "List of all jobs.",
"description": "List of all job IDs.",
"schema": {
"type": "array",
"items": {
"$ref": "#/components/schemas/Job"
"type": "string"
}
}
}
@@ -343,7 +343,7 @@
},
"ScriptType": {
"type": "string",
"enum": ["HeroScript", "RhaiSAL", "RhaiDSL"],
"enum": ["OSIS", "SAL", "V", "Python"],
"description": "The type of script to execute."
},
"JobStatus": {

View File

@@ -0,0 +1,42 @@
[package]
name = "hero-openrpc-client"
version = "0.1.0"
edition = "2021"
[[bin]]
name = "hero-openrpc-client"
path = "cmd/main.rs"
[dependencies]
# Core dependencies
tokio = { version = "1.0", features = ["full"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
anyhow = "1.0"
thiserror = "1.0"
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
clap = { version = "4.0", features = ["derive"] }
# JSON-RPC dependencies
jsonrpsee = { version = "0.21", features = [
"client",
"macros"
] }
async-trait = "0.1"
# Hero dependencies
hero_job = { path = "../../../core/job" }
# Authentication and crypto
secp256k1 = { version = "0.28", features = ["rand", "recovery"] }
hex = "0.4"
sha2 = "0.10"
rand = "0.8"
# CLI utilities
dialoguer = "0.11"
colored = "2.0"
# Async utilities
futures = "0.3"

View File

@@ -0,0 +1,489 @@
use anyhow::Result;
use clap::{Parser, Subcommand};
use colored::*;
use dialoguer::{Input, Select, Confirm, MultiSelect};
use hero_job::ScriptType;
use hero_openrpc_client::{
AuthHelper, ClientTransport, HeroOpenRpcClient, JobParams,
};
use std::path::PathBuf;
use tracing::{error, info, Level};
use tracing_subscriber;
#[derive(Parser)]
#[command(name = "hero-openrpc-client")]
#[command(about = "Hero OpenRPC Client - Interactive JSON-RPC client")]
struct Cli {
#[command(subcommand)]
command: Commands,
/// Private key for authentication (hex format)
#[arg(long)]
private_key: Option<String>,
/// Generate a new private key and exit
#[arg(long)]
generate_key: bool,
/// Log level
#[arg(long, default_value = "info")]
log_level: String,
}
#[derive(Subcommand)]
enum Commands {
/// Connect to WebSocket server
Websocket {
/// Server URL
#[arg(long, default_value = "ws://127.0.0.1:9944")]
url: String,
},
}
/// Available RPC methods with descriptions
#[derive(Debug, Clone)]
struct RpcMethod {
name: &'static str,
description: &'static str,
requires_auth: bool,
}
const RPC_METHODS: &[RpcMethod] = &[
RpcMethod {
name: "fetch_nonce",
description: "Fetch a nonce for authentication",
requires_auth: false,
},
RpcMethod {
name: "authenticate",
description: "Authenticate with public key and signature",
requires_auth: false,
},
RpcMethod {
name: "whoami",
description: "Get authentication status and user information",
requires_auth: true,
},
RpcMethod {
name: "play",
description: "Execute a Rhai script immediately",
requires_auth: true,
},
RpcMethod {
name: "create_job",
description: "Create a new job without starting it",
requires_auth: true,
},
RpcMethod {
name: "start_job",
description: "Start a previously created job",
requires_auth: true,
},
RpcMethod {
name: "run_job",
description: "Create and run a job, returning result when complete",
requires_auth: true,
},
RpcMethod {
name: "get_job_status",
description: "Get the current status of a job",
requires_auth: true,
},
RpcMethod {
name: "get_job_output",
description: "Get the output of a completed job",
requires_auth: true,
},
RpcMethod {
name: "get_job_logs",
description: "Get the logs of a job",
requires_auth: true,
},
RpcMethod {
name: "list_jobs",
description: "List all jobs in the system",
requires_auth: true,
},
RpcMethod {
name: "stop_job",
description: "Stop a running job",
requires_auth: true,
},
RpcMethod {
name: "delete_job",
description: "Delete a job from the system",
requires_auth: true,
},
RpcMethod {
name: "clear_all_jobs",
description: "Clear all jobs from the system",
requires_auth: true,
},
];
#[tokio::main]
async fn main() -> Result<()> {
let cli = Cli::parse();
// Initialize tracing
let log_level = match cli.log_level.to_lowercase().as_str() {
"trace" => Level::TRACE,
"debug" => Level::DEBUG,
"info" => Level::INFO,
"warn" => Level::WARN,
"error" => Level::ERROR,
_ => Level::INFO,
};
tracing_subscriber::fmt()
.with_max_level(log_level)
.init();
// Handle key generation
if cli.generate_key {
let auth_helper = AuthHelper::generate()?;
println!("{}", "Generated new private key:".green().bold());
println!("Private Key: {}", auth_helper.private_key_hex().yellow());
println!("Public Key: {}", auth_helper.public_key_hex().cyan());
println!();
println!("{}", "Save the private key securely and use it with --private-key".bright_yellow());
return Ok(());
}
let transport = match cli.command {
Commands::Websocket { url } => {
println!("{} {}", "Connecting to WebSocket server:".green(), url.cyan());
ClientTransport::WebSocket(url)
}
};
// Connect to the server
let client = HeroOpenRpcClient::connect(transport).await?;
println!("{}", "Connected successfully!".green().bold());
// Handle authentication if private key is provided
let mut authenticated = false;
if let Some(private_key) = cli.private_key {
println!("{}", "Authenticating...".yellow());
match client.authenticate_with_key(&private_key).await {
Ok(true) => {
println!("{}", "Authentication successful!".green().bold());
authenticated = true;
}
Ok(false) => {
println!("{}", "Authentication failed!".red().bold());
}
Err(e) => {
error!("Authentication error: {}", e);
println!("{} {}", "Authentication error:".red().bold(), e);
}
}
} else {
println!("{}", "No private key provided. Some methods will require authentication.".yellow());
println!("{}", "Use --generate-key to create a new key or --private-key to use an existing one.".bright_yellow());
}
println!();
// Interactive loop
loop {
// Filter methods based on authentication status
let available_methods: Vec<&RpcMethod> = RPC_METHODS
.iter()
.filter(|method| !method.requires_auth || authenticated)
.collect();
if available_methods.is_empty() {
println!("{}", "No methods available. Please authenticate first.".red());
break;
}
// Display method selection
let method_names: Vec<String> = available_methods
.iter()
.map(|method| {
if method.requires_auth && !authenticated {
format!("{} {} (requires auth)", method.name.red(), method.description)
} else {
format!("{} {}", method.name.green(), method.description)
}
})
.collect();
let selection = Select::new()
.with_prompt("Select an RPC method to call")
.items(&method_names)
.default(0)
.interact_opt()?;
let Some(selection) = selection else {
println!("{}", "Goodbye!".cyan());
break;
};
let selected_method = available_methods[selection];
println!();
println!("{} {}", "Selected method:".bold(), selected_method.name.green());
// Handle method-specific parameter collection and execution
match execute_method(&client, selected_method.name).await {
Ok(_) => {}
Err(e) => {
error!("Method execution failed: {}", e);
println!("{} {}", "Error:".red().bold(), e);
}
}
println!();
// Ask if user wants to continue
if !Confirm::new()
.with_prompt("Do you want to call another method?")
.default(true)
.interact()?
{
break;
}
println!();
}
println!("{}", "Goodbye!".cyan().bold());
Ok(())
}
async fn execute_method(client: &HeroOpenRpcClient, method_name: &str) -> Result<()> {
match method_name {
"fetch_nonce" => {
let pubkey: String = Input::new()
.with_prompt("Public key (hex)")
.interact_text()?;
let result = client.fetch_nonce(pubkey).await?;
println!("{} {}", "Nonce:".green().bold(), result.yellow());
}
"authenticate" => {
let pubkey: String = Input::new()
.with_prompt("Public key (hex)")
.interact_text()?;
let signature: String = Input::new()
.with_prompt("Signature (hex)")
.interact_text()?;
let nonce: String = Input::new()
.with_prompt("Nonce (hex) - fetch via fetch_nonce first")
.interact_text()?;
let result = client.authenticate(pubkey, signature, nonce).await?;
println!("{} {}", "Authentication result:".green().bold(),
if result { "Success".green() } else { "Failed".red() });
}
"whoami" => {
let result = client.whoami().await?;
println!("{} {}", "User info:".green().bold(), result.cyan());
}
"play" => {
let script: String = Input::new()
.with_prompt("Rhai script to execute")
.interact_text()?;
let result = client.play(script).await?;
println!("{} {}", "Script output:".green().bold(), result.output.cyan());
}
"create_job" => {
let script: String = Input::new()
.with_prompt("Script content")
.interact_text()?;
let script_types = ["OSIS", "SAL", "V", "Python"];
let script_type_selection = Select::new()
.with_prompt("Script type")
.items(&script_types)
.default(0)
.interact()?;
let script_type = match script_type_selection {
0 => ScriptType::OSIS,
1 => ScriptType::SAL,
2 => ScriptType::V,
_ => ScriptType::Python,
};
let add_prerequisites = Confirm::new()
.with_prompt("Add prerequisites?")
.default(false)
.interact()?;
let prerequisites = if add_prerequisites {
let prereq_input: String = Input::new()
.with_prompt("Prerequisites (comma-separated job IDs)")
.interact_text()?;
Some(prereq_input.split(',').map(|s| s.trim().to_string()).collect())
} else {
None
};
let caller_id: String = Input::new()
.with_prompt("Caller ID")
.interact_text()?;
let context_id: String = Input::new()
.with_prompt("Context ID")
.interact_text()?;
let specify_timeout = Confirm::new()
.with_prompt("Specify timeout (seconds)?")
.default(false)
.interact()?;
let timeout = if specify_timeout {
let t: u64 = Input::new()
.with_prompt("Timeout (seconds)")
.interact_text()?;
Some(t)
} else {
None
};
let job_params = JobParams {
script,
script_type,
caller_id,
context_id,
timeout,
prerequisites,
};
let result = client.create_job(job_params).await?;
println!("{} {}", "Created job ID:".green().bold(), result.yellow());
}
"start_job" => {
let job_id: String = Input::new()
.with_prompt("Job ID to start")
.interact_text()?;
let result = client.start_job(job_id).await?;
println!("{} {}", "Start result:".green().bold(),
if result.success { "Success".green() } else { "Failed".red() });
}
"run_job" => {
let script: String = Input::new()
.with_prompt("Script content")
.interact_text()?;
let script_types = ["OSIS", "SAL", "V", "Python"];
let script_type_selection = Select::new()
.with_prompt("Script type")
.items(&script_types)
.default(0)
.interact()?;
let script_type = match script_type_selection {
0 => ScriptType::OSIS,
1 => ScriptType::SAL,
2 => ScriptType::V,
_ => ScriptType::Python,
};
let add_prerequisites = Confirm::new()
.with_prompt("Add prerequisites?")
.default(false)
.interact()?;
let prerequisites = if add_prerequisites {
let prereq_input: String = Input::new()
.with_prompt("Prerequisites (comma-separated job IDs)")
.interact_text()?;
Some(prereq_input.split(',').map(|s| s.trim().to_string()).collect())
} else {
None
};
let result = client.run_job(script, script_type, prerequisites).await?;
println!("{} {}", "Job result:".green().bold(), result.cyan());
}
"get_job_status" => {
let job_id: String = Input::new()
.with_prompt("Job ID")
.interact_text()?;
let result = client.get_job_status(job_id).await?;
println!("{} {:?}", "Job status:".green().bold(), result);
}
"get_job_output" => {
let job_id: String = Input::new()
.with_prompt("Job ID")
.interact_text()?;
let result = client.get_job_output(job_id).await?;
println!("{} {}", "Job output:".green().bold(), result.cyan());
}
"get_job_logs" => {
let job_id: String = Input::new()
.with_prompt("Job ID")
.interact_text()?;
let result = client.get_job_logs(job_id).await?;
match result.logs {
Some(logs) => println!("{} {}", "Job logs:".green().bold(), logs.cyan()),
None => println!("{} {}", "Job logs:".green().bold(), "(no logs)".yellow()),
}
}
"list_jobs" => {
let result = client.list_jobs().await?;
println!("{}", "Job IDs:".green().bold());
for id in result {
println!(" {}", id.yellow());
}
}
"stop_job" => {
let job_id: String = Input::new()
.with_prompt("Job ID to stop")
.interact_text()?;
client.stop_job(job_id.clone()).await?;
println!("{} {}", "Stopped job:".green().bold(), job_id.yellow());
}
"delete_job" => {
let job_id: String = Input::new()
.with_prompt("Job ID to delete")
.interact_text()?;
client.delete_job(job_id.clone()).await?;
println!("{} {}", "Deleted job:".green().bold(), job_id.yellow());
}
"clear_all_jobs" => {
let confirm = Confirm::new()
.with_prompt("Are you sure you want to clear ALL jobs?")
.default(false)
.interact()?;
if confirm {
client.clear_all_jobs().await?;
println!("{}", "Cleared all jobs".green().bold());
} else {
println!("{}", "Operation cancelled".yellow());
}
}
_ => {
println!("{} {}", "Unknown method:".red().bold(), method_name);
}
}
Ok(())
}

View File

@@ -0,0 +1,81 @@
use anyhow::{anyhow, Result};
use secp256k1::{Message, PublicKey, ecdsa::Signature, Secp256k1, SecretKey};
use sha2::{Digest, Sha256};
/// Helper for authentication operations
pub struct AuthHelper {
secret_key: SecretKey,
public_key: PublicKey,
secp: Secp256k1<secp256k1::All>,
}
impl AuthHelper {
/// Create a new auth helper from a private key hex string
pub fn new(private_key_hex: &str) -> Result<Self> {
let secp = Secp256k1::new();
let secret_key_bytes = hex::decode(private_key_hex)
.map_err(|_| anyhow!("Invalid private key hex format"))?;
let secret_key = SecretKey::from_slice(&secret_key_bytes)
.map_err(|_| anyhow!("Invalid private key"))?;
let public_key = PublicKey::from_secret_key(&secp, &secret_key);
Ok(Self {
secret_key,
public_key,
secp,
})
}
/// Generate a new random private key
pub fn generate() -> Result<Self> {
let secp = Secp256k1::new();
let (secret_key, public_key) = secp.generate_keypair(&mut rand::thread_rng());
Ok(Self {
secret_key,
public_key,
secp,
})
}
/// Get the public key as a hex string
pub fn public_key_hex(&self) -> String {
hex::encode(self.public_key.serialize())
}
/// Get the private key as a hex string
pub fn private_key_hex(&self) -> String {
hex::encode(self.secret_key.secret_bytes())
}
/// Sign a message and return the signature as hex
pub fn sign_message(&self, message: &str) -> Result<String> {
let message_hash = Sha256::digest(message.as_bytes());
let message = Message::from_slice(&message_hash)
.map_err(|_| anyhow!("Failed to create message from hash"))?;
let signature = self.secp.sign_ecdsa(&message, &self.secret_key);
Ok(hex::encode(signature.serialize_compact()))
}
/// Verify a signature against a message
pub fn verify_signature(&self, message: &str, signature_hex: &str) -> Result<bool> {
let message_hash = Sha256::digest(message.as_bytes());
let message = Message::from_slice(&message_hash)
.map_err(|_| anyhow!("Failed to create message from hash"))?;
let signature_bytes = hex::decode(signature_hex)
.map_err(|_| anyhow!("Invalid signature hex format"))?;
let signature = Signature::from_compact(&signature_bytes)
.map_err(|_| anyhow!("Invalid signature format"))?;
match self.secp.verify_ecdsa(&message, &signature, &self.public_key) {
Ok(_) => Ok(true),
Err(_) => Ok(false),
}
}
}

View File

@@ -0,0 +1,212 @@
use anyhow::Result;
use async_trait::async_trait;
use hero_job::{JobStatus, ScriptType};
use jsonrpsee::core::client::ClientT;
use jsonrpsee::core::ClientError;
use jsonrpsee::proc_macros::rpc;
use jsonrpsee::rpc_params;
use jsonrpsee::ws_client::{WsClient, WsClientBuilder};
use std::path::PathBuf;
use tracing::{error, info};
mod auth;
mod types;
pub use auth::*;
pub use types::*;
/// Transport configuration for the client
#[derive(Debug, Clone)]
pub enum ClientTransport {
WebSocket(String),
}
/// OpenRPC client trait defining all available methods
#[rpc(client)]
pub trait OpenRpcClient {
// Authentication methods
#[method(name = "fetch_nonce")]
async fn fetch_nonce(&self, pubkey: String) -> Result<String, ClientError>;
#[method(name = "authenticate")]
async fn authenticate(
&self,
pubkey: String,
signature: String,
nonce: String,
) -> Result<bool, ClientError>;
#[method(name = "whoami")]
async fn whoami(&self) -> Result<String, ClientError>;
// Script execution
#[method(name = "play")]
async fn play(&self, script: String) -> Result<PlayResult, ClientError>;
// Job management
#[method(name = "create_job")]
async fn create_job(&self, job: JobParams) -> Result<String, ClientError>;
#[method(name = "start_job")]
async fn start_job(&self, job_id: String) -> Result<StartJobResult, ClientError>;
#[method(name = "run_job")]
async fn run_job(
&self,
script: String,
script_type: ScriptType,
prerequisites: Option<Vec<String>>,
) -> Result<String, ClientError>;
#[method(name = "get_job_status")]
async fn get_job_status(&self, job_id: String) -> Result<JobStatus, ClientError>;
#[method(name = "get_job_output")]
async fn get_job_output(&self, job_id: String) -> Result<String, ClientError>;
#[method(name = "get_job_logs")]
async fn get_job_logs(&self, job_id: String) -> Result<JobLogsResult, ClientError>;
#[method(name = "list_jobs")]
async fn list_jobs(&self) -> Result<Vec<String>, ClientError>;
#[method(name = "stop_job")]
async fn stop_job(&self, job_id: String) -> Result<(), ClientError>;
#[method(name = "delete_job")]
async fn delete_job(&self, job_id: String) -> Result<(), ClientError>;
#[method(name = "clear_all_jobs")]
async fn clear_all_jobs(&self) -> Result<(), ClientError>;
}
/// Wrapper client that can use WebSocket transport
pub struct HeroOpenRpcClient {
client: WsClient,
}
impl HeroOpenRpcClient {
/// Connect to the OpenRPC server using the specified transport
pub async fn connect(transport: ClientTransport) -> Result<Self> {
match transport {
ClientTransport::WebSocket(url) => {
info!("Connecting to WebSocket server at {}", url);
let client = WsClientBuilder::default()
.build(&url)
.await?;
Ok(Self { client })
}
}
}
/// Get the underlying client for making RPC calls
pub fn client(&self) -> &WsClient {
&self.client
}
/// Authenticate with the server using a private key
pub async fn authenticate_with_key(&self, private_key: &str) -> Result<bool> {
let auth_helper = AuthHelper::new(private_key)?;
// Get nonce
let pubkey = auth_helper.public_key_hex();
let nonce: String = self.client.fetch_nonce(pubkey.clone()).await?;
// Sign nonce
let signature = auth_helper.sign_message(&nonce)?;
// Authenticate
let result = self.client.authenticate(pubkey, signature, nonce).await?;
if result {
info!("Authentication successful");
} else {
error!("Authentication failed");
}
Ok(result)
}
}
// Implement delegation methods on HeroOpenRpcClient to use the generated trait methods
impl HeroOpenRpcClient {
/// Delegate to fetch_nonce on the underlying client
pub async fn fetch_nonce(&self, pubkey: String) -> Result<String, ClientError> {
self.client.fetch_nonce(pubkey).await
}
/// Delegate to authenticate on the underlying client
pub async fn authenticate(
&self,
pubkey: String,
signature: String,
nonce: String,
) -> Result<bool, ClientError> {
self.client.authenticate(pubkey, signature, nonce).await
}
/// Delegate to whoami on the underlying client
pub async fn whoami(&self) -> Result<String, ClientError> {
self.client.whoami().await
}
/// Delegate to play on the underlying client
pub async fn play(&self, script: String) -> Result<PlayResult, ClientError> {
self.client.play(script).await
}
/// Delegate to create_job on the underlying client
pub async fn create_job(&self, job: JobParams) -> Result<String, ClientError> {
self.client.create_job(job).await
}
/// Delegate to start_job on the underlying client
pub async fn start_job(&self, job_id: String) -> Result<StartJobResult, ClientError> {
self.client.start_job(job_id).await
}
/// Delegate to run_job on the underlying client
pub async fn run_job(
&self,
script: String,
script_type: ScriptType,
prerequisites: Option<Vec<String>>,
) -> Result<String, ClientError> {
self.client.run_job(script, script_type, prerequisites).await
}
/// Delegate to get_job_status on the underlying client
pub async fn get_job_status(&self, job_id: String) -> Result<JobStatus, ClientError> {
self.client.get_job_status(job_id).await
}
/// Delegate to get_job_output on the underlying client
pub async fn get_job_output(&self, job_id: String) -> Result<String, ClientError> {
self.client.get_job_output(job_id).await
}
/// Delegate to get_job_logs on the underlying client
pub async fn get_job_logs(&self, job_id: String) -> Result<JobLogsResult, ClientError> {
self.client.get_job_logs(job_id).await
}
/// Delegate to list_jobs on the underlying client
pub async fn list_jobs(&self) -> Result<Vec<String>, ClientError> {
self.client.list_jobs().await
}
/// Delegate to stop_job on the underlying client
pub async fn stop_job(&self, job_id: String) -> Result<(), ClientError> {
self.client.stop_job(job_id).await
}
/// Delegate to delete_job on the underlying client
pub async fn delete_job(&self, job_id: String) -> Result<(), ClientError> {
self.client.delete_job(job_id).await
}
/// Delegate to clear_all_jobs on the underlying client
pub async fn clear_all_jobs(&self) -> Result<(), ClientError> {
self.client.clear_all_jobs().await
}
}

View File

@@ -0,0 +1,31 @@
use hero_job::ScriptType;
use serde::{Deserialize, Serialize};
/** Parameters for creating a job (must mirror server DTO) */
#[derive(Debug, Serialize, Deserialize)]
pub struct JobParams {
pub script: String,
pub script_type: ScriptType,
pub caller_id: String,
pub context_id: String,
pub timeout: Option<u64>, // seconds
pub prerequisites: Option<Vec<String>>,
}
/// Result of script execution
#[derive(Debug, Serialize, Deserialize)]
pub struct PlayResult {
pub output: String,
}
/// Result of starting a job
#[derive(Debug, Serialize, Deserialize)]
pub struct StartJobResult {
pub success: bool,
}
/** Result of getting job logs */
#[derive(Debug, Serialize, Deserialize)]
pub struct JobLogsResult {
pub logs: Option<String>,
}

View File

@@ -0,0 +1,44 @@
[package]
name = "hero-openrpc-server"
version = "0.1.0"
edition = "2021"
[[bin]]
name = "hero-openrpc-server"
path = "cmd/main.rs"
[dependencies]
# Core dependencies
tokio = { version = "1.0", features = ["full"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
anyhow = "1.0"
thiserror = "1.0"
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
clap = { version = "4.0", features = ["derive"] }
# JSON-RPC dependencies
jsonrpsee = { version = "0.21", features = ["server", "macros"] }
jsonrpsee-types = "0.21"
uuid = { version = "1.6", features = ["v4", "serde"] }
chrono = { version = "0.4", features = ["serde"] }
# Hero dependencies
hero_supervisor = { path = "../../../core/supervisor" }
hero_job = { path = "../../../core/job" }
# Authentication and crypto
secp256k1 = { version = "0.28", features = ["rand", "recovery"] }
hex = "0.4"
sha2 = "0.10"
rand = "0.8"
# Async utilities
futures = "0.3"
# Test dependencies
[dev-dependencies]
tokio-test = "0.4"
uuid = { version = "1.6", features = ["v4"] }

View File

@@ -0,0 +1,81 @@
use anyhow::Result;
use clap::{Parser, Subcommand};
use hero_openrpc_server::{OpenRpcServer, OpenRpcServerConfig, Transport};
use std::net::SocketAddr;
use std::path::PathBuf;
use tracing::{info, Level};
use tracing_subscriber;
#[derive(Parser)]
#[command(name = "hero-openrpc-server")]
#[command(about = "Hero OpenRPC Server - JSON-RPC over HTTP/WS")]
struct Cli {
#[command(subcommand)]
command: Commands,
/// Path to supervisor configuration file
#[arg(long)]
supervisor_config: Option<PathBuf>,
/// Database path for supervisor
#[arg(long, default_value = "./supervisor.db")]
db_path: PathBuf,
/// Log level
#[arg(long, default_value = "info")]
log_level: String,
}
#[derive(Subcommand)]
enum Commands {
/// Start WebSocket server
Websocket {
/// Address to bind to
#[arg(long, default_value = "127.0.0.1:9944")]
addr: SocketAddr,
},
}
#[tokio::main]
async fn main() -> Result<()> {
let cli = Cli::parse();
// Initialize tracing
let log_level = match cli.log_level.to_lowercase().as_str() {
"trace" => Level::TRACE,
"debug" => Level::DEBUG,
"info" => Level::INFO,
"warn" => Level::WARN,
"error" => Level::ERROR,
_ => Level::INFO,
};
tracing_subscriber::fmt()
.with_max_level(log_level)
.init();
let transport = match cli.command {
Commands::Websocket { addr } => {
info!("Starting WebSocket server on {}", addr);
Transport::WebSocket(addr)
}
};
let config = OpenRpcServerConfig {
transport: transport.clone(),
supervisor_config_path: cli.supervisor_config,
db_path: cli.db_path,
};
// Create and start the server
let server = OpenRpcServer::new(config.clone()).await?;
let handle = server.start(config).await?;
info!("Server started successfully");
// Wait for the server to finish
handle.stopped().await;
info!("Server stopped");
Ok(())
}

View File

@@ -0,0 +1,131 @@
use anyhow::{anyhow, Result};
use secp256k1::{Message, PublicKey, Secp256k1, ecdsa::Signature};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use std::time::{SystemTime, UNIX_EPOCH};
/// Nonce response structure
#[derive(Debug, Serialize, Deserialize)]
pub struct NonceResponse {
pub nonce: String,
pub timestamp: u64,
}
/// Authentication manager for handling nonces and signature verification
#[derive(Debug)]
pub struct AuthManager {
nonces: HashMap<String, NonceResponse>,
authenticated_keys: HashMap<String, u64>, // pubkey -> timestamp
}
impl AuthManager {
/// Create a new authentication manager
pub fn new() -> Self {
Self {
nonces: HashMap::new(),
authenticated_keys: HashMap::new(),
}
}
/// Generate a nonce for a given public key
pub fn generate_nonce(&mut self, pubkey: &str) -> String {
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs();
let nonce = format!("{}:{}", pubkey, timestamp);
let nonce_hash = format!("{:x}", Sha256::digest(nonce.as_bytes()));
self.nonces.insert(
pubkey.to_string(),
NonceResponse {
nonce: nonce_hash.clone(),
timestamp,
},
);
nonce_hash
}
/// Verify a signature against a stored nonce
pub fn verify_signature(&mut self, pubkey: &str, signature: &str) -> Result<bool> {
// Get the nonce for this public key
let nonce_response = self
.nonces
.get(pubkey)
.ok_or_else(|| anyhow!("No nonce found for public key"))?;
// Check if nonce is not too old (5 minutes)
let current_time = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs();
if current_time - nonce_response.timestamp > 300 {
return Err(anyhow!("Nonce expired"));
}
// Parse the public key
let pubkey_bytes = hex::decode(pubkey)
.map_err(|_| anyhow!("Invalid public key format"))?;
let secp = Secp256k1::new();
let public_key = PublicKey::from_slice(&pubkey_bytes)
.map_err(|_| anyhow!("Invalid public key"))?;
// Parse the signature
let signature_bytes = hex::decode(signature)
.map_err(|_| anyhow!("Invalid signature format"))?;
let signature = Signature::from_compact(&signature_bytes)
.map_err(|_| anyhow!("Invalid signature"))?;
// Create message hash from nonce
let message_hash = Sha256::digest(nonce_response.nonce.as_bytes());
let message = Message::from_slice(&message_hash)
.map_err(|_| anyhow!("Failed to create message"))?;
// Verify the signature
match secp.verify_ecdsa(&message, &signature, &public_key) {
Ok(_) => {
// Mark this key as authenticated
self.authenticated_keys.insert(pubkey.to_string(), current_time);
// Remove the used nonce
self.nonces.remove(pubkey);
Ok(true)
}
Err(_) => Ok(false),
}
}
/// Check if a public key is currently authenticated
pub fn is_authenticated(&self, pubkey: &str) -> bool {
if let Some(&timestamp) = self.authenticated_keys.get(pubkey) {
let current_time = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs();
// Authentication is valid for 1 hour
current_time - timestamp < 3600
} else {
false
}
}
/// Remove expired authentications
pub fn cleanup_expired(&mut self) {
let current_time = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs();
// Remove expired nonces (older than 5 minutes)
self.nonces.retain(|_, nonce| current_time - nonce.timestamp <= 300);
// Remove expired authentications (older than 1 hour)
self.authenticated_keys.retain(|_, &mut timestamp| current_time - timestamp <= 3600);
}
}

View File

@@ -0,0 +1,479 @@
use anyhow::Result;
use hero_job::{Job, JobBuilder, JobStatus, ScriptType};
use hero_supervisor::{Supervisor, SupervisorBuilder, SupervisorError};
use jsonrpsee::core::async_trait;
use jsonrpsee::proc_macros::rpc;
use jsonrpsee::server::{ServerBuilder, ServerHandle};
use jsonrpsee::RpcModule;
use jsonrpsee_types::error::ErrorCode;
use std::net::SocketAddr;
use std::path::PathBuf;
use std::sync::Arc;
use tokio::sync::RwLock;
use tracing::error;
fn map_sup_error_to_rpc(e: &SupervisorError) -> ErrorCode {
match e {
SupervisorError::InvalidInput(_) | SupervisorError::JobError(_) => ErrorCode::InvalidParams,
SupervisorError::Timeout(_) => ErrorCode::ServerError(-32002),
_ => ErrorCode::InternalError,
}
}
mod auth;
pub mod types;
pub use auth::*;
pub use types::*;
/** Transport type for the OpenRPC server */
#[derive(Debug, Clone)]
pub enum Transport {
WebSocket(SocketAddr),
}
/// OpenRPC server configuration
#[derive(Debug, Clone)]
pub struct OpenRpcServerConfig {
pub transport: Transport,
pub supervisor_config_path: Option<PathBuf>,
pub db_path: PathBuf,
}
/// Main OpenRPC server state
#[derive(Clone)]
pub struct OpenRpcServer {
supervisor: Arc<RwLock<Supervisor>>,
auth_manager: Arc<RwLock<AuthManager>>,
}
/// OpenRPC trait defining all available methods
#[rpc(server)]
pub trait OpenRpcApi {
// Authentication methods
#[method(name = "fetch_nonce")]
async fn fetch_nonce(&self, public_key: String) -> Result<String, ErrorCode>;
#[method(name = "authenticate")]
async fn authenticate(&self, public_key: String, signature: String, nonce: String) -> Result<bool, ErrorCode>;
#[method(name = "whoami")]
async fn whoami(&self) -> Result<String, ErrorCode>;
// Script execution
#[method(name = "play")]
async fn play(&self, script: String) -> Result<PlayResult, ErrorCode>;
// Job management
#[method(name = "create_job")]
async fn create_job(&self, job_params: JobParams) -> Result<String, ErrorCode>;
#[method(name = "start_job")]
async fn start_job(&self, job_id: String) -> Result<StartJobResult, ErrorCode>;
#[method(name = "run_job")]
async fn run_job(
&self,
script: String,
script_type: ScriptType,
prerequisites: Option<Vec<String>>,
) -> Result<String, ErrorCode>;
#[method(name = "get_job_status")]
async fn get_job_status(&self, job_id: String) -> Result<JobStatus, ErrorCode>;
#[method(name = "get_job_output")]
async fn get_job_output(&self, job_id: String) -> Result<String, ErrorCode>;
#[method(name = "get_job_logs")]
async fn get_job_logs(&self, job_id: String) -> Result<JobLogsResult, ErrorCode>;
#[method(name = "list_jobs")]
async fn list_jobs(&self) -> Result<Vec<String>, ErrorCode>;
#[method(name = "stop_job")]
async fn stop_job(&self, job_id: String) -> Result<(), ErrorCode>;
#[method(name = "delete_job")]
async fn delete_job(&self, job_id: String) -> Result<(), ErrorCode>;
#[method(name = "clear_all_jobs")]
async fn clear_all_jobs(&self) -> Result<(), ErrorCode>;
}
impl OpenRpcServer {
/// Create a new OpenRPC server instance
pub async fn new(config: OpenRpcServerConfig) -> Result<Self> {
let supervisor = if let Some(config_path) = config.supervisor_config_path {
// Load supervisor from config file
SupervisorBuilder::from_toml(&config_path)?
.build().await?
} else {
// Create default supervisor with Redis URL
SupervisorBuilder::new()
.redis_url("redis://localhost:6379")
.build().await?
};
Ok(Self {
supervisor: Arc::new(RwLock::new(supervisor)),
auth_manager: Arc::new(RwLock::new(AuthManager::new())),
})
}
/// Start the OpenRPC server on the given SocketAddr (HTTP/WS only)
pub async fn start_on(self, addr: SocketAddr) -> Result<ServerHandle> {
let mut module = RpcModule::new(());
// Register all the RPC methods
let server_clone = self.clone();
module.register_async_method("fetch_nonce", move |params, _| {
let server = server_clone.clone();
async move {
let public_key: String = params.one().map_err(|_| ErrorCode::InvalidParams)?;
server.fetch_nonce(public_key).await
}
})?;
let server_clone = self.clone();
module.register_async_method("authenticate", move |params, _| {
let server = server_clone.clone();
async move {
let (public_key, signature, nonce): (String, String, String) = params.parse().map_err(|_| ErrorCode::InvalidParams)?;
server.authenticate(public_key, signature, nonce).await
}
})?;
let server_clone = self.clone();
module.register_async_method("whoami", move |_params, _| {
let server = server_clone.clone();
async move {
server.whoami().await
}
})?;
let server_clone = self.clone();
module.register_async_method("play", move |params, _| {
let server = server_clone.clone();
async move {
let script: String = params.one().map_err(|_| ErrorCode::InvalidParams)?;
server.play(script).await
}
})?;
let server_clone = self.clone();
module.register_async_method("create_job", move |params, _| {
let server = server_clone.clone();
async move {
let job: JobParams = params.one().map_err(|_| ErrorCode::InvalidParams)?;
server.create_job(job).await
}
})?;
let server_clone = self.clone();
module.register_async_method("start_job", move |params, _| {
let server = server_clone.clone();
async move {
let job_id: String = params.one().map_err(|_| ErrorCode::InvalidParams)?;
server.start_job(job_id).await
}
})?;
let server_clone = self.clone();
module.register_async_method("run_job", move |params, _| {
let server = server_clone.clone();
async move {
let (script, script_type, prerequisites): (String, ScriptType, Option<Vec<String>>) = params.parse().map_err(|_| ErrorCode::InvalidParams)?;
server.run_job(script, script_type, prerequisites).await
}
})?;
let server_clone = self.clone();
module.register_async_method("get_job_status", move |params, _| {
let server = server_clone.clone();
async move {
let job_id: String = params.one().map_err(|_| ErrorCode::InvalidParams)?;
server.get_job_status(job_id).await
}
})?;
let server_clone = self.clone();
module.register_async_method("get_job_output", move |params, _| {
let server = server_clone.clone();
async move {
let job_id: String = params.one().map_err(|_| ErrorCode::InvalidParams)?;
server.get_job_output(job_id).await
}
})?;
let server_clone = self.clone();
module.register_async_method("get_job_logs", move |params, _| {
let server = server_clone.clone();
async move {
let job_id: String = params.one().map_err(|_| ErrorCode::InvalidParams)?;
server.get_job_logs(job_id).await
}
})?;
let server_clone = self.clone();
module.register_async_method("list_jobs", move |_params, _| {
let server = server_clone.clone();
async move {
// No parameters expected; ignore any provided params for robustness
server.list_jobs().await
}
})?;
let server_clone = self.clone();
module.register_async_method("stop_job", move |params, _| {
let server = server_clone.clone();
async move {
let job_id: String = params.one().map_err(|_| ErrorCode::InvalidParams)?;
server.stop_job(job_id).await
}
})?;
let server_clone = self.clone();
module.register_async_method("delete_job", move |params, _| {
let server = server_clone.clone();
async move {
let job_id: String = params.one().map_err(|_| ErrorCode::InvalidParams)?;
server.delete_job(job_id).await
}
})?;
let server_clone = self.clone();
module.register_async_method("clear_all_jobs", move |params, _| {
let server = server_clone.clone();
async move {
let _: () = params.parse().map_err(|_| ErrorCode::InvalidParams)?;
server.clear_all_jobs().await
}
})?;
let server = ServerBuilder::default()
.build(addr)
.await?;
let handle = server.start(module);
Ok(handle)
}
/// Start the OpenRPC server (config wrapper)
pub async fn start(self, config: OpenRpcServerConfig) -> Result<ServerHandle> {
match config.transport {
Transport::WebSocket(addr) => self.start_on(addr).await,
}
}
}
#[async_trait]
impl OpenRpcApiServer for OpenRpcServer {
async fn fetch_nonce(&self, public_key: String) -> Result<String, ErrorCode> {
let mut auth_manager = self.auth_manager.write().await;
let nonce = auth_manager.generate_nonce(&public_key);
Ok(nonce)
}
async fn authenticate(
&self,
public_key: String,
signature: String,
_nonce: String,
) -> Result<bool, ErrorCode> {
let mut auth_manager = self.auth_manager.write().await;
match auth_manager.verify_signature(&public_key, &signature) {
Ok(is_valid) => Ok(is_valid),
Err(e) => {
error!("Authentication error: {}", e);
Ok(false)
}
}
}
async fn whoami(&self) -> Result<String, ErrorCode> {
let _auth_manager = self.auth_manager.read().await;
// For now, return basic info - in a real implementation,
// you'd track authenticated sessions
Ok(serde_json::json!({
"authenticated": true,
"user_id": "anonymous"
}).to_string())
}
async fn play(&self, script: String) -> Result<PlayResult, ErrorCode> {
let output = self.run_job(script, ScriptType::SAL, None).await?;
Ok(PlayResult { output })
}
async fn create_job(&self, job_params: JobParams) -> Result<String, ErrorCode> {
let supervisor = self.supervisor.read().await;
// Use JobBuilder to create a Job instance
let mut builder = hero_job::JobBuilder::new()
.caller_id(&job_params.caller_id)
.context_id(&job_params.context_id)
.script(&job_params.script)
.script_type(job_params.script_type);
// Set timeout if provided
if let Some(timeout_secs) = job_params.timeout {
builder = builder.timeout(std::time::Duration::from_secs(timeout_secs));
}
// Set prerequisites if provided
if let Some(prerequisites) = job_params.prerequisites {
builder = builder.prerequisites(prerequisites);
}
// Build the job
let job = match builder.build() {
Ok(job) => job,
Err(e) => {
error!("Failed to build job: {}", e);
return Err(ErrorCode::InvalidParams);
}
};
let job_id = job.id.clone();
// Create the job using the supervisor
match supervisor.create_job(&job).await {
Ok(_) => Ok(job_id),
Err(e) => {
error!("Failed to create job: {}", e);
Err(ErrorCode::InternalError)
}
}
}
async fn start_job(&self, job_id: String) -> Result<StartJobResult, ErrorCode> {
let supervisor = self.supervisor.read().await;
match supervisor.start_job(&job_id).await {
Ok(_) => Ok(StartJobResult { success: true }),
Err(e) => {
error!("Failed to start job {}: {}", job_id, e);
Ok(StartJobResult { success: false })
}
}
}
async fn run_job(
&self,
script: String,
script_type: ScriptType,
prerequisites: Option<Vec<String>>,
) -> Result<String, ErrorCode> {
let supervisor = self.supervisor.read().await;
// Build job with defaults and optional prerequisites
let mut builder = JobBuilder::new()
.caller_id("rpc-caller")
.context_id("rpc-context")
.script(&script)
.script_type(script_type)
.timeout(std::time::Duration::from_secs(30));
if let Some(prs) = prerequisites {
builder = builder.prerequisites(prs);
}
let job = match builder.build() {
Ok(j) => j,
Err(e) => {
error!("Failed to build job in run_job: {}", e);
return Err(ErrorCode::InvalidParams);
}
};
match supervisor.run_job_and_await_result(&job).await {
Ok(output) => Ok(output),
Err(e) => {
error!("run_job failed: {}", e);
Err(map_sup_error_to_rpc(&e))
}
}
}
async fn get_job_status(&self, job_id: String) -> Result<JobStatus, ErrorCode> {
let supervisor = self.supervisor.read().await;
match supervisor.get_job_status(&job_id).await {
Ok(status) => Ok(status),
Err(e) => {
error!("Failed to get job status for {}: {}", job_id, e);
Err(map_sup_error_to_rpc(&e))
}
}
}
async fn get_job_output(&self, job_id: String) -> Result<String, ErrorCode> {
let supervisor = self.supervisor.read().await;
match supervisor.get_job_output(&job_id).await {
Ok(output) => Ok(output.unwrap_or_else(|| "No output available".to_string())),
Err(e) => {
error!("Failed to get job output for {}: {}", job_id, e);
Err(map_sup_error_to_rpc(&e))
}
}
}
async fn get_job_logs(&self, job_id: String) -> Result<JobLogsResult, ErrorCode> {
let supervisor = self.supervisor.read().await;
match supervisor.get_job_logs(&job_id).await {
Ok(logs_opt) => Ok(JobLogsResult { logs: logs_opt }),
Err(e) => {
error!("Failed to get job logs for {}: {}", job_id, e);
Err(map_sup_error_to_rpc(&e))
}
}
}
async fn list_jobs(&self) -> Result<Vec<String>, ErrorCode> {
let supervisor = self.supervisor.read().await;
match supervisor.list_jobs().await {
Ok(job_ids) => Ok(job_ids),
Err(e) => {
error!("Failed to list jobs: {}", e);
Err(map_sup_error_to_rpc(&e))
}
}
}
async fn stop_job(&self, job_id: String) -> Result<(), ErrorCode> {
let supervisor = self.supervisor.read().await;
match supervisor.stop_job(&job_id).await {
Ok(_) => Ok(()),
Err(e) => {
error!("Failed to stop job {}: {}", job_id, e);
Err(map_sup_error_to_rpc(&e))
}
}
}
async fn delete_job(&self, job_id: String) -> Result<(), ErrorCode> {
let supervisor = self.supervisor.read().await;
match supervisor.delete_job(&job_id).await {
Ok(_) => Ok(()),
Err(e) => {
error!("Failed to delete job {}: {}", job_id, e);
Err(map_sup_error_to_rpc(&e))
}
}
}
async fn clear_all_jobs(&self) -> Result<(), ErrorCode> {
let supervisor = self.supervisor.read().await;
match supervisor.clear_all_jobs().await {
Ok(_) => Ok(()),
Err(e) => {
error!("Failed to clear all jobs: {}", e);
Err(map_sup_error_to_rpc(&e))
}
}
}
}

View File

@@ -0,0 +1,31 @@
use hero_job::ScriptType;
use serde::{Deserialize, Serialize};
/// Parameters for creating a job
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct JobParams {
pub script: String,
pub script_type: ScriptType,
pub caller_id: String,
pub context_id: String,
pub timeout: Option<u64>, // timeout in seconds
pub prerequisites: Option<Vec<String>>,
}
/// Result of script execution
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct PlayResult {
pub output: String,
}
/// Result of starting a job
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct StartJobResult {
pub success: bool,
}
/** Result of getting job logs */
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct JobLogsResult {
pub logs: Option<String>,
}

View File

@@ -0,0 +1,412 @@
use hero_openrpc_server::{OpenRpcServer, OpenRpcServerConfig, OpenRpcApiServer, Transport, types::*};
use hero_supervisor::{Supervisor, SupervisorBuilder};
use hero_job::{JobBuilder, JobStatus, ScriptType};
use jsonrpsee_types::error::ErrorCode;
use std::sync::Arc;
use tokio::sync::RwLock;
use std::time::Duration;
/// Helper function to create a test supervisor
async fn create_test_supervisor() -> Arc<RwLock<Supervisor>> {
let supervisor = SupervisorBuilder::new()
.redis_url("redis://localhost:6379")
.build()
.await
.expect("Failed to create test supervisor");
Arc::new(RwLock::new(supervisor))
}
/// Helper function to create a test OpenRPC server
async fn create_test_server() -> OpenRpcServer {
use std::net::SocketAddr;
use std::path::PathBuf;
let config = OpenRpcServerConfig {
transport: Transport::WebSocket("127.0.0.1:0".parse::<SocketAddr>().unwrap()),
supervisor_config_path: None,
db_path: PathBuf::from("/tmp/test_openrpc.db"),
};
OpenRpcServer::new(config).await.expect("Failed to create OpenRPC server")
}
#[tokio::test]
async fn test_fetch_nonce() {
let server = create_test_server().await;
let public_key = "test_public_key".to_string();
let result = server.fetch_nonce(public_key).await;
assert!(result.is_ok());
let nonce = result.unwrap();
assert!(!nonce.is_empty());
assert_eq!(nonce.len(), 64); // Should be a 32-byte hex string
}
#[tokio::test]
async fn test_create_job_success() {
let server = create_test_server().await;
let job_params = JobParams {
script: "print('Hello, World!');".to_string(),
script_type: ScriptType::OSIS,
caller_id: "test_caller".to_string(),
context_id: "test_context".to_string(),
timeout: Some(60),
prerequisites: None,
};
let result = server.create_job(job_params).await;
assert!(result.is_ok());
let job_id = result.unwrap();
assert!(!job_id.is_empty());
// Job ID should be a valid UUID format
assert!(uuid::Uuid::parse_str(&job_id).is_ok());
}
#[tokio::test]
async fn test_create_job_with_prerequisites() {
let server = create_test_server().await;
let job_params = JobParams {
script: "print('Job with prerequisites');".to_string(),
script_type: ScriptType::SAL,
caller_id: "test_caller".to_string(),
context_id: "test_context".to_string(),
timeout: Some(120),
prerequisites: Some(vec!["prereq_job_1".to_string(), "prereq_job_2".to_string()]),
};
let result = server.create_job(job_params).await;
assert!(result.is_ok());
let job_id = result.unwrap();
assert!(!job_id.is_empty());
}
#[tokio::test]
async fn test_create_job_invalid_params() {
let server = create_test_server().await;
// Test with empty caller_id (should fail JobBuilder validation)
let job_params = JobParams {
script: "print('Test');".to_string(),
script_type: ScriptType::OSIS,
caller_id: "".to_string(), // Empty caller_id should fail
context_id: "test_context".to_string(),
timeout: Some(60),
prerequisites: None,
};
let result = server.create_job(job_params).await;
assert!(result.is_err());
assert_eq!(result.unwrap_err(), ErrorCode::InvalidParams);
}
#[tokio::test]
async fn test_start_job() {
let server = create_test_server().await;
// First create a job
let job_params = JobParams {
script: "print('Test job');".to_string(),
script_type: ScriptType::OSIS,
caller_id: "test_caller".to_string(),
context_id: "test_context".to_string(),
timeout: Some(60),
prerequisites: None,
};
let job_id = server.create_job(job_params).await.unwrap();
// Then start the job
let result = server.start_job(job_id).await;
assert!(result.is_ok());
let start_result = result.unwrap();
assert!(start_result.success);
}
#[tokio::test]
async fn test_get_job_status() {
let server = create_test_server().await;
// First create a job
let job_params = JobParams {
script: "print('Status test');".to_string(),
script_type: ScriptType::OSIS,
caller_id: "test_caller".to_string(),
context_id: "test_context".to_string(),
timeout: Some(60),
prerequisites: None,
};
let job_id = server.create_job(job_params).await.unwrap();
// Get job status
let result = server.get_job_status(job_id).await;
assert!(result.is_ok());
let status = result.unwrap();
// Status should be one of the valid JobStatus variants
match status {
JobStatus::Dispatched | JobStatus::WaitingForPrerequisites |
JobStatus::Started | JobStatus::Error | JobStatus::Finished => {
// Valid status
}
}
}
#[tokio::test]
async fn test_get_job_output() {
let server = create_test_server().await;
// First create a job
let job_params = JobParams {
script: "print('Output test');".to_string(),
script_type: ScriptType::OSIS,
caller_id: "test_caller".to_string(),
context_id: "test_context".to_string(),
timeout: Some(60),
prerequisites: None,
};
let job_id = server.create_job(job_params).await.unwrap();
// Get job output
let result = server.get_job_output(job_id).await;
assert!(result.is_ok());
let output = result.unwrap();
assert!(!output.is_empty());
}
#[tokio::test]
async fn test_list_jobs() {
let server = create_test_server().await;
// Create a few jobs first
for i in 0..3 {
let job_params = JobParams {
script: format!("print('Job {}');", i),
script_type: ScriptType::OSIS,
caller_id: "test_caller".to_string(),
context_id: "test_context".to_string(),
timeout: Some(60),
prerequisites: None,
};
let _ = server.create_job(job_params).await.unwrap();
}
// List all jobs
let result = server.list_jobs().await;
assert!(result.is_ok());
let job_ids = result.unwrap();
assert!(job_ids.len() >= 3); // Should have at least the 3 jobs we created
// Verify job IDs are valid UUIDs
for id in job_ids {
assert!(!id.is_empty());
assert!(uuid::Uuid::parse_str(&id).is_ok());
}
}
#[tokio::test]
async fn test_stop_job() {
let server = create_test_server().await;
// First create and start a job
let job_params = JobParams {
script: "print('Stop test');".to_string(),
script_type: ScriptType::OSIS,
caller_id: "test_caller".to_string(),
context_id: "test_context".to_string(),
timeout: Some(60),
prerequisites: None,
};
let job_id = server.create_job(job_params).await.unwrap();
let _ = server.start_job(job_id.clone()).await.unwrap();
// Stop the job
let result = server.stop_job(job_id).await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_delete_job() {
let server = create_test_server().await;
// First create a job
let job_params = JobParams {
script: "print('Delete test');".to_string(),
script_type: ScriptType::OSIS,
caller_id: "test_caller".to_string(),
context_id: "test_context".to_string(),
timeout: Some(60),
prerequisites: None,
};
let job_id = server.create_job(job_params).await.unwrap();
// Delete the job
let result = server.delete_job(job_id).await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_clear_all_jobs() {
let server = create_test_server().await;
// Create a few jobs first
for i in 0..3 {
let job_params = JobParams {
script: format!("print('Clear test {}');", i),
script_type: ScriptType::OSIS,
caller_id: "test_caller".to_string(),
context_id: "test_context".to_string(),
timeout: Some(60),
prerequisites: None,
};
let _ = server.create_job(job_params).await.unwrap();
}
// Clear all jobs
let result = server.clear_all_jobs().await;
assert!(result.is_ok());
// Verify jobs are cleared
let jobs = server.list_jobs().await.unwrap();
assert_eq!(jobs.len(), 0);
}
#[tokio::test]
async fn test_run_job() {
let server = create_test_server().await;
let script = "print('Run job test');".to_string();
let script_type = ScriptType::OSIS;
let prerequisites = None;
let result = server.run_job(script, script_type, prerequisites).await;
assert!(result.is_ok());
let output = result.unwrap();
assert!(!output.is_empty());
assert!(output.contains("Run job test"));
}
#[tokio::test]
async fn test_play_script() {
let server = create_test_server().await;
let script = "print('Play script test');".to_string();
let result = server.play(script.clone()).await;
assert!(result.is_ok());
let play_result = result.unwrap();
assert!(!play_result.output.is_empty());
assert!(play_result.output.contains(&script));
}
#[tokio::test]
async fn test_get_job_logs() {
let server = create_test_server().await;
// First create a job
let job_params = JobParams {
script: "print('Logs test');".to_string(),
script_type: ScriptType::OSIS,
caller_id: "test_caller".to_string(),
context_id: "test_context".to_string(),
timeout: Some(60),
prerequisites: None,
};
let job_id = server.create_job(job_params).await.unwrap();
// Get job logs
let result = server.get_job_logs(job_id).await;
assert!(result.is_ok());
let logs_result = result.unwrap();
match logs_result.logs {
Some(ref logs) => assert!(!logs.is_empty()),
None => {} // acceptable when no logs are available
}
}
#[tokio::test]
async fn test_job_builder_integration() {
// Test that JobBuilder is working correctly with all the fields
let job_params = JobParams {
script: "print('JobBuilder test');".to_string(),
script_type: ScriptType::V,
caller_id: "test_caller".to_string(),
context_id: "test_context".to_string(),
timeout: Some(300),
prerequisites: Some(vec!["prereq1".to_string(), "prereq2".to_string()]),
};
// Build job using JobBuilder (similar to what the server does)
let mut builder = JobBuilder::new()
.caller_id(&job_params.caller_id)
.context_id(&job_params.context_id)
.script(&job_params.script)
.script_type(job_params.script_type);
if let Some(timeout_secs) = job_params.timeout {
builder = builder.timeout(Duration::from_secs(timeout_secs));
}
if let Some(prerequisites) = job_params.prerequisites {
builder = builder.prerequisites(prerequisites);
}
let job = builder.build();
assert!(job.is_ok());
let job = job.unwrap();
assert_eq!(job.caller_id, "test_caller");
assert_eq!(job.context_id, "test_context");
assert_eq!(job.script, "print('JobBuilder test');");
assert_eq!(job.script_type, ScriptType::V);
assert_eq!(job.timeout, Duration::from_secs(300));
assert_eq!(job.prerequisites, vec!["prereq1".to_string(), "prereq2".to_string()]);
}
#[tokio::test]
async fn test_error_handling() {
let server = create_test_server().await;
// Test getting status for non-existent job
let result = server.get_job_status("non_existent_job".to_string()).await;
// Should return an error or handle gracefully
match result {
Ok(_) => {
// Some implementations might return a default status
},
Err(error_code) => {
assert_eq!(error_code, ErrorCode::InvalidParams);
}
}
// Test getting output for non-existent job
let result = server.get_job_output("non_existent_job".to_string()).await;
match result {
Ok(output) => {
// Should return "No output available" or similar
assert!(output.contains("No output available") || output.is_empty());
},
Err(error_code) => {
assert_eq!(error_code, ErrorCode::InvalidParams);
}
}
}

View File

@@ -1,6 +1,18 @@
[package]
name = "hero-client-unix"
version = "0.1.0"
edition = "2024"
edition = "2021"
[dependencies]
anyhow = "1.0"
clap = { version = "4.5", features = ["derive"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] }
# JSON-RPC async client and params types
jsonrpsee = { version = "0.21", features = ["macros", "async-client"] }
jsonrpsee-types = "0.21"
# IPC transport
reth-ipc = { git = "https://github.com/paradigmxyz/reth", package = "reth-ipc" }

View File

@@ -1,3 +1,124 @@
fn main() {
println!("Hello, world!");
use std::path::PathBuf;
use anyhow::Result;
use clap::Parser;
use jsonrpsee::core::client::ClientT;
use jsonrpsee::rpc_params;
use reth_ipc::client::IpcClientBuilder;
use serde_json::Value;
use tracing_subscriber::EnvFilter;
/// Simple IPC (Unix socket) JSON-RPC client for manual testing.
///
/// Examples:
/// - Call method without params:
/// hero-client-unix --socket /tmp/baobab.ipc --method whoami
///
/// - Call method with positional params (as JSON array):
/// hero-client-unix --socket /tmp/baobab.ipc --method authenticate --params '["pubkey","signature","nonce"]'
///
/// - Call method with single object param:
/// hero-client-unix --socket /tmp/baobab.ipc --method create_job --params '{"job_id":"abc"}'
#[derive(Parser, Debug)]
#[command(name = "hero-client-unix", version, about = "IPC JSON-RPC client")]
struct Args {
/// Filesystem path to the Unix domain socket
#[arg(long, default_value = "/tmp/baobab.ipc", env = "HERO_IPC_SOCKET")]
socket: PathBuf,
/// JSON-RPC method name to call
#[arg(long)]
method: String,
/// JSON string for params. Either an array for positional params or an object for named params.
/// Defaults to [] (no params).
#[arg(long, default_value = "[]")]
params: String,
/// Log filter (e.g., info, debug, trace)
#[arg(long, default_value = "info", env = "RUST_LOG")]
log: String,
}
#[tokio::main]
async fn main() -> Result<()> {
let args = Args::parse();
tracing_subscriber::FmtSubscriber::builder()
.with_env_filter(EnvFilter::new(args.log.clone()))
.try_init()
.expect("setting default subscriber failed");
let socket_str = args.socket.to_string_lossy().to_string();
let client = IpcClientBuilder::default().build(&socket_str).await?;
let params_value: Value = serde_json::from_str(&args.params)?;
// We deserialize responses to serde_json::Value for generality.
// You can set a concrete type instead if needed.
let result: Value = match params_value {
Value::Array(arr) => match arr.len() {
0 => client.request(&args.method, rpc_params![]).await?,
1 => client.request(&args.method, rpc_params![arr[0].clone()]).await?,
2 => client.request(&args.method, rpc_params![arr[0].clone(), arr[1].clone()]).await?,
3 => client
.request(&args.method, rpc_params![arr[0].clone(), arr[1].clone(), arr[2].clone()])
.await?,
4 => client
.request(
&args.method,
rpc_params![arr[0].clone(), arr[1].clone(), arr[2].clone(), arr[3].clone()],
)
.await?,
5 => client
.request(
&args.method,
rpc_params![
arr[0].clone(),
arr[1].clone(),
arr[2].clone(),
arr[3].clone(),
arr[4].clone()
],
)
.await?,
6 => client
.request(
&args.method,
rpc_params![
arr[0].clone(),
arr[1].clone(),
arr[2].clone(),
arr[3].clone(),
arr[4].clone(),
arr[5].clone()
],
)
.await?,
7 => client
.request(
&args.method,
rpc_params![
arr[0].clone(),
arr[1].clone(),
arr[2].clone(),
arr[3].clone(),
arr[4].clone(),
arr[5].clone(),
arr[6].clone()
],
)
.await?,
_ => {
// Fallback: send entire array as a single param to avoid combinatorial explosion.
// Adjust if your server expects strictly positional expansion beyond 7 items.
client.request(&args.method, rpc_params![Value::Array(arr)]).await?
}
},
// Single non-array param (object, string, number, etc.)
other => client.request(&args.method, rpc_params![other]).await?,
};
println!("{}", serde_json::to_string_pretty(&result)?);
Ok(())
}

View File

@@ -1,6 +1,14 @@
[package]
name = "hero-server-unix"
version = "0.1.0"
edition = "2024"
edition = "2021"
[dependencies]
anyhow = "1.0"
clap = { version = "4.5", features = ["derive"] }
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
tokio = { version = "1", features = ["macros", "rt-multi-thread", "signal"] }
# Reuse the OpenRPC server crate that registers all methods and now supports IPC
hero-openrpc-server = { path = "../../openrpc/server" }

View File

@@ -1,3 +1,64 @@
fn main() {
println!("Hello, world!");
use std::path::PathBuf;
use clap::Parser;
use tracing_subscriber::EnvFilter;
use hero_openrpc_server::{OpenRpcServer, OpenRpcServerConfig, Transport};
/// IPC (Unix socket) JSON-RPC server launcher.
///
/// This binary starts the OpenRPC server over a Unix domain socket using the reth-ipc transport.
#[derive(Parser, Debug)]
#[command(name = "hero-server-unix", version, about = "Start the JSON-RPC IPC server")]
struct Args {
/// Filesystem path to the Unix domain socket
#[arg(long, default_value = "/tmp/baobab.ipc", env = "HERO_IPC_SOCKET")]
socket_path: PathBuf,
/// Optional path to a supervisor configuration file
#[arg(long)]
supervisor_config: Option<PathBuf>,
/// Database path (reserved for future use)
#[arg(long, default_value = "./db", env = "HERO_DB_PATH")]
db_path: PathBuf,
/// Log filter (e.g., info, debug, trace)
#[arg(long, default_value = "info", env = "RUST_LOG")]
log: String,
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let args = Args::parse();
// Initialize tracing with provided log filter
tracing_subscriber::FmtSubscriber::builder()
.with_env_filter(EnvFilter::new(args.log.clone()))
.try_init()
.expect("setting default subscriber failed");
let cfg = OpenRpcServerConfig {
transport: Transport::Unix(args.socket_path.clone()),
supervisor_config_path: args.supervisor_config.clone(),
db_path: args.db_path.clone(),
};
// Build server state
let server = OpenRpcServer::new(cfg.clone()).await?;
// Start IPC server
let handle = server.start(cfg).await?;
tracing::info!(
"IPC server started on {} (press Ctrl+C to stop)",
args.socket_path.display()
);
// Run until stopped
tokio::spawn(handle.stopped());
tokio::signal::ctrl_c().await?;
tracing::info!("Shutting down IPC server");
Ok(())
}

View File

@@ -0,0 +1,3 @@
# Baobab WASM App
The Baobab WASM app is a simple web interface for interacting with the backend. It provides a simple way to supervise workers, view and manage jobs, and to execute scripts.

View File

@@ -5,7 +5,7 @@ An OpenRPC WebSocket Server to interface with the [cores](../../core) of authori
- [OpenRPC Specification](openrpc.json) defines the API.
- There are RPC Operations specified to authorize a websocket connection.
- Authorized clients can manage jobs.
- The server uses the [supervisor] to dispatch [jobs] to the [workers].
- The server uses the [supervisor] to dispatch [jobs] to the [actors].
## Circles

View File

@@ -88,7 +88,6 @@ async fn main() -> std::io::Result<()> {
cert: args.cert.clone(),
key: args.key.clone(),
tls_port: args.tls_port,
webhooks: args.webhooks,
circles: std::collections::HashMap::new(), // Empty circles when using CLI
}
};
@@ -151,7 +150,6 @@ async fn main() -> std::io::Result<()> {
}
println!(" Authentication: {}", if config.auth { "ENABLED" } else { "DISABLED" });
println!(" TLS/WSS: {}", if config.tls { "ENABLED" } else { "DISABLED" });
println!(" Webhooks: {}", if config.webhooks { "ENABLED" } else { "DISABLED" });
println!(" Circles configured: {}", config.circles.len());
if config.tls {
@@ -161,12 +159,6 @@ async fn main() -> std::io::Result<()> {
}
}
if config.webhooks {
println!(" Webhook secrets loaded from environment variables:");
println!(" - STRIPE_WEBHOOK_SECRET");
println!(" - IDENFY_WEBHOOK_SECRET");
}
if config.auth && !config.circles.is_empty() {
println!(" Configured circles:");
for (circle_name, members) in &config.circles {

View File

@@ -30,7 +30,7 @@ graph TB
subgraph "Backend"
L[Redis]
M[Rhai Worker]
M[Rhai Actor]
end
A --> |POST /webhooks/stripe/{circle_pk}| E
@@ -94,7 +94,7 @@ sequenceDiagram
participant WV as Webhook Verifier
participant SD as Script Supervisor
participant RC as RhaiSupervisor
participant RW as Rhai Worker
participant RW as Rhai Actor
WS->>CS: POST /webhooks/stripe/{circle_pk}
CS->>CS: Extract circle_pk from URL

View File

@@ -1,8 +1,8 @@
use circle_ws_lib::{spawn_circle_server, ServerConfig};
use rhailib_engine::create_heromodels_engine;
use baobab_engine::create_heromodels_engine;
use futures_util::{SinkExt, StreamExt};
use heromodels::db::hero::OurDB;
use rhailib_worker::spawn_rhai_worker;
use baobab_actor::spawn_rhai_actor;
use serde_json::json;
use std::sync::Arc;
use tokio::sync::mpsc;
@@ -14,13 +14,13 @@ async fn test_server_startup_and_play() {
let circle_pk = Uuid::new_v4().to_string();
let redis_url = "redis://127.0.0.1/";
// --- Worker Setup ---
// --- Actor Setup ---
let (shutdown_tx, shutdown_rx) = mpsc::channel(1);
let db = Arc::new(OurDB::new("file:memdb_test_server?mode=memory&cache=shared", true).unwrap());
let engine = create_heromodels_engine();
let worker_id = Uuid::new_v4().to_string();
let worker_handle = spawn_rhai_worker(
worker_id,
let actor_id = Uuid::new_v4().to_string();
let actor_handle = spawn_rhai_actor(
actor_id,
circle_pk.to_string(),
engine,
redis_url.to_string(),
@@ -37,7 +37,7 @@ async fn test_server_startup_and_play() {
let (server_task, server_handle) = spawn_circle_server(config).unwrap();
let server_join_handle = tokio::spawn(server_task);
// Give server and worker a moment to start
// Give server and actor a moment to start
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
// --- Client Connection and Test ---
@@ -72,5 +72,5 @@ async fn test_server_startup_and_play() {
server_handle.stop(true).await;
let _ = server_join_handle.await;
let _ = shutdown_tx.send(()).await;
let _ = worker_handle.await;
let _ = actor_handle.await;
}

View File

@@ -0,0 +1,127 @@
// Copyright 2019-2021 Parity Technologies (UK) Ltd.
//
// Permission is hereby granted, free of charge, to any
// person obtaining a copy of this software and associated
// documentation files (the "Software"), to deal in the
// Software without restriction, including without
// limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software
// is furnished to do so, subject to the following
// conditions:
//
// The above copyright notice and this permission notice
// shall be included in all copies or substantial portions
// of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
use std::net::SocketAddr;
use std::time::Duration;
use futures::{Stream, StreamExt};
use jsonrpsee::core::DeserializeOwned;
use jsonrpsee::core::client::{Subscription, SubscriptionClientT};
use jsonrpsee::rpc_params;
use jsonrpsee::server::{RpcModule, Server};
use jsonrpsee::ws_client::WsClientBuilder;
use tokio_stream::wrappers::BroadcastStream;
use tokio_stream::wrappers::errors::BroadcastStreamRecvError;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
tracing_subscriber::FmtSubscriber::builder()
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
.try_init()
.expect("setting default subscriber failed");
let addr = run_server().await?;
let url = format!("ws://{}", addr);
let client = WsClientBuilder::default().build(&url).await?;
let sub: Subscription<i32> = client.subscribe("subscribe_hello", rpc_params![], "unsubscribe_hello").await?;
// drop oldest messages from subscription:
let mut sub = drop_oldest_when_lagging(sub, 10);
// Simulate that polling takes a long time.
tokio::time::sleep(Duration::from_secs(1)).await;
// The subscription starts from zero but you can
// notice that many items have been replaced
// because the subscription wasn't polled.
for _ in 0..10 {
match sub.next().await.unwrap() {
Ok(n) => {
tracing::info!("recv={n}");
}
Err(e) => {
tracing::info!("{e}");
}
};
}
Ok(())
}
fn drop_oldest_when_lagging<T: Clone + DeserializeOwned + Send + Sync + 'static>(
mut sub: Subscription<T>,
buffer_size: usize,
) -> impl Stream<Item = Result<T, BroadcastStreamRecvError>> {
let (tx, rx) = tokio::sync::broadcast::channel(buffer_size);
tokio::spawn(async move {
// Poll the subscription which ignores errors.
while let Some(n) = sub.next().await {
let msg = match n {
Ok(msg) => msg,
Err(e) => {
tracing::error!("Failed to decode the subscription message: {e}");
continue;
}
};
if tx.send(msg).is_err() {
return;
}
}
});
BroadcastStream::new(rx)
}
async fn run_server() -> anyhow::Result<SocketAddr> {
let server = Server::builder().build("127.0.0.1:0").await?;
let mut module = RpcModule::new(());
module
.register_subscription("subscribe_hello", "s_hello", "unsubscribe_hello", |_, pending, _, _| async move {
let sub = pending.accept().await.unwrap();
for i in 0..usize::MAX {
let json = serde_json::value::to_raw_value(&i).unwrap();
sub.send(json).await.unwrap();
tokio::time::sleep(Duration::from_millis(10)).await;
}
Ok(())
})
.unwrap();
let addr = server.local_addr()?;
let handle = server.start(module);
// In this example we don't care about doing shutdown so let's it run forever.
// You may use the `ServerHandle` to shut it down or manage it yourself.
tokio::spawn(handle.stopped());
Ok(addr)
}

Some files were not shown because too many files have changed in this diff Show More