//! End-to-End Integration Tests for Hero Supervisor //! //! Tests all OpenRPC client methods against a running supervisor instance. //! The supervisor binary is automatically started and stopped for each test run. //! //! **IMPORTANT**: Run with `--test-threads=1` to ensure tests run sequentially: //! ``` //! cargo test --test supervisor -- --test-threads=1 //! ``` use hero_supervisor_openrpc_client::SupervisorClient; use hero_job::{Job, JobBuilder}; use std::sync::Once; use std::process::Child; /// Test configuration const SUPERVISOR_URL: &str = "http://127.0.0.1:3031"; const ADMIN_SECRET: &str = "test-admin-secret-for-e2e-tests"; const TEST_RUNNER_NAME: &str = "test-runner"; use std::sync::Mutex; use lazy_static::lazy_static; lazy_static! { static ref SUPERVISOR_PROCESS: Mutex> = Mutex::new(None); } /// Global initialization flag static INIT: Once = Once::new(); /// Initialize and start the supervisor binary (called once) async fn init_supervisor() { INIT.call_once(|| { // Register cleanup handler let _ = std::panic::catch_unwind(|| { ctrlc::set_handler(move || { cleanup_supervisor(); std::process::exit(0); }).ok(); }); // Use escargot to build and get the binary path let binary = escargot::CargoBuild::new() .bin("supervisor") .package("hero-supervisor") .run() .expect("Failed to build supervisor binary"); // Start the supervisor binary let child = binary .command() .args(&[ "--admin-secret", ADMIN_SECRET, "--port", "3031", ]) .spawn() .expect("Failed to start supervisor"); *SUPERVISOR_PROCESS.lock().unwrap() = Some(child); // Wait for server to be ready with simple TCP check use std::net::TcpStream; use std::time::Duration; println!("⏳ Waiting for supervisor to start..."); for i in 0..30 { std::thread::sleep(Duration::from_millis(500)); // Try to connect to the port if TcpStream::connect_timeout( &"127.0.0.1:3031".parse().unwrap(), Duration::from_millis(100) ).is_ok() { // Give it more time to fully initialize std::thread::sleep(Duration::from_secs(2)); println!("✅ Supervisor ready after ~{}ms", (i * 500) + 2000); return; } } panic!("Supervisor failed to start within 15 seconds"); }); } /// Cleanup supervisor process fn cleanup_supervisor() { if let Ok(mut guard) = SUPERVISOR_PROCESS.lock() { if let Some(mut child) = guard.take() { println!("🧹 Cleaning up supervisor process..."); let _ = child.kill(); let _ = child.wait(); } } } /// Helper to create a test client async fn create_client() -> SupervisorClient { // Ensure supervisor is running init_supervisor().await; SupervisorClient::builder() .url(SUPERVISOR_URL) .secret(ADMIN_SECRET) .build() .expect("Failed to create supervisor client") } /// Helper to create a test job (always uses TEST_RUNNER_NAME) fn create_test_job(payload: &str) -> Job { JobBuilder::new() .caller_id("e2e-test") .context_id("test-context") .runner(TEST_RUNNER_NAME) .payload(payload) .executor("rhai") .timeout(30) .build() .expect("Failed to build test job") } #[tokio::test] async fn test_01_rpc_discover() { println!("\n🧪 Test: rpc.discover"); let client = create_client().await; let result = client.discover().await; assert!(result.is_ok(), "rpc.discover should succeed"); let spec = result.unwrap(); // Verify it's a valid OpenRPC spec assert!(spec.get("openrpc").is_some(), "Should have openrpc field"); assert!(spec.get("methods").is_some(), "Should have methods field"); println!("✅ rpc.discover works"); } #[tokio::test] async fn test_02_runner_register() { println!("\n🧪 Test: runner.register"); let client = create_client().await; // Register a test runner let result = client.runner_create(TEST_RUNNER_NAME).await; // Should succeed or already exist match result { Ok(()) => { println!("✅ runner.register works - registered: {}", TEST_RUNNER_NAME); } Err(e) => { // If it fails, it might already exist, which is okay println!("⚠️ runner.register: {:?} (may already exist)", e); } } } #[tokio::test] async fn test_03_runner_list() { println!("\n🧪 Test: runner.list"); let client = create_client().await; // First ensure our test runner exists let _ = client.runner_create(TEST_RUNNER_NAME).await; // List all runners let result = client.runner_list().await; if let Err(ref e) = result { println!(" Error: {:?}", e); } assert!(result.is_ok(), "runner.list should succeed"); let runners = result.unwrap(); assert!(!runners.is_empty(), "Should have at least one runner"); assert!(runners.contains(&TEST_RUNNER_NAME.to_string()), "Should contain our test runner"); println!("✅ runner.list works - found {} runners", runners.len()); for runner in &runners { println!(" - {}", runner); } } #[tokio::test] async fn test_04_jobs_create() { println!("\n🧪 Test: jobs.create"); let client = create_client().await; // Ensure runner exists let _ = client.runner_create(TEST_RUNNER_NAME).await; // Create a job without running it let job = create_test_job("print('test job');"); let result = client.job_create(job).await; match &result { Ok(_) => {}, Err(e) => println!(" Error: {:?}", e), } assert!(result.is_ok(), "jobs.create should succeed"); let job_id = result.unwrap(); assert!(!job_id.is_empty(), "Should return a job ID"); println!("✅ jobs.create works - created job: {}", job_id); } #[tokio::test] async fn test_05_jobs_list() { println!("\n🧪 Test: jobs.list"); let client = create_client().await; // Create a job first let _ = client.runner_create(TEST_RUNNER_NAME).await; let job = create_test_job("print('list test');"); let _ = client.job_create(job).await; // List all jobs let result = client.job_list().await; assert!(result.is_ok(), "jobs.list should succeed"); let jobs = result.unwrap(); println!("✅ jobs.list works - found {} jobs", jobs.len()); } #[tokio::test] async fn test_06_job_run_simple() { println!("\n🧪 Test: job.run (simple script)"); let client = create_client().await; // Ensure runner exists let _ = client.runner_create(TEST_RUNNER_NAME).await; // Run a simple job let job = create_test_job(r#" print("Hello from test!"); 42 "#); let result = client.job_run(job, Some(30)).await; // Note: This will timeout if no runner is actually connected to Redis // but we're testing the API call itself match result { Ok(response) => { println!("✅ job.run works - job_id: {}, status: {}", response.job_id, response.status); } Err(e) => { println!("⚠️ job.run: {:?} (runner may not be connected)", e); // This is expected if no actual runner is listening } } } #[tokio::test] async fn test_07_job_status() { println!("\n🧪 Test: job.status"); let client = create_client().await; // Create a job first let _ = client.runner_create(TEST_RUNNER_NAME).await; let job = create_test_job("print('status test');"); let job_id = client.job_create(job).await.expect("Failed to create job"); // Get job status let result = client.job_status(&job_id).await; if let Err(ref e) = result { println!(" Error: {:?}", e); } assert!(result.is_ok(), "job.status should succeed"); let status = result.unwrap(); println!("✅ job.status works - job: {}, status: {:?}", job_id, status); } #[tokio::test] async fn test_08_job_get() { println!("\n🧪 Test: job.get"); let client = create_client().await; // Create a job first let _ = client.runner_create(TEST_RUNNER_NAME).await; let original_job = create_test_job("print('get test');"); let job_id = client.job_create(original_job.clone()).await .expect("Failed to create job"); // Get the job let result = client.job_get(&job_id).await; assert!(result.is_ok(), "job.get should succeed"); let job = result.unwrap(); assert_eq!(job.id, job_id); println!("✅ job.get works - retrieved job: {}", job.id); } #[tokio::test] async fn test_09_job_delete() { println!("\n🧪 Test: job.delete"); let client = create_client().await; // Create a job first let _ = client.runner_create(TEST_RUNNER_NAME).await; let job = create_test_job("print('delete test');"); let job_id = client.job_create(job).await.expect("Failed to create job"); // Delete the job let result = client.job_delete(&job_id).await; if let Err(ref e) = result { println!(" Error: {:?}", e); } assert!(result.is_ok(), "job.delete should succeed"); println!("✅ job.delete works - deleted job: {}", job_id); // Verify it's gone let get_result = client.job_get(&job_id).await; assert!(get_result.is_err(), "Job should not exist after deletion"); } #[tokio::test] async fn test_10_auth_verify() { println!("\n🧪 Test: auth.verify"); let client = create_client().await; let result = client.auth_verify().await; assert!(result.is_ok(), "auth.verify should succeed with valid key"); let auth_info = result.unwrap(); println!("✅ auth.verify works"); println!(" Scope: {}", auth_info.scope); println!(" Name: {}", auth_info.name.unwrap_or_else(|| "N/A".to_string())); } #[tokio::test] async fn test_11_auth_key_create() { println!("\n🧪 Test: auth.key.create"); let client = create_client().await; use hero_supervisor_openrpc_client::GenerateApiKeyParams; let params = GenerateApiKeyParams { name: "test-key".to_string(), scope: "user".to_string(), }; let result = client.key_generate(params).await; assert!(result.is_ok(), "auth.key.create should succeed"); let api_key = result.unwrap(); assert!(!api_key.key.is_empty(), "Should return a key"); assert_eq!(api_key.name, "test-key"); assert_eq!(api_key.scope, "user"); println!("✅ auth.key.create works - created key: {}...", &api_key.key[..api_key.key.len().min(8)]); } #[tokio::test] async fn test_12_auth_key_list() { println!("\n🧪 Test: auth.key.list"); let client = create_client().await; // Create a key first use hero_supervisor_openrpc_client::GenerateApiKeyParams; let params = GenerateApiKeyParams { name: "list-test-key".to_string(), scope: "user".to_string(), }; let _ = client.key_generate(params).await; let result = client.key_list().await; assert!(result.is_ok(), "auth.key.list should succeed"); let keys = result.unwrap(); println!("✅ auth.key.list works - found {} keys", keys.len()); for key in &keys { println!(" - {} ({}): {}...", key.name, key.scope, &key.key[..key.key.len().min(8)]); } } #[tokio::test] async fn test_13_auth_key_remove() { println!("\n🧪 Test: auth.key.remove"); let client = create_client().await; // Create a key first use hero_supervisor_openrpc_client::GenerateApiKeyParams; let params = GenerateApiKeyParams { name: "remove-test-key".to_string(), scope: "user".to_string(), }; let api_key = client.key_generate(params) .await .expect("Failed to create key"); // Remove it (use name as the key_id, not the key value) let result = client.key_delete(api_key.name.clone()).await; if let Err(ref e) = result { println!(" Error: {:?}", e); } assert!(result.is_ok(), "auth.key.remove should succeed"); println!("✅ auth.key.remove works - removed key: {}...", &api_key.key[..api_key.key.len().min(8)]); } #[tokio::test] async fn test_14_runner_remove() { println!("\n🧪 Test: runner.remove"); let client = create_client().await; // Register a runner to remove let runner_name = "test-runner-to-remove"; let _ = client.runner_create(runner_name).await; // Remove it let result = client.runner_remove(runner_name).await; assert!(result.is_ok(), "runner.remove should succeed"); println!("✅ runner.remove works - removed: {}", runner_name); // Verify it's gone let runners = client.runner_list().await.unwrap(); assert!(!runners.contains(&runner_name.to_string()), "Runner should not exist after removal"); } #[tokio::test] async fn test_15_supervisor_info() { println!("\n🧪 Test: supervisor.info"); let client = create_client().await; let result = client.get_supervisor_info().await; if let Err(ref e) = result { println!(" Error: {:?}", e); } assert!(result.is_ok(), "supervisor.info should succeed"); let info = result.unwrap(); println!("✅ supervisor.info works"); println!(" Server URL: {}", info.server_url); } /// Integration test that runs a complete workflow #[tokio::test] async fn test_99_complete_workflow() { println!("\n🧪 Test: Complete Workflow"); let client = create_client().await; // 1. Register runner println!(" 1. Registering runner..."); let _ = client.runner_create("workflow-runner").await; // 2. List runners println!(" 2. Listing runners..."); let runners = client.runner_list().await.unwrap(); assert!(runners.contains(&"workflow-runner".to_string())); // 3. Create API key println!(" 3. Creating API key..."); use hero_supervisor_openrpc_client::GenerateApiKeyParams; let params = GenerateApiKeyParams { name: "workflow-key".to_string(), scope: "user".to_string(), }; let api_key = client.key_generate(params).await.unwrap(); // 4. Verify auth println!(" 4. Verifying auth..."); let _ = client.auth_verify().await.unwrap(); // 5. Create job println!(" 5. Creating job..."); let job = create_test_job("print('workflow test');"); let job_id = client.job_create(job).await.unwrap(); // 6. Get job status println!(" 6. Getting job status..."); let _status = client.job_status(&job_id).await.unwrap(); // 7. List all jobs println!(" 7. Listing all jobs..."); let jobs = client.job_list().await.unwrap(); assert!(!jobs.is_empty()); // 8. Delete job println!(" 8. Deleting job..."); let _ = client.job_delete(&job_id).await.unwrap(); // 9. Remove API key println!(" 9. Removing API key..."); let _ = client.key_delete(api_key.name).await.unwrap(); // 10. Remove runner println!(" 10. Removing runner..."); let _ = client.runner_remove("workflow-runner").await.unwrap(); println!("✅ Complete workflow test passed!"); } /// Final test that ensures cleanup happens /// This test runs last (test_zz prefix ensures it runs after test_99) #[tokio::test] async fn test_zz_cleanup() { println!("🧹 Running cleanup..."); cleanup_supervisor(); // Wait a bit to ensure process is killed tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; // Verify port is free use std::net::TcpStream; let port_free = TcpStream::connect_timeout( &"127.0.0.1:3031".parse().unwrap(), std::time::Duration::from_millis(100) ).is_err(); assert!(port_free, "Port 3031 should be free after cleanup"); println!("✅ Cleanup complete - port 3031 is free"); }