horus/tests/supervisor.rs

//! End-to-End Integration Tests for Hero Supervisor
//! 
//! Tests all OpenRPC client methods against a running supervisor instance.
//! The supervisor binary is automatically started and stopped for each test run.
//!
//! **IMPORTANT**: Run with `--test-threads=1` to ensure tests run sequentially:
//! ```
//! cargo test --test supervisor -- --test-threads=1
//! ```

use hero_supervisor_openrpc_client::SupervisorClient;
use hero_job::{Job, JobBuilder};
use std::sync::Once;
use std::process::Child;

/// Test configuration
const SUPERVISOR_URL: &str = "http://127.0.0.1:3031";
const ADMIN_SECRET: &str = "test-admin-secret-for-e2e-tests";
const TEST_RUNNER_NAME: &str = "test-runner";

use std::sync::Mutex;
use lazy_static::lazy_static;

lazy_static! {
    static ref SUPERVISOR_PROCESS: Mutex<Option<Child>> = Mutex::new(None);
}

/// Global initialization flag
static INIT: Once = Once::new();

/// Initialize and start the supervisor binary (called once)
async fn init_supervisor() {
    INIT.call_once(|| {
        // Register cleanup handler
        let _ = std::panic::catch_unwind(|| {
            ctrlc::set_handler(move || {
                cleanup_supervisor();
                std::process::exit(0);
            }).ok();
        });
        
        // Use escargot to build and get the binary path
        let binary = escargot::CargoBuild::new()
            .bin("supervisor")
            .package("hero-supervisor")
            .run()
            .expect("Failed to build supervisor binary");
        
        // Start the supervisor binary
        let child = binary
            .command()
            .args(&[
                "--admin-secret",
                ADMIN_SECRET,
                "--port",
                "3031",
            ])
            .spawn()
            .expect("Failed to start supervisor");
        
        *SUPERVISOR_PROCESS.lock().unwrap() = Some(child);
        
        // Wait for server to be ready with simple TCP check
        use std::net::TcpStream;
        use std::time::Duration;
        
        println!("⏳ Waiting for supervisor to start...");
        
        for i in 0..30 {
            std::thread::sleep(Duration::from_millis(500));
            
            // Try to connect to the port
            if TcpStream::connect_timeout(
                &"127.0.0.1:3031".parse().unwrap(),
                Duration::from_millis(100)
            ).is_ok() {
                // Give it more time to fully initialize
                std::thread::sleep(Duration::from_secs(2));
                println!("✅ Supervisor ready after ~{}ms", (i * 500) + 2000);
                return;
            }
        }
        
        panic!("Supervisor failed to start within 15 seconds");
    });
}

/// Cleanup supervisor process
fn cleanup_supervisor() {
    if let Ok(mut guard) = SUPERVISOR_PROCESS.lock() {
        if let Some(mut child) = guard.take() {
            println!("🧹 Cleaning up supervisor process...");
            let _ = child.kill();
            let _ = child.wait();
        }
    }
}

/// Helper to create a test client
async fn create_client() -> SupervisorClient {
    // Ensure supervisor is running
    init_supervisor().await;
    
    SupervisorClient::builder()
        .url(SUPERVISOR_URL)
        .secret(ADMIN_SECRET)
        .build()
        .expect("Failed to create supervisor client")
}

/// Helper to create a test job (always uses TEST_RUNNER_NAME)
fn create_test_job(payload: &str) -> Job {
    JobBuilder::new()
        .caller_id("e2e-test")
        .context_id("test-context")
        .runner(TEST_RUNNER_NAME)
        .payload(payload)
        .executor("rhai")
        .timeout(30)
        .build()
        .expect("Failed to build test job")
}

#[tokio::test]
async fn test_01_rpc_discover() {
    println!("\n🧪 Test: rpc.discover");
    
    let client = create_client().await;
    let result = client.discover().await;
    
    assert!(result.is_ok(), "rpc.discover should succeed");
    let spec = result.unwrap();
    
    // Verify it's a valid OpenRPC spec
    assert!(spec.get("openrpc").is_some(), "Should have openrpc field");
    assert!(spec.get("methods").is_some(), "Should have methods field");
    
    println!("✅ rpc.discover works");
}

#[tokio::test]
async fn test_02_runner_register() {
    println!("\n🧪 Test: runner.register");
    
    let client = create_client().await;
    
    // Register a test runner
    let result = client.runner_create(TEST_RUNNER_NAME).await;
    
    // Should succeed or already exist
    match result {
        Ok(()) => {
            println!("✅ runner.register works - registered: {}", TEST_RUNNER_NAME);
        }
        Err(e) => {
            // If it fails, it might already exist, which is okay
            println!("⚠️  runner.register: {:?} (may already exist)", e);
        }
    }
}

#[tokio::test]
async fn test_03_runner_list() {
    println!("\n🧪 Test: runner.list");
    
    let client = create_client().await;
    
    // First ensure our test runner exists
    let _ = client.runner_create(TEST_RUNNER_NAME).await;
    
    // List all runners
    let result = client.runner_list().await;
    
    if let Err(ref e) = result {
        println!("   Error: {:?}", e);
    }
    assert!(result.is_ok(), "runner.list should succeed");
    let runners = result.unwrap();
    
    assert!(!runners.is_empty(), "Should have at least one runner");
    assert!(runners.contains(&TEST_RUNNER_NAME.to_string()), 
            "Should contain our test runner");
    
    println!("✅ runner.list works - found {} runners", runners.len());
    for runner in &runners {
        println!("   - {}", runner);
    }
}

#[tokio::test]
async fn test_04_jobs_create() {
    println!("\n🧪 Test: jobs.create");
    
    let client = create_client().await;
    
    // Ensure runner exists
    let _ = client.runner_create(TEST_RUNNER_NAME).await;
    
    // Create a job without running it
    let job = create_test_job("print('test job');");
    let result = client.job_create(job).await;
    
    match &result {
        Ok(_) => {},
        Err(e) => println!("   Error: {:?}", e),
    }
    assert!(result.is_ok(), "jobs.create should succeed");
    let job_id = result.unwrap();
    
    assert!(!job_id.is_empty(), "Should return a job ID");
    println!("✅ jobs.create works - created job: {}", job_id);
}

#[tokio::test]
async fn test_05_jobs_list() {
    println!("\n🧪 Test: jobs.list");
    
    let client = create_client().await;
    
    // Create a job first
    let _ = client.runner_create(TEST_RUNNER_NAME).await;
    let job = create_test_job("print('list test');");
    let _ = client.job_create(job).await;
    
    // List all jobs
    let result = client.job_list().await;
    
    assert!(result.is_ok(), "jobs.list should succeed");
    let jobs = result.unwrap();
    
    println!("✅ jobs.list works - found {} jobs", jobs.len());
}

#[tokio::test]
async fn test_06_job_run_simple() {
    println!("\n🧪 Test: job.run (simple script)");
    
    let client = create_client().await;
    
    // Ensure runner exists
    let _ = client.runner_create(TEST_RUNNER_NAME).await;
    
    // Run a simple job
    let job = create_test_job(r#"
        print("Hello from test!");
        42
    "#);
    
    let result = client.job_run(job, Some(30)).await;
    
    // Note: This will timeout if no runner is actually connected to Redis
    // but we're testing the API call itself
    match result {
        Ok(response) => {
            println!("✅ job.run works - job_id: {}, status: {}", 
                     response.job_id, response.status);
        }
        Err(e) => {
            println!("⚠️  job.run: {:?} (runner may not be connected)", e);
            // This is expected if no actual runner is listening
        }
    }
}

#[tokio::test]
async fn test_07_job_status() {
    println!("\n🧪 Test: job.status");
    
    let client = create_client().await;
    
    // Create a job first
    let _ = client.runner_create(TEST_RUNNER_NAME).await;
    let job = create_test_job("print('status test');");
    let job_id = client.job_create(job).await.expect("Failed to create job");
    
    // Get job status
    let result = client.job_status(&job_id).await;
    
    if let Err(ref e) = result {
        println!("   Error: {:?}", e);
    }
    assert!(result.is_ok(), "job.status should succeed");
    let status = result.unwrap();
    
    println!("✅ job.status works - job: {}, status: {:?}", 
             job_id, status);
}

#[tokio::test]
async fn test_08_job_get() {
    println!("\n🧪 Test: job.get");
    
    let client = create_client().await;
    
    // Create a job first
    let _ = client.runner_create(TEST_RUNNER_NAME).await;
    let original_job = create_test_job("print('get test');");
    let job_id = client.job_create(original_job.clone()).await
        .expect("Failed to create job");
    
    // Get the job
    let result = client.job_get(&job_id).await;
    
    assert!(result.is_ok(), "job.get should succeed");
    let job = result.unwrap();
    
    assert_eq!(job.id, job_id);
    println!("✅ job.get works - retrieved job: {}", job.id);
}

#[tokio::test]
async fn test_09_job_delete() {
    println!("\n🧪 Test: job.delete");
    
    let client = create_client().await;
    
    // Create a job first
    let _ = client.runner_create(TEST_RUNNER_NAME).await;
    let job = create_test_job("print('delete test');");
    let job_id = client.job_create(job).await.expect("Failed to create job");
    
    // Delete the job
    let result = client.job_delete(&job_id).await;
    
    if let Err(ref e) = result {
        println!("   Error: {:?}", e);
    }
    assert!(result.is_ok(), "job.delete should succeed");
    println!("✅ job.delete works - deleted job: {}", job_id);
    
    // Verify it's gone
    let get_result = client.job_get(&job_id).await;
    assert!(get_result.is_err(), "Job should not exist after deletion");
}

#[tokio::test]
async fn test_10_auth_verify() {
    println!("\n🧪 Test: auth.verify");
    
    let client = create_client().await;
    
    let result = client.auth_verify().await;
    
    assert!(result.is_ok(), "auth.verify should succeed with valid key");
    let auth_info = result.unwrap();
    
    println!("✅ auth.verify works");
    println!("   Scope: {}", auth_info.scope);
    println!("   Name: {}", auth_info.name.unwrap_or_else(|| "N/A".to_string()));
}

#[tokio::test]
async fn test_11_auth_key_create() {
    println!("\n🧪 Test: auth.key.create");
    
    let client = create_client().await;
    
    use hero_supervisor_openrpc_client::GenerateApiKeyParams;
    let params = GenerateApiKeyParams {
        name: "test-key".to_string(),
        scope: "user".to_string(),
    };
    let result = client.key_generate(params).await;
    
    assert!(result.is_ok(), "auth.key.create should succeed");
    let api_key = result.unwrap();
    
    assert!(!api_key.key.is_empty(), "Should return a key");
    assert_eq!(api_key.name, "test-key");
    assert_eq!(api_key.scope, "user");
    
    println!("✅ auth.key.create works - created key: {}...", 
             &api_key.key[..api_key.key.len().min(8)]);
}

#[tokio::test]
async fn test_12_auth_key_list() {
    println!("\n🧪 Test: auth.key.list");
    
    let client = create_client().await;
    
    // Create a key first
    use hero_supervisor_openrpc_client::GenerateApiKeyParams;
    let params = GenerateApiKeyParams {
        name: "list-test-key".to_string(),
        scope: "user".to_string(),
    };
    let _ = client.key_generate(params).await;
    
    let result = client.key_list().await;
    
    assert!(result.is_ok(), "auth.key.list should succeed");
    let keys = result.unwrap();
    
    println!("✅ auth.key.list works - found {} keys", keys.len());
    for key in &keys {
        println!("   - {} ({}): {}...", key.name, key.scope, 
                 &key.key[..key.key.len().min(8)]);
    }
}

#[tokio::test]
async fn test_13_auth_key_remove() {
    println!("\n🧪 Test: auth.key.remove");
    
    let client = create_client().await;
    
    // Create a key first
    use hero_supervisor_openrpc_client::GenerateApiKeyParams;
    let params = GenerateApiKeyParams {
        name: "remove-test-key".to_string(),
        scope: "user".to_string(),
    };
    let api_key = client.key_generate(params)
        .await
        .expect("Failed to create key");
    
    // Remove it (use name as the key_id, not the key value)
    let result = client.key_delete(api_key.name.clone()).await;
    
    if let Err(ref e) = result {
        println!("   Error: {:?}", e);
    }
    assert!(result.is_ok(), "auth.key.remove should succeed");
    println!("✅ auth.key.remove works - removed key: {}...", 
             &api_key.key[..api_key.key.len().min(8)]);
}

#[tokio::test]
async fn test_14_runner_remove() {
    println!("\n🧪 Test: runner.remove");
    
    let client = create_client().await;
    
    // Register a runner to remove
    let runner_name = "test-runner-to-remove";
    let _ = client.runner_create(runner_name).await;
    
    // Remove it
    let result = client.runner_remove(runner_name).await;
    
    assert!(result.is_ok(), "runner.remove should succeed");
    println!("✅ runner.remove works - removed: {}", runner_name);
    
    // Verify it's gone
    let runners = client.runner_list().await.unwrap();
    assert!(!runners.contains(&runner_name.to_string()), 
            "Runner should not exist after removal");
}

#[tokio::test]
async fn test_15_supervisor_info() {
    println!("\n🧪 Test: supervisor.info");
    
    let client = create_client().await;
    
    let result = client.get_supervisor_info().await;
    
    if let Err(ref e) = result {
        println!("   Error: {:?}", e);
    }
    assert!(result.is_ok(), "supervisor.info should succeed");
    let info = result.unwrap();
    
    println!("✅ supervisor.info works");
    println!("   Server URL: {}", info.server_url);
}

/// Integration test that runs a complete workflow
#[tokio::test]
async fn test_99_complete_workflow() {
    println!("\n🧪 Test: Complete Workflow");
    
    let client = create_client().await;
    
    // 1. Register runner
    println!("  1. Registering runner...");
    let _ = client.runner_create("workflow-runner").await;
    
    // 2. List runners
    println!("  2. Listing runners...");
    let runners = client.runner_list().await.unwrap();
    assert!(runners.contains(&"workflow-runner".to_string()));
    
    // 3. Create API key
    println!("  3. Creating API key...");
    use hero_supervisor_openrpc_client::GenerateApiKeyParams;
    let params = GenerateApiKeyParams {
        name: "workflow-key".to_string(),
        scope: "user".to_string(),
    };
    let api_key = client.key_generate(params).await.unwrap();
    
    // 4. Verify auth
    println!("  4. Verifying auth...");
    let _ = client.auth_verify().await.unwrap();
    
    // 5. Create job
    println!("  5. Creating job...");
    let job = create_test_job("print('workflow test');");
    let job_id = client.job_create(job).await.unwrap();
    
    // 6. Get job status
    println!("  6. Getting job status...");
    let _status = client.job_status(&job_id).await.unwrap();
    
    // 7. List all jobs
    println!("  7. Listing all jobs...");
    let jobs = client.job_list().await.unwrap();
    assert!(!jobs.is_empty());
    
    // 8. Delete job
    println!("  8. Deleting job...");
    let _ = client.job_delete(&job_id).await.unwrap();
    
    // 9. Remove API key
    println!("  9. Removing API key...");
    let _ = client.key_delete(api_key.name).await.unwrap();
    
    // 10. Remove runner
    println!("  10. Removing runner...");
    let _ = client.runner_remove("workflow-runner").await.unwrap();
    
    println!("✅ Complete workflow test passed!");
}

/// Final test that ensures cleanup happens
/// This test runs last (test_zz prefix ensures it runs after test_99)
#[tokio::test]
async fn test_zz_cleanup() {
    println!("🧹 Running cleanup...");
    cleanup_supervisor();
    
    // Wait a bit to ensure process is killed
    tokio::time::sleep(tokio::time::Duration::from_millis(500)).await;
    
    // Verify port is free
    use std::net::TcpStream;
    let port_free = TcpStream::connect_timeout(
        &"127.0.0.1:3031".parse().unwrap(),
        std::time::Duration::from_millis(100)
    ).is_err();
    
    assert!(port_free, "Port 3031 should be free after cleanup");
    println!("✅ Cleanup complete - port 3031 is free");
}