Reenable Linux-on-macOS runner tests with in-process broker

.github/workflows/ci.yml

@@ -172,6 +172,10 @@ jobs:
       - run: ./.github/tools/github_actions_run_cargo clippy --all-targets --all-features $MACOS_AARCH64_CRATES
       - run: ./.github/tools/github_actions_run_cargo build $MACOS_AARCH64_CRATES
       - run: ./.github/tools/github_actions_run_cargo nextest $MACOS_AARCH64_CRATES
+      - name: Test Linux-on-macOS runner with in-process broker
+        run: >-
+          ./.github/tools/github_actions_run_cargo nextest
+          -p litebox_runner_linux_on_macos_userland --features test-broker
       - name: Test native macOS runner with in-process broker
         run: >-
           ./.github/tools/github_actions_run_cargo nextest

litebox_runner_linux_on_macos_userland/Cargo.toml

@@ -7,15 +7,23 @@ edition = "2024"
 anyhow = "1"
 clap = { version = "4", features = ["derive"] }
 litebox = { path = "../litebox", version = "0.1.0" }
+litebox_broker_core = { path = "../litebox_broker_core", version = "0.1.0", features = ["test-support"], optional = true }
+litebox_broker_host = { path = "../litebox_broker_host", version = "0.1.0", features = ["test-support"], optional = true }
+litebox_broker_local = { path = "../litebox_broker_local", version = "0.1.0", optional = true }
+litebox_broker_protocol = { path = "../litebox_broker_protocol", version = "0.1.0", optional = true }
 litebox_common_linux = { path = "../litebox_common_linux", version = "0.1.0" }
 litebox_platform_macos_userland = { path = "../litebox_platform_macos_userland", version = "0.1.0", features = ["subpage_compat"] }
-litebox_shim_linux = { path = "../litebox_shim_linux", version = "0.1.0", default-features = false, features = ["alarm_fallback"] }
+litebox_shim_linux = { path = "../litebox_shim_linux", version = "0.1.0", default-features = false }
 litebox_util_log = { path = "../litebox_util_log", version = "0.1.0", features = ["backend_tracing"] }
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 
 [target.'cfg(all(target_os = "macos", target_arch = "aarch64"))'.dev-dependencies]
 tempfile = "3"
 litebox_syscall_rewriter = { path = "../litebox_syscall_rewriter", version = "0.1.0", default-features = false }
 
+[features]
+# Development-only in-process broker fixture.
+test-broker = ["dep:litebox_broker_core", "dep:litebox_broker_host", "dep:litebox_broker_local", "dep:litebox_broker_protocol"]
+
 [lints]
 workspace = true

litebox_runner_linux_on_macos_userland/src/lib.rs

@@ -4,11 +4,20 @@
 //! Run AArch64 Linux PIE programs on an AArch64 macOS host.
 #![cfg(all(target_os = "macos", target_arch = "aarch64"))]
 
+#[cfg(feature = "test-broker")]
+use anyhow::Context as _;
 use anyhow::{Result, bail};
 use clap::Parser;
+#[cfg(feature = "test-broker")]
+use litebox_platform_macos_userland::MacosUserland4K;
 use litebox_platform_macos_userland::{GuestAbi, set_guest_abi};
+#[cfg(feature = "test-broker")]
+use std::ffi::CString;
 use std::path::PathBuf;
 
+#[cfg(feature = "test-broker")]
+mod test_broker;
+
 #[derive(Parser, Debug)]
 #[command(about = "AArch64 Linux runner for macOS; broker support is required")]
 pub struct CliArgs {
@@ -33,7 +42,6 @@ pub struct CliArgs {
     pub program_from_tar: bool,
 }
 
-/// Returns an error until the macOS runner can connect to a broker.
 pub fn run(cli_args: CliArgs) -> Result<i32> {
     set_guest_abi(GuestAbi::Linux);
     tracing_subscriber::fmt()
@@ -46,6 +54,72 @@ pub fn run(cli_args: CliArgs) -> Result<i32> {
         )
         .init();
 
-    let _ = cli_args;
-    bail!("filesystem startup on macOS requires broker support")
+    #[cfg(not(feature = "test-broker"))]
+    {
+        let _ = cli_args;
+        bail!("filesystem startup on macOS requires broker support")
+    }
+
+    #[cfg(feature = "test-broker")]
+    {
+        let requested_program = cli_args
+            .program_and_arguments
+            .first()
+            .context("missing program")?;
+        if cli_args.program_from_tar && !requested_program.starts_with('/') {
+            bail!("program path in --initial-files must be absolute, got: {requested_program}");
+        }
+        let platform = MacosUserland4K::new();
+        let host_program =
+            (!cli_args.program_from_tar).then(|| std::path::Path::new(requested_program));
+        let setup = test_broker::setup(platform, cli_args.initial_files.as_deref(), host_program)?;
+        let program_path = if cli_args.program_from_tar {
+            requested_program.as_str()
+        } else {
+            setup.program_path.as_str()
+        };
+        let argv = cli_args
+            .program_and_arguments
+            .iter()
+            .map(|value| CString::new(value.as_bytes()))
+            .collect::<Result<Vec<_>, _>>()
+            .context("NUL in program argument")?;
+        let mut environment = cli_args.environment_variables;
+        if cli_args.forward_environment_variables {
+            environment.extend(std::env::vars().map(|(key, value)| format!("{key}={value}")));
+        }
+        let envp = environment
+            .iter()
+            .map(|value| CString::new(value.as_bytes()))
+            .collect::<Result<Vec<_>, _>>()
+            .context("NUL in environment entry")?;
+        let shim = setup.builder.build();
+        let program = shim
+            .load_program(
+                litebox_common_linux::TaskParams {
+                    pid: setup.process_id,
+                    ppid: 0,
+                    uid: 1000,
+                    euid: 1000,
+                    gid: 1000,
+                    egid: 1000,
+                },
+                setup.initial_thread,
+                program_path,
+                argv,
+                envp,
+            )
+            .context("loading Linux program")?;
+        // SAFETY: the loader owns valid mappings and has finalized Linux gates;
+        // entrypoints retain those mappings until guest execution stops.
+        unsafe {
+            litebox_platform_macos_userland::run_thread(
+                program.entrypoints,
+                &mut litebox_common_linux::PtRegs::default(),
+            );
+        }
+        let exit_code = program.process.wait_for_unix_shell_exit_code();
+        test_broker::flush_output(&setup.stdio)?;
+        Ok(exit_code)
+    }
 }

litebox_runner_linux_on_macos_userland/src/test_broker.rs

@@ -0,0 +1,146 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+//! In-process broker fixture for the test-broker runner feature.
+
+use anyhow::{Context as _, Result, anyhow};
+use litebox::LiteBox;
+use litebox_broker_core::{
+    ObjectRights, PolicyEngine,
+    fs::{
+        composer::Composer,
+        devices::Devices,
+        in_mem::{InMem, InitialNode},
+        overlay::Overlay,
+        resolver::Resolver,
+        tar_ro::{EMPTY_TAR_FILE, TarRo},
+    },
+    random::{RandomProvider, RandomProviderError},
+    test_support::{TestBrokerCoreBuilder, TestStdioProvider},
+};
+use litebox_broker_host::test_support::InProcessBrokerSetup;
+use litebox_broker_local::BrokerLocal;
+use litebox_broker_protocol::{
+    fs::{FileMode, FileUser},
+    stdio::StdioOutputStream,
+};
+use litebox_platform_macos_userland::MacosUserland4K;
+use litebox_shim_linux::LinuxShimBuilder;
+use std::{
+    borrow::Cow,
+    io::{Read as _, Write as _},
+    path::Path,
+    sync::Arc,
+};
+
+const HOST_PROGRAM_PATH: &str = "/.litebox-host-program";
+
+struct TestRandomProvider;
+
+impl RandomProvider for TestRandomProvider {
+    fn fill(&self, output: &mut [u8]) -> Result<(), RandomProviderError> {
+        output.fill(0x5a);
+        Ok(())
+    }
+}
+
+pub(crate) struct Setup {
+    pub(crate) builder: LinuxShimBuilder<MacosUserland4K>,
+    pub(crate) process_id: i32,
+    pub(crate) initial_thread: litebox::thread::Thread,
+    pub(crate) program_path: String,
+    pub(crate) stdio: Arc<TestStdioProvider>,
+}
+
+pub(crate) fn setup(
+    platform: &'static MacosUserland4K,
+    initial_files: Option<&Path>,
+    host_program: Option<&Path>,
+) -> Result<Setup> {
+    let stdio = Arc::new(TestStdioProvider::default());
+    let mut input = Vec::new();
+    std::io::stdin()
+        .read_to_end(&mut input)
+        .context("reading test input")?;
+    stdio.push_input(&input);
+
+    let directory = || InitialNode::Directory {
+        mode: FileMode::RWXU | FileMode::RWXG | FileMode::RWXO,
+        owner: FileUser::ROOT,
+    };
+    let mut entries = vec![("/tmp".to_owned(), directory())];
+    if let Some(path) = host_program {
+        entries.push((
+            HOST_PROGRAM_PATH.to_owned(),
+            InitialNode::File {
+                mode: FileMode::RUSR
+                    | FileMode::WUSR
+                    | FileMode::XUSR
+                    | FileMode::RGRP
+                    | FileMode::XGRP
+                    | FileMode::ROTH
+                    | FileMode::XOTH,
+                owner: FileUser::ROOT,
+                data: std::fs::read(path)
+                    .with_context(|| format!("reading guest program {}", path.display()))?
+                    .into(),
+            },
+        ));
+    }
+    let tar_data = match initial_files {
+        Some(path) => Cow::Owned(
+            std::fs::read(path)
+                .with_context(|| format!("reading initial files {}", path.display()))?,
+        ),
+        None => Cow::Borrowed(EMPTY_TAR_FILE),
+    };
+    let upper = InMem::<MacosUserland4K>::new_initialized(entries);
+    let fs = Composer::builder()
+        .mount_nestable("/", |allocators| {
+            Overlay::<MacosUserland4K>::new(
+                upper,
+                TarRo::new(tar_data, allocators.next()),
+                allocators.next(),
+            )
+        })
+        .mount("/dev", Devices::new)
+        .build()
+        .map_err(|error| anyhow!("test filesystem: {error:?}"))?;
+    let core = TestBrokerCoreBuilder::new(PolicyEngine::with_unauthenticated_rights(
+        ObjectRights::all(),
+    ))
+    .with_random_provider(Arc::new(TestRandomProvider))
+    .with_stdio_provider(stdio.clone())
+    .with_file_service(Arc::new(Resolver::<MacosUserland4K, _>::new(fs)))
+    .build()
+    .map_err(|error| anyhow!("test broker: {error:?}"))?;
+    let setup = InProcessBrokerSetup::new(core);
+    let readiness = setup.readiness_sink();
+    let (local, _startup, ()) = BrokerLocal::negotiate(setup, |setup| {
+        let memory = setup.shared_memory();
+        Ok((setup.activate(), memory, ()))
+    })
+    .map_err(|error| anyhow!("test broker negotiation: {error:?}"))?;
+    let (litebox, process_id, initial_thread) =
+        LiteBox::new_process_with_broker_local(platform, local);
+    let process_id = i32::try_from(process_id.0).context("process ID does not fit pid_t")?;
+    readiness.attach(litebox.broker_notification_dispatcher());
+
+    Ok(Setup {
+        builder: LinuxShimBuilder::new_with_litebox(platform, litebox, process_id),
+        process_id,
+        initial_thread,
+        program_path: host_program.map_or_else(String::new, |_| HOST_PROGRAM_PATH.to_owned()),
+        stdio,
+    })
+}
+
+pub(crate) fn flush_output(stdio: &TestStdioProvider) -> Result<()> {
+    for (stream, bytes) in stdio.writes() {
+        match stream {
+            StdioOutputStream::Stdout => std::io::stdout().write_all(&bytes)?,
+            StdioOutputStream::Stderr => std::io::stderr().write_all(&bytes)?,
+        }
+    }
+    Ok(())
+}

litebox_runner_linux_on_macos_userland/tests/loader.rs

@@ -53,15 +53,16 @@ fn run_program(name: &str, aot: bool) {
             .args(["-Z", "--initial-files"])
             .arg(&archive)
             .args(["--env", "LD_LIBRARY_PATH=/lib/aarch64-linux-gnu"]);
-        if from_tar {
-            command
-                .arg("--program-from-tar")
-                .arg(format!("/bin/{name}"));
+        let expected_argv0 = if from_tar {
+            let program = format!("/bin/{name}");
+            command.arg("--program-from-tar").arg(&program);
+            program
         } else {
-            command.arg(root.join("bin").join(name));
-        }
+            let program = root.join("bin").join(name);
+            command.arg(&program);
+            program.display().to_string()
+        };
         let output = command.output().unwrap();
-        // Stdout requires a broker; these tests check successful execution.
         assert_eq!(
             output.status.code(),
             Some(0),
@@ -70,29 +71,51 @@ fn run_program(name: &str, aot: bool) {
             String::from_utf8_lossy(&output.stdout),
             String::from_utf8_lossy(&output.stderr)
         );
+        if name == "hello_world_dyn" {
+            let stdout = String::from_utf8(output.stdout).unwrap();
+            let expected_prefix = format!(
+                "argv[0] = {expected_argv0}\nenvp[0] = LD_LIBRARY_PATH=/lib/aarch64-linux-gnu\nElapsed time: "
+            );
+            assert!(
+                stdout.starts_with(&expected_prefix) && stdout.ends_with(" seconds\n"),
+                "unexpected guest stdout: {stdout:?}"
+            );
+        }
     }
 }
 
 #[test]
-#[ignore = "macOS runner requires broker support"]
+#[cfg_attr(
+    not(feature = "test-broker"),
+    ignore = "macOS runner requires broker support"
+)]
 fn test_load_exec_dynamic() {
     run_program("hello_world_dyn", false);
 }
 
 #[test]
-#[ignore = "macOS runner requires broker support"]
+#[cfg_attr(
+    not(feature = "test-broker"),
+    ignore = "macOS runner requires broker support"
+)]
 fn test_load_exec_dynamic_pthreads() {
     run_program("hello_thread", false);
 }
 
 #[test]
-#[ignore = "macOS runner requires broker support"]
+#[cfg_attr(
+    not(feature = "test-broker"),
+    ignore = "macOS runner requires broker support"
+)]
 fn test_syscall_rewriter() {
     run_program("hello_world_dyn", true);
 }
 
 #[test]
-#[ignore = "macOS runner requires broker support"]
+#[cfg_attr(
+    not(feature = "test-broker"),
+    ignore = "macOS runner requires broker support"
+)]
 fn test_syscall_rewriter_pthreads() {
     run_program("hello_thread", true);
 }