Implement dynamic linking and loading of side modules

Cargo.toml

@@ -181,7 +181,7 @@ engine = ["universal"]
 universal = []
 cache = ["wasmer-cache"]
 wast = ["wasmer-wast"]
-wasi = ["wasmer-wasix"]
+wasi = ["wasmer-wasix", "tokio"]
 wat = ["wasmer/wat"]
 compiler = ["wasmer/compiler", "backend", "wasmer-compiler/translator"]
 singlepass = ["compiler", "wasmer-compiler-singlepass", "wasmer/singlepass"]

examples/wasi.rs

@@ -14,10 +14,13 @@
 //!
 //! Ready?
 
-use std::io::Read;
+use std::{io::Read, sync::Arc};
 
-use wasmer::{Module, Store};
-use wasmer_wasix::{Pipe, WasiEnv};
+use wasmer_wasix::{
+    runners::wasi::{RuntimeOrEngine, WasiRunner},
+    runtime::task_manager::tokio::TokioTaskManager,
+    Pipe, PluggableRuntime, Runtime,
+};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let wasm_path = concat!(
@@ -27,28 +30,50 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Let's declare the Wasm module with the text representation.
     let wasm_bytes = std::fs::read(wasm_path)?;
 
-    // Create a Store.
-    let mut store = Store::default();
+    // We optionally need a tokio runtime and a WASI runtime. This doesn't need to
+    // happen though; see the wasi-pipes example for an alternate approach. Things
+    // such as the file system or networking can be configured on the runtime.
+    let tokio_runtime = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .unwrap();
+    let _guard = tokio_runtime.enter();
+    let tokio_task_manager = TokioTaskManager::new(tokio_runtime.handle().clone());
+    let runtime = PluggableRuntime::new(Arc::new(tokio_task_manager));
 
     println!("Compiling module...");
     // Let's compile the Wasm module.
-    let module = Module::new(&store, wasm_bytes)?;
+    let module = runtime.load_module_sync(&wasm_bytes[..])?;
 
+    // Create a pipe for the module's stdout.
     let (stdout_tx, mut stdout_rx) = Pipe::channel();
 
-    // Run the module.
-    WasiEnv::builder("hello")
-        // .args(&["world"])
-        // .env("KEY", "Value")
-        .stdout(Box::new(stdout_tx))
-        .run_with_store(module, &mut store)?;
+    {
+        // Create a WASI runner. We use a scope to make sure the runner is dropped
+        // as soon as we are done with it; otherwise, it will keep the stdout pipe
+        // open.
+        let mut runner = WasiRunner::new();
+        runner.with_stdout(Box::new(stdout_tx));
 
-    eprintln!("Run complete - reading output");
+        println!("Running module...");
+        // Now, run the module.
+        runner.run_wasm(
+            RuntimeOrEngine::Runtime(Arc::new(runtime)),
+            "hello",
+            module,
+            wasmer_types::ModuleHash::xxhash(wasm_bytes),
+        )?;
+    }
+
+    println!("Run complete - reading output");
 
     let mut buf = String::new();
     stdout_rx.read_to_string(&mut buf).unwrap();
 
-    eprintln!("Output: {buf}");
+    println!("Output: {buf}");
+
+    // Verify the module wrote the correct thing, for the test below
+    assert_eq!(buf, "Hello, world!\n");
 
     Ok(())
 }

examples/wasi_manual_setup.rs

@@ -45,6 +45,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     let mut wasi_env = WasiEnv::builder("hello")
         // .args(&["world"])
         // .env("KEY", "Value")
+        .engine(store.engine().clone())
         .finalize(&mut store)?;
 
     println!("Instantiating module with WASI imports...");

examples/wasi_pipes.rs

@@ -12,8 +12,12 @@
 //! Ready?
 
 use std::io::{Read, Write};
-use wasmer::{Module, Store};
-use wasmer_wasix::{Pipe, WasiEnv};
+
+use wasmer::Module;
+use wasmer_wasix::{
+    runners::wasi::{RuntimeOrEngine, WasiRunner},
+    Pipe,
+};
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
     let wasm_path = concat!(
@@ -23,12 +27,12 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Let's declare the Wasm module with the text representation.
     let wasm_bytes = std::fs::read(wasm_path)?;
 
-    // Create a Store.
-    let mut store = Store::default();
+    // We need at least an engine to be able to compile the module.
+    let engine = wasmer::Engine::default();
 
     println!("Compiling module...");
     // Let's compile the Wasm module.
-    let module = Module::new(&store, wasm_bytes)?;
+    let module = Module::new(&engine, &wasm_bytes[..])?;
 
     let msg = "racecar go zoom";
     println!("Writing \"{}\" to the WASI stdin...", msg);
@@ -38,18 +42,35 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     // To write to the stdin
     writeln!(stdin_sender, "{}", msg)?;
 
-    println!("Running module...");
-    // First, we create the `WasiEnv` with the stdio pipes
-    WasiEnv::builder("hello")
-        .stdin(Box::new(stdin_reader))
-        .stdout(Box::new(stdout_sender))
-        .run_with_store(module, &mut store)?;
+    {
+        // Create a WASI runner. We use a scope to make sure the runner is dropped
+        // as soon as we are done with it; otherwise, it will keep the stdout pipe
+        // open.
+        let mut runner = WasiRunner::new();
+
+        // Configure the WasiRunner with the stdio pipes.
+        runner
+            .with_stdin(Box::new(stdin_reader))
+            .with_stdout(Box::new(stdout_sender));
+
+        // Now, run the module.
+        println!("Running module...");
+        runner.run_wasm(
+            RuntimeOrEngine::Engine(engine),
+            "hello",
+            module,
+            wasmer_types::ModuleHash::xxhash(wasm_bytes),
+        )?;
+    }
 
     // To read from the stdout
     let mut buf = String::new();
     stdout_reader.read_to_string(&mut buf)?;
     println!("Read \"{}\" from the WASI stdout!", buf.trim());
 
+    // Verify the module wrote the correct thing, for the test below
+    assert_eq!(buf.trim(), "mooz og racecar");
+
     Ok(())
 }
 

lib/api/build.rs

@@ -56,7 +56,7 @@ fn build_wamr() {
             .expect("failed to extract wamr zip file");
         let _ = std::fs::remove_dir_all(&wamr_dir);
         std::fs::rename(zip_dir.join(ZIP_NAME), &wamr_dir)
-            .expect(&format!("failed to rename wamr dir: {zip_dir:?}"));
+            .unwrap_or_else(|e| panic!("failed to rename wamr dir: {zip_dir:?} due to: {e:?}"));
     } else {
         println!("cargo::rerun-if-changed={}", wamr_dir.display());
     }

lib/c-api/src/wasm_c_api/wasi/mod.rs

@@ -275,7 +275,7 @@ fn prepare_webc_env(
     let handle = runtime.handle().clone();
     let _guard = handle.enter();
     let mut rt = PluggableRuntime::new(Arc::new(TokioTaskManager::new(runtime)));
-    rt.set_engine(Some(store_mut.engine().clone()));
+    rt.set_engine(store_mut.engine().clone());
 
     let slice = unsafe { std::slice::from_raw_parts(bytes, len) };
     let volumes = WebC::parse_volumes_from_fileblock(slice).ok()?;
@@ -348,7 +348,7 @@ pub unsafe extern "C" fn wasi_env_new(
     let handle = runtime.handle().clone();
     let _guard = handle.enter();
     let mut rt = PluggableRuntime::new(Arc::new(TokioTaskManager::new(runtime)));
-    rt.set_engine(Some(store_mut.engine().clone()));
+    rt.set_engine(store_mut.engine().clone());
 
     if !config.inherit_stdout {
         config.builder.set_stdout(Box::new(Pipe::channel().0));

lib/cli/src/commands/run/mod.rs

@@ -45,7 +45,7 @@ use wasmer_wasix::{
     runners::{
         dcgi::{DcgiInstanceFactory, DcgiRunner},
         dproxy::DProxyRunner,
-        wasi::WasiRunner,
+        wasi::{RuntimeOrEngine, WasiRunner},
         wcgi::{self, AbortHandle, NoOpWcgiCallbacks, WcgiRunner},
         MappedCommand, MappedDirectory, Runner,
     },
@@ -393,7 +393,7 @@ impl Run {
         runtime: Arc<dyn Runtime + Send + Sync>,
     ) -> Result<(), Error> {
         let mut runner = self.build_wasi_runner(&runtime)?;
-        runner.run_command(command_name, pkg, runtime)
+        Runner::run_command(&mut runner, command_name, pkg, runtime)
     }
 
     fn run_wcgi(
@@ -584,7 +584,12 @@ impl Run {
         let program_name = wasm_path.display().to_string();
 
         let runner = self.build_wasi_runner(&runtime)?;
-        runner.run_wasm(runtime, &program_name, module, module_hash)
+        runner.run_wasm(
+            RuntimeOrEngine::Runtime(runtime),
+            &program_name,
+            module,
+            module_hash,
+        )
     }
 
     #[allow(unused_variables)]

lib/cli/src/commands/run/wasi.rs

@@ -680,7 +680,7 @@ impl Wasi {
         rt.set_package_loader(package_loader)
             .set_module_cache(module_cache)
             .set_source(registry)
-            .set_engine(Some(engine));
+            .set_engine(engine);
 
         Ok(rt)
     }

lib/swift/src/lib.rs

@@ -4,7 +4,10 @@ use virtual_fs::{AsyncReadExt, AsyncSeekExt};
 use wasmer_package::utils::from_bytes;
 use wasmer_wasix::{
     bin_factory::BinaryPackage,
-    runners::{wasi::WasiRunner, Runner},
+    runners::{
+        wasi::{RuntimeOrEngine, WasiRunner},
+        Runner,
+    },
     runtime::{package_loader::BuiltinPackageLoader, task_manager::tokio::TokioTaskManager},
     PluggableRuntime,
 };
@@ -37,8 +40,7 @@ pub fn run_package(webc_bytes: Vec<u8>, args: Vec<String>) -> Result<String, Was
     let tasks = TokioTaskManager::new(tokio_rt.handle().clone());
     let tasks = Arc::new(tasks);
     let mut rt = PluggableRuntime::new(Arc::clone(&tasks) as Arc<_>);
-    rt.set_engine(Some(wasmer::Engine::default()))
-        .set_package_loader(BuiltinPackageLoader::new());
+    rt.set_package_loader(BuiltinPackageLoader::new());
 
     let pkg = tokio_rt
         .handle()
@@ -61,7 +63,7 @@ pub fn run_package(webc_bytes: Vec<u8>, args: Vec<String>) -> Result<String, Was
             .with_stdin(Box::<virtual_fs::NullFile>::default())
             .with_stdout(Box::new(stdout_2) as Box<_>)
             .with_stderr(Box::<virtual_fs::NullFile>::default())
-            .run_command(&entrypoint, &pkg, Arc::new(rt))
+            .run_command(&entrypoint, &pkg, RuntimeOrEngine::Runtime(Arc::new(rt)))
     });
 
     let _ = handle.join();

lib/types/src/module_hash.rs

@@ -73,7 +73,7 @@ impl ModuleHash {
         Ok(Self::sha256_from_bytes(hash))
     }
 
-    /// Generate a new [`ModuleCache`] based on the XXHash hash of some bytes.
+    /// Generate a new [`ModuleHash`] based on the XXHash hash of some bytes.
     pub fn xxhash(wasm: impl AsRef<[u8]>) -> Self {
         let wasm = wasm.as_ref();
 
@@ -82,7 +82,7 @@ impl ModuleHash {
         Self::XXHash(hash.to_ne_bytes())
     }
 
-    /// Generate a new [`ModuleCache`] based on the Sha256 hash of some bytes.
+    /// Generate a new [`ModuleHash`] based on the Sha256 hash of some bytes.
     pub fn sha256(wasm: impl AsRef<[u8]>) -> Self {
         let wasm = wasm.as_ref();
 
@@ -91,6 +91,13 @@ impl ModuleHash {
         Self::Sha256(hash)
     }
 
+    /// Generate a random [`ModuleHash`]. For when you don't care about caches.
+    pub fn random() -> Self {
+        let mut bytes = [0_u8; 8];
+        getrandom::getrandom(&mut bytes).unwrap();
+        Self::XXHash(bytes)
+    }
+
     /// Get the raw hash.
     pub fn as_bytes(&self) -> &[u8] {
         match self {

lib/wasi-types/src/wasi/bindings.rs

@@ -2417,6 +2417,24 @@ impl<M: MemorySize> core::fmt::Debug for ProcSpawnFdOp<M> {
     }
 }
 
+pub type DlHandle = u32;
+
+wai_bindgen_rust::bitflags::bitflags! {
+    pub struct DlFlags : u32 {
+        const LAZY = 1 << 0;
+        const NOW = 1 << 1;
+        const GLOBAL = 1 << 2;
+        const NOLOAD = 1 << 3;
+        const NODELETE = 1 << 4;
+        const DEEPBIND = 1 << 5;
+    }
+}
+impl DlFlags {
+    pub fn from_bits_preserve(bits: u32) -> Self {
+        Self { bits }
+    }
+}
+
 #[repr(C)]
 #[derive(Copy, Clone)]
 pub struct AddrUnspec {

lib/wasi-types/src/wasi/bindings_manual.rs

@@ -391,3 +391,21 @@ unsafe impl wasmer::FromToNativeWasmType for JoinFlags {
         false
     }
 }
+
+// TODO: if necessary, must be implemented in wit-bindgen
+unsafe impl wasmer::FromToNativeWasmType for DlFlags {
+    type Native = i32;
+
+    fn to_native(self) -> Self::Native {
+        self.bits() as i32
+    }
+
+    fn from_native(n: Self::Native) -> Self {
+        Self::from_bits_truncate(n as u32)
+    }
+
+    fn is_from_store(&self, _store: &impl wasmer::AsStoreRef) -> bool {
+        // TODO: find correct implementation
+        false
+    }
+}

lib/wasix/Cargo.toml

@@ -122,6 +122,7 @@ hyper-util = { version = "0.1.5", features = [
 http-body-util = { version = "0.1.1", optional = true }
 toml = { workspace = true }
 pin-utils = "0.1.0"
+wasmparser = { workspace = true }
 
 [target.'cfg(not(any(target_arch = "riscv64", target_arch = "loongarch64")))'.dependencies.reqwest]
 workspace = true