Merge pull request #28 from n0-computer/extism

WIP: extism integration example

Author: b5

.gitignore

@@ -5,3 +5,4 @@ target/
 # Added by cargo
 
 /target
+Cargo.lock

extism/host/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "extism-host"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+anyhow = "1.0.79"
+extism = "1.0.0"
+tokio = { version = "1", features = ["full"] }
+iroh-extism-host-functions = { path = "../iroh-extism-host-functions" }

extism/host/src/main.rs

@@ -0,0 +1,34 @@
+use extism::*;
+
+fn main() -> anyhow::Result<()> {
+    let tokio_rt = tokio::runtime::Builder::new_multi_thread()
+        .thread_name("main-runtime")
+        .worker_threads(2)
+        .enable_all()
+        .build()?;
+    let rt = tokio_rt.handle().clone();
+
+    let ticket = std::env::args().nth(1).expect("missing ticket");
+
+    let iroh = rt.block_on(async {
+        let iroh_path = iroh_extism_host_functions::default_iroh_extism_data_root().await?;
+        iroh_extism_host_functions::create_iroh(iroh_path).await
+    })?;
+    println!("iroh node id: {:?}", iroh.node_id());
+
+    let file = Wasm::file("../plugin/target/wasm32-unknown-unknown/debug/plugin.wasm");
+    let manifest = Manifest::new([file]);
+
+    let plugin = PluginBuilder::new(manifest).with_wasi(true);
+
+    let mut plugin =
+        iroh_extism_host_functions::add_all_host_functions(rt, plugin, iroh).build()?;
+
+    let res = plugin
+        .call::<&str, &str>("print_hai_and_get_ticket", &ticket)
+        .unwrap();
+
+    println!("Received iroh data:\n\n{}", res);
+
+    Ok(())
+}

extism/iroh-extism-host-functions/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "iroh-extism-host-functions"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.79"
+dirs-next = "2.0.0"
+extism = "1.0.0"
+futures = "0.3.29"
+iroh = { version = "0.12.0", default-features = false }
+tokio = { version = "1", features = ["full"] }
+tokio-util = { version = "0.7", features = ["codec", "io-util", "io", "time"] }

extism/iroh-extism-host-functions/src/lib.rs

@@ -0,0 +1,96 @@
+use std::path::PathBuf;
+use std::str::FromStr;
+
+use anyhow::{anyhow, Result};
+use extism::*;
+use futures::stream::StreamExt;
+use iroh::net::key::SecretKey;
+use iroh::node::Node;
+use iroh::rpc_protocol::{DownloadLocation, SetTagOption};
+use iroh::util::path::IrohPaths;
+
+type IrohNode = Node<iroh::bytes::store::flat::Store>;
+
+const IROH_EXTISM_DATA_DIR: &str = "iroh-extism";
+
+pub async fn default_iroh_extism_data_root() -> Result<PathBuf> {
+    if let Some(val) = std::env::var_os("IROH_EXTISM_DATA_DIR") {
+        return Ok(PathBuf::from(val));
+    }
+    let path = dirs_next::data_dir().ok_or_else(|| {
+        anyhow!("operating environment provides no directory for application data")
+    })?;
+
+    Ok(path.join(IROH_EXTISM_DATA_DIR))
+}
+
+pub async fn create_iroh(path: PathBuf) -> Result<IrohNode> {
+    tokio::fs::create_dir_all(&path).await?;
+
+    let blob_dir = path.join(IrohPaths::BaoFlatStoreDir);
+    tokio::fs::create_dir_all(&blob_dir).await?;
+
+    let db = iroh::bytes::store::flat::Store::load(blob_dir).await?;
+    let doc_store = iroh::sync::store::fs::Store::new(path.join(IrohPaths::DocsDatabase))?;
+    let node = iroh::node::Node::builder(db, doc_store)
+        .secret_key(SecretKey::generate())
+        .spawn()
+        .await?;
+
+    Ok(node)
+}
+
+// host_fn!(iroh_blob_get_node_id(user_data: Node<iroh::bytes::store::flat::Store>) -> String {
+//     Ok(user_data.node_id().to_string())
+// });
+
+struct Context {
+    rt: tokio::runtime::Handle,
+    iroh: IrohNode,
+}
+
+host_fn!(iroh_blob_get_ticket(user_data: Context; ticket: &str) -> Vec<u8> {
+    let ctx = user_data.get()?;
+    let ctx = ctx.lock().unwrap();
+
+    let (node_addr, hash, format) = iroh::ticket::BlobTicket::from_str(ticket).map_err(|_| anyhow!("invalid ticket"))?.into_parts();
+
+    if format != iroh::rpc_protocol::BlobFormat::Raw {
+        return Err(anyhow!("can only get raw bytes for now, not HashSequences (collections)"));
+    }
+    let client = ctx.iroh.client();
+    let buf = ctx.rt.block_on(async move {
+        let mut stream = client.blobs.download(iroh::rpc_protocol::BlobDownloadRequest {
+            hash,
+            format,
+            peer: node_addr,
+            out: DownloadLocation::Internal,
+            tag: SetTagOption::Auto,
+        }).await?;
+        while stream.next().await.is_some() {}
+
+        let buffer = client.blobs.read(hash).await?.read_to_bytes().await?;
+        anyhow::Ok(buffer.to_vec())
+    })?;
+
+    Ok(buf)
+});
+
+pub fn add_all_host_functions(
+    rt: tokio::runtime::Handle,
+    b: PluginBuilder,
+    iroh: IrohNode,
+) -> PluginBuilder {
+    let ctx = UserData::new(Context {
+        rt,
+        iroh,
+    });
+
+    b.with_function(
+        "iroh_blob_get_ticket",
+        [PTR],
+        [PTR],
+        ctx,
+        iroh_blob_get_ticket,
+    )
+}

extism/plugin/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "plugin"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+crate_type = ["cdylib"]
+
+[dependencies]
+extism-pdk = "1.0.1"

extism/plugin/src/lib.rs

@@ -0,0 +1,13 @@
+use extism_pdk::*;
+
+#[host_fn]
+extern "ExtismHost" {
+    fn iroh_blob_get_ticket(ticket: String) -> Vec<u8>;
+}
+
+#[plugin_fn]
+pub fn print_hai_and_get_ticket(ticket: String) -> FnResult<Vec<u8>> {
+    println!("Hai from a wasm plugin!");
+    let v = unsafe { iroh_blob_get_ticket(ticket.into()) }?;
+    Ok(v)
+}

extism/readme.md

@@ -0,0 +1,46 @@
+# iroh + exsitm
+
+This is an example of running iroh thorugh [exitsm](https://extism.org/) as a set of [host functions](https://extism.org/docs/concepts/host-functions). It's written entirely in rust, but the purpose is to show how one would build an API that any plugin could use to gain access to a plugin-host-provided iroh instance.
+
+## Requirements
+
+You'll need the latest version of the rust toolchain, and the `wasm32-unknown-unknown` target. Install it with:
+
+```
+rustup target add wasm32-unknown-unknown
+```
+
+## Running the Example
+
+There are three 
+
+1. Build the WASM plugin
+2. Get a blob ticket
+3. run the host
+
+### 1. Build the WASM plugin
+
+`cd` to `extism/plugin` and run `cargo build --target wasm32-unknown-unknown`. If you get an error, check the requirements section above. This will generate a WASM plugin that actually calls the blob download host function.
+
+### 2. Get a blob ticket
+
+This example needs a `Raw` blob ticket, which you can get from `iroh blob add` on a raw file, or just use this one, which should be hosted on iroh.network:
+```
+blobabk62aofuwwwu5zb5ocvzj5v3rtqt6siglyuhoxhqtu4fxravvoteajcnb2hi4dthixs65ltmuys2mjomrsxe4bonfzg62bonzsxi53pojvs4lydaac2cyt22erablaraaa5ciqbfiaqj7ya6cbpuaaaaaaaaaaaahjceagucztuhgez4qucv2733xphmgpc2nkgj54od2vuygn6sz4zzxo6ce
+```
+(it's a text file that says hello)
+
+### 3. Run the host
+
+The host code will actually call the example. `cd` to `./extism/host` and run
+
+```
+cargo run <TICKET>
+```
+
+pasting in the ticket you'd like to fetch. It'll fetch the bytes & print it out, proving this host can provide iroh to extism plugins!
+
+
+## Future work
+
+What would be truly neat would be to reverse the direction, and write a plugin that brings iroh to a host. For that we'd need to be able to compile some/all of iroh to WASM, which we're [working on](https://github.com/n0-computer/iroh/issues/1803).