oci-loader and path-loader components

.github/workflows/ci.yaml

@@ -19,6 +19,12 @@ jobs:
         uses: cargo-bins/cargo-binstall@main
       - name: Install wasmtime
         run: cargo binstall --force wasmtime-cli
+      - name: Install static-config
+        run: cargo binstall --force static-config
+      - name: Install wac-cli
+        # hack to add map support to wac, remove once this PR lands
+        run: cargo +stable install --force --git https://github.com/salmans/wac --branch map-defined-type wac-cli
+        # run: cargo binstall --force wac-cli
       - name: Install wkg
         run: cargo binstall --force wkg
       - name: Install wasm-tools

Cargo.toml

@@ -1,10 +1,7 @@
 [workspace]
 resolver = "2"
 members = [
-    "components/*",
-]
-exclude = [
-    "components/wit",
+    "components/*/src/..",
 ]
 
 [workspace.dependencies]

Makefile

@@ -4,7 +4,11 @@ export RUST_BACKTRACE ?= 1
 export WASMTIME_BACKTRACE_DETAILS ?= 1
 WKG_CONFIG_FILE ?= $(dir $(abspath $(lastword $(MAKEFILE_LIST)))).config/wasm-pkg/config.toml
 
-COMPONENTS = $(sort $(notdir $(patsubst %/,%,$(dir $(wildcard  components/*/Cargo.toml)))))
+CARGO_COMPONENTS = $(sort $(notdir $(patsubst %/,%,$(dir $(wildcard  components/*/Cargo.toml)))))
+CONFIG_COMPONENTS = $(sort $(notdir $(patsubst %/,%,$(dir $(wildcard  components/*/*.properties)))))
+WAC_COMPONENTS = $(sort $(notdir $(patsubst %/,%,$(dir $(wildcard  components/*/*.wac)))))
+WKG_COMPONENTS = $(sort $(notdir $(patsubst %/,%,$(dir $(wildcard  components/*/*.wkg)))))
+COMPONENTS = $(sort $(CARGO_COMPONENTS) $(CONFIG_COMPONENTS) $(WAC_COMPONENTS) $(WKG_COMPONENTS))
 
 .PHONY: all
 all: components
@@ -27,26 +31,52 @@ define BUILD_COMPONENT
 .PHONY: components/$1
 components/$1: lib/$1.wasm lib/$1.debug.wasm
 
+ifneq ($(wildcard components/$1/Cargo.toml),)
+
 lib/$1.wasm: Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f)
-	@$(eval target := $(shell yq -r '.package.default-target // "wasm32-unknown-unknown"' components/$1/Cargo.toml))
-	cargo build -p $1 --target $(target) --release
-ifeq ($(target),wasm32-unknown-unknown)
-	wasm-tools component new target/$(target)/release/$(subst -,_,$1).wasm -o lib/$1.wasm
-else
-	cp target/$(target)/release/$(subst -,_,$1).wasm lib/$1.wasm
-endif
+	cargo build -p $1 --target wasm32-unknown-unknown --release
+	wasm-tools component new target/wasm32-unknown-unknown/release/$(subst -,_,$1).wasm -o lib/$1.wasm
 	cp components/$1/README.md lib/$1.wasm.md
 
 lib/$1.debug.wasm: Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f)
-	@$(eval target := $(shell yq -r '.package.default-target // "wasm32-unknown-unknown"' components/$1/Cargo.toml))
-	cargo build --target $(target) -p $1
-ifeq ($(target),wasm32-unknown-unknown)
-	wasm-tools component new target/$(target)/debug/$(subst -,_,$1).wasm -o lib/$1.debug.wasm
-else
-	cp target/$(target)/debug/$(subst -,_,$1).wasm lib/$1.debug.wasm
-endif
+	cargo build --target wasm32-unknown-unknown -p $1
+	wasm-tools component new target/wasm32-unknown-unknown/debug/$(subst -,_,$1).wasm -o lib/$1.debug.wasm
 	cp components/$1/README.md lib/$1.debug.wasm.md
 
+else ifneq ($(wildcard components/$1/$1.properties),)
+
+lib/$1.wasm: components/$1/$1.properties components/$1/README.md
+	static-config -f components/$1/$1.properties -o lib/$1.wasm
+	cp components/$1/README.md lib/$1.wasm.md
+
+lib/$1.debug.wasm: components/$1/$1.properties components/$1/README.md
+	static-config -f components/$1/$1.properties -o lib/$1.debug.wasm
+	cp components/$1/README.md lib/$1.debug.wasm.md
+
+else ifneq ($(wildcard components/$1/$1.wac),)
+
+WAC_DEPS_$1 := $(shell wac parse components/$1/$1.wac 2> /dev/null | jq -r '[.. | .package?.name? | strings | select(startswith("local:")) | sub("^local:"; "")] | unique[]')
+
+lib/$1.wasm: components/$1/$1.wac components/$1/README.md $$(foreach component,$$(WAC_DEPS_$1),lib/$$(component).wasm)
+	wac compose $$(foreach component,$$(WAC_DEPS_$1),-d local:$$(component)=lib/$$(component).wasm) -o lib/$1.wasm components/$1/$1.wac
+	cp components/$1/README.md lib/$1.wasm.md
+
+lib/$1.debug.wasm: components/$1/$1.wac components/$1/README.md $$(foreach component,$$(WAC_DEPS_$1),lib/$$(component).debug.wasm)
+	wac compose $$(foreach component,$$(WAC_DEPS_$1),-d local:$$(component)=lib/$$(component).debug.wasm) -o lib/$1.debug.wasm components/$1/$1.wac
+	cp components/$1/README.md lib/$1.debug.wasm.md
+
+else ifneq ($(wildcard components/$1/$1.wkg),)
+
+lib/$1.wasm: components/$1/$1.wkg components/$1/README.md
+	wkg oci pull $(shell cat components/$1/$1.wkg 2> /dev/null | head -1) -o lib/$1.wasm
+	cp components/$1/README.md lib/$1.wasm.md
+
+lib/$1.debug.wasm: components/$1/$1.wkg components/$1/README.md
+	wkg oci pull $(shell cat components/$1/$1.wkg  2> /dev/null | tail -1 2> /dev/null) -o lib/$1.debug.wasm
+	cp components/$1/README.md lib/$1.debug.wasm.md
+
+endif
+
 endef
 
 $(foreach component,$(COMPONENTS),$(eval $(call BUILD_COMPONENT,$(component))))
@@ -65,7 +95,7 @@ components/wit/deps: wit/deps components/wkg.toml $(WKG_CONFIG_FILE) $(shell fin
 	( cd components && wkg fetch --config $(WKG_CONFIG_FILE) )
 
 .PHONY: publish ## Publish each component in the lib directory
-publish: $(shell find lib -maxdepth 1 -type f -name "*.wasm" | sed -e 's:^lib/:publish-:g')
+publish: $(shell find lib -maxdepth 1 -type f -name "*.wasm" ! -name "dep-*" ! -name "test-*" | sed -e 's:^lib/:publish-:g')
 
 .PHONY: publish-%
 publish-%:

README.md

@@ -18,9 +18,11 @@ A [dev container](https://containers.dev) is available that contains the necessa
 
 Prereqs:
 - a rust toolchain
+- [`jq`](https://jqlang.org)
+- [`static-config`](https://github.com/componentized/static-config)
+- [`wac`](https://github.com/bytecodealliance/wac)
 - [`wasm-tools`](https://github.com/bytecodealliance/wasm-tools)
 - [`wkg`](https://github.com/bytecodealliance/wasm-pkg-tools)
-- [`yq`](https://github.com/mikefarah/yq)
 
 ```sh
 make components
@@ -31,6 +33,9 @@ make components
 - [`extract-wit`](./components/extract-wit/)
 - [`filesystem-loader`](./components/filesystem-loader/)
 - [`http-loader`](./components/http-loader/)
+- [`oci-loader`](./components/oci-loader/)
+- [`path-loader`](./components/path-loader/)
+- [`path-loader-router`](./components/path-loader-router/)
 - [`wac-loader`](./components/wac-loader/)
 
 ## Community

components/dep-componentized-http-client/README.md

@@ -0,0 +1,3 @@
+# `dep-componentized-http-client`
+
+from https://github.com/componentized/http/tree/main/components/http-client

components/dep-componentized-http-client/dep-componentized-http-client.wkg

@@ -0,0 +1,2 @@
+ghcr.io/componentized/http/http-client:0.1.0-dev@sha256:c8ddd9363ae5db549f366fdf9339b64342909a234083d96edbc187f168192e93
+ghcr.io/componentized/http/http-client:0.1.0-dev_debug@sha256:938b063bbe3c219c671f01af6abadc55b4b6035752188befb7eae77e385a028b

components/dep-componentized-oci-client/README.md

@@ -0,0 +1,3 @@
+# `dep-componentized-oci-client`
+
+from https://github.com/componentized/oci/tree/main/components/client

components/dep-componentized-oci-client/dep-componentized-oci-client.wkg

@@ -0,0 +1,2 @@
+ghcr.io/componentized/oci/client:0.0.0-dev@sha256:c370def08aa59bb73f7124c094b045b49d489e3631afeadbe474106bf80d5447
+ghcr.io/componentized/oci/client:0.0.0-dev_debug@sha256:d256ffa0c6ae712d5fae2501587cc74f383767d38d0ca3b96754586b62b6782b

components/filesystem-loader/Cargo.toml

@@ -1,13 +1,9 @@
-cargo-features = ["per-package-target"]
-
 [package]
 name = "filesystem-loader"
 version = "0.1.0"
 edition = "2021"
 license = "Apache-2.0"
 
-default-target = "wasm32-wasip3"
-
 [lib]
 crate-type = ["cdylib"]
 

components/filesystem-loader/src/lib.rs

@@ -1,25 +1,158 @@
 #![no_main]
 
-use std::{fs, io};
+use std::path::Path;
 
 use crate::{
-    exports::componentized::component::path_loader::Guest,
-    exports::componentized::component::types::{Component, Error},
+    exports::componentized::component::{
+        path_loader::Guest,
+        types::{Component, Error},
+    },
+    wasi::filesystem::{preopens, types as filesystem},
 };
 
 pub(crate) struct FilesystemLoader;
 
 impl Guest for FilesystemLoader {
     #[allow(async_fn_in_trait)]
     async fn load(path: String) -> Result<Component, Error> {
-        let component = fs::read(path)?;
+        let (dir, path) = resolve(&path)?;
+        let file = dir
+            .open_at(
+                filesystem::PathFlags::SYMLINK_FOLLOW,
+                path,
+                filesystem::OpenFlags::empty(),
+                filesystem::DescriptorFlags::READ,
+            )
+            .await?;
+        let (data, result) = file.read_via_stream(0);
+        let component = data.collect().await;
+        result.await?;
         Ok(component)
     }
 }
 
-impl From<io::Error> for Error {
-    fn from(value: io::Error) -> Self {
-        Self::Other(Some(value.to_string()))
+/// Resolve a path to the preopened directory that contains it and the path relative to that
+/// directory. When multiple preopens match, the most specific one wins.
+fn resolve(path: &str) -> Result<(filesystem::Descriptor, String), Error> {
+    let path = Path::new(path);
+    let mut resolved: Option<(usize, filesystem::Descriptor, String)> = None;
+    for (dir, name) in preopens::get_directories() {
+        let prefix = Path::new(&name);
+        let relative = if path.is_relative() && prefix == Path::new(".") {
+            Some(path)
+        } else {
+            path.strip_prefix(prefix).ok()
+        };
+        let Some(relative) = relative.and_then(Path::to_str) else {
+            continue;
+        };
+        let specificity = prefix.components().count();
+        if resolved.as_ref().is_none_or(|(s, _, _)| specificity > *s) {
+            resolved = Some((specificity, dir, relative.to_string()));
+        }
+    }
+    resolved.map(|(_, dir, path)| (dir, path)).ok_or_else(|| {
+        Error::Other(Some(format!(
+            "filesystem: no preopen for path {}",
+            path.display()
+        )))
+    })
+}
+
+impl From<filesystem::ErrorCode> for Error {
+    fn from(value: filesystem::ErrorCode) -> Self {
+        match value {
+            filesystem::ErrorCode::Access => Self::Other(Some("filesystem: access".to_string())),
+            filesystem::ErrorCode::Already => Self::Other(Some("filesystem: already".to_string())),
+            filesystem::ErrorCode::BadDescriptor => {
+                Self::Other(Some("filesystem: bad-descriptor".to_string()))
+            }
+            filesystem::ErrorCode::Busy => Self::Other(Some("filesystem: busy".to_string())),
+            filesystem::ErrorCode::Deadlock => {
+                Self::Other(Some("filesystem: deadlock".to_string()))
+            }
+            filesystem::ErrorCode::Quota => Self::Other(Some("filesystem: quota".to_string())),
+            filesystem::ErrorCode::Exist => Self::Other(Some("filesystem: exist".to_string())),
+            filesystem::ErrorCode::FileTooLarge => {
+                Self::Other(Some("filesystem: file-too-large".to_string()))
+            }
+            filesystem::ErrorCode::IllegalByteSequence => {
+                Self::Other(Some("filesystem: illegal-byte-sequence".to_string()))
+            }
+            filesystem::ErrorCode::InProgress => {
+                Self::Other(Some("filesystem: in-progress".to_string()))
+            }
+            filesystem::ErrorCode::Interrupted => {
+                Self::Other(Some("filesystem: interrupted".to_string()))
+            }
+            filesystem::ErrorCode::Invalid => Self::Other(Some("filesystem: invalid".to_string())),
+            filesystem::ErrorCode::Io => Self::Other(Some("filesystem: io".to_string())),
+            filesystem::ErrorCode::IsDirectory => {
+                Self::Other(Some("filesystem: is-directory".to_string()))
+            }
+            filesystem::ErrorCode::Loop => Self::Other(Some("filesystem: loop".to_string())),
+            filesystem::ErrorCode::TooManyLinks => {
+                Self::Other(Some("filesystem: too-many-links".to_string()))
+            }
+            filesystem::ErrorCode::MessageSize => {
+                Self::Other(Some("filesystem: message-size".to_string()))
+            }
+            filesystem::ErrorCode::NameTooLong => {
+                Self::Other(Some("filesystem: name-too-long".to_string()))
+            }
+            filesystem::ErrorCode::NoDevice => {
+                Self::Other(Some("filesystem: no-device".to_string()))
+            }
+            filesystem::ErrorCode::NoEntry => Self::Other(Some("filesystem: no-entry".to_string())),
+            filesystem::ErrorCode::NoLock => Self::Other(Some("filesystem: no-lock".to_string())),
+            filesystem::ErrorCode::InsufficientMemory => {
+                Self::Other(Some("filesystem: insufficient-memory".to_string()))
+            }
+            filesystem::ErrorCode::InsufficientSpace => {
+                Self::Other(Some("filesystem: insufficient-space".to_string()))
+            }
+            filesystem::ErrorCode::NotDirectory => {
+                Self::Other(Some("filesystem: not-directory".to_string()))
+            }
+            filesystem::ErrorCode::NotEmpty => {
+                Self::Other(Some("filesystem: not-empty".to_string()))
+            }
+            filesystem::ErrorCode::NotRecoverable => {
+                Self::Other(Some("filesystem: not-recoverable".to_string()))
+            }
+            filesystem::ErrorCode::Unsupported => {
+                Self::Other(Some("filesystem: unsupported".to_string()))
+            }
+            filesystem::ErrorCode::NoTty => Self::Other(Some("filesystem: no-tty".to_string())),
+            filesystem::ErrorCode::NoSuchDevice => {
+                Self::Other(Some("filesystem: no-such-device".to_string()))
+            }
+            filesystem::ErrorCode::Overflow => {
+                Self::Other(Some("filesystem: overflow".to_string()))
+            }
+            filesystem::ErrorCode::NotPermitted => {
+                Self::Other(Some("filesystem: not-permitted".to_string()))
+            }
+            filesystem::ErrorCode::Pipe => Self::Other(Some("filesystem: pipe".to_string())),
+            filesystem::ErrorCode::ReadOnly => {
+                Self::Other(Some("filesystem: read-only".to_string()))
+            }
+            filesystem::ErrorCode::InvalidSeek => {
+                Self::Other(Some("filesystem: invalid-seek".to_string()))
+            }
+            filesystem::ErrorCode::TextFileBusy => {
+                Self::Other(Some("filesystem: text-file-busy".to_string()))
+            }
+            filesystem::ErrorCode::CrossDevice => {
+                Self::Other(Some("filesystem: cross-device".to_string()))
+            }
+            filesystem::ErrorCode::Other(None) => {
+                Self::Other(Some("filesystem: other".to_string()))
+            }
+            filesystem::ErrorCode::Other(Some(message)) => {
+                Self::Other(Some(format!("filesystem: other: {message}")))
+            }
+        }
     }
 }
 

components/oci-loader/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "oci-loader"
+version = "0.1.0"
+edition = "2021"
+license = "Apache-2.0"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+wit-bindgen = { workspace = true }

components/oci-loader/README.md

@@ -0,0 +1,3 @@
+# `oci-loader`
+
+Loads a component from an OCI registry by reference.