From b9d40cd9b0b40874fb9287de0f930628f7d14c96 Mon Sep 17 00:00:00 2001 From: Scott Andrews Date: Tue, 22 Sep 2026 16:56:43 -0400 Subject: [PATCH 1/3] oci-loader and path-loader components The oci-loader uses the componentized/oci client to load a wasm component from an OCI registry. The path-loader uses the other loader components routing based on the path. Using a wac fork until map support lands. Signed-off-by: Scott Andrews --- .github/workflows/ci.yaml | 6 + Cargo.lock | 14 + Cargo.toml | 1 + Makefile | 33 +- README.md | 5 + components/oci-loader/Cargo.toml | 11 + components/oci-loader/README.md | 3 + components/oci-loader/src/lib.rs | 134 ++++++++ components/path-loader-router/Cargo.toml | 11 + components/path-loader-router/README.md | 14 + components/path-loader-router/src/lib.rs | 40 +++ components/path-loader/README.md | 14 + components/path-loader/path-loader.wac | 8 + .../componentized-oci-0.0.0-dev/package.wit | 287 ++++++++++++++++++ components/wit/worlds.wit | 24 ++ components/wkg.lock | 9 + 16 files changed, 611 insertions(+), 3 deletions(-) create mode 100644 components/oci-loader/Cargo.toml create mode 100644 components/oci-loader/README.md create mode 100644 components/oci-loader/src/lib.rs create mode 100644 components/path-loader-router/Cargo.toml create mode 100644 components/path-loader-router/README.md create mode 100644 components/path-loader-router/src/lib.rs create mode 100644 components/path-loader/README.md create mode 100644 components/path-loader/path-loader.wac create mode 100644 components/wit/deps/componentized-oci-0.0.0-dev/package.wit diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8cf9672..8edd45b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,6 +19,12 @@ jobs: uses: cargo-bins/cargo-binstall@main - name: Install wasmtime run: cargo binstall --force wasmtime-cli + - name: Install static-config + run: cargo binstall --force static-config + - name: Install wac-cli + # hack to add map support to wac, remove once this PR lands + run: cargo install --git https://github.com/salmans/wac --branch map-defined-type wac-cli + # run: cargo binstall --force wac-cli - name: Install wkg run: cargo binstall --force wkg - name: Install wasm-tools diff --git a/Cargo.lock b/Cargo.lock index fd045d8..5f2a71d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -386,6 +386,20 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "oci-loader" +version = "0.1.0" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "path-loader-router" +version = "0.1.0" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "percent-encoding" version = "2.3.2" diff --git a/Cargo.toml b/Cargo.toml index 6bc315b..4150d88 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "components/*", ] exclude = [ + "components/path-loader", "components/wit", ] diff --git a/Makefile b/Makefile index 1b3db27..555db8b 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,10 @@ export RUST_BACKTRACE ?= 1 export WASMTIME_BACKTRACE_DETAILS ?= 1 WKG_CONFIG_FILE ?= $(dir $(abspath $(lastword $(MAKEFILE_LIST)))).config/wasm-pkg/config.toml -COMPONENTS = $(sort $(notdir $(patsubst %/,%,$(dir $(wildcard components/*/Cargo.toml))))) +CARGO_COMPONENTS = $(sort $(notdir $(patsubst %/,%,$(dir $(wildcard components/*/Cargo.toml))))) +CONFIG_COMPONENTS = $(sort $(notdir $(patsubst %/,%,$(dir $(wildcard components/*/*.properties))))) +WAC_COMPONENTS = $(sort $(notdir $(patsubst %/,%,$(dir $(wildcard components/*/*.wac))))) +COMPONENTS = $(CARGO_COMPONENTS) $(CONFIG_COMPONENTS) $(WAC_COMPONENTS) .PHONY: all all: components @@ -27,8 +30,10 @@ define BUILD_COMPONENT .PHONY: components/$1 components/$1: lib/$1.wasm lib/$1.debug.wasm +ifneq ($(wildcard components/$1/Cargo.toml),) + lib/$1.wasm: Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f) - @$(eval target := $(shell yq -r '.package.default-target // "wasm32-unknown-unknown"' components/$1/Cargo.toml)) + @$(eval target := $(shell yq -r '.package.default-target // "wasm32-unknown-unknown"' components/$1/Cargo.toml 2> /dev/null)) cargo build -p $1 --target $(target) --release ifeq ($(target),wasm32-unknown-unknown) wasm-tools component new target/$(target)/release/$(subst -,_,$1).wasm -o lib/$1.wasm @@ -38,7 +43,7 @@ endif cp components/$1/README.md lib/$1.wasm.md lib/$1.debug.wasm: Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f) - @$(eval target := $(shell yq -r '.package.default-target // "wasm32-unknown-unknown"' components/$1/Cargo.toml)) + @$(eval target := $(shell yq -r '.package.default-target // "wasm32-unknown-unknown"' components/$1/Cargo.toml 2> /dev/null)) cargo build --target $(target) -p $1 ifeq ($(target),wasm32-unknown-unknown) wasm-tools component new target/$(target)/debug/$(subst -,_,$1).wasm -o lib/$1.debug.wasm @@ -47,6 +52,28 @@ else endif cp components/$1/README.md lib/$1.debug.wasm.md +else ifneq ($(wildcard components/$1/$1.properties),) + +lib/$1.wasm: components/$1/$1.properties components/$1/README.md + static-config -f components/$1/$1.properties -o lib/$1.wasm + cp components/$1/README.md lib/$1.wasm.md + +lib/$1.debug.wasm: components/$1/$1.properties components/$1/README.md + static-config -f components/$1/$1.properties -o lib/$1.debug.wasm + cp components/$1/README.md lib/$1.debug.wasm.md + +else ifneq ($(wildcard components/$1/$1.wac),) + +lib/$1.wasm: components/$1/$1.wac components/$1/README.md $(foreach component,$(CARGO_COMPONENTS),lib/$(component).wasm) $(foreach component,$(CONFIG_COMPONENTS),lib/$(component).wasm) + wac compose $(foreach component,$(COMPONENTS),-d local:$(component)=lib/$(component).wasm) -o lib/$1.wasm components/$1/$1.wac + cp components/$1/README.md lib/$1.wasm.md + +lib/$1.debug.wasm: components/$1/$1.wac components/$1/README.md $(foreach component,$(CARGO_COMPONENTS),lib/$(component).debug.wasm) $(foreach component,$(CONFIG_COMPONENTS),lib/$(component).debug.wasm) + wac compose $(foreach component,$(COMPONENTS),-d local:$(component)=lib/$(component).debug.wasm) -o lib/$1.debug.wasm components/$1/$1.wac + cp components/$1/README.md lib/$1.debug.wasm.md + +endif + endef $(foreach component,$(COMPONENTS),$(eval $(call BUILD_COMPONENT,$(component)))) diff --git a/README.md b/README.md index 7f2a5d5..0e3fe8c 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ A [dev container](https://containers.dev) is available that contains the necessa Prereqs: - a rust toolchain +- [`static-config`](https://github.com/componentized/static-config) +- [`wac`](https://github.com/bytecodealliance/wac) - [`wasm-tools`](https://github.com/bytecodealliance/wasm-tools) - [`wkg`](https://github.com/bytecodealliance/wasm-pkg-tools) - [`yq`](https://github.com/mikefarah/yq) @@ -31,6 +33,9 @@ make components - [`extract-wit`](./components/extract-wit/) - [`filesystem-loader`](./components/filesystem-loader/) - [`http-loader`](./components/http-loader/) +- [`oci-loader`](./components/oci-loader/) +- [`path-loader`](./components/path-loader/) +- [`path-loader-router`](./components/path-loader-router/) - [`wac-loader`](./components/wac-loader/) ## Community diff --git a/components/oci-loader/Cargo.toml b/components/oci-loader/Cargo.toml new file mode 100644 index 0000000..cfb2534 --- /dev/null +++ b/components/oci-loader/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "oci-loader" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +wit-bindgen = { workspace = true } diff --git a/components/oci-loader/README.md b/components/oci-loader/README.md new file mode 100644 index 0000000..9408ca5 --- /dev/null +++ b/components/oci-loader/README.md @@ -0,0 +1,3 @@ +# `oci-loader` + +Loads a component from an OCI registry by reference. diff --git a/components/oci-loader/src/lib.rs b/components/oci-loader/src/lib.rs new file mode 100644 index 0000000..16f24f7 --- /dev/null +++ b/components/oci-loader/src/lib.rs @@ -0,0 +1,134 @@ +#![no_main] + +use crate::{ + componentized::oci::client::{self as oci, Digest, Reference}, + exports::componentized::component::{ + path_loader::Guest, + types::{Component, Error}, + }, +}; + +pub(crate) struct OCILoader; + +impl Guest for OCILoader { + #[allow(async_fn_in_trait)] + async fn load(path: String) -> Result { + let manifest_reference = oci::parse_reference(&path)?; + let manifest = match oci::get_manifest(manifest_reference.clone()).await? { + oci::Manifest::OciImageV1(oci_image_manifest_v1) => oci_image_manifest_v1, + _ => Err(Error::Other(Some("unexpected manifest".to_string())))?, + }; + match manifest.config.media_type { + oci::MediaType::ApplicationVndWasmConfigV0(oci::MediaTypeSuffix::Json) => {} + _ => Err(Error::Other(Some( + "unexpected config media type".to_string(), + )))?, + }; + if manifest.layers.len() != 1 { + Err(Error::Other(Some("unknown artifact layout".to_string())))? + } + let component_descriptor = manifest.layers.first().unwrap(); + match component_descriptor.media_type { + oci::MediaType::ApplicationWasm => {} + _ => Err(Error::Other(Some( + "unknown artifact media type".to_string(), + )))?, + } + + let config_reference = manifest_reference.with_digest(&manifest.config.digest); + let config = match oci::get_config(config_reference).await? { + oci::Config::WasmV0(wasm_config_v0) => wasm_config_v0, + _ => Err(Error::Other(Some("unexpected config".to_string())))?, + }; + if config.component.is_none() { + Err(Error::Other(Some("not a component".to_string())))? + } + + let component_reference = + manifest_reference.with_digest(&manifest.layers.first().unwrap().digest); + let component = oci::get_blob(component_reference).await?; + if component_descriptor.size != component.len() as u64 { + Err(Error::Other(Some(format!( + "component size mismatch: expected {expected}, actual {actual}", + expected = component_descriptor.size, + actual = component.len() + ))))? + } + + Ok(component) + } +} + +impl Reference { + fn with_digest(&self, digest: &Digest) -> Self { + Self { + registry: self.registry.clone(), + repository: self.repository.clone(), + tag: None, + digest: Some(digest.clone()), + } + } +} + +impl From for Error { + fn from(value: oci::ErrorCode) -> Self { + match value { + oci::ErrorCode::BlobUnknown(message) => { + Self::Other(Some(format!("OCI error blob-unknown: {message}"))) + } + oci::ErrorCode::BlobUploadInvalid(message) => { + Self::Other(Some(format!("OCI error blob-upload-invalid: {message}"))) + } + oci::ErrorCode::BlobUploadUnknown(message) => { + Self::Other(Some(format!("OCI error blob-upload-unknown: {message}"))) + } + oci::ErrorCode::DigestInvalid(message) => { + Self::Other(Some(format!("OCI error digest-invalid: {message}"))) + } + oci::ErrorCode::ManifestBlobUnknown(message) => { + Self::Other(Some(format!("OCI error manifest-blob-unknown: {message}"))) + } + oci::ErrorCode::ManifestInvalid(message) => { + Self::Other(Some(format!("OCI error manifest-invalid: {message}"))) + } + oci::ErrorCode::ManifestUnknown(message) => { + Self::Other(Some(format!("OCI error manifest-unknown: {message}"))) + } + oci::ErrorCode::NameInvalid(message) => { + Self::Other(Some(format!("OCI error name-invalid: {message}"))) + } + oci::ErrorCode::NameUnknown(message) => { + Self::Other(Some(format!("OCI error name-unknown: {message}"))) + } + oci::ErrorCode::SizeInvalid(message) => { + Self::Other(Some(format!("OCI error size-invalid: {message}"))) + } + oci::ErrorCode::Unauthorized(message) => { + Self::Other(Some(format!("OCI error unauthorized: {message}"))) + } + oci::ErrorCode::Denied(message) => { + Self::Other(Some(format!("OCI error denied: {message}"))) + } + oci::ErrorCode::Unsupported(message) => { + Self::Other(Some(format!("OCI error unsupported: {message}"))) + } + oci::ErrorCode::Toomanyrequests(_) => { + // TODO detect and retry request if within a grace period + Self::Other(Some(format!("OCI error toomanyrequests"))) + } + oci::ErrorCode::Other(Some(message)) => { + Self::Other(Some(format!("OCI error other: {message}"))) + } + oci::ErrorCode::Other(None) => Self::Other(Some(format!("OCI error other"))), + } + } +} + +wit_bindgen::generate!({ + path: "../wit", + world: "oci-loader", + merge_structurally_equal_types: true, + generate_all +}); + +export!(OCILoader); diff --git a/components/path-loader-router/Cargo.toml b/components/path-loader-router/Cargo.toml new file mode 100644 index 0000000..ae29e06 --- /dev/null +++ b/components/path-loader-router/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "path-loader-router" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +wit-bindgen = { workspace = true } diff --git a/components/path-loader-router/README.md b/components/path-loader-router/README.md new file mode 100644 index 0000000..a1c07c1 --- /dev/null +++ b/components/path-loader-router/README.md @@ -0,0 +1,14 @@ +# `path-loader-router` + +Routes a component loader based on the path. + +See [`path-loader`](../path-loader/) for the router with the loader components installed. + +Based on the path prefix the request is routed to an appropriate loader: + +- `http://` -> http-loader: scheme is preserved +- `https://` -> http-loader: scheme is preserved +- `file:///` -> filesystem-loader: scheme is dropped, last '/' is preserved with path +- `file://` -> invalid, other hosts are not supported +- `file:` -> filesystem-loader: scheme is dropped +- `oci:` -> oci-loader: scheme is dropped diff --git a/components/path-loader-router/src/lib.rs b/components/path-loader-router/src/lib.rs new file mode 100644 index 0000000..2b93263 --- /dev/null +++ b/components/path-loader-router/src/lib.rs @@ -0,0 +1,40 @@ +#![no_main] + +use crate::exports::componentized::component::{ + path_loader::Guest, + types::{Component, Error}, +}; + +pub(crate) struct PathLoader; + +impl Guest for PathLoader { + #[allow(async_fn_in_trait)] + async fn load(path: String) -> Result { + if path.starts_with("http:") { + http_loader::load(path).await + } else if path.starts_with("https:") { + http_loader::load(path).await + } else if path.starts_with("file:///") { + filesystem_loader::load(path[7..].to_string()).await + } else if path.starts_with("file://") { + Err(Error::Other(Some(format!( + "invalid filesystem-path loader: {path}" + )))) + } else if path.starts_with("file:") { + filesystem_loader::load(path[5..].to_string()).await + } else if path.starts_with("oci:") { + oci_loader::load(path[4..].to_string()).await + } else { + Err(Error::Other(Some(format!("unknown path loader: {path}")))) + } + } +} + +wit_bindgen::generate!({ + path: "../wit", + world: "path-loader", + merge_structurally_equal_types: true, + generate_all +}); + +export!(PathLoader); diff --git a/components/path-loader/README.md b/components/path-loader/README.md new file mode 100644 index 0000000..9782106 --- /dev/null +++ b/components/path-loader/README.md @@ -0,0 +1,14 @@ +# `path-loader` + +Loads a component from a path. + +See [`path-loader-router`](../path-loader-router/) for the router without the loader components installed. + +Based on the path prefix the request is routed to an appropriate loader: + +- `http://` -> http-loader: scheme is preserved +- `https://` -> http-loader: scheme is preserved +- `file:///` -> filesystem-loader: scheme is dropped, last '/' is preserved with path +- `file://` -> invalid, other hosts are not supported +- `file:` -> filesystem-loader: scheme is dropped +- `oci:` -> oci-loader: scheme is dropped diff --git a/components/path-loader/path-loader.wac b/components/path-loader/path-loader.wac new file mode 100644 index 0000000..d5210aa --- /dev/null +++ b/components/path-loader/path-loader.wac @@ -0,0 +1,8 @@ +package componentized:component; + +export new local:path-loader-router{ + filesystem-loader: new local:filesystem-loader{ ... }.path-loader, + http-loader: new local:http-loader{ ... }.path-loader, + oci-loader: new local:oci-loader{ ... }.path-loader, + ... +}...; diff --git a/components/wit/deps/componentized-oci-0.0.0-dev/package.wit b/components/wit/deps/componentized-oci-0.0.0-dev/package.wit new file mode 100644 index 0000000..fae6a8e --- /dev/null +++ b/components/wit/deps/componentized-oci-0.0.0-dev/package.wit @@ -0,0 +1,287 @@ +package componentized:oci@0.0.0-dev; + +interface client { + use wasi:clocks/system-clock@0.3.0.{instant}; + + variant retry-after { + date(instant), + delay-seconds(u32), + } + + variant error-code { + /// blob unknown to registry (code-1 `BLOB_UNKNOWN`) + blob-unknown(string), + /// blob upload invalid (code-2 `BLOB_UPLOAD_INVALID`) + blob-upload-invalid(string), + /// blob upload unknown to registry (code-3`BLOB_UPLOAD_UNKNOWN`) + blob-upload-unknown(string), + /// provided digest did not match uploaded content (code-4 `DIGEST_INVALID`) + digest-invalid(string), + /// manifest references a manifest or blob unknown to registry (code-5 `MANIFEST_BLOB_UNKNOWN`) + manifest-blob-unknown(string), + /// manifest invalid (code-6 `MANIFEST_INVALID`) + manifest-invalid(string), + /// manifest unknown to registry (code-7 `MANIFEST_UNKNOWN`) + manifest-unknown(string), + /// invalid repository name (code-8 `NAME_INVALID`) + name-invalid(string), + /// repository name not known to registry (code-9 `NAME_UNKNOWN`) + name-unknown(string), + /// provided length did not match content length (code-10 `SIZE_INVALID`) + size-invalid(string), + /// authentication required (code-11 `UNAUTHORIZED`) + unauthorized(string), + /// requested access to the resource is denied (code-12 `DENIED`) + denied(string), + /// the operation is unsupported (code-13 `UNSUPPORTED`) + unsupported(string), + /// too many requests (code-14 `TOOMANYREQUESTS`) + toomanyrequests(option), + /// other + other(option), + } + + variant schema-version { + v2, + other(option), + } + + variant media-type-suffix { + /// json encoded + json, + /// compressed with gzip + gzip, + /// compressed with zstd + zstd, + /// other encoding + other(option), + } + + /// common media types for oci artifacts + variant media-type { + /// content descriptor + application-vnd-oci-descriptor-v1(media-type-suffix), + /// oci layout + application-vnd-oci-layout-header-v1(media-type-suffix), + /// image index + application-vnd-oci-image-index-v1(media-type-suffix), + /// image manifest + application-vnd-oci-image-manifest-v1(media-type-suffix), + /// image config + application-vnd-oci-image-config-v1(media-type-suffix), + /// layer, as a tar archive + application-vnd-oci-image-layer-v1-tar(media-type-suffix), + /// empty for unused descriptors + application-vnd-oci-empty-v1(media-type-suffix), + /// layer, as a tar archive + application-vnd-oci-image-layer-nondistributable-v1-tar(media-type-suffix), + /// wasm config + application-vnd-wasm-config-v0(media-type-suffix), + /// wasm + application-wasm, + /// other + other(string), + } + + record oci-image-index-manifest-v1-manifest-platform { + /// CPU architecture which the binaries in this image are built to run on + architecture: string, + /// name of the operating system which the image is built to run on + os: string, + /// version of the operating system + os-version: option, + /// mandatory OS features + os-features: option>, + /// variant of the CPU + %variant: option, + } + + record oci-image-config-v1-config { + /// username or UID which is a platform-specific structure that allows specific control over which user the process run as + user: option, + /// set of ports to expose from a container running this image + exposed-ports: option>, + /// values act as defaults and are merged with any specified when creating a container + env: option>, + /// arguments to use as the command to execute when the container starts + entrypoint: option>, + /// arguments to the entrypoint of the container + cmd: option>, + /// directories describing where the process is likely to write data specific to a container instance + volumes: option>, + /// current working directory of the entrypoint process in the container + working-dir: option, + /// arbitrary metadata for the container + labels: option>, + /// system call signal that will be sent to the container to exit + stop-signal: option, + /// deprecated: present only for legacy compatibility + args-escaped: option, + } + + record oci-image-config-v1-content-addresses { + /// must be 'layers' + %type: string, + /// layer content hashes, in order from first to last + diff-ids: list, + } + + record oci-image-config-v1-history-entry { + /// combined date and time at which the layer was created + created: option, + /// author of the build point + author: option, + /// command which created the layer + created-by: option, + /// custom message set when creating the layer + comment: option, + /// mark if the history item created a filesystem diff + empty-layer: option, + } + + record oci-image-config-v1 { + /// combined date and time at which the image was created + created: option, + /// name and/or email address of the person or entity which created and is responsible for maintaining the image + author: option, + /// CPU architecture which the binaries in this image are built to run on + architecture: string, + /// name of the operating system which the image is built to run on + os: string, + /// version of the operating system + os-version: option, + /// mandatory OS features + os-features: option>, + /// variant of the specified CPU architecture + %variant: option, + /// execution parameters which should be used as a base when running a container using the image + config: option, + /// layer content addresses used by the image + rootfs: oci-image-config-v1-content-addresses, + /// history of each layer. The array is ordered from first to last + history: option>, + } + + record wasm-config-v0-component { + exports: list, + imports: list, + target: option, + } + + record digest { + algorithm: string, + encoded: string, + } + + record oci-descriptor-v1 { + /// media type of the referenced content + media-type: media-type, + /// digest of the targeted content + digest: digest, + /// size, in bytes, of the raw content + size: u64, + /// list of URIs from which this object MAY be downloaded + urls: option>, + /// arbitrary metadata for this descriptor + annotations: option>, + } + + record oci-image-manifest-v1 { + /// manifest schema version + schema-version: schema-version, + /// must contain the media type 'application/vnd.oci.image.manifest.v1+json'. + media-type: media-type, + /// type of an artifact when the manifest is used for an artifact + artifact-type: option, + /// configuration object for a container + config: oci-descriptor-v1, + /// list of layer descriptors + layers: list, + /// weak association to another manifest + subject: option, + /// arbitrary metadata for the image manifest + annotations: option>, + } + + record oci-image-index-manifest-v1-manifest { + /// media type of the subject manifest + media-type: media-type, + /// minimum runtime requirements of the image + platform: option, + /// descriptor of another manifest + subject: option, + /// arbitrary metadata for the image manifest + annotations: option>, + } + + record oci-image-index-manifest-v1 { + /// manifest schema version + schema-version: schema-version, + /// must contain the media type 'application/vnd.oci.image.index.v1+json'. + media-type: media-type, + /// type of an artifact when the manifest is used for an artifact + artifact-type: option, + /// manifests for specific platforms + manifests: list, + /// weak association to another manifest + subject: option, + /// arbitrary metadata for the image manifest + annotations: option>, + } + + variant manifest { + /// oci image + oci-image-v1(oci-image-manifest-v1), + /// oci image index + oci-image-index-v1(oci-image-index-manifest-v1), + /// other + other(list), + } + + record wasm-config-v0 { + /// combined date and time at which the image was created + created: option, + /// name and/or email address of the person or entity which created and is responsible for maintaining the image + author: option, + /// must match a GOARCH value + architecture: string, + /// must match a GOOS value + os: string, + /// digests of the layers in the same order + layer-digests: list, + /// component metadata + component: option, + } + + variant config { + /// oci iamge + oci-image-v1(oci-image-config-v1), + /// wasm module + wasm-v0(wasm-config-v0), + /// other + other(list), + } + + record reference { + registry: string, + repository: string, + tag: option, + digest: option, + } + + parse-reference: func(reference: string) -> result; + + resolve-digest: async func(reference: reference) -> result; + + get-blob: async func(reference: reference) -> result, error-code>; + + get-config: async func(reference: reference) -> result; + + get-manifest: async func(reference: reference) -> result; +} + +world imports { + import wasi:clocks/types@0.3.0; + import wasi:clocks/system-clock@0.3.0; + import client; +} diff --git a/components/wit/worlds.wit b/components/wit/worlds.wit index 6b1cacb..21607ad 100644 --- a/components/wit/worlds.wit +++ b/components/wit/worlds.wit @@ -17,6 +17,30 @@ world http-loader { export componentized:component/path-loader@0.0.0-dev; } +world oci-loader { + import componentized:oci/client@0.0.0-dev; + export componentized:component/types@0.0.0-dev; + export componentized:component/path-loader@0.0.0-dev; +} + +world path-loader { + import filesystem-loader: interface { + use componentized:component/types@0.0.0-dev.{component, error}; + load: async func(path: string) -> result; + } + import http-loader: interface { + use componentized:component/types@0.0.0-dev.{component, error}; + load: async func(path: string) -> result; + } + import oci-loader: interface { + use componentized:component/types@0.0.0-dev.{component, error}; + load: async func(path: string) -> result; + } + + export componentized:component/types@0.0.0-dev; + export componentized:component/path-loader@0.0.0-dev; +} + world wac-loader { export componentized:component/types@0.0.0-dev; export componentized:component/wac-loader@0.0.0-dev; diff --git a/components/wkg.lock b/components/wkg.lock index 57c4824..d09052f 100644 --- a/components/wkg.lock +++ b/components/wkg.lock @@ -11,6 +11,15 @@ requirement = "=0.1.0-dev" version = "0.1.0-dev" digest = "sha256:1c43656521dfd238a67af2288f7abcf67e2ebd950352a90c396dcf11b25c4440" +[[packages]] +name = "componentized:oci" +registry = "componentized.dev" + +[[packages.versions]] +requirement = "=0.0.0-dev" +version = "0.0.0-dev" +digest = "sha256:6f30d69218537da0ea518d1176e6c8a340746a2472975e9460959859e118abf6" + [[packages]] name = "wasi:filesystem" registry = "wasi.dev" From 54144ba80a676b3ef7c7c041563f9d037a7e3f66 Mon Sep 17 00:00:00 2001 From: Scott Andrews Date: Tue, 22 Sep 2026 17:04:21 -0400 Subject: [PATCH 2/3] use stable toolchain to install wac Signed-off-by: Scott Andrews --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8edd45b..a4f9611 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -23,7 +23,7 @@ jobs: run: cargo binstall --force static-config - name: Install wac-cli # hack to add map support to wac, remove once this PR lands - run: cargo install --git https://github.com/salmans/wac --branch map-defined-type wac-cli + run: cargo +stable install --force --git https://github.com/salmans/wac --branch map-defined-type wac-cli # run: cargo binstall --force wac-cli - name: Install wkg run: cargo binstall --force wkg From 8040e1d7b0b1ac14df7232efe9e83ffa5dcef48c Mon Sep 17 00:00:00 2001 From: Scott Andrews Date: Thu, 24 Sep 2026 12:26:43 -0400 Subject: [PATCH 3/3] Create test harness for path-loader `test-loader` is a wasi:cli/command which accepts a single argument, the path to load, loads that path and dumps the bytes to stdout. The Makefile now supports fetching "third party" components via wkg. Creating components via wac scripts is further enhanced so that only the dependencies of that script are built. This is done by parsing the wac script for components prefixed with 'local:', the remainder of the name is treated as a component make target. make components/test-loader && \ wasmtime run \ -Wcomponent-model-map \ -Shttp \ --dir=. \ ./lib/test-loader.wasm \ oci://ghcr.io/componentized/config/empty:0.2.1 \ | wasm-tools component wit Signed-off-by: Scott Andrews --- Cargo.lock | 7 + Cargo.toml | 6 +- Makefile | 43 +-- README.md | 2 +- .../dep-componentized-http-client/README.md | 3 + .../dep-componentized-http-client.wkg | 2 + .../dep-componentized-oci-client/README.md | 3 + .../dep-componentized-oci-client.wkg | 2 + components/filesystem-loader/Cargo.toml | 4 - components/filesystem-loader/src/lib.rs | 147 +++++++++- components/oci-loader/src/lib.rs | 11 +- components/path-loader-router/README.md | 2 +- components/path-loader-router/src/lib.rs | 4 +- components/path-loader/README.md | 2 +- components/test-loader-harness/Cargo.toml | 11 + components/test-loader-harness/README.md | 3 + components/test-loader-harness/src/lib.rs | 63 +++++ components/test-loader/README.md | 3 + components/test-loader/test-loader.wac | 17 ++ .../componentized-oci-0.0.0-dev/package.wit | 82 +++--- .../wit/deps/wasi-cli-0.3.0/package.wit | 256 ++++++++++++++++++ .../wit/deps/wasi-clocks-0.3.0/package.wit | 24 ++ .../wit/deps/wasi-random-0.3.0/package.wit | 18 ++ .../wit/deps/wasi-sockets-0.3.0/package.wit | 116 ++++++++ components/wit/worlds.wit | 8 + components/wkg.lock | 11 +- rust-toolchain.toml | 4 +- 27 files changed, 772 insertions(+), 82 deletions(-) create mode 100644 components/dep-componentized-http-client/README.md create mode 100644 components/dep-componentized-http-client/dep-componentized-http-client.wkg create mode 100644 components/dep-componentized-oci-client/README.md create mode 100644 components/dep-componentized-oci-client/dep-componentized-oci-client.wkg create mode 100644 components/test-loader-harness/Cargo.toml create mode 100644 components/test-loader-harness/README.md create mode 100644 components/test-loader-harness/src/lib.rs create mode 100644 components/test-loader/README.md create mode 100644 components/test-loader/test-loader.wac create mode 100644 components/wit/deps/wasi-cli-0.3.0/package.wit create mode 100644 components/wit/deps/wasi-random-0.3.0/package.wit create mode 100644 components/wit/deps/wasi-sockets-0.3.0/package.wit diff --git a/Cargo.lock b/Cargo.lock index 5f2a71d..855c801 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -572,6 +572,13 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "test-loader-harness" +version = "0.1.0" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "thiserror" version = "1.0.69" diff --git a/Cargo.toml b/Cargo.toml index 4150d88..3ddd01d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,7 @@ [workspace] resolver = "2" members = [ - "components/*", -] -exclude = [ - "components/path-loader", - "components/wit", + "components/*/src/..", ] [workspace.dependencies] diff --git a/Makefile b/Makefile index 555db8b..27bff83 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,8 @@ WKG_CONFIG_FILE ?= $(dir $(abspath $(lastword $(MAKEFILE_LIST)))).config/wasm-pk CARGO_COMPONENTS = $(sort $(notdir $(patsubst %/,%,$(dir $(wildcard components/*/Cargo.toml))))) CONFIG_COMPONENTS = $(sort $(notdir $(patsubst %/,%,$(dir $(wildcard components/*/*.properties))))) WAC_COMPONENTS = $(sort $(notdir $(patsubst %/,%,$(dir $(wildcard components/*/*.wac))))) -COMPONENTS = $(CARGO_COMPONENTS) $(CONFIG_COMPONENTS) $(WAC_COMPONENTS) +WKG_COMPONENTS = $(sort $(notdir $(patsubst %/,%,$(dir $(wildcard components/*/*.wkg))))) +COMPONENTS = $(sort $(CARGO_COMPONENTS) $(CONFIG_COMPONENTS) $(WAC_COMPONENTS) $(WKG_COMPONENTS)) .PHONY: all all: components @@ -33,23 +34,13 @@ components/$1: lib/$1.wasm lib/$1.debug.wasm ifneq ($(wildcard components/$1/Cargo.toml),) lib/$1.wasm: Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f) - @$(eval target := $(shell yq -r '.package.default-target // "wasm32-unknown-unknown"' components/$1/Cargo.toml 2> /dev/null)) - cargo build -p $1 --target $(target) --release -ifeq ($(target),wasm32-unknown-unknown) - wasm-tools component new target/$(target)/release/$(subst -,_,$1).wasm -o lib/$1.wasm -else - cp target/$(target)/release/$(subst -,_,$1).wasm lib/$1.wasm -endif + cargo build -p $1 --target wasm32-unknown-unknown --release + wasm-tools component new target/wasm32-unknown-unknown/release/$(subst -,_,$1).wasm -o lib/$1.wasm cp components/$1/README.md lib/$1.wasm.md lib/$1.debug.wasm: Cargo.toml Cargo.lock components/wit/deps $(shell find components/$1 -type f) - @$(eval target := $(shell yq -r '.package.default-target // "wasm32-unknown-unknown"' components/$1/Cargo.toml 2> /dev/null)) - cargo build --target $(target) -p $1 -ifeq ($(target),wasm32-unknown-unknown) - wasm-tools component new target/$(target)/debug/$(subst -,_,$1).wasm -o lib/$1.debug.wasm -else - cp target/$(target)/debug/$(subst -,_,$1).wasm lib/$1.debug.wasm -endif + cargo build --target wasm32-unknown-unknown -p $1 + wasm-tools component new target/wasm32-unknown-unknown/debug/$(subst -,_,$1).wasm -o lib/$1.debug.wasm cp components/$1/README.md lib/$1.debug.wasm.md else ifneq ($(wildcard components/$1/$1.properties),) @@ -64,12 +55,24 @@ lib/$1.debug.wasm: components/$1/$1.properties components/$1/README.md else ifneq ($(wildcard components/$1/$1.wac),) -lib/$1.wasm: components/$1/$1.wac components/$1/README.md $(foreach component,$(CARGO_COMPONENTS),lib/$(component).wasm) $(foreach component,$(CONFIG_COMPONENTS),lib/$(component).wasm) - wac compose $(foreach component,$(COMPONENTS),-d local:$(component)=lib/$(component).wasm) -o lib/$1.wasm components/$1/$1.wac +WAC_DEPS_$1 := $(shell wac parse components/$1/$1.wac 2> /dev/null | jq -r '[.. | .package?.name? | strings | select(startswith("local:")) | sub("^local:"; "")] | unique[]') + +lib/$1.wasm: components/$1/$1.wac components/$1/README.md $$(foreach component,$$(WAC_DEPS_$1),lib/$$(component).wasm) + wac compose $$(foreach component,$$(WAC_DEPS_$1),-d local:$$(component)=lib/$$(component).wasm) -o lib/$1.wasm components/$1/$1.wac + cp components/$1/README.md lib/$1.wasm.md + +lib/$1.debug.wasm: components/$1/$1.wac components/$1/README.md $$(foreach component,$$(WAC_DEPS_$1),lib/$$(component).debug.wasm) + wac compose $$(foreach component,$$(WAC_DEPS_$1),-d local:$$(component)=lib/$$(component).debug.wasm) -o lib/$1.debug.wasm components/$1/$1.wac + cp components/$1/README.md lib/$1.debug.wasm.md + +else ifneq ($(wildcard components/$1/$1.wkg),) + +lib/$1.wasm: components/$1/$1.wkg components/$1/README.md + wkg oci pull $(shell cat components/$1/$1.wkg 2> /dev/null | head -1) -o lib/$1.wasm cp components/$1/README.md lib/$1.wasm.md -lib/$1.debug.wasm: components/$1/$1.wac components/$1/README.md $(foreach component,$(CARGO_COMPONENTS),lib/$(component).debug.wasm) $(foreach component,$(CONFIG_COMPONENTS),lib/$(component).debug.wasm) - wac compose $(foreach component,$(COMPONENTS),-d local:$(component)=lib/$(component).debug.wasm) -o lib/$1.debug.wasm components/$1/$1.wac +lib/$1.debug.wasm: components/$1/$1.wkg components/$1/README.md + wkg oci pull $(shell cat components/$1/$1.wkg 2> /dev/null | tail -1 2> /dev/null) -o lib/$1.debug.wasm cp components/$1/README.md lib/$1.debug.wasm.md endif @@ -92,7 +95,7 @@ components/wit/deps: wit/deps components/wkg.toml $(WKG_CONFIG_FILE) $(shell fin ( cd components && wkg fetch --config $(WKG_CONFIG_FILE) ) .PHONY: publish ## Publish each component in the lib directory -publish: $(shell find lib -maxdepth 1 -type f -name "*.wasm" | sed -e 's:^lib/:publish-:g') +publish: $(shell find lib -maxdepth 1 -type f -name "*.wasm" ! -name "dep-*" ! -name "test-*" | sed -e 's:^lib/:publish-:g') .PHONY: publish-% publish-%: diff --git a/README.md b/README.md index 0e3fe8c..e4c295b 100644 --- a/README.md +++ b/README.md @@ -18,11 +18,11 @@ A [dev container](https://containers.dev) is available that contains the necessa Prereqs: - a rust toolchain +- [`jq`](https://jqlang.org) - [`static-config`](https://github.com/componentized/static-config) - [`wac`](https://github.com/bytecodealliance/wac) - [`wasm-tools`](https://github.com/bytecodealliance/wasm-tools) - [`wkg`](https://github.com/bytecodealliance/wasm-pkg-tools) -- [`yq`](https://github.com/mikefarah/yq) ```sh make components diff --git a/components/dep-componentized-http-client/README.md b/components/dep-componentized-http-client/README.md new file mode 100644 index 0000000..ef9fc65 --- /dev/null +++ b/components/dep-componentized-http-client/README.md @@ -0,0 +1,3 @@ +# `dep-componentized-http-client` + +from https://github.com/componentized/http/tree/main/components/http-client diff --git a/components/dep-componentized-http-client/dep-componentized-http-client.wkg b/components/dep-componentized-http-client/dep-componentized-http-client.wkg new file mode 100644 index 0000000..f1fceaa --- /dev/null +++ b/components/dep-componentized-http-client/dep-componentized-http-client.wkg @@ -0,0 +1,2 @@ +ghcr.io/componentized/http/http-client:0.1.0-dev@sha256:c8ddd9363ae5db549f366fdf9339b64342909a234083d96edbc187f168192e93 +ghcr.io/componentized/http/http-client:0.1.0-dev_debug@sha256:938b063bbe3c219c671f01af6abadc55b4b6035752188befb7eae77e385a028b diff --git a/components/dep-componentized-oci-client/README.md b/components/dep-componentized-oci-client/README.md new file mode 100644 index 0000000..e7d9359 --- /dev/null +++ b/components/dep-componentized-oci-client/README.md @@ -0,0 +1,3 @@ +# `dep-componentized-oci-client` + +from https://github.com/componentized/oci/tree/main/components/client diff --git a/components/dep-componentized-oci-client/dep-componentized-oci-client.wkg b/components/dep-componentized-oci-client/dep-componentized-oci-client.wkg new file mode 100644 index 0000000..d2eb861 --- /dev/null +++ b/components/dep-componentized-oci-client/dep-componentized-oci-client.wkg @@ -0,0 +1,2 @@ +ghcr.io/componentized/oci/client:0.0.0-dev@sha256:c370def08aa59bb73f7124c094b045b49d489e3631afeadbe474106bf80d5447 +ghcr.io/componentized/oci/client:0.0.0-dev_debug@sha256:d256ffa0c6ae712d5fae2501587cc74f383767d38d0ca3b96754586b62b6782b diff --git a/components/filesystem-loader/Cargo.toml b/components/filesystem-loader/Cargo.toml index f5808c3..8b868ee 100644 --- a/components/filesystem-loader/Cargo.toml +++ b/components/filesystem-loader/Cargo.toml @@ -1,13 +1,9 @@ -cargo-features = ["per-package-target"] - [package] name = "filesystem-loader" version = "0.1.0" edition = "2021" license = "Apache-2.0" -default-target = "wasm32-wasip3" - [lib] crate-type = ["cdylib"] diff --git a/components/filesystem-loader/src/lib.rs b/components/filesystem-loader/src/lib.rs index 6c71a6b..41595b6 100644 --- a/components/filesystem-loader/src/lib.rs +++ b/components/filesystem-loader/src/lib.rs @@ -1,10 +1,13 @@ #![no_main] -use std::{fs, io}; +use std::path::Path; use crate::{ - exports::componentized::component::path_loader::Guest, - exports::componentized::component::types::{Component, Error}, + exports::componentized::component::{ + path_loader::Guest, + types::{Component, Error}, + }, + wasi::filesystem::{preopens, types as filesystem}, }; pub(crate) struct FilesystemLoader; @@ -12,14 +15,144 @@ pub(crate) struct FilesystemLoader; impl Guest for FilesystemLoader { #[allow(async_fn_in_trait)] async fn load(path: String) -> Result { - let component = fs::read(path)?; + let (dir, path) = resolve(&path)?; + let file = dir + .open_at( + filesystem::PathFlags::SYMLINK_FOLLOW, + path, + filesystem::OpenFlags::empty(), + filesystem::DescriptorFlags::READ, + ) + .await?; + let (data, result) = file.read_via_stream(0); + let component = data.collect().await; + result.await?; Ok(component) } } -impl From for Error { - fn from(value: io::Error) -> Self { - Self::Other(Some(value.to_string())) +/// Resolve a path to the preopened directory that contains it and the path relative to that +/// directory. When multiple preopens match, the most specific one wins. +fn resolve(path: &str) -> Result<(filesystem::Descriptor, String), Error> { + let path = Path::new(path); + let mut resolved: Option<(usize, filesystem::Descriptor, String)> = None; + for (dir, name) in preopens::get_directories() { + let prefix = Path::new(&name); + let relative = if path.is_relative() && prefix == Path::new(".") { + Some(path) + } else { + path.strip_prefix(prefix).ok() + }; + let Some(relative) = relative.and_then(Path::to_str) else { + continue; + }; + let specificity = prefix.components().count(); + if resolved.as_ref().is_none_or(|(s, _, _)| specificity > *s) { + resolved = Some((specificity, dir, relative.to_string())); + } + } + resolved.map(|(_, dir, path)| (dir, path)).ok_or_else(|| { + Error::Other(Some(format!( + "filesystem: no preopen for path {}", + path.display() + ))) + }) +} + +impl From for Error { + fn from(value: filesystem::ErrorCode) -> Self { + match value { + filesystem::ErrorCode::Access => Self::Other(Some("filesystem: access".to_string())), + filesystem::ErrorCode::Already => Self::Other(Some("filesystem: already".to_string())), + filesystem::ErrorCode::BadDescriptor => { + Self::Other(Some("filesystem: bad-descriptor".to_string())) + } + filesystem::ErrorCode::Busy => Self::Other(Some("filesystem: busy".to_string())), + filesystem::ErrorCode::Deadlock => { + Self::Other(Some("filesystem: deadlock".to_string())) + } + filesystem::ErrorCode::Quota => Self::Other(Some("filesystem: quota".to_string())), + filesystem::ErrorCode::Exist => Self::Other(Some("filesystem: exist".to_string())), + filesystem::ErrorCode::FileTooLarge => { + Self::Other(Some("filesystem: file-too-large".to_string())) + } + filesystem::ErrorCode::IllegalByteSequence => { + Self::Other(Some("filesystem: illegal-byte-sequence".to_string())) + } + filesystem::ErrorCode::InProgress => { + Self::Other(Some("filesystem: in-progress".to_string())) + } + filesystem::ErrorCode::Interrupted => { + Self::Other(Some("filesystem: interrupted".to_string())) + } + filesystem::ErrorCode::Invalid => Self::Other(Some("filesystem: invalid".to_string())), + filesystem::ErrorCode::Io => Self::Other(Some("filesystem: io".to_string())), + filesystem::ErrorCode::IsDirectory => { + Self::Other(Some("filesystem: is-directory".to_string())) + } + filesystem::ErrorCode::Loop => Self::Other(Some("filesystem: loop".to_string())), + filesystem::ErrorCode::TooManyLinks => { + Self::Other(Some("filesystem: too-many-links".to_string())) + } + filesystem::ErrorCode::MessageSize => { + Self::Other(Some("filesystem: message-size".to_string())) + } + filesystem::ErrorCode::NameTooLong => { + Self::Other(Some("filesystem: name-too-long".to_string())) + } + filesystem::ErrorCode::NoDevice => { + Self::Other(Some("filesystem: no-device".to_string())) + } + filesystem::ErrorCode::NoEntry => Self::Other(Some("filesystem: no-entry".to_string())), + filesystem::ErrorCode::NoLock => Self::Other(Some("filesystem: no-lock".to_string())), + filesystem::ErrorCode::InsufficientMemory => { + Self::Other(Some("filesystem: insufficient-memory".to_string())) + } + filesystem::ErrorCode::InsufficientSpace => { + Self::Other(Some("filesystem: insufficient-space".to_string())) + } + filesystem::ErrorCode::NotDirectory => { + Self::Other(Some("filesystem: not-directory".to_string())) + } + filesystem::ErrorCode::NotEmpty => { + Self::Other(Some("filesystem: not-empty".to_string())) + } + filesystem::ErrorCode::NotRecoverable => { + Self::Other(Some("filesystem: not-recoverable".to_string())) + } + filesystem::ErrorCode::Unsupported => { + Self::Other(Some("filesystem: unsupported".to_string())) + } + filesystem::ErrorCode::NoTty => Self::Other(Some("filesystem: no-tty".to_string())), + filesystem::ErrorCode::NoSuchDevice => { + Self::Other(Some("filesystem: no-such-device".to_string())) + } + filesystem::ErrorCode::Overflow => { + Self::Other(Some("filesystem: overflow".to_string())) + } + filesystem::ErrorCode::NotPermitted => { + Self::Other(Some("filesystem: not-permitted".to_string())) + } + filesystem::ErrorCode::Pipe => Self::Other(Some("filesystem: pipe".to_string())), + filesystem::ErrorCode::ReadOnly => { + Self::Other(Some("filesystem: read-only".to_string())) + } + filesystem::ErrorCode::InvalidSeek => { + Self::Other(Some("filesystem: invalid-seek".to_string())) + } + filesystem::ErrorCode::TextFileBusy => { + Self::Other(Some("filesystem: text-file-busy".to_string())) + } + filesystem::ErrorCode::CrossDevice => { + Self::Other(Some("filesystem: cross-device".to_string())) + } + filesystem::ErrorCode::Other(None) => { + Self::Other(Some("filesystem: other".to_string())) + } + filesystem::ErrorCode::Other(Some(message)) => { + Self::Other(Some(format!("filesystem: other: {message}"))) + } + } } } diff --git a/components/oci-loader/src/lib.rs b/components/oci-loader/src/lib.rs index 16f24f7..ce6c81b 100644 --- a/components/oci-loader/src/lib.rs +++ b/components/oci-loader/src/lib.rs @@ -36,10 +36,13 @@ impl Guest for OCILoader { } let config_reference = manifest_reference.with_digest(&manifest.config.digest); - let config = match oci::get_config(config_reference).await? { - oci::Config::WasmV0(wasm_config_v0) => wasm_config_v0, - _ => Err(Error::Other(Some("unexpected config".to_string())))?, - }; + let config = + match oci::get_config(config_reference, Some(manifest.config.media_type.clone())) + .await? + { + oci::Config::WasmV0(wasm_config_v0) => wasm_config_v0, + _ => Err(Error::Other(Some("unexpected config".to_string())))?, + }; if config.component.is_none() { Err(Error::Other(Some("not a component".to_string())))? } diff --git a/components/path-loader-router/README.md b/components/path-loader-router/README.md index a1c07c1..34410ea 100644 --- a/components/path-loader-router/README.md +++ b/components/path-loader-router/README.md @@ -11,4 +11,4 @@ Based on the path prefix the request is routed to an appropriate loader: - `file:///` -> filesystem-loader: scheme is dropped, last '/' is preserved with path - `file://` -> invalid, other hosts are not supported - `file:` -> filesystem-loader: scheme is dropped -- `oci:` -> oci-loader: scheme is dropped +- `oci://` -> oci-loader: scheme is dropped diff --git a/components/path-loader-router/src/lib.rs b/components/path-loader-router/src/lib.rs index 2b93263..0d5962a 100644 --- a/components/path-loader-router/src/lib.rs +++ b/components/path-loader-router/src/lib.rs @@ -22,8 +22,8 @@ impl Guest for PathLoader { )))) } else if path.starts_with("file:") { filesystem_loader::load(path[5..].to_string()).await - } else if path.starts_with("oci:") { - oci_loader::load(path[4..].to_string()).await + } else if path.starts_with("oci://") { + oci_loader::load(path[6..].to_string()).await } else { Err(Error::Other(Some(format!("unknown path loader: {path}")))) } diff --git a/components/path-loader/README.md b/components/path-loader/README.md index 9782106..894afd3 100644 --- a/components/path-loader/README.md +++ b/components/path-loader/README.md @@ -11,4 +11,4 @@ Based on the path prefix the request is routed to an appropriate loader: - `file:///` -> filesystem-loader: scheme is dropped, last '/' is preserved with path - `file://` -> invalid, other hosts are not supported - `file:` -> filesystem-loader: scheme is dropped -- `oci:` -> oci-loader: scheme is dropped +- `oci://` -> oci-loader: scheme is dropped diff --git a/components/test-loader-harness/Cargo.toml b/components/test-loader-harness/Cargo.toml new file mode 100644 index 0000000..ed8bd7b --- /dev/null +++ b/components/test-loader-harness/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "test-loader-harness" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +wit-bindgen = { workspace = true } diff --git a/components/test-loader-harness/README.md b/components/test-loader-harness/README.md new file mode 100644 index 0000000..7e343ec --- /dev/null +++ b/components/test-loader-harness/README.md @@ -0,0 +1,3 @@ +# `test-loader-core` + +Test harness for a path loader. diff --git a/components/test-loader-harness/src/lib.rs b/components/test-loader-harness/src/lib.rs new file mode 100644 index 0000000..68df4b7 --- /dev/null +++ b/components/test-loader-harness/src/lib.rs @@ -0,0 +1,63 @@ +#![no_main] + +use crate::{ + componentized::component::{path_loader, types::Error}, + exports::wasi::cli::run::Guest, + wasi::cli::{environment, stderr, stdout}, +}; + +/// Writes `$bytes` (a `Vec`) to a wasi:cli output stream (`stdout` or `stderr`). +/// +/// Must be used within an async context. Evaluates to `Result<(), ()>`, which is `Err` +/// if the host reports an error or stops reading before all bytes are written. +macro_rules! write_to { + ($stream:ident, $bytes:expr) => { + async { + let (mut tx, rx) = wit_stream::new(); + // hand the reader to the host before writing, otherwise the write never completes + let result = $stream::write_via_stream(rx); + let remaining = tx.write_all($bytes).await; + // close the stream so the host sees EOF and resolves `result` + drop(tx); + match result.await { + Ok(()) if remaining.is_empty() => Ok(()), + _ => Err(()), + } + } + .await + }; +} + +struct LoaderHarness; + +impl Guest for LoaderHarness { + #[doc = " Run the program."] + #[allow(async_fn_in_trait)] + async fn run() -> Result<(), ()> { + let args = environment::get_arguments(); + let path = args.get(1).expect("path argument required"); + + match path_loader::load(path.to_string()).await { + Ok(component) => { + write_to!(stdout, component) + } + Err(e) => { + let message = match e { + Error::Other(Some(message)) => message, + Error::Other(None) => "--unknown error--".to_string(), + }; + let _ = write_to!(stderr, format!("{}\n", message).into_bytes()); + Err(()) + } + } + } +} + +wit_bindgen::generate!({ + path: "../wit", + world: "test-loader-harness", + merge_structurally_equal_types: true, + generate_all +}); + +export!(LoaderHarness); diff --git a/components/test-loader/README.md b/components/test-loader/README.md new file mode 100644 index 0000000..53f1954 --- /dev/null +++ b/components/test-loader/README.md @@ -0,0 +1,3 @@ +# `test-loader` + +Test harness for a path loader. diff --git a/components/test-loader/test-loader.wac b/components/test-loader/test-loader.wac new file mode 100644 index 0000000..e7dfad3 --- /dev/null +++ b/components/test-loader/test-loader.wac @@ -0,0 +1,17 @@ +package componentized:component; + +let http-client = new local:dep-componentized-http-client { ... }; + +let oci-client = new local:dep-componentized-oci-client { + client: http-client.client, + ... +}; + +export new local:test-loader-harness { + path-loader: new local:path-loader { + "componentized:http/client@0.1.0-dev": http-client.client, + "componentized:oci/client@0.0.0-dev": oci-client.client, + ... + }.path-loader, + ... +}...; diff --git a/components/wit/deps/componentized-oci-0.0.0-dev/package.wit b/components/wit/deps/componentized-oci-0.0.0-dev/package.wit index fae6a8e..2630638 100644 --- a/components/wit/deps/componentized-oci-0.0.0-dev/package.wit +++ b/components/wit/deps/componentized-oci-0.0.0-dev/package.wit @@ -119,13 +119,6 @@ interface client { args-escaped: option, } - record oci-image-config-v1-content-addresses { - /// must be 'layers' - %type: string, - /// layer content hashes, in order from first to last - diff-ids: list, - } - record oci-image-config-v1-history-entry { /// combined date and time at which the layer was created created: option, @@ -139,29 +132,6 @@ interface client { empty-layer: option, } - record oci-image-config-v1 { - /// combined date and time at which the image was created - created: option, - /// name and/or email address of the person or entity which created and is responsible for maintaining the image - author: option, - /// CPU architecture which the binaries in this image are built to run on - architecture: string, - /// name of the operating system which the image is built to run on - os: string, - /// version of the operating system - os-version: option, - /// mandatory OS features - os-features: option>, - /// variant of the specified CPU architecture - %variant: option, - /// execution parameters which should be used as a base when running a container using the image - config: option, - /// layer content addresses used by the image - rootfs: oci-image-config-v1-content-addresses, - /// history of each layer. The array is ordered from first to last - history: option>, - } - record wasm-config-v0-component { exports: list, imports: list, @@ -184,6 +154,10 @@ interface client { urls: option>, /// arbitrary metadata for this descriptor annotations: option>, + /// embedded representation of the referenced content + data: option, + /// type of an artifact when the descriptor points to an artifact + artifact-type: option, } record oci-image-manifest-v1 { @@ -204,14 +178,24 @@ interface client { } record oci-image-index-manifest-v1-manifest { - /// media type of the subject manifest + /// media type of the referenced content media-type: media-type, + /// digest of the targeted content + digest: digest, + /// size, in bytes, of the raw content + size: u64, + /// list of URIs from which this object MAY be downloaded + urls: option>, + /// arbitrary metadata for this descriptor + annotations: option>, + /// embedded representation of the referenced content + data: option, + /// type of an artifact when the descriptor points to an artifact + artifact-type: option, /// minimum runtime requirements of the image platform: option, /// descriptor of another manifest subject: option, - /// arbitrary metadata for the image manifest - annotations: option>, } record oci-image-index-manifest-v1 { @@ -238,6 +222,36 @@ interface client { other(list), } + record oci-image-config-v1-content-addresses { + /// must be 'layers' + %type: string, + /// layer content hashes, in order from first to last + diff-ids: list, + } + + record oci-image-config-v1 { + /// combined date and time at which the image was created + created: option, + /// name and/or email address of the person or entity which created and is responsible for maintaining the image + author: option, + /// CPU architecture which the binaries in this image are built to run on + architecture: string, + /// name of the operating system which the image is built to run on + os: string, + /// version of the operating system + os-version: option, + /// mandatory OS features + os-features: option>, + /// variant of the specified CPU architecture + %variant: option, + /// execution parameters which should be used as a base when running a container using the image + config: option, + /// layer content addresses used by the image + rootfs: oci-image-config-v1-content-addresses, + /// history of each layer. The array is ordered from first to last + history: option>, + } + record wasm-config-v0 { /// combined date and time at which the image was created created: option, @@ -275,7 +289,7 @@ interface client { get-blob: async func(reference: reference) -> result, error-code>; - get-config: async func(reference: reference) -> result; + get-config: async func(reference: reference, default-media-type: option) -> result; get-manifest: async func(reference: reference) -> result; } diff --git a/components/wit/deps/wasi-cli-0.3.0/package.wit b/components/wit/deps/wasi-cli-0.3.0/package.wit new file mode 100644 index 0000000..7aae56c --- /dev/null +++ b/components/wit/deps/wasi-cli-0.3.0/package.wit @@ -0,0 +1,256 @@ +package wasi:cli@0.3.0; + +@since(version = 0.3.0) +interface environment { + /// Get the POSIX-style environment variables. + /// + /// Each environment variable is provided as a pair of string variable names + /// and string value. + /// + /// Morally, these are a value import, but until value imports are available + /// in the component model, this import function should return the same + /// values each time it is called. + @since(version = 0.3.0) + get-environment: func() -> list>; + + /// Get the POSIX-style arguments to the program. + @since(version = 0.3.0) + get-arguments: func() -> list; + + /// Return a path that programs should use as their initial current working + /// directory, interpreting `.` as shorthand for this. + @since(version = 0.3.0) + get-initial-cwd: func() -> option; +} + +@since(version = 0.3.0) +interface exit { + /// Exit the current instance and any linked instances. + @since(version = 0.3.0) + exit: func(status: result); + + /// Exit the current instance and any linked instances, reporting the + /// specified status code to the host. + /// + /// The meaning of the code depends on the context, with 0 usually meaning + /// "success", and other values indicating various types of failure. + /// + /// This function does not return; the effect is analogous to a trap, but + /// without the connotation that something bad has happened. + @since(version = 0.3.0) + exit-with-code: func(status-code: u8); +} + +@since(version = 0.3.0) +interface run { + /// Run the program. + @since(version = 0.3.0) + run: async func() -> result; +} + +@since(version = 0.3.0) +interface types { + @since(version = 0.3.0) + enum error-code { + /// Input/output error + io, + /// Invalid or incomplete multibyte or wide character + illegal-byte-sequence, + /// Broken pipe + pipe, + } +} + +@since(version = 0.3.0) +interface stdin { + use types.{error-code}; + + /// Return a stream for reading from stdin. + /// + /// This function returns a stream which provides data read from stdin, + /// and a future to signal read results. + /// + /// If the stream's readable end is dropped the future will resolve to success. + /// + /// If the stream's writable end is dropped the future will either resolve to + /// success if stdin was closed by the writer or to an error-code if reading + /// failed for some other reason. + /// + /// Multiple streams may be active at the same time. The behavior of concurrent + /// reads is implementation-specific. + @since(version = 0.3.0) + read-via-stream: func() -> tuple, future>>; +} + +@since(version = 0.3.0) +interface stdout { + use types.{error-code}; + + /// Write the given stream to stdout. + /// + /// If the stream's writable end is dropped this function will either return + /// success once the entire contents of the stream have been written or an + /// error-code representing a failure. + /// + /// Otherwise if there is an error the readable end of the stream will be + /// dropped and this function will return an error-code. + @since(version = 0.3.0) + write-via-stream: func(data: stream) -> future>; +} + +@since(version = 0.3.0) +interface stderr { + use types.{error-code}; + + /// Write the given stream to stderr. + /// + /// If the stream's writable end is dropped this function will either return + /// success once the entire contents of the stream have been written or an + /// error-code representing a failure. + /// + /// Otherwise if there is an error the readable end of the stream will be + /// dropped and this function will return an error-code. + @since(version = 0.3.0) + write-via-stream: func(data: stream) -> future>; +} + +/// Terminal input. +/// +/// In the future, this may include functions for disabling echoing, +/// disabling input buffering so that keyboard events are sent through +/// immediately, querying supported features, and so on. +@since(version = 0.3.0) +interface terminal-input { + /// The input side of a terminal. + @since(version = 0.3.0) + resource terminal-input; +} + +/// Terminal output. +/// +/// In the future, this may include functions for querying the terminal +/// size, being notified of terminal size changes, querying supported +/// features, and so on. +@since(version = 0.3.0) +interface terminal-output { + /// The output side of a terminal. + @since(version = 0.3.0) + resource terminal-output; +} + +/// An interface providing an optional `terminal-input` for stdin as a +/// link-time authority. +@since(version = 0.3.0) +interface terminal-stdin { + @since(version = 0.3.0) + use terminal-input.{terminal-input}; + + /// If stdin is connected to a terminal, return a `terminal-input` handle + /// allowing further interaction with it. + @since(version = 0.3.0) + get-terminal-stdin: func() -> option; +} + +/// An interface providing an optional `terminal-output` for stdout as a +/// link-time authority. +@since(version = 0.3.0) +interface terminal-stdout { + @since(version = 0.3.0) + use terminal-output.{terminal-output}; + + /// If stdout is connected to a terminal, return a `terminal-output` handle + /// allowing further interaction with it. + @since(version = 0.3.0) + get-terminal-stdout: func() -> option; +} + +/// An interface providing an optional `terminal-output` for stderr as a +/// link-time authority. +@since(version = 0.3.0) +interface terminal-stderr { + @since(version = 0.3.0) + use terminal-output.{terminal-output}; + + /// If stderr is connected to a terminal, return a `terminal-output` handle + /// allowing further interaction with it. + @since(version = 0.3.0) + get-terminal-stderr: func() -> option; +} + +@since(version = 0.3.0) +world imports { + @since(version = 0.3.0) + import environment; + @since(version = 0.3.0) + import exit; + @since(version = 0.3.0) + import types; + @since(version = 0.3.0) + import stdin; + @since(version = 0.3.0) + import stdout; + @since(version = 0.3.0) + import stderr; + @since(version = 0.3.0) + import terminal-input; + @since(version = 0.3.0) + import terminal-output; + @since(version = 0.3.0) + import terminal-stdin; + @since(version = 0.3.0) + import terminal-stdout; + @since(version = 0.3.0) + import terminal-stderr; + import wasi:clocks/types@0.3.0; + import wasi:clocks/monotonic-clock@0.3.0; + import wasi:clocks/system-clock@0.3.0; + @unstable(feature = clocks-timezone) + import wasi:clocks/timezone@0.3.0; + import wasi:filesystem/types@0.3.0; + import wasi:filesystem/preopens@0.3.0; + import wasi:sockets/types@0.3.0; + import wasi:sockets/ip-name-lookup@0.3.0; + import wasi:random/random@0.3.0; + import wasi:random/insecure@0.3.0; + import wasi:random/insecure-seed@0.3.0; +} +@since(version = 0.3.0) +world command { + @since(version = 0.3.0) + import environment; + @since(version = 0.3.0) + import exit; + @since(version = 0.3.0) + import types; + @since(version = 0.3.0) + import stdin; + @since(version = 0.3.0) + import stdout; + @since(version = 0.3.0) + import stderr; + @since(version = 0.3.0) + import terminal-input; + @since(version = 0.3.0) + import terminal-output; + @since(version = 0.3.0) + import terminal-stdin; + @since(version = 0.3.0) + import terminal-stdout; + @since(version = 0.3.0) + import terminal-stderr; + import wasi:clocks/types@0.3.0; + import wasi:clocks/monotonic-clock@0.3.0; + import wasi:clocks/system-clock@0.3.0; + @unstable(feature = clocks-timezone) + import wasi:clocks/timezone@0.3.0; + import wasi:filesystem/types@0.3.0; + import wasi:filesystem/preopens@0.3.0; + import wasi:sockets/types@0.3.0; + import wasi:sockets/ip-name-lookup@0.3.0; + import wasi:random/random@0.3.0; + import wasi:random/insecure@0.3.0; + import wasi:random/insecure-seed@0.3.0; + + @since(version = 0.3.0) + export run; +} diff --git a/components/wit/deps/wasi-clocks-0.3.0/package.wit b/components/wit/deps/wasi-clocks-0.3.0/package.wit index 2594e02..475dc0f 100644 --- a/components/wit/deps/wasi-clocks-0.3.0/package.wit +++ b/components/wit/deps/wasi-clocks-0.3.0/package.wit @@ -17,3 +17,27 @@ interface system-clock { get-resolution: func() -> duration; } +interface monotonic-clock { + use types.{duration}; + + type mark = u64; + + now: func() -> mark; + + get-resolution: func() -> duration; + + wait-until: async func(when: mark); + + wait-for: async func(how-long: duration); +} + +interface timezone { + use system-clock.{instant}; + + iana-id: func() -> option; + + utc-offset: func(when: instant) -> option; + + to-debug-string: func() -> string; +} + diff --git a/components/wit/deps/wasi-random-0.3.0/package.wit b/components/wit/deps/wasi-random-0.3.0/package.wit new file mode 100644 index 0000000..f6cfb81 --- /dev/null +++ b/components/wit/deps/wasi-random-0.3.0/package.wit @@ -0,0 +1,18 @@ +package wasi:random@0.3.0; + +interface random { + get-random-bytes: func(max-len: u64) -> list; + + get-random-u64: func() -> u64; +} + +interface insecure { + get-insecure-random-bytes: func(max-len: u64) -> list; + + get-insecure-random-u64: func() -> u64; +} + +interface insecure-seed { + get-insecure-seed: func() -> tuple; +} + diff --git a/components/wit/deps/wasi-sockets-0.3.0/package.wit b/components/wit/deps/wasi-sockets-0.3.0/package.wit new file mode 100644 index 0000000..da14ae8 --- /dev/null +++ b/components/wit/deps/wasi-sockets-0.3.0/package.wit @@ -0,0 +1,116 @@ +package wasi:sockets@0.3.0; + +interface types { + use wasi:clocks/types@0.3.0.{duration}; + + variant error-code { + access-denied, + not-supported, + invalid-argument, + out-of-memory, + timeout, + invalid-state, + address-not-bindable, + address-in-use, + remote-unreachable, + connection-refused, + connection-broken, + connection-reset, + connection-aborted, + datagram-too-large, + other(option), + } + + enum ip-address-family { + ipv4, + ipv6, + } + + type ipv4-address = tuple; + + type ipv6-address = tuple; + + variant ip-address { + ipv4(ipv4-address), + ipv6(ipv6-address), + } + + record ipv4-socket-address { + port: u16, + address: ipv4-address, + } + + record ipv6-socket-address { + port: u16, + flow-info: u32, + address: ipv6-address, + scope-id: u32, + } + + variant ip-socket-address { + ipv4(ipv4-socket-address), + ipv6(ipv6-socket-address), + } + + resource tcp-socket { + create: static func(address-family: ip-address-family) -> result; + bind: func(local-address: ip-socket-address) -> result<_, error-code>; + connect: async func(remote-address: ip-socket-address) -> result<_, error-code>; + listen: func() -> result, error-code>; + send: func(data: stream) -> future>; + receive: func() -> tuple, future>>; + get-local-address: func() -> result; + get-remote-address: func() -> result; + get-is-listening: func() -> bool; + get-address-family: func() -> ip-address-family; + set-listen-backlog-size: func(value: u64) -> result<_, error-code>; + get-keep-alive-enabled: func() -> result; + set-keep-alive-enabled: func(value: bool) -> result<_, error-code>; + get-keep-alive-idle-time: func() -> result; + set-keep-alive-idle-time: func(value: duration) -> result<_, error-code>; + get-keep-alive-interval: func() -> result; + set-keep-alive-interval: func(value: duration) -> result<_, error-code>; + get-keep-alive-count: func() -> result; + set-keep-alive-count: func(value: u32) -> result<_, error-code>; + get-hop-limit: func() -> result; + set-hop-limit: func(value: u8) -> result<_, error-code>; + get-receive-buffer-size: func() -> result; + set-receive-buffer-size: func(value: u64) -> result<_, error-code>; + get-send-buffer-size: func() -> result; + set-send-buffer-size: func(value: u64) -> result<_, error-code>; + } + + resource udp-socket { + create: static func(address-family: ip-address-family) -> result; + bind: func(local-address: ip-socket-address) -> result<_, error-code>; + connect: func(remote-address: ip-socket-address) -> result<_, error-code>; + disconnect: func() -> result<_, error-code>; + send: async func(data: list, remote-address: option) -> result<_, error-code>; + receive: async func() -> result, ip-socket-address>, error-code>; + get-local-address: func() -> result; + get-remote-address: func() -> result; + get-address-family: func() -> ip-address-family; + get-unicast-hop-limit: func() -> result; + set-unicast-hop-limit: func(value: u8) -> result<_, error-code>; + get-receive-buffer-size: func() -> result; + set-receive-buffer-size: func(value: u64) -> result<_, error-code>; + get-send-buffer-size: func() -> result; + set-send-buffer-size: func(value: u64) -> result<_, error-code>; + } +} + +interface ip-name-lookup { + use types.{ip-address}; + + variant error-code { + access-denied, + invalid-argument, + name-unresolvable, + temporary-resolver-failure, + permanent-resolver-failure, + other(option), + } + + resolve-addresses: async func(name: string) -> result, error-code>; +} + diff --git a/components/wit/worlds.wit b/components/wit/worlds.wit index 21607ad..81abea2 100644 --- a/components/wit/worlds.wit +++ b/components/wit/worlds.wit @@ -45,3 +45,11 @@ world wac-loader { export componentized:component/types@0.0.0-dev; export componentized:component/wac-loader@0.0.0-dev; } + +world test-loader-harness { + import componentized:component/path-loader@0.0.0-dev; + import wasi:cli/environment@0.3.0; + import wasi:cli/stdout@0.3.0; + import wasi:cli/stderr@0.3.0; + export wasi:cli/run@0.3.0; +} diff --git a/components/wkg.lock b/components/wkg.lock index d09052f..428244a 100644 --- a/components/wkg.lock +++ b/components/wkg.lock @@ -18,7 +18,16 @@ registry = "componentized.dev" [[packages.versions]] requirement = "=0.0.0-dev" version = "0.0.0-dev" -digest = "sha256:6f30d69218537da0ea518d1176e6c8a340746a2472975e9460959859e118abf6" +digest = "sha256:fe33e6ac5f37a1a5f37b249c382b28e5224d4f2b28a7b6f8499a8bb11762e400" + +[[packages]] +name = "wasi:cli" +registry = "wasi.dev" + +[[packages.versions]] +requirement = "=0.3.0" +version = "0.3.0" +digest = "sha256:3a2d58743b67a057f7210b9a73d3b21b34ec7895ffe34d1cb092927307156e40" [[packages]] name = "wasi:filesystem" diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 90bb313..9dbbb5c 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2026-09-18" -targets = [ "wasm32-unknown-unknown", "wasm32-wasip3" ] +channel = "1.98" +targets = [ "wasm32-unknown-unknown" ]