Merge pull request #42 from containers/add-all-docs

Enable missing_docs lint, add missing docs

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ concurrency:
 env:
   CARGO_TERM_COLOR: always
   # Pinned toolchain for linting
-  ACTIONS_LINTS_TOOLCHAIN: 1.76.0
+  ACTIONS_LINTS_TOOLCHAIN: 1.86.0
 
 jobs:
   tests-stable:
@@ -26,7 +26,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install toolchain
         uses: dtolnay/rust-toolchain@v1
         with:
@@ -34,15 +34,18 @@ jobs:
       - name: Cache build artifacts
         uses: Swatinem/rust-cache@v2
       - name: cargo build
-        run: cargo build --all-targets
+        run: cargo build --all-targets --all-features
       - name: cargo test
-        run: cargo test --all-targets
+        run: cargo test --all-targets --all-features
+      # https://github.com/rust-lang/cargo/issues/6669
+      - name: cargo test --doc
+        run: cargo test --doc --all-features
   linting:
     name: Lints, pinned toolchain
     runs-on: ubuntu-latest
     steps:
       - name: Check out repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install toolchain
         uses: dtolnay/rust-toolchain@v1
         with:
@@ -53,4 +56,4 @@ jobs:
       - name: cargo fmt (check)
         run: cargo fmt -- --check -l
       - name: cargo clippy (warnings)
-        run: cargo clippy --all-targets -- -D warnings
+        run: cargo clippy --all-targets --all-features -- -D warnings

Cargo.toml

@@ -1,23 +1,31 @@
 [package]
 name = "ocidir"
 description = "A Rust library for reading and writing OCI (opencontainers) layout directories"
-version = "0.2.0"
+version = "0.5.0"
 edition = "2021"
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/containers/ocidir-rs"
 keywords = ["oci", "opencontainers", "docker", "podman", "containers"]
 
 [dependencies]
-anyhow = "1.0"
 camino = "1.0.4"
 chrono = "0.4.19"
-olpc-cjson = "0.1.1"
 cap-std-ext = "4.0"
-flate2 = { features = ["zlib"], default_features = false, version = "1.0.20" }
-fn-error-context = "0.2.0"
+flate2 = { features = ["zlib"], default-features = false, version = "1.0.20" }
 hex = "0.4.3"
 openssl = "0.10.33"
 serde = { features = ["derive"], version = "1.0.125" }
 serde_json = "1.0.64"
 tar = "0.4.38"
-oci-spec = "0.6.5"
+thiserror = "2"
+oci-spec = "0.8.0"
+zstd = { version = "0.13.2", optional = true }
+canon-json = "0.2.0"
+
+[dev-dependencies]
+anyhow = "1.0.89"
+cap-tempfile = "3.3.0"
+
+[features]
+zstd = ["dep:zstd"]
+zstdmt = ["zstd", "zstd/zstdmt"]

README.md

@@ -4,6 +4,36 @@
 
 [crates-badge]: https://img.shields.io/crates/v/ocidir.svg
 [crates-url]: https://crates.io/crates/ocidir
+[![docs.rs](https://docs.rs/ocidir/badge.svg)](https://docs.rs/ocidir)
 
-A low level Rust library for reading and writing
-[OCI directories](https://github.com/opencontainers/image-spec/).
+# Read and write to OCI image layout directories
+
+This library contains medium and low-level APIs for working with
+[OCI images], which are basically a directory with blobs and JSON files
+for metadata.
+
+## Dependency on cap-std
+
+This library makes use of [cap-std] to operate in a capability-oriented
+fashion. In practice, the code in this project is well tested and would
+not traverse outside its own path root. However, using capabilities
+is a generally good idea when operating in the container ecosystem,
+in particular when actively processing tar streams.
+
+## Examples
+
+To access an existing OCI directory:
+
+```rust,no_run
+# use ocidir::cap_std;
+# use anyhow::{anyhow, Result};
+# fn main() -> anyhow::Result<()> {
+let d = cap_std::fs::Dir::open_ambient_dir("/path/to/ocidir", cap_std::ambient_authority())?;
+let d = ocidir::OciDir::open(d)?;
+println!("{:?}", d.read_index()?);
+# Ok(())
+# }
+```
+
+[cap-std]: https://docs.rs/cap-std/
+[OCI images]: https://github.com/opencontainers/image-spec

examples/custom_compressor.rs

@@ -0,0 +1,61 @@
+/// Example that shows how to use a custom compression and media type for image layers.
+/// The example below does no compression.
+use std::{env, io, path::PathBuf};
+
+use oci_spec::image::Platform;
+use ocidir::{cap_std::fs::Dir, BlobWriter, OciDir, WriteComplete};
+
+struct NoCompression<'a>(BlobWriter<'a>);
+
+impl io::Write for NoCompression<'_> {
+    fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
+        self.0.write(buf)
+    }
+
+    fn flush(&mut self) -> io::Result<()> {
+        self.0.flush()
+    }
+}
+
+impl<'a> WriteComplete<BlobWriter<'a>> for NoCompression<'a> {
+    fn complete(self) -> io::Result<BlobWriter<'a>> {
+        Ok(self.0)
+    }
+}
+
+fn main() {
+    let dir = Dir::open_ambient_dir(env::temp_dir(), ocidir::cap_std::ambient_authority()).unwrap();
+    let oci_dir = OciDir::ensure(dir).unwrap();
+
+    let mut manifest = oci_dir.new_empty_manifest().unwrap().build().unwrap();
+    let mut config = ocidir::oci_spec::image::ImageConfigurationBuilder::default()
+        .build()
+        .unwrap();
+
+    // Add the src as a layer
+    let writer = oci_dir
+        .create_custom_layer(
+            |bw| Ok(NoCompression(bw)),
+            oci_spec::image::MediaType::ImageLayer,
+        )
+        .unwrap();
+    let mut builder = tar::Builder::new(writer);
+    builder.follow_symlinks(false);
+
+    builder
+        .append_dir_all(".", PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src"))
+        .unwrap();
+
+    let layer = builder.into_inner().unwrap().complete().unwrap();
+    oci_dir.push_layer(&mut manifest, &mut config, layer, "src", None);
+
+    println!(
+        "Created image with manifest: {}",
+        manifest.to_string_pretty().unwrap()
+    );
+
+    // Add the image manifest
+    let _descriptor = oci_dir
+        .insert_manifest_and_config(manifest.clone(), config, None, Platform::default())
+        .unwrap();
+}

examples/ocidir-import-tar.rs

@@ -0,0 +1,80 @@
+//! # Import a pre-generated tarball, wrapping with OCI metadata into an OCI directory.
+//!
+//! This little exmaple shows a bit of how to use the ocidir API. But it has a use case
+//! for low-level testing of OCI runtimes by injecting arbitrary tarballs.
+
+use std::io::BufReader;
+use std::{fs::File, path::Path};
+
+use anyhow::Context;
+use cap_tempfile::cap_std;
+use chrono::Utc;
+use oci_spec::image::{MediaType, Platform};
+use ocidir::OciDir;
+
+fn import(oci_dir: &OciDir, name: &str, src: File) -> anyhow::Result<()> {
+    let mtime = src.metadata()?.modified()?;
+    let mut input_tar = BufReader::new(src);
+    let created = chrono::DateTime::<Utc>::from(mtime);
+
+    let mut manifest = oci_dir.new_empty_manifest().unwrap().build().unwrap();
+    let mut config = ocidir::oci_spec::image::ImageConfigurationBuilder::default()
+        .build()
+        .unwrap();
+
+    // Add the src as a layer
+    let mut writer = oci_dir.create_blob().unwrap();
+    std::io::copy(&mut input_tar, &mut writer)?;
+
+    let blob = writer.complete()?;
+    let descriptor = blob
+        .descriptor()
+        .media_type(MediaType::ImageLayer)
+        .build()
+        .unwrap();
+    let blob_digest = descriptor.digest().to_string();
+    manifest.layers_mut().push(descriptor);
+    let mut rootfs = config.rootfs().clone();
+    rootfs.diff_ids_mut().push(blob_digest);
+    config.set_rootfs(rootfs);
+    let h = oci_spec::image::HistoryBuilder::default()
+        .created(created.to_rfc3339_opts(chrono::SecondsFormat::Secs, true))
+        .created_by(name.to_string())
+        .build()
+        .unwrap();
+    config.history_mut().push(h);
+
+    println!(
+        "Created image with manifest: {}",
+        manifest.to_string_pretty().unwrap()
+    );
+
+    // Add the image manifest
+    let _descriptor = oci_dir
+        .insert_manifest_and_config(
+            manifest.clone(),
+            config,
+            Some("latest"),
+            Platform::default(),
+        )
+        .unwrap();
+
+    Ok(())
+}
+
+fn main() -> anyhow::Result<()> {
+    let args = std::env::args().collect::<Vec<_>>();
+    let ocidir = args[1].as_str();
+    let path = Path::new(args[2].as_str());
+    let Some(name) = path.file_stem().and_then(|v| v.to_str()) else {
+        anyhow::bail!("Invalid path: {path:?}");
+    };
+    let f = File::open(path).with_context(|| format!("Opening {path:?}"))?;
+
+    let dir = &cap_std::fs::Dir::open_ambient_dir(ocidir, cap_std::ambient_authority())
+        .with_context(|| format!("Opening {ocidir}"))?;
+    let oci_dir = OciDir::ensure(dir.try_clone()?)?;
+
+    import(&oci_dir, name, f)?;
+    Ok(())
+}

examples/zstd.rs

@@ -0,0 +1,55 @@
+#[cfg(feature = "zstdmt")]
+fn main() {
+    use std::{env, path::PathBuf};
+
+    use cap_tempfile::TempDir;
+    use oci_spec::image::Platform;
+    use ocidir::OciDir;
+    let dir = TempDir::new(ocidir::cap_std::ambient_authority()).unwrap();
+    let oci_dir = OciDir::ensure(dir.try_clone().unwrap()).unwrap();
+
+    let mut manifest = oci_dir.new_empty_manifest().unwrap().build().unwrap();
+    let mut config = ocidir::oci_spec::image::ImageConfigurationBuilder::default()
+        .build()
+        .unwrap();
+
+    // Add the src as a layer
+    let writer = oci_dir.create_layer_zstd(Some(0)).unwrap();
+    let mut builder = tar::Builder::new(writer);
+    builder.follow_symlinks(false);
+
+    builder
+        .append_dir_all(".", PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src"))
+        .unwrap();
+
+    let layer = builder.into_inner().unwrap().complete().unwrap();
+    oci_dir.push_layer(&mut manifest, &mut config, layer, "src", None);
+
+    // Add the examples as a layer, using multithreaded compression
+    let writer = oci_dir.create_layer_zstd_multithread(Some(0), 4).unwrap();
+    let mut builder = tar::Builder::new(writer);
+    builder.follow_symlinks(false);
+    builder
+        .append_dir_all(
+            ".",
+            PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("examples"),
+        )
+        .unwrap();
+    let layer = builder.into_inner().unwrap().complete().unwrap();
+    oci_dir.push_layer(&mut manifest, &mut config, layer, "examples", None);
+
+    println!(
+        "Created image with manifest: {}",
+        manifest.to_string_pretty().unwrap()
+    );
+
+    // Add the image manifest
+    let _descriptor = oci_dir
+        .insert_manifest_and_config(manifest.clone(), config, None, Platform::default())
+        .unwrap();
+}
+
+#[cfg(not(feature = "zstdmt"))]
+fn main() {
+    println!("Run this example with `cargo run --example zstd --features zstdmt`");
+}