diff --git a/Cargo.toml b/Cargo.toml index 4b5a400..dccaf0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,10 +28,10 @@ rpm-qa = "0.3" serde = { version = "1", features = ["derive"] } serde_json = "1" tar = "0.4" +tempfile = "3" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } [dev-dependencies] fs-set-times = "0.20.3" maplit = "1" -tempfile = "3" diff --git a/Containerfile b/Containerfile index 7442530..4b7198e 100644 --- a/Containerfile +++ b/Containerfile @@ -51,6 +51,6 @@ RUN for db in /rootfs/var/lib/rpm/rpmdb.sqlite \ RUN --mount=type=bind,target=/run/src,rw \ chunkah build --rootfs /rootfs \ --config-str '{"Config": {"Entrypoint": ["/usr/bin/chunkah"], "Env": ["CHUNKAH_ROOTFS=/chunkah"], "WorkingDir": "/srv"}}' \ - > /run/src/out.ociarchive + --output oci:/run/src/out -FROM oci-archive:out.ociarchive +FROM oci:out diff --git a/Containerfile.splitter b/Containerfile.splitter index fbd3901..5b773a1 100644 --- a/Containerfile.splitter +++ b/Containerfile.splitter @@ -10,7 +10,7 @@ FROM ${CHUNKAH} AS chunkah ARG CHUNKAH_CONFIG_STR ARG CHUNKAH_ARGS RUN --mount=type=bind,target=/run/src,rw \ - --mount=from=target,target=/chunkah \ - chunkah build ${CHUNKAH_ARGS} > /run/src/out.ociarchive + --mount=from=target,target=/chunkah,ro \ + chunkah build ${CHUNKAH_ARGS} --output oci:/run/src/out -FROM oci-archive:out.ociarchive +FROM oci:out diff --git a/README-dev.md b/README-dev.md index 5049c3d..b785bd7 100644 --- a/README-dev.md +++ b/README-dev.md @@ -25,6 +25,7 @@ with content-based layers. - [Understanding components](#understanding-components) - [Customizing the layers](#customizing-the-layers) - [Limiting the number of layers](#limiting-the-number-of-layers) + - [Output options](#output-options) - [Building from a raw rootfs](#building-from-a-raw-rootfs) - [Customizing the OCI image config and annotations](#customizing-the-oci-image-config-and-annotations) - [Pruning and filtering](#pruning-and-filtering) @@ -122,7 +123,8 @@ argument. > RFE][buildah-rfe]). > > For Buildah versions before v1.44, this also requires `-v $(pwd):/run/src -> --security-opt=label=disable`. +> --security-opt=label=disable`. Note that this will leave behind an `out/` +> directory in the current directory. Another option is using the chunkah image directly and image mounts: @@ -139,6 +141,11 @@ The `-t`/`--tag` option sets the image name in the OCI archive so that `podman load` automatically tags the loaded image. Without it, the image is loaded as an unnamed image identified only by its digest. +For large images, this approach is less efficient than using +`Containerfile.splitter`, which avoids the overhead of tarring and untarring +the OCI archive (though the same efficiency can be gained by using +[`--output oci:PATH`](#output-options) and importing it using `skopeo copy`.) + #### Using Docker/Moby You can use the chunkah image directly using image mounts (requires v28+): @@ -166,9 +173,9 @@ docker load -i out.dockerarchive ### Splitting an image at build time (buildah/podman only) -This uses a method called the "`FROM oci-archive:` trick", for lack of a better -term. It has the massive advantage of being compatible with a regular `buildah -build` flow and also makes it more natural to apply configs to the image, but is +This uses a method called the "`FROM oci:` trick", for lack of a better term. +It has the massive advantage of being compatible with a regular `buildah build` +flow and also makes it more natural to apply configs to the image, but is specific to the Podman ecosystem. This *will not* work with Docker. ```Dockerfile @@ -182,11 +189,11 @@ RUN dnf install -y git-core && dnf clean all FROM quay.io/coreos/chunkah AS chunkah ARG CHUNKAH_CONFIG_STR -RUN --mount=from=builder,src=/,target=/chunkah,ro \ - --mount=type=bind,target=/run/src,rw \ - chunkah build > /run/src/out.ociarchive +RUN --mount=type=bind,target=/run/src,rw \ + --mount=from=builder,target=/chunkah,ro \ + chunkah build --output oci:/run/src/out -FROM oci-archive:out.ociarchive +FROM oci:out ENTRYPOINT ["git"] ``` @@ -196,7 +203,8 @@ ENTRYPOINT ["git"] > `--jobs` option. > > For Buildah versions before v1.44, this also requires `-v $(pwd):/run/src -> --security-opt=label=disable`. +> --security-opt=label=disable`. Note that this will leave behind an `out/` +> directory in the current directory. > [!NOTE] @@ -260,6 +268,25 @@ content-based layers. Too many layers may mean excessive processing and overhead when pushing/pulling the image. Note that containers-storage has a hard limit of 500 layers. +### Output options + +By default, chunkah writes an OCI archive to stdout. The `-o`/`--output` flag +controls where and in what format the image is written. It supports an optional +transport prefix: + +- `--output PATH` or `--output oci-archive:PATH` — write an OCI archive to a + file instead of stdout. +- `--output oci:PATH` — write the OCI directory layout directly to disk. This + avoids the overhead of tarring and untarring the archive, which is + particularly useful in the buildah `FROM oci:` flow (see [Splitting an image + at build time](#splitting-an-image-at-build-time-buildahpodman-only)). + +By default, layers are uncompressed (since in the common case the OCI +archive/directory is immediately imported into a container storage backend, +which would immediately uncompress it). Use `--compressed` to enable gzip +compression for layers (and the OCI archive itself, if applicable). The +compression level can be tuned with `--compression-level` (0-9, default 6). + ### Building from a raw rootfs For completeness, note it's of course also possible to split any arbitrary @@ -276,13 +303,8 @@ podman run --rm -v /path/to/rootfs:/chunkah:z \ > be expensive for a large rootfs. You can use `--security-opt=label=disable` to > avoid this, but it disables SELinux separation with the chunkah container. -When running chunkah directly in this way, the OCI archive is written to stdout -by default. Use `-o`/`--output` to write to a file instead (whose directory -would then have to be mounted in). - -By default, layers and the OCI archive are uncompressed. Use `--compressed` -to enable gzip compression for both. The compression level can be tuned with -`--compression-level` (0-9, default 6). +See [Output options](#output-options) for controlling the output format and +compression. ### Customizing the OCI image config and annotations @@ -363,9 +385,9 @@ RUN --mount=from=builder,src=/,target=/chunkah,ro \ --mount=type=bind,target=/run/src,rw \ chunkah build --prune /sysroot/ --max-layers 128 \ --label ostree.commit- --label ostree.final-diffid- \ - > /run/src/out.ociarchive + --output oci:/run/src/out -FROM oci-archive:out.ociarchive +FROM oci:out ``` ### Debugging diff --git a/src/cmd_build.rs b/src/cmd_build.rs index dcc1382..24e5911 100644 --- a/src/cmd_build.rs +++ b/src/cmd_build.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, HashMap}; use std::num::NonZeroUsize; use anyhow::{Context, Result}; -use camino::Utf8PathBuf; +use camino::{Utf8Path, Utf8PathBuf}; use cap_std_ext::cap_std::ambient_authority; use cap_std_ext::cap_std::fs::Dir; use clap::Parser; @@ -14,14 +14,28 @@ use crate::ocibuilder::{Builder, Compression}; use crate::packing::{PackItem, calculate_packing}; use crate::utils; +/// Parsed output target for the built OCI image. +enum OutputTarget { + /// OCI archive to stdout. + Stdout, + /// OCI archive to a file. + OciArchive(Utf8PathBuf), + /// OCI directory layout. + OciDir(Utf8PathBuf), +} + #[derive(Parser, Default)] pub struct BuildArgs { /// Path to the rootfs to build from #[arg(long, env = "CHUNKAH_ROOTFS", hide_env_values = true)] rootfs: Utf8PathBuf, - /// Output file path (defaults to stdout) - #[arg(short, long, value_name = "PATH")] + /// Output path with optional transport prefix + /// + /// Supports `oci:PATH` for OCI directory layout and `oci-archive:PATH` for + /// OCI archive. If no prefix is given, defaults to `oci-archive`. If not + /// specified at all, the OCI archive is written to stdout. + #[arg(short, long, value_name = "[oci:|oci-archive:]PATH")] output: Option, /// Maximum number of layers to output @@ -69,10 +83,11 @@ pub struct BuildArgs { )] source_date_epoch: Option, - /// Compress layers and the OCI archive with gzip + /// Compress layers (and the OCI archive, if applicable) with gzip /// - /// By default, layers and the OCI archive are uncompressed. This flag - /// enables gzip compression for both. + /// By default, layers are uncompressed. This flag enables gzip compression + /// for layers. For `oci-archive` format, the archive itself is also + /// compressed. #[arg(long)] compressed: bool, @@ -105,7 +120,7 @@ pub struct BuildArgs { #[arg(long = "prune", value_name = "PATH")] prune: Vec, - /// Tag to apply to the image in the OCI archive + /// Tag to apply to the image /// /// Sets the org.opencontainers.image.ref.name annotation on the manifest /// descriptor in index.json, so that `podman load`/`docker load` tags the @@ -166,6 +181,8 @@ impl BuildArgs { } pub fn run(args: &BuildArgs) -> Result<()> { + let output_target = parse_output_target(args.output.as_deref())?; + tracing::info!(rootfs = %args.rootfs, "starting build"); const CONTAINERS_STORAGE_LAYER_LIMIT: usize = 500; @@ -273,14 +290,21 @@ pub fn run(args: &BuildArgs) -> Result<()> { builder = builder.tag(tag.clone()); } - if let Some(output_path) = &args.output { - tracing::info!(output = %output_path, "writing to file"); - let mut file = std::fs::File::create(output_path) - .with_context(|| format!("creating output file {}", output_path))?; - builder.build(&mut file)?; - } else { - tracing::info!("writing to stdout"); - builder.build(&mut std::io::stdout().lock())?; + match output_target { + OutputTarget::OciDir(ref path) => { + // no logging needed here; build_to_oci_dir already logs + builder.build_to_oci_dir(path)?; + } + OutputTarget::OciArchive(ref path) => { + tracing::info!(output = %path, "writing to file"); + let mut file = std::fs::File::create(path) + .with_context(|| format!("creating output file {}", path))?; + builder.build_to_oci_archive(&mut file)?; + } + OutputTarget::Stdout => { + tracing::info!("writing to stdout"); + builder.build_to_oci_archive(&mut std::io::stdout().lock())?; + } } if let Some(path) = &args.write_peak_mem_to { @@ -293,6 +317,31 @@ pub fn run(args: &BuildArgs) -> Result<()> { Ok(()) } +/// Parse the `--output` value into an [`OutputTarget`]. +fn parse_output_target(output: Option<&Utf8Path>) -> Result { + match output.map(|o| o.as_str()) { + None => Ok(OutputTarget::Stdout), + Some(s) => match s.split_once(':') { + Some(("oci-archive", path)) => { + anyhow::ensure!(!path.is_empty(), "output path cannot be empty"); + Ok(OutputTarget::OciArchive(Utf8PathBuf::from(path))) + } + Some(("oci", path)) => { + anyhow::ensure!(!path.is_empty(), "output path cannot be empty"); + let path = Utf8PathBuf::from(path); + anyhow::ensure!(!path.exists(), "output path already exists: {path}"); + Ok(OutputTarget::OciDir(path)) + } + Some((transport, _)) => { + // technically breaks paths with literal ':'... let's see if anyone complains; they + // can always just redirect from stdout instead + anyhow::bail!("unknown transport: {transport}"); + } + None => Ok(OutputTarget::OciArchive(Utf8PathBuf::from(s))), + }, + } +} + /// Check whether the file map contains an OSTree sysroot repo. /// The scanner always includes parent directories, so the exact path /// will be present if any children under it were scanned. @@ -621,7 +670,7 @@ mod tests { .annotations(parsed.annotations) .config(image_config); let mut output = Vec::new(); - builder.build(&mut output).unwrap(); + builder.build_to_oci_archive(&mut output).unwrap(); // now extract it back out and open it as an ocidir let oci_tempdir = tempfile::tempdir().unwrap(); diff --git a/src/ocibuilder.rs b/src/ocibuilder.rs index 03590b2..cc3ff02 100644 --- a/src/ocibuilder.rs +++ b/src/ocibuilder.rs @@ -4,7 +4,9 @@ use std::num::NonZeroUsize; use std::sync::atomic::{AtomicUsize, Ordering}; use anyhow::{Context, Result}; +use camino::Utf8Path; use cap_std_ext::cap_std::fs::Dir; +use cap_std_ext::cap_tempfile; use ocidir::oci_spec::image as oci_image; use crate::components::Component; @@ -23,8 +25,6 @@ pub enum Compression { pub struct Builder { /// The rootfs to build from. rootfs: Dir, - /// The OCI directory to build into. - oci_dir: cap_std_ext::cap_tempfile::TempDir, /// The components to include in the image, ordered by stability descending. components: Vec<(String, Component)>, /// Compression settings for layers and archive. @@ -49,12 +49,8 @@ struct ComponentLayer { impl Builder { /// Create a new Builder with required parameters. pub fn new(rootfs: &Dir, components: Vec<(String, Component)>) -> Result { - let oci_dir = cap_std_ext::cap_tempfile::tempdir(cap_std_ext::cap_std::ambient_authority()) - .context("creating temp directory")?; - Ok(Self { rootfs: rootfs.try_clone().context("cloning rootfs")?, - oci_dir, components, compression: Compression::default(), threads: NonZeroUsize::MIN, @@ -98,9 +94,12 @@ impl Builder { self } - /// Build the OCI image and write it to the given output. - pub fn build(self, output: &mut W) -> Result<()> { - self.build_oci_dir().context("building OCI directory")?; + /// Build the OCI image and write it as an OCI archive to the given output. + pub fn build_to_oci_archive(self, output: &mut W) -> Result<()> { + let oci_dir = cap_tempfile::tempdir(cap_std_ext::cap_std::ambient_authority()) + .context("creating temp directory")?; + self.build_oci_dir(&oci_dir) + .context("building OCI directory")?; let compressed = !matches!(self.compression, Compression::None); tracing::info!(compressed = compressed, "writing OCI archive"); @@ -112,16 +111,43 @@ impl Builder { } }; - crate::tar::write_oci_archive(&self.oci_dir, &mut *output, compression) + crate::tar::write_oci_archive(&oci_dir, &mut *output, compression) .context("writing OCI archive")?; output.flush().context("flushing output") } - fn build_oci_dir(&self) -> Result<()> { + /// Build the OCI image and write it as an OCI directory layout. + pub fn build_to_oci_dir(self, output: &Utf8Path) -> Result<()> { + // Allocate the tempdir in the same dir as the target for rename(). + let parent = output + .parent() + .filter(|p| !p.as_str().is_empty()) + .unwrap_or(Utf8Path::new(".")); + // Notice here we use `tempfile::TempDir` rather than `cap_tempfile::TempDir` because we + // need an actually addressable path for rename(). `cap_tempfile` intentionally doesn't + // expose paths. + let mut temp_dir = tempfile::TempDir::with_prefix_in("chunkah-", parent.as_std_path()) + .context("creating temp directory")?; let oci_dir = - ocidir::OciDir::ensure(self.oci_dir.try_clone().context("cloning temp directory")?) - .context("creating OCI directory")?; + Dir::open_ambient_dir(temp_dir.path(), cap_std_ext::cap_std::ambient_authority()) + .context("opening temp directory")?; + self.build_oci_dir(&oci_dir) + .context("building OCI directory")?; + + tracing::info!(output = %output, "writing OCI directory"); + std::fs::rename(temp_dir.path(), output.as_std_path()) + .with_context(|| format!("renaming temp directory to {output}"))?; + // Disarm cleanup now that the rename succeeded. TempDir needs a .persist()... + temp_dir.disable_cleanup(true); + Ok(()) + } + + /// The underlying function called by build_to_oci_archive() and build_to_oci_dir() that does + /// all the heavy-lifting to actually build the image. + fn build_oci_dir(&self, dir: &Dir) -> Result<()> { + let oci_dir = ocidir::OciDir::ensure(dir.try_clone().context("cloning temp directory")?) + .context("creating OCI directory")?; // first, let's create an empty manifest let mut manifest = oci_dir @@ -133,7 +159,7 @@ impl Builder { let mut config = self.config.clone().unwrap_or_default(); // this is the important bit: we add all the layers - self.add_components(&mut manifest, &mut config) + self.add_components(dir, &mut manifest, &mut config) .context("adding layers to OCI directory")?; if let Some(annotations) = &self.annotations { @@ -157,6 +183,7 @@ impl Builder { /// Write layers to the OCI directory in parallel and update the manifest and config. fn add_components( &self, + oci_dir: &Dir, manifest: &mut oci_image::ImageManifest, config: &mut oci_image::ImageConfiguration, ) -> Result<()> { @@ -197,7 +224,7 @@ impl Builder { } let (name, component) = components[i]; let result = self - .write_component_layer(name, component) + .write_component_layer(oci_dir, name, component) .with_context(|| format!("adding component {name}")); results.push((i, result)); } @@ -216,7 +243,7 @@ impl Builder { // sort back based on index for reproducible builds results.sort_by_key(|(idx, _)| *idx); - let oci_dir = ocidir::OciDir::open(self.oci_dir.try_clone().context("cloning oci_dir")?) + let oci_dir = ocidir::OciDir::open(oci_dir.try_clone().context("cloning oci_dir")?) .context("opening OCI directory")?; for (_, result) in results { @@ -235,8 +262,13 @@ impl Builder { /// Write a single component as a tar layer. Returns the layer metadata for /// later assembly into the manifest and config. - fn write_component_layer(&self, name: &str, component: &Component) -> Result { - let oci_dir = ocidir::OciDir::open(self.oci_dir.try_clone().context("cloning oci_dir")?) + fn write_component_layer( + &self, + oci_dir: &Dir, + name: &str, + component: &Component, + ) -> Result { + let oci_dir = ocidir::OciDir::open(oci_dir.try_clone().context("cloning oci_dir")?) .context("opening OCI directory")?; tracing::debug!(component = name, "creating tar layer"); let mut tar_builder = @@ -397,7 +429,7 @@ mod tests { .compression(Compression::None) .config(config); let mut output = Vec::new(); - builder.build(&mut output).unwrap(); + builder.build_to_oci_archive(&mut output).unwrap(); // Extract to tempdir let oci_tempdir = tempfile::tempdir().unwrap(); diff --git a/tests/e2e/test-buildtime.sh b/tests/e2e/test-buildtime.sh index f6f22bc..a245212 100755 --- a/tests/e2e/test-buildtime.sh +++ b/tests/e2e/test-buildtime.sh @@ -28,7 +28,9 @@ RUN dnf install -y jq acl attr && dnf clean all EOF buildah build -t "${ROOTFS_IMAGE}" -f Containerfile.rootfs . -# build chunked image using FROM oci-archive: trick +# build chunked image using FROM oci-archive: trick (note the use of oci-archive +# instead of oci is intended here to make sure we have coverage for this now +# that the Containerfile.splitter approach has moved to oci) cat > Containerfile <