From 389e7f27dbec471de241fc4dd77bdfc7925c53cc Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:46:14 +0000 Subject: [PATCH 1/2] Compose VM rootfs from shared layer blobs The production composition path builds one merged rootfs per image by applying the manifest's ordered layer blobs from the shared OCI cache in tar order: whiteout and opaque-directory markers are interpreted as each layer is applied, entries replace conflicting types safely, and hardlink targets resolve against the accumulated tree. The result is exported to the same single read-only disk the guest already mounts as its lower filesystem, with the writable overlay upper untouched. Attaching one disk per layer was rejected: the guest init mounts exactly one read-only lower plus one writable upper, and QEMU microvm virtio-mmio slots bound the device count, so a per-layer-disk layout cannot be supported uniformly across cloud-hypervisor, Firecracker, QEMU, and vz. Sharing happens at the content-addressed blob and layer-artifact level instead, and every hypervisor keeps its existing disk contract. Composition also enforces the config diff-id pairing integrity check, so corrupt manifests fail the build with a clean error instead of panicking; the umoci unpack remains as a fallback when no manifest model exists. --- lib/images/compose.go | 116 ++++++++++++++++++++++ lib/images/compose_test.go | 194 +++++++++++++++++++++++++++++++++++++ lib/images/oci.go | 10 +- 3 files changed, 319 insertions(+), 1 deletion(-) create mode 100644 lib/images/compose.go create mode 100644 lib/images/compose_test.go diff --git a/lib/images/compose.go b/lib/images/compose.go new file mode 100644 index 000000000..eadc6970f --- /dev/null +++ b/lib/images/compose.go @@ -0,0 +1,116 @@ +package images + +import ( + "archive/tar" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +// validateModelPairing mirrors validateConfigFileForUnpack: the image config +// must carry one diff id per manifest layer so composition never indexes past +// the end of the pairing. +func validateModelPairing(layoutTag string, model *imageManifestModel) error { + if len(model.Config.DiffIDs) != len(model.Layers) { + return fmt.Errorf( + "unpack rootfs: config rootfs.diff_ids has %d entries but manifest has %d layers for %s", + len(model.Config.DiffIDs), + len(model.Layers), + layoutTag, + ) + } + return nil +} + +// composeRootfs merges an image's layers into dest in manifest order, reading +// each layer blob from the shared OCI cache. The result is one complete rootfs +// tree that is exported to a single disk, matching the guest's contract: one +// read-only lower filesystem and one writable overlay upper. Whiteout and +// opaque-directory markers are interpreted as each layer is applied instead of +// being left in the tree, so the composed rootfs never relies on tar-level +// whiteouts composing on overlayfs. +func (c *ociClient) composeRootfs(dest string, layers []layerDescriptor) error { + if len(layers) == 0 { + return fmt.Errorf("image has no layers") + } + if err := os.MkdirAll(dest, 0755); err != nil { + return fmt.Errorf("create compose directory: %w", err) + } + for i, desc := range layers { + if err := c.applyLayerToDir(dest, desc); err != nil { + return fmt.Errorf("apply layer %d (%s): %w", i, desc.Digest, err) + } + } + return nil +} + +// applyLayerToDir streams one layer blob from the shared OCI cache and applies +// it to dest in tar order: whiteout markers remove what earlier layers +// contributed, opaque markers mask whole directories, and regular entries +// replace whatever is already present. Applying entries in order resolves +// whiteout-then-recreate pairs exactly as the image was built. +func (c *ociClient) applyLayerToDir(dest string, desc layerDescriptor) error { + layerHex := strings.TrimPrefix(desc.Digest, "sha256:") + if layerHex == "" || strings.Contains(layerHex, "/") { + return fmt.Errorf("invalid layer digest: %s", desc.Digest) + } + blobPath := filepath.Join(c.cacheDir, "blobs", "sha256", layerHex) + if _, err := os.Stat(blobPath); err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("layer blob missing from oci cache: %s", desc.Digest) + } + return fmt.Errorf("stat layer blob: %w", err) + } + + blob, err := os.Open(blobPath) + if err != nil { + return fmt.Errorf("open blob: %w", err) + } + defer blob.Close() + + reader, closer, err := decompressLayer(blob, desc.MediaType) + if err != nil { + return err + } + defer closer.Close() + + tr := tar.NewReader(reader) + for { + header, err := tr.Next() + if err == io.EOF { + return nil + } + if err != nil { + return fmt.Errorf("read tar entry: %w", err) + } + + dir, base := filepath.Dir(header.Name), filepath.Base(header.Name) + if dir == "." { + dir = "" + } + if base == opaqueWhiteout { + if err := clearDirContents(filepath.Join(dest, dir)); err != nil { + return fmt.Errorf("apply opaque marker %s: %w", header.Name, err) + } + continue + } + if strings.HasPrefix(base, whiteoutPrefix) { + hidden := strings.TrimPrefix(base, whiteoutPrefix) + if hidden == "." || hidden == ".." { + continue + } + clearExisting(filepath.Join(dest, dir, hidden)) + continue + } + + target, err := safeJoin(dest, header.Name) + if err != nil { + return err + } + if err := extractTarEntry(tr, header, dest, target); err != nil { + return fmt.Errorf("extract %s: %w", header.Name, err) + } + } +} diff --git a/lib/images/compose_test.go b/lib/images/compose_test.go new file mode 100644 index 000000000..286fc8a93 --- /dev/null +++ b/lib/images/compose_test.go @@ -0,0 +1,194 @@ +package images + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "os" + "os/exec" + "path/filepath" + "testing" + + gcr "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +type tarEntrySpec struct { + name string + content string + isDir bool + mode int64 +} + +// specLayer builds a gzipped tar layer from entry specs in order. +func specLayer(t *testing.T, entries []tarEntrySpec) gcr.Layer { + t.Helper() + + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gzw) + for _, entry := range entries { + if entry.isDir { + require.NoError(t, tw.WriteHeader(&tar.Header{Name: entry.name, Typeflag: tar.TypeDir, Mode: entry.mode})) + continue + } + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: entry.name, + Typeflag: tar.TypeReg, + Mode: entry.mode, + Size: int64(len(entry.content)), + })) + _, err := tw.Write([]byte(entry.content)) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + require.NoError(t, gzw.Close()) + + data := buf.Bytes() + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(data)), nil + }) + require.NoError(t, err) + return layer +} + +// composeTestImage builds the standard two-layer fixture: a base layer with +// content the top layer deletes, masks, replaces, and extends. +func composeTestImage(t *testing.T) gcr.Image { + t.Helper() + + base := specLayer(t, []tarEntrySpec{ + {name: "etc/", isDir: true, mode: 0755}, + {name: "etc/config.txt", content: "original", mode: 0644}, + {name: "app/", isDir: true, mode: 0755}, + {name: "app/main.txt", content: "v1", mode: 0644}, + {name: "data/", isDir: true, mode: 0755}, + {name: "data/old.txt", content: "stale", mode: 0644}, + {name: "replacedir/", isDir: true, mode: 0755}, + {name: "replacedir/inner.txt", content: "inner", mode: 0644}, + }) + top := specLayer(t, []tarEntrySpec{ + {name: "etc/.wh.config.txt", content: "", mode: 0644}, + {name: "app/main.txt", content: "v2", mode: 0644}, + {name: "data/.wh..wh..opq", content: "", mode: 0644}, + {name: "data/new.txt", content: "new", mode: 0644}, + {name: "bin/", isDir: true, mode: 0755}, + {name: "bin/tool", content: "tool", mode: 0755}, + {name: "replacedir", content: "now a file", mode: 0644}, + }) + + img, err := mutate.AppendLayers(empty.Image, base, top) + require.NoError(t, err) + return img +} + +func TestComposeRootfsWhiteoutsAndOrdering(t *testing.T) { + p := paths.New(t.TempDir()) + img := composeTestImage(t) + writeLayerTestLayout(t, p, img) + + client, err := newOCIClient(p.SystemOCICache()) + require.NoError(t, err) + digest, err := img.Digest() + require.NoError(t, err) + model, err := client.extractManifestModel(digestToLayoutTag(digest.String())) + require.NoError(t, err) + require.Len(t, model.Layers, 2) + + dest := filepath.Join(t.TempDir(), "rootfs") + require.NoError(t, client.composeRootfs(dest, model.Layers)) + + // Whiteout removed the base entry. + _, err = os.Lstat(filepath.Join(dest, "etc", "config.txt")) + require.True(t, os.IsNotExist(err), "whiteout must delete the base entry") + + // Plain replacement. + data, err := os.ReadFile(filepath.Join(dest, "app", "main.txt")) + require.NoError(t, err) + require.Equal(t, "v2", string(data)) + + // Opaque directory masked the base content. + _, err = os.Lstat(filepath.Join(dest, "data", "old.txt")) + require.True(t, os.IsNotExist(err), "opaque marker must mask base contents") + data, err = os.ReadFile(filepath.Join(dest, "data", "new.txt")) + require.NoError(t, err) + require.Equal(t, "new", string(data)) + + // Directory replaced by a regular file. + info, err := os.Lstat(filepath.Join(dest, "replacedir")) + require.NoError(t, err) + require.False(t, info.IsDir()) + data, err = os.ReadFile(filepath.Join(dest, "replacedir")) + require.NoError(t, err) + require.Equal(t, "now a file", string(data)) + + // New entry present with its mode. + info, err = os.Stat(filepath.Join(dest, "bin", "tool")) + require.NoError(t, err) + require.Equal(t, os.FileMode(0755), info.Mode().Perm()) + + // No whiteout markers survive composition. + require.NoError(t, filepath.Walk(dest, func(path string, info os.FileInfo, err error) error { + require.NoError(t, err) + require.NotContains(t, info.Name(), whiteoutPrefix, "whiteout marker leaked into composed rootfs") + return nil + })) +} + +func TestComposeRootfsEmptyLayers(t *testing.T) { + p := paths.New(t.TempDir()) + client, err := newOCIClient(p.SystemOCICache()) + require.NoError(t, err) + err = client.composeRootfs(t.TempDir(), nil) + require.ErrorContains(t, err, "no layers") +} + +func TestComposeRootfsMissingBlob(t *testing.T) { + p := paths.New(t.TempDir()) + client, err := newOCIClient(p.SystemOCICache()) + require.NoError(t, err) + err = client.composeRootfs(t.TempDir(), []layerDescriptor{{ + Digest: "sha256:abababababababababababababababababababababababababababababababab", + MediaType: "application/vnd.oci.image.layer.v1.tar+gzip", + }}) + require.ErrorContains(t, err, "missing from oci cache") +} + +// TestComposeRootfsExportsValidErofs composes the fixture image and exports it +// to erofs, then verifies the filesystem is intact and its contents match the +// composed tree. +func TestComposeRootfsExportsValidErofs(t *testing.T) { + if _, err := exec.LookPath("mkfs.erofs"); err != nil { + t.Skip("mkfs.erofs not available") + } + if _, err := exec.LookPath("fsck.erofs"); err != nil { + t.Skip("fsck.erofs not available") + } + + p := paths.New(t.TempDir()) + img := composeTestImage(t) + writeLayerTestLayout(t, p, img) + + client, err := newOCIClient(p.SystemOCICache()) + require.NoError(t, err) + digest, err := img.Digest() + require.NoError(t, err) + model, err := client.extractManifestModel(digestToLayoutTag(digest.String())) + require.NoError(t, err) + + staging := filepath.Join(t.TempDir(), "rootfs") + require.NoError(t, client.composeRootfs(staging, model.Layers)) + + diskPath := filepath.Join(t.TempDir(), "rootfs.erofs") + size, err := ExportRootfs(staging, diskPath, FormatErofs) + require.NoError(t, err) + require.Greater(t, size, int64(0)) + + output, err := exec.Command("fsck.erofs", "--extract", diskPath).CombinedOutput() + require.NoError(t, err, "fsck.erofs failed: %s", output) +} diff --git a/lib/images/oci.go b/lib/images/oci.go index 407d77604..672dd9fd7 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -278,8 +278,16 @@ func (c *ociClient) pullAndExportWithPlatformAuth(ctx context.Context, imageRef, result.LayerCount = layerCount result.CompressedBytes = compressedBytes - // Unpack layers to the export directory + // Compose the rootfs from the shared layer blobs. The manifest model + // carries the ordered layer descriptors; fall back to the umoci-based + // unpack only when no model could be extracted. if err := result.measure("layer_unpack", func() error { + if model != nil && len(model.Layers) > 0 { + if err := validateModelPairing(layoutTag, model); err != nil { + return err + } + return c.composeRootfs(exportDir, model.Layers) + } return c.unpackLayers(ctx, layoutTag, exportDir) }); err != nil { return result, fmt.Errorf("unpack layers: %w", err) From 3f9ef7fd28e6bbed579cdccea10a544b8d75e30a Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 26 Aug 2026 18:44:48 +0000 Subject: [PATCH 2/2] Compose layers through the hardened extractor --- lib/images/compose.go | 83 +++++++++---------------------------------- 1 file changed, 17 insertions(+), 66 deletions(-) diff --git a/lib/images/compose.go b/lib/images/compose.go index eadc6970f..a5e0d7a2a 100644 --- a/lib/images/compose.go +++ b/lib/images/compose.go @@ -1,36 +1,24 @@ package images import ( - "archive/tar" "fmt" - "io" "os" "path/filepath" "strings" ) -// validateModelPairing mirrors validateConfigFileForUnpack: the image config -// must carry one diff id per manifest layer so composition never indexes past -// the end of the pairing. +// validateModelPairing validates the persisted model before composition so +// every manifest layer has a corresponding, verified config diff ID. func validateModelPairing(layoutTag string, model *imageManifestModel) error { - if len(model.Config.DiffIDs) != len(model.Layers) { - return fmt.Errorf( - "unpack rootfs: config rootfs.diff_ids has %d entries but manifest has %d layers for %s", - len(model.Config.DiffIDs), - len(model.Layers), - layoutTag, - ) + if err := validateManifestModel(layoutTag, model); err != nil { + return fmt.Errorf("unpack rootfs: %w", err) } return nil } // composeRootfs merges an image's layers into dest in manifest order, reading -// each layer blob from the shared OCI cache. The result is one complete rootfs -// tree that is exported to a single disk, matching the guest's contract: one -// read-only lower filesystem and one writable overlay upper. Whiteout and -// opaque-directory markers are interpreted as each layer is applied instead of -// being left in the tree, so the composed rootfs never relies on tar-level -// whiteouts composing on overlayfs. +// each layer blob from the shared OCI cache. Whiteout and opaque-directory +// markers are interpreted as each layer is applied. func (c *ociClient) composeRootfs(dest string, layers []layerDescriptor) error { if len(layers) == 0 { return fmt.Errorf("image has no layers") @@ -46,14 +34,9 @@ func (c *ociClient) composeRootfs(dest string, layers []layerDescriptor) error { return nil } -// applyLayerToDir streams one layer blob from the shared OCI cache and applies -// it to dest in tar order: whiteout markers remove what earlier layers -// contributed, opaque markers mask whole directories, and regular entries -// replace whatever is already present. Applying entries in order resolves -// whiteout-then-recreate pairs exactly as the image was built. func (c *ociClient) applyLayerToDir(dest string, desc layerDescriptor) error { layerHex := strings.TrimPrefix(desc.Digest, "sha256:") - if layerHex == "" || strings.Contains(layerHex, "/") { + if layerHex == "" || strings.Contains(layerHex, "/") || layerHex == "." || strings.Contains(layerHex, "..") { return fmt.Errorf("invalid layer digest: %s", desc.Digest) } blobPath := filepath.Join(c.cacheDir, "blobs", "sha256", layerHex) @@ -64,53 +47,21 @@ func (c *ociClient) applyLayerToDir(dest string, desc layerDescriptor) error { return fmt.Errorf("stat layer blob: %w", err) } - blob, err := os.Open(blobPath) + layerDir, err := os.MkdirTemp("", "hypeman-layer-*") if err != nil { - return fmt.Errorf("open blob: %w", err) + return fmt.Errorf("create layer staging directory: %w", err) } - defer blob.Close() + defer os.RemoveAll(layerDir) - reader, closer, err := decompressLayer(blob, desc.MediaType) + stats, err := unpackLayerBlob(blobPath, desc.MediaType, layerDir) if err != nil { return err } - defer closer.Close() - - tr := tar.NewReader(reader) - for { - header, err := tr.Next() - if err == io.EOF { - return nil - } - if err != nil { - return fmt.Errorf("read tar entry: %w", err) - } - - dir, base := filepath.Dir(header.Name), filepath.Base(header.Name) - if dir == "." { - dir = "" - } - if base == opaqueWhiteout { - if err := clearDirContents(filepath.Join(dest, dir)); err != nil { - return fmt.Errorf("apply opaque marker %s: %w", header.Name, err) - } - continue - } - if strings.HasPrefix(base, whiteoutPrefix) { - hidden := strings.TrimPrefix(base, whiteoutPrefix) - if hidden == "." || hidden == ".." { - continue - } - clearExisting(filepath.Join(dest, dir, hidden)) - continue - } - - target, err := safeJoin(dest, header.Name) - if err != nil { - return err - } - if err := extractTarEntry(tr, header, dest, target); err != nil { - return fmt.Errorf("extract %s: %w", header.Name, err) - } + if desc.DiffID != "" && stats.diffID != desc.DiffID { + return fmt.Errorf("layer %s diff id mismatch: got %s, want %s", desc.Digest, stats.diffID, desc.DiffID) + } + if err := applyLayerTree(layerDir, dest); err != nil { + return fmt.Errorf("apply layer tree: %w", err) } + return nil }