diff --git a/lib/images/compose.go b/lib/images/compose.go new file mode 100644 index 000000000..a5e0d7a2a --- /dev/null +++ b/lib/images/compose.go @@ -0,0 +1,67 @@ +package images + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// validateModelPairing validates the persisted model before composition so +// every manifest layer has a corresponding, verified config diff ID. +func validateModelPairing(layoutTag string, model *imageManifestModel) error { + if err := validateManifestModel(layoutTag, model); err != nil { + return fmt.Errorf("unpack rootfs: %w", err) + } + return nil +} + +// composeRootfs merges an image's layers into dest in manifest order, reading +// each layer blob from the shared OCI cache. Whiteout and opaque-directory +// markers are interpreted as each layer is applied. +func (c *ociClient) composeRootfs(dest string, layers []layerDescriptor) error { + if len(layers) == 0 { + return fmt.Errorf("image has no layers") + } + if err := os.MkdirAll(dest, 0755); err != nil { + return fmt.Errorf("create compose directory: %w", err) + } + for i, desc := range layers { + if err := c.applyLayerToDir(dest, desc); err != nil { + return fmt.Errorf("apply layer %d (%s): %w", i, desc.Digest, err) + } + } + return nil +} + +func (c *ociClient) applyLayerToDir(dest string, desc layerDescriptor) error { + layerHex := strings.TrimPrefix(desc.Digest, "sha256:") + if layerHex == "" || strings.Contains(layerHex, "/") || layerHex == "." || strings.Contains(layerHex, "..") { + return fmt.Errorf("invalid layer digest: %s", desc.Digest) + } + blobPath := filepath.Join(c.cacheDir, "blobs", "sha256", layerHex) + if _, err := os.Stat(blobPath); err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("layer blob missing from oci cache: %s", desc.Digest) + } + return fmt.Errorf("stat layer blob: %w", err) + } + + layerDir, err := os.MkdirTemp("", "hypeman-layer-*") + if err != nil { + return fmt.Errorf("create layer staging directory: %w", err) + } + defer os.RemoveAll(layerDir) + + stats, err := unpackLayerBlob(blobPath, desc.MediaType, layerDir) + if err != nil { + return err + } + if desc.DiffID != "" && stats.diffID != desc.DiffID { + return fmt.Errorf("layer %s diff id mismatch: got %s, want %s", desc.Digest, stats.diffID, desc.DiffID) + } + if err := applyLayerTree(layerDir, dest); err != nil { + return fmt.Errorf("apply layer tree: %w", err) + } + return nil +} diff --git a/lib/images/compose_test.go b/lib/images/compose_test.go new file mode 100644 index 000000000..286fc8a93 --- /dev/null +++ b/lib/images/compose_test.go @@ -0,0 +1,194 @@ +package images + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "os" + "os/exec" + "path/filepath" + "testing" + + gcr "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +type tarEntrySpec struct { + name string + content string + isDir bool + mode int64 +} + +// specLayer builds a gzipped tar layer from entry specs in order. +func specLayer(t *testing.T, entries []tarEntrySpec) gcr.Layer { + t.Helper() + + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gzw) + for _, entry := range entries { + if entry.isDir { + require.NoError(t, tw.WriteHeader(&tar.Header{Name: entry.name, Typeflag: tar.TypeDir, Mode: entry.mode})) + continue + } + require.NoError(t, tw.WriteHeader(&tar.Header{ + Name: entry.name, + Typeflag: tar.TypeReg, + Mode: entry.mode, + Size: int64(len(entry.content)), + })) + _, err := tw.Write([]byte(entry.content)) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + require.NoError(t, gzw.Close()) + + data := buf.Bytes() + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(data)), nil + }) + require.NoError(t, err) + return layer +} + +// composeTestImage builds the standard two-layer fixture: a base layer with +// content the top layer deletes, masks, replaces, and extends. +func composeTestImage(t *testing.T) gcr.Image { + t.Helper() + + base := specLayer(t, []tarEntrySpec{ + {name: "etc/", isDir: true, mode: 0755}, + {name: "etc/config.txt", content: "original", mode: 0644}, + {name: "app/", isDir: true, mode: 0755}, + {name: "app/main.txt", content: "v1", mode: 0644}, + {name: "data/", isDir: true, mode: 0755}, + {name: "data/old.txt", content: "stale", mode: 0644}, + {name: "replacedir/", isDir: true, mode: 0755}, + {name: "replacedir/inner.txt", content: "inner", mode: 0644}, + }) + top := specLayer(t, []tarEntrySpec{ + {name: "etc/.wh.config.txt", content: "", mode: 0644}, + {name: "app/main.txt", content: "v2", mode: 0644}, + {name: "data/.wh..wh..opq", content: "", mode: 0644}, + {name: "data/new.txt", content: "new", mode: 0644}, + {name: "bin/", isDir: true, mode: 0755}, + {name: "bin/tool", content: "tool", mode: 0755}, + {name: "replacedir", content: "now a file", mode: 0644}, + }) + + img, err := mutate.AppendLayers(empty.Image, base, top) + require.NoError(t, err) + return img +} + +func TestComposeRootfsWhiteoutsAndOrdering(t *testing.T) { + p := paths.New(t.TempDir()) + img := composeTestImage(t) + writeLayerTestLayout(t, p, img) + + client, err := newOCIClient(p.SystemOCICache()) + require.NoError(t, err) + digest, err := img.Digest() + require.NoError(t, err) + model, err := client.extractManifestModel(digestToLayoutTag(digest.String())) + require.NoError(t, err) + require.Len(t, model.Layers, 2) + + dest := filepath.Join(t.TempDir(), "rootfs") + require.NoError(t, client.composeRootfs(dest, model.Layers)) + + // Whiteout removed the base entry. + _, err = os.Lstat(filepath.Join(dest, "etc", "config.txt")) + require.True(t, os.IsNotExist(err), "whiteout must delete the base entry") + + // Plain replacement. + data, err := os.ReadFile(filepath.Join(dest, "app", "main.txt")) + require.NoError(t, err) + require.Equal(t, "v2", string(data)) + + // Opaque directory masked the base content. + _, err = os.Lstat(filepath.Join(dest, "data", "old.txt")) + require.True(t, os.IsNotExist(err), "opaque marker must mask base contents") + data, err = os.ReadFile(filepath.Join(dest, "data", "new.txt")) + require.NoError(t, err) + require.Equal(t, "new", string(data)) + + // Directory replaced by a regular file. + info, err := os.Lstat(filepath.Join(dest, "replacedir")) + require.NoError(t, err) + require.False(t, info.IsDir()) + data, err = os.ReadFile(filepath.Join(dest, "replacedir")) + require.NoError(t, err) + require.Equal(t, "now a file", string(data)) + + // New entry present with its mode. + info, err = os.Stat(filepath.Join(dest, "bin", "tool")) + require.NoError(t, err) + require.Equal(t, os.FileMode(0755), info.Mode().Perm()) + + // No whiteout markers survive composition. + require.NoError(t, filepath.Walk(dest, func(path string, info os.FileInfo, err error) error { + require.NoError(t, err) + require.NotContains(t, info.Name(), whiteoutPrefix, "whiteout marker leaked into composed rootfs") + return nil + })) +} + +func TestComposeRootfsEmptyLayers(t *testing.T) { + p := paths.New(t.TempDir()) + client, err := newOCIClient(p.SystemOCICache()) + require.NoError(t, err) + err = client.composeRootfs(t.TempDir(), nil) + require.ErrorContains(t, err, "no layers") +} + +func TestComposeRootfsMissingBlob(t *testing.T) { + p := paths.New(t.TempDir()) + client, err := newOCIClient(p.SystemOCICache()) + require.NoError(t, err) + err = client.composeRootfs(t.TempDir(), []layerDescriptor{{ + Digest: "sha256:abababababababababababababababababababababababababababababababab", + MediaType: "application/vnd.oci.image.layer.v1.tar+gzip", + }}) + require.ErrorContains(t, err, "missing from oci cache") +} + +// TestComposeRootfsExportsValidErofs composes the fixture image and exports it +// to erofs, then verifies the filesystem is intact and its contents match the +// composed tree. +func TestComposeRootfsExportsValidErofs(t *testing.T) { + if _, err := exec.LookPath("mkfs.erofs"); err != nil { + t.Skip("mkfs.erofs not available") + } + if _, err := exec.LookPath("fsck.erofs"); err != nil { + t.Skip("fsck.erofs not available") + } + + p := paths.New(t.TempDir()) + img := composeTestImage(t) + writeLayerTestLayout(t, p, img) + + client, err := newOCIClient(p.SystemOCICache()) + require.NoError(t, err) + digest, err := img.Digest() + require.NoError(t, err) + model, err := client.extractManifestModel(digestToLayoutTag(digest.String())) + require.NoError(t, err) + + staging := filepath.Join(t.TempDir(), "rootfs") + require.NoError(t, client.composeRootfs(staging, model.Layers)) + + diskPath := filepath.Join(t.TempDir(), "rootfs.erofs") + size, err := ExportRootfs(staging, diskPath, FormatErofs) + require.NoError(t, err) + require.Greater(t, size, int64(0)) + + output, err := exec.Command("fsck.erofs", "--extract", diskPath).CombinedOutput() + require.NoError(t, err, "fsck.erofs failed: %s", output) +} diff --git a/lib/images/oci.go b/lib/images/oci.go index 407d77604..672dd9fd7 100644 --- a/lib/images/oci.go +++ b/lib/images/oci.go @@ -278,8 +278,16 @@ func (c *ociClient) pullAndExportWithPlatformAuth(ctx context.Context, imageRef, result.LayerCount = layerCount result.CompressedBytes = compressedBytes - // Unpack layers to the export directory + // Compose the rootfs from the shared layer blobs. The manifest model + // carries the ordered layer descriptors; fall back to the umoci-based + // unpack only when no model could be extracted. if err := result.measure("layer_unpack", func() error { + if model != nil && len(model.Layers) > 0 { + if err := validateModelPairing(layoutTag, model); err != nil { + return err + } + return c.composeRootfs(exportDir, model.Layers) + } return c.unpackLayers(ctx, layoutTag, exportDir) }); err != nil { return result, fmt.Errorf("unpack layers: %w", err)