From f0fc2ca209a29e1a996eb323b61328fa709f3f65 Mon Sep 17 00:00:00 2001 From: Matthew Penner Date: Tue, 2 May 2023 16:22:58 -0600 Subject: [PATCH] image: multi-manifest index support Signed-off-by: Matthew Penner --- client.go | 56 ++++++-- pkg/image/image_index.go | 42 ++++++ pkg/image/oci/directory_provider.go | 127 ++++++++++++++++-- pkg/image/oci/directory_provider_test.go | 38 +++++- pkg/image/oci/tarball_provider.go | 27 +++- pkg/image/oci/tarball_provider_test.go | 6 +- ...fe0ee5dae9bd8caf8efa6c8144f7f2a43889dc513b | 0 ...fe0ee5dae9bd8caf8efa6c8144f7f2a43889dc513c | 0 .../valid_manifest_index/index.json | 24 ++++ pkg/image/provider.go | 5 + 10 files changed, 299 insertions(+), 26 deletions(-) create mode 100644 pkg/image/image_index.go create mode 100644 pkg/image/oci/test-fixtures/valid_manifest_index/blobs/sha256/f67dcc5fc786f04f0743abfe0ee5dae9bd8caf8efa6c8144f7f2a43889dc513b create mode 100644 pkg/image/oci/test-fixtures/valid_manifest_index/blobs/sha256/f67dcc5fc786f04f0743abfe0ee5dae9bd8caf8efa6c8144f7f2a43889dc513c create mode 100644 pkg/image/oci/test-fixtures/valid_manifest_index/index.json diff --git a/client.go b/client.go index b5056898..8d947972 100644 --- a/client.go +++ b/client.go @@ -104,6 +104,42 @@ func GetImageFromSource(ctx context.Context, imgStr string, source image.Source, return img, nil } +// GetImageIndexFromSource returns an image index from the explicitly provided source. +func GetImageIndexFromSource(ctx context.Context, imgStr string, source image.Source, options ...Option) (*image.Index, error) { + log.Debugf("image index: source=%+v location=%+v", source, imgStr) + + var cfg config + for _, option := range options { + if option == nil { + continue + } + if err := option(&cfg); err != nil { + return nil, fmt.Errorf("unable to parse option: %w", err) + } + } + + provider, cleanup, err := selectImageProvider(imgStr, source, cfg) + if cleanup != nil { + defer cleanup() + } + if err != nil { + return nil, err + } + + var indexProvider image.IndexProvider + var ok bool + if indexProvider, ok = provider.(image.IndexProvider); !ok { + return nil, fmt.Errorf("provider doesn't support image indexes") + } + + index, err := indexProvider.ProvideIndex(ctx, cfg.AdditionalMetadata...) + if err != nil { + return nil, fmt.Errorf("unable to use %s source: %w", source, err) + } + + return index, nil +} + // nolint:funlen func selectImageProvider(imgStr string, source image.Source, cfg config) (image.Provider, func(), error) { var provider image.Provider @@ -168,15 +204,9 @@ func selectImageProvider(imgStr string, source image.Source, cfg config) (image. return nil, cleanup, err } case image.OciDirectorySource: - if cfg.Platform != nil { - return nil, cleanup, platformSelectionUnsupported - } - provider = oci.NewProviderFromPath(imgStr, tempDirGenerator) + provider = oci.NewProviderFromPath(imgStr, tempDirGenerator, cfg.Platform) case image.OciTarballSource: - if cfg.Platform != nil { - return nil, cleanup, platformSelectionUnsupported - } - provider = oci.NewProviderFromTarball(imgStr, tempDirGenerator) + provider = oci.NewProviderFromTarball(imgStr, tempDirGenerator, cfg.Platform) case image.OciRegistrySource: defaultPlatformIfNil(&cfg) provider = oci.NewProviderFromRegistry(imgStr, tempDirGenerator, cfg.Registry, cfg.Platform) @@ -216,6 +246,16 @@ func GetImage(ctx context.Context, userStr string, options ...Option) (*image.Im return GetImageFromSource(ctx, imgStr, source, options...) } +// GetImageIndex parses the user provided image string and provides an index object; +// note: the source where the image should be referenced from is automatically inferred. +func GetImageIndex(ctx context.Context, userStr string, options ...Option) (*image.Index, error) { + source, imgStr, err := image.DetectSource(userStr) + if err != nil { + return nil, err + } + return GetImageIndexFromSource(ctx, imgStr, source, options...) +} + func SetLogger(logger logger.Logger) { log.Log = logger } diff --git a/pkg/image/image_index.go b/pkg/image/image_index.go new file mode 100644 index 00000000..89228d12 --- /dev/null +++ b/pkg/image/image_index.go @@ -0,0 +1,42 @@ +package image + +import ( + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/hashicorp/go-multierror" +) + +// Index represents a container image index. +type Index struct { + // index is the raw index manifest and content provider from the GCR lib + index v1.ImageIndex + // images is a list of images associated with an index. + images []*Image +} + +// NewIndex provides a new image index object. +func NewIndex(index v1.ImageIndex, images []*Image) *Index { + return &Index{ + index: index, + images: images, + } +} + +// Images returns a list of images associated with an index. +func (i *Index) Images() []*Image { + return i.images +} + +// Cleanup removes all temporary files created from parsing the index and associated images. +// Future calls to image will not function correctly after this call. +func (i *Index) Cleanup() error { + if i == nil { + return nil + } + var errs error + for _, img := range i.images { + if err := img.Cleanup(); err != nil { + errs = multierror.Append(errs, err) + } + } + return errs +} diff --git a/pkg/image/oci/directory_provider.go b/pkg/image/oci/directory_provider.go index 549d5870..7624aeb4 100644 --- a/pkg/image/oci/directory_provider.go +++ b/pkg/image/oci/directory_provider.go @@ -2,7 +2,9 @@ package oci import ( "context" + "errors" "fmt" + "strconv" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/layout" @@ -15,13 +17,15 @@ import ( type DirectoryImageProvider struct { path string tmpDirGen *file.TempDirGenerator + platform *image.Platform } // NewProviderFromPath creates a new provider instance for the specific image already at the given path. -func NewProviderFromPath(path string, tmpDirGen *file.TempDirGenerator) *DirectoryImageProvider { +func NewProviderFromPath(path string, tmpDirGen *file.TempDirGenerator, platform *image.Platform) *DirectoryImageProvider { return &DirectoryImageProvider{ path: path, tmpDirGen: tmpDirGen, + platform: platform, } } @@ -42,24 +46,65 @@ func (p *DirectoryImageProvider) Provide(_ context.Context, userMetadata ...imag return nil, fmt.Errorf("unable to parse OCI directory indexManifest: %w", err) } - // for now, lets only support one image indexManifest (it is not clear how to handle multiple manifests) - if len(indexManifest.Manifests) != 1 { - if len(indexManifest.Manifests) == 0 { - return nil, fmt.Errorf("unexpected number of OCI directory manifests (found %d)", len(indexManifest.Manifests)) - } - // if all the manifests have the same digest, then we can treat this as a single image - if !checkManifestDigestsEqual(indexManifest.Manifests) { - return nil, fmt.Errorf("unexpected number of OCI directory manifests (found %d)", len(indexManifest.Manifests)) + manifestLen := len(indexManifest.Manifests) + if manifestLen < 1 { + return nil, errors.New("expected at least one OCI manifest (found 0)") + } + + var manifest v1.Descriptor + if manifestLen > 1 { + if p.platform == nil { + // if all the manifests have the same digest, then we can treat this as a single image + if !checkManifestDigestsEqual(indexManifest.Manifests) { + // TODO: default to the current OS? + return nil, errors.New("when a OCI manifest contains multiple references, a platform selector is required") + } + manifest = indexManifest.Manifests[0] + } else { + var found bool + for _, m := range indexManifest.Manifests { + if m.Platform == nil { + continue + } + + // Check if the manifest's platform matches our selector. + if m.Platform.OS != p.platform.OS { + continue + } + + // Check if the manifest's architecture matches our selector. + if m.Platform.Architecture != p.platform.Architecture { + continue + } + + // Check if the manifest's variant matches our selector. + if m.Platform.Variant != p.platform.Variant { + continue + } + + // TODO: there is the possibility that multiple manifests may match. + // Do we continue iterating all of them and check if multiple matches + // exist then throw an error? + manifest = m + found = true + break + } + + if !found { + return nil, fmt.Errorf("unable to find a OCI manifest matching the given platform (platform: %s)", p.platform) + } } + } else { + // Only one manifest exists, so use it. + manifest = indexManifest.Manifests[0] } - manifest := indexManifest.Manifests[0] img, err := pathObj.Image(manifest.Digest) if err != nil { return nil, fmt.Errorf("unable to parse OCI directory as an image: %w", err) } - var metadata = []image.AdditionalMetadata{ + metadata := []image.AdditionalMetadata{ image.WithManifestDigest(manifest.Digest.String()), } @@ -80,6 +125,66 @@ func (p *DirectoryImageProvider) Provide(_ context.Context, userMetadata ...imag return image.New(img, p.tmpDirGen, contentTempDir, metadata...), nil } +// ProvideIndex provides an image index that represents the OCI image as a directory. +func (p *DirectoryImageProvider) ProvideIndex(_ context.Context, userMetadata ...image.AdditionalMetadata) (*image.Index, error) { + pathObj, err := layout.FromPath(p.path) + if err != nil { + return nil, fmt.Errorf("unable to read image from OCI directory path %q: %w", p.path, err) + } + + index, err := layout.ImageIndexFromPath(p.path) + if err != nil { + return nil, fmt.Errorf("unable to parse OCI directory index: %w", err) + } + + indexManifest, err := index.IndexManifest() + if err != nil { + return nil, fmt.Errorf("unable to parse OCI directory indexManifest: %w", err) + } + + if len(indexManifest.Manifests) < 1 { + return nil, fmt.Errorf("expected at least one OCI directory manifests (found %d)", len(indexManifest.Manifests)) + } + + images := make([]*image.Image, len(indexManifest.Manifests)) + for i, manifest := range indexManifest.Manifests { + img, err := pathObj.Image(manifest.Digest) + if err != nil { + return nil, fmt.Errorf("unable to parse OCI directory as an image: %w", err) + } + + metadata := []image.AdditionalMetadata{ + image.WithManifestDigest(manifest.Digest.String()), + } + if manifest.Platform != nil { + if manifest.Platform.Architecture != "" { + metadata = append(metadata, image.WithArchitecture(manifest.Platform.Architecture, manifest.Platform.Variant)) + } + if manifest.Platform.OS != "" { + metadata = append(metadata, image.WithOS(manifest.Platform.OS)) + } + } + + // make a best-effort attempt at getting the raw indexManifest + rawManifest, err := img.RawManifest() + if err == nil { + metadata = append(metadata, image.WithManifest(rawManifest)) + } + + // apply user-supplied metadata last to override any default behavior + metadata = append(metadata, userMetadata...) + + contentTempDir, err := p.tmpDirGen.NewDirectory("oci-dir-image-" + strconv.Itoa(i)) + if err != nil { + return nil, err + } + + images[i] = image.New(img, p.tmpDirGen, contentTempDir, metadata...) + } + + return image.NewIndex(index, images), nil +} + func checkManifestDigestsEqual(manifests []v1.Descriptor) bool { if len(manifests) < 1 { return false diff --git a/pkg/image/oci/directory_provider_test.go b/pkg/image/oci/directory_provider_test.go index f4fe8c8b..a58c2709 100644 --- a/pkg/image/oci/directory_provider_test.go +++ b/pkg/image/oci/directory_provider_test.go @@ -14,7 +14,7 @@ func Test_NewProviderFromPath(t *testing.T) { generator := file.TempDirGenerator{} //WHEN - provider := NewProviderFromPath(path, &generator) + provider := NewProviderFromPath(path, &generator, nil) //THEN assert.NotNil(t, provider.path) @@ -33,10 +33,11 @@ func Test_Directory_Provide(t *testing.T) { {"reads valid oci manifest with no images", "test-fixtures/no_manifests", true}, {"reads a fully correct manifest", "test-fixtures/valid_manifest", false}, {"reads a fully correct manifest with equal digests", "test-fixtures/valid_manifest", false}, + {"fails to read a fully correct manifest index with more than one manifest", "test-fixtures/valid_manifest_index", true}, } for _, tc := range tests { - provider := NewProviderFromPath(tc.path, file.NewTempDirGenerator("tempDir")) + provider := NewProviderFromPath(tc.path, file.NewTempDirGenerator("tempDir"), nil) t.Run(tc.name, func(t *testing.T) { //WHEN image, err := provider.Provide(nil) @@ -53,3 +54,36 @@ func Test_Directory_Provide(t *testing.T) { }) } } + +func Test_Directory_ProvideIndex(t *testing.T) { + //GIVEN + tests := []struct { + name string + path string + expectedErr bool + }{ + {"fails to read from path", "", true}, + {"reads invalid oci manifest", "test-fixtures/invalid_file", true}, + {"reads valid oci manifest with no images", "test-fixtures/no_manifests", true}, + {"reads a fully correct manifest", "test-fixtures/valid_manifest", false}, + {"reads a fully correct manifest index with more than one manifest", "test-fixtures/valid_manifest_index", false}, + } + + for _, tc := range tests { + provider := NewProviderFromPath(tc.path, file.NewTempDirGenerator("tempDir"), nil) + t.Run(tc.name, func(t *testing.T) { + //WHEN + image, err := provider.ProvideIndex(nil) + + //THEN + if tc.expectedErr { + assert.Error(t, err) + assert.Nil(t, image) + } else { + assert.NoError(t, err) + assert.NotNil(t, image) + } + + }) + } +} diff --git a/pkg/image/oci/tarball_provider.go b/pkg/image/oci/tarball_provider.go index 74a29278..a5674003 100644 --- a/pkg/image/oci/tarball_provider.go +++ b/pkg/image/oci/tarball_provider.go @@ -13,13 +13,15 @@ import ( type TarballImageProvider struct { path string tmpDirGen *file.TempDirGenerator + platform *image.Platform } // NewProviderFromTarball creates a new provider instance for the specific image tarball already at the given path. -func NewProviderFromTarball(path string, tmpDirGen *file.TempDirGenerator) *TarballImageProvider { +func NewProviderFromTarball(path string, tmpDirGen *file.TempDirGenerator, platform *image.Platform) *TarballImageProvider { return &TarballImageProvider{ path: path, tmpDirGen: tmpDirGen, + platform: platform, } } @@ -41,5 +43,26 @@ func (p *TarballImageProvider) Provide(ctx context.Context, metadata ...image.Ad return nil, err } - return NewProviderFromPath(tempDir, p.tmpDirGen).Provide(ctx, metadata...) + return NewProviderFromPath(tempDir, p.tmpDirGen, p.platform).Provide(ctx, metadata...) +} + +// ProvideIndex provides an image index that represents the OCI image index from a tarball. +func (p *TarballImageProvider) ProvideIndex(ctx context.Context, metadata ...image.AdditionalMetadata) (*image.Index, error) { + // note: we are untaring the image and using the existing directory provider, we could probably enhance the google + // container registry lib to do this without needing to untar to a temp dir (https://github.com/google/go-containerregistry/issues/726) + f, err := os.Open(p.path) + if err != nil { + return nil, fmt.Errorf("unable to open OCI tarball: %w", err) + } + + tempDir, err := p.tmpDirGen.NewDirectory("oci-tarball-image") + if err != nil { + return nil, err + } + + if err = file.UntarToDirectory(f, tempDir); err != nil { + return nil, err + } + + return NewProviderFromPath(tempDir, p.tmpDirGen, p.platform).ProvideIndex(ctx, metadata...) } diff --git a/pkg/image/oci/tarball_provider_test.go b/pkg/image/oci/tarball_provider_test.go index ae2707ca..203e3f71 100644 --- a/pkg/image/oci/tarball_provider_test.go +++ b/pkg/image/oci/tarball_provider_test.go @@ -14,7 +14,7 @@ func Test_NewProviderFromTarball(t *testing.T) { generator := file.TempDirGenerator{} //WHEN - provider := NewProviderFromTarball(path, &generator) + provider := NewProviderFromTarball(path, &generator, nil) //THEN assert.NotNil(t, provider.path) @@ -23,7 +23,7 @@ func Test_NewProviderFromTarball(t *testing.T) { func Test_TarballProvide(t *testing.T) { //GIVEN - provider := NewProviderFromTarball("test-fixtures/file.tar", file.NewTempDirGenerator("tempDir")) + provider := NewProviderFromTarball("test-fixtures/file.tar", file.NewTempDirGenerator("tempDir"), nil) //WHEN image, err := provider.Provide(nil) @@ -35,7 +35,7 @@ func Test_TarballProvide(t *testing.T) { func Test_TarballProvide_Fails(t *testing.T) { //GIVEN - provider := NewProviderFromTarball("", file.NewTempDirGenerator("tempDir")) + provider := NewProviderFromTarball("", file.NewTempDirGenerator("tempDir"), nil) //WHEN image, err := provider.Provide(nil) diff --git a/pkg/image/oci/test-fixtures/valid_manifest_index/blobs/sha256/f67dcc5fc786f04f0743abfe0ee5dae9bd8caf8efa6c8144f7f2a43889dc513b b/pkg/image/oci/test-fixtures/valid_manifest_index/blobs/sha256/f67dcc5fc786f04f0743abfe0ee5dae9bd8caf8efa6c8144f7f2a43889dc513b new file mode 100644 index 00000000..e69de29b diff --git a/pkg/image/oci/test-fixtures/valid_manifest_index/blobs/sha256/f67dcc5fc786f04f0743abfe0ee5dae9bd8caf8efa6c8144f7f2a43889dc513c b/pkg/image/oci/test-fixtures/valid_manifest_index/blobs/sha256/f67dcc5fc786f04f0743abfe0ee5dae9bd8caf8efa6c8144f7f2a43889dc513c new file mode 100644 index 00000000..e69de29b diff --git a/pkg/image/oci/test-fixtures/valid_manifest_index/index.json b/pkg/image/oci/test-fixtures/valid_manifest_index/index.json new file mode 100644 index 00000000..94be4bea --- /dev/null +++ b/pkg/image/oci/test-fixtures/valid_manifest_index/index.json @@ -0,0 +1,24 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", + "manifests": [ + { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "size": 424, + "digest": "sha256:f67dcc5fc786f04f0743abfe0ee5dae9bd8caf8efa6c8144f7f2a43889dc513b", + "platform": { + "architecture": "arm64", + "os": "linux" + } + }, + { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "size": 424, + "digest": "sha256:f67dcc5fc786f04f0743abfe0ee5dae9bd8caf8efa6c8144f7f2a43889dc513c", + "platform": { + "architecture": "amd64", + "os": "linux" + } + } + ] +} diff --git a/pkg/image/provider.go b/pkg/image/provider.go index a546cb32..2d2c7f53 100644 --- a/pkg/image/provider.go +++ b/pkg/image/provider.go @@ -7,3 +7,8 @@ import "context" type Provider interface { Provide(context.Context, ...AdditionalMetadata) (*Image, error) } + +// IndexProvider is an abstraction for any object that provides image indexes. +type IndexProvider interface { + ProvideIndex(context.Context, ...AdditionalMetadata) (*Index, error) +}