InTheForest/moby
1
0
Fork 0
mirror of https://github.com/moby/moby.git synced 2026-01-11 18:51:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add support for multiple platforms in image export and loading.

Browse Source 
Currently the image export and load APIs can be used to export or load all
platforms for the image, or a single specified platform.

This commit updates the API so that it accepts a list of platforms to export or
load, thereby giving clients the ability to export only selected platforms of an
image into a tar file, or load selected platforms from a tar file.

Unit and integration tests were updated accordingly.

As this requires a daemon API change, the API version was bumped.

Signed-off-by: Cesar Talledo <cesar.talledo@docker.com>
This commit is contained in:
Cesar Talledo
2025-05-28 13:53:08 -07:00
committed by Sebastiaan van Stijn
parent c55a163523
commit fcc8209e12
12 changed files with 482 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
api/swagger.yaml
View File

@@ -10443,7 +10443,10 @@ paths:
type: "string"
required: true
- name: "platform"
type: "string"
type: "array"
items:
type: "string"
collectionFormat: "multi"
in: "query"
description: |
JSON encoded OCI platform describing a platform which will be used
@@ -10488,13 +10491,16 @@ paths:
items:
type: "string"
- name: "platform"
type: "string"
type: "array"
items:
type: "string"
collectionFormat: "multi"
in: "query"
description: |
JSON encoded OCI platform describing a platform which will be used
to select a platform-specific image to be saved if the image is
multi-platform.
If not provided, the full multi-platform image will be saved.
JSON encoded OCI platform(s) which will be used to select the
platform-specific image(s) to be saved if the image is
multi-platform. If not provided, the full multi-platform image
will be saved.
Example: `{"os": "linux", "architecture": "arm", "variant": "v5"}`
tags: ["Image"]
@@ -10530,13 +10536,16 @@ paths:
type: "boolean"
default: false
- name: "platform"
type: "string"
type: "array"
items:
type: "string"
collectionFormat: "multi"
in: "query"
description: |
JSON encoded OCI platform describing a platform which will be used
to select a platform-specific image to be load if the image is
multi-platform.
If not provided, the full multi-platform image will be loaded.
JSON encoded OCI platform(s) which will be used to select the
platform-specific image(s) to load if the image is
multi-platform. If not provided, the full multi-platform image
will be loaded.
Example: `{"os": "linux", "architecture": "arm", "variant": "v5"}`
tags: ["Image"]

67
daemon/containerd/image_exporter.go
View File

@@ -31,8 +31,15 @@ import (
// outStream is the writer which the images are written to.
//
// TODO(thaJeztah): produce JSON stream progress response and image events; see https://github.com/moby/moby/issues/43910
func (i *ImageService) ExportImage(ctx context.Context, names []string, platform *ocispec.Platform, outStream io.Writer) error {
pm := i.matchRequestedOrDefault(platforms.OnlyStrict, platform)
func (i *ImageService) ExportImage(ctx context.Context, names []string, platformList []ocispec.Platform, outStream io.Writer) error {
var pm platforms.MatchComparer
// Get the platform matcher for the requested platforms
if len(platformList) == 0 {
pm = matchAllWithPreference(i.hostPlatformMatcher())
} else {
pm = matchAnyWithPreference(i.hostPlatformMatcher(), platformList)
}
opts := []archive.ExportOpt{
archive.WithSkipNonDistributableBlobs(),
@@ -65,8 +72,12 @@ func (i *ImageService) ExportImage(ctx context.Context, names []string, platform
exportImage := func(ctx context.Context, img c8dimages.Image, ref reference.Named) error {
target := img.Target
if platform != nil {
newTarget, err := i.getPushDescriptor(ctx, img, platform)
// If a single platform is requested, export the manifest for the specific platform only
// (single-level index). Otherwise export the full index (two-level, nested). Note that
// since opts includes WithPlatform and WithSkipMissing, the index will contain the
// requested platforms only, and only if they are available in the content store.
if len(platformList) == 1 {
newTarget, err := i.getPushDescriptor(ctx, img, &platformList[0])
if err != nil {
return errors.Wrap(err, "no suitable export target found")
}
@@ -229,14 +240,23 @@ func (i *ImageService) leaseContent(ctx context.Context, store content.Store, de
// LoadImage uploads a set of images into the repository. This is the
// complement of ExportImage. The input stream is an uncompressed tar
// ball containing images and metadata.
func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, platform *ocispec.Platform, outStream io.Writer, quiet bool) error {
func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, platformList []ocispec.Platform, outStream io.Writer, quiet bool) error {
decompressed, err := compression.DecompressStream(inTar)
if err != nil {
return errors.Wrap(err, "failed to decompress input tar archive")
}
defer decompressed.Close()
pm := i.matchRequestedOrDefault(platforms.OnlyStrict, platform)
specificPlatforms := len(platformList) > 0
// Get the platform matcher for the requested platforms
var pm platforms.MatchComparer
if specificPlatforms {
pm = platforms.Any(platformList...)
} else {
// All platforms
pm = matchAllWithPreference(i.hostPlatformMatcher())
}
opts := []containerd.ImportOpt{
containerd.WithImportPlatform(pm),
@@ -266,16 +286,19 @@ func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, platf
}),
}
if platform == nil {
if !specificPlatforms {
// Allow variants to be missing if no specific platform is requested.
opts = append(opts, containerd.WithSkipMissing())
}
imgs, err := i.client.Import(ctx, decompressed, opts...)
if err != nil {
if platform != nil {
p := platforms.FormatAll(*platform)
log.G(ctx).WithFields(log.Fields{"error": err, "platform": p}).Debug("failed to import image to containerd")
if specificPlatforms {
platformNames := make([]string, 0, len(platformList))
for _, p := range platformList {
platformNames = append(platformNames, platforms.FormatAll(p))
}
log.G(ctx).WithFields(log.Fields{"error": err, "platform(s)": platformNames}).Debug("failed to import image to containerd")
// Note: ErrEmptyWalk will not be returned in most cases as
// index.json will contain a descriptor of the actual OCI index or
@@ -284,22 +307,26 @@ func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, platf
// doesn't have a platform set, so it won't be filtered out by the
// FilterPlatform containerd handler.
if errors.Is(err, c8dimages.ErrEmptyWalk) {
return errdefs.NotFound(errors.Wrapf(err, "requested platform (%s) not found", p))
return errdefs.NotFound(errors.Wrapf(err, "requested platform(s) (%v) not found", platformNames))
}
if cerrdefs.IsNotFound(err) {
return errdefs.NotFound(errors.Wrapf(err, "requested platform (%s) found, but some content is missing", p))
return errdefs.NotFound(errors.Wrapf(err, "requested platform(s) (%v) found, but some content is missing", platformNames))
}
}
log.G(ctx).WithError(err).Debug("failed to import image to containerd")
return errdefs.System(err)
}
if platform != nil {
// Verify that the requested platform is available for the loaded images.
if specificPlatforms {
// Verify that the requested platform(s) are available for the loaded images.
// While the ideal behavior here would be to verify whether the input
// archive actually supplied them, we're not able to determine that
// as the imported index is not returned by the import operation.
if err := i.verifyImagesProvidePlatform(ctx, imgs, *platform, pm); err != nil {
platformNames := make([]string, 0, len(platformList))
for _, p := range platformList {
platformNames = append(platformNames, platforms.FormatAll(p))
}
if err := i.verifyImagesProvidePlatform(ctx, imgs, platformNames, pm); err != nil {
return err
}
}
@@ -308,7 +335,7 @@ func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, platf
// Unpack only an image of the host platform
unpackPm := i.hostPlatformMatcher()
// If a load of specific platform is requested, unpack it
if platform != nil {
if specificPlatforms {
unpackPm = pm
}
@@ -378,9 +405,9 @@ func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, platf
// verifyImagesProvidePlatform checks if the requested platform is loaded.
// If the requested platform is not loaded, it returns an error.
func (i *ImageService) verifyImagesProvidePlatform(ctx context.Context, imgs []c8dimages.Image, platform ocispec.Platform, pm platforms.Matcher) error {
func (i *ImageService) verifyImagesProvidePlatform(ctx context.Context, imgs []c8dimages.Image, platformNames []string, pm platforms.Matcher) error {
if len(imgs) == 0 {
return errdefs.NotFound(fmt.Errorf("no images providing the requested platform %s found", platforms.FormatAll(platform)))
return errdefs.NotFound(fmt.Errorf("no images providing the requested platform(s) found: %v", platformNames))
}
var incompleteImgs []string
for _, img := range imgs {
@@ -399,7 +426,7 @@ func (i *ImageService) verifyImagesProvidePlatform(ctx context.Context, imgs []c
}
available, err := platformImg.CheckContentAvailable(ctx)
if err != nil {
return errors.Wrapf(err, "failed to determine image content availability for platform %s", platforms.FormatAll(platform))
return errors.Wrapf(err, "failed to determine image content availability for platform(s) %s", platformNames)
}
if available {
@@ -427,5 +454,5 @@ func (i *ImageService) verifyImagesProvidePlatform(ctx context.Context, imgs []c
msg = "images [%s] were loaded, but don't provide the requested platform (%s)"
}
return errdefs.NotFound(fmt.Errorf(msg, strings.Join(incompleteImgs, ", "), platforms.FormatAll(platform)))
return errdefs.NotFound(fmt.Errorf(msg, strings.Join(incompleteImgs, ", "), platformNames))
}

108
daemon/containerd/image_load_test.go
View File

@@ -3,6 +3,7 @@ package containerd
import (
"bytes"
"context"
"fmt"
"math/rand"
"os"
"path/filepath"
@@ -16,15 +17,18 @@ import (
"github.com/docker/docker/internal/testutils/labelstore"
"github.com/docker/docker/internal/testutils/specialimage"
"github.com/moby/go-archive"
"github.com/moby/moby/api/types/backend"
"github.com/moby/moby/api/types/image"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"gotest.tools/v3/assert"
is "gotest.tools/v3/assert/cmp"
)
func TestImageLoadMissing(t *testing.T) {
func TestImageLoad(t *testing.T) {
linuxAmd64 := ocispec.Platform{OS: "linux", Architecture: "amd64"}
linuxArm64 := ocispec.Platform{OS: "linux", Architecture: "arm64"}
linuxArmv5 := ocispec.Platform{OS: "linux", Architecture: "arm", Variant: "v5"}
linuxRiscv64 := ocispec.Platform{OS: "linux", Architecture: "riskv64"}
ctx := namespaces.WithNamespace(context.TODO(), "testing-"+t.Name())
@@ -35,7 +39,7 @@ func TestImageLoadMissing(t *testing.T) {
// Mock the daemon platform.
imgSvc.defaultPlatformOverride = platforms.Only(linuxAmd64)
tryLoad := func(ctx context.Context, t *testing.T, dir string, platform ocispec.Platform) error {
tryLoad := func(ctx context.Context, t *testing.T, dir string, platformList []ocispec.Platform) error {
tarRc, err := archive.Tar(dir, archive.Uncompressed)
assert.NilError(t, err)
defer tarRc.Close()
@@ -46,10 +50,19 @@ func TestImageLoadMissing(t *testing.T) {
t.Log(buf.String())
}()
return imgSvc.LoadImage(ctx, tarRc, &platform, &buf, true)
return imgSvc.LoadImage(ctx, tarRc, platformList, &buf, true)
}
clearStore := func(ctx context.Context, t *testing.T) {
cleanup := func(ctx context.Context, t *testing.T) {
// Remove all existing images to start fresh
images, err := imgSvc.Images(ctx, image.ListOptions{})
assert.NilError(t, err)
for _, img := range images {
_, err := imgSvc.ImageDelete(ctx, img.ID, image.RemoveOptions{PruneChildren: true})
assert.NilError(t, err)
}
// Remove all content from the store
assert.NilError(t, store.Walk(ctx, func(info content.Info) error {
return store.Delete(ctx, info.Digest)
}), "failed to delete all content")
@@ -60,39 +73,64 @@ func TestImageLoadMissing(t *testing.T) {
_, err := specialimage.EmptyIndex(imgDataDir)
assert.NilError(t, err)
err = tryLoad(ctx, t, imgDataDir, linuxAmd64)
assert.Check(t, is.Error(err, "image emptyindex:latest was loaded, but doesn't provide the requested platform (linux/amd64)"))
err = tryLoad(ctx, t, imgDataDir, []ocispec.Platform{linuxAmd64})
assert.Check(t, is.Error(err, "image emptyindex:latest was loaded, but doesn't provide the requested platform ([linux/amd64])"))
assert.Check(t, is.ErrorType(err, cerrdefs.IsNotFound))
})
clearStore(ctx, t)
cleanup(ctx, t)
t.Run("single platform", func(t *testing.T) {
imgDataDir := t.TempDir()
r := rand.NewSource(0x9127371238)
_, err := specialimage.RandomSinglePlatform(imgDataDir, linuxAmd64, r)
_, err = specialimage.RandomSinglePlatform(imgDataDir, linuxAmd64, r)
assert.NilError(t, err)
err = tryLoad(ctx, t, imgDataDir, linuxArm64)
assert.Check(t, is.ErrorContains(err, "doesn't provide the requested platform (linux/arm64)"))
platforms := []ocispec.Platform{linuxAmd64}
err = tryLoad(ctx, t, imgDataDir, platforms)
assert.NilError(t, err)
err = tryLoad(ctx, t, imgDataDir, []ocispec.Platform{linuxArm64})
assert.Check(t, is.ErrorContains(err, "doesn't provide the requested platform ([linux/arm64])"))
assert.Check(t, is.ErrorType(err, cerrdefs.IsNotFound))
})
cleanup(ctx, t)
clearStore(ctx, t)
t.Run("2 platform image", func(t *testing.T) {
t.Run("multi-platform image", func(t *testing.T) {
imgDataDir := t.TempDir()
_, mfstDescs, err := specialimage.MultiPlatform(imgDataDir, "multiplatform:latest", []ocispec.Platform{linuxAmd64, linuxArm64})
imgRef := "multiplatform:latest"
_, mfstDescs, err := specialimage.MultiPlatform(imgDataDir, imgRef, []ocispec.Platform{linuxAmd64, linuxArm64, linuxRiscv64})
assert.NilError(t, err)
t.Run("one platform in index", func(t *testing.T) {
platforms := []ocispec.Platform{linuxAmd64}
err = tryLoad(ctx, t, imgDataDir, platforms)
assert.NilError(t, err)
// verify that the loaded image has the correct platform
err = verifyImagePlatforms(ctx, imgSvc, imgRef, platforms)
assert.NilError(t, err)
})
cleanup(ctx, t)
t.Run("all platforms in index", func(t *testing.T) {
platforms := []ocispec.Platform{linuxAmd64, linuxArm64, linuxRiscv64}
err = tryLoad(ctx, t, imgDataDir, platforms)
assert.NilError(t, err)
// verify that the loaded image has the correct platforms
err = verifyImagePlatforms(ctx, imgSvc, imgRef, platforms)
assert.NilError(t, err)
})
cleanup(ctx, t)
t.Run("platform not included in index", func(t *testing.T) {
err = tryLoad(ctx, t, imgDataDir, linuxArmv5)
assert.Check(t, is.Error(err, "image multiplatform:latest was loaded, but doesn't provide the requested platform (linux/arm/v5)"))
err = tryLoad(ctx, t, imgDataDir, []ocispec.Platform{linuxArmv5})
assert.Check(t, is.Error(err, "image multiplatform:latest was loaded, but doesn't provide the requested platform ([linux/arm/v5])"))
assert.Check(t, is.ErrorType(err, cerrdefs.IsNotFound))
})
cleanup(ctx, t)
clearStore(ctx, t)
t.Run("platform blobs missing", func(t *testing.T) {
t.Run("platform included but blobs missing", func(t *testing.T) {
// Assumption: arm64 image is second in the index (implementation detail of specialimage.MultiPlatform)
mfstDesc := mfstDescs[1]
assert.Assert(t, mfstDesc.Platform.Architecture == linuxArm64.Architecture)
@@ -104,9 +142,37 @@ func TestImageLoadMissing(t *testing.T) {
mfstPath := filepath.Join(imgDataDir, "blobs/sha256", mfstDesc.Digest.Encoded())
assert.NilError(t, os.Remove(mfstPath))
err = tryLoad(ctx, t, imgDataDir, linuxArm64)
assert.Check(t, is.ErrorContains(err, "requested platform (linux/arm64) found, but some content is missing"))
err = tryLoad(ctx, t, imgDataDir, []ocispec.Platform{linuxArm64})
assert.Check(t, is.ErrorContains(err, "requested platform(s) ([linux/arm64]) found, but some content is missing"))
assert.Check(t, is.ErrorType(err, cerrdefs.IsNotFound))
})
cleanup(ctx, t)
})
}
func verifyImagePlatforms(ctx context.Context, imgSvc *ImageService, imgRef string, expectedPlatforms []ocispec.Platform) error {
// get the manifest(s) for the image
img, err := imgSvc.ImageInspect(ctx, imgRef, backend.ImageInspectOpts{Manifests: true})
if err != nil {
return err
}
// verify that the image manifest has the expected platforms
for _, ep := range expectedPlatforms {
want := platforms.FormatAll(ep)
found := false
for _, m := range img.Manifests {
if m.Descriptor.Platform != nil {
got := platforms.FormatAll(*m.Descriptor.Platform)
if got == want {
found = true
break
}
}
}
if !found {
return fmt.Errorf("expected platform %q not found in loaded images", want)
}
}
return nil
}

47
daemon/containerd/image_save_test.go
View File

@@ -31,12 +31,17 @@ func TestImageMultiplatformSaveShallowWithNative(t *testing.T) {
Architecture: "arm64",
}
riscv64 := platforms.Platform{
OS: "linux",
Architecture: "riscv64",
}
imgSvc := fakeImageService(t, ctx, store)
// Mock the native platform.
imgSvc.defaultPlatformOverride = platforms.Only(native)
idx, _, err := specialimage.PartialMultiPlatform(contentDir, "partial-with-native:latest", specialimage.PartialOpts{
Stored: []ocispec.Platform{native},
Stored: []ocispec.Platform{native, riscv64},
Missing: []ocispec.Platform{arm64},
})
assert.NilError(t, err)
@@ -49,13 +54,21 @@ func TestImageMultiplatformSaveShallowWithNative(t *testing.T) {
assert.NilError(t, err)
})
t.Run("export native", func(t *testing.T) {
err = imgSvc.ExportImage(ctx, []string{img.Name}, &native, io.Discard)
err = imgSvc.ExportImage(ctx, []string{img.Name}, []ocispec.Platform{native}, io.Discard)
assert.NilError(t, err)
})
t.Run("export multiple platforms", func(t *testing.T) {
err = imgSvc.ExportImage(ctx, []string{img.Name}, []ocispec.Platform{native, riscv64}, io.Discard)
assert.NilError(t, err)
})
t.Run("export missing", func(t *testing.T) {
err = imgSvc.ExportImage(ctx, []string{img.Name}, &arm64, io.Discard)
err = imgSvc.ExportImage(ctx, []string{img.Name}, []ocispec.Platform{arm64}, io.Discard)
assert.Check(t, is.ErrorType(err, cerrdefs.IsNotFound))
})
t.Run("export multiple platforms with some missing", func(t *testing.T) {
err = imgSvc.ExportImage(ctx, []string{img.Name}, []ocispec.Platform{arm64, riscv64}, io.Discard)
assert.NilError(t, err)
})
}
func TestImageMultiplatformSaveShallowWithoutNative(t *testing.T) {
@@ -74,12 +87,22 @@ func TestImageMultiplatformSaveShallowWithoutNative(t *testing.T) {
Architecture: "arm64",
}
riscv64 := platforms.Platform{
OS: "linux",
Architecture: "riscv64",
}
s390x := platforms.Platform{
OS: "linux",
Architecture: "s390x",
}
imgSvc := fakeImageService(t, ctx, store)
// Mock the native platform.
imgSvc.defaultPlatformOverride = platforms.Only(native)
idx, _, err := specialimage.PartialMultiPlatform(contentDir, "partial-without-native:latest", specialimage.PartialOpts{
Stored: []ocispec.Platform{arm64},
Stored: []ocispec.Platform{arm64, riscv64},
Missing: []ocispec.Platform{native},
})
assert.NilError(t, err)
@@ -93,11 +116,23 @@ func TestImageMultiplatformSaveShallowWithoutNative(t *testing.T) {
assert.NilError(t, err)
})
t.Run("export native", func(t *testing.T) {
err = imgSvc.ExportImage(ctx, []string{img.Name}, &native, io.Discard)
err = imgSvc.ExportImage(ctx, []string{img.Name}, []ocispec.Platform{native}, io.Discard)
assert.Check(t, is.ErrorType(err, cerrdefs.IsNotFound))
})
t.Run("export arm64", func(t *testing.T) {
err = imgSvc.ExportImage(ctx, []string{img.Name}, &arm64, io.Discard)
err = imgSvc.ExportImage(ctx, []string{img.Name}, []ocispec.Platform{arm64}, io.Discard)
assert.NilError(t, err)
})
t.Run("export multiple platforms", func(t *testing.T) {
err = imgSvc.ExportImage(ctx, []string{img.Name}, []ocispec.Platform{arm64, riscv64}, io.Discard)
assert.NilError(t, err)
})
t.Run("export multiple platforms with some missing", func(t *testing.T) {
err = imgSvc.ExportImage(ctx, []string{img.Name}, []ocispec.Platform{arm64, native}, io.Discard)
assert.Check(t, is.ErrorType(err, cerrdefs.IsNotFound))
})
t.Run("export non existing platform", func(t *testing.T) {
err = imgSvc.ExportImage(ctx, []string{img.Name}, []ocispec.Platform{s390x}, io.Discard)
assert.Check(t, is.ErrorType(err, cerrdefs.IsNotFound))
})
}

31
daemon/containerd/platform_matchers.go
View File

@@ -5,13 +5,14 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
// allPlatformsWithPreferenceMatcher returns a platform matcher that matches all
// platforms but orders platforms to match the preferred matcher first.
// It implements the platforms.MatchComparer interface.
type allPlatformsWithPreferenceMatcher struct {
preferred platforms.MatchComparer
}
// matchAllWithPreference will return a platform matcher that matches all
// platforms but will order platforms matching the preferred matcher first.
func matchAllWithPreference(preferred platforms.MatchComparer) platforms.MatchComparer {
func matchAllWithPreference(preferred platforms.MatchComparer) allPlatformsWithPreferenceMatcher {
return allPlatformsWithPreferenceMatcher{
preferred: preferred,
}
@@ -25,6 +26,29 @@ func (c allPlatformsWithPreferenceMatcher) Less(p1, p2 ocispec.Platform) bool {
return c.preferred.Less(p1, p2)
}
// platformsWithPreferenceMatcher is a platform matcher that matches any of the
// given platforms, but orders platforms to match the preferred matcher first.
// It implements the platforms.MatchComparer interface.
type platformsWithPreferenceMatcher struct {
platformList []ocispec.Platform
preferred platforms.MatchComparer
}
func matchAnyWithPreference(preferred platforms.MatchComparer, platformList []ocispec.Platform) platformsWithPreferenceMatcher {
return platformsWithPreferenceMatcher{
platformList: platformList,
preferred: preferred,
}
}
func (c platformsWithPreferenceMatcher) Match(p ocispec.Platform) bool {
return platforms.Any(c.platformList...).Match(p)
}
func (c platformsWithPreferenceMatcher) Less(p1, p2 ocispec.Platform) bool {
return c.preferred.Less(p1, p2)
}
// platformMatcherWithRequestedPlatform is a platform matcher that also
// contains the platform that was requested by the user in the context
// in which the matcher was created.
@@ -36,6 +60,7 @@ type platformMatcherWithRequestedPlatform struct {
type matchComparerProvider func(ocispec.Platform) platforms.MatchComparer
// TODO(ctalledo): move this to a more appropriate place (e.g., next to the other ImageService methods).
func (i *ImageService) matchRequestedOrDefault(
fpm matchComparerProvider, // function to create a platform matcher if platform is not nil
platform *ocispec.Platform, // input platform, nil if not specified

32
daemon/containerd/platform_matchers_test.go
View File

@@ -1,6 +1,7 @@
package containerd
import (
"reflect"
"runtime"
"testing"
@@ -167,3 +168,34 @@ func testOnlyAndOnlyStrict(t *testing.T, daemonPlatform platforms.MatchComparer,
}
})
}
func TestPlatformsWithPreferenceMatcher(t *testing.T) {
platformList := []ocispec.Platform{
pLinuxAmd64,
pLinuxArmv5,
pLinuxArmv6,
pLinuxArm64,
pWindowsAmd64,
}
// Use pLinuxArm64 as the preferred platform
preferred := platforms.Only(pLinuxArm64)
matcher := matchAnyWithPreference(preferred, platformList)
// Should match all platforms in the list
for _, p := range platformList {
assert.Assert(t, matcher.Match(p), "matcher should match platform: %v", platforms.Format(p))
}
// Should not match a platform not in the list
notInList := ocispec.Platform{OS: "linux", Architecture: "s390x"}
assert.Assert(t, !matcher.Match(notInList), "matcher should not match platform: %v", platforms.Format(notInList))
// Test Less: preferred should be less than others
for _, p := range platformList {
if reflect.DeepEqual(p, pLinuxArm64) {
continue
}
assert.Assert(t, matcher.Less(pLinuxArm64, p), "preferred platform should be less than %v", platforms.Format(p))
assert.Assert(t, !matcher.Less(p, pLinuxArm64), "%v should not be less than preferred platform", platforms.Format(p))
}
}

4
daemon/image_service.go
View File

@@ -30,8 +30,8 @@ type ImageService interface {
PushImage(ctx context.Context, ref reference.Named, platform *ocispec.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error
CreateImage(ctx context.Context, config []byte, parent string, contentStoreDigest digest.Digest) (builder.Image, error)
ImageDelete(ctx context.Context, imageRef string, options imagetype.RemoveOptions) ([]imagetype.DeleteResponse, error)
ExportImage(ctx context.Context, names []string, platform *ocispec.Platform, outStream io.Writer) error
LoadImage(ctx context.Context, inTar io.ReadCloser, platform *ocispec.Platform, outStream io.Writer, quiet bool) error
ExportImage(ctx context.Context, names []string, platformList []ocispec.Platform, outStream io.Writer) error
LoadImage(ctx context.Context, inTar io.ReadCloser, platformList []ocispec.Platform, outStream io.Writer, quiet bool) error
Images(ctx context.Context, opts imagetype.ListOptions) ([]*imagetype.Summary, error)
LogImageEvent(ctx context.Context, imageID, refName string, action events.Action)
CountImages(ctx context.Context) int

22
daemon/images/image_exporter.go
View File

@@ -4,8 +4,10 @@ import (
"context"
"io"
"github.com/docker/docker/errdefs"
"github.com/docker/docker/image/tarexport"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
// ExportImage exports a list of images to the given output stream. The
@@ -13,7 +15,15 @@ import (
// stream. All images with the given tag and all versions containing
// the same tag are exported. names is the set of tags to export, and
// outStream is the writer which the images are written to.
func (i *ImageService) ExportImage(ctx context.Context, names []string, platform *ocispec.Platform, outStream io.Writer) error {
func (i *ImageService) ExportImage(ctx context.Context, names []string, platformList []ocispec.Platform, outStream io.Writer) error {
var platform *ocispec.Platform
if len(platformList) > 1 {
return errdefs.InvalidParameter(errors.New("multiple platforms not supported for this image store; use a multi-platform image store such as containerd-snapshotter"))
} else if len(platformList) == 1 {
platform = &platformList[0]
}
imageExporter := tarexport.NewTarExporter(i.imageStore, i.layerStore, i.referenceStore, i, platform)
return imageExporter.Save(ctx, names, outStream)
}
@@ -21,7 +31,15 @@ func (i *ImageService) ExportImage(ctx context.Context, names []string, platform
// LoadImage uploads a set of images into the repository. This is the
// complement of ExportImage. The input stream is an uncompressed tar
// ball containing images and metadata.
func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, platform *ocispec.Platform, outStream io.Writer, quiet bool) error {
func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, platformList []ocispec.Platform, outStream io.Writer, quiet bool) error {
var platform *ocispec.Platform
if len(platformList) > 1 {
return errdefs.InvalidParameter(errors.New("multiple platforms not supported for this image store; use a multi-platform image store such as containerd-snapshotter"))
} else if len(platformList) == 1 {
platform = &platformList[0]
}
imageExporter := tarexport.NewTarExporter(i.imageStore, i.layerStore, i.referenceStore, i, platform)
return imageExporter.Load(ctx, inTar, outStream, quiet)
}

4
daemon/server/router/image/backend.go
View File

@@ -32,9 +32,9 @@ type imageBackend interface {
}
type importExportBackend interface {
LoadImage(ctx context.Context, inTar io.ReadCloser, platform *ocispec.Platform, outStream io.Writer, quiet bool) error
LoadImage(ctx context.Context, inTar io.ReadCloser, platformList []ocispec.Platform, outStream io.Writer, quiet bool) error
ImportImage(ctx context.Context, ref reference.Named, platform *ocispec.Platform, msg string, layerReader io.Reader, changes []string) (dockerimage.ID, error)
ExportImage(ctx context.Context, names []string, platform *ocispec.Platform, outStream io.Writer) error
ExportImage(ctx context.Context, names []string, platformList []ocispec.Platform, outStream io.Writer) error
}
type registryBackend interface {

47
daemon/server/router/image/image_routes.go
View File

@@ -235,6 +235,7 @@ func (ir *imageRouter) getImagesGet(ctx context.Context, w http.ResponseWriter,
output := ioutils.NewWriteFlusher(w)
defer output.Close()
var names []string
if name, ok := vars["name"]; ok {
names = []string{name}
@@ -242,27 +243,28 @@ func (ir *imageRouter) getImagesGet(ctx context.Context, w http.ResponseWriter,
names = r.Form["names"]
}
var platform *ocispec.Platform
var platformList []ocispec.Platform
// platform param was introduce in API version 1.48
if versions.GreaterThanOrEqualTo(httputils.VersionFromContext(ctx), "1.48") {
if formPlatforms := r.Form["platform"]; len(formPlatforms) > 1 {
// TODO(thaJeztah): remove once we support multiple platforms: see https://github.com/moby/moby/issues/48759
return errdefs.InvalidParameter(errors.New("multiple platform parameters not supported"))
var err error
formPlatforms := r.Form["platform"]
// multi-platform params were introduced in API version 1.51
if versions.LessThan(httputils.VersionFromContext(ctx), "1.51") && len(formPlatforms) > 1 {
return errdefs.InvalidParameter(errors.New("multiple platform parameters are not supported in this API version; use API version 1.51 or later."))
}
if formPlatform := r.Form.Get("platform"); formPlatform != "" {
p, err := httputils.DecodePlatform(formPlatform)
if err != nil {
return err
}
platform = p
platformList, err = httputils.DecodePlatforms(formPlatforms)
if err != nil {
return err
}
}
if err := ir.backend.ExportImage(ctx, names, platform, output); err != nil {
if err := ir.backend.ExportImage(ctx, names, platformList, output); err != nil {
if !output.Flushed() {
return err
}
_, _ = output.Write(streamformatter.FormatError(err))
}
return nil
}
@@ -271,18 +273,18 @@ func (ir *imageRouter) postImagesLoad(ctx context.Context, w http.ResponseWriter
return err
}
var platform *ocispec.Platform
var platformList []ocispec.Platform
// platform param was introduce in API version 1.48
if versions.GreaterThanOrEqualTo(httputils.VersionFromContext(ctx), "1.48") {
if formPlatforms := r.Form["platform"]; len(formPlatforms) > 1 {
// TODO(thaJeztah): remove once we support multiple platforms: see https://github.com/moby/moby/issues/48759
return errdefs.InvalidParameter(errors.New("multiple platform parameters not supported"))
var err error
formPlatforms := r.Form["platform"]
// multi-platform params were introduced in API version 1.51
if versions.LessThan(httputils.VersionFromContext(ctx), "1.51") && len(formPlatforms) > 1 {
return errdefs.InvalidParameter(errors.New("multiple platform parameters are not supported in this API version; use API version 1.51 or later."))
}
if formPlatform := r.Form.Get("platform"); formPlatform != "" {
p, err := httputils.DecodePlatform(formPlatform)
if err != nil {
return err
}
platform = p
platformList, err = httputils.DecodePlatforms(formPlatforms)
if err != nil {
return err
}
}
quiet := httputils.BoolValueOrDefault(r, "quiet", true)
@@ -291,7 +293,8 @@ func (ir *imageRouter) postImagesLoad(ctx context.Context, w http.ResponseWriter
output := ioutils.NewWriteFlusher(w)
defer output.Close()
if err := ir.backend.LoadImage(ctx, r.Body, platform, output, quiet); err != nil {
if err := ir.backend.LoadImage(ctx, r.Body, platformList, output, quiet); err != nil {
_, _ = output.Write(streamformatter.FormatError(err))
}
return nil

172
integration/image/save_test.go
View File

@@ -13,7 +13,6 @@ import (
"testing"
"time"
cerrdefs "github.com/containerd/errdefs"
"github.com/cpuguy83/tar2go"
"github.com/docker/docker/integration/internal/build"
"github.com/docker/docker/integration/internal/container"
@@ -23,6 +22,7 @@ import (
"github.com/docker/docker/testutil/fakecontext"
"github.com/moby/go-archive/compression"
containertypes "github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/image"
"github.com/moby/moby/api/types/versions"
"github.com/moby/moby/client"
"github.com/opencontainers/go-digest"
@@ -213,23 +213,167 @@ func TestSaveOCI(t *testing.T) {
}
}
// TODO(thaJeztah): this test currently only checks invalid cases; update this test to use a table-test and test both valid and invalid platform options.
func TestSavePlatform(t *testing.T) {
ctx := setupTest(t)
func TestSaveAndLoadPlatform(t *testing.T) {
skip.If(t, testEnv.DaemonInfo.OSType == "windows", "The test image is a Linux image")
t.Parallel()
ctx := setupTest(t)
apiClient := testEnv.APIClient()
const repoName = "busybox:latest"
_, err := apiClient.ImageInspect(ctx, repoName)
assert.NilError(t, err)
const repoName = "alpine:latest"
_, err = apiClient.ImageSave(ctx, []string{repoName}, client.ImageSaveWithPlatforms(
ocispec.Platform{Architecture: "amd64", OS: "linux"},
ocispec.Platform{Architecture: "arm64", OS: "linux", Variant: "v8"},
))
assert.Check(t, is.ErrorType(err, cerrdefs.IsInvalidArgument))
assert.Check(t, is.Error(err, "Error response from daemon: multiple platform parameters not supported"))
type testCase struct {
testName string
containerdStoreOnly bool
pullPlatforms []string
savePlatforms []ocispec.Platform
loadPlatforms []ocispec.Platform
expectedSavedPlatforms []ocispec.Platform
expectedLoadedPlatforms []ocispec.Platform // expected platforms to be saved, if empty, all pulled platforms are expected to be saved
}
testCases := []testCase{
{
testName: "With no platforms specified",
containerdStoreOnly: true,
pullPlatforms: []string{"linux/amd64", "linux/riscv64", "linux/arm64/v8"},
savePlatforms: nil,
loadPlatforms: nil,
expectedSavedPlatforms: []ocispec.Platform{
{OS: "linux", Architecture: "amd64"},
{OS: "linux", Architecture: "riscv64"},
{OS: "linux", Architecture: "arm64", Variant: "v8"},
},
expectedLoadedPlatforms: []ocispec.Platform{
{OS: "linux", Architecture: "amd64"},
{OS: "linux", Architecture: "riscv64"},
{OS: "linux", Architecture: "arm64", Variant: "v8"},
},
},
{
testName: "With single pulled platform",
pullPlatforms: []string{"linux/amd64"},
savePlatforms: []ocispec.Platform{{OS: "linux", Architecture: "amd64"}},
loadPlatforms: []ocispec.Platform{{OS: "linux", Architecture: "amd64"}},
expectedSavedPlatforms: []ocispec.Platform{{OS: "linux", Architecture: "amd64"}},
expectedLoadedPlatforms: []ocispec.Platform{{OS: "linux", Architecture: "amd64"}},
},
{
testName: "With single platform save and load",
containerdStoreOnly: true,
pullPlatforms: []string{"linux/amd64", "linux/riscv64", "linux/arm64/v8"},
savePlatforms: []ocispec.Platform{{OS: "linux", Architecture: "amd64"}},
loadPlatforms: []ocispec.Platform{{OS: "linux", Architecture: "amd64"}},
expectedSavedPlatforms: []ocispec.Platform{{OS: "linux", Architecture: "amd64"}},
expectedLoadedPlatforms: []ocispec.Platform{{OS: "linux", Architecture: "amd64"}},
},
{
testName: "With multiple platforms save and load",
containerdStoreOnly: true,
pullPlatforms: []string{"linux/amd64", "linux/riscv64", "linux/arm64/v8"},
savePlatforms: []ocispec.Platform{
{OS: "linux", Architecture: "arm64", Variant: "v8"},
{OS: "linux", Architecture: "riscv64"},
},
loadPlatforms: []ocispec.Platform{
{OS: "linux", Architecture: "arm64", Variant: "v8"},
{OS: "linux", Architecture: "riscv64"},
},
expectedSavedPlatforms: []ocispec.Platform{
{OS: "linux", Architecture: "arm64", Variant: "v8"},
{OS: "linux", Architecture: "riscv64"},
},
expectedLoadedPlatforms: []ocispec.Platform{
{OS: "linux", Architecture: "arm64", Variant: "v8"},
{OS: "linux", Architecture: "riscv64"},
},
},
{
testName: "With mixed platform save and load",
containerdStoreOnly: true,
pullPlatforms: []string{"linux/amd64", "linux/riscv64", "linux/arm64/v8"},
savePlatforms: []ocispec.Platform{
{OS: "linux", Architecture: "arm64", Variant: "v8"},
{OS: "linux", Architecture: "riscv64"},
},
loadPlatforms: []ocispec.Platform{
{OS: "linux", Architecture: "riscv64"},
},
expectedSavedPlatforms: []ocispec.Platform{
{OS: "linux", Architecture: "arm64", Variant: "v8"},
{OS: "linux", Architecture: "riscv64"},
},
expectedLoadedPlatforms: []ocispec.Platform{
{OS: "linux", Architecture: "riscv64"},
},
},
}
for _, tc := range testCases {
if tc.containerdStoreOnly && !testEnv.UsingSnapshotter() {
continue
}
t.Run(tc.testName, func(t *testing.T) {
// pull the image
for _, p := range tc.pullPlatforms {
resp, err := apiClient.ImagePull(ctx, repoName, image.PullOptions{Platform: p})
assert.NilError(t, err)
_, err = io.ReadAll(resp)
assert.NilError(t, err)
resp.Close()
}
// export the image
rdr, err := apiClient.ImageSave(ctx, []string{repoName}, client.ImageSaveWithPlatforms(tc.savePlatforms...))
assert.NilError(t, err)
// remove the pulled image
_, err = apiClient.ImageRemove(ctx, repoName, image.RemoveOptions{})
assert.NilError(t, err)
// load the full exported image (all platforms in it)
_, err = apiClient.ImageLoad(ctx, rdr)
assert.NilError(t, err)
rdr.Close()
// verify the loaded image has all the expected saved platforms
for _, p := range tc.expectedSavedPlatforms {
inspectResponse, err := apiClient.ImageInspect(ctx, repoName, client.ImageInspectWithPlatform(&p))
assert.NilError(t, err)
assert.Check(t, is.Equal(inspectResponse.Os, p.OS))
assert.Check(t, is.Equal(inspectResponse.Architecture, p.Architecture))
}
// pull the image again (start fresh)
for _, p := range tc.pullPlatforms {
resp, err := apiClient.ImagePull(ctx, repoName, image.PullOptions{Platform: p})
assert.NilError(t, err)
_, err = io.ReadAll(resp)
assert.NilError(t, err)
resp.Close()
}
// export the image
rdr, err = apiClient.ImageSave(ctx, []string{repoName}, client.ImageSaveWithPlatforms(tc.savePlatforms...))
assert.NilError(t, err)
// remove the pulled image
_, err = apiClient.ImageRemove(ctx, repoName, image.RemoveOptions{})
assert.NilError(t, err)
// load the exported image on the specified platforms only
_, err = apiClient.ImageLoad(ctx, rdr, client.ImageLoadWithPlatforms(tc.loadPlatforms...))
assert.NilError(t, err)
rdr.Close()
// verify the image was loaded for the specified platforms
for _, p := range tc.expectedLoadedPlatforms {
inspectResponse, err := apiClient.ImageInspect(ctx, repoName, client.ImageInspectWithPlatform(&p))
assert.NilError(t, err)
assert.Check(t, is.Equal(inspectResponse.Os, p.OS))
assert.Check(t, is.Equal(inspectResponse.Architecture, p.Architecture))
}
})
}
}
func TestSaveRepoWithMultipleImages(t *testing.T) {

31
vendor/github.com/moby/moby/api/swagger.yaml generated vendored
View File

@@ -10443,7 +10443,10 @@ paths:
type: "string"
required: true
- name: "platform"
type: "string"
type: "array"
items:
type: "string"
collectionFormat: "multi"
in: "query"
description: |
JSON encoded OCI platform describing a platform which will be used
@@ -10488,13 +10491,16 @@ paths:
items:
type: "string"
- name: "platform"
type: "string"
type: "array"
items:
type: "string"
collectionFormat: "multi"
in: "query"
description: |
JSON encoded OCI platform describing a platform which will be used
to select a platform-specific image to be saved if the image is
multi-platform.
If not provided, the full multi-platform image will be saved.
JSON encoded OCI platform(s) which will be used to select the
platform-specific image(s) to be saved if the image is
multi-platform. If not provided, the full multi-platform image
will be saved.
Example: `{"os": "linux", "architecture": "arm", "variant": "v5"}`
tags: ["Image"]
@@ -10530,13 +10536,16 @@ paths:
type: "boolean"
default: false
- name: "platform"
type: "string"
type: "array"
items:
type: "string"
collectionFormat: "multi"
in: "query"
description: |
JSON encoded OCI platform describing a platform which will be used
to select a platform-specific image to be load if the image is
multi-platform.
If not provided, the full multi-platform image will be loaded.
JSON encoded OCI platform(s) which will be used to select the
platform-specific image(s) to load if the image is
multi-platform. If not provided, the full multi-platform image
will be loaded.
Example: `{"os": "linux", "architecture": "arm", "variant": "v5"}`
tags: ["Image"]