Download support of images with multi-arch manifest

Currently we only support 'application/vnd.docker.distribution.manifest.v2+json'
manifest images download, with more multi-arch images used, we need to support
download images with 'application/vnd.docker.distribution.manifest.list.v2+json'
format(aka "fat manifest"), else we will fail to download those multi-arch ones.

This PR adds 'application/vnd.docker.distribution.manifest.list.v2+json' manifest
support, thus we can download both multi-arch and legacy images.

Signed-off-by: Dennis Chen <dennis.chen@arm.com>
(cherry picked from commit 0af5db511e)
Signed-off-by: Andrew Hsu <andrewhsu@docker.com>

api/common.go

@@ -3,7 +3,7 @@ package api
 // Common constants for daemon and client.
 const (
 	// DefaultVersion of Current REST API
-	DefaultVersion string = "1.36"
+	DefaultVersion string = "1.35"
 
 	// NoBaseImageSpecifier is the symbol used by the FROM
 	// command to specify that no base image is to be used.

api/swagger.yaml

@@ -19,10 +19,10 @@ produces:
 consumes:
   - "application/json"
   - "text/plain"
-basePath: "/v1.36"
+basePath: "/v1.35"
 info:
   title: "Docker Engine API"
-  version: "1.36"
+  version: "1.35"
   x-logo:
     url: "https://docs.docker.com/images/logo-docker-main.png"
   description: |
@@ -49,8 +49,8 @@ info:
     the URL is not supported by the daemon, a HTTP `400 Bad Request` error message
     is returned.
 
-    If you omit the version-prefix, the current version of the API (v1.36) is used.
-    For example, calling `/info` is the same as calling `/v1.36/info`. Using the
+    If you omit the version-prefix, the current version of the API (v1.35) is used.
+    For example, calling `/info` is the same as calling `/v1.35/info`. Using the
     API without a version-prefix is deprecated and will be removed in a future release.
 
     Engine releases in the near future should support this version of the API,

contrib/download-frozen-image-v2.sh

@@ -6,7 +6,6 @@ set -eo pipefail
 
 # debian                           latest              f6fab3b798be        10 weeks ago        85.1 MB
 # debian                           latest              f6fab3b798be3174f45aa1eb731f8182705555f89c9026d8c1ef230cbf8301dd   10 weeks ago        85.1 MB
-
 if ! command -v curl &> /dev/null; then
 	echo >&2 'error: "curl" not found!'
 	exit 1
@@ -80,6 +79,109 @@ fetch_blob() {
 	fi
 }
 
+# handle 'application/vnd.docker.distribution.manifest.v2+json' manifest
+handle_single_manifest_v2() {
+	local manifestJson="$1"; shift
+
+	local configDigest="$(echo "$manifestJson" | jq --raw-output '.config.digest')"
+	local imageId="${configDigest#*:}" # strip off "sha256:"
+
+	local configFile="$imageId.json"
+	fetch_blob "$token" "$image" "$configDigest" "$dir/$configFile" -s
+
+	local layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.layers[]')"
+	local IFS="$newlineIFS"
+	local layers=( $layersFs )
+	unset IFS
+
+	echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..."
+	local layerId=
+	local layerFiles=()
+	for i in "${!layers[@]}"; do
+		local layerMeta="${layers[$i]}"
+
+		local layerMediaType="$(echo "$layerMeta" | jq --raw-output '.mediaType')"
+		local layerDigest="$(echo "$layerMeta" | jq --raw-output '.digest')"
+
+		# save the previous layer's ID
+		local parentId="$layerId"
+		# create a new fake layer ID based on this layer's digest and the previous layer's fake ID
+		layerId="$(echo "$parentId"$'\n'"$layerDigest" | sha256sum | cut -d' ' -f1)"
+		# this accounts for the possibility that an image contains the same layer twice (and thus has a duplicate digest value)
+
+		mkdir -p "$dir/$layerId"
+		echo '1.0' > "$dir/$layerId/VERSION"
+
+		if [ ! -s "$dir/$layerId/json" ]; then
+			local parentJson="$(printf ', parent: "%s"' "$parentId")"
+			local addJson="$(printf '{ id: "%s"%s }' "$layerId" "${parentId:+$parentJson}")"
+			# this starter JSON is taken directly from Docker's own "docker save" output for unimportant layers
+			jq "$addJson + ." > "$dir/$layerId/json" <<-'EOJSON'
+				{
+					"created": "0001-01-01T00:00:00Z",
+					"container_config": {
+						"Hostname": "",
+						"Domainname": "",
+						"User": "",
+						"AttachStdin": false,
+						"AttachStdout": false,
+						"AttachStderr": false,
+						"Tty": false,
+						"OpenStdin": false,
+						"StdinOnce": false,
+						"Env": null,
+						"Cmd": null,
+						"Image": "",
+						"Volumes": null,
+						"WorkingDir": "",
+						"Entrypoint": null,
+						"OnBuild": null,
+						"Labels": null
+					}
+				}
+			EOJSON
+		fi
+
+		case "$layerMediaType" in
+			application/vnd.docker.image.rootfs.diff.tar.gzip)
+				local layerTar="$layerId/layer.tar"
+				layerFiles=( "${layerFiles[@]}" "$layerTar" )
+				# TODO figure out why "-C -" doesn't work here
+				# "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume."
+				# "HTTP/1.1 416 Requested Range Not Satisfiable"
+				if [ -f "$dir/$layerTar" ]; then
+					# TODO hackpatch for no -C support :'(
+					echo "skipping existing ${layerId:0:12}"
+					continue
+				fi
+				local token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"
+				fetch_blob "$token" "$image" "$layerDigest" "$dir/$layerTar" --progress
+				;;
+
+			*)
+				echo >&2 "error: unknown layer mediaType ($imageIdentifier, $layerDigest): '$layerMediaType'"
+				exit 1
+				;;
+		esac
+	done
+
+	# change "$imageId" to be the ID of the last layer we added (needed for old-style "repositories" file which is created later -- specifically for older Docker daemons)
+	imageId="$layerId"
+
+	# munge the top layer image manifest to have the appropriate image configuration for older daemons
+	local imageOldConfig="$(jq --raw-output --compact-output '{ id: .id } + if .parent then { parent: .parent } else {} end' "$dir/$imageId/json")"
+	jq --raw-output "$imageOldConfig + del(.history, .rootfs)" "$dir/$configFile" > "$dir/$imageId/json"
+
+	local manifestJsonEntry="$(
+		echo '{}' | jq --raw-output '. + {
+			Config: "'"$configFile"'",
+			RepoTags: ["'"${image#library\/}:$tag"'"],
+			Layers: '"$(echo '[]' | jq --raw-output ".$(for layerFile in "${layerFiles[@]}"; do echo " + [ \"$layerFile\" ]"; done)")"'
+		}'
+	)"
+	manifestJsonEntries=( "${manifestJsonEntries[@]}" "$manifestJsonEntry" )
+}
+
 while [ $# -gt 0 ]; do
 	imageTag="$1"
 	shift
@@ -101,6 +203,7 @@ while [ $# -gt 0 ]; do
 		curl -fsSL \
 			-H "Authorization: Bearer $token" \
 			-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
+			-H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
 			-H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \
 			"$registryBase/v2/$image/manifests/$digest"
 	)"
@@ -119,105 +222,40 @@ while [ $# -gt 0 ]; do
 
 			case "$mediaType" in
 				application/vnd.docker.distribution.manifest.v2+json)
-					configDigest="$(echo "$manifestJson" | jq --raw-output '.config.digest')"
-					imageId="${configDigest#*:}" # strip off "sha256:"
-
-					configFile="$imageId.json"
-					fetch_blob "$token" "$image" "$configDigest" "$dir/$configFile" -s
-
-					layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.layers[]')"
+					handle_single_manifest_v2 "$manifestJson"
+					;;
+				application/vnd.docker.distribution.manifest.list.v2+json)
+					layersFs="$(echo "$manifestJson" | jq --raw-output --compact-output '.manifests[]')"
 					IFS="$newlineIFS"
 					layers=( $layersFs )
 					unset IFS
 
-					echo "Downloading '$imageIdentifier' (${#layers[@]} layers)..."
-					layerId=
-					layerFiles=()
+					found=""
+					# parse first level multi-arch manifest
 					for i in "${!layers[@]}"; do
 						layerMeta="${layers[$i]}"
-
-						layerMediaType="$(echo "$layerMeta" | jq --raw-output '.mediaType')"
-						layerDigest="$(echo "$layerMeta" | jq --raw-output '.digest')"
-
-						# save the previous layer's ID
-						parentId="$layerId"
-						# create a new fake layer ID based on this layer's digest and the previous layer's fake ID
-						layerId="$(echo "$parentId"$'\n'"$layerDigest" | sha256sum | cut -d' ' -f1)"
-						# this accounts for the possibility that an image contains the same layer twice (and thus has a duplicate digest value)
-
-						mkdir -p "$dir/$layerId"
-						echo '1.0' > "$dir/$layerId/VERSION"
-
-						if [ ! -s "$dir/$layerId/json" ]; then
-							parentJson="$(printf ', parent: "%s"' "$parentId")"
-							addJson="$(printf '{ id: "%s"%s }' "$layerId" "${parentId:+$parentJson}")"
-							# this starter JSON is taken directly from Docker's own "docker save" output for unimportant layers
-							jq "$addJson + ." > "$dir/$layerId/json" <<-'EOJSON'
-								{
-									"created": "0001-01-01T00:00:00Z",
-									"container_config": {
-										"Hostname": "",
-										"Domainname": "",
-										"User": "",
-										"AttachStdin": false,
-										"AttachStdout": false,
-										"AttachStderr": false,
-										"Tty": false,
-										"OpenStdin": false,
-										"StdinOnce": false,
-										"Env": null,
-										"Cmd": null,
-										"Image": "",
-										"Volumes": null,
-										"WorkingDir": "",
-										"Entrypoint": null,
-										"OnBuild": null,
-										"Labels": null
-									}
-								}
-							EOJSON
+						maniArch="$(echo "$layerMeta" | jq --raw-output '.platform.architecture')"
+						if [ "$maniArch" = "$(go env GOARCH)" ]; then
+							digest="$(echo "$layerMeta" | jq --raw-output '.digest')"
+							# get second level single manifest
+							submanifestJson="$(
+								curl -fsSL \
+									-H "Authorization: Bearer $token" \
+									-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
+									-H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
+									-H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \
+									"$registryBase/v2/$image/manifests/$digest"
+							)"
+							handle_single_manifest_v2 "$submanifestJson"
+							found="found"
+							break
 						fi
-
-						case "$layerMediaType" in
-							application/vnd.docker.image.rootfs.diff.tar.gzip)
-								layerTar="$layerId/layer.tar"
-								layerFiles=( "${layerFiles[@]}" "$layerTar" )
-								# TODO figure out why "-C -" doesn't work here
-								# "curl: (33) HTTP server doesn't seem to support byte ranges. Cannot resume."
-								# "HTTP/1.1 416 Requested Range Not Satisfiable"
-								if [ -f "$dir/$layerTar" ]; then
-									# TODO hackpatch for no -C support :'(
-									echo "skipping existing ${layerId:0:12}"
-									continue
-								fi
-								token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"
-								fetch_blob "$token" "$image" "$layerDigest" "$dir/$layerTar" --progress
-								;;
-
-							*)
-								echo >&2 "error: unknown layer mediaType ($imageIdentifier, $layerDigest): '$layerMediaType'"
-								exit 1
-								;;
-						esac
 					done
-
-					# change "$imageId" to be the ID of the last layer we added (needed for old-style "repositories" file which is created later -- specifically for older Docker daemons)
-					imageId="$layerId"
-
-					# munge the top layer image manifest to have the appropriate image configuration for older daemons
-					imageOldConfig="$(jq --raw-output --compact-output '{ id: .id } + if .parent then { parent: .parent } else {} end' "$dir/$imageId/json")"
-					jq --raw-output "$imageOldConfig + del(.history, .rootfs)" "$dir/$configFile" > "$dir/$imageId/json"
-
-					manifestJsonEntry="$(
-						echo '{}' | jq --raw-output '. + {
-							Config: "'"$configFile"'",
-							RepoTags: ["'"${image#library\/}:$tag"'"],
-							Layers: '"$(echo '[]' | jq --raw-output ".$(for layerFile in "${layerFiles[@]}"; do echo " + [ \"$layerFile\" ]"; done)")"'
-						}'
-					)"
-					manifestJsonEntries=( "${manifestJsonEntries[@]}" "$manifestJsonEntry" )
+					if [ -z "$found" ]; then
+						echo >&2 "error: manifest for $maniArch is not found"
+						exit 1
+					fi
 					;;
-
 				*)
 					echo >&2 "error: unknown manifest mediaType ($imageIdentifier): '$mediaType'"
 					exit 1

docs/api/version-history.md

@@ -13,11 +13,6 @@ keywords: "API, Docker, rcli, REST, documentation"
      will be rejected.
 -->
 
-## v1.36 API changes
-
-[Docker Engine API v1.36](https://docs.docker.com/engine/api/v1.36/) documentation
-
-
 ## v1.35 API changes
 
 [Docker Engine API v1.35](https://docs.docker.com/engine/api/v1.35/) documentation

hack/make/.detect-daemon-osarch

@@ -64,7 +64,9 @@ case "$PACKAGE_ARCH" in
 		;;
 	*)
 		DOCKERFILE="Dockerfile.$PACKAGE_ARCH"
-		TEST_IMAGE_NAMESPACE="$PACKAGE_ARCH"
+		if [ "$PACKAGE_ARCH" != "aarch64" ]; then
+			TEST_IMAGE_NAMESPACE="$PACKAGE_ARCH"
+		fi
 		;;
 esac
 export DOCKERFILE TEST_IMAGE_NAMESPACE

integration-cli/docker_cli_build_test.go

@@ -4221,6 +4221,7 @@ func (s *DockerTrustSuite) TestBuildContextDirIsSymlink(c *check.C) {
 }
 
 func (s *DockerTrustSuite) TestTrustedBuildTagFromReleasesRole(c *check.C) {
+	c.Skip("Blacklisting for Docker CE")
 	testRequires(c, NotaryHosting)
 
 	latestTag := s.setupTrustedImage(c, "trusted-build-releases-role")
@@ -4252,6 +4253,7 @@ func (s *DockerTrustSuite) TestTrustedBuildTagFromReleasesRole(c *check.C) {
 }
 
 func (s *DockerTrustSuite) TestTrustedBuildTagIgnoresOtherDelegationRoles(c *check.C) {
+	c.Skip("Blacklisting for Docker CE")
 	testRequires(c, NotaryHosting)
 
 	latestTag := s.setupTrustedImage(c, "trusted-build-releases-role")

integration-cli/docker_cli_pull_trusted_test.go

@@ -133,6 +133,7 @@ func (s *DockerTrustSuite) TestTrustedPullDelete(c *check.C) {
 }
 
 func (s *DockerTrustSuite) TestTrustedPullReadsFromReleasesRole(c *check.C) {
+	c.Skip("Blacklisting for Docker CE")
 	testRequires(c, NotaryHosting)
 	repoName := fmt.Sprintf("%v/dockerclireleasesdelegationpulling/trusted", privateRegistryURL)
 	targetName := fmt.Sprintf("%s:latest", repoName)
@@ -188,6 +189,7 @@ func (s *DockerTrustSuite) TestTrustedPullReadsFromReleasesRole(c *check.C) {
 }
 
 func (s *DockerTrustSuite) TestTrustedPullIgnoresOtherDelegationRoles(c *check.C) {
+	c.Skip("Blacklisting for Docker CE")
 	testRequires(c, NotaryHosting)
 	repoName := fmt.Sprintf("%v/dockerclipullotherdelegation/trusted", privateRegistryURL)
 	targetName := fmt.Sprintf("%s:latest", repoName)

integration-cli/docker_cli_push_test.go

@@ -282,6 +282,7 @@ func (s *DockerSchema1RegistrySuite) TestCrossRepositoryLayerPushNotSupported(c
 }
 
 func (s *DockerTrustSuite) TestTrustedPush(c *check.C) {
+	c.Skip("Blacklisting for Docker CE")
 	repoName := fmt.Sprintf("%v/dockerclitrusted/pushtest:latest", privateRegistryURL)
 	// tag the image and upload it to the private registry
 	cli.DockerCmd(c, "tag", "busybox", repoName)
@@ -366,6 +367,7 @@ func (s *DockerTrustSuite) TestTrustedPushWithExistingSignedTag(c *check.C) {
 }
 
 func (s *DockerTrustSuite) TestTrustedPushWithIncorrectPassphraseForNonRoot(c *check.C) {
+	c.Skip("Blacklisting for Docker CE")
 	repoName := fmt.Sprintf("%v/dockercliincorretpwd/trusted:latest", privateRegistryURL)
 	// tag the image and upload it to the private registry
 	cli.DockerCmd(c, "tag", "busybox", repoName)

integration-cli/trust_server_test.go

@@ -41,7 +41,7 @@ const notaryHost = "localhost:4443"
 const notaryURL = "https://" + notaryHost
 
 var SuccessTagging = icmd.Expected{
-	Out: "Tagging",
+	Err: "Tagging",
 }
 
 var SuccessSigningAndPushing = icmd.Expected{