Merge pull request #46044 from neersighted/init_cleanup_24.0

[24.0 backport] Upstart & sysvinit cleanup

.github/actions/setup-runner/action.yml

@@ -13,7 +13,7 @@ runs:
       shell: bash
     - run: |
         if [ ! -e /etc/docker/daemon.json ]; then
-         echo '{}' | tee /etc/docker/daemon.json >/dev/null
+         echo '{}' | sudo tee /etc/docker/daemon.json >/dev/null
         fi
         DOCKERD_CONFIG=$(jq '.+{"experimental":true,"live-restore":true,"ipv6":true,"fixed-cidr-v6":"2001:db8:1::/64"}' /etc/docker/daemon.json)
         sudo tee /etc/docker/daemon.json <<<"$DOCKERD_CONFIG" >/dev/null

.github/workflows/.windows.yml

@@ -15,7 +15,7 @@ on:
         default: false
 
 env:
-  GO_VERSION: "1.20.4"
+  GO_VERSION: "1.20.6"
   GOTESTLIST_VERSION: v0.3.1
   TESTSTAT_VERSION: v0.1.3
   WINDOWS_BASE_IMAGE: mcr.microsoft.com/windows/servercore

.github/workflows/bin-image.yml

@@ -0,0 +1,180 @@
+name: bin-image
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'master'
+      - '[0-9]+.[0-9]+'
+    tags:
+      - 'v*'
+  pull_request:
+
+env:
+  MOBYBIN_REPO_SLUG: moby/moby-bin
+  PLATFORM: Moby Engine
+  PRODUCT: Moby
+  DEFAULT_PRODUCT_LICENSE: Moby
+  PACKAGER_NAME: Moby
+
+jobs:
+  validate-dco:
+    uses: ./.github/workflows/.dco.yml
+
+  prepare:
+    runs-on: ubuntu-20.04
+    outputs:
+      platforms: ${{ steps.platforms.outputs.matrix }}
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.MOBYBIN_REPO_SLUG }}
+          ### versioning strategy
+          ## push semver tag v23.0.0
+          # moby/moby-bin:23.0.0
+          # moby/moby-bin:latest
+          ## push semver prelease tag v23.0.0-beta.1
+          # moby/moby-bin:23.0.0-beta.1
+          ## push on master
+          # moby/moby-bin:master
+          ## push on 23.0 branch
+          # moby/moby-bin:23.0
+          ## any push
+          # moby/moby-bin:sha-ad132f5
+          tags: |
+            type=semver,pattern={{version}}
+            type=ref,event=branch
+            type=ref,event=pr
+            type=sha
+      -
+        name: Rename meta bake definition file
+        run: |
+          mv "${{ steps.meta.outputs.bake-file }}" "/tmp/bake-meta.json"
+      -
+        name: Upload meta bake definition
+        uses: actions/upload-artifact@v3
+        with:
+          name: bake-meta
+          path: /tmp/bake-meta.json
+          if-no-files-found: error
+          retention-days: 1
+      -
+        name: Create platforms matrix
+        id: platforms
+        run: |
+          echo "matrix=$(docker buildx bake bin-image-cross --print | jq -cr '.target."bin-image-cross".platforms')" >>${GITHUB_OUTPUT}
+
+  build:
+    runs-on: ubuntu-20.04
+    needs:
+      - validate-dco
+      - prepare
+    strategy:
+      fail-fast: false
+      matrix:
+        platform: ${{ fromJson(needs.prepare.outputs.platforms) }}
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      -
+        name: Download meta bake definition
+        uses: actions/download-artifact@v3
+        with:
+          name: bake-meta
+          path: /tmp
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_MOBYBIN_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_MOBYBIN_TOKEN }}
+      -
+        name: Build
+        id: bake
+        uses: docker/bake-action@v3
+        with:
+          files: |
+            ./docker-bake.hcl
+            /tmp/bake-meta.json
+          targets: bin-image
+          set: |
+            *.platform=${{ matrix.platform }}
+            *.output=type=image,name=${{ env.MOBYBIN_REPO_SLUG }},push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }}
+            *.tags=
+      -
+        name: Export digest
+        if: github.event_name != 'pull_request'
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ fromJSON(steps.bake.outputs.metadata)['bin-image']['containerimage.digest'] }}"
+          touch "/tmp/digests/${digest#sha256:}"
+      -
+        name: Upload digest
+        if: github.event_name != 'pull_request'
+        uses: actions/upload-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    runs-on: ubuntu-20.04
+    if: github.event_name != 'pull_request'
+    needs:
+      - build
+    steps:
+      -
+        name: Download meta bake definition
+        uses: actions/download-artifact@v3
+        with:
+          name: bake-meta
+          path: /tmp
+      -
+        name: Download digests
+        uses: actions/download-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      -
+        name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_MOBYBIN_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_MOBYBIN_TOKEN }}
+      -
+        name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          set -x
+          docker buildx imagetools create $(jq -cr '.target."docker-metadata-action".tags | map("-t " + .) | join(" ")' /tmp/bake-meta.json) \
+            $(printf '${{ env.MOBYBIN_REPO_SLUG }}@sha256:%s ' *)
+      -
+        name: Inspect image
+        run: |
+          set -x
+          docker buildx imagetools inspect ${{ env.MOBYBIN_REPO_SLUG }}:$(jq -cr '.target."docker-metadata-action".args.DOCKER_META_VERSION' /tmp/bake-meta.json)

.github/workflows/buildkit.yml

@@ -13,6 +13,8 @@ on:
   pull_request:
 
 env:
+  # FIXME(thaJeztah): update to newer go versions once BuildKit's vendoring has the fix from https://github.com/moby/moby/pull/45942
+  GO_VERSION: "1.20.5"
   DESTDIR: ./build
 
 jobs:
@@ -47,8 +49,6 @@ jobs:
   test:
     runs-on: ubuntu-20.04
     timeout-minutes: 120
-    env:
-      BUILDKIT_REPO: moby/buildkit
     needs:
       - build
     strategy:
@@ -82,7 +82,7 @@ jobs:
       -
         name: BuildKit ref
         run: |
-          echo "BUILDKIT_REF=$(./hack/buildkit-ref)" >> $GITHUB_ENV
+          echo "$(./hack/buildkit-ref)" >> $GITHUB_ENV
         working-directory: moby
       -
         name: Checkout BuildKit ${{ env.BUILDKIT_REF }}
@@ -106,7 +106,7 @@ jobs:
       -
         name: Update daemon.json
         run: |
-          sudo rm /etc/docker/daemon.json
+          sudo rm -f /etc/docker/daemon.json
           sudo service docker restart
           docker version
           docker info

.github/workflows/test.yml

@@ -15,7 +15,7 @@ on:
   pull_request:
 
 env:
-  GO_VERSION: "1.20.4"
+  GO_VERSION: "1.20.6"
   GOTESTLIST_VERSION: v0.3.1
   TESTSTAT_VERSION: v0.1.3
   ITG_CLI_MATRIX_SIZE: 6

Dockerfile

@@ -1,12 +1,18 @@
 # syntax=docker/dockerfile:1
 
-ARG GO_VERSION=1.20.4
+ARG GO_VERSION=1.20.6
 ARG BASE_DEBIAN_DISTRO="bullseye"
 ARG GOLANG_IMAGE="golang:${GO_VERSION}-${BASE_DEBIAN_DISTRO}"
 ARG XX_VERSION=1.2.1
 
 ARG VPNKIT_VERSION=0.5.0
-ARG DOCKERCLI_VERSION=v17.06.2-ce
+
+ARG DOCKERCLI_REPOSITORY="https://github.com/docker/cli.git"
+ARG DOCKERCLI_VERSION=v24.0.2
+# cli version used for integration-cli tests
+ARG DOCKERCLI_INTEGRATION_REPOSITORY="https://github.com/docker/cli.git"
+ARG DOCKERCLI_INTEGRATION_VERSION=v17.06.2-ce
+ARG BUILDX_VERSION=0.11.2
 
 ARG SYSTEMD="false"
 ARG DEBIAN_FRONTEND=noninteractive
@@ -27,8 +33,7 @@ FROM --platform=$BUILDPLATFORM ${GOLANG_IMAGE} AS base
 COPY --from=xx / /
 RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
 ARG APT_MIRROR
-RUN sed -ri "s/(httpredir|deb).debian.org/${APT_MIRROR:-deb.debian.org}/g" /etc/apt/sources.list \
- && sed -ri "s/(security).debian.org/${APT_MIRROR:-security.debian.org}/g" /etc/apt/sources.list
+RUN test -n "$APT_MIRROR" && sed -ri "s#(httpredir|deb|security).debian.org#${APT_MIRROR}#g" /etc/apt/sources.list || true
 ARG DEBIAN_FRONTEND
 RUN apt-get update && apt-get install --no-install-recommends -y file
 ENV GO111MODULE=off
@@ -192,7 +197,7 @@ RUN git init . && git remote add origin "https://github.com/containerd/container
 # When updating the binary version you may also need to update the vendor
 # version to pick up bug fixes or new APIs, however, usually the Go packages
 # are built from a commit from the master branch.
-ARG CONTAINERD_VERSION=v1.7.0
+ARG CONTAINERD_VERSION=v1.7.1
 RUN git fetch -q --depth 1 origin "${CONTAINERD_VERSION}" +refs/tags/*:refs/tags/* && git checkout -q FETCH_HEAD
 
 FROM base AS containerd-build
@@ -243,34 +248,29 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
         GOBIN=/build/ GO111MODULE=on go install "mvdan.cc/sh/v3/cmd/shfmt@${SHFMT_VERSION}" \
      && /build/shfmt --version
 
-# dockercli
-FROM base AS dockercli-src
-WORKDIR /tmp/dockercli
-RUN git init . && git remote add origin "https://github.com/docker/cli.git"
-ARG DOCKERCLI_VERSION
-RUN git fetch -q --depth 1 origin "${DOCKERCLI_VERSION}" +refs/tags/*:refs/tags/* && git checkout -q FETCH_HEAD
-RUN [ -d ./components/cli ] && mv ./components/cli /usr/src/dockercli || mv /tmp/dockercli /usr/src/dockercli
-WORKDIR /usr/src/dockercli
-
 FROM base AS dockercli
 WORKDIR /go/src/github.com/docker/cli
+ARG DOCKERCLI_REPOSITORY
 ARG DOCKERCLI_VERSION
-ARG DOCKERCLI_CHANNEL=stable
 ARG TARGETPLATFORM
-RUN xx-apt-get install -y --no-install-recommends gcc libc6-dev
-RUN --mount=from=dockercli-src,src=/usr/src/dockercli,rw \
-    --mount=type=cache,target=/root/.cache/go-build,id=dockercli-build-$TARGETPLATFORM <<EOT
-  set -e
-  DOWNLOAD_URL="https://download.docker.com/linux/static/${DOCKERCLI_CHANNEL}/$(xx-info march)/docker-${DOCKERCLI_VERSION#v}.tgz"
-  if curl --head --silent --fail "${DOWNLOAD_URL}" 1>/dev/null 2>&1; then
-    mkdir /build
-    curl -Ls "${DOWNLOAD_URL}" | tar -xz docker/docker
-    mv docker/docker /build/docker
-  else
-    CGO_ENABLED=0 xx-go build -o /build/docker ./cmd/docker
-  fi
-  xx-verify /build/docker
-EOT
+RUN --mount=source=hack/dockerfile/cli.sh,target=/download-or-build-cli.sh \
+    --mount=type=cache,id=dockercli-git-$TARGETPLATFORM,sharing=locked,target=./.git \
+    --mount=type=cache,target=/root/.cache/go-build,id=dockercli-build-$TARGETPLATFORM \
+        rm -f ./.git/*.lock \
+     && /download-or-build-cli.sh ${DOCKERCLI_VERSION} ${DOCKERCLI_REPOSITORY} /build \
+     && /build/docker --version
+
+FROM base AS dockercli-integration
+WORKDIR /go/src/github.com/docker/cli
+ARG DOCKERCLI_INTEGRATION_REPOSITORY
+ARG DOCKERCLI_INTEGRATION_VERSION
+ARG TARGETPLATFORM
+RUN --mount=source=hack/dockerfile/cli.sh,target=/download-or-build-cli.sh \
+    --mount=type=cache,id=dockercli-git-$TARGETPLATFORM,sharing=locked,target=./.git \
+    --mount=type=cache,target=/root/.cache/go-build,id=dockercli-build-$TARGETPLATFORM \
+        rm -f ./.git/*.lock \
+     && /download-or-build-cli.sh ${DOCKERCLI_INTEGRATION_VERSION} ${DOCKERCLI_INTEGRATION_REPOSITORY} /build \
+     && /build/docker --version
 
 # runc
 FROM base AS runc-src
@@ -280,7 +280,7 @@ RUN git init . && git remote add origin "https://github.com/opencontainers/runc.
 # that is used. If you need to update runc, open a pull request in the containerd
 # project first, and update both after that is merged. When updating RUNC_VERSION,
 # consider updating runc in vendor.mod accordingly.
-ARG RUNC_VERSION=v1.1.7
+ARG RUNC_VERSION=v1.1.8
 RUN git fetch -q --depth 1 origin "${RUNC_VERSION}" +refs/tags/*:refs/tags/* && git checkout -q FETCH_HEAD
 
 FROM base AS runc-build
@@ -368,8 +368,8 @@ RUN --mount=from=rootlesskit-src,src=/usr/src/rootlesskit,rw \
   xx-go build -o /build/rootlesskit-docker-proxy -ldflags="$([ "$DOCKER_STATIC" != "1" ] && echo "-linkmode=external")" ./cmd/rootlesskit-docker-proxy
   xx-verify $([ "$DOCKER_STATIC" = "1" ] && echo "--static") /build/rootlesskit-docker-proxy
 EOT
-COPY ./contrib/dockerd-rootless.sh /build/
-COPY ./contrib/dockerd-rootless-setuptool.sh /build/
+COPY --link ./contrib/dockerd-rootless.sh /build/
+COPY --link ./contrib/dockerd-rootless-setuptool.sh /build/
 
 FROM rootlesskit-build AS rootlesskit-linux
 FROM binary-dummy AS rootlesskit-windows
@@ -437,28 +437,36 @@ FROM binary-dummy AS containerutil-linux
 FROM containerutil-build AS containerutil-windows-amd64
 FROM containerutil-windows-${TARGETARCH} AS containerutil-windows
 FROM containerutil-${TARGETOS} AS containerutil
+FROM docker/buildx-bin:${BUILDX_VERSION} as buildx
 
 FROM base AS dev-systemd-false
-COPY --from=dockercli     /build/ /usr/local/cli
-COPY --from=frozen-images /build/ /docker-frozen-images
-COPY --from=swagger       /build/ /usr/local/bin/
-COPY --from=delve         /build/ /usr/local/bin/
-COPY --from=tomll         /build/ /usr/local/bin/
-COPY --from=gowinres      /build/ /usr/local/bin/
-COPY --from=tini          /build/ /usr/local/bin/
-COPY --from=registry      /build/ /usr/local/bin/
-COPY --from=criu          /build/ /usr/local/bin/
-COPY --from=gotestsum     /build/ /usr/local/bin/
-COPY --from=golangci_lint /build/ /usr/local/bin/
-COPY --from=shfmt         /build/ /usr/local/bin/
-COPY --from=runc          /build/ /usr/local/bin/
-COPY --from=containerd    /build/ /usr/local/bin/
-COPY --from=rootlesskit   /build/ /usr/local/bin/
-COPY --from=vpnkit        /       /usr/local/bin/
-COPY --from=containerutil /build/ /usr/local/bin/
-COPY --from=crun          /build/ /usr/local/bin/
-COPY hack/dockerfile/etc/docker/  /etc/docker/
+COPY --link --from=frozen-images /build/ /docker-frozen-images
+COPY --link --from=swagger       /build/ /usr/local/bin/
+COPY --link --from=delve         /build/ /usr/local/bin/
+COPY --link --from=tomll         /build/ /usr/local/bin/
+COPY --link --from=gowinres      /build/ /usr/local/bin/
+COPY --link --from=tini          /build/ /usr/local/bin/
+COPY --link --from=registry      /build/ /usr/local/bin/
+
+# Skip the CRIU stage for now, as the opensuse package repository is sometimes
+# unstable, and we're currently not using it in CI.
+#
+# FIXME(thaJeztah): re-enable this stage when https://github.com/moby/moby/issues/38963 is resolved (see https://github.com/moby/moby/pull/38984)
+# COPY --link --from=criu          /build/ /usr/local/bin/
+COPY --link --from=gotestsum     /build/ /usr/local/bin/
+COPY --link --from=golangci_lint /build/ /usr/local/bin/
+COPY --link --from=shfmt         /build/ /usr/local/bin/
+COPY --link --from=runc          /build/ /usr/local/bin/
+COPY --link --from=containerd    /build/ /usr/local/bin/
+COPY --link --from=rootlesskit   /build/ /usr/local/bin/
+COPY --link --from=vpnkit        /       /usr/local/bin/
+COPY --link --from=containerutil /build/ /usr/local/bin/
+COPY --link --from=crun          /build/ /usr/local/bin/
+COPY --link hack/dockerfile/etc/docker/  /etc/docker/
+COPY --link --from=buildx        /buildx /usr/local/libexec/docker/cli-plugins/docker-buildx
+
 ENV PATH=/usr/local/cli:$PATH
+ENV TEST_CLIENT_BINARY=/usr/local/cli-integration/docker
 ENV CONTAINERD_ADDRESS=/run/docker/containerd/containerd.sock
 ENV CONTAINERD_NAMESPACE=moby
 WORKDIR /go/src/github.com/docker/docker
@@ -544,6 +552,8 @@ RUN --mount=type=cache,sharing=locked,id=moby-dev-aptlib,target=/var/lib/apt \
             libsecret-1-dev \
             libsystemd-dev \
             libudev-dev
+COPY --link --from=dockercli             /build/ /usr/local/cli
+COPY --link --from=dockercli-integration /build/ /usr/local/cli-integration
 
 FROM base AS build
 COPY --from=gowinres /build/ /usr/local/bin/
@@ -615,13 +625,13 @@ COPY --from=build /build/ /
 # usage:
 # > docker buildx bake all
 FROM scratch AS all
-COPY --from=tini          /build/ /
-COPY --from=runc          /build/ /
-COPY --from=containerd    /build/ /
-COPY --from=rootlesskit   /build/ /
-COPY --from=containerutil /build/ /
-COPY --from=vpnkit        /       /
-COPY --from=build         /build  /
+COPY --link --from=tini          /build/ /
+COPY --link --from=runc          /build/ /
+COPY --link --from=containerd    /build/ /
+COPY --link --from=rootlesskit   /build/ /
+COPY --link --from=containerutil /build/ /
+COPY --link --from=vpnkit        /       /
+COPY --link --from=build         /build  /
 
 # smoke tests
 # usage:
@@ -641,4 +651,4 @@ EOT
 # > make shell
 # > SYSTEMD=true make shell
 FROM dev-base AS dev
-COPY . .
+COPY --link . .

Dockerfile.e2e

@@ -71,8 +71,8 @@ RUN apk --no-cache add \
     tar \
     xz
 
-COPY hack/test/e2e-run.sh       /scripts/run.sh
-COPY hack/make/.ensure-emptyfs  /scripts/ensure-emptyfs.sh
+COPY hack/test/e2e-run.sh           /scripts/run.sh
+COPY hack/make/.build-empty-images  /scripts/build-empty-images.sh
 
 COPY integration/testdata       /tests/integration/testdata
 COPY integration/build/testdata /tests/integration/build/testdata

Dockerfile.simple

@@ -5,7 +5,7 @@
 
 # This represents the bare minimum required to build and test Docker.
 
-ARG GO_VERSION=1.20.4
+ARG GO_VERSION=1.20.6
 
 ARG BASE_DEBIAN_DISTRO="bullseye"
 ARG GOLANG_IMAGE="golang:${GO_VERSION}-${BASE_DEBIAN_DISTRO}"
@@ -13,9 +13,9 @@ ARG GOLANG_IMAGE="golang:${GO_VERSION}-${BASE_DEBIAN_DISTRO}"
 FROM ${GOLANG_IMAGE}
 ENV GO111MODULE=off
 
-# allow replacing httpredir or deb mirror
-ARG APT_MIRROR=deb.debian.org
-RUN sed -ri "s/(httpredir|deb).debian.org/$APT_MIRROR/g" /etc/apt/sources.list
+# allow replacing debian mirror
+ARG APT_MIRROR
+RUN test -n "$APT_MIRROR" && sed -ri "s#(httpredir|deb|security).debian.org#${APT_MIRROR}#g" /etc/apt/sources.list || true
 
 # Compile and runtime deps
 # https://github.com/docker/docker/blob/master/project/PACKAGERS.md#build-dependencies

Dockerfile.windows

@@ -165,10 +165,10 @@ FROM microsoft/windowsservercore
 # Use PowerShell as the default shell
 SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
 
-ARG GO_VERSION=1.20.4
+ARG GO_VERSION=1.20.6
 ARG GOTESTSUM_VERSION=v1.8.2
 ARG GOWINRES_VERSION=v0.3.0
-ARG CONTAINERD_VERSION=v1.7.0
+ARG CONTAINERD_VERSION=v1.7.1
 
 # Environment variable notes:
 #  - GO_VERSION must be consistent with 'Dockerfile' used by Linux.

Jenkinsfile

@@ -17,7 +17,6 @@ pipeline {
         DOCKER_BUILDKIT     = '1'
         DOCKER_EXPERIMENTAL = '1'
         DOCKER_GRAPHDRIVER  = 'overlay2'
-        APT_MIRROR          = 'cdn-fastly.deb.debian.org'
         CHECK_CONFIG_COMMIT = '33a3680e08d1007e72c3b3f1454f823d8e9948ee'
         TESTDEBUG           = '0'
         TIMEOUT             = '120m'
@@ -78,7 +77,7 @@ pipeline {
                         stage("Build dev image") {
                             steps {
                                 sh '''
-                                docker build --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} .
+                                docker build --force-rm -t docker:${GIT_COMMIT} .
                                 '''
                             }
                         }
@@ -191,7 +190,7 @@ pipeline {
                         stage("Build dev image") {
                             steps {
                                 sh '''
-                                docker build --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} .
+                                docker build --force-rm -t docker:${GIT_COMMIT} .
                                 '''
                             }
                         }
@@ -278,7 +277,7 @@ pipeline {
                         stage("Build dev image") {
                             steps {
                                 sh '''
-                                docker buildx build --load --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} .
+                                docker buildx build --load --force-rm -t docker:${GIT_COMMIT} .
                                 '''
                             }
                         }
@@ -391,7 +390,7 @@ pipeline {
                         stage("Build dev image") {
                             steps {
                                 sh '''
-                                docker buildx build --load --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} .
+                                docker buildx build --load --force-rm -t docker:${GIT_COMMIT} .
                                 '''
                             }
                         }
@@ -476,7 +475,7 @@ pipeline {
                         }
                         stage("Build dev image") {
                             steps {
-                                sh 'docker build --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} .'
+                                sh 'docker build --force-rm -t docker:${GIT_COMMIT} .'
                             }
                         }
                         stage("Unit tests") {

Makefile

@@ -41,6 +41,10 @@ DOCKER_ENVS := \
 	-e DOCKER_BUILDKIT \
 	-e DOCKER_BASH_COMPLETION_PATH \
 	-e DOCKER_CLI_PATH \
+	-e DOCKERCLI_VERSION \
+	-e DOCKERCLI_REPOSITORY \
+	-e DOCKERCLI_INTEGRATION_VERSION \
+	-e DOCKERCLI_INTEGRATION_REPOSITORY \
 	-e DOCKER_DEBUG \
 	-e DOCKER_EXPERIMENTAL \
 	-e DOCKER_GITCOMMIT \
@@ -136,6 +140,11 @@ endif
 DOCKER_RUN_DOCKER := $(DOCKER_FLAGS) "$(DOCKER_IMAGE)"
 
 DOCKER_BUILD_ARGS += --build-arg=GO_VERSION
+DOCKER_BUILD_ARGS += --build-arg=APT_MIRROR
+DOCKER_BUILD_ARGS += --build-arg=DOCKERCLI_VERSION
+DOCKER_BUILD_ARGS += --build-arg=DOCKERCLI_REPOSITORY
+DOCKER_BUILD_ARGS += --build-arg=DOCKERCLI_INTEGRATION_VERSION
+DOCKER_BUILD_ARGS += --build-arg=DOCKERCLI_INTEGRATION_REPOSITORY
 ifdef DOCKER_SYSTEMD
 DOCKER_BUILD_ARGS += --build-arg=SYSTEMD=true
 endif

api/server/router/container/container_routes.go

@@ -21,7 +21,7 @@ import (
 	containerpkg "github.com/docker/docker/container"
 	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/pkg/ioutils"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/websocket"
@@ -44,7 +44,7 @@ func (s *containerRouter) postCommit(ctx context.Context, w http.ResponseWriter,
 	}
 
 	config, _, _, err := s.decoder.DecodeConfig(r.Body)
-	if err != nil && err != io.EOF { // Do not fail if body is empty.
+	if err != nil && !errors.Is(err, io.EOF) { // Do not fail if body is empty.
 		return err
 	}
 
@@ -486,6 +486,9 @@ func (s *containerRouter) postContainersCreate(ctx context.Context, w http.Respo
 
 	config, hostConfig, networkingConfig, err := s.decoder.DecodeConfig(r.Body)
 	if err != nil {
+		if errors.Is(err, io.EOF) {
+			return errdefs.InvalidParameter(errors.New("invalid JSON: got EOF while reading request body"))
+		}
 		return err
 	}
 	version := httputils.VersionFromContext(ctx)
@@ -566,7 +569,7 @@ func (s *containerRouter) postContainersCreate(ctx context.Context, w http.Respo
 		hostConfig.Annotations = nil
 	}
 
-	var platform *specs.Platform
+	var platform *ocispec.Platform
 	if versions.GreaterThanOrEqualTo(version, "1.41") {
 		if v := r.Form.Get("platform"); v != "" {
 			p, err := platforms.Parse(v)

api/server/router/distribution/distribution_routes.go

@@ -12,7 +12,7 @@ import (
 	"github.com/docker/docker/api/server/httputils"
 	"github.com/docker/docker/api/types/registry"
 	"github.com/docker/docker/errdefs"
-	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
 
@@ -61,7 +61,7 @@ func (s *distributionRouter) getDistributionInfo(ctx context.Context, w http.Res
 		if err != nil {
 			return err
 		}
-		distributionInspect.Descriptor = v1.Descriptor{
+		distributionInspect.Descriptor = ocispec.Descriptor{
 			MediaType: descriptor.MediaType,
 			Digest:    descriptor.Digest,
 			Size:      descriptor.Size,
@@ -107,7 +107,7 @@ func (s *distributionRouter) getDistributionInfo(ctx context.Context, w http.Res
 	switch mnfstObj := mnfst.(type) {
 	case *manifestlist.DeserializedManifestList:
 		for _, m := range mnfstObj.Manifests {
-			distributionInspect.Platforms = append(distributionInspect.Platforms, v1.Platform{
+			distributionInspect.Platforms = append(distributionInspect.Platforms, ocispec.Platform{
 				Architecture: m.Platform.Architecture,
 				OS:           m.Platform.OS,
 				OSVersion:    m.Platform.OSVersion,
@@ -117,7 +117,7 @@ func (s *distributionRouter) getDistributionInfo(ctx context.Context, w http.Res
 		}
 	case *schema2.DeserializedManifest:
 		configJSON, err := blobsrvc.Get(ctx, mnfstObj.Config.Digest)
-		var platform v1.Platform
+		var platform ocispec.Platform
 		if err == nil {
 			err := json.Unmarshal(configJSON, &platform)
 			if err == nil && (platform.OS != "" || platform.Architecture != "") {
@@ -125,7 +125,7 @@ func (s *distributionRouter) getDistributionInfo(ctx context.Context, w http.Res
 			}
 		}
 	case *schema1.SignedManifest:
-		platform := v1.Platform{
+		platform := ocispec.Platform{
 			Architecture: mnfstObj.Architecture,
 			OS:           "linux",
 		}

api/server/router/image/backend.go

@@ -10,7 +10,7 @@ import (
 	"github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/api/types/registry"
 	dockerimage "github.com/docker/docker/image"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // Backend is all the methods that need to be implemented
@@ -32,12 +32,12 @@ type imageBackend interface {
 
 type importExportBackend interface {
 	LoadImage(ctx context.Context, inTar io.ReadCloser, outStream io.Writer, quiet bool) error
-	ImportImage(ctx context.Context, ref reference.Named, platform *specs.Platform, msg string, layerReader io.Reader, changes []string) (dockerimage.ID, error)
+	ImportImage(ctx context.Context, ref reference.Named, platform *ocispec.Platform, msg string, layerReader io.Reader, changes []string) (dockerimage.ID, error)
 	ExportImage(ctx context.Context, names []string, outStream io.Writer) error
 }
 
 type registryBackend interface {
-	PullImage(ctx context.Context, image, tag string, platform *specs.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error
+	PullImage(ctx context.Context, image, tag string, platform *ocispec.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error
 	PushImage(ctx context.Context, ref reference.Named, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error
 }
 

api/server/router/image/image_routes.go

@@ -24,7 +24,7 @@ import (
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/docker/docker/pkg/progress"
 	"github.com/docker/docker/pkg/streamformatter"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
 
@@ -41,7 +41,7 @@ func (ir *imageRouter) postImagesCreate(ctx context.Context, w http.ResponseWrit
 		comment     = r.Form.Get("message")
 		progressErr error
 		output      = ioutils.NewWriteFlusher(w)
-		platform    *specs.Platform
+		platform    *ocispec.Platform
 	)
 	defer output.Close()
 
@@ -282,6 +282,14 @@ func (ir *imageRouter) toImageInspect(img *image.Image) (*types.ImageInspect, er
 		comment = img.History[len(img.History)-1].Comment
 	}
 
+	// Make sure we output empty arrays instead of nil.
+	if repoTags == nil {
+		repoTags = []string{}
+	}
+	if repoDigests == nil {
+		repoDigests = []string{}
+	}
+
 	return &types.ImageInspect{
 		ID:              img.ID().String(),
 		RepoTags:        repoTags,

api/swagger.yaml

@@ -5162,42 +5162,8 @@ definitions:
       ServerVersion:
         description: |
           Version string of the daemon.
-
-          > **Note**: the [standalone Swarm API](https://docs.docker.com/swarm/swarm-api/)
-          > returns the Swarm version instead of the daemon  version, for example
-          > `swarm/1.2.8`.
         type: "string"
-        example: "17.06.0-ce"
-      ClusterStore:
-        description: |
-          URL of the distributed storage backend.
-
-
-          The storage backend is used for multihost networking (to store
-          network and endpoint information) and by the node discovery mechanism.
-
-          <p><br /></p>
-
-          > **Deprecated**: This field is only propagated when using standalone Swarm
-          > mode, and overlay networking using an external k/v store. Overlay
-          > networks with Swarm mode enabled use the built-in raft store, and
-          > this field will be empty.
-        type: "string"
-        example: "consul://consul.corp.example.com:8600/some/path"
-      ClusterAdvertise:
-        description: |
-          The network endpoint that the Engine advertises for the purpose of
-          node discovery. ClusterAdvertise is a `host:port` combination on which
-          the daemon is reachable by other hosts.
-
-          <p><br /></p>
-
-          > **Deprecated**: This field is only propagated when using standalone Swarm
-          > mode, and overlay networking using an external k/v store. Overlay
-          > networks with Swarm mode enabled use the built-in raft store, and
-          > this field will be empty.
-        type: "string"
-        example: "node5.corp.example.com:8000"
+        example: "24.0.2"
       Runtimes:
         description: |
           List of [OCI compliant](https://github.com/opencontainers/runtime-spec)
@@ -9930,7 +9896,9 @@ paths:
               Id: "22be93d5babb089c5aab8dbc369042fad48ff791584ca2da2100db837a1c7c30"
               Warning: ""
         403:
-          description: "operation not supported for pre-defined networks"
+          description: |
+            Forbidden operation. This happens when trying to create a network named after a pre-defined network,
+            or when trying to create an overlay network on a daemon which is not part of a Swarm cluster.
           schema:
             $ref: "#/definitions/ErrorResponse"
         404:
@@ -10393,6 +10361,12 @@ paths:
             default if omitted.
           required: true
           type: "string"
+        - name: "force"
+          in: "query"
+          description: |
+            Force disable a plugin even if still in use.
+          required: false
+          type: "boolean"
       tags: ["Plugin"]
   /plugins/{name}/upgrade:
     post:

api/types/backend/build.go

@@ -6,7 +6,7 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/registry"
 	"github.com/docker/docker/pkg/streamformatter"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // PullOption defines different modes for accessing images
@@ -42,5 +42,5 @@ type GetImageAndLayerOptions struct {
 	PullOption PullOption
 	AuthConfig map[string]registry.AuthConfig
 	Output     io.Writer
-	Platform   *specs.Platform
+	Platform   *ocispec.Platform
 }

api/types/configs.go

@@ -3,7 +3,7 @@ package types // import "github.com/docker/docker/api/types"
 import (
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/network"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // configs holds structs used for internal communication between the
@@ -16,7 +16,7 @@ type ContainerCreateConfig struct {
 	Config           *container.Config
 	HostConfig       *container.HostConfig
 	NetworkingConfig *network.NetworkingConfig
-	Platform         *specs.Platform
+	Platform         *ocispec.Platform
 	AdjustCPUShares  bool
 }
 

api/types/image/opts.go

@@ -1,9 +1,9 @@
 package image
 
-import specs "github.com/opencontainers/image-spec/specs-go/v1"
+import ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 
 // GetImageOpts holds parameters to inspect an image.
 type GetImageOpts struct {
-	Platform *specs.Platform
+	Platform *ocispec.Platform
 	Details  bool
 }

api/types/registry/registry.go

@@ -4,7 +4,7 @@ import (
 	"encoding/json"
 	"net"
 
-	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // ServiceConfig stores daemon registry services configuration.
@@ -113,8 +113,8 @@ type SearchResults struct {
 type DistributionInspect struct {
 	// Descriptor contains information about the manifest, including
 	// the content addressable digest
-	Descriptor v1.Descriptor
+	Descriptor ocispec.Descriptor
 	// Platforms contains the list of platforms supported by the image,
 	// obtained by parsing the manifest
-	Platforms []v1.Platform
+	Platforms []ocispec.Platform
 }

builder/builder-next/adapters/localinlinecache/inlinecache.go

@@ -18,7 +18,7 @@ import (
 	"github.com/moby/buildkit/solver"
 	"github.com/moby/buildkit/worker"
 	"github.com/opencontainers/go-digest"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
 
@@ -26,9 +26,9 @@ import (
 func ResolveCacheImporterFunc(sm *session.Manager, resolverFunc docker.RegistryHosts, cs content.Store, rs reference.Store, is imagestore.Store) remotecache.ResolveCacheImporterFunc {
 	upstream := registryremotecache.ResolveCacheImporterFunc(sm, cs, resolverFunc)
 
-	return func(ctx context.Context, group session.Group, attrs map[string]string) (remotecache.Importer, specs.Descriptor, error) {
+	return func(ctx context.Context, group session.Group, attrs map[string]string) (remotecache.Importer, ocispec.Descriptor, error) {
 		if dt, err := tryImportLocal(rs, is, attrs["ref"]); err == nil {
-			return newLocalImporter(dt), specs.Descriptor{}, nil
+			return newLocalImporter(dt), ocispec.Descriptor{}, nil
 		}
 		return upstream(ctx, group, attrs)
 	}
@@ -59,7 +59,7 @@ type localImporter struct {
 	dt []byte
 }
 
-func (li *localImporter) Resolve(ctx context.Context, _ specs.Descriptor, id string, w worker.Worker) (solver.CacheManager, error) {
+func (li *localImporter) Resolve(ctx context.Context, _ ocispec.Descriptor, id string, w worker.Worker) (solver.CacheManager, error) {
 	cc := v1.NewCacheChains()
 	if err := li.importInlineCache(ctx, li.dt, cc); err != nil {
 		return nil, err
@@ -96,7 +96,7 @@ func (li *localImporter) importInlineCache(ctx context.Context, dt []byte, cc so
 	layers := v1.DescriptorProvider{}
 	for i, diffID := range img.Rootfs.DiffIDs {
 		dgst := digest.Digest(diffID.String())
-		desc := specs.Descriptor{
+		desc := ocispec.Descriptor{
 			Digest:      dgst,
 			Size:        -1,
 			MediaType:   images.MediaTypeDockerSchema2Layer,
@@ -157,6 +157,6 @@ func parseCreatedLayerInfo(img image) ([]string, []string, error) {
 type emptyProvider struct {
 }
 
-func (p *emptyProvider) ReaderAt(ctx context.Context, dec specs.Descriptor) (content.ReaderAt, error) {
+func (p *emptyProvider) ReaderAt(ctx context.Context, dec ocispec.Descriptor) (content.ReaderAt, error) {
 	return nil, errors.Errorf("ReaderAt not implemented for empty provider")
 }

builder/builder-next/builder.go

@@ -78,6 +78,7 @@ var cacheFields = map[string]bool{
 type Opt struct {
 	SessionManager      *session.Manager
 	Root                string
+	EngineID            string
 	Dist                images.DistributionServices
 	ImageTagger         mobyexporter.ImageTagger
 	NetworkController   *libnetwork.Controller
@@ -354,11 +355,7 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.
 	exporterName := ""
 	exporterAttrs := map[string]string{}
 	if len(opt.Options.Outputs) == 0 {
-		if b.useSnapshotter {
-			exporterName = client.ExporterImage
-		} else {
-			exporterName = exporter.Moby
-		}
+		exporterName = exporter.Moby
 	} else {
 		// cacheonly is a special type for triggering skipping all exporters
 		if opt.Options.Outputs[0].Type != "cacheonly" {

builder/builder-next/controller.go

@@ -16,10 +16,10 @@ import (
 	"github.com/docker/docker/builder/builder-next/adapters/containerimage"
 	"github.com/docker/docker/builder/builder-next/adapters/localinlinecache"
 	"github.com/docker/docker/builder/builder-next/adapters/snapshot"
-	"github.com/docker/docker/builder/builder-next/exporter"
 	"github.com/docker/docker/builder/builder-next/exporter/mobyexporter"
 	"github.com/docker/docker/builder/builder-next/imagerefchecker"
 	mobyworker "github.com/docker/docker/builder/builder-next/worker"
+	wlabel "github.com/docker/docker/builder/builder-next/worker/label"
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/daemon/graphdriver"
 	units "github.com/docker/go-units"
@@ -96,6 +96,7 @@ func newSnapshotterController(ctx context.Context, rt http.RoundTripper, opt Opt
 
 	wo.GCPolicy = policy
 	wo.RegistryHosts = opt.RegistryHosts
+	wo.Labels = getLabels(opt, wo.Labels)
 
 	exec, err := newExecutor(opt.Root, opt.DefaultCgroupParent, opt.NetworkController, dns, opt.Rootless, opt.IdentityMapping, opt.ApparmorProfile)
 	if err != nil {
@@ -303,7 +304,7 @@ func newGraphDriverController(ctx context.Context, rt http.RoundTripper, opt Opt
 		return nil, errors.Errorf("snapshotter doesn't support differ")
 	}
 
-	leases, err := lm.List(ctx, "labels.\"buildkit/lease.temporary\"")
+	leases, err := lm.List(ctx, `labels."buildkit/lease.temporary"`)
 	if err != nil {
 		return nil, err
 	}
@@ -312,7 +313,7 @@ func newGraphDriverController(ctx context.Context, rt http.RoundTripper, opt Opt
 	}
 
 	wopt := mobyworker.Opt{
-		ID:                exporter.Moby,
+		ID:                opt.EngineID,
 		ContentStore:      store,
 		CacheManager:      cm,
 		GCPolicy:          gcPolicy,
@@ -326,6 +327,7 @@ func newGraphDriverController(ctx context.Context, rt http.RoundTripper, opt Opt
 		Layers:            layers,
 		Platforms:         archutil.SupportedPlatforms(true),
 		LeaseManager:      lm,
+		Labels:            getLabels(opt, nil),
 	}
 
 	wc := &worker.Controller{}
@@ -412,3 +414,11 @@ func getEntitlements(conf config.BuilderConfig) []string {
 	}
 	return ents
 }
+
+func getLabels(opt Opt, labels map[string]string) map[string]string {
+	if labels == nil {
+		labels = make(map[string]string)
+	}
+	labels[wlabel.HostGatewayIP] = opt.DNSConfig.HostGatewayIP.String()
+	return labels
+}

builder/builder-next/worker/label/label.go

@@ -0,0 +1,9 @@
+package label
+
+// Pre-defined label keys similar to BuildKit ones
+// https://github.com/moby/buildkit/blob/v0.11.6/worker/label/label.go#L3-L16
+const (
+	prefix = "org.mobyproject.buildkit.worker.moby."
+
+	HostGatewayIP = prefix + "host-gateway-ip"
+)

builder/builder-next/worker/worker.go

@@ -50,7 +50,7 @@ import (
 )
 
 func init() {
-	version.Version = "v0.11.6"
+	version.Version = "v0.11.6+0a15675913b7"
 }
 
 const labelCreatedAt = "buildkit/createdat"

builder/builder.go

@@ -14,6 +14,7 @@ import (
 	containerpkg "github.com/docker/docker/container"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/layer"
+	"github.com/opencontainers/go-digest"
 )
 
 const (
@@ -45,7 +46,7 @@ type Backend interface {
 	// ContainerCreateWorkdir creates the workdir
 	ContainerCreateWorkdir(containerID string) error
 
-	CreateImage(config []byte, parent string) (Image, error)
+	CreateImage(ctx context.Context, config []byte, parent string, contentStoreDigest digest.Digest) (Image, error)
 
 	ImageCacheBuilder
 }
@@ -104,6 +105,7 @@ type ROLayer interface {
 	Release() error
 	NewRWLayer() (RWLayer, error)
 	DiffID() layer.DiffID
+	ContentStoreDigest() digest.Digest
 }
 
 // RWLayer is active layer that can be read/modified

builder/dockerfile/builder.go

@@ -21,7 +21,7 @@ import (
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 	"github.com/moby/buildkit/frontend/dockerfile/parser"
 	"github.com/moby/buildkit/frontend/dockerfile/shell"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sync/syncmap"
@@ -125,7 +125,7 @@ type Builder struct {
 	pathCache        pathCache
 	containerManager *containerManager
 	imageProber      ImageProber
-	platform         *specs.Platform
+	platform         *ocispec.Platform
 }
 
 // newBuilder creates a new Dockerfile builder from an optional dockerfile and a Options.

builder/dockerfile/copy.go

@@ -24,7 +24,7 @@ import (
 	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/docker/docker/pkg/system"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
 
@@ -74,7 +74,7 @@ type copier struct {
 	source      builder.Source
 	pathCache   pathCache
 	download    sourceDownloader
-	platform    *specs.Platform
+	platform    *ocispec.Platform
 	// for cleanup. TODO: having copier.cleanup() is error prone and hard to
 	// follow. Code calling performCopy should manage the lifecycle of its params.
 	// Copier should take override source as input, not imageMount.
@@ -86,7 +86,7 @@ func copierFromDispatchRequest(req dispatchRequest, download sourceDownloader, i
 	platform := req.builder.platform
 	if platform == nil {
 		// May be nil if not explicitly set in API/dockerfile
-		platform = &specs.Platform{}
+		platform = &ocispec.Platform{}
 	}
 	if platform.OS == "" {
 		// Default to the dispatch requests operating system if not explicit in API/dockerfile

builder/dockerfile/copy_test.go

@@ -53,31 +53,31 @@ func TestGetFilenameForDownload(t *testing.T) {
 		expected    string
 	}{
 		{
-			path:     "http://www.example.com/",
+			path:     "https://www.example.com/",
 			expected: "",
 		},
 		{
-			path:     "http://www.example.com/xyz",
+			path:     "https://www.example.com/xyz",
 			expected: "xyz",
 		},
 		{
-			path:     "http://www.example.com/xyz.html",
+			path:     "https://www.example.com/xyz.html",
 			expected: "xyz.html",
 		},
 		{
-			path:     "http://www.example.com/xyz/",
+			path:     "https://www.example.com/xyz/",
 			expected: "",
 		},
 		{
-			path:     "http://www.example.com/xyz/uvw",
+			path:     "https://www.example.com/xyz/uvw",
 			expected: "uvw",
 		},
 		{
-			path:     "http://www.example.com/xyz/uvw.html",
+			path:     "https://www.example.com/xyz/uvw.html",
 			expected: "uvw.html",
 		},
 		{
-			path:     "http://www.example.com/xyz/uvw/",
+			path:     "https://www.example.com/xyz/uvw/",
 			expected: "",
 		},
 		{
@@ -114,23 +114,23 @@ func TestGetFilenameForDownload(t *testing.T) {
 			expected:    "xyz.html",
 		},
 		{
-			disposition: "attachment; filename=\"xyz\"",
+			disposition: `attachment; filename="xyz"`,
 			expected:    "xyz",
 		},
 		{
-			disposition: "attachment; filename=\"xyz.html\"",
+			disposition: `attachment; filename="xyz.html"`,
 			expected:    "xyz.html",
 		},
 		{
-			disposition: "attachment; filename=\"/xyz.html\"",
+			disposition: `attachment; filename="/xyz.html"`,
 			expected:    "xyz.html",
 		},
 		{
-			disposition: "attachment; filename=\"/xyz/uvw\"",
+			disposition: `attachment; filename="/xyz/uvw"`,
 			expected:    "uvw",
 		},
 		{
-			disposition: "attachment; filename=\"Naïve file.txt\"",
+			disposition: `attachment; filename="Naïve file.txt"`,
 			expected:    "Naïve file.txt",
 		},
 	}

builder/dockerfile/copy_windows.go

@@ -15,8 +15,8 @@ import (
 )
 
 var pathDenyList = map[string]bool{
-	"c:\\":        true,
-	"c:\\windows": true,
+	`c:\`:        true,
+	`c:\windows`: true,
 }
 
 func init() {

builder/dockerfile/dispatchers.go

@@ -28,7 +28,7 @@ import (
 	"github.com/moby/buildkit/frontend/dockerfile/parser"
 	"github.com/moby/buildkit/frontend/dockerfile/shell"
 	"github.com/moby/sys/signal"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
 
@@ -158,7 +158,7 @@ func initializeStage(ctx context.Context, d dispatchRequest, cmd *instructions.S
 		return err
 	}
 
-	var platform *specs.Platform
+	var platform *ocispec.Platform
 	if v := cmd.Platform; v != "" {
 		v, err := d.getExpandedString(d.shlex, v)
 		if err != nil {
@@ -232,7 +232,7 @@ func (d *dispatchRequest) getExpandedString(shlex *shell.Lex, str string) (strin
 	return name, nil
 }
 
-func (d *dispatchRequest) getImageOrStage(ctx context.Context, name string, platform *specs.Platform) (builder.Image, error) {
+func (d *dispatchRequest) getImageOrStage(ctx context.Context, name string, platform *ocispec.Platform) (builder.Image, error) {
 	var localOnly bool
 	if im, ok := d.stages.getByName(name); ok {
 		name = im.Image
@@ -266,7 +266,7 @@ func (d *dispatchRequest) getImageOrStage(ctx context.Context, name string, plat
 	return imageMount.Image(), nil
 }
 
-func (d *dispatchRequest) getFromImage(ctx context.Context, shlex *shell.Lex, basename string, platform *specs.Platform) (builder.Image, error) {
+func (d *dispatchRequest) getFromImage(ctx context.Context, shlex *shell.Lex, basename string, platform *ocispec.Platform) (builder.Image, error) {
 	name, err := d.getExpandedString(shlex, basename)
 	if err != nil {
 		return nil, err

builder/dockerfile/evaluator_test.go

@@ -21,8 +21,11 @@ type dispatchTestCase struct {
 	files               map[string]string
 }
 
-func init() {
-	reexec.Init()
+func TestMain(m *testing.M) {
+	if reexec.Init() {
+		return
+	}
+	os.Exit(m.Run())
 }
 
 func TestDispatch(t *testing.T) {

builder/dockerfile/imagecontext.go

@@ -8,12 +8,12 @@ import (
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/builder"
 	dockerimage "github.com/docker/docker/image"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
-type getAndMountFunc func(context.Context, string, bool, *specs.Platform) (builder.Image, builder.ROLayer, error)
+type getAndMountFunc func(context.Context, string, bool, *ocispec.Platform) (builder.Image, builder.ROLayer, error)
 
 // imageSources mounts images and provides a cache for mounted images. It tracks
 // all images so they can be unmounted at the end of the build.
@@ -24,7 +24,7 @@ type imageSources struct {
 }
 
 func newImageSources(options builderOptions) *imageSources {
-	getAndMount := func(ctx context.Context, idOrRef string, localOnly bool, platform *specs.Platform) (builder.Image, builder.ROLayer, error) {
+	getAndMount := func(ctx context.Context, idOrRef string, localOnly bool, platform *ocispec.Platform) (builder.Image, builder.ROLayer, error) {
 		pullOption := backend.PullOptionNoPull
 		if !localOnly {
 			if options.Options.PullParent {
@@ -47,7 +47,7 @@ func newImageSources(options builderOptions) *imageSources {
 	}
 }
 
-func (m *imageSources) Get(ctx context.Context, idOrRef string, localOnly bool, platform *specs.Platform) (*imageMount, error) {
+func (m *imageSources) Get(ctx context.Context, idOrRef string, localOnly bool, platform *ocispec.Platform) (*imageMount, error) {
 	if im, ok := m.byImageID[idOrRef]; ok {
 		return im, nil
 	}
@@ -71,7 +71,7 @@ func (m *imageSources) Unmount() (retErr error) {
 	return
 }
 
-func (m *imageSources) Add(im *imageMount, platform *specs.Platform) {
+func (m *imageSources) Add(im *imageMount, platform *ocispec.Platform) {
 	switch im.image {
 	case nil:
 		// Set the platform for scratch images

builder/dockerfile/internals.go

@@ -19,7 +19,7 @@ import (
 	"github.com/docker/docker/pkg/chrootarchive"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/go-connections/nat"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -63,7 +63,7 @@ func (b *Builder) commitContainer(ctx context.Context, dispatchState *dispatchSt
 	return err
 }
 
-func (b *Builder) exportImage(state *dispatchState, layer builder.RWLayer, parent builder.Image, runConfig *container.Config) error {
+func (b *Builder) exportImage(ctx context.Context, state *dispatchState, layer builder.RWLayer, parent builder.Image, runConfig *container.Config) error {
 	newLayer, err := layer.Commit()
 	if err != nil {
 		return err
@@ -74,7 +74,7 @@ func (b *Builder) exportImage(state *dispatchState, layer builder.RWLayer, paren
 		return errors.Errorf("unexpected image type")
 	}
 
-	platform := &specs.Platform{
+	platform := &ocispec.Platform{
 		OS:           parentImage.OS,
 		Architecture: parentImage.Architecture,
 		Variant:      parentImage.Variant,
@@ -98,7 +98,15 @@ func (b *Builder) exportImage(state *dispatchState, layer builder.RWLayer, paren
 		return errors.Wrap(err, "failed to encode image config")
 	}
 
-	exportedImage, err := b.docker.CreateImage(config, state.imageID)
+	// when writing the new image's manifest, we now need to pass in the new layer's digest.
+	// before the containerd store work this was unnecessary since we get the layer id
+	// from the image's RootFS ChainID -- see:
+	// https://github.com/moby/moby/blob/8cf66ed7322fa885ef99c4c044fa23e1727301dc/image/store.go#L162
+	// however, with the containerd store we can't do this. An alternative implementation here
+	// without changing the signature would be to get the layer digest by walking the content store
+	// and filtering the objects to find the layer with the DiffID we want, but that has performance
+	// implications that should be called out/investigated
+	exportedImage, err := b.docker.CreateImage(ctx, config, state.imageID, newLayer.ContentStoreDigest())
 	if err != nil {
 		return errors.Wrapf(err, "failed to export image")
 	}
@@ -170,7 +178,7 @@ func (b *Builder) performCopy(ctx context.Context, req dispatchRequest, inst cop
 			return errors.Wrapf(err, "failed to copy files")
 		}
 	}
-	return b.exportImage(state, rwLayer, imageMount.Image(), runConfigWithCommentCmd)
+	return b.exportImage(ctx, state, rwLayer, imageMount.Image(), runConfigWithCommentCmd)
 }
 
 func createDestInfo(workingDir string, inst copyInstruction, rwLayer builder.RWLayer, platform string) (copyInfo, error) {

builder/dockerfile/internals_test.go

@@ -1,6 +1,7 @@
 package dockerfile // import "github.com/docker/docker/builder/dockerfile"
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"runtime"
@@ -193,6 +194,7 @@ type MockROLayer struct {
 	diffID layer.DiffID
 }
 
+func (l *MockROLayer) ContentStoreDigest() digest.Digest    { return "" }
 func (l *MockROLayer) Release() error                       { return nil }
 func (l *MockROLayer) NewRWLayer() (builder.RWLayer, error) { return nil, nil }
 func (l *MockROLayer) DiffID() layer.DiffID                 { return l.diffID }
@@ -217,6 +219,6 @@ func TestExportImage(t *testing.T) {
 		imageSources: getMockImageSource(nil, nil, nil),
 		docker:       getMockBuildBackend(),
 	}
-	err := b.exportImage(ds, layer, parentImage, runConfig)
+	err := b.exportImage(context.TODO(), ds, layer, parentImage, runConfig)
 	assert.NilError(t, err)
 }

builder/dockerfile/mockbackend_test.go

@@ -13,6 +13,7 @@ import (
 	containerpkg "github.com/docker/docker/container"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/layer"
+	"github.com/opencontainers/go-digest"
 )
 
 // MockBackend implements the builder.Backend interface for unit testing
@@ -80,7 +81,7 @@ func (m *MockBackend) MakeImageCache(ctx context.Context, cacheFrom []string) (b
 	return nil, nil
 }
 
-func (m *MockBackend) CreateImage(config []byte, parent string) (builder.Image, error) {
+func (m *MockBackend) CreateImage(ctx context.Context, config []byte, parent string, layerDigest digest.Digest) (builder.Image, error) {
 	return &mockImage{id: "test"}, nil
 }
 
@@ -119,6 +120,10 @@ func (mic *mockImageCache) GetCache(parentID string, cfg *container.Config) (str
 
 type mockLayer struct{}
 
+func (l *mockLayer) ContentStoreDigest() digest.Digest {
+	return ""
+}
+
 func (l *mockLayer) Release() error {
 	return nil
 }

builder/remotecontext/tarsum.go

@@ -15,7 +15,9 @@ type hashed interface {
 	Digest() digest.Digest
 }
 
-// CachableSource is a source that contains cache records for its contents
+// CachableSource is a source that contains cache records for its contents.
+//
+// Deprecated: this type was used for the experimental "stream" support for the classic builder, which is no longer supported.
 type CachableSource struct {
 	mu   sync.Mutex
 	root string
@@ -23,7 +25,9 @@ type CachableSource struct {
 	txn  *iradix.Txn
 }
 
-// NewCachableSource creates new CachableSource
+// NewCachableSource creates new CachableSource.
+//
+// Deprecated: this type was used for the experimental "stream" support for the classic builder, which is no longer supported.
 func NewCachableSource(root string) *CachableSource {
 	ts := &CachableSource{
 		tree: iradix.New(),

builder/remotecontext/tarsum_test.go

@@ -17,8 +17,11 @@ const (
 	contents = "contents test"
 )
 
-func init() {
-	reexec.Init()
+func TestMain(m *testing.M) {
+	if reexec.Init() {
+		return
+	}
+	os.Exit(m.Run())
 }
 
 func TestCloseRootDirectory(t *testing.T) {

cli/debug/debug_test.go

@@ -14,20 +14,20 @@ func TestEnable(t *testing.T) {
 	}()
 	Enable()
 	if os.Getenv("DEBUG") != "1" {
-		t.Fatalf("expected DEBUG=1, got %s\n", os.Getenv("DEBUG"))
+		t.Fatalf("expected DEBUG=1, got %s", os.Getenv("DEBUG"))
 	}
 	if logrus.GetLevel() != logrus.DebugLevel {
-		t.Fatalf("expected log level %v, got %v\n", logrus.DebugLevel, logrus.GetLevel())
+		t.Fatalf("expected log level %v, got %v", logrus.DebugLevel, logrus.GetLevel())
 	}
 }
 
 func TestDisable(t *testing.T) {
 	Disable()
 	if os.Getenv("DEBUG") != "" {
-		t.Fatalf("expected DEBUG=\"\", got %s\n", os.Getenv("DEBUG"))
+		t.Fatalf(`expected DEBUG="", got %s`, os.Getenv("DEBUG"))
 	}
 	if logrus.GetLevel() != logrus.InfoLevel {
-		t.Fatalf("expected log level %v, got %v\n", logrus.InfoLevel, logrus.GetLevel())
+		t.Fatalf("expected log level %v, got %v", logrus.InfoLevel, logrus.GetLevel())
 	}
 }
 

client/client.go

@@ -56,6 +56,36 @@ import (
 	"github.com/pkg/errors"
 )
 
+// DummyHost is a hostname used for local communication.
+//
+// It acts as a valid formatted hostname for local connections (such as "unix://"
+// or "npipe://") which do not require a hostname. It should never be resolved,
+// but uses the special-purpose ".localhost" TLD (as defined in [RFC 2606, Section 2]
+// and [RFC 6761, Section 6.3]).
+//
+// [RFC 7230, Section 5.4] defines that an empty header must be used for such
+// cases:
+//
+//	If the authority component is missing or undefined for the target URI,
+//	then a client MUST send a Host header field with an empty field-value.
+//
+// However, [Go stdlib] enforces the semantics of HTTP(S) over TCP, does not
+// allow an empty header to be used, and requires req.URL.Scheme to be either
+// "http" or "https".
+//
+// For further details, refer to:
+//
+//   - https://github.com/docker/engine-api/issues/189
+//   - https://github.com/golang/go/issues/13624
+//   - https://github.com/golang/go/issues/61076
+//   - https://github.com/moby/moby/issues/45935
+//
+// [RFC 2606, Section 2]: https://www.rfc-editor.org/rfc/rfc2606.html#section-2
+// [RFC 6761, Section 6.3]: https://www.rfc-editor.org/rfc/rfc6761#section-6.3
+// [RFC 7230, Section 5.4]: https://datatracker.ietf.org/doc/html/rfc7230#section-5.4
+// [Go stdlib]: https://github.com/golang/go/blob/6244b1946bc2101b01955468f1be502dbadd6807/src/net/http/transport.go#L558-L569
+const DummyHost = "api.moby.localhost"
+
 // ErrRedirect is the error returned by checkRedirect when the request is non-GET.
 var ErrRedirect = errors.New("unexpected redirect in response")
 

client/container_create.go

@@ -9,7 +9,7 @@ import (
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/api/types/versions"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 type configWrapper struct {
@@ -20,7 +20,7 @@ type configWrapper struct {
 
 // ContainerCreate creates a new container based on the given configuration.
 // It can be associated with a name, but it's not mandatory.
-func (cli *Client) ContainerCreate(ctx context.Context, config *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, platform *specs.Platform, containerName string) (container.CreateResponse, error) {
+func (cli *Client) ContainerCreate(ctx context.Context, config *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, platform *ocispec.Platform, containerName string) (container.CreateResponse, error) {
 	var response container.CreateResponse
 
 	if err := cli.NewVersionError("1.25", "stop timeout"); config != nil && config.StopTimeout != nil && err != nil {
@@ -75,7 +75,7 @@ func (cli *Client) ContainerCreate(ctx context.Context, config *container.Config
 // Similar to containerd's platforms.Format(), but does allow components to be
 // omitted (e.g. pass "architecture" only, without "os":
 // https://github.com/containerd/containerd/blob/v1.5.2/platforms/platforms.go#L243-L263
-func formatPlatform(platform *specs.Platform) string {
+func formatPlatform(platform *ocispec.Platform) string {
 	if platform == nil {
 		return ""
 	}

client/hijack.go

@@ -23,14 +23,10 @@ func (cli *Client) postHijacked(ctx context.Context, path string, query url.Valu
 	if err != nil {
 		return types.HijackedResponse{}, err
 	}
-
-	apiPath := cli.getAPIPath(ctx, path, query)
-	req, err := http.NewRequest(http.MethodPost, apiPath, bodyEncoded)
+	req, err := cli.buildRequest(http.MethodPost, cli.getAPIPath(ctx, path, query), bodyEncoded, headers)
 	if err != nil {
 		return types.HijackedResponse{}, err
 	}
-	req = cli.addHeaders(req, headers)
-
 	conn, mediaType, err := cli.setupHijackConn(ctx, req, "tcp")
 	if err != nil {
 		return types.HijackedResponse{}, err
@@ -64,7 +60,6 @@ func fallbackDial(proto, addr string, tlsConfig *tls.Config) (net.Conn, error) {
 }
 
 func (cli *Client) setupHijackConn(ctx context.Context, req *http.Request, proto string) (net.Conn, string, error) {
-	req.Host = cli.addr
 	req.Header.Set("Connection", "Upgrade")
 	req.Header.Set("Upgrade", proto)
 
@@ -80,8 +75,8 @@ func (cli *Client) setupHijackConn(ctx context.Context, req *http.Request, proto
 	// state. Setting TCP KeepAlive on the socket connection will prohibit
 	// ECONNTIMEOUT unless the socket connection truly is broken
 	if tcpConn, ok := conn.(*net.TCPConn); ok {
-		tcpConn.SetKeepAlive(true)
-		tcpConn.SetKeepAlivePeriod(30 * time.Second)
+		_ = tcpConn.SetKeepAlive(true)
+		_ = tcpConn.SetKeepAlivePeriod(30 * time.Second)
 	}
 
 	clientconn := httputil.NewClientConn(conn, nil)
@@ -96,7 +91,7 @@ func (cli *Client) setupHijackConn(ctx context.Context, req *http.Request, proto
 			return nil, "", err
 		}
 		if resp.StatusCode != http.StatusSwitchingProtocols {
-			resp.Body.Close()
+			_ = resp.Body.Close()
 			return nil, "", fmt.Errorf("unable to upgrade to %s, received %d", proto, resp.StatusCode)
 		}
 	}

client/image_tag_test.go

@@ -42,7 +42,7 @@ func TestImageTagInvalidSourceImageName(t *testing.T) {
 	}
 
 	err := client.ImageTag(context.Background(), "invalid_source_image_name_", "repo:tag")
-	if err == nil || err.Error() != "Error parsing reference: \"invalid_source_image_name_\" is not a valid repository/tag: invalid reference format" {
+	if err == nil || err.Error() != `Error parsing reference: "invalid_source_image_name_" is not a valid repository/tag: invalid reference format` {
 		t.Fatalf("expected Parsing Reference Error, got %v", err)
 	}
 }

client/interface.go

@@ -15,7 +15,7 @@ import (
 	"github.com/docker/docker/api/types/registry"
 	"github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/api/types/volume"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // CommonAPIClient is the common methods between stable and experimental versions of APIClient.
@@ -47,7 +47,7 @@ type CommonAPIClient interface {
 type ContainerAPIClient interface {
 	ContainerAttach(ctx context.Context, container string, options types.ContainerAttachOptions) (types.HijackedResponse, error)
 	ContainerCommit(ctx context.Context, container string, options types.ContainerCommitOptions) (types.IDResponse, error)
-	ContainerCreate(ctx context.Context, config *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, platform *specs.Platform, containerName string) (container.CreateResponse, error)
+	ContainerCreate(ctx context.Context, config *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, platform *ocispec.Platform, containerName string) (container.CreateResponse, error)
 	ContainerDiff(ctx context.Context, container string) ([]container.FilesystemChange, error)
 	ContainerExecAttach(ctx context.Context, execID string, config types.ExecStartCheck) (types.HijackedResponse, error)
 	ContainerExecCreate(ctx context.Context, container string, config types.ExecConfig) (types.IDResponse, error)

client/request.go

@@ -96,16 +96,14 @@ func (cli *Client) buildRequest(method, path string, body io.Reader, headers hea
 		return nil, err
 	}
 	req = cli.addHeaders(req, headers)
+	req.URL.Scheme = cli.scheme
+	req.URL.Host = cli.addr
 
 	if cli.proto == "unix" || cli.proto == "npipe" {
-		// For local communications, it doesn't matter what the host is. We just
-		// need a valid and meaningful host name. (See #189)
-		req.Host = "docker"
+		// Override host header for non-tcp connections.
+		req.Host = DummyHost
 	}
 
-	req.URL.Host = cli.addr
-	req.URL.Scheme = cli.scheme
-
 	if expectedPayload && req.Header.Get("Content-Type") == "" {
 		req.Header.Set("Content-Type", "text/plain")
 	}

client/request_test.go

@@ -28,24 +28,24 @@ func TestSetHostHeader(t *testing.T) {
 		expectedURLHost string
 	}{
 		{
-			"unix:///var/run/docker.sock",
-			"docker",
-			"/var/run/docker.sock",
+			host:            "unix:///var/run/docker.sock",
+			expectedHost:    DummyHost,
+			expectedURLHost: "/var/run/docker.sock",
 		},
 		{
-			"npipe:////./pipe/docker_engine",
-			"docker",
-			"//./pipe/docker_engine",
+			host:            "npipe:////./pipe/docker_engine",
+			expectedHost:    DummyHost,
+			expectedURLHost: "//./pipe/docker_engine",
 		},
 		{
-			"tcp://0.0.0.0:4243",
-			"",
-			"0.0.0.0:4243",
+			host:            "tcp://0.0.0.0:4243",
+			expectedHost:    "",
+			expectedURLHost: "0.0.0.0:4243",
 		},
 		{
-			"tcp://localhost:4243",
-			"",
-			"localhost:4243",
+			host:            "tcp://localhost:4243",
+			expectedHost:    "",
+			expectedURLHost: "localhost:4243",
 		},
 	}
 

client/service_create_test.go

@@ -15,7 +15,7 @@ import (
 	"github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/errdefs"
 	"github.com/opencontainers/go-digest"
-	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"gotest.tools/v3/assert"
 	is "gotest.tools/v3/assert/cmp"
 )
@@ -91,10 +91,10 @@ func TestServiceCreateCompatiblePlatforms(t *testing.T) {
 				}, nil
 			} else if strings.HasPrefix(req.URL.Path, "/v1.30/distribution/") {
 				b, err := json.Marshal(registrytypes.DistributionInspect{
-					Descriptor: v1.Descriptor{
+					Descriptor: ocispec.Descriptor{
 						Digest: "sha256:c0537ff6a5218ef531ece93d4984efc99bbf3f7497c0a7726c88e2bb7584dc96",
 					},
-					Platforms: []v1.Platform{
+					Platforms: []ocispec.Platform{
 						{
 							Architecture: "amd64",
 							OS:           "linux",
@@ -171,7 +171,7 @@ func TestServiceCreateDigestPinning(t *testing.T) {
 			} else if strings.HasPrefix(req.URL.Path, "/v1.30/distribution/") {
 				// resolvable images
 				b, err := json.Marshal(registrytypes.DistributionInspect{
-					Descriptor: v1.Descriptor{
+					Descriptor: ocispec.Descriptor{
 						Digest: digest.Digest(dgst),
 					},
 				})

cmd/dockerd/daemon.go

@@ -351,6 +351,7 @@ func newRouterOptions(ctx context.Context, config *config.Config, d *daemon.Daem
 	bk, err := buildkit.New(ctx, buildkit.Opt{
 		SessionManager:      sm,
 		Root:                filepath.Join(config.Root, "buildkit"),
+		EngineID:            d.ID(),
 		Dist:                d.DistributionServices(),
 		ImageTagger:         d.ImageService(),
 		NetworkController:   d.NetworkController(),

contrib/apparmor/main.go

@@ -6,13 +6,9 @@ import (
 	"os"
 	"path"
 	"text/template"
-
-	"github.com/docker/docker/pkg/aaparser"
 )
 
-type profileData struct {
-	Version int
-}
+type profileData struct{}
 
 func main() {
 	if len(os.Args) < 2 {
@@ -22,15 +18,6 @@ func main() {
 	// parse the arg
 	apparmorProfilePath := os.Args[1]
 
-	version, err := aaparser.GetVersion()
-	if err != nil {
-		log.Fatal(err)
-	}
-	data := profileData{
-		Version: version,
-	}
-	fmt.Printf("apparmor_parser is of version %+v\n", data)
-
 	// parse the template
 	compiled, err := template.New("apparmor_profile").Parse(dockerProfileTemplate)
 	if err != nil {
@@ -48,6 +35,7 @@ func main() {
 	}
 	defer f.Close()
 
+	data := profileData{}
 	if err := compiled.Execute(f, data); err != nil {
 		log.Fatalf("executing template failed: %v", err)
 	}

contrib/apparmor/template.go

@@ -149,9 +149,7 @@ profile /usr/bin/docker (attach_disconnected, complain) {
   }
   # xz works via pipes, so we do not need access to the filesystem.
   profile /usr/bin/xz (complain) {
-{{if ge .Version 209000}}
     signal (receive) peer=/usr/bin/docker,
-{{end}}
     /etc/ld.so.cache r,
     /lib/** rm,
     /usr/bin/xz rm,

contrib/busybox/Dockerfile

@@ -10,10 +10,10 @@
 # To publish: Needs someone with publishing rights
 ARG WINDOWS_BASE_IMAGE=mcr.microsoft.com/windows/servercore
 ARG WINDOWS_BASE_IMAGE_TAG=ltsc2022
-ARG BUSYBOX_VERSION=FRP-3329-gcf0fa4d13
+ARG BUSYBOX_VERSION=FRP-5007-g82accfc19
 
 # Checksum taken from https://frippery.org/files/busybox/SHA256SUM
-ARG BUSYBOX_SHA256SUM=bfaeb88638e580fc522a68e69072e305308f9747563e51fa085eec60ca39a5ae
+ARG BUSYBOX_SHA256SUM=2d6fff0b2de5c034c92990d696c0d85a677b8a75931fa1ec30694fbf1f1df5c9
 
 FROM ${WINDOWS_BASE_IMAGE}:${WINDOWS_BASE_IMAGE_TAG}
 RUN mkdir C:\tmp && mkdir C:\bin

contrib/check-config.sh

@@ -351,7 +351,7 @@ echo "  - \"$(wrap_color 'overlay' blue)\":"
 check_flags VXLAN BRIDGE_VLAN_FILTERING | sed 's/^/    /'
 echo '      Optional (for encrypted networks):'
 check_flags CRYPTO CRYPTO_AEAD CRYPTO_GCM CRYPTO_SEQIV CRYPTO_GHASH \
-	XFRM XFRM_USER XFRM_ALGO INET_ESP | sed 's/^/      /'
+	XFRM XFRM_USER XFRM_ALGO INET_ESP NETFILTER_XT_MATCH_BPF | sed 's/^/      /'
 if [ "$kernelMajor" -lt 5 ] || [ "$kernelMajor" -eq 5 -a "$kernelMinor" -le 3 ]; then
 	check_flags INET_XFRM_MODE_TRANSPORT | sed 's/^/      /'
 fi

contrib/init/sysvinit-debian/docker

@@ -44,14 +44,6 @@ if [ ! -x $DOCKERD ]; then
 	exit 1
 fi
 
-check_init() {
-	# see also init_is_upstart in /lib/lsb/init-functions (which isn't available in Ubuntu 12.04, or we'd use it directly)
-	if [ -x /sbin/initctl ] && /sbin/initctl version 2> /dev/null | grep -q upstart; then
-		log_failure_msg "$DOCKER_DESC is managed via upstart, try using service $BASE $1"
-		exit 1
-	fi
-}
-
 fail_unless_root() {
 	if [ "$(id -u)" != '0' ]; then
 		log_failure_msg "$DOCKER_DESC must be run as root"
@@ -59,37 +51,10 @@ fail_unless_root() {
 	fi
 }
 
-cgroupfs_mount() {
-	# see also https://github.com/tianon/cgroupfs-mount/blob/master/cgroupfs-mount
-	if grep -v '^#' /etc/fstab | grep -q cgroup \
-		|| [ ! -e /proc/cgroups ] \
-		|| [ ! -d /sys/fs/cgroup ]; then
-		return
-	fi
-	if ! mountpoint -q /sys/fs/cgroup; then
-		mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup
-	fi
-	(
-		cd /sys/fs/cgroup
-		for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do
-			mkdir -p $sys
-			if ! mountpoint -q $sys; then
-				if ! mount -n -t cgroup -o $sys cgroup $sys; then
-					rmdir $sys || true
-				fi
-			fi
-		done
-	)
-}
-
 case "$1" in
 	start)
-		check_init
-
 		fail_unless_root
 
-		cgroupfs_mount
-
 		touch "$DOCKER_LOGFILE"
 		chgrp docker "$DOCKER_LOGFILE"
 
@@ -117,7 +82,6 @@ case "$1" in
 		;;
 
 	stop)
-		check_init
 		fail_unless_root
 		if [ -f "$DOCKER_SSD_PIDFILE" ]; then
 			log_begin_msg "Stopping $DOCKER_DESC: $BASE"
@@ -129,7 +93,6 @@ case "$1" in
 		;;
 
 	restart)
-		check_init
 		fail_unless_root
 		docker_pid=$(cat "$DOCKER_SSD_PIDFILE" 2> /dev/null || true)
 		[ -n "$docker_pid" ] \
@@ -139,13 +102,11 @@ case "$1" in
 		;;
 
 	force-reload)
-		check_init
 		fail_unless_root
 		$0 restart
 		;;
 
 	status)
-		check_init
 		status_of_proc -p "$DOCKER_SSD_PIDFILE" "$DOCKERD" "$DOCKER_DESC"
 		;;
 

contrib/init/sysvinit-debian/docker.default

@@ -1,4 +1,4 @@
-# Docker Upstart and SysVinit configuration file
+# Docker SysVinit configuration file
 
 #
 # THIS FILE DOES NOT APPLY TO SYSTEMD

contrib/init/upstart/docker.conf

@@ -1,72 +0,0 @@
-description "Docker daemon"
-
-start on (filesystem and net-device-up IFACE!=lo)
-stop on runlevel [!2345]
-
-limit nofile 524288 1048576
-
-# Having non-zero limits causes performance problems due to accounting overhead
-# in the kernel. We recommend using cgroups to do container-local accounting.
-limit nproc unlimited unlimited
-
-respawn
-
-kill timeout 20
-
-pre-start script
-	# see also https://github.com/tianon/cgroupfs-mount/blob/master/cgroupfs-mount
-	if grep -v '^#' /etc/fstab | grep -q cgroup \
-		|| [ ! -e /proc/cgroups ] \
-		|| [ ! -d /sys/fs/cgroup ]; then
-		exit 0
-	fi
-	if ! mountpoint -q /sys/fs/cgroup; then
-		mount -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup
-	fi
-	(
-		cd /sys/fs/cgroup
-		for sys in $(awk '!/^#/ { if ($4 == 1) print $1 }' /proc/cgroups); do
-			mkdir -p $sys
-			if ! mountpoint -q $sys; then
-				if ! mount -n -t cgroup -o $sys cgroup $sys; then
-					rmdir $sys || true
-				fi
-			fi
-		done
-	)
-end script
-
-script
-	# modify these in /etc/default/$UPSTART_JOB (/etc/default/docker)
-	DOCKERD=/usr/bin/dockerd
-	DOCKER_OPTS=
-	if [ -f /etc/default/$UPSTART_JOB ]; then
-		. /etc/default/$UPSTART_JOB
-	fi
-	exec "$DOCKERD" $DOCKER_OPTS --raw-logs
-end script
-
-# Don't emit "started" event until docker.sock is ready.
-# See https://github.com/docker/docker/issues/6647
-post-start script
-	DOCKER_OPTS=
-	DOCKER_SOCKET=
-	if [ -f /etc/default/$UPSTART_JOB ]; then
-		. /etc/default/$UPSTART_JOB
-	fi
-
-	if ! printf "%s" "$DOCKER_OPTS" | grep -qE -e '-H|--host'; then
-		DOCKER_SOCKET=/var/run/docker.sock
-	else
-		DOCKER_SOCKET=$(printf "%s" "$DOCKER_OPTS" | grep -oP -e '(-H|--host)\W*unix://\K(\S+)' | sed 1q)
-	fi
-
-	if [ -n "$DOCKER_SOCKET" ]; then
-		while ! [ -e "$DOCKER_SOCKET" ]; do
-			initctl status $UPSTART_JOB | grep -qE "(stop|respawn)/" && exit 1
-			echo "Waiting for $DOCKER_SOCKET"
-			sleep 0.1
-		done
-		echo "$DOCKER_SOCKET is up"
-	fi
-end script

daemon/cluster/cluster.go

@@ -359,7 +359,7 @@ func (c *Cluster) errNoManager(st nodeState) error {
 		if st.err == errSwarmCertificatesExpired {
 			return errSwarmCertificatesExpired
 		}
-		return errors.WithStack(notAvailableError("This node is not a swarm manager. Use \"docker swarm init\" or \"docker swarm join\" to connect this node to swarm and try again."))
+		return errors.WithStack(notAvailableError(`This node is not a swarm manager. Use "docker swarm init" or "docker swarm join" to connect this node to swarm and try again.`))
 	}
 	if st.swarmNode.Manager() != nil {
 		return errors.WithStack(notAvailableError("This node is not a swarm manager. Manager is being prepared or has trouble connecting to the cluster."))

daemon/cluster/convert/node.go

@@ -58,13 +58,20 @@ func NodeFromGRPC(n swarmapi.Node) types.Node {
 		}
 		for _, csi := range n.Description.CSIInfo {
 			if csi != nil {
+				convertedInfo := types.NodeCSIInfo{
+					PluginName:        csi.PluginName,
+					NodeID:            csi.NodeID,
+					MaxVolumesPerNode: csi.MaxVolumesPerNode,
+				}
+
+				if csi.AccessibleTopology != nil {
+					convertedInfo.AccessibleTopology = &types.Topology{
+						Segments: csi.AccessibleTopology.Segments,
+					}
+				}
+
 				node.Description.CSIInfo = append(
-					node.Description.CSIInfo,
-					types.NodeCSIInfo{
-						PluginName:        csi.PluginName,
-						NodeID:            csi.NodeID,
-						MaxVolumesPerNode: csi.MaxVolumesPerNode,
-					},
+					node.Description.CSIInfo, convertedInfo,
 				)
 			}
 		}

daemon/cluster/convert/node_test.go

@@ -0,0 +1,60 @@
+package convert
+
+import (
+	"testing"
+
+	types "github.com/docker/docker/api/types/swarm"
+	swarmapi "github.com/moby/swarmkit/v2/api"
+	"gotest.tools/v3/assert"
+)
+
+// TestNodeCSIInfoFromGRPC tests that conversion of the NodeCSIInfo from the
+// gRPC to the Docker types is correct.
+func TestNodeCSIInfoFromGRPC(t *testing.T) {
+	node := &swarmapi.Node{
+		ID: "someID",
+		Description: &swarmapi.NodeDescription{
+			CSIInfo: []*swarmapi.NodeCSIInfo{
+				&swarmapi.NodeCSIInfo{
+					PluginName:        "plugin1",
+					NodeID:            "p1n1",
+					MaxVolumesPerNode: 1,
+				},
+				&swarmapi.NodeCSIInfo{
+					PluginName:        "plugin2",
+					NodeID:            "p2n1",
+					MaxVolumesPerNode: 2,
+					AccessibleTopology: &swarmapi.Topology{
+						Segments: map[string]string{
+							"a": "1",
+							"b": "2",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	expected := []types.NodeCSIInfo{
+		{
+			PluginName:        "plugin1",
+			NodeID:            "p1n1",
+			MaxVolumesPerNode: 1,
+		},
+		{
+			PluginName:        "plugin2",
+			NodeID:            "p2n1",
+			MaxVolumesPerNode: 2,
+			AccessibleTopology: &types.Topology{
+				Segments: map[string]string{
+					"a": "1",
+					"b": "2",
+				},
+			},
+		},
+	}
+
+	actual := NodeFromGRPC(*node)
+
+	assert.DeepEqual(t, actual.Description.CSIInfo, expected)
+}

daemon/cluster/errors.go

@@ -5,13 +5,13 @@ const (
 	errNoSwarm notAvailableError = "This node is not part of a swarm"
 
 	// errSwarmExists is returned on initialize or join request for a cluster that has already been activated
-	errSwarmExists notAvailableError = "This node is already part of a swarm. Use \"docker swarm leave\" to leave this swarm and join another one."
+	errSwarmExists notAvailableError = `This node is already part of a swarm. Use "docker swarm leave" to leave this swarm and join another one.`
 
 	// errSwarmJoinTimeoutReached is returned when cluster join could not complete before timeout was reached.
-	errSwarmJoinTimeoutReached notAvailableError = "Timeout was reached before node joined. The attempt to join the swarm will continue in the background. Use the \"docker info\" command to see the current swarm status of your node."
+	errSwarmJoinTimeoutReached notAvailableError = `Timeout was reached before node joined. The attempt to join the swarm will continue in the background. Use the "docker info" command to see the current swarm status of your node.`
 
 	// errSwarmLocked is returned if the swarm is encrypted and needs a key to unlock it.
-	errSwarmLocked notAvailableError = "Swarm is encrypted and needs to be unlocked before it can be used. Please use \"docker swarm unlock\" to unlock it."
+	errSwarmLocked notAvailableError = `Swarm is encrypted and needs to be unlocked before it can be used. Please use "docker swarm unlock" to unlock it.`
 
 	// errSwarmCertificatesExpired is returned if docker was not started for the whole validity period and they had no chance to renew automatically.
 	errSwarmCertificatesExpired notAvailableError = "Swarm certificates have expired. To replace them, leave the swarm and join again."

daemon/cluster/executor/backend.go

@@ -27,7 +27,7 @@ import (
 	"github.com/docker/docker/plugin"
 	volumeopts "github.com/docker/docker/volume/service/opts"
 	"github.com/moby/swarmkit/v2/agent/exec"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // Backend defines the executor component for a swarm agent.
@@ -75,7 +75,7 @@ type VolumeBackend interface {
 
 // ImageBackend is used by an executor to perform image operations
 type ImageBackend interface {
-	PullImage(ctx context.Context, image, tag string, platform *specs.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error
+	PullImage(ctx context.Context, image, tag string, platform *ocispec.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error
 	GetRepository(context.Context, reference.Named, *registry.AuthConfig) (distribution.Repository, error)
 	GetImage(ctx context.Context, refOrID string, options opts.GetImageOpts) (*image.Image, error)
 }

daemon/cluster/services.go

@@ -456,7 +456,7 @@ func (c *Cluster) ServiceLogs(ctx context.Context, selector *backend.LogSelector
 	} else {
 		t, err := strconv.Atoi(config.Tail)
 		if err != nil {
-			return nil, errors.New("tail value must be a positive integer or \"all\"")
+			return nil, errors.New(`tail value must be a positive integer or "all"`)
 		}
 		if t < 0 {
 			return nil, errors.New("negative tail values not supported")

daemon/containerd/cache.go

@@ -2,11 +2,92 @@ package containerd
 
 import (
 	"context"
+	"reflect"
+	"strings"
 
+	"github.com/docker/docker/api/types/container"
+	imagetype "github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/builder"
+	"github.com/docker/docker/image"
 )
 
 // MakeImageCache creates a stateful image cache.
 func (i *ImageService) MakeImageCache(ctx context.Context, cacheFrom []string) (builder.ImageCache, error) {
-	panic("not implemented")
+	images := []*image.Image{}
+	for _, c := range cacheFrom {
+		im, err := i.GetImage(ctx, c, imagetype.GetImageOpts{})
+		if err != nil {
+			return nil, err
+		}
+		images = append(images, im)
+	}
+	return &imageCache{images: images, c: i}, nil
+}
+
+type imageCache struct {
+	images []*image.Image
+	c      *ImageService
+}
+
+func (ic *imageCache) GetCache(parentID string, cfg *container.Config) (imageID string, err error) {
+	ctx := context.TODO()
+
+	if parentID == "" {
+		// TODO handle "parentless" image cache lookups ("FROM scratch")
+		return "", nil
+	}
+
+	parent, err := ic.c.GetImage(ctx, parentID, imagetype.GetImageOpts{})
+	if err != nil {
+		return "", err
+	}
+
+	for _, localCachedImage := range ic.images {
+		if isMatch(localCachedImage, parent, cfg) {
+			return localCachedImage.ID().String(), nil
+		}
+	}
+
+	children, err := ic.c.Children(ctx, parent.ID())
+	if err != nil {
+		return "", err
+	}
+
+	for _, children := range children {
+		childImage, err := ic.c.GetImage(ctx, children.String(), imagetype.GetImageOpts{})
+		if err != nil {
+			return "", err
+		}
+
+		if isMatch(childImage, parent, cfg) {
+			return children.String(), nil
+		}
+	}
+
+	return "", nil
+}
+
+// isMatch checks whether a given target can be used as cache for the given
+// parent image/config combination.
+// A target can only be an immediate child of the given parent image. For
+// a parent image with `n` history entries, a valid target must have `n+1`
+// entries and the extra entry must match the provided config
+func isMatch(target, parent *image.Image, cfg *container.Config) bool {
+	if target == nil || parent == nil || cfg == nil {
+		return false
+	}
+
+	if len(target.History) != len(parent.History)+1 ||
+		len(target.RootFS.DiffIDs) != len(parent.RootFS.DiffIDs)+1 {
+		return false
+	}
+
+	for i := range parent.History {
+		if !reflect.DeepEqual(parent.History[i], target.History[i]) {
+			return false
+		}
+	}
+
+	childCreatedBy := target.History[len(target.History)-1].CreatedBy
+	return childCreatedBy == strings.Join(cfg.Cmd, " ")
 }

daemon/containerd/handlers.go

@@ -9,16 +9,13 @@ import (
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
-// walkPresentChildren is a simple wrapper for containerdimages.Walk with
-// presentChildrenHandler wrapping a simple handler that only operates on
-// walked Descriptor and doesn't return any errror.
+// walkPresentChildren is a simple wrapper for containerdimages.Walk with presentChildrenHandler.
 // This is only a convenient helper to reduce boilerplate.
-func (i *ImageService) walkPresentChildren(ctx context.Context, target ocispec.Descriptor, f func(context.Context, ocispec.Descriptor)) error {
+func (i *ImageService) walkPresentChildren(ctx context.Context, target ocispec.Descriptor, f func(context.Context, ocispec.Descriptor) error) error {
 	store := i.client.ContentStore()
 	return containerdimages.Walk(ctx, presentChildrenHandler(store, containerdimages.HandlerFunc(
 		func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
-			f(ctx, desc)
-			return nil, nil
+			return nil, f(ctx, desc)
 		})), target)
 }
 

daemon/containerd/image.go

@@ -2,14 +2,13 @@ package containerd
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"regexp"
+	"sort"
 	"strconv"
 	"sync/atomic"
 	"time"
 
-	"github.com/containerd/containerd/content"
 	cerrdefs "github.com/containerd/containerd/errdefs"
 	containerdimages "github.com/containerd/containerd/images"
 	cplatforms "github.com/containerd/containerd/platforms"
@@ -33,7 +32,7 @@ var truncatedID = regexp.MustCompile(`^([a-f0-9]{4,64})$`)
 
 // GetImage returns an image corresponding to the image referred to by refOrID.
 func (i *ImageService) GetImage(ctx context.Context, refOrID string, options imagetype.GetImageOpts) (*image.Image, error) {
-	desc, err := i.resolveDescriptor(ctx, refOrID)
+	desc, err := i.resolveImage(ctx, refOrID)
 	if err != nil {
 		return nil, err
 	}
@@ -44,20 +43,31 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima
 	}
 
 	cs := i.client.ContentStore()
-	conf, err := containerdimages.Config(ctx, cs, desc, platform)
+
+	var presentImages []ocispec.Image
+	err = i.walkImageManifests(ctx, desc, func(img *ImageManifest) error {
+		conf, err := img.Config(ctx)
+		if err != nil {
+			return err
+		}
+		var ociimage ocispec.Image
+		if err := readConfig(ctx, cs, conf, &ociimage); err != nil {
+			return err
+		}
+		presentImages = append(presentImages, ociimage)
+		return nil
+	})
 	if err != nil {
 		return nil, err
 	}
-
-	imageConfigBytes, err := content.ReadBlob(ctx, cs, conf)
-	if err != nil {
-		return nil, err
+	if len(presentImages) == 0 {
+		return nil, errdefs.NotFound(errors.New("failed to find image manifest"))
 	}
 
-	var ociimage ocispec.Image
-	if err := json.Unmarshal(imageConfigBytes, &ociimage); err != nil {
-		return nil, err
-	}
+	sort.SliceStable(presentImages, func(i, j int) bool {
+		return platform.Less(presentImages[i].Platform, presentImages[j].Platform)
+	})
+	ociimage := presentImages[0]
 
 	rootfs := image.NewRootFS()
 	for _, id := range ociimage.RootFS.DiffIDs {
@@ -68,11 +78,31 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima
 		exposedPorts[nat.Port(k)] = v
 	}
 
-	img := image.NewImage(image.ID(desc.Digest))
+	derefTimeSafely := func(t *time.Time) time.Time {
+		if t != nil {
+			return *t
+		}
+		return time.Time{}
+	}
+
+	var imgHistory []image.History
+	for _, h := range ociimage.History {
+		imgHistory = append(imgHistory, image.History{
+			Created:    derefTimeSafely(h.Created),
+			Author:     h.Author,
+			CreatedBy:  h.CreatedBy,
+			Comment:    h.Comment,
+			EmptyLayer: h.EmptyLayer,
+		})
+	}
+
+	img := image.NewImage(image.ID(desc.Target.Digest))
 	img.V1Image = image.V1Image{
-		ID:           string(desc.Digest),
+		ID:           string(desc.Target.Digest),
 		OS:           ociimage.OS,
 		Architecture: ociimage.Architecture,
+		Created:      derefTimeSafely(ociimage.Created),
+		Variant:      ociimage.Variant,
 		Config: &containertypes.Config{
 			Entrypoint:   ociimage.Config.Entrypoint,
 			Env:          ociimage.Config.Env,
@@ -87,20 +117,21 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima
 	}
 
 	img.RootFS = rootfs
+	img.History = imgHistory
 
 	if options.Details {
 		lastUpdated := time.Unix(0, 0)
-		size, err := i.size(ctx, desc, platform)
+		size, err := i.size(ctx, desc.Target, platform)
 		if err != nil {
 			return nil, err
 		}
 
-		tagged, err := i.client.ImageService().List(ctx, "target.digest=="+desc.Digest.String())
+		tagged, err := i.client.ImageService().List(ctx, "target.digest=="+desc.Target.Digest.String())
 		if err != nil {
 			return nil, err
 		}
 
-		// Each image will result in 2 references (named and digested).
+		// Usually each image will result in 2 references (named and digested).
 		refs := make([]reference.Named, 0, len(tagged)*2)
 		for _, i := range tagged {
 			if i.UpdatedAt.After(lastUpdated) {
@@ -125,7 +156,12 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima
 			}
 			refs = append(refs, name)
 
-			digested, err := reference.WithDigest(reference.TrimNamed(name), desc.Digest)
+			if _, ok := name.(reference.Digested); ok {
+				// Image name already contains a digest, so no need to create a digested reference.
+				continue
+			}
+
+			digested, err := reference.WithDigest(reference.TrimNamed(name), desc.Target.Digest)
 			if err != nil {
 				// This could only happen if digest is invalid, but considering that
 				// we get it from the Descriptor it's highly unlikely.
@@ -244,6 +280,24 @@ func (i *ImageService) resolveImage(ctx context.Context, refOrID string) (contai
 			return containerdimages.Image{}, images.ErrImageDoesNotExist{Ref: parsed}
 		}
 
+		// If reference is both Named and Digested, make sure we don't match
+		// images with a different repository even if digest matches.
+		// For example, busybox@sha256:abcdef..., shouldn't match asdf@sha256:abcdef...
+		if parsedNamed, ok := parsed.(reference.Named); ok {
+			for _, img := range imgs {
+				imgNamed, err := reference.ParseNormalizedNamed(img.Name)
+				if err != nil {
+					logrus.WithError(err).WithField("image", img.Name).Warn("image with invalid name encountered")
+					continue
+				}
+
+				if parsedNamed.Name() == imgNamed.Name() {
+					return img, nil
+				}
+			}
+			return containerdimages.Image{}, images.ErrImageDoesNotExist{Ref: parsed}
+		}
+
 		return imgs[0], nil
 	}
 

daemon/containerd/image_builder.go

@@ -2,23 +2,512 @@ package containerd
 
 import (
 	"context"
-	"errors"
+	"fmt"
+	"io"
+	"os"
+	"runtime"
+	"time"
 
+	"github.com/containerd/containerd"
+	cerrdefs "github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/leases"
+	"github.com/containerd/containerd/mount"
+	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/containerd/rootfs"
+	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types/backend"
+	imagetypes "github.com/docker/docker/api/types/image"
+	"github.com/docker/docker/api/types/registry"
+	registrypkg "github.com/docker/docker/registry"
+
+	// "github.com/docker/docker/api/types/container"
+	containerdimages "github.com/containerd/containerd/images"
+	"github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/errdefs"
+	dimage "github.com/docker/docker/image"
+	"github.com/docker/docker/layer"
+	"github.com/docker/docker/pkg/progress"
+	"github.com/docker/docker/pkg/streamformatter"
+	"github.com/docker/docker/pkg/stringid"
+	"github.com/docker/docker/pkg/system"
+	"github.com/opencontainers/go-digest"
+	"github.com/opencontainers/image-spec/identity"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/sirupsen/logrus"
 )
 
 // GetImageAndReleasableLayer returns an image and releaseable layer for a
 // reference or ID. Every call to GetImageAndReleasableLayer MUST call
 // releasableLayer.Release() to prevent leaking of layers.
 func (i *ImageService) GetImageAndReleasableLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ROLayer, error) {
-	return nil, nil, errdefs.NotImplemented(errors.New("not implemented"))
+	if refOrID == "" { // from SCRATCH
+		os := runtime.GOOS
+		if runtime.GOOS == "windows" {
+			os = "linux"
+		}
+		if opts.Platform != nil {
+			os = opts.Platform.OS
+		}
+		if !system.IsOSSupported(os) {
+			return nil, nil, system.ErrNotSupportedOperatingSystem
+		}
+		return nil, &rolayer{
+			key:         "",
+			c:           i.client,
+			snapshotter: i.snapshotter,
+			diffID:      "",
+			root:        "",
+		}, nil
+	}
+
+	if opts.PullOption != backend.PullOptionForcePull {
+		// TODO(laurazard): same as below
+		img, err := i.GetImage(ctx, refOrID, image.GetImageOpts{Platform: opts.Platform})
+		if err != nil && opts.PullOption == backend.PullOptionNoPull {
+			return nil, nil, err
+		}
+		imgDesc, err := i.resolveDescriptor(ctx, refOrID)
+		if err != nil && !errdefs.IsNotFound(err) {
+			return nil, nil, err
+		}
+		if img != nil {
+			if !system.IsOSSupported(img.OperatingSystem()) {
+				return nil, nil, system.ErrNotSupportedOperatingSystem
+			}
+
+			layer, err := newROLayerForImage(ctx, &imgDesc, i, opts, refOrID, opts.Platform)
+			if err != nil {
+				return nil, nil, err
+			}
+
+			return img, layer, nil
+		}
+	}
+
+	ctx, _, err := i.client.WithLease(ctx, leases.WithRandomID(), leases.WithExpiration(1*time.Hour))
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to create lease for commit: %w", err)
+	}
+
+	// TODO(laurazard): do we really need a new method here to pull the image?
+	imgDesc, err := i.pullForBuilder(ctx, refOrID, opts.AuthConfig, opts.Output, opts.Platform)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// TODO(laurazard): pullForBuilder should return whatever we
+	// need here instead of having to go and get it again
+	img, err := i.GetImage(ctx, refOrID, imagetypes.GetImageOpts{
+		Platform: opts.Platform,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
+	layer, err := newROLayerForImage(ctx, imgDesc, i, opts, refOrID, opts.Platform)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return img, layer, nil
+}
+
+func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConfigs map[string]registry.AuthConfig, output io.Writer, platform *ocispec.Platform) (*ocispec.Descriptor, error) {
+	ref, err := reference.ParseNormalizedNamed(name)
+	if err != nil {
+		return nil, err
+	}
+	taggedRef := reference.TagNameOnly(ref)
+
+	pullRegistryAuth := &registry.AuthConfig{}
+	if len(authConfigs) > 0 {
+		// The request came with a full auth config, use it
+		repoInfo, err := i.registryService.ResolveRepository(ref)
+		if err != nil {
+			return nil, err
+		}
+
+		resolvedConfig := registrypkg.ResolveAuthConfig(authConfigs, repoInfo.Index)
+		pullRegistryAuth = &resolvedConfig
+	}
+
+	if err := i.PullImage(ctx, ref.Name(), taggedRef.(reference.NamedTagged).Tag(), platform, nil, pullRegistryAuth, output); err != nil {
+		return nil, err
+	}
+
+	img, err := i.GetImage(ctx, name, imagetypes.GetImageOpts{Platform: platform})
+	if err != nil {
+		if errdefs.IsNotFound(err) && img != nil && platform != nil {
+			imgPlat := ocispec.Platform{
+				OS:           img.OS,
+				Architecture: img.BaseImgArch(),
+				Variant:      img.BaseImgVariant(),
+			}
+
+			p := *platform
+			if !platforms.Only(p).Match(imgPlat) {
+				po := streamformatter.NewJSONProgressOutput(output, false)
+				progress.Messagef(po, "", `
+WARNING: Pulled image with specified platform (%s), but the resulting image's configured platform (%s) does not match.
+This is most likely caused by a bug in the build system that created the fetched image (%s).
+Please notify the image author to correct the configuration.`,
+					platforms.Format(p), platforms.Format(imgPlat), name,
+				)
+				logrus.WithError(err).WithField("image", name).Warn("Ignoring error about platform mismatch where the manifest list points to an image whose configuration does not match the platform in the manifest.")
+			}
+		} else {
+			return nil, err
+		}
+	}
+
+	if !system.IsOSSupported(img.OperatingSystem()) {
+		return nil, system.ErrNotSupportedOperatingSystem
+	}
+
+	imgDesc, err := i.resolveDescriptor(ctx, name)
+	if err != nil {
+		return nil, err
+	}
+
+	return &imgDesc, err
+}
+
+func newROLayerForImage(ctx context.Context, imgDesc *ocispec.Descriptor, i *ImageService, opts backend.GetImageAndLayerOptions, refOrID string, platform *ocispec.Platform) (builder.ROLayer, error) {
+	if imgDesc == nil {
+		return nil, fmt.Errorf("can't make an RO layer for a nil image :'(")
+	}
+
+	platMatcher := platforms.Default()
+	if platform != nil {
+		platMatcher = platforms.Only(*platform)
+	}
+
+	// this needs it's own context + lease so that it doesn't get cleaned before we're ready
+	confDesc, err := containerdimages.Config(ctx, i.client.ContentStore(), *imgDesc, platMatcher)
+	if err != nil {
+		return nil, err
+	}
+
+	diffIDs, err := containerdimages.RootFS(ctx, i.client.ContentStore(), confDesc)
+	if err != nil {
+		return nil, err
+	}
+	parent := identity.ChainID(diffIDs).String()
+
+	s := i.client.SnapshotService(i.snapshotter)
+	key := stringid.GenerateRandomID()
+	ctx, _, err = i.client.WithLease(ctx, leases.WithRandomID(), leases.WithExpiration(1*time.Hour))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create lease for commit: %w", err)
+	}
+	mounts, err := s.View(ctx, key, parent)
+	if err != nil {
+		return nil, err
+	}
+
+	tempMountLocation := os.TempDir()
+	root, err := os.MkdirTemp(tempMountLocation, "rootfs-mount")
+	if err != nil {
+		return nil, err
+	}
+
+	if err := mount.All(mounts, root); err != nil {
+		return nil, err
+	}
+
+	return &rolayer{
+		key:                key,
+		c:                  i.client,
+		snapshotter:        i.snapshotter,
+		diffID:             digest.Digest(parent),
+		root:               root,
+		contentStoreDigest: "",
+	}, nil
+}
+
+type rolayer struct {
+	key                string
+	c                  *containerd.Client
+	snapshotter        string
+	diffID             digest.Digest
+	root               string
+	contentStoreDigest digest.Digest
+}
+
+func (rl *rolayer) ContentStoreDigest() digest.Digest {
+	return rl.contentStoreDigest
+}
+
+func (rl *rolayer) DiffID() layer.DiffID {
+	if rl.diffID == "" {
+		return layer.DigestSHA256EmptyTar
+	}
+	return layer.DiffID(rl.diffID)
+}
+
+func (rl *rolayer) Release() error {
+	snapshotter := rl.c.SnapshotService(rl.snapshotter)
+	err := snapshotter.Remove(context.TODO(), rl.key)
+	if err != nil && !cerrdefs.IsNotFound(err) {
+		return err
+	}
+
+	if rl.root == "" { // nothing to release
+		return nil
+	}
+	if err := mount.UnmountAll(rl.root, 0); err != nil {
+		logrus.WithError(err).WithField("root", rl.root).Error("failed to unmount ROLayer")
+		return err
+	}
+	if err := os.Remove(rl.root); err != nil {
+		logrus.WithError(err).WithField("dir", rl.root).Error("failed to remove mount temp dir")
+		return err
+	}
+	rl.root = ""
+	return nil
+}
+
+// NewRWLayer creates a new read-write layer for the builder
+func (rl *rolayer) NewRWLayer() (builder.RWLayer, error) {
+	snapshotter := rl.c.SnapshotService(rl.snapshotter)
+
+	// we need this here for the prepared snapshots or
+	// we'll have racy behaviour where sometimes they
+	// will get GC'd before we commit/use them
+	ctx, _, err := rl.c.WithLease(context.TODO(), leases.WithRandomID(), leases.WithExpiration(1*time.Hour))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create lease for commit: %w", err)
+	}
+
+	key := stringid.GenerateRandomID()
+	mounts, err := snapshotter.Prepare(ctx, key, rl.diffID.String())
+	if err != nil {
+		return nil, err
+	}
+
+	root, err := os.MkdirTemp(os.TempDir(), "rootfs-mount")
+	if err != nil {
+		return nil, err
+	}
+	if err := mount.All(mounts, root); err != nil {
+		return nil, err
+	}
+
+	return &rwlayer{
+		key:         key,
+		parent:      rl.key,
+		c:           rl.c,
+		snapshotter: rl.snapshotter,
+		root:        root,
+	}, nil
+}
+
+type rwlayer struct {
+	key         string
+	parent      string
+	c           *containerd.Client
+	snapshotter string
+	root        string
+}
+
+func (rw *rwlayer) Root() string {
+	return rw.root
+}
+
+func (rw *rwlayer) Commit() (builder.ROLayer, error) {
+	// we need this here for the prepared snapshots or
+	// we'll have racy behaviour where sometimes they
+	// will get GC'd before we commit/use them
+	ctx, _, err := rw.c.WithLease(context.TODO(), leases.WithRandomID(), leases.WithExpiration(1*time.Hour))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create lease for commit: %w", err)
+	}
+	snapshotter := rw.c.SnapshotService(rw.snapshotter)
+
+	key := stringid.GenerateRandomID()
+	err = snapshotter.Commit(ctx, key, rw.key)
+	if err != nil && !cerrdefs.IsAlreadyExists(err) {
+		return nil, err
+	}
+
+	differ := rw.c.DiffService()
+	desc, err := rootfs.CreateDiff(ctx, key, snapshotter, differ)
+	if err != nil {
+		return nil, err
+	}
+	info, err := rw.c.ContentStore().Info(ctx, desc.Digest)
+	if err != nil {
+		return nil, err
+	}
+	diffIDStr, ok := info.Labels["containerd.io/uncompressed"]
+	if !ok {
+		return nil, fmt.Errorf("invalid differ response with no diffID")
+	}
+	diffID, err := digest.Parse(diffIDStr)
+	if err != nil {
+		return nil, err
+	}
+
+	return &rolayer{
+		key:                key,
+		c:                  rw.c,
+		snapshotter:        rw.snapshotter,
+		diffID:             diffID,
+		root:               "",
+		contentStoreDigest: desc.Digest,
+	}, nil
+}
+
+func (rw *rwlayer) Release() error {
+	snapshotter := rw.c.SnapshotService(rw.snapshotter)
+	err := snapshotter.Remove(context.TODO(), rw.key)
+	if err != nil && !cerrdefs.IsNotFound(err) {
+		return err
+	}
+
+	if rw.root == "" { // nothing to release
+		return nil
+	}
+	if err := mount.UnmountAll(rw.root, 0); err != nil {
+		logrus.WithError(err).WithField("root", rw.root).Error("failed to unmount ROLayer")
+		return err
+	}
+	if err := os.Remove(rw.root); err != nil {
+		logrus.WithError(err).WithField("dir", rw.root).Error("failed to remove mount temp dir")
+		return err
+	}
+	rw.root = ""
+	return nil
 }
 
 // CreateImage creates a new image by adding a config and ID to the image store.
 // This is similar to LoadImage() except that it receives JSON encoded bytes of
 // an image instead of a tar archive.
-func (i *ImageService) CreateImage(config []byte, parent string) (builder.Image, error) {
-	return nil, errdefs.NotImplemented(errors.New("not implemented"))
+func (i *ImageService) CreateImage(ctx context.Context, config []byte, parent string, layerDigest digest.Digest) (builder.Image, error) {
+	imgToCreate, err := dimage.NewFromJSON(config)
+	if err != nil {
+		return nil, err
+	}
+
+	rootfs := ocispec.RootFS{
+		Type:    imgToCreate.RootFS.Type,
+		DiffIDs: []digest.Digest{},
+	}
+	for _, diffId := range imgToCreate.RootFS.DiffIDs {
+		rootfs.DiffIDs = append(rootfs.DiffIDs, digest.Digest(diffId))
+	}
+	exposedPorts := make(map[string]struct{}, len(imgToCreate.Config.ExposedPorts))
+	for k, v := range imgToCreate.Config.ExposedPorts {
+		exposedPorts[string(k)] = v
+	}
+
+	var ociHistory []ocispec.History
+	for _, history := range imgToCreate.History {
+		created := history.Created
+		ociHistory = append(ociHistory, ocispec.History{
+			Created:    &created,
+			CreatedBy:  history.CreatedBy,
+			Author:     history.Author,
+			Comment:    history.Comment,
+			EmptyLayer: history.EmptyLayer,
+		})
+	}
+
+	// make an ocispec.Image from the docker/image.Image
+	ociImgToCreate := ocispec.Image{
+		Created: &imgToCreate.Created,
+		Author:  imgToCreate.Author,
+		Platform: ocispec.Platform{
+			Architecture: imgToCreate.Architecture,
+			Variant:      imgToCreate.Variant,
+			OS:           imgToCreate.OS,
+			OSVersion:    imgToCreate.OSVersion,
+			OSFeatures:   imgToCreate.OSFeatures,
+		},
+		Config: ocispec.ImageConfig{
+			User:         imgToCreate.Config.User,
+			ExposedPorts: exposedPorts,
+			Env:          imgToCreate.Config.Env,
+			Entrypoint:   imgToCreate.Config.Entrypoint,
+			Cmd:          imgToCreate.Config.Cmd,
+			Volumes:      imgToCreate.Config.Volumes,
+			WorkingDir:   imgToCreate.Config.WorkingDir,
+			Labels:       imgToCreate.Config.Labels,
+			StopSignal:   imgToCreate.Config.StopSignal,
+		},
+		RootFS:  rootfs,
+		History: ociHistory,
+	}
+
+	var layers []ocispec.Descriptor
+	// if the image has a parent, we need to start with the parents layers descriptors
+	if parent != "" {
+		parentDesc, err := i.resolveDescriptor(ctx, parent)
+		if err != nil {
+			return nil, err
+		}
+		parentImageManifest, err := containerdimages.Manifest(ctx, i.client.ContentStore(), parentDesc, platforms.Default())
+		if err != nil {
+			return nil, err
+		}
+
+		layers = parentImageManifest.Layers
+	}
+
+	// get the info for the new layers
+	info, err := i.client.ContentStore().Info(ctx, layerDigest)
+	if err != nil {
+		return nil, err
+	}
+
+	// append the new layer descriptor
+	layers = append(layers,
+		ocispec.Descriptor{
+			MediaType: containerdimages.MediaTypeDockerSchema2LayerGzip,
+			Digest:    layerDigest,
+			Size:      info.Size,
+		},
+	)
+
+	// necessary to prevent the contents from being GC'd
+	// between writing them here and creating an image
+	ctx, done, err := i.client.WithLease(ctx, leases.WithRandomID(), leases.WithExpiration(1*time.Hour))
+	if err != nil {
+		return nil, err
+	}
+	defer done(ctx)
+
+	commitManifestDesc, err := writeContentsForImage(ctx, i.snapshotter, i.client.ContentStore(), ociImgToCreate, layers)
+	if err != nil {
+		return nil, err
+	}
+
+	// image create
+	img := containerdimages.Image{
+		Name:      danglingImageName(commitManifestDesc.Digest),
+		Target:    commitManifestDesc,
+		CreatedAt: time.Now(),
+	}
+
+	createdImage, err := i.client.ImageService().Update(ctx, img)
+	if err != nil {
+		if !cerrdefs.IsNotFound(err) {
+			return nil, err
+		}
+
+		if createdImage, err = i.client.ImageService().Create(ctx, img); err != nil {
+			return nil, fmt.Errorf("failed to create new image: %w", err)
+		}
+	}
+
+	if err := i.unpackImage(ctx, createdImage, platforms.DefaultSpec()); err != nil {
+		return nil, err
+	}
+
+	newImage := dimage.NewImage(dimage.ID(createdImage.Target.Digest))
+	newImage.V1Image = imgToCreate.V1Image
+	newImage.V1Image.ID = string(createdImage.Target.Digest)
+	newImage.History = imgToCreate.History
+	return newImage, nil
 }

daemon/containerd/image_children.go

@@ -127,3 +127,71 @@ func isRootfsChildOf(child ocispec.RootFS, parent ocispec.RootFS) bool {
 
 	return true
 }
+
+// parents returns a slice of image IDs whose entire rootfs contents match,
+// in order, the childs first layers, excluding images with the exact same
+// rootfs.
+//
+// Called from image_delete.go to prune dangling parents.
+func (i *ImageService) parents(ctx context.Context, id image.ID) ([]imageWithRootfs, error) {
+	target, err := i.resolveDescriptor(ctx, id.String())
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get child image")
+	}
+
+	cs := i.client.ContentStore()
+
+	allPlatforms, err := containerdimages.Platforms(ctx, cs, target)
+	if err != nil {
+		return nil, errdefs.System(errors.Wrap(err, "failed to list platforms supported by image"))
+	}
+
+	var childRootFS []ocispec.RootFS
+	for _, platform := range allPlatforms {
+		rootfs, err := platformRootfs(ctx, cs, target, platform)
+		if err != nil {
+			if cerrdefs.IsNotFound(err) {
+				continue
+			}
+			return nil, errdefs.System(errors.Wrap(err, "failed to get platform-specific rootfs"))
+		}
+
+		childRootFS = append(childRootFS, rootfs)
+	}
+
+	imgs, err := i.client.ImageService().List(ctx)
+	if err != nil {
+		return nil, errdefs.System(errors.Wrap(err, "failed to list all images"))
+	}
+
+	var parents []imageWithRootfs
+	for _, img := range imgs {
+	nextImage:
+		for _, platform := range allPlatforms {
+			rootfs, err := platformRootfs(ctx, cs, img.Target, platform)
+			if err != nil {
+				if cerrdefs.IsNotFound(err) {
+					continue
+				}
+				return nil, errdefs.System(errors.Wrap(err, "failed to get platform-specific rootfs"))
+			}
+
+			for _, childRoot := range childRootFS {
+				if isRootfsChildOf(childRoot, rootfs) {
+					parents = append(parents, imageWithRootfs{
+						img:    img,
+						rootfs: rootfs,
+					})
+					break nextImage
+				}
+			}
+		}
+	}
+
+	return parents, nil
+}
+
+type imageWithRootfs struct {
+	img    containerdimages.Image
+	rootfs ocispec.RootFS
+}

daemon/containerd/image_commit.go

@@ -6,9 +6,9 @@ import (
 	"crypto/rand"
 	"encoding/base64"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/containerd/containerd/content"
@@ -19,7 +19,6 @@ import (
 	"github.com/containerd/containerd/rootfs"
 	"github.com/containerd/containerd/snapshots"
 	"github.com/docker/docker/api/types/backend"
-	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/image"
 	"github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/identity"
@@ -132,20 +131,23 @@ func generateCommitImageConfig(baseConfig ocispec.Image, diffID digest.Digest, o
 	}
 	logrus.Debugf("generateCommitImageConfig(): arch=%q, os=%q", arch, os)
 	return ocispec.Image{
-		Architecture: arch,
-		OS:           os,
-		Created:      &createdTime,
-		Author:       opts.Author,
-		Config:       containerConfigToOciImageConfig(opts.Config),
+		Platform: ocispec.Platform{
+			Architecture: arch,
+			OS:           os,
+		},
+		Created: &createdTime,
+		Author:  opts.Author,
+		Config:  containerConfigToOciImageConfig(opts.Config),
 		RootFS: ocispec.RootFS{
 			Type:    "layers",
 			DiffIDs: append(baseConfig.RootFS.DiffIDs, diffID),
 		},
 		History: append(baseConfig.History, ocispec.History{
-			Created:    &createdTime,
-			CreatedBy:  "", // FIXME(ndeloof) ?
-			Author:     opts.Author,
-			Comment:    opts.Comment,
+			Created:   &createdTime,
+			CreatedBy: strings.Join(opts.ContainerConfig.Cmd, " "),
+			Author:    opts.Author,
+			Comment:   opts.Comment,
+			// TODO(laurazard): this check might be incorrect
 			EmptyLayer: diffID == "",
 		}),
 	}
@@ -297,5 +299,13 @@ func uniquePart() string {
 //
 // This is a temporary shim. Should be removed when builder stops using commit.
 func (i *ImageService) CommitBuildStep(ctx context.Context, c backend.CommitConfig) (image.ID, error) {
-	return "", errdefs.NotImplemented(errors.New("not implemented"))
+	ctr := i.containers.Get(c.ContainerID)
+	if ctr == nil {
+		// TODO: use typed error
+		return "", fmt.Errorf("container not found: %s", c.ContainerID)
+	}
+	c.ContainerMountLabel = ctr.MountLabel
+	c.ContainerOS = ctr.OS
+	c.ParentImageID = string(ctr.ImageID)
+	return i.CommitImage(ctx, c)
 }

daemon/containerd/image_delete.go

@@ -2,10 +2,16 @@ package containerd
 
 import (
 	"context"
+	"fmt"
+	"sort"
+	"strings"
 
 	"github.com/containerd/containerd/images"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/container"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/pkg/stringid"
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/sirupsen/logrus"
@@ -30,8 +36,6 @@ import (
 // are divided into two categories grouped by their severity:
 //
 // Hard Conflict:
-//   - a pull or build using the image.
-//   - any descendant image.
 //   - any running container using the image.
 //
 // Soft Conflict:
@@ -45,8 +49,6 @@ import (
 // meaning any delete conflicts will cause the image to not be deleted and the
 // conflict will not be reported.
 //
-// TODO(thaJeztah): implement ImageDelete "force" options; see https://github.com/moby/moby/issues/43850
-// TODO(thaJeztah): implement ImageDelete "prune" options; see https://github.com/moby/moby/issues/43849
 // TODO(thaJeztah): image delete should send prometheus counters; see https://github.com/moby/moby/issues/45268
 func (i *ImageService) ImageDelete(ctx context.Context, imageRef string, force, prune bool) ([]types.ImageDeleteResponseItem, error) {
 	parsedRef, err := reference.ParseNormalizedNamed(imageRef)
@@ -59,28 +61,279 @@ func (i *ImageService) ImageDelete(ctx context.Context, imageRef string, force,
 		return nil, err
 	}
 
-	possiblyDeletedConfigs := map[digest.Digest]struct{}{}
-	if err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, d ocispec.Descriptor) {
-		if images.IsConfigType(d.MediaType) {
-			possiblyDeletedConfigs[d.Digest] = struct{}{}
-		}
-	}); err != nil {
-		return nil, err
+	imgID := image.ID(img.Target.Digest)
+
+	if isImageIDPrefix(imgID.String(), imageRef) {
+		return i.deleteAll(ctx, img, force, prune)
 	}
 
-	err = i.client.ImageService().Delete(ctx, img.Name, images.SynchronousDelete())
+	singleRef, err := i.isSingleReference(ctx, img)
 	if err != nil {
 		return nil, err
 	}
-
-	// Workaround for: https://github.com/moby/buildkit/issues/3797
-	if err := i.unleaseSnapshotsFromDeletedConfigs(context.Background(), possiblyDeletedConfigs); err != nil {
-		logrus.WithError(err).Warn("failed to unlease snapshots")
+	if !singleRef {
+		err := i.client.ImageService().Delete(ctx, img.Name)
+		if err != nil {
+			return nil, err
+		}
+		i.LogImageEvent(imgID.String(), imgID.String(), "untag")
+		records := []types.ImageDeleteResponseItem{{Untagged: reference.FamiliarString(reference.TagNameOnly(parsedRef))}}
+		return records, nil
 	}
 
-	imgID := string(img.Target.Digest)
-	i.LogImageEvent(imgID, imgID, "untag")
-	i.LogImageEvent(imgID, imgID, "delete")
+	using := func(c *container.Container) bool {
+		return c.ImageID == imgID
+	}
+	ctr := i.containers.First(using)
+	if ctr != nil {
+		if !force {
+			// If we removed the repository reference then
+			// this image would remain "dangling" and since
+			// we really want to avoid that the client must
+			// explicitly force its removal.
+			refString := reference.FamiliarString(reference.TagNameOnly(parsedRef))
+			err := &imageDeleteConflict{
+				reference: refString,
+				used:      true,
+				message: fmt.Sprintf("container %s is using its referenced image %s",
+					stringid.TruncateID(ctr.ID),
+					stringid.TruncateID(imgID.String())),
+			}
+			return nil, err
+		}
 
-	return []types.ImageDeleteResponseItem{{Untagged: reference.FamiliarString(parsedRef)}}, nil
+		err := i.softImageDelete(ctx, img)
+		if err != nil {
+			return nil, err
+		}
+
+		i.LogImageEvent(imgID.String(), imgID.String(), "untag")
+		records := []types.ImageDeleteResponseItem{{Untagged: reference.FamiliarString(reference.TagNameOnly(parsedRef))}}
+		return records, nil
+	}
+
+	return i.deleteAll(ctx, img, force, prune)
+}
+
+// deleteAll deletes the image from the daemon, and if prune is true,
+// also deletes dangling parents if there is no conflict in doing so.
+// Parent images are removed quietly, and if there is any issue/conflict
+// it is logged but does not halt execution/an error is not returned.
+func (i *ImageService) deleteAll(ctx context.Context, img images.Image, force, prune bool) ([]types.ImageDeleteResponseItem, error) {
+	var records []types.ImageDeleteResponseItem
+
+	// Workaround for: https://github.com/moby/buildkit/issues/3797
+	possiblyDeletedConfigs := map[digest.Digest]struct{}{}
+	err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, d ocispec.Descriptor) error {
+		if images.IsConfigType(d.MediaType) {
+			possiblyDeletedConfigs[d.Digest] = struct{}{}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err := i.unleaseSnapshotsFromDeletedConfigs(context.Background(), possiblyDeletedConfigs); err != nil {
+			logrus.WithError(err).Warn("failed to unlease snapshots")
+		}
+	}()
+
+	imgID := img.Target.Digest.String()
+
+	var parents []imageWithRootfs
+	if prune {
+		parents, err = i.parents(ctx, image.ID(imgID))
+		if err != nil {
+			logrus.WithError(err).Warn("failed to get image parents")
+		}
+		sortParentsByAffinity(parents)
+	}
+
+	imageRefs, err := i.client.ImageService().List(ctx, "target.digest=="+imgID)
+	if err != nil {
+		return nil, err
+	}
+	for _, imageRef := range imageRefs {
+		if err := i.imageDeleteHelper(ctx, imageRef, &records, force); err != nil {
+			return records, err
+		}
+	}
+	i.LogImageEvent(imgID, imgID, "delete")
+	records = append(records, types.ImageDeleteResponseItem{Deleted: imgID})
+
+	for _, parent := range parents {
+		if !isDanglingImage(parent.img) {
+			break
+		}
+		err = i.imageDeleteHelper(ctx, parent.img, &records, false)
+		if err != nil {
+			logrus.WithError(err).Warn("failed to remove image parent")
+			break
+		}
+		parentID := parent.img.Target.Digest.String()
+		i.LogImageEvent(parentID, parentID, "delete")
+		records = append(records, types.ImageDeleteResponseItem{Deleted: parentID})
+	}
+
+	return records, nil
+}
+
+// isImageIDPrefix returns whether the given
+// possiblePrefix is a prefix of the given imageID.
+func isImageIDPrefix(imageID, possiblePrefix string) bool {
+	if strings.HasPrefix(imageID, possiblePrefix) {
+		return true
+	}
+	if i := strings.IndexRune(imageID, ':'); i >= 0 {
+		return strings.HasPrefix(imageID[i+1:], possiblePrefix)
+	}
+	return false
+}
+
+func sortParentsByAffinity(parents []imageWithRootfs) {
+	sort.Slice(parents, func(i, j int) bool {
+		lenRootfsI := len(parents[i].rootfs.DiffIDs)
+		lenRootfsJ := len(parents[j].rootfs.DiffIDs)
+		if lenRootfsI == lenRootfsJ {
+			return isDanglingImage(parents[i].img)
+		}
+		return lenRootfsI > lenRootfsJ
+	})
+}
+
+// isSingleReference returns true if there are no other images in the
+// daemon targeting the same content as `img` that are not dangling.
+func (i *ImageService) isSingleReference(ctx context.Context, img images.Image) (bool, error) {
+	refs, err := i.client.ImageService().List(ctx, "target.digest=="+img.Target.Digest.String())
+	if err != nil {
+		return false, err
+	}
+	for _, ref := range refs {
+		if !isDanglingImage(ref) && ref.Name != img.Name {
+			return false, nil
+		}
+	}
+	return true, nil
+}
+
+type conflictType int
+
+const (
+	conflictRunningContainer conflictType = 1 << iota
+	conflictActiveReference
+	conflictStoppedContainer
+	conflictHard = conflictRunningContainer
+	conflictSoft = conflictActiveReference | conflictStoppedContainer
+)
+
+// imageDeleteHelper attempts to delete the given image from this daemon.
+// If the image has any hard delete conflicts (running containers using
+// the image) then it cannot be deleted. If the image has any soft delete
+// conflicts (any tags/digests referencing the image or any stopped container
+// using the image) then it can only be deleted if force is true. Any deleted
+// images and untagged references are appended to the given records. If any
+// error or conflict is encountered, it will be returned immediately without
+// deleting the image.
+func (i *ImageService) imageDeleteHelper(ctx context.Context, img images.Image, records *[]types.ImageDeleteResponseItem, force bool) error {
+	// First, determine if this image has any conflicts. Ignore soft conflicts
+	// if force is true.
+	c := conflictHard
+	if !force {
+		c |= conflictSoft
+	}
+
+	imgID := image.ID(img.Target.Digest)
+
+	err := i.checkImageDeleteConflict(ctx, imgID, c)
+	if err != nil {
+		return err
+	}
+
+	untaggedRef, err := reference.ParseAnyReference(img.Name)
+	if err != nil {
+		return err
+	}
+	err = i.client.ImageService().Delete(ctx, img.Name, images.SynchronousDelete())
+	if err != nil {
+		return err
+	}
+
+	i.LogImageEvent(imgID.String(), imgID.String(), "untag")
+	*records = append(*records, types.ImageDeleteResponseItem{Untagged: reference.FamiliarString(untaggedRef)})
+
+	return nil
+}
+
+// ImageDeleteConflict holds a soft or hard conflict and associated
+// error. A hard conflict represents a running container using the
+// image, while a soft conflict is any tags/digests referencing the
+// given image or any stopped container using the image.
+// Implements the error interface.
+type imageDeleteConflict struct {
+	hard      bool
+	used      bool
+	reference string
+	message   string
+}
+
+func (idc *imageDeleteConflict) Error() string {
+	var forceMsg string
+	if idc.hard {
+		forceMsg = "cannot be forced"
+	} else {
+		forceMsg = "must be forced"
+	}
+	return fmt.Sprintf("conflict: unable to delete %s (%s) - %s", idc.reference, forceMsg, idc.message)
+}
+
+func (imageDeleteConflict) Conflict() {}
+
+// checkImageDeleteConflict returns a conflict representing
+// any issue preventing deletion of the given image ID, and
+// nil if there are none. It takes a bitmask representing a
+// filter for which conflict types the caller cares about,
+// and will only check for these conflict types.
+func (i *ImageService) checkImageDeleteConflict(ctx context.Context, imgID image.ID, mask conflictType) error {
+	if mask&conflictRunningContainer != 0 {
+		running := func(c *container.Container) bool {
+			return c.ImageID == imgID && c.IsRunning()
+		}
+		if ctr := i.containers.First(running); ctr != nil {
+			return &imageDeleteConflict{
+				reference: stringid.TruncateID(imgID.String()),
+				hard:      true,
+				used:      true,
+				message:   fmt.Sprintf("image is being used by running container %s", stringid.TruncateID(ctr.ID)),
+			}
+		}
+	}
+
+	if mask&conflictStoppedContainer != 0 {
+		stopped := func(c *container.Container) bool {
+			return !c.IsRunning() && c.ImageID == imgID
+		}
+		if ctr := i.containers.First(stopped); ctr != nil {
+			return &imageDeleteConflict{
+				reference: stringid.TruncateID(imgID.String()),
+				used:      true,
+				message:   fmt.Sprintf("image is being used by stopped container %s", stringid.TruncateID(ctr.ID)),
+			}
+		}
+	}
+
+	if mask&conflictActiveReference != 0 {
+		refs, err := i.client.ImageService().List(ctx, "target.digest=="+imgID.String())
+		if err != nil {
+			return err
+		}
+		if len(refs) > 1 {
+			return &imageDeleteConflict{
+				reference: stringid.TruncateID(imgID.String()),
+				message:   "image is referenced in multiple repositories",
+			}
+		}
+	}
+
+	return nil
 }

daemon/containerd/image_exporter.go

@@ -6,9 +6,11 @@ import (
 	"io"
 
 	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/content"
 	cerrdefs "github.com/containerd/containerd/errdefs"
 	containerdimages "github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/images/archive"
+	"github.com/containerd/containerd/leases"
 	"github.com/containerd/containerd/mount"
 	cplatforms "github.com/containerd/containerd/platforms"
 	"github.com/docker/distribution/reference"
@@ -18,6 +20,7 @@ import (
 	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/opencontainers/image-spec/specs-go"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
@@ -56,11 +59,17 @@ func (i *ImageService) ExportImage(ctx context.Context, names []string, outStrea
 		archive.WithPlatform(platform),
 	}
 
-	ctx, release, err := i.client.WithLease(ctx)
+	contentStore := i.client.ContentStore()
+	leasesManager := i.client.LeasesService()
+	lease, err := leasesManager.Create(ctx, leases.WithRandomID())
 	if err != nil {
 		return errdefs.System(err)
 	}
-	defer release(ctx)
+	defer func() {
+		if err := leasesManager.Delete(ctx, lease); err != nil {
+			logrus.WithError(err).Warn("cleaning up lease")
+		}
+	}()
 
 	for _, name := range names {
 		target, err := i.resolveDescriptor(ctx, name)
@@ -68,6 +77,10 @@ func (i *ImageService) ExportImage(ctx context.Context, names []string, outStrea
 			return err
 		}
 
+		if err = leaseContent(ctx, contentStore, leasesManager, lease, target); err != nil {
+			return err
+		}
+
 		// We may not have locally all the platforms that are specified in the index.
 		// Export only those manifests that we have.
 		// TODO(vvoland): Reconsider this when `--platform` is added.
@@ -99,6 +112,30 @@ func (i *ImageService) ExportImage(ctx context.Context, names []string, outStrea
 	return i.client.Export(ctx, outStream, opts...)
 }
 
+// leaseContent will add a resource to the lease for each child of the descriptor making sure that it and
+// its children won't be deleted while the lease exists
+func leaseContent(ctx context.Context, store content.Store, leasesManager leases.Manager, lease leases.Lease, desc ocispec.Descriptor) error {
+	return containerdimages.Walk(ctx, containerdimages.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
+		_, err := store.Info(ctx, desc.Digest)
+		if err != nil {
+			if errors.Is(err, cerrdefs.ErrNotFound) {
+				return nil, nil
+			}
+			return nil, errdefs.System(err)
+		}
+
+		r := leases.Resource{
+			ID:   desc.Digest.String(),
+			Type: "content",
+		}
+		if err := leasesManager.AddResource(ctx, lease, r); err != nil {
+			return nil, errdefs.System(err)
+		}
+
+		return containerdimages.Children(ctx, store, desc)
+	}), desc)
+}
+
 // LoadImage uploads a set of images into the repository. This is the
 // complement of ExportImage.  The input stream is an uncompressed tar
 // ball containing images and metadata.
@@ -109,7 +146,7 @@ func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, outSt
 
 		// Create an additional image with dangling name for imported images...
 		containerd.WithDigestRef(danglingImageName),
-		/// ... but only if they don't have a name or it's invalid.
+		// / ... but only if they don't have a name or it's invalid.
 		containerd.WithSkipDigestRef(func(nameFromArchive string) bool {
 			if nameFromArchive == "" {
 				return false
@@ -125,16 +162,9 @@ func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, outSt
 		return errdefs.System(err)
 	}
 
-	store := i.client.ContentStore()
 	progress := streamformatter.NewStdoutWriter(outStream)
 
 	for _, img := range imgs {
-		allPlatforms, err := containerdimages.Platforms(ctx, store, img.Target)
-		if err != nil {
-			logrus.WithError(err).WithField("image", img.Name).Debug("failed to get image platforms")
-			return errdefs.Unknown(err)
-		}
-
 		name := img.Name
 		loadedMsg := "Loaded image"
 
@@ -145,17 +175,25 @@ func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, outSt
 			name = reference.FamiliarName(reference.TagNameOnly(named))
 		}
 
-		for _, platform := range allPlatforms {
+		err = i.walkImageManifests(ctx, img, func(platformImg *ImageManifest) error {
 			logger := logrus.WithFields(logrus.Fields{
-				"platform": platform,
 				"image":    name,
+				"manifest": platformImg.Target().Digest,
 			})
-			platformImg := containerd.NewImageWithPlatform(i.client, img, cplatforms.OnlyStrict(platform))
+
+			if isPseudo, err := platformImg.IsPseudoImage(ctx); isPseudo || err != nil {
+				if err != nil {
+					logger.WithError(err).Warn("failed to read manifest")
+				} else {
+					logger.Debug("don't unpack non-image manifest")
+				}
+				return nil
+			}
 
 			unpacked, err := platformImg.IsUnpacked(ctx, i.snapshotter)
 			if err != nil {
-				logger.WithError(err).Debug("failed to check if image is unpacked")
-				continue
+				logger.WithError(err).Warn("failed to check if image is unpacked")
+				return nil
 			}
 
 			if !unpacked {
@@ -166,6 +204,10 @@ func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, outSt
 				}
 			}
 			logger.WithField("alreadyUnpacked", unpacked).WithError(err).Debug("unpack")
+			return nil
+		})
+		if err != nil {
+			return errors.Wrap(err, "failed to unpack loaded image")
 		}
 
 		fmt.Fprintf(progress, "%s: %s\n", loadedMsg, name)

daemon/containerd/image_history.go

@@ -2,13 +2,12 @@ package containerd
 
 import (
 	"context"
-	"encoding/json"
+	"sort"
 
-	"github.com/containerd/containerd/content"
-	containerdimages "github.com/containerd/containerd/images"
 	cplatforms "github.com/containerd/containerd/platforms"
 	"github.com/docker/distribution/reference"
 	imagetype "github.com/docker/docker/api/types/image"
+	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/pkg/platforms"
 	"github.com/opencontainers/image-spec/identity"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -18,32 +17,39 @@ import (
 // ImageHistory returns a slice of HistoryResponseItem structures for the
 // specified image name by walking the image lineage.
 func (i *ImageService) ImageHistory(ctx context.Context, name string) ([]*imagetype.HistoryResponseItem, error) {
-	desc, err := i.resolveDescriptor(ctx, name)
+	desc, err := i.resolveImage(ctx, name)
 	if err != nil {
 		return nil, err
 	}
 
 	cs := i.client.ContentStore()
-	// TODO: pass the platform from the cli
-	conf, err := containerdimages.Config(ctx, cs, desc, platforms.AllPlatformsWithPreference(cplatforms.Default()))
+	// TODO: pass platform in from the CLI
+	platform := platforms.AllPlatformsWithPreference(cplatforms.Default())
+
+	var presentImages []ocispec.Image
+	err = i.walkImageManifests(ctx, desc, func(img *ImageManifest) error {
+		conf, err := img.Config(ctx)
+		if err != nil {
+			return err
+		}
+		var ociimage ocispec.Image
+		if err := readConfig(ctx, cs, conf, &ociimage); err != nil {
+			return err
+		}
+		presentImages = append(presentImages, ociimage)
+		return nil
+	})
 	if err != nil {
 		return nil, err
 	}
-
-	diffIDs, err := containerdimages.RootFS(ctx, cs, conf)
-	if err != nil {
-		return nil, err
+	if len(presentImages) == 0 {
+		return nil, errdefs.NotFound(errors.New("failed to find image manifest"))
 	}
 
-	blob, err := content.ReadBlob(ctx, cs, conf)
-	if err != nil {
-		return nil, err
-	}
-
-	var image ocispec.Image
-	if err := json.Unmarshal(blob, &image); err != nil {
-		return nil, err
-	}
+	sort.SliceStable(presentImages, func(i, j int) bool {
+		return platform.Less(presentImages[i].Platform, presentImages[j].Platform)
+	})
+	ociimage := presentImages[0]
 
 	var (
 		history []*imagetype.HistoryResponseItem
@@ -51,6 +57,7 @@ func (i *ImageService) ImageHistory(ctx context.Context, name string) ([]*imaget
 	)
 	s := i.client.SnapshotService(i.snapshotter)
 
+	diffIDs := ociimage.RootFS.DiffIDs
 	for i := range diffIDs {
 		chainID := identity.ChainID(diffIDs[0 : i+1]).String()
 
@@ -62,7 +69,7 @@ func (i *ImageService) ImageHistory(ctx context.Context, name string) ([]*imaget
 		sizes = append(sizes, use.Size)
 	}
 
-	for _, h := range image.History {
+	for _, h := range ociimage.History {
 		size := int64(0)
 		if !h.EmptyLayer {
 			if len(sizes) == 0 {
@@ -83,20 +90,23 @@ func (i *ImageService) ImageHistory(ctx context.Context, name string) ([]*imaget
 	}
 
 	if len(history) != 0 {
-		history[0].ID = desc.Digest.String()
+		history[0].ID = desc.Target.Digest.String()
 
-		tagged, err := i.client.ImageService().List(ctx, "target.digest=="+desc.Digest.String())
+		tagged, err := i.client.ImageService().List(ctx, "target.digest=="+desc.Target.Digest.String())
 		if err != nil {
 			return nil, err
 		}
 
-		tags := make([]string, len(tagged))
-		for i, t := range tagged {
+		var tags []string
+		for _, t := range tagged {
+			if isDanglingImage(t) {
+				continue
+			}
 			name, err := reference.ParseNamed(t.Name)
 			if err != nil {
 				return nil, err
 			}
-			tags[i] = reference.FamiliarString(name)
+			tags = append(tags, reference.FamiliarString(name))
 		}
 		history[0].Tags = tags
 	}

daemon/containerd/image_import.go

@@ -86,11 +86,10 @@ func (i *ImageService) ImportImage(ctx context.Context, ref reference.Named, pla
 	ociCfg := containerConfigToOciImageConfig(imageConfig)
 	createdAt := time.Now()
 	config := ocispec.Image{
-		Architecture: platform.Architecture,
-		OS:           platform.OS,
-		Created:      &createdAt,
-		Author:       "",
-		Config:       ociCfg,
+		Platform: *platform,
+		Created:  &createdAt,
+		Author:   "",
+		Config:   ociCfg,
 		RootFS: ocispec.RootFS{
 			Type:    "layers",
 			DiffIDs: []digest.Digest{uncompressedDigest},
@@ -393,8 +392,11 @@ func containerConfigToOciImageConfig(cfg *container.Config) ocispec.ImageConfig
 		StopSignal:  cfg.StopSignal,
 		ArgsEscaped: cfg.ArgsEscaped,
 	}
-	for k, v := range cfg.ExposedPorts {
-		ociCfg.ExposedPorts[string(k)] = v
+	if len(cfg.ExposedPorts) > 0 {
+		ociCfg.ExposedPorts = map[string]struct{}{}
+		for k, v := range cfg.ExposedPorts {
+			ociCfg.ExposedPorts[string(k)] = v
+		}
 	}
 
 	return ociCfg

daemon/containerd/image_import_test.go

@@ -0,0 +1,22 @@
+package containerd
+
+import (
+	"testing"
+
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/go-connections/nat"
+	"gotest.tools/v3/assert"
+	is "gotest.tools/v3/assert/cmp"
+)
+
+// regression test for https://github.com/moby/moby/issues/45904
+func TestContainerConfigToOciImageConfig(t *testing.T) {
+	ociCFG := containerConfigToOciImageConfig(&container.Config{
+		ExposedPorts: nat.PortSet{
+			"80/tcp": struct{}{},
+		},
+	})
+
+	expected := map[string]struct{}{"80/tcp": {}}
+	assert.Check(t, is.DeepEqual(ociCFG.ExposedPorts, expected))
+}

daemon/containerd/image_list.go

@@ -6,20 +6,17 @@ import (
 	"strings"
 	"time"
 
-	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/content"
 	cerrdefs "github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/labels"
-	"github.com/containerd/containerd/platforms"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	timetypes "github.com/docker/docker/api/types/time"
-	"github.com/moby/buildkit/util/attestation"
 	"github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/identity"
-	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -87,72 +84,41 @@ func (i *ImageService) Images(ctx context.Context, opts types.ImageListOptions)
 			continue
 		}
 
-		err := images.Walk(ctx, images.HandlerFunc(func(ctx context.Context, desc v1.Descriptor) ([]v1.Descriptor, error) {
-			if images.IsIndexType(desc.MediaType) {
-				return images.Children(ctx, contentStore, desc)
+		err := i.walkImageManifests(ctx, img, func(img *ImageManifest) error {
+			if isPseudo, err := img.IsPseudoImage(ctx); isPseudo || err != nil {
+				return err
 			}
 
-			if images.IsManifestType(desc.MediaType) {
-				// Ignore buildkit attestation manifests
-				// https://github.com/moby/buildkit/blob/v0.11.4/docs/attestations/attestation-storage.md
-				// This would have also been caught by the isImageManifest call below, but it requires
-				// an additional content read and deserialization of Manifest.
-				if _, has := desc.Annotations[attestation.DockerAnnotationReferenceType]; has {
-					return nil, nil
-				}
+			available, err := img.CheckContentAvailable(ctx)
+			if err != nil {
+				logrus.WithFields(logrus.Fields{
+					logrus.ErrorKey: err,
+					"manifest":      img.Target(),
+					"image":         img.Name(),
+				}).Warn("checking availability of platform specific manifest failed")
+				return nil
+			}
 
-				mfst, err := images.Manifest(ctx, contentStore, desc, platforms.All)
-				if err != nil {
-					if cerrdefs.IsNotFound(err) {
-						return nil, nil
-					}
-					return nil, err
-				}
+			if !available {
+				return nil
+			}
 
-				if !isImageManifest(mfst) {
-					return nil, nil
-				}
+			image, chainIDs, err := i.singlePlatformImage(ctx, contentStore, img)
+			if err != nil {
+				return err
+			}
 
-				platform, err := getManifestPlatform(ctx, contentStore, desc, mfst.Config)
-				if err != nil {
-					if cerrdefs.IsNotFound(err) {
-						return nil, nil
-					}
-					return nil, err
-				}
+			summaries = append(summaries, image)
 
-				pm := platforms.OnlyStrict(platform)
-				available, _, _, missing, err := images.Check(ctx, contentStore, img.Target, pm)
-				if err != nil {
-					logrus.WithFields(logrus.Fields{
-						logrus.ErrorKey: err,
-						"platform":      platform,
-						"image":         img.Target,
-					}).Warn("checking availability of platform content failed")
-					return nil, nil
-				}
-				if !available || len(missing) > 0 {
-					return nil, nil
-				}
-
-				c8dImage := containerd.NewImageWithPlatform(i.client, img, pm)
-				image, chainIDs, err := i.singlePlatformImage(ctx, contentStore, c8dImage)
-				if err != nil {
-					return nil, err
-				}
-
-				summaries = append(summaries, image)
-
-				if opts.SharedSize {
-					root = append(root, &chainIDs)
-					for _, id := range chainIDs {
-						layers[id] = layers[id] + 1
-					}
+			if opts.SharedSize {
+				root = append(root, &chainIDs)
+				for _, id := range chainIDs {
+					layers[id] = layers[id] + 1
 				}
 			}
 
-			return nil, nil
-		}), img.Target)
+			return nil
+		})
 
 		if err != nil {
 			return nil, err
@@ -173,7 +139,7 @@ func (i *ImageService) Images(ctx context.Context, opts types.ImageListOptions)
 	return summaries, nil
 }
 
-func (i *ImageService) singlePlatformImage(ctx context.Context, contentStore content.Store, image containerd.Image) (*types.ImageSummary, []digest.Digest, error) {
+func (i *ImageService) singlePlatformImage(ctx context.Context, contentStore content.Store, image *ImageManifest) (*types.ImageSummary, []digest.Digest, error) {
 	diffIDs, err := image.RootFS(ctx)
 	if err != nil {
 		return nil, nil, err
@@ -418,7 +384,7 @@ func setupLabelFilter(store content.Store, fltrs filters.Args) (func(image image
 		// processing more content (otherwise it will run for all children).
 		// It will be returned once a matching config is found.
 		errFoundConfig := errors.New("success, found matching config")
-		err := images.Dispatch(ctx, presentChildrenHandler(store, images.HandlerFunc(func(ctx context.Context, desc v1.Descriptor) (subdescs []v1.Descriptor, err error) {
+		err := images.Dispatch(ctx, presentChildrenHandler(store, images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) (subdescs []ocispec.Descriptor, err error) {
 			if !images.IsConfigType(desc.MediaType) {
 				return nil, nil
 			}
@@ -509,35 +475,8 @@ func computeSharedSize(chainIDs []digest.Digest, layers map[digest.Digest]int, s
 	return sharedSize, nil
 }
 
-// getManifestPlatform returns a platform specified by the manifest descriptor
-// or reads it from its config.
-func getManifestPlatform(ctx context.Context, store content.Provider, manifestDesc, configDesc v1.Descriptor) (v1.Platform, error) {
-	var platform v1.Platform
-	if manifestDesc.Platform != nil {
-		platform = *manifestDesc.Platform
-	} else {
-		// Config is technically a v1.Image, but it has the same member as v1.Platform
-		// which makes the v1.Platform a subset of Image so we can unmarshal directly.
-		if err := readConfig(ctx, store, configDesc, &platform); err != nil {
-			return platform, err
-		}
-	}
-	return platforms.Normalize(platform), nil
-}
-
-// isImageManifest returns true if the manifest has any layer that is a known image layer.
-// Some manifests use the image media type for compatibility, even if they are not a real image.
-func isImageManifest(mfst v1.Manifest) bool {
-	for _, l := range mfst.Layers {
-		if images.IsLayerType(l.MediaType) {
-			return true
-		}
-	}
-	return false
-}
-
 // readConfig reads content pointed by the descriptor and unmarshals it into a specified output.
-func readConfig(ctx context.Context, store content.Provider, desc v1.Descriptor, out interface{}) error {
+func readConfig(ctx context.Context, store content.Provider, desc ocispec.Descriptor, out interface{}) error {
 	data, err := content.ReadBlob(ctx, store, desc)
 	if err != nil {
 		return errors.Wrapf(err, "failed to read config content")

daemon/containerd/image_manifest.go

@@ -0,0 +1,151 @@
+package containerd
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/images"
+	containerdimages "github.com/containerd/containerd/images"
+	cplatforms "github.com/containerd/containerd/platforms"
+	"github.com/docker/docker/errdefs"
+	"github.com/moby/buildkit/util/attestation"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+var (
+	errNotManifestOrIndex = errdefs.InvalidParameter(errors.New("descriptor is neither a manifest or index"))
+	errNotManifest        = errdefs.InvalidParameter(errors.New("descriptor isn't a manifest"))
+)
+
+// walkImageManifests calls the handler for each locally present manifest in
+// the image. The image implements the containerd.Image interface, but all
+// operations act on the specific manifest instead of the index.
+func (i *ImageService) walkImageManifests(ctx context.Context, img containerdimages.Image, handler func(img *ImageManifest) error) error {
+	desc := img.Target
+
+	handleManifest := func(ctx context.Context, d ocispec.Descriptor) error {
+		platformImg, err := i.NewImageManifest(ctx, img, d)
+		if err != nil {
+			if err == errNotManifest {
+				return nil
+			}
+			return err
+		}
+		return handler(platformImg)
+	}
+
+	if containerdimages.IsManifestType(desc.MediaType) {
+		return handleManifest(ctx, desc)
+	}
+
+	if containerdimages.IsIndexType(desc.MediaType) {
+		return i.walkPresentChildren(ctx, desc, handleManifest)
+	}
+
+	return errNotManifestOrIndex
+}
+
+type ImageManifest struct {
+	containerd.Image
+
+	// Parent of the manifest (index/manifest list)
+	RealTarget ocispec.Descriptor
+
+	manifest *ocispec.Manifest
+}
+
+func (i *ImageService) NewImageManifest(ctx context.Context, img containerdimages.Image, manifestDesc ocispec.Descriptor) (*ImageManifest, error) {
+	if !containerdimages.IsManifestType(manifestDesc.MediaType) {
+		return nil, errNotManifest
+	}
+
+	parent := img.Target
+	img.Target = manifestDesc
+
+	c8dImg := containerd.NewImageWithPlatform(i.client, img, cplatforms.All)
+	return &ImageManifest{
+		Image:      c8dImg,
+		RealTarget: parent,
+	}, nil
+}
+
+func (im *ImageManifest) Metadata() containerdimages.Image {
+	md := im.Image.Metadata()
+	md.Target = im.RealTarget
+	return md
+}
+
+// IsPseudoImage returns false if the manifest has no layers or any of its layers is a known image layer.
+// Some manifests use the image media type for compatibility, even if they are not a real image.
+func (im *ImageManifest) IsPseudoImage(ctx context.Context) (bool, error) {
+	desc := im.Target()
+
+	// Quick check for buildkit attestation manifests
+	// https://github.com/moby/buildkit/blob/v0.11.4/docs/attestations/attestation-storage.md
+	// This would have also been caught by the layer check below, but it requires
+	// an additional content read and deserialization of Manifest.
+	if _, has := desc.Annotations[attestation.DockerAnnotationReferenceType]; has {
+		return true, nil
+	}
+
+	mfst, err := im.Manifest(ctx)
+	if err != nil {
+		return true, err
+	}
+	if len(mfst.Layers) == 0 {
+		return false, nil
+	}
+	for _, l := range mfst.Layers {
+		if images.IsLayerType(l.MediaType) {
+			return false, nil
+		}
+	}
+	return true, nil
+}
+
+func (im *ImageManifest) Manifest(ctx context.Context) (ocispec.Manifest, error) {
+	if im.manifest != nil {
+		return *im.manifest, nil
+	}
+
+	mfst, err := readManifest(ctx, im.ContentStore(), im.Target())
+	if err != nil {
+		return ocispec.Manifest{}, err
+	}
+
+	im.manifest = &mfst
+	return mfst, nil
+}
+
+func (im *ImageManifest) CheckContentAvailable(ctx context.Context) (bool, error) {
+	// The target is already a platform-specific manifest, so no need to match platform.
+	pm := cplatforms.All
+
+	available, _, _, missing, err := containerdimages.Check(ctx, im.ContentStore(), im.Target(), pm)
+	if err != nil {
+		return false, err
+	}
+
+	if !available || len(missing) > 0 {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+func readManifest(ctx context.Context, store content.Provider, desc ocispec.Descriptor) (ocispec.Manifest, error) {
+	p, err := content.ReadBlob(ctx, store, desc)
+	if err != nil {
+		return ocispec.Manifest{}, err
+	}
+
+	var mfst ocispec.Manifest
+	if err := json.Unmarshal(p, &mfst); err != nil {
+		return ocispec.Manifest{}, err
+	}
+
+	return mfst, nil
+}

daemon/containerd/image_prune.go

@@ -2,6 +2,7 @@ package containerd
 
 import (
 	"context"
+	"strings"
 
 	cerrdefs "github.com/containerd/containerd/errdefs"
 	containerdimages "github.com/containerd/containerd/images"
@@ -69,35 +70,50 @@ func (i *ImageService) pruneUnused(ctx context.Context, filterFunc imageFilterFu
 		return nil, err
 	}
 
+	// How many images make reference to a particular target digest.
+	digestRefCount := map[digest.Digest]int{}
+	// Images considered for pruning.
 	imagesToPrune := map[string]containerdimages.Image{}
 	for _, img := range allImages {
+		digestRefCount[img.Target.Digest] += 1
+
 		if !danglingOnly || isDanglingImage(img) {
-			imagesToPrune[img.Name] = img
+			canBePruned := filterFunc(img)
+			logrus.WithFields(logrus.Fields{
+				"image":       img.Name,
+				"canBePruned": canBePruned,
+			}).Debug("considering image for pruning")
+
+			if canBePruned {
+				imagesToPrune[img.Name] = img
+			}
+
 		}
 	}
 
-	// Apply filters
-	for name, img := range imagesToPrune {
-		filteredOut := !filterFunc(img)
-		logrus.WithFields(logrus.Fields{
-			"image":       name,
-			"filteredOut": filteredOut,
-		}).Debug("filtering image")
+	// Image specified by digests that are used by containers.
+	usedDigests := map[digest.Digest]struct{}{}
 
-		if filteredOut {
-			delete(imagesToPrune, name)
-		}
-	}
-
-	containers := i.containers.List()
-
-	var errs error
 	// Exclude images used by existing containers
-	for _, ctr := range containers {
+	for _, ctr := range i.containers.List() {
+		// If the original image was deleted, make sure we don't delete the dangling image
+		delete(imagesToPrune, danglingImageName(ctr.ImageID.Digest()))
+
 		// Config.Image is the image reference passed by user.
-		// For example: container created by `docker run alpine` will have Image="alpine"
-		// Warning: This doesn't handle truncated ids:
-		//          `docker run 124c7d2` will have Image="124c7d270790"
+		// Config.ImageID is the resolved content digest based on the user's Config.Image.
+		// For example: container created by:
+		//           `docker run alpine` will have Config.Image="alpine"
+		//           `docker run 82d1e9d` will have Config.Image="82d1e9d"
+		// but both will have ImageID="sha256:82d1e9d7ed48a7523bdebc18cf6290bdb97b82302a8a9c27d4fe885949ea94d1"
+		imageDgst := ctr.ImageID.Digest()
+
+		// If user didn't specify an explicit image, mark the digest as used.
+		normalizedImageID := "sha256:" + strings.TrimPrefix(ctr.Config.Image, "sha256:")
+		if strings.HasPrefix(imageDgst.String(), normalizedImageID) {
+			usedDigests[imageDgst] = struct{}{}
+			continue
+		}
+
 		ref, err := reference.ParseNormalizedNamed(ctr.Config.Image)
 		logrus.WithFields(logrus.Fields{
 			"ctr":          ctr.ID,
@@ -106,12 +122,28 @@ func (i *ImageService) pruneUnused(ctx context.Context, filterFunc imageFilterFu
 		}).Debug("filtering container's image")
 
 		if err == nil {
+			// If user provided a specific image name, exclude that image.
 			name := reference.TagNameOnly(ref)
 			delete(imagesToPrune, name.String())
 		}
 	}
 
+	// Create dangling images for images that will be deleted but are still in use.
+	for _, img := range imagesToPrune {
+		dgst := img.Target.Digest
+
+		digestRefCount[dgst] -= 1
+		if digestRefCount[dgst] == 0 {
+			if _, isUsed := usedDigests[dgst]; isUsed {
+				if err := i.ensureDanglingImage(ctx, img); err != nil {
+					return &report, errors.Wrapf(err, "failed to create ensure dangling image for %s", img.Name)
+				}
+			}
+		}
+	}
+
 	possiblyDeletedConfigs := map[digest.Digest]struct{}{}
+	var errs error
 
 	// Workaround for https://github.com/moby/buildkit/issues/3797
 	defer func() {
@@ -125,11 +157,12 @@ func (i *ImageService) pruneUnused(ctx context.Context, filterFunc imageFilterFu
 
 		blobs := []ocispec.Descriptor{}
 
-		err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, desc ocispec.Descriptor) {
+		err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, desc ocispec.Descriptor) error {
 			blobs = append(blobs, desc)
 			if containerdimages.IsConfigType(desc.MediaType) {
 				possiblyDeletedConfigs[desc.Digest] = struct{}{}
 			}
+			return nil
 		})
 		if err != nil {
 			errs = multierror.Append(errs, err)
@@ -186,10 +219,11 @@ func (i *ImageService) unleaseSnapshotsFromDeletedConfigs(ctx context.Context, p
 
 	var errs error
 	for _, img := range all {
-		err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, desc ocispec.Descriptor) {
+		err := i.walkPresentChildren(ctx, img.Target, func(_ context.Context, desc ocispec.Descriptor) error {
 			if containerdimages.IsConfigType(desc.MediaType) {
 				delete(possiblyDeletedConfigs, desc.Digest)
 			}
+			return nil
 		})
 		if err != nil {
 			errs = multierror.Append(errs, err)

daemon/containerd/image_pull.go

@@ -14,13 +14,13 @@ import (
 	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/opencontainers/go-digest"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/sirupsen/logrus"
 )
 
 // PullImage initiates a pull operation. image is the repository name to pull, and
 // tagOrDigest may be either empty, or indicate a specific tag or digest to pull.
-func (i *ImageService) PullImage(ctx context.Context, image, tagOrDigest string, platform *specs.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error {
+func (i *ImageService) PullImage(ctx context.Context, image, tagOrDigest string, platform *ocispec.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error {
 	var opts []containerd.RemoteOpt
 	if platform != nil {
 		opts = append(opts, containerd.WithPlatform(platforms.Format(*platform)))
@@ -45,11 +45,11 @@ func (i *ImageService) PullImage(ctx context.Context, image, tagOrDigest string,
 		}
 	}
 
-	resolver, _ := i.newResolverFromAuthConfig(authConfig)
+	resolver, _ := i.newResolverFromAuthConfig(ctx, authConfig)
 	opts = append(opts, containerd.WithResolver(resolver))
 
 	jobs := newJobs()
-	h := images.HandlerFunc(func(ctx context.Context, desc specs.Descriptor) ([]specs.Descriptor, error) {
+	h := images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
 		if desc.MediaType != images.MediaTypeDockerSchema1Manifest {
 			jobs.Add(desc)
 		}

daemon/containerd/image_push.go

@@ -62,7 +62,7 @@ func (i *ImageService) PushImage(ctx context.Context, targetRef reference.Named,
 	target := img.Target
 	store := i.client.ContentStore()
 
-	resolver, tracker := i.newResolverFromAuthConfig(authConfig)
+	resolver, tracker := i.newResolverFromAuthConfig(ctx, authConfig)
 	progress := pushProgress{Tracker: tracker}
 	jobsQueue := newJobs()
 	finishProgress := jobsQueue.showProgress(ctx, out, combinedProgress([]progressUpdater{

daemon/containerd/image_snapshot.go

@@ -3,16 +3,17 @@ package containerd
 import (
 	"context"
 
+	"github.com/containerd/containerd"
 	containerdimages "github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/leases"
 	"github.com/containerd/containerd/platforms"
 	"github.com/opencontainers/image-spec/identity"
-	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // PrepareSnapshot prepares a snapshot from a parent image for a container
-func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, parentImage string, platform *v1.Platform) error {
-	desc, err := i.resolveDescriptor(ctx, parentImage)
+func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, parentImage string, platform *ocispec.Platform) error {
+	img, err := i.resolveImage(ctx, parentImage)
 	if err != nil {
 		return err
 	}
@@ -24,7 +25,19 @@ func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, parentIma
 		matcher = platforms.Only(*platform)
 	}
 
-	desc, err = containerdimages.Config(ctx, cs, desc, matcher)
+	platformImg := containerd.NewImageWithPlatform(i.client, img, matcher)
+	unpacked, err := platformImg.IsUnpacked(ctx, i.snapshotter)
+	if err != nil {
+		return err
+	}
+
+	if !unpacked {
+		if err := platformImg.Unpack(ctx, i.snapshotter); err != nil {
+			return err
+		}
+	}
+
+	desc, err := containerdimages.Config(ctx, cs, img.Target, matcher)
 	if err != nil {
 		return err
 	}

daemon/containerd/mount.go

@@ -3,9 +3,7 @@ package containerd
 import (
 	"context"
 	"fmt"
-	"os"
 
-	"github.com/containerd/containerd/mount"
 	"github.com/docker/docker/container"
 	"github.com/sirupsen/logrus"
 )
@@ -19,17 +17,13 @@ func (i *ImageService) Mount(ctx context.Context, container *container.Container
 		return err
 	}
 
-	// The temporary location will be under /var/lib/docker/... because
-	// we set the `TMPDIR`
-	root, err := os.MkdirTemp("", fmt.Sprintf("%s_rootfs-mount", container.ID))
-	if err != nil {
-		return fmt.Errorf("failed to create temp dir: %w", err)
-	}
-
-	if err := mount.All(mounts, root); err != nil {
+	var root string
+	if root, err = i.refCountMounter.Mount(mounts, container.ID); err != nil {
 		return fmt.Errorf("failed to mount %s: %w", root, err)
 	}
 
+	logrus.WithField("container", container.ID).Debugf("container mounted via snapshotter: %v", root)
+
 	container.BaseFS = root
 	return nil
 }
@@ -38,15 +32,10 @@ func (i *ImageService) Mount(ctx context.Context, container *container.Container
 func (i *ImageService) Unmount(ctx context.Context, container *container.Container) error {
 	root := container.BaseFS
 
-	if err := mount.UnmountAll(root, 0); err != nil {
+	if err := i.refCountMounter.Unmount(root); err != nil {
+		logrus.WithField("container", container.ID).WithError(err).Error("error unmounting container")
 		return fmt.Errorf("failed to unmount %s: %w", root, err)
 	}
 
-	if err := os.Remove(root); err != nil {
-		logrus.WithError(err).WithField("dir", root).Error("failed to remove mount temp dir")
-	}
-
-	container.BaseFS = ""
-
 	return nil
 }

daemon/containerd/progress.go

@@ -152,7 +152,7 @@ func (p pullProgress) UpdateProgress(ctx context.Context, ongoing *jobs, out pro
 		} else if p.ShowExists {
 			out.WriteProgress(progress.Progress{
 				ID:         stringid.TruncateID(j.Digest.Encoded()),
-				Action:     "Exists",
+				Action:     "Already exists",
 				HideCounts: true,
 				LastUpdate: true,
 			})

daemon/containerd/resolver.go

@@ -1,30 +1,42 @@
 package containerd
 
 import (
+	"context"
 	"crypto/tls"
 	"errors"
 	"net/http"
 
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/containerd/remotes/docker"
+	"github.com/containerd/containerd/version"
 	registrytypes "github.com/docker/docker/api/types/registry"
+	"github.com/docker/docker/dockerversion"
+	"github.com/docker/docker/pkg/useragent"
 	"github.com/docker/docker/registry"
 	"github.com/sirupsen/logrus"
 )
 
-func (i *ImageService) newResolverFromAuthConfig(authConfig *registrytypes.AuthConfig) (remotes.Resolver, docker.StatusTracker) {
+func (i *ImageService) newResolverFromAuthConfig(ctx context.Context, authConfig *registrytypes.AuthConfig) (remotes.Resolver, docker.StatusTracker) {
 	tracker := docker.NewInMemoryTracker()
 	hostsFn := i.registryHosts.RegistryHosts()
 
 	hosts := hostsWrapper(hostsFn, authConfig, i.registryService)
+	headers := http.Header{}
+	headers.Set("User-Agent", dockerversion.DockerUserAgent(ctx, useragent.VersionInfo{Name: "containerd-client", Version: version.Version}, useragent.VersionInfo{Name: "storage-driver", Version: i.snapshotter}))
 
 	return docker.NewResolver(docker.ResolverOptions{
 		Hosts:   hosts,
 		Tracker: tracker,
+		Headers: headers,
 	}), tracker
 }
 
-func hostsWrapper(hostsFn docker.RegistryHosts, authConfig *registrytypes.AuthConfig, regService RegistryConfigProvider) docker.RegistryHosts {
+func hostsWrapper(hostsFn docker.RegistryHosts, optAuthConfig *registrytypes.AuthConfig, regService RegistryConfigProvider) docker.RegistryHosts {
+	var authorizer docker.Authorizer
+	if optAuthConfig != nil {
+		authorizer = docker.NewDockerAuthorizer(authorizationCredsFromAuthConfig(*optAuthConfig))
+	}
+
 	return func(n string) ([]docker.RegistryHost, error) {
 		hosts, err := hostsFn(n)
 		if err != nil {
@@ -33,12 +45,7 @@ func hostsWrapper(hostsFn docker.RegistryHosts, authConfig *registrytypes.AuthCo
 
 		for i := range hosts {
 			if hosts[i].Authorizer == nil {
-				var opts []docker.AuthorizerOpt
-				if authConfig != nil {
-					opts = append(opts, authorizationCredsFromAuthConfig(*authConfig))
-				}
-				hosts[i].Authorizer = docker.NewDockerAuthorizer(opts...)
-
+				hosts[i].Authorizer = authorizer
 				isInsecure := regService.IsInsecureRegistry(hosts[i].Host)
 				if hosts[i].Client.Transport != nil && isInsecure {
 					hosts[i].Client.Transport = httpFallback{super: hosts[i].Client.Transport}
@@ -51,13 +58,16 @@ func hostsWrapper(hostsFn docker.RegistryHosts, authConfig *registrytypes.AuthCo
 
 func authorizationCredsFromAuthConfig(authConfig registrytypes.AuthConfig) docker.AuthorizerOpt {
 	cfgHost := registry.ConvertToHostname(authConfig.ServerAddress)
-	if cfgHost == registry.IndexHostname {
+	if cfgHost == "" || cfgHost == registry.IndexHostname {
 		cfgHost = registry.DefaultRegistryHost
 	}
 
 	return docker.WithAuthCreds(func(host string) (string, string, error) {
 		if cfgHost != host {
-			logrus.WithField("host", host).WithField("cfgHost", cfgHost).Warn("Host doesn't match")
+			logrus.WithFields(logrus.Fields{
+				"host":    host,
+				"cfgHost": cfgHost,
+			}).Warn("Host doesn't match")
 			return "", "", nil
 		}
 		if authConfig.IdentityToken != "" {

daemon/containerd/service.go

@@ -10,13 +10,16 @@ import (
 	"github.com/containerd/containerd/plugin"
 	"github.com/containerd/containerd/remotes/docker"
 	"github.com/containerd/containerd/snapshots"
+	"github.com/docker/distribution/reference"
 	imagetypes "github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/container"
 	daemonevents "github.com/docker/docker/daemon/events"
 	"github.com/docker/docker/daemon/images"
+	"github.com/docker/docker/daemon/snapshotter"
 	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/layer"
+	"github.com/docker/docker/registry"
 	"github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/identity"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -33,6 +36,7 @@ type ImageService struct {
 	registryService RegistryConfigProvider
 	eventsService   *daemonevents.Events
 	pruneRunning    atomic.Bool
+	refCountMounter snapshotter.Mounter
 }
 
 type RegistryHostsProvider interface {
@@ -41,15 +45,17 @@ type RegistryHostsProvider interface {
 
 type RegistryConfigProvider interface {
 	IsInsecureRegistry(host string) bool
+	ResolveRepository(name reference.Named) (*registry.RepositoryInfo, error)
 }
 
 type ImageServiceConfig struct {
-	Client        *containerd.Client
-	Containers    container.Store
-	Snapshotter   string
-	HostsProvider RegistryHostsProvider
-	Registry      RegistryConfigProvider
-	EventsService *daemonevents.Events
+	Client          *containerd.Client
+	Containers      container.Store
+	Snapshotter     string
+	HostsProvider   RegistryHostsProvider
+	Registry        RegistryConfigProvider
+	EventsService   *daemonevents.Events
+	RefCountMounter snapshotter.Mounter
 }
 
 // NewService creates a new ImageService.
@@ -61,6 +67,7 @@ func NewService(config ImageServiceConfig) *ImageService {
 		registryHosts:   config.HostsProvider,
 		registryService: config.Registry,
 		eventsService:   config.EventsService,
+		refCountMounter: config.RefCountMounter,
 	}
 }
 

daemon/containerd/soft_delete.go

@@ -7,6 +7,7 @@ import (
 	containerdimages "github.com/containerd/containerd/images"
 	"github.com/docker/docker/errdefs"
 	"github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
 
@@ -18,10 +19,10 @@ func (i *ImageService) softImageDelete(ctx context.Context, img containerdimages
 
 	// If the image already exists, persist it as dangling image
 	// but only if no other image has the same target.
-	digest := img.Target.Digest.String()
-	imgs, err := is.List(ctx, "target.digest=="+digest)
+	dgst := img.Target.Digest.String()
+	imgs, err := is.List(ctx, "target.digest=="+dgst)
 	if err != nil {
-		return errdefs.System(errors.Wrapf(err, "failed to check if there are images targeting digest %s", digest))
+		return errdefs.System(errors.Wrapf(err, "failed to check if there are images targeting digest %s", dgst))
 	}
 
 	// From this point explicitly ignore the passed context
@@ -29,19 +30,12 @@ func (i *ImageService) softImageDelete(ctx context.Context, img containerdimages
 
 	// Create dangling image if this is the last image pointing to this target.
 	if len(imgs) == 1 {
-		danglingImage := img
-
-		danglingImage.Name = danglingImageName(img.Target.Digest)
-		delete(danglingImage.Labels, "io.containerd.image.name")
-		delete(danglingImage.Labels, "org.opencontainers.image.ref.name")
-
-		_, err = is.Create(context.Background(), danglingImage)
+		err = i.ensureDanglingImage(context.Background(), img)
 
 		// Error out in case we couldn't persist the old image.
-		// If it already exists, then just continue.
-		if err != nil && !cerrdefs.IsAlreadyExists(err) {
+		if err != nil {
 			return errdefs.System(errors.Wrapf(err, "failed to create a dangling image for the replaced image %s with digest %s",
-				danglingImage.Name, danglingImage.Target.Digest.String()))
+				img.Name, img.Target.Digest.String()))
 		}
 	}
 
@@ -56,6 +50,29 @@ func (i *ImageService) softImageDelete(ctx context.Context, img containerdimages
 	return nil
 }
 
+func (i *ImageService) ensureDanglingImage(ctx context.Context, from containerdimages.Image) error {
+	danglingImage := from
+
+	danglingImage.Labels = make(map[string]string)
+	for k, v := range from.Labels {
+		switch k {
+		case containerdimages.AnnotationImageName, ocispec.AnnotationRefName:
+			// Don't copy name labels.
+		default:
+			danglingImage.Labels[k] = v
+		}
+	}
+	danglingImage.Name = danglingImageName(from.Target.Digest)
+
+	_, err := i.client.ImageService().Create(context.Background(), danglingImage)
+	// If it already exists, then just continue.
+	if cerrdefs.IsAlreadyExists(err) {
+		return nil
+	}
+
+	return err
+}
+
 func danglingImageName(digest digest.Digest) string {
 	return "moby-dangling@" + digest.String()
 }

daemon/create.go

@@ -19,7 +19,7 @@ import (
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/runconfig"
-	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/opencontainers/selinux/go-selinux"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -74,7 +74,7 @@ func (daemon *Daemon) containerCreate(ctx context.Context, opts createOpts) (con
 		}
 		if img != nil {
 			p := maximumSpec()
-			imgPlat := v1.Platform{
+			imgPlat := ocispec.Platform{
 				OS:           img.OS,
 				Architecture: img.Architecture,
 				Variant:      img.Variant,
@@ -117,7 +117,7 @@ func (daemon *Daemon) create(ctx context.Context, opts createOpts) (retC *contai
 	var (
 		ctr         *container.Container
 		img         *image.Image
-		imgManifest *v1.Descriptor
+		imgManifest *ocispec.Descriptor
 		imgID       image.ID
 		err         error
 		os          = runtime.GOOS
@@ -345,7 +345,7 @@ func verifyNetworkingConfig(nwConfig *networktypes.NetworkingConfig) error {
 }
 
 // maximumSpec returns the distribution platform with maximum compatibility for the current node.
-func maximumSpec() v1.Platform {
+func maximumSpec() ocispec.Platform {
 	p := platforms.DefaultSpec()
 	if p.Architecture == "amd64" {
 		p.Variant = archvariant.AMD64Variant()

daemon/daemon.go

@@ -40,6 +40,7 @@ import (
 	"github.com/docker/docker/daemon/images"
 	dlogger "github.com/docker/docker/daemon/logger"
 	"github.com/docker/docker/daemon/network"
+	"github.com/docker/docker/daemon/snapshotter"
 	"github.com/docker/docker/daemon/stats"
 	"github.com/docker/docker/distribution"
 	dmetadata "github.com/docker/docker/distribution/metadata"
@@ -129,6 +130,13 @@ type Daemon struct {
 	// It stores metadata for the content store (used for manifest caching)
 	// This needs to be closed on daemon exit
 	mdDB *bbolt.DB
+
+	usesSnapshotter bool
+}
+
+// ID returns the daemon id
+func (daemon *Daemon) ID() string {
+	return daemon.id
 }
 
 // StoreHosts stores the addresses the daemon is listening on
@@ -153,16 +161,7 @@ func (daemon *Daemon) Features() *map[string]bool {
 
 // UsesSnapshotter returns true if feature flag to use containerd snapshotter is enabled
 func (daemon *Daemon) UsesSnapshotter() bool {
-	// TEST_INTEGRATION_USE_SNAPSHOTTER is used for integration tests only.
-	if os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "" {
-		return true
-	}
-	if daemon.configStore.Features != nil {
-		if b, ok := daemon.configStore.Features["containerd-snapshotter"]; ok {
-			return b
-		}
-	}
-	return false
+	return daemon.usesSnapshotter
 }
 
 // RegistryHosts returns registry configuration in containerd resolvers format
@@ -421,6 +420,8 @@ func (daemon *Daemon) restore() error {
 					if es != nil {
 						ces.ExitCode = int(es.ExitCode())
 						ces.ExitedAt = es.ExitTime()
+					} else {
+						ces.ExitCode = 255
 					}
 					c.SetStopped(&ces)
 					daemon.Cleanup(c)
@@ -796,6 +797,13 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		startupDone: make(chan struct{}),
 	}
 
+	// TEST_INTEGRATION_USE_SNAPSHOTTER is used for integration tests only.
+	if os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "" {
+		d.usesSnapshotter = true
+	} else {
+		d.usesSnapshotter = config.Features["containerd-snapshotter"]
+	}
+
 	// Ensure the daemon is properly shutdown if there is a failure during
 	// initialization
 	defer func() {
@@ -1018,12 +1026,13 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 			return nil, err
 		}
 		d.imageService = ctrd.NewService(ctrd.ImageServiceConfig{
-			Client:        d.containerdCli,
-			Containers:    d.containers,
-			Snapshotter:   driverName,
-			HostsProvider: d,
-			Registry:      d.registryService,
-			EventsService: d.EventsService,
+			Client:          d.containerdCli,
+			Containers:      d.containers,
+			Snapshotter:     driverName,
+			HostsProvider:   d,
+			Registry:        d.registryService,
+			EventsService:   d.EventsService,
+			RefCountMounter: snapshotter.NewMounter(config.Root, driverName, idMapping),
 		})
 	} else {
 		layerStore, err := layer.NewStoreFromOptions(layer.StoreOptions{

daemon/daemon_unix.go

@@ -105,7 +105,10 @@ func getMemoryResources(config containertypes.Resources) *specs.LinuxMemory {
 		memory.KernelTCP = &config.KernelMemoryTCP
 	}
 
-	return &memory
+	if memory != (specs.LinuxMemory{}) {
+		return &memory
+	}
+	return nil
 }
 
 func getPidsLimit(config containertypes.Resources) *specs.LinuxPids {
@@ -127,7 +130,7 @@ func getCPUResources(config containertypes.Resources) (*specs.LinuxCPU, error) {
 	if config.CPUShares < 0 {
 		return nil, fmt.Errorf("shares: invalid argument")
 	}
-	if config.CPUShares >= 0 {
+	if config.CPUShares > 0 {
 		shares := uint64(config.CPUShares)
 		cpu.Shares = &shares
 	}
@@ -168,7 +171,10 @@ func getCPUResources(config containertypes.Resources) (*specs.LinuxCPU, error) {
 		cpu.RealtimeRuntime = &c
 	}
 
-	return &cpu, nil
+	if cpu != (specs.LinuxCPU{}) {
+		return &cpu, nil
+	}
+	return nil, nil
 }
 
 func getBlkioWeightDevices(config containertypes.Resources) ([]specs.LinuxWeightDevice, error) {
@@ -695,7 +701,7 @@ func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes.
 	if hostConfig.CgroupParent != "" && UsingSystemd(daemon.configStore) {
 		// CgroupParent for systemd cgroup should be named as "xxx.slice"
 		if len(hostConfig.CgroupParent) <= 6 || !strings.HasSuffix(hostConfig.CgroupParent, ".slice") {
-			return warnings, fmt.Errorf("cgroup-parent for systemd cgroup should be a valid slice named as \"xxx.slice\"")
+			return warnings, fmt.Errorf(`cgroup-parent for systemd cgroup should be a valid slice named as "xxx.slice"`)
 		}
 	}
 	if hostConfig.Runtime == "" {
@@ -748,7 +754,7 @@ func verifyDaemonSettings(conf *config.Config) error {
 	}
 	if conf.CgroupParent != "" && UsingSystemd(conf) {
 		if len(conf.CgroupParent) <= 6 || !strings.HasSuffix(conf.CgroupParent, ".slice") {
-			return fmt.Errorf("cgroup-parent for systemd cgroup should be a valid slice named as \"xxx.slice\"")
+			return fmt.Errorf(`cgroup-parent for systemd cgroup should be a valid slice named as "xxx.slice"`)
 		}
 	}
 
@@ -1063,7 +1069,7 @@ func initBridgeDriver(controller *libnetwork.Controller, config *config.Config)
 		libnetwork.NetworkOptionIpam("default", "", v4Conf, v6Conf, nil),
 		libnetwork.NetworkOptionDeferIPv6Alloc(deferIPv6Alloc))
 	if err != nil {
-		return fmt.Errorf("Error creating default \"bridge\" network: %v", err)
+		return fmt.Errorf(`error creating default "bridge" network: %v`, err)
 	}
 	return nil
 }
@@ -1394,19 +1400,13 @@ func (daemon *Daemon) registerLinks(container *container.Container, hostConfig *
 // conditionalMountOnStart is a platform specific helper function during the
 // container start to call mount.
 func (daemon *Daemon) conditionalMountOnStart(container *container.Container) error {
-	if !daemon.UsesSnapshotter() {
-		return daemon.Mount(container)
-	}
-	return nil
+	return daemon.Mount(container)
 }
 
 // conditionalUnmountOnCleanup is a platform specific helper function called
 // during the cleanup of a container to unmount.
 func (daemon *Daemon) conditionalUnmountOnCleanup(container *container.Container) error {
-	if !daemon.UsesSnapshotter() {
-		return daemon.Unmount(container)
-	}
-	return nil
+	return daemon.Unmount(container)
 }
 
 // setDefaultIsolation determines the default isolation mode for the

daemon/daemon_unix_test.go

@@ -148,7 +148,7 @@ func TestParseSecurityOptWithDeprecatedColon(t *testing.T) {
 		t.Fatalf("Unexpected parseSecurityOpt error: %v", err)
 	}
 	if opts.AppArmorProfile != "test_profile" {
-		t.Fatalf("Unexpected AppArmorProfile, expected: \"test_profile\", got %q", opts.AppArmorProfile)
+		t.Fatalf(`Unexpected AppArmorProfile, expected: "test_profile", got %q`, opts.AppArmorProfile)
 	}
 
 	// test seccomp

daemon/graphdriver/fuse-overlayfs/fuseoverlayfs_test.go

@@ -9,7 +9,6 @@ import (
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/daemon/graphdriver/graphtest"
 	"github.com/docker/docker/pkg/archive"
-	"github.com/docker/docker/pkg/reexec"
 )
 
 func init() {
@@ -17,8 +16,6 @@ func init() {
 	// errors or hangs to be debugged directly from the test process.
 	untar = archive.UntarUncompressed
 	graphdriver.ApplyUncompressedLayer = archive.ApplyUncompressedLayer
-
-	reexec.Init()
 }
 
 // This avoids creating a new driver for each test if all tests are run

daemon/graphdriver/overlay2/overlay_test.go

@@ -10,7 +10,6 @@ import (
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/daemon/graphdriver/graphtest"
 	"github.com/docker/docker/pkg/archive"
-	"github.com/docker/docker/pkg/reexec"
 )
 
 func init() {
@@ -18,8 +17,6 @@ func init() {
 	// errors or hangs to be debugged directly from the test process.
 	untar = archive.UntarUncompressed
 	graphdriver.ApplyUncompressedLayer = archive.ApplyUncompressedLayer
-
-	reexec.Init()
 }
 
 func skipIfNaive(t *testing.T) {

daemon/graphdriver/vfs/vfs_test.go

@@ -7,14 +7,8 @@ import (
 	"testing"
 
 	"github.com/docker/docker/daemon/graphdriver/graphtest"
-
-	"github.com/docker/docker/pkg/reexec"
 )
 
-func init() {
-	reexec.Init()
-}
-
 // This avoids creating a new driver for each test if all tests are run
 // Make sure to put new tests between TestVfsSetup and TestVfsTeardown
 func TestVfsSetup(t *testing.T) {

daemon/graphdriver/zfs/zfs.go

@@ -16,7 +16,7 @@ import (
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/parsers"
-	zfs "github.com/mistifyio/go-zfs"
+	zfs "github.com/mistifyio/go-zfs/v3"
 	"github.com/moby/locker"
 	"github.com/moby/sys/mount"
 	"github.com/moby/sys/mountinfo"

daemon/health.go

@@ -160,7 +160,6 @@ func (p *cmdProbe) run(ctx context.Context, d *Daemon, cntr *container.Container
 		info.Lock()
 		defer info.Unlock()
 		if info.ExitCode == nil {
-			info.Unlock()
 			return 0, fmt.Errorf("healthcheck for container %s has no exit code", cntr.ID)
 		}
 		return *info.ExitCode, nil

daemon/image_service.go

@@ -16,7 +16,8 @@ import (
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/layer"
 	"github.com/docker/docker/pkg/archive"
-	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // ImageService is a temporary interface to assist in the migration to the
@@ -25,9 +26,9 @@ import (
 type ImageService interface {
 	// Images
 
-	PullImage(ctx context.Context, name, tag string, platform *v1.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error
+	PullImage(ctx context.Context, name, tag string, platform *ocispec.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error
 	PushImage(ctx context.Context, ref reference.Named, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error
-	CreateImage(config []byte, parent string) (builder.Image, error)
+	CreateImage(ctx context.Context, config []byte, parent string, contentStoreDigest digest.Digest) (builder.Image, error)
 	ImageDelete(ctx context.Context, imageRef string, force, prune bool) ([]types.ImageDeleteResponseItem, error)
 	ExportImage(ctx context.Context, names []string, outStream io.Writer) error
 	PerformWithBaseFS(ctx context.Context, c *container.Container, fn func(string) error) error
@@ -36,7 +37,7 @@ type ImageService interface {
 	LogImageEvent(imageID, refName, action string)
 	CountImages() int
 	ImagesPrune(ctx context.Context, pruneFilters filters.Args) (*types.ImagesPruneReport, error)
-	ImportImage(ctx context.Context, ref reference.Named, platform *v1.Platform, msg string, layerReader io.Reader, changes []string) (image.ID, error)
+	ImportImage(ctx context.Context, ref reference.Named, platform *ocispec.Platform, msg string, layerReader io.Reader, changes []string) (image.ID, error)
 	TagImage(ctx context.Context, imageID image.ID, newTag reference.Named) error
 	GetImage(ctx context.Context, refOrID string, options imagetype.GetImageOpts) (*image.Image, error)
 	ImageHistory(ctx context.Context, name string) ([]*imagetype.HistoryResponseItem, error)
@@ -45,8 +46,8 @@ type ImageService interface {
 
 	// Containerd related methods
 
-	PrepareSnapshot(ctx context.Context, id string, image string, platform *v1.Platform) error
-	GetImageManifest(ctx context.Context, refOrID string, options imagetype.GetImageOpts) (*v1.Descriptor, error)
+	PrepareSnapshot(ctx context.Context, id string, image string, platform *ocispec.Platform) error
+	GetImageManifest(ctx context.Context, refOrID string, options imagetype.GetImageOpts) (*ocispec.Descriptor, error)
 
 	// Layers
 

daemon/images/image.go

@@ -17,8 +17,7 @@ import (
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/layer"
 	"github.com/opencontainers/go-digest"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
-	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -40,19 +39,19 @@ func (e ErrImageDoesNotExist) Error() string {
 func (e ErrImageDoesNotExist) NotFound() {}
 
 type manifestList struct {
-	Manifests []specs.Descriptor `json:"manifests"`
+	Manifests []ocispec.Descriptor `json:"manifests"`
 }
 
 type manifest struct {
-	Config specs.Descriptor `json:"config"`
+	Config ocispec.Descriptor `json:"config"`
 }
 
-func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, image string, platform *v1.Platform) error {
+func (i *ImageService) PrepareSnapshot(ctx context.Context, id string, image string, platform *ocispec.Platform) error {
 	// Only makes sense when conatinerd image store is used
 	panic("not implemented")
 }
 
-func (i *ImageService) manifestMatchesPlatform(ctx context.Context, img *image.Image, platform specs.Platform) (bool, error) {
+func (i *ImageService) manifestMatchesPlatform(ctx context.Context, img *image.Image, platform ocispec.Platform) (bool, error) {
 	logger := logrus.WithField("image", img.ID).WithField("desiredPlatform", platforms.Format(platform))
 
 	ls, leaseErr := i.leases.ListResources(ctx, leases.Lease{ID: imageKey(img.ID().String())})
@@ -81,7 +80,7 @@ func (i *ImageService) manifestMatchesPlatform(ctx context.Context, img *image.I
 			continue
 		}
 
-		ra, err := i.content.ReaderAt(ctx, specs.Descriptor{Digest: digest.Digest(r.ID)})
+		ra, err := i.content.ReaderAt(ctx, ocispec.Descriptor{Digest: digest.Digest(r.ID)})
 		if err != nil {
 			if cerrdefs.IsNotFound(err) {
 				continue
@@ -107,12 +106,12 @@ func (i *ImageService) manifestMatchesPlatform(ctx context.Context, img *image.I
 
 		for _, md := range ml.Manifests {
 			switch md.MediaType {
-			case specs.MediaTypeImageManifest, images.MediaTypeDockerSchema2Manifest:
+			case ocispec.MediaTypeImageManifest, images.MediaTypeDockerSchema2Manifest:
 			default:
 				continue
 			}
 
-			p := specs.Platform{
+			p := ocispec.Platform{
 				Architecture: md.Platform.Architecture,
 				OS:           md.Platform.OS,
 				Variant:      md.Platform.Variant,
@@ -124,7 +123,7 @@ func (i *ImageService) manifestMatchesPlatform(ctx context.Context, img *image.I
 
 			// Here we have a platform match for the referenced manifest, let's make sure the manifest is actually for the image config we are using.
 
-			ra, err := i.content.ReaderAt(ctx, specs.Descriptor{Digest: md.Digest})
+			ra, err := i.content.ReaderAt(ctx, ocispec.Descriptor{Digest: md.Digest})
 			if err != nil {
 				logger.WithField("otherDigest", md.Digest).WithError(err).Error("Could not get reader for manifest")
 				continue
@@ -192,7 +191,7 @@ func (i *ImageService) GetImage(ctx context.Context, refOrID string, options ima
 	return img, nil
 }
 
-func (i *ImageService) GetImageManifest(ctx context.Context, refOrID string, options imagetypes.GetImageOpts) (*v1.Descriptor, error) {
+func (i *ImageService) GetImageManifest(ctx context.Context, refOrID string, options imagetypes.GetImageOpts) (*ocispec.Descriptor, error) {
 	panic("not implemented")
 }
 
@@ -202,7 +201,7 @@ func (i *ImageService) getImage(ctx context.Context, refOrID string, options ima
 			return
 		}
 
-		imgPlat := specs.Platform{
+		imgPlat := ocispec.Platform{
 			OS:           retImg.OS,
 			Architecture: retImg.Architecture,
 			Variant:      retImg.Variant,
@@ -272,16 +271,16 @@ func (i *ImageService) getImage(ctx context.Context, refOrID string, options ima
 // The reason for this is that CPU variant is not even if the official image config spec as of this writing.
 // See: https://github.com/opencontainers/image-spec/pull/809
 // Since Docker tends to compare platforms from the image config, we need to handle this case.
-func OnlyPlatformWithFallback(p specs.Platform) platforms.Matcher {
+func OnlyPlatformWithFallback(p ocispec.Platform) platforms.Matcher {
 	return &onlyFallbackMatcher{only: platforms.Only(p), p: platforms.Normalize(p)}
 }
 
 type onlyFallbackMatcher struct {
 	only platforms.Matcher
-	p    specs.Platform
+	p    ocispec.Platform
 }
 
-func (m *onlyFallbackMatcher) Match(other specs.Platform) bool {
+func (m *onlyFallbackMatcher) Match(other ocispec.Platform) bool {
 	if m.only.Match(other) {
 		// It matches, no reason to fallback
 		return true

daemon/images/image_builder.go

@@ -19,7 +19,8 @@ import (
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/docker/pkg/system"
 	registrypkg "github.com/docker/docker/registry"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -30,6 +31,10 @@ type roLayer struct {
 	roLayer    layer.Layer
 }
 
+func (l *roLayer) ContentStoreDigest() digest.Digest {
+	return ""
+}
+
 func (l *roLayer) DiffID() layer.DiffID {
 	if l.roLayer == nil {
 		return layer.DigestSHA256EmptyTar
@@ -144,7 +149,7 @@ func newROLayerForImage(img *image.Image, layerStore layer.Store) (builder.ROLay
 }
 
 // TODO: could this use the regular daemon PullImage ?
-func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConfigs map[string]registry.AuthConfig, output io.Writer, platform *specs.Platform) (*image.Image, error) {
+func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConfigs map[string]registry.AuthConfig, output io.Writer, platform *ocispec.Platform) (*image.Image, error) {
 	ref, err := reference.ParseNormalizedNamed(name)
 	if err != nil {
 		return nil, err
@@ -169,7 +174,7 @@ func (i *ImageService) pullForBuilder(ctx context.Context, name string, authConf
 
 	img, err := i.GetImage(ctx, name, imagetypes.GetImageOpts{Platform: platform})
 	if errdefs.IsNotFound(err) && img != nil && platform != nil {
-		imgPlat := specs.Platform{
+		imgPlat := ocispec.Platform{
 			OS:           img.OS,
 			Architecture: img.BaseImgArch(),
 			Variant:      img.BaseImgVariant(),
@@ -241,7 +246,7 @@ func (i *ImageService) GetImageAndReleasableLayer(ctx context.Context, refOrID s
 // CreateImage creates a new image by adding a config and ID to the image store.
 // This is similar to LoadImage() except that it receives JSON encoded bytes of
 // an image instead of a tar archive.
-func (i *ImageService) CreateImage(config []byte, parent string) (builder.Image, error) {
+func (i *ImageService) CreateImage(ctx context.Context, config []byte, parent string, _ digest.Digest) (builder.Image, error) {
 	id, err := i.imageStore.Create(config)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to create image")

daemon/images/image_import.go

@@ -16,7 +16,7 @@ import (
 	"github.com/docker/docker/layer"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/system"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // ImportImage imports an image, getting the archived layer data from layerReader.
@@ -26,7 +26,7 @@ import (
 // If the platform is nil, the default host platform is used.
 // Message is used as the image's history comment.
 // Image configuration is derived from the dockerfile instructions in changes.
-func (i *ImageService) ImportImage(ctx context.Context, newRef reference.Named, platform *specs.Platform, msg string, layerReader io.Reader, changes []string) (image.ID, error) {
+func (i *ImageService) ImportImage(ctx context.Context, newRef reference.Named, platform *ocispec.Platform, msg string, layerReader io.Reader, changes []string) (image.ID, error) {
 	if platform == nil {
 		def := platforms.DefaultSpec()
 		platform = &def

daemon/images/image_pull.go

@@ -17,14 +17,14 @@ import (
 	"github.com/docker/docker/pkg/progress"
 	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/opencontainers/go-digest"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
 // PullImage initiates a pull operation. image is the repository name to pull, and
 // tag may be either empty, or indicate a specific tag to pull.
-func (i *ImageService) PullImage(ctx context.Context, image, tag string, platform *specs.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error {
+func (i *ImageService) PullImage(ctx context.Context, image, tag string, platform *ocispec.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error {
 	start := time.Now()
 	// Special case: "pull -a" may send an image name with a
 	// trailing :. This is ugly, but let's not break API
@@ -63,7 +63,7 @@ func (i *ImageService) PullImage(ctx context.Context, image, tag string, platfor
 		// we allow the image to have a non-matching architecture. The code
 		// below checks for this situation, and returns a warning to the client,
 		// as well as logging it to the daemon logs.
-		img, err := i.GetImage(ctx, image, imagetypes.GetImageOpts{Platform: platform})
+		img, err := i.GetImage(ctx, ref.String(), imagetypes.GetImageOpts{Platform: platform})
 
 		// Note that this is a special case where GetImage returns both an image
 		// and an error: https://github.com/docker/docker/blob/v20.10.7/daemon/images/image.go#L175-L183
@@ -79,7 +79,7 @@ func (i *ImageService) PullImage(ctx context.Context, image, tag string, platfor
 	return nil
 }
 
-func (i *ImageService) pullImageWithReference(ctx context.Context, ref reference.Named, platform *specs.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error {
+func (i *ImageService) pullImageWithReference(ctx context.Context, ref reference.Named, platform *ocispec.Platform, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error {
 	// Include a buffer so that slow client connections don't affect
 	// transfer performance.
 	progressChan := make(chan progress.Progress, 100)

daemon/images/images_test.go

@@ -3,30 +3,30 @@ package images
 import (
 	"testing"
 
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"gotest.tools/v3/assert"
 )
 
 func TestOnlyPlatformWithFallback(t *testing.T) {
-	p := specs.Platform{
+	p := ocispec.Platform{
 		OS:           "linux",
 		Architecture: "arm",
 		Variant:      "v8",
 	}
 
 	// Check no variant
-	assert.Assert(t, OnlyPlatformWithFallback(p).Match(specs.Platform{
+	assert.Assert(t, OnlyPlatformWithFallback(p).Match(ocispec.Platform{
 		OS:           p.OS,
 		Architecture: p.Architecture,
 	}))
 	// check with variant
-	assert.Assert(t, OnlyPlatformWithFallback(p).Match(specs.Platform{
+	assert.Assert(t, OnlyPlatformWithFallback(p).Match(ocispec.Platform{
 		OS:           p.OS,
 		Architecture: p.Architecture,
 		Variant:      p.Variant,
 	}))
 	// Make sure non-matches are false.
-	assert.Assert(t, !OnlyPlatformWithFallback(p).Match(specs.Platform{
+	assert.Assert(t, !OnlyPlatformWithFallback(p).Match(ocispec.Platform{
 		OS:           p.OS,
 		Architecture: "amd64",
 	}))

daemon/images/store_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/containerd/containerd/namespaces"
 	"github.com/docker/docker/image"
 	"github.com/opencontainers/go-digest"
-	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"go.etcd.io/bbolt"
 	"gotest.tools/v3/assert"
 	is "gotest.tools/v3/assert/cmp"
@@ -96,7 +96,7 @@ func TestContentStoreForPull(t *testing.T) {
 	}
 
 	data := []byte(`{}`)
-	desc := v1.Descriptor{
+	desc := ocispec.Descriptor{
 		Digest: digest.Canonical.FromBytes(data),
 		Size:   int64(len(data)),
 	}

daemon/monitor.go

@@ -39,7 +39,10 @@ func (daemon *Daemon) handleContainerExit(c *container.Container, e *libcontaine
 		es, err := tsk.Delete(ctx)
 		cancel()
 		if err != nil {
-			logrus.WithError(err).WithField("container", c.ID).Warnf("failed to delete container from containerd")
+			logrus.WithFields(logrus.Fields{
+				logrus.ErrorKey: err,
+				"container":     c.ID,
+			}).Warn("failed to delete container from containerd")
 		} else {
 			exitStatus = container.ExitStatus{
 				ExitCode: int(es.ExitCode()),
@@ -66,29 +69,32 @@ func (daemon *Daemon) handleContainerExit(c *container.Container, e *libcontaine
 	execDuration := time.Since(c.StartedAt)
 	restart, wait, err := c.RestartManager().ShouldRestart(uint32(exitStatus.ExitCode), daemonShutdown || c.HasBeenManuallyStopped, execDuration)
 	if err != nil {
-		logrus.WithError(err).
-			WithField("container", c.ID).
-			WithField("restartCount", c.RestartCount).
-			WithField("exitStatus", exitStatus).
-			WithField("daemonShuttingDown", daemonShutdown).
-			WithField("hasBeenManuallyStopped", c.HasBeenManuallyStopped).
-			WithField("execDuration", execDuration).
-			Warn("ShouldRestart failed, container will not be restarted")
+		logrus.WithFields(logrus.Fields{
+			logrus.ErrorKey:          err,
+			"container":              c.ID,
+			"restartCount":           c.RestartCount,
+			"exitStatus":             exitStatus,
+			"daemonShuttingDown":     daemonShutdown,
+			"hasBeenManuallyStopped": c.HasBeenManuallyStopped,
+			"execDuration":           execDuration,
+		}).Warn("ShouldRestart failed, container will not be restarted")
 		restart = false
 	}
 
 	attributes := map[string]string{
-		"exitCode": strconv.Itoa(exitStatus.ExitCode),
+		"exitCode":     strconv.Itoa(exitStatus.ExitCode),
+		"execDuration": strconv.Itoa(int(execDuration.Seconds())),
 	}
 	daemon.Cleanup(c)
 
 	if restart {
 		c.RestartCount++
-		logrus.WithField("container", c.ID).
-			WithField("restartCount", c.RestartCount).
-			WithField("exitStatus", exitStatus).
-			WithField("manualRestart", c.HasBeenManuallyRestarted).
-			Debug("Restarting container")
+		logrus.WithFields(logrus.Fields{
+			"container":     c.ID,
+			"restartCount":  c.RestartCount,
+			"exitStatus":    exitStatus,
+			"manualRestart": c.HasBeenManuallyRestarted,
+		}).Debug("Restarting container")
 		c.SetRestarting(&exitStatus)
 	} else {
 		c.SetStopped(&exitStatus)
@@ -169,7 +175,7 @@ func (daemon *Daemon) ProcessEvent(id string, e libcontainerdtypes.EventType, ei
 			// Remove the exec command from the container's store only and not the
 			// daemon's store so that the exec command can be inspected. Remove it
 			// before mutating execConfig to maintain the invariant that
-			// c.ExecCommands only contain execs in the Running state.
+			// c.ExecCommands only contains execs that have not exited.
 			c.ExecCommands.Delete(execConfig.ID)
 
 			execConfig.ExitCode = &ec
@@ -185,14 +191,25 @@ func (daemon *Daemon) ProcessEvent(id string, e libcontainerdtypes.EventType, ei
 
 			exitCode = ec
 
-			go func() {
-				if _, err := execConfig.Process.Delete(context.Background()); err != nil {
-					logrus.WithError(err).WithFields(logrus.Fields{
-						"container": ei.ContainerID,
-						"process":   ei.ProcessID,
-					}).Warn("failed to delete process")
-				}
-			}()
+			// If the exec failed at start in such a way that containerd
+			// publishes an exit event for it, we will race processing the event
+			// with daemon.ContainerExecStart() removing the exec from
+			// c.ExecCommands. If we win the race, we will find that there is no
+			// process to clean up. (And ContainerExecStart will clobber the
+			// exit code we set.) Prevent a nil-dereferenc panic in that
+			// situation to restore the status quo where this is merely a
+			// logical race condition.
+			if execConfig.Process != nil {
+				go func() {
+					if _, err := execConfig.Process.Delete(context.Background()); err != nil {
+						logrus.WithFields(logrus.Fields{
+							logrus.ErrorKey: err,
+							"container":     ei.ContainerID,
+							"process":       ei.ProcessID,
+						}).Warn("failed to delete process")
+					}
+				}()
+			}
 		}
 		attributes := map[string]string{
 			"execID":   ei.ProcessID,
@@ -210,8 +227,10 @@ func (daemon *Daemon) ProcessEvent(id string, e libcontainerdtypes.EventType, ei
 				if errdefs.IsNotFound(err) {
 					// The container was started by not-docker and so could have been deleted by
 					// not-docker before we got around to loading it from containerd.
-					logrus.WithField("container", c.ID).WithError(err).
-						Debug("could not load containerd container for start event")
+					logrus.WithFields(logrus.Fields{
+						logrus.ErrorKey: err,
+						"container":     c.ID,
+					}).Debug("could not load containerd container for start event")
 					return nil
 				}
 				return err
@@ -219,8 +238,10 @@ func (daemon *Daemon) ProcessEvent(id string, e libcontainerdtypes.EventType, ei
 			tsk, err := ctr.Task(context.Background())
 			if err != nil {
 				if errdefs.IsNotFound(err) {
-					logrus.WithField("container", c.ID).WithError(err).
-						Debug("failed to load task for externally-started container")
+					logrus.WithFields(logrus.Fields{
+						logrus.ErrorKey: err,
+						"container":     c.ID,
+					}).Debug("failed to load task for externally-started container")
 					return nil
 				}
 				return err
@@ -285,5 +306,5 @@ func (daemon *Daemon) autoRemove(c *container.Container) {
 		return
 	}
 
-	logrus.WithError(err).WithField("container", c.ID).Error("error removing container")
+	logrus.WithFields(logrus.Fields{logrus.ErrorKey: err, "container": c.ID}).Error("error removing container")
 }