Merge pull request #299 from thaJeztah/19.03_backport_scrub

[19.03 backport] DebugRequestMiddleware: unconditionally scrub data field

Dockerfile

@@ -26,7 +26,7 @@
 
 ARG CROSS="false"
 
-FROM golang:1.12.4 AS base
+FROM golang:1.12.5 AS base
 # allow replacing httpredir or deb mirror
 ARG APT_MIRROR=deb.debian.org
 RUN sed -ri "s/(httpredir|deb).debian.org/$APT_MIRROR/g" /etc/apt/sources.list
@@ -51,6 +51,11 @@ RUN apt-get update && apt-get install -y \
 	&& make PREFIX=/build/ install-criu
 
 FROM base AS registry
+# Install two versions of the registry. The first is an older version that
+# only supports schema1 manifests. The second is a newer version that supports
+# both. This allows integration-cli tests to cover push/pull with both schema1
+# and schema2 manifests.
+ENV REGISTRY_COMMIT_SCHEMA1 ec87e9b6971d831f0eff752ddb54fb64693e51cd
 ENV REGISTRY_COMMIT 47a064d4195a9b56133891bbb13620c3ac83a827
 RUN set -x \
 	&& export GOPATH="$(mktemp -d)" \
@@ -58,6 +63,13 @@ RUN set -x \
 	&& (cd "$GOPATH/src/github.com/docker/distribution" && git checkout -q "$REGISTRY_COMMIT") \
 	&& GOPATH="$GOPATH/src/github.com/docker/distribution/Godeps/_workspace:$GOPATH" \
 		go build -buildmode=pie -o /build/registry-v2 github.com/docker/distribution/cmd/registry \
+	&& case $(dpkg --print-architecture) in \
+		amd64|ppc64*|s390x) \
+		(cd "$GOPATH/src/github.com/docker/distribution" && git checkout -q "$REGISTRY_COMMIT_SCHEMA1"); \
+		GOPATH="$GOPATH/src/github.com/docker/distribution/Godeps/_workspace:$GOPATH"; \
+			go build -buildmode=pie -o /build/registry-v2-schema1 github.com/docker/distribution/cmd/registry; \
+		;; \
+	   esac \
 	&& rm -rf "$GOPATH"
 
 

Dockerfile.e2e

@@ -1,5 +1,4 @@
-## Step 1: Build tests
-FROM golang:1.12.4-alpine as builder
+FROM golang:1.12.5-alpine as base
 
 RUN apk --no-cache add \
     bash \
@@ -9,37 +8,55 @@ RUN apk --no-cache add \
     lvm2-dev \
     jq
 
+RUN mkdir -p /build/
 RUN mkdir -p /go/src/github.com/docker/docker/
 WORKDIR /go/src/github.com/docker/docker/
 
-# Generate frozen images
-COPY contrib/download-frozen-image-v2.sh contrib/download-frozen-image-v2.sh
-RUN contrib/download-frozen-image-v2.sh /output/docker-frozen-images \
+FROM base AS frozen-images
+# Get useful and necessary Hub images so we can "docker load" locally instead of pulling
+COPY contrib/download-frozen-image-v2.sh /
+RUN /download-frozen-image-v2.sh /build \
 	buildpack-deps:jessie@sha256:dd86dced7c9cd2a724e779730f0a53f93b7ef42228d4344b25ce9a42a1486251 \
 	busybox:latest@sha256:bbc3a03235220b170ba48a157dd097dd1379299370e1ed99ce976df0355d24f0 \
 	busybox:glibc@sha256:0b55a30394294ab23b9afd58fab94e61a923f5834fba7ddbae7f8e0c11ba85e6 \
 	debian:jessie@sha256:287a20c5f73087ab406e6b364833e3fb7b3ae63ca0eb3486555dc27ed32c6e60 \
 	hello-world:latest@sha256:be0cd392e45be79ffeffa6b05338b98ebb16c87b255f48e297ec7f98e123905c
+# See also ensureFrozenImagesLinux() in "integration-cli/fixtures_linux_daemon_test.go" (which needs to be updated when adding images to this list)
 
-# Install dockercli
-# Please edit hack/dockerfile/install/<name>.installer to update them.
-COPY hack/dockerfile/install hack/dockerfile/install
-RUN ./hack/dockerfile/install/install.sh dockercli
-
-# Set tag and add sources
-ARG DOCKER_GITCOMMIT
-ENV DOCKER_GITCOMMIT=${DOCKER_GITCOMMIT:-undefined}
-ADD . .
+FROM base AS dockercli
+ENV INSTALL_BINARY_NAME=dockercli
+COPY hack/dockerfile/install/install.sh ./install.sh
+COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
+RUN PREFIX=/build ./install.sh $INSTALL_BINARY_NAME
 
 # Build DockerSuite.TestBuild* dependency
-RUN CGO_ENABLED=0 go build -buildmode=pie -o /output/httpserver github.com/docker/docker/contrib/httpserver
+FROM base AS contrib
+COPY contrib/syscall-test           /build/syscall-test
+COPY contrib/httpserver/Dockerfile  /build/httpserver/Dockerfile
+COPY contrib/httpserver             contrib/httpserver
+RUN CGO_ENABLED=0 go build -buildmode=pie -o /build/httpserver/httpserver github.com/docker/docker/contrib/httpserver
 
-# Build the integration tests and copy the resulting binaries to /output/tests
+# Build the integration tests and copy the resulting binaries to /build/tests
+FROM base AS builder
+
+# Set tag and add sources
+COPY . .
+# Copy test sources tests that use assert can print errors
+RUN mkdir -p /build${PWD} && find integration integration-cli -name \*_test.go -exec cp --parents '{}' /build${PWD} \;
+# Build and install test binaries
+ARG DOCKER_GITCOMMIT=undefined
 RUN hack/make.sh build-integration-test-binary
-RUN mkdir -p /output/tests && find . -name test.main -exec cp --parents '{}' /output/tests \;
+RUN mkdir -p /build/tests && find . -name test.main -exec cp --parents '{}' /build/tests \;
 
-## Step 2: Generate testing image
-FROM alpine:3.8 as runner
+## Generate testing image
+FROM alpine:3.9 as runner
+
+ENV DOCKER_REMOTE_DAEMON=1
+ENV DOCKER_INTEGRATION_DAEMON_DEST=/
+ENTRYPOINT ["/scripts/run.sh"]
+
+# Add an unprivileged user to be used for tests which need it
+RUN addgroup docker && adduser -D -G docker unprivilegeduser -s /bin/ash
 
 # GNU tar is used for generating the emptyfs image
 RUN apk --no-cache add \
@@ -52,21 +69,14 @@ RUN apk --no-cache add \
     tar \
     xz
 
-# Add an unprivileged user to be used for tests which need it
-RUN addgroup docker && adduser -D -G docker unprivilegeduser -s /bin/ash
+COPY hack/test/e2e-run.sh       /scripts/run.sh
+COPY hack/make/.ensure-emptyfs  /scripts/ensure-emptyfs.sh
 
-COPY contrib/httpserver/Dockerfile /tests/contrib/httpserver/Dockerfile
-COPY contrib/syscall-test /tests/contrib/syscall-test
-COPY integration-cli/fixtures /tests/integration-cli/fixtures
+COPY integration/testdata       /tests/integration/testdata
+COPY integration/build/testdata /tests/integration/build/testdata
+COPY integration-cli/fixtures   /tests/integration-cli/fixtures
 
-COPY hack/test/e2e-run.sh /scripts/run.sh
-COPY hack/make/.ensure-emptyfs /scripts/ensure-emptyfs.sh
-
-COPY --from=builder /output/docker-frozen-images /docker-frozen-images
-COPY --from=builder /output/httpserver /tests/contrib/httpserver/httpserver
-COPY --from=builder /output/tests /tests
-COPY --from=builder /usr/local/bin/docker /usr/bin/docker
-
-ENV DOCKER_REMOTE_DAEMON=1 DOCKER_INTEGRATION_DAEMON_DEST=/
-
-ENTRYPOINT ["/scripts/run.sh"]
+COPY --from=frozen-images /build/ /docker-frozen-images
+COPY --from=dockercli     /build/ /usr/bin/
+COPY --from=contrib       /build/ /tests/contrib/
+COPY --from=builder       /build/ /

Dockerfile.simple

@@ -5,7 +5,7 @@
 
 # This represents the bare minimum required to build and test Docker.
 
-FROM golang:1.12.4
+FROM golang:1.12.5
 
 # allow replacing httpredir or deb mirror
 ARG APT_MIRROR=deb.debian.org

Dockerfile.windows

@@ -168,7 +168,7 @@ SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPref
 # Environment variable notes:
 #  - GO_VERSION must be consistent with 'Dockerfile' used by Linux.
 #  - FROM_DOCKERFILE is used for detection of building within a container.
-ENV GO_VERSION=1.12.4 `
+ENV GO_VERSION=1.12.5 `
     GIT_VERSION=2.11.1 `
     GOPATH=C:\go `
     FROM_DOCKERFILE=1

api/server/backend/build/backend.go

@@ -91,7 +91,9 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
 		stdout := config.ProgressWriter.StdoutFormatter
 		fmt.Fprintf(stdout, "Successfully built %s\n", stringid.TruncateID(imageID))
 	}
-	err = tagger.TagImages(image.ID(imageID))
+	if imageID != "" {
+		err = tagger.TagImages(image.ID(imageID))
+	}
 	return imageID, err
 }
 

api/server/middleware/debug.go

@@ -41,7 +41,7 @@ func DebugRequestMiddleware(handler func(ctx context.Context, w http.ResponseWri
 
 		var postForm map[string]interface{}
 		if err := json.Unmarshal(b, &postForm); err == nil {
-			maskSecretKeys(postForm, r.RequestURI)
+			maskSecretKeys(postForm)
 			formStr, errMarshal := json.Marshal(postForm)
 			if errMarshal == nil {
 				logrus.Debugf("form data: %s", string(formStr))
@@ -54,41 +54,37 @@ func DebugRequestMiddleware(handler func(ctx context.Context, w http.ResponseWri
 	}
 }
 
-func maskSecretKeys(inp interface{}, path string) {
-	// Remove any query string from the path
-	idx := strings.Index(path, "?")
-	if idx != -1 {
-		path = path[:idx]
-	}
-	// Remove trailing / characters
-	path = strings.TrimRight(path, "/")
-
+func maskSecretKeys(inp interface{}) {
 	if arr, ok := inp.([]interface{}); ok {
 		for _, f := range arr {
-			maskSecretKeys(f, path)
+			maskSecretKeys(f)
 		}
 		return
 	}
 
 	if form, ok := inp.(map[string]interface{}); ok {
+		scrub := []string{
+			// Note: The Data field contains the base64-encoded secret in 'secret'
+			// and 'config' create and update requests. Currently, no other POST
+			// API endpoints use a data field, so we scrub this field unconditionally.
+			// Change this handling to be conditional if a new endpoint is added
+			// in future where this field should not be scrubbed.
+			"data",
+			"jointoken",
+			"password",
+			"secret",
+			"signingcakey",
+			"unlockkey",
+		}
 	loop0:
 		for k, v := range form {
-			for _, m := range []string{"password", "secret", "jointoken", "unlockkey", "signingcakey"} {
+			for _, m := range scrub {
 				if strings.EqualFold(m, k) {
 					form[k] = "*****"
 					continue loop0
 				}
 			}
-			maskSecretKeys(v, path)
-		}
-
-		// Route-specific redactions
-		if strings.HasSuffix(path, "/secrets/create") {
-			for k := range form {
-				if k == "Data" {
-					form[k] = "*****"
-				}
-			}
+			maskSecretKeys(v)
 		}
 	}
 }

api/server/middleware/debug_test.go

@@ -9,31 +9,25 @@ import (
 
 func TestMaskSecretKeys(t *testing.T) {
 	tests := []struct {
-		path     string
+		doc      string
 		input    map[string]interface{}
 		expected map[string]interface{}
 	}{
 		{
-			path:     "/v1.30/secrets/create",
+			doc:      "secret/config create and update requests",
 			input:    map[string]interface{}{"Data": "foo", "Name": "name", "Labels": map[string]interface{}{}},
 			expected: map[string]interface{}{"Data": "*****", "Name": "name", "Labels": map[string]interface{}{}},
 		},
 		{
-			path:     "/v1.30/secrets/create//",
-			input:    map[string]interface{}{"Data": "foo", "Name": "name", "Labels": map[string]interface{}{}},
-			expected: map[string]interface{}{"Data": "*****", "Name": "name", "Labels": map[string]interface{}{}},
-		},
-
-		{
-			path:     "/secrets/create?key=val",
-			input:    map[string]interface{}{"Data": "foo", "Name": "name", "Labels": map[string]interface{}{}},
-			expected: map[string]interface{}{"Data": "*****", "Name": "name", "Labels": map[string]interface{}{}},
-		},
-		{
-			path: "/v1.30/some/other/path",
+			doc: "masking other fields (recursively)",
 			input: map[string]interface{}{
-				"password": "pass",
+				"password":     "pass",
+				"secret":       "secret",
+				"jointoken":    "jointoken",
+				"unlockkey":    "unlockkey",
+				"signingcakey": "signingcakey",
 				"other": map[string]interface{}{
+					"password":     "pass",
 					"secret":       "secret",
 					"jointoken":    "jointoken",
 					"unlockkey":    "unlockkey",
@@ -41,8 +35,13 @@ func TestMaskSecretKeys(t *testing.T) {
 				},
 			},
 			expected: map[string]interface{}{
-				"password": "*****",
+				"password":     "*****",
+				"secret":       "*****",
+				"jointoken":    "*****",
+				"unlockkey":    "*****",
+				"signingcakey": "*****",
 				"other": map[string]interface{}{
+					"password":     "*****",
 					"secret":       "*****",
 					"jointoken":    "*****",
 					"unlockkey":    "*****",
@@ -50,10 +49,27 @@ func TestMaskSecretKeys(t *testing.T) {
 				},
 			},
 		},
+		{
+			doc: "case insensitive field matching",
+			input: map[string]interface{}{
+				"PASSWORD": "pass",
+				"other": map[string]interface{}{
+					"PASSWORD": "pass",
+				},
+			},
+			expected: map[string]interface{}{
+				"PASSWORD": "*****",
+				"other": map[string]interface{}{
+					"PASSWORD": "*****",
+				},
+			},
+		},
 	}
 
 	for _, testcase := range tests {
-		maskSecretKeys(testcase.input, testcase.path)
-		assert.Check(t, is.DeepEqual(testcase.expected, testcase.input))
+		t.Run(testcase.doc, func(t *testing.T) {
+			maskSecretKeys(testcase.input)
+			assert.Check(t, is.DeepEqual(testcase.expected, testcase.input))
+		})
 	}
 }

api/swagger.yaml

@@ -463,10 +463,6 @@ definitions:
         type: "array"
         items:
           $ref: "#/definitions/DeviceRequest"
-      DiskQuota:
-        description: "Disk limit (in bytes)."
-        type: "integer"
-        format: "int64"
       KernelMemory:
         description: "Kernel memory limit in bytes."
         type: "integer"
@@ -1145,6 +1141,7 @@ definitions:
     type: "object"
     additionalProperties:
       type: "array"
+      x-nullable: true
       items:
         $ref: "#/definitions/PortBinding"
     example:
@@ -1169,7 +1166,6 @@ definitions:
       PortBinding represents a binding between a host IP address and a host
       port.
     type: "object"
-    x-nullable: true
     properties:
       HostIp:
         description: "Host IP address that the container's port is mapped to."
@@ -3809,7 +3805,7 @@ definitions:
         description: |
           The driver to use for managing cgroups.
         type: "string"
-        enum: ["cgroupfs", "systemd"]
+        enum: ["cgroupfs", "systemd", "none"]
         default: "cgroupfs"
         example: "cgroupfs"
       NEventsListener:
@@ -4044,7 +4040,7 @@ definitions:
       SecurityOptions:
         description: |
           List of security features that are enabled on the daemon, such as
-          apparmor, seccomp, SELinux, and user-namespaces (userns).
+          apparmor, seccomp, SELinux, user-namespaces (userns), and rootless.
 
           Additional configuration options for each security feature may
           be present, and are included as a comma-separated list of key/value
@@ -4057,6 +4053,7 @@ definitions:
           - "name=seccomp,profile=default"
           - "name=selinux"
           - "name=userns"
+          - "name=rootless"
       ProductLicense:
         description: |
           Reports a summary of the product license on the daemon.
@@ -5466,7 +5463,7 @@ paths:
   /containers/{id}/resize:
     post:
       summary: "Resize a container TTY"
-      description: "Resize the TTY for a container. You must restart the container for the resize to take effect."
+      description: "Resize the TTY for a container."
       operationId: "ContainerResize"
       consumes:
         - "application/octet-stream"
@@ -6220,12 +6217,17 @@ paths:
           in: "query"
           description: "If “1”, “true”, or “True” then it will be an error if unpacking the given content would cause an existing directory to be replaced with a non-directory and vice versa."
           type: "string"
+        - name: "copyUIDGID"
+          in: "query"
+          description: "If “1”, “true”, then it will copy UID/GID maps to the dest file or dir"
+          type: "string"
         - name: "inputStream"
           in: "body"
           required: true
           description: "The input stream must be a tar archive compressed with one of the following algorithms: identity (no compression), gzip, bzip2, xz."
           schema:
             type: "string"
+            format: "binary"
       tags: ["Container"]
   /containers/prune:
     post:
@@ -9110,7 +9112,9 @@ paths:
                 type: "string"
               RemoteAddrs:
                 description: "Addresses of manager nodes already participating in the swarm."
-                type: "string"
+                type: "array"
+                items:
+                  type: "string"
               JoinToken:
                 description: "Secret token for joining this swarm."
                 type: "string"

api/templates/server/operation.gotmpl

@@ -1,4 +1,4 @@
-package {{ .Package }}
+package {{ .Package }} // import "github.com/docker/docker/api/types/{{ .Package }}"
 
 // ----------------------------------------------------------------------------
 // DO NOT EDIT THIS FILE

api/types/container/container_changes.go

@@ -1,4 +1,4 @@
-package container
+package container // import "github.com/docker/docker/api/types/container"
 
 // ----------------------------------------------------------------------------
 // DO NOT EDIT THIS FILE

api/types/container/container_create.go

@@ -1,4 +1,4 @@
-package container
+package container // import "github.com/docker/docker/api/types/container"
 
 // ----------------------------------------------------------------------------
 // DO NOT EDIT THIS FILE

api/types/container/container_top.go

@@ -1,4 +1,4 @@
-package container
+package container // import "github.com/docker/docker/api/types/container"
 
 // ----------------------------------------------------------------------------
 // DO NOT EDIT THIS FILE

api/types/container/container_update.go

@@ -1,4 +1,4 @@
-package container
+package container // import "github.com/docker/docker/api/types/container"
 
 // ----------------------------------------------------------------------------
 // DO NOT EDIT THIS FILE

api/types/container/container_wait.go

@@ -1,4 +1,4 @@
-package container
+package container // import "github.com/docker/docker/api/types/container"
 
 // ----------------------------------------------------------------------------
 // DO NOT EDIT THIS FILE

api/types/container/host_config.go

@@ -338,7 +338,6 @@ type Resources struct {
 	Devices              []DeviceMapping // List of devices to map inside the container
 	DeviceCgroupRules    []string        // List of rule to be added to the device cgroup
 	DeviceRequests       []DeviceRequest // List of device requests for device drivers
-	DiskQuota            int64           // Disk limit (in bytes)
 	KernelMemory         int64           // Kernel memory limit (in bytes)
 	KernelMemoryTCP      int64           // Hard limit for kernel TCP buffer memory (in bytes)
 	MemoryReservation    int64           // Memory soft limit (in bytes)

api/types/image/image_history.go

@@ -1,4 +1,4 @@
-package image
+package image // import "github.com/docker/docker/api/types/image"
 
 // ----------------------------------------------------------------------------
 // DO NOT EDIT THIS FILE

api/types/volume/volume_create.go

@@ -1,4 +1,4 @@
-package volume
+package volume // import "github.com/docker/docker/api/types/volume"
 
 // ----------------------------------------------------------------------------
 // DO NOT EDIT THIS FILE

api/types/volume/volume_list.go

@@ -1,4 +1,4 @@
-package volume
+package volume // import "github.com/docker/docker/api/types/volume"
 
 // ----------------------------------------------------------------------------
 // DO NOT EDIT THIS FILE

builder/builder-next/adapters/containerimage/pull.go

@@ -135,7 +135,7 @@ func (is *imageSource) resolveRemote(ctx context.Context, ref string, platform *
 		dt   []byte
 	}
 	res, err := is.g.Do(ctx, ref, func(ctx context.Context) (interface{}, error) {
-		dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx, is.ResolverOpt, ref, sm), is.ContentStore, platform)
+		dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx, is.ResolverOpt, ref, sm), is.ContentStore, nil, platform)
 		if err != nil {
 			return nil, err
 		}
@@ -545,10 +545,10 @@ func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
 
 	r := image.NewRootFS()
 	rootFS, release, err := p.is.DownloadManager.Download(ctx, *r, runtime.GOOS, layers, pkgprogress.ChanOutput(pchan))
+	stopProgress()
 	if err != nil {
 		return nil, err
 	}
-	stopProgress()
 
 	ref, err := p.is.CacheAccessor.GetFromSnapshotter(ctx, string(rootFS.ChainID()), cache.WithDescription(fmt.Sprintf("pulled from %s", p.ref)))
 	release()
@@ -842,7 +842,7 @@ func (r *resolverCache) Add(ctx context.Context, ref string, resolver remotes.Re
 	r.mu.Lock()
 	defer r.mu.Unlock()
 
-	ref = r.domain(ref) + "-" + session.FromContext(ctx)
+	ref = r.repo(ref) + "-" + session.FromContext(ctx)
 
 	cr, ok := r.m[ref]
 	cr.timeout = time.Now().Add(time.Minute)
@@ -855,19 +855,19 @@ func (r *resolverCache) Add(ctx context.Context, ref string, resolver remotes.Re
 	return &cr
 }
 
-func (r *resolverCache) domain(refStr string) string {
+func (r *resolverCache) repo(refStr string) string {
 	ref, err := distreference.ParseNormalizedNamed(refStr)
 	if err != nil {
 		return refStr
 	}
-	return distreference.Domain(ref)
+	return ref.Name()
 }
 
 func (r *resolverCache) Get(ctx context.Context, ref string) remotes.Resolver {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 
-	ref = r.domain(ref) + "-" + session.FromContext(ctx)
+	ref = r.repo(ref) + "-" + session.FromContext(ctx)
 
 	cr, ok := r.m[ref]
 	if !ok {

builder/builder-next/adapters/snapshot/snapshot.go

@@ -74,6 +74,10 @@ func NewSnapshotter(opt Opt) (snapshot.SnapshotterBase, error) {
 	return s, nil
 }
 
+func (s *snapshotter) Name() string {
+	return "default"
+}
+
 func (s *snapshotter) IdentityMapping() *idtools.IdentityMapping {
 	return nil
 }

builder/builder-next/executor_unix.go

@@ -31,6 +31,7 @@ func newExecutor(root, cgroupParent string, net libnetwork.NetworkController, ro
 		CommandCandidates:   []string{"runc"},
 		DefaultCgroupParent: cgroupParent,
 		Rootless:            rootless,
+		NoPivot:             os.Getenv("DOCKER_RAMDISK") != "",
 	}, networkProviders)
 }
 

client/ping.go

@@ -31,6 +31,8 @@ func (cli *Client) Ping(ctx context.Context) (types.Ping, error) {
 			// Server handled the request, so parse the response
 			return parsePingResponse(cli, serverResp)
 		}
+	} else if IsErrConnectionFailed(err) {
+		return ping, err
 	}
 
 	req, err = cli.buildRequest("GET", path.Join(cli.basePath, "/_ping"), nil, nil)

cmd/dockerd/config.go

@@ -12,6 +12,8 @@ import (
 const (
 	// defaultShutdownTimeout is the default shutdown timeout for the daemon
 	defaultShutdownTimeout = 15
+	// defaultTrustKeyFile is the default filename for the trust key
+	defaultTrustKeyFile = "key.json"
 )
 
 // installCommonConfigFlags adds flags to the pflag.FlagSet to configure the daemon

cmd/dockerd/config_common_unix.go

@@ -9,12 +9,11 @@ import (
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/opts"
 	"github.com/docker/docker/pkg/homedir"
-	"github.com/docker/docker/rootless"
 	"github.com/spf13/pflag"
 )
 
 func getDefaultPidFile() (string, error) {
-	if !rootless.RunningWithNonRootUsername() {
+	if !honorXDG {
 		return "/var/run/docker.pid", nil
 	}
 	runtimeDir, err := homedir.GetRuntimeDir()
@@ -25,7 +24,7 @@ func getDefaultPidFile() (string, error) {
 }
 
 func getDefaultDataRoot() (string, error) {
-	if !rootless.RunningWithNonRootUsername() {
+	if !honorXDG {
 		return "/var/lib/docker", nil
 	}
 	dataHome, err := homedir.GetDataHome()
@@ -36,7 +35,7 @@ func getDefaultDataRoot() (string, error) {
 }
 
 func getDefaultExecRoot() (string, error) {
-	if !rootless.RunningWithNonRootUsername() {
+	if !honorXDG {
 		return "/var/run/docker", nil
 	}
 	runtimeDir, err := homedir.GetRuntimeDir()

cmd/dockerd/config_unix.go

@@ -3,10 +3,13 @@
 package main
 
 import (
+	"os/exec"
+
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/opts"
 	"github.com/docker/docker/rootless"
 	"github.com/docker/go-units"
+	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
 )
 
@@ -35,7 +38,16 @@ func installConfigFlags(conf *config.Config, flags *pflag.FlagSet) error {
 	flags.BoolVar(&conf.BridgeConfig.EnableIPv6, "ipv6", false, "Enable IPv6 networking")
 	flags.StringVar(&conf.BridgeConfig.FixedCIDRv6, "fixed-cidr-v6", "", "IPv6 subnet for fixed IPs")
 	flags.BoolVar(&conf.BridgeConfig.EnableUserlandProxy, "userland-proxy", true, "Use userland proxy for loopback traffic")
-	flags.StringVar(&conf.BridgeConfig.UserlandProxyPath, "userland-proxy-path", "", "Path to the userland proxy binary")
+	defaultUserlandProxyPath := ""
+	if rootless.RunningWithRootlessKit() {
+		var err error
+		// use rootlesskit-docker-proxy for exposing the ports in RootlessKit netns to the initial namespace.
+		defaultUserlandProxyPath, err = exec.LookPath(rootless.RootlessKitDockerProxyBinary)
+		if err != nil {
+			return errors.Wrapf(err, "running with RootlessKit, but %s not installed", rootless.RootlessKitDockerProxyBinary)
+		}
+	}
+	flags.StringVar(&conf.BridgeConfig.UserlandProxyPath, "userland-proxy-path", defaultUserlandProxyPath, "Path to the userland proxy binary")
 	flags.StringVar(&conf.CgroupParent, "cgroup-parent", "", "Set parent cgroup for all containers")
 	flags.StringVar(&conf.RemappedRoot, "userns-remap", "", "User/Group setting for user namespaces")
 	flags.BoolVar(&conf.LiveRestoreEnabled, "live-restore", false, "Enable live restore of docker when containers are still running")
@@ -49,7 +61,8 @@ func installConfigFlags(conf *config.Config, flags *pflag.FlagSet) error {
 	flags.BoolVar(&conf.NoNewPrivileges, "no-new-privileges", false, "Set no-new-privileges by default for new containers")
 	flags.StringVar(&conf.IpcMode, "default-ipc-mode", config.DefaultIpcMode, `Default mode for containers ipc ("shareable" | "private")`)
 	flags.Var(&conf.NetworkConfig.DefaultAddressPools, "default-address-pool", "Default address pools for node specific local networks")
-	// Mostly users don't need to set this flag explicitly.
-	flags.BoolVar(&conf.Rootless, "rootless", rootless.RunningWithNonRootUsername(), "Enable rootless mode (experimental)")
+	// rootless needs to be explicitly specified for running "rootful" dockerd in rootless dockerd (#38702)
+	// Note that defaultUserlandProxyPath and honorXDG are configured according to the value of rootless.RunningWithRootlessKit, not the value of --rootless.
+	flags.BoolVar(&conf.Rootless, "rootless", rootless.RunningWithRootlessKit(), "Enable rootless mode; typically used with RootlessKit (experimental)")
 	return nil
 }

cmd/dockerd/daemon.go

@@ -91,6 +91,8 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
 		return err
 	}
 
+	logrus.Info("Starting up")
+
 	cli.configFile = &opts.configFile
 	cli.flags = opts.flags
 
@@ -103,6 +105,12 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
 		if cli.Config.IsRootless() {
 			logrus.Warn("Running in rootless mode. Cgroups, AppArmor, and CRIU are disabled.")
 		}
+		if rootless.RunningWithRootlessKit() {
+			logrus.Info("Running with RootlessKit integration")
+			if !cli.Config.IsRootless() {
+				return fmt.Errorf("rootless mode needs to be enabled for running with RootlessKit")
+			}
+		}
 	} else {
 		if cli.Config.IsRootless() {
 			return fmt.Errorf("rootless mode is supported only when running in experimental mode")
@@ -260,6 +268,7 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
 		return errors.Wrap(errAPI, "shutting down due to ServeAPI error")
 	}
 
+	logrus.Info("Daemon shutdown complete")
 	return nil
 }
 
@@ -415,6 +424,14 @@ func loadDaemonCliConfig(opts *daemonOptions) (*config.Config, error) {
 		conf.CommonTLSOptions.KeyFile = opts.TLSOptions.KeyFile
 	}
 
+	if conf.TrustKeyPath == "" {
+		daemonConfDir, err := getDaemonConfDir(conf.Root)
+		if err != nil {
+			return nil, err
+		}
+		conf.TrustKeyPath = filepath.Join(daemonConfDir, defaultTrustKeyFile)
+	}
+
 	if flags.Changed("graph") && flags.Changed("data-root") {
 		return nil, errors.New(`cannot specify both "--graph" and "--data-root" option`)
 	}
@@ -591,7 +608,7 @@ func loadListeners(cli *DaemonCli, serverConfig *apiserver.Config) ([]string, er
 	var hosts []string
 	for i := 0; i < len(cli.Config.Hosts); i++ {
 		var err error
-		if cli.Config.Hosts[i], err = dopts.ParseHost(cli.Config.TLS, rootless.RunningWithNonRootUsername(), cli.Config.Hosts[i]); err != nil {
+		if cli.Config.Hosts[i], err = dopts.ParseHost(cli.Config.TLS, honorXDG, cli.Config.Hosts[i]); err != nil {
 			return nil, errors.Wrapf(err, "error parsing -H %s", cli.Config.Hosts[i])
 		}
 
@@ -668,9 +685,9 @@ func validateAuthzPlugins(requestedPlugins []string, pg plugingetter.PluginGette
 	return nil
 }
 
-func systemContainerdRunning(isRootless bool) (string, bool, error) {
+func systemContainerdRunning(honorXDG bool) (string, bool, error) {
 	addr := containerddefaults.DefaultAddress
-	if isRootless {
+	if honorXDG {
 		runtimeDir, err := homedir.GetRuntimeDir()
 		if err != nil {
 			return "", false, err

cmd/dockerd/daemon_unix.go

@@ -18,14 +18,14 @@ import (
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/libcontainerd/supervisor"
 	"github.com/docker/docker/pkg/homedir"
-	"github.com/docker/docker/rootless"
 	"github.com/docker/libnetwork/portallocator"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 )
 
 func getDefaultDaemonConfigDir() (string, error) {
-	if !rootless.RunningWithNonRootUsername() {
+	if !honorXDG {
 		return "/etc/docker", nil
 	}
 	// NOTE: CLI uses ~/.docker while the daemon uses ~/.config/docker, because
@@ -58,6 +58,10 @@ func setDefaultUmask() error {
 	return nil
 }
 
+func getDaemonConfDir(_ string) (string, error) {
+	return getDefaultDaemonConfigDir()
+}
+
 func (cli *DaemonCli) getPlatformContainerdDaemonOpts() ([]supervisor.DaemonOpt, error) {
 	opts := []supervisor.DaemonOpt{
 		supervisor.WithOOMScore(cli.Config.OOMScoreAdjust),
@@ -148,11 +152,12 @@ func newCgroupParent(config *config.Config) string {
 func (cli *DaemonCli) initContainerD(ctx context.Context) (func(time.Duration) error, error) {
 	var waitForShutdown func(time.Duration) error
 	if cli.Config.ContainerdAddr == "" {
-		systemContainerdAddr, ok, err := systemContainerdRunning(cli.Config.IsRootless())
+		systemContainerdAddr, ok, err := systemContainerdRunning(honorXDG)
 		if err != nil {
 			return nil, errors.Wrap(err, "could not determine whether the system containerd is running")
 		}
 		if !ok {
+			logrus.Debug("Containerd not running, starting daemon managed containerd")
 			opts, err := cli.getContainerdDaemonOpts()
 			if err != nil {
 				return nil, errors.Wrap(err, "failed to generate containerd options")
@@ -162,6 +167,7 @@ func (cli *DaemonCli) initContainerD(ctx context.Context) (func(time.Duration) e
 			if err != nil {
 				return nil, errors.Wrap(err, "failed to start containerd")
 			}
+			logrus.Debug("Started daemon managed containerd")
 			cli.Config.ContainerdAddr = r.Address()
 
 			// Try to wait for containerd to shutdown

cmd/dockerd/daemon_windows.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/docker/docker/daemon/config"
@@ -23,6 +24,10 @@ func setDefaultUmask() error {
 	return nil
 }
 
+func getDaemonConfDir(root string) (string, error) {
+	return filepath.Join(root, `\config`), nil
+}
+
 // preNotifySystem sends a message to the host when the API is active, but before the daemon is
 func preNotifySystem() {
 	// start the service now to prevent timeouts waiting for daemon to start

cmd/dockerd/docker.go

@@ -10,11 +10,16 @@ import (
 	"github.com/docker/docker/pkg/jsonmessage"
 	"github.com/docker/docker/pkg/reexec"
 	"github.com/docker/docker/pkg/term"
+	"github.com/docker/docker/rootless"
 	"github.com/moby/buildkit/util/apicaps"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
+var (
+	honorXDG bool
+)
+
 func newDaemonCommand() (*cobra.Command, error) {
 	opts := newDaemonOptions(config.New())
 
@@ -53,6 +58,14 @@ func init() {
 	if dockerversion.ProductName != "" {
 		apicaps.ExportedProduct = dockerversion.ProductName
 	}
+	// When running with RootlessKit, $XDG_RUNTIME_DIR, $XDG_DATA_HOME, and $XDG_CONFIG_HOME needs to be
+	// honored as the default dirs, because we are unlikely to have permissions to access the system-wide
+	// directories.
+	//
+	// Note that even running with --rootless, when not running with RootlessKit, honorXDG needs to be kept false,
+	// because the system-wide directories in the current mount namespace are expected to be accessible.
+	// ("rootful" dockerd in rootless dockerd, #38702)
+	honorXDG = rootless.RunningWithRootlessKit()
 }
 
 func main() {

container/container_unix.go

@@ -136,7 +136,7 @@ func (container *Container) CopyImagePathContent(v volume.Volume, destination st
 		return err
 	}
 
-	id := stringid.GenerateNonCryptoID()
+	id := stringid.GenerateRandomID()
 	path, err := v.Mount(id)
 	if err != nil {
 		return err

container/container_windows.go

@@ -153,7 +153,6 @@ func (container *Container) UpdateContainer(hostConfig *containertypes.HostConfi
 		resources.CpusetMems != "" ||
 		len(resources.Devices) != 0 ||
 		len(resources.DeviceCgroupRules) != 0 ||
-		resources.DiskQuota != 0 ||
 		resources.KernelMemory != 0 ||
 		resources.MemoryReservation != 0 ||
 		resources.MemorySwap != 0 ||

contrib/dockerd-rootless.sh

@@ -9,7 +9,9 @@
 # External dependencies:
 # * newuidmap and newgidmap needs to be installed.
 # * /etc/subuid and /etc/subgid needs to be configured for the current user.
-# * Either slirp4netns (v0.3+) or VPNKit needs to be installed.
+# * Either one of slirp4netns (v0.3+), VPNKit, lxc-user-nic needs to be installed.
+#   slirp4netns is used by default if installed. Otherwise fallsback to VPNKit.
+#   The default value can be overridden with $DOCKERD_ROOTLESS_ROOTLESSKIT_NET=(slirp4netns|vpnkit|lxc-user-nic)
 #
 # See the documentation for the further information.
 
@@ -35,24 +37,32 @@ if [ -z $rootlesskit ]; then
 	exit 1
 fi
 
-net=""
-mtu=""
-if which slirp4netns >/dev/null 2>&1; then
-	if slirp4netns --help | grep -- --disable-host-loopback; then
-		net=slirp4netns
-		mtu=65520
-	else
-		echo "slirp4netns does not support --disable-host-loopback. Falling back to VPNKit."
+: "${DOCKERD_ROOTLESS_ROOTLESSKIT_NET:=}"
+: "${DOCKERD_ROOTLESS_ROOTLESSKIT_MTU:=}"
+net=$DOCKERD_ROOTLESS_ROOTLESSKIT_NET
+mtu=$DOCKERD_ROOTLESS_ROOTLESSKIT_MTU
+if [ -z $net ]; then
+	if which slirp4netns >/dev/null 2>&1; then
+		if slirp4netns --help | grep -- --disable-host-loopback; then
+			net=slirp4netns
+			if [ -z $mtu ]; then
+				mtu=65520
+			fi
+		else
+			echo "slirp4netns does not support --disable-host-loopback. Falling back to VPNKit."
+		fi
+	fi
+	if [ -z $net ]; then
+		if which vpnkit >/dev/null 2>&1; then
+			net=vpnkit
+		else
+			echo "Either slirp4netns (v0.3+) or vpnkit needs to be installed"
+			exit 1
+		fi
 	fi
 fi
-if [ -z $net ]; then
-	if which vpnkit >/dev/null 2>&1; then
-		net=vpnkit
-		mtu=1500
-	else
-		echo "Either slirp4netns (v0.3+) or vpnkit needs to be installed"
-		exit 1
-	fi
+if [ -z $mtu ]; then
+	mtu=1500
 fi
 
 if [ -z $_DOCKERD_ROOTLESS_CHILD ]; then
@@ -65,8 +75,9 @@ if [ -z $_DOCKERD_ROOTLESS_CHILD ]; then
 	#         namespace from being unexpectedly unmounted when `/etc/resolv.conf` is recreated on the host
 	#         (by either systemd-networkd or NetworkManager)
 	# * /run: copy-up is required so that we can create /run/docker (hardcoded for plugins) in our namespace
-	$rootlesskit \
-		--net=$net --mtu=$mtu --disable-host-loopback --port-driver=builtin \
+	exec $rootlesskit \
+		--net=$net --mtu=$mtu \
+		--disable-host-loopback --port-driver=builtin \
 		--copy-up=/etc --copy-up=/run \
 		$DOCKERD_ROOTLESS_ROOTLESSKIT_FLAGS \
 		$0 $@
@@ -75,5 +86,5 @@ else
 	# remove the symlinks for the existing files in the parent namespace if any,
 	# so that we can create our own files in our mount namespace.
 	rm -f /run/docker /run/xtables.lock
-	dockerd $@
+	exec dockerd $@
 fi

daemon/archive.go

@@ -31,18 +31,19 @@ type archiver interface {
 }
 
 // helper functions to extract or archive
-func extractArchive(i interface{}, src io.Reader, dst string, opts *archive.TarOptions) error {
+func extractArchive(i interface{}, src io.Reader, dst string, opts *archive.TarOptions, root string) error {
 	if ea, ok := i.(extractor); ok {
 		return ea.ExtractArchive(src, dst, opts)
 	}
-	return chrootarchive.Untar(src, dst, opts)
+
+	return chrootarchive.UntarWithRoot(src, dst, opts, root)
 }
 
-func archivePath(i interface{}, src string, opts *archive.TarOptions) (io.ReadCloser, error) {
+func archivePath(i interface{}, src string, opts *archive.TarOptions, root string) (io.ReadCloser, error) {
 	if ap, ok := i.(archiver); ok {
 		return ap.ArchivePath(src, opts)
 	}
-	return archive.TarWithOptions(src, opts)
+	return chrootarchive.Tar(src, opts, root)
 }
 
 // ContainerCopy performs a deprecated operation of archiving the resource at
@@ -235,10 +236,16 @@ func (daemon *Daemon) containerArchivePath(container *container.Container, path
 	if driver.Base(resolvedPath) == "." {
 		resolvedPath += string(driver.Separator()) + "."
 	}
-	sourceDir, sourceBase := driver.Dir(resolvedPath), driver.Base(resolvedPath)
+
+	sourceDir := resolvedPath
+	sourceBase := "."
+
+	if stat.Mode&os.ModeDir == 0 { // not dir
+		sourceDir, sourceBase = driver.Split(resolvedPath)
+	}
 	opts := archive.TarResourceRebaseOpts(sourceBase, driver.Base(absPath))
 
-	data, err := archivePath(driver, sourceDir, opts)
+	data, err := archivePath(driver, sourceDir, opts, container.BaseFS.Path())
 	if err != nil {
 		return nil, nil, err
 	}
@@ -367,7 +374,7 @@ func (daemon *Daemon) containerExtractToDir(container *container.Container, path
 		}
 	}
 
-	if err := extractArchive(driver, content, resolvedPath, options); err != nil {
+	if err := extractArchive(driver, content, resolvedPath, options, container.BaseFS.Path()); err != nil {
 		return err
 	}
 
@@ -425,14 +432,11 @@ func (daemon *Daemon) containerCopy(container *container.Container, resource str
 		d, f := driver.Split(basePath)
 		basePath = d
 		filter = []string{f}
-	} else {
-		filter = []string{driver.Base(basePath)}
-		basePath = driver.Dir(basePath)
 	}
 	archive, err := archivePath(driver, basePath, &archive.TarOptions{
 		Compression:  archive.Uncompressed,
 		IncludeFiles: filter,
-	})
+	}, container.BaseFS.Path())
 	if err != nil {
 		return nil, err
 	}

daemon/cluster/executor/container/container.go

@@ -6,7 +6,6 @@ import (
 	"net"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/sirupsen/logrus"
 
@@ -31,10 +30,6 @@ import (
 )
 
 const (
-	// Explicitly use the kernel's default setting for CPU quota of 100ms.
-	// https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
-	cpuQuotaPeriod = 100 * time.Millisecond
-
 	// systemLabelPrefix represents the reserved namespace for system labels.
 	systemLabelPrefix = "com.docker.swarm"
 )
@@ -451,9 +446,7 @@ func (c *containerConfig) resources() enginecontainer.Resources {
 	}
 
 	if r.Limits.NanoCPUs > 0 {
-		// CPU Period must be set in microseconds.
-		resources.CPUPeriod = int64(cpuQuotaPeriod / time.Microsecond)
-		resources.CPUQuota = r.Limits.NanoCPUs * resources.CPUPeriod / 1e9
+		resources.NanoCPUs = r.Limits.NanoCPUs
 	}
 
 	return resources

daemon/cluster/executor/container/controller.go

@@ -369,11 +369,17 @@ func (r *controller) Shutdown(ctx context.Context) error {
 	}
 
 	if err := r.adapter.shutdown(ctx); err != nil {
-		if isUnknownContainer(err) || isStoppedContainer(err) {
-			return nil
+		if !(isUnknownContainer(err) || isStoppedContainer(err)) {
+			return err
 		}
+	}
 
-		return err
+	// Try removing networks referenced in this task in case this
+	// task is the last one referencing it
+	if err := r.adapter.removeNetworks(ctx); err != nil {
+		if !isUnknownContainer(err) {
+			return err
+		}
 	}
 
 	return nil
@@ -419,15 +425,6 @@ func (r *controller) Remove(ctx context.Context) error {
 		log.G(ctx).WithError(err).Debug("shutdown failed on removal")
 	}
 
-	// Try removing networks referenced in this task in case this
-	// task is the last one referencing it
-	if err := r.adapter.removeNetworks(ctx); err != nil {
-		if isUnknownContainer(err) {
-			return nil
-		}
-		return err
-	}
-
 	if err := r.adapter.remove(ctx); err != nil {
 		if isUnknownContainer(err) {
 			return nil

daemon/cluster/nodes.go

@@ -8,6 +8,7 @@ import (
 	"github.com/docker/docker/daemon/cluster/convert"
 	"github.com/docker/docker/errdefs"
 	swarmapi "github.com/docker/swarmkit/api"
+	"google.golang.org/grpc"
 )
 
 // GetNodes returns a list of all nodes known to a cluster.
@@ -30,7 +31,9 @@ func (c *Cluster) GetNodes(options apitypes.NodeListOptions) ([]types.Node, erro
 
 	r, err := state.controlClient.ListNodes(
 		ctx,
-		&swarmapi.ListNodesRequest{Filters: filters})
+		&swarmapi.ListNodesRequest{Filters: filters},
+		grpc.MaxCallRecvMsgSize(defaultRecvSizeForListResponse),
+	)
 	if err != nil {
 		return nil, err
 	}

daemon/cluster/secrets.go

@@ -7,6 +7,7 @@ import (
 	types "github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/daemon/cluster/convert"
 	swarmapi "github.com/docker/swarmkit/api"
+	"google.golang.org/grpc"
 )
 
 // GetSecret returns a secret from a managed swarm cluster
@@ -44,7 +45,9 @@ func (c *Cluster) GetSecrets(options apitypes.SecretListOptions) ([]types.Secret
 	defer cancel()
 
 	r, err := state.controlClient.ListSecrets(ctx,
-		&swarmapi.ListSecretsRequest{Filters: filters})
+		&swarmapi.ListSecretsRequest{Filters: filters},
+		grpc.MaxCallRecvMsgSize(defaultRecvSizeForListResponse),
+	)
 	if err != nil {
 		return nil, err
 	}

daemon/config/config.go

@@ -63,6 +63,8 @@ var flatOptions = map[string]bool{
 var skipValidateOptions = map[string]bool{
 	"features": true,
 	"builder":  true,
+	// Corresponding flag has been removed because it was already unusable
+	"deprecated-key-path": true,
 }
 
 // skipDuplicates contains configuration keys that
@@ -134,6 +136,12 @@ type CommonConfig struct {
 	SocketGroup           string                    `json:"group,omitempty"`
 	CorsHeaders           string                    `json:"api-cors-header,omitempty"`
 
+	// TrustKeyPath is used to generate the daemon ID and for signing schema 1 manifests
+	// when pushing to a registry which does not support schema 2. This field is marked as
+	// deprecated because schema 1 manifests are deprecated in favor of schema 2 and the
+	// daemon ID will use a dedicated identifier not shared with exported signatures.
+	TrustKeyPath string `json:"deprecated-key-path,omitempty"`
+
 	// LiveRestoreEnabled determines whether we should keep containers
 	// alive upon daemon shutdown/start
 	LiveRestoreEnabled bool `json:"live-restore,omitempty"`
@@ -239,7 +247,6 @@ func New() *Config {
 	config := Config{}
 	config.LogConfig.Config = make(map[string]string)
 	config.ClusterOpts = make(map[string]string)
-
 	return &config
 }
 

daemon/container_operations.go

@@ -354,6 +354,15 @@ func (daemon *Daemon) findAndAttachNetwork(container *container.Container, idOrN
 		if container.Managed || !n.Info().Dynamic() {
 			return n, nil, nil
 		}
+		// Throw an error if the container is already attached to the network
+		if container.NetworkSettings.Networks != nil {
+			networkName := n.Name()
+			containerName := strings.TrimPrefix(container.Name, "/")
+			if network, ok := container.NetworkSettings.Networks[networkName]; ok && network.EndpointID != "" {
+				err := fmt.Errorf("%s is already attached to network %s", containerName, networkName)
+				return n, nil, errdefs.Conflict(err)
+			}
+		}
 	}
 
 	var addresses []string

daemon/create_unix.go

@@ -41,7 +41,7 @@ func (daemon *Daemon) createContainerOSSpecificSettings(container *container.Con
 	}
 
 	for spec := range config.Volumes {
-		name := stringid.GenerateNonCryptoID()
+		name := stringid.GenerateRandomID()
 		destination := filepath.Clean(spec)
 
 		// Skip volumes for which we already have something mounted on that

daemon/create_windows.go

@@ -38,7 +38,7 @@ func (daemon *Daemon) createContainerOSSpecificSettings(container *container.Con
 
 		// If the mountpoint doesn't have a name, generate one.
 		if len(mp.Name) == 0 {
-			mp.Name = stringid.GenerateNonCryptoID()
+			mp.Name = stringid.GenerateRandomID()
 		}
 
 		// Skip volumes for which we already have something mounted on that

daemon/daemon.go

@@ -960,7 +960,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		return nil, err
 	}
 
-	uuid, err := loadOrCreateUUID(filepath.Join(config.Root, "engine_uuid"))
+	trustKey, err := loadOrCreateTrustKey(config.TrustKeyPath)
 	if err != nil {
 		return nil, err
 	}
@@ -1005,7 +1005,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		return nil, errors.New("Devices cgroup isn't mounted")
 	}
 
-	d.ID = uuid
+	d.ID = trustKey.PublicKey().KeyID()
 	d.repository = daemonRepo
 	d.containers = container.NewMemoryStore()
 	if d.containersReplica, err = container.NewViewDB(); err != nil {
@@ -1036,6 +1036,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		MaxConcurrentUploads:      *config.MaxConcurrentUploads,
 		ReferenceStore:            rs,
 		RegistryService:           registryService,
+		TrustKey:                  trustKey,
 	})
 
 	go d.execCommandGC()

daemon/daemon_linux.go

@@ -9,18 +9,13 @@ import (
 	"strings"
 
 	"github.com/docker/docker/daemon/config"
-	"github.com/docker/docker/internal/procfs"
 	"github.com/docker/docker/pkg/fileutils"
 	"github.com/docker/docker/pkg/mount"
+	"github.com/docker/libnetwork/resolvconf"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
-const (
-	defaultResolvConf   = "/etc/resolv.conf"
-	alternateResolvConf = "/run/systemd/resolve/resolv.conf"
-)
-
 // On Linux, plugins use a static path for storing execution state,
 // instead of deriving path from daemon's exec-root. This is because
 // plugin socket files are created here and they cannot exceed max
@@ -148,20 +143,5 @@ func setupResolvConf(config *config.Config) {
 	if config.ResolvConf != "" {
 		return
 	}
-
-	config.ResolvConf = defaultResolvConf
-	pids, err := procfs.PidOf("systemd-resolved")
-	if err != nil {
-		logrus.Errorf("unable to check systemd-resolved status: %s", err)
-		return
-	}
-	if len(pids) > 0 && pids[0] > 0 {
-		_, err := os.Stat(alternateResolvConf)
-		if err == nil {
-			logrus.Infof("systemd-resolved is running, so using resolvconf: %s", alternateResolvConf)
-			config.ResolvConf = alternateResolvConf
-			return
-		}
-		logrus.Infof("systemd-resolved is running, but %s is not present, fallback to %s", alternateResolvConf, defaultResolvConf)
-	}
+	config.ResolvConf = resolvconf.Path()
 }

daemon/daemon_unix.go

@@ -73,6 +73,7 @@ const (
 	// constant for cgroup drivers
 	cgroupFsDriver      = "cgroupfs"
 	cgroupSystemdDriver = "systemd"
+	cgroupNoneDriver    = "none"
 
 	// DefaultRuntimeName is the default runtime to be used by
 	// containerd if none is specified
@@ -191,8 +192,8 @@ func getBlkioWeightDevices(config containertypes.Resources) ([]specs.LinuxWeight
 		}
 		weight := weightDevice.Weight
 		d := specs.LinuxWeightDevice{Weight: &weight}
-		d.Major = int64(stat.Rdev / 256)
-		d.Minor = int64(stat.Rdev % 256)
+		d.Major = int64(unix.Major(stat.Rdev))
+		d.Minor = int64(unix.Minor(stat.Rdev))
 		blkioWeightDevices = append(blkioWeightDevices, d)
 	}
 
@@ -262,8 +263,8 @@ func getBlkioThrottleDevices(devs []*blkiodev.ThrottleDevice) ([]specs.LinuxThro
 			return nil, err
 		}
 		d := specs.LinuxThrottleDevice{Rate: d.Rate}
-		d.Major = int64(stat.Rdev / 256)
-		d.Minor = int64(stat.Rdev % 256)
+		d.Major = int64(unix.Major(stat.Rdev))
+		d.Minor = int64(unix.Minor(stat.Rdev))
 		throttleDevices = append(throttleDevices, d)
 	}
 
@@ -575,6 +576,9 @@ func verifyPlatformContainerResources(resources *containertypes.Resources, sysIn
 }
 
 func (daemon *Daemon) getCgroupDriver() string {
+	if daemon.Rootless() {
+		return cgroupNoneDriver
+	}
 	cgroupDriver := cgroupFsDriver
 
 	if UsingSystemd(daemon.configStore) {
@@ -601,6 +605,9 @@ func VerifyCgroupDriver(config *config.Config) error {
 	if cd == "" || cd == cgroupFsDriver || cd == cgroupSystemdDriver {
 		return nil
 	}
+	if cd == cgroupNoneDriver {
+		return fmt.Errorf("native.cgroupdriver option %s is internally used and cannot be specified manually", cd)
+	}
 	return fmt.Errorf("native.cgroupdriver option %s not supported", cd)
 }
 

daemon/daemon_unix_test.go

@@ -6,12 +6,15 @@ import (
 	"errors"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"testing"
 
+	"github.com/docker/docker/api/types/blkiodev"
 	containertypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/container"
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/pkg/sysinfo"
+	"golang.org/x/sys/unix"
 	"gotest.tools/assert"
 	is "gotest.tools/assert/cmp"
 )
@@ -376,3 +379,61 @@ func sysInfo(t *testing.T, opts ...func(*sysinfo.SysInfo)) sysinfo.SysInfo {
 	}
 	return si
 }
+
+const (
+	// prepare major 0x1FD(509 in decimal) and minor 0x130(304)
+	DEVNO  = 0x11FD30
+	MAJOR  = 509
+	MINOR  = 304
+	WEIGHT = 1024
+)
+
+func deviceTypeMock(t *testing.T, testAndCheck func(string)) {
+	if os.Getuid() != 0 {
+		t.Skip("root required") // for mknod
+	}
+
+	t.Parallel()
+
+	tempDir, err := ioutil.TempDir("", "tempDevDir"+t.Name())
+	assert.NilError(t, err, "create temp file")
+	tempFile := filepath.Join(tempDir, "dev")
+
+	defer os.RemoveAll(tempDir)
+
+	if err = unix.Mknod(tempFile, unix.S_IFCHR, DEVNO); err != nil {
+		t.Fatalf("mknod error %s(%x): %v", tempFile, DEVNO, err)
+	}
+
+	testAndCheck(tempFile)
+}
+
+func TestGetBlkioWeightDevices(t *testing.T) {
+	deviceTypeMock(t, func(tempFile string) {
+		mockResource := containertypes.Resources{
+			BlkioWeightDevice: []*blkiodev.WeightDevice{{Path: tempFile, Weight: WEIGHT}},
+		}
+
+		weightDevs, err := getBlkioWeightDevices(mockResource)
+
+		assert.NilError(t, err, "getBlkioWeightDevices")
+		assert.Check(t, is.Len(weightDevs, 1), "getBlkioWeightDevices")
+		assert.Check(t, weightDevs[0].Major == MAJOR, "get major device type")
+		assert.Check(t, weightDevs[0].Minor == MINOR, "get minor device type")
+		assert.Check(t, *weightDevs[0].Weight == WEIGHT, "get device weight")
+	})
+}
+
+func TestGetBlkioThrottleDevices(t *testing.T) {
+	deviceTypeMock(t, func(tempFile string) {
+		mockDevs := []*blkiodev.ThrottleDevice{{Path: tempFile, Rate: WEIGHT}}
+
+		retDevs, err := getBlkioThrottleDevices(mockDevs)
+
+		assert.NilError(t, err, "getBlkioThrottleDevices")
+		assert.Check(t, is.Len(retDevs, 1), "getBlkioThrottleDevices")
+		assert.Check(t, retDevs[0].Major == MAJOR, "get major device type")
+		assert.Check(t, retDevs[0].Minor == MINOR, "get minor device type")
+		assert.Check(t, retDevs[0].Rate == WEIGHT, "get device rate")
+	})
+}

daemon/exec/exec.go

@@ -39,7 +39,7 @@ type Config struct {
 // NewConfig initializes the a new exec configuration
 func NewConfig() *Config {
 	return &Config{
-		ID:           stringid.GenerateNonCryptoID(),
+		ID:           stringid.GenerateRandomID(),
 		StreamConfig: stream.NewConfig(),
 		Started:      make(chan struct{}),
 	}

daemon/export.go

@@ -70,7 +70,7 @@ func (daemon *Daemon) containerExport(container *container.Container) (arch io.R
 		Compression: archive.Uncompressed,
 		UIDMaps:     daemon.idMapping.UIDs(),
 		GIDMaps:     daemon.idMapping.GIDs(),
-	})
+	}, basefs.Path())
 	if err != nil {
 		rwlayer.Unmount()
 		return nil, err

daemon/graphdriver/aufs/aufs.go

@@ -72,7 +72,6 @@ func init() {
 
 // Driver contains information about the filesystem mounted.
 type Driver struct {
-	sync.Mutex
 	root          string
 	uidMaps       []idtools.IDMap
 	gidMaps       []idtools.IDMap
@@ -81,6 +80,7 @@ type Driver struct {
 	pathCache     map[string]string
 	naiveDiff     graphdriver.DiffDriver
 	locker        *locker.Locker
+	mntL          sync.Mutex
 }
 
 // Init returns a new AUFS driver.
@@ -327,11 +327,11 @@ func (a *Driver) Remove(id string) error {
 			break
 		}
 
-		if err != unix.EBUSY {
-			return errors.Wrapf(err, "aufs: unmount error: %s", mountpoint)
+		if errors.Cause(err) != unix.EBUSY {
+			return errors.Wrap(err, "aufs: unmount error")
 		}
 		if retries >= 5 {
-			return errors.Wrapf(err, "aufs: unmount error after retries: %s", mountpoint)
+			return errors.Wrap(err, "aufs: unmount error after retries")
 		}
 		// If unmount returns EBUSY, it could be a transient error. Sleep and retry.
 		retries++
@@ -437,7 +437,7 @@ func (a *Driver) Put(id string) error {
 
 	err := a.unmount(m)
 	if err != nil {
-		logger.Debugf("Failed to unmount %s aufs: %v", id, err)
+		logger.WithError(err).WithField("method", "Put()").Warn()
 	}
 	return err
 }
@@ -547,9 +547,6 @@ func (a *Driver) getParentLayerPaths(id string) ([]string, error) {
 }
 
 func (a *Driver) mount(id string, target string, mountLabel string, layers []string) error {
-	a.Lock()
-	defer a.Unlock()
-
 	// If the id is mounted or we get an error return
 	if mounted, err := a.mounted(target); err != nil || mounted {
 		return err
@@ -564,9 +561,6 @@ func (a *Driver) mount(id string, target string, mountLabel string, layers []str
 }
 
 func (a *Driver) unmount(mountPath string) error {
-	a.Lock()
-	defer a.Unlock()
-
 	if mounted, err := a.mounted(mountPath); err != nil || !mounted {
 		return err
 	}
@@ -579,23 +573,20 @@ func (a *Driver) mounted(mountpoint string) (bool, error) {
 
 // Cleanup aufs and unmount all mountpoints
 func (a *Driver) Cleanup() error {
-	var dirs []string
-	if err := filepath.Walk(a.mntPath(), func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if !info.IsDir() {
-			return nil
-		}
-		dirs = append(dirs, path)
-		return nil
-	}); err != nil {
-		return err
+	dir := a.mntPath()
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return errors.Wrap(err, "aufs readdir error")
 	}
+	for _, f := range files {
+		if !f.IsDir() {
+			continue
+		}
+
+		m := path.Join(dir, f.Name())
 
-	for _, m := range dirs {
 		if err := a.unmount(m); err != nil {
-			logger.Debugf("error unmounting %s: %s", m, err)
+			logger.WithError(err).WithField("method", "Cleanup()").Warn()
 		}
 	}
 	return mount.RecursiveUnmount(a.root)
@@ -604,7 +595,7 @@ func (a *Driver) Cleanup() error {
 func (a *Driver) aufsMount(ro []string, rw, target, mountLabel string) (err error) {
 	defer func() {
 		if err != nil {
-			Unmount(target)
+			mount.Unmount(target)
 		}
 	}()
 
@@ -632,14 +623,29 @@ func (a *Driver) aufsMount(ro []string, rw, target, mountLabel string) (err erro
 		opts += ",dirperm1"
 	}
 	data := label.FormatMountLabel(fmt.Sprintf("%s,%s", string(b[:bp]), opts), mountLabel)
-	if err = unix.Mount("none", target, "aufs", 0, data); err != nil {
+	a.mntL.Lock()
+	err = unix.Mount("none", target, "aufs", 0, data)
+	a.mntL.Unlock()
+	if err != nil {
+		err = errors.Wrap(err, "mount target="+target+" data="+data)
 		return
 	}
 
-	for ; index < len(ro); index++ {
-		layer := fmt.Sprintf(":%s=ro+wh", ro[index])
-		data := label.FormatMountLabel(fmt.Sprintf("append%s", layer), mountLabel)
-		if err = unix.Mount("none", target, "aufs", unix.MS_REMOUNT, data); err != nil {
+	for index < len(ro) {
+		bp = 0
+		for ; index < len(ro); index++ {
+			layer := fmt.Sprintf("append:%s=ro+wh,", ro[index])
+			if bp+len(layer) > len(b) {
+				break
+			}
+			bp += copy(b[bp:], layer)
+		}
+		data := label.FormatMountLabel(string(b[:bp]), mountLabel)
+		a.mntL.Lock()
+		err = unix.Mount("none", target, "aufs", unix.MS_REMOUNT, data)
+		a.mntL.Unlock()
+		if err != nil {
+			err = errors.Wrap(err, "mount target="+target+" flags=MS_REMOUNT data="+data)
 			return
 		}
 	}

daemon/graphdriver/aufs/aufs_test.go

@@ -731,7 +731,7 @@ func BenchmarkConcurrentAccess(b *testing.B) {
 	// create a bunch of ids
 	var ids []string
 	for i := 0; i < numConcurrent; i++ {
-		ids = append(ids, stringid.GenerateNonCryptoID())
+		ids = append(ids, stringid.GenerateRandomID())
 	}
 
 	if err := d.Create(ids[0], "", nil); err != nil {

daemon/graphdriver/aufs/mount.go

@@ -4,14 +4,38 @@ package aufs // import "github.com/docker/docker/daemon/graphdriver/aufs"
 
 import (
 	"os/exec"
+	"syscall"
 
-	"golang.org/x/sys/unix"
+	"github.com/docker/docker/pkg/mount"
 )
 
 // Unmount the target specified.
 func Unmount(target string) error {
-	if err := exec.Command("auplink", target, "flush").Run(); err != nil {
-		logger.WithError(err).Warnf("Couldn't run auplink before unmount %s", target)
+	const (
+		EINVAL  = 22 // if auplink returns this,
+		retries = 3  // retry a few times
+	)
+
+	for i := 0; ; i++ {
+		out, err := exec.Command("auplink", target, "flush").CombinedOutput()
+		if err == nil {
+			break
+		}
+		rc := 0
+		if exiterr, ok := err.(*exec.ExitError); ok {
+			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
+				rc = status.ExitStatus()
+			}
+		}
+		if i >= retries || rc != EINVAL {
+			logger.WithError(err).WithField("method", "Unmount").Warnf("auplink flush failed: %s", out)
+			break
+		}
+		// auplink failed to find target in /proc/self/mounts because
+		// kernel can't guarantee continuity while reading from it
+		// while mounts table is being changed
+		logger.Debugf("auplink flush error (retrying %d/%d): %s", i+1, retries, out)
 	}
-	return unix.Unmount(target, 0)
+
+	return mount.Unmount(target)
 }

daemon/graphdriver/windows/windows.go

@@ -338,11 +338,14 @@ func (d *Driver) Remove(id string) error {
 		// If permission denied, it's possible that the scratch is still mounted, an
 		// artifact after a hard daemon crash for example. Worth a shot to try detaching it
 		// before retrying the rename.
-		if detachErr := vhd.DetachVhd(filepath.Join(layerPath, "sandbox.vhdx")); detachErr != nil {
-			return errors.Wrapf(err, "failed to detach VHD: %s", detachErr)
-		}
-		if renameErr := os.Rename(layerPath, tmpLayerPath); renameErr != nil && !os.IsNotExist(renameErr) {
-			return errors.Wrapf(err, "second rename attempt following detach failed: %s", renameErr)
+		sandbox := filepath.Join(layerPath, "sandbox.vhdx")
+		if _, statErr := os.Stat(sandbox); statErr == nil {
+			if detachErr := vhd.DetachVhd(sandbox); detachErr != nil {
+				return errors.Wrapf(err, "failed to detach VHD: %s", detachErr)
+			}
+			if renameErr := os.Rename(layerPath, tmpLayerPath); renameErr != nil && !os.IsNotExist(renameErr) {
+				return errors.Wrapf(err, "second rename attempt following detach failed: %s", renameErr)
+			}
 		}
 	}
 	if err := hcsshim.DestroyLayer(d.info, tmpID); err != nil {

daemon/images/image_push.go

@@ -54,6 +54,7 @@ func (i *ImageService) PushImage(ctx context.Context, image, tag string, metaHea
 		},
 		ConfigMediaType: schema2.MediaTypeImageConfig,
 		LayerStores:     distribution.NewLayerProvidersFromStores(i.layerStores),
+		TrustKey:        i.trustKey,
 		UploadManager:   i.uploadManager,
 	}
 

daemon/images/service.go

@@ -14,6 +14,7 @@ import (
 	"github.com/docker/docker/layer"
 	dockerreference "github.com/docker/docker/reference"
 	"github.com/docker/docker/registry"
+	"github.com/docker/libtrust"
 	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -39,6 +40,7 @@ type ImageServiceConfig struct {
 	MaxConcurrentUploads      int
 	ReferenceStore            dockerreference.Store
 	RegistryService           registry.Service
+	TrustKey                  libtrust.PrivateKey
 }
 
 // NewImageService returns a new ImageService from a configuration
@@ -54,6 +56,7 @@ func NewImageService(config ImageServiceConfig) *ImageService {
 		layerStores:               config.LayerStores,
 		referenceStore:            config.ReferenceStore,
 		registryService:           config.RegistryService,
+		trustKey:                  config.TrustKey,
 		uploadManager:             xfer.NewLayerUploadManager(config.MaxConcurrentUploads),
 	}
 }
@@ -69,6 +72,7 @@ type ImageService struct {
 	pruneRunning              int32
 	referenceStore            dockerreference.Store
 	registryService           registry.Service
+	trustKey                  libtrust.PrivateKey
 	uploadManager             *xfer.LayerUploadManager
 }
 

daemon/logger/plugin.go

@@ -81,7 +81,7 @@ func makePluginCreator(name string, l logPlugin, scopePath func(s string) string
 			return nil, err
 		}
 
-		id := stringid.GenerateNonCryptoID()
+		id := stringid.GenerateRandomID()
 		a := &pluginAdapter{
 			driverName: name,
 			id:         id,

daemon/names.go

@@ -38,7 +38,7 @@ func (daemon *Daemon) registerName(container *container.Container) error {
 func (daemon *Daemon) generateIDAndName(name string) (string, string, error) {
 	var (
 		err error
-		id  = stringid.GenerateNonCryptoID()
+		id  = stringid.GenerateRandomID()
 	)
 
 	if name == "" {

daemon/trustkey.go

@@ -0,0 +1,57 @@
+package daemon // import "github.com/docker/docker/daemon"
+
+import (
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/pkg/ioutils"
+	"github.com/docker/docker/pkg/system"
+	"github.com/docker/libtrust"
+)
+
+// LoadOrCreateTrustKey attempts to load the libtrust key at the given path,
+// otherwise generates a new one
+// TODO: this should use more of libtrust.LoadOrCreateTrustKey which may need
+// a refactor or this function to be moved into libtrust
+func loadOrCreateTrustKey(trustKeyPath string) (libtrust.PrivateKey, error) {
+	err := system.MkdirAll(filepath.Dir(trustKeyPath), 0755, "")
+	if err != nil {
+		return nil, err
+	}
+	trustKey, err := libtrust.LoadKeyFile(trustKeyPath)
+	if err == libtrust.ErrKeyFileDoesNotExist {
+		trustKey, err = libtrust.GenerateECP256PrivateKey()
+		if err != nil {
+			return nil, fmt.Errorf("Error generating key: %s", err)
+		}
+		encodedKey, err := serializePrivateKey(trustKey, filepath.Ext(trustKeyPath))
+		if err != nil {
+			return nil, fmt.Errorf("Error serializing key: %s", err)
+		}
+		if err := ioutils.AtomicWriteFile(trustKeyPath, encodedKey, os.FileMode(0600)); err != nil {
+			return nil, fmt.Errorf("Error saving key file: %s", err)
+		}
+	} else if err != nil {
+		return nil, fmt.Errorf("Error loading key file %s: %s", trustKeyPath, err)
+	}
+	return trustKey, nil
+}
+
+func serializePrivateKey(key libtrust.PrivateKey, ext string) (encoded []byte, err error) {
+	if ext == ".json" || ext == ".jwk" {
+		encoded, err = json.Marshal(key)
+		if err != nil {
+			return nil, fmt.Errorf("unable to encode private key JWK: %s", err)
+		}
+	} else {
+		pemBlock, err := key.PEMBlock()
+		if err != nil {
+			return nil, fmt.Errorf("unable to encode private key PEM: %s", err)
+		}
+		encoded = pem.EncodeToMemory(pemBlock)
+	}
+	return
+}

daemon/trustkey_test.go

@@ -0,0 +1,71 @@
+package daemon // import "github.com/docker/docker/daemon"
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"gotest.tools/assert"
+	is "gotest.tools/assert/cmp"
+	"gotest.tools/fs"
+)
+
+// LoadOrCreateTrustKey
+func TestLoadOrCreateTrustKeyInvalidKeyFile(t *testing.T) {
+	tmpKeyFolderPath, err := ioutil.TempDir("", "api-trustkey-test")
+	assert.NilError(t, err)
+	defer os.RemoveAll(tmpKeyFolderPath)
+
+	tmpKeyFile, err := ioutil.TempFile(tmpKeyFolderPath, "keyfile")
+	assert.NilError(t, err)
+
+	_, err = loadOrCreateTrustKey(tmpKeyFile.Name())
+	assert.Check(t, is.ErrorContains(err, "Error loading key file"))
+}
+
+func TestLoadOrCreateTrustKeyCreateKeyWhenFileDoesNotExist(t *testing.T) {
+	tmpKeyFolderPath := fs.NewDir(t, "api-trustkey-test")
+	defer tmpKeyFolderPath.Remove()
+
+	// Without the need to create the folder hierarchy
+	tmpKeyFile := tmpKeyFolderPath.Join("keyfile")
+
+	key, err := loadOrCreateTrustKey(tmpKeyFile)
+	assert.NilError(t, err)
+	assert.Check(t, key != nil)
+
+	_, err = os.Stat(tmpKeyFile)
+	assert.NilError(t, err, "key file doesn't exist")
+}
+
+func TestLoadOrCreateTrustKeyCreateKeyWhenDirectoryDoesNotExist(t *testing.T) {
+	tmpKeyFolderPath := fs.NewDir(t, "api-trustkey-test")
+	defer tmpKeyFolderPath.Remove()
+	tmpKeyFile := tmpKeyFolderPath.Join("folder/hierarchy/keyfile")
+
+	key, err := loadOrCreateTrustKey(tmpKeyFile)
+	assert.NilError(t, err)
+	assert.Check(t, key != nil)
+
+	_, err = os.Stat(tmpKeyFile)
+	assert.NilError(t, err, "key file doesn't exist")
+}
+
+func TestLoadOrCreateTrustKeyCreateKeyNoPath(t *testing.T) {
+	defer os.Remove("keyfile")
+	key, err := loadOrCreateTrustKey("keyfile")
+	assert.NilError(t, err)
+	assert.Check(t, key != nil)
+
+	_, err = os.Stat("keyfile")
+	assert.NilError(t, err, "key file doesn't exist")
+}
+
+func TestLoadOrCreateTrustKeyLoadValidKey(t *testing.T) {
+	tmpKeyFile := filepath.Join("testdata", "keyfile")
+	key, err := loadOrCreateTrustKey(tmpKeyFile)
+	assert.NilError(t, err)
+	expected := "AWX2:I27X:WQFX:IOMK:CNAK:O7PW:VYNB:ZLKC:CVAE:YJP2:SI4A:XXAY"
+	assert.Check(t, is.Contains(key.String(), expected))
+}

daemon/uuid.go

@@ -1,35 +0,0 @@
-package daemon // import "github.com/docker/docker/daemon"
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-
-	"github.com/docker/docker/pkg/ioutils"
-	"github.com/google/uuid"
-)
-
-func loadOrCreateUUID(path string) (string, error) {
-	err := os.MkdirAll(filepath.Dir(path), 0755)
-	if err != nil {
-		return "", err
-	}
-	var id string
-	idb, err := ioutil.ReadFile(path)
-	if os.IsNotExist(err) {
-		id = uuid.New().String()
-		if err := ioutils.AtomicWriteFile(path, []byte(id), os.FileMode(0600)); err != nil {
-			return "", fmt.Errorf("Error saving uuid file: %s", err)
-		}
-	} else if err != nil {
-		return "", fmt.Errorf("Error loading uuid file %s: %s", path, err)
-	} else {
-		idp, err := uuid.Parse(string(idb))
-		if err != nil {
-			return "", fmt.Errorf("Error parsing uuid in file %s: %s", path, err)
-		}
-		id = idp.String()
-	}
-	return id, nil
-}

distribution/config.go

@@ -18,6 +18,7 @@ import (
 	"github.com/docker/docker/pkg/system"
 	refstore "github.com/docker/docker/reference"
 	"github.com/docker/docker/registry"
+	"github.com/docker/libtrust"
 	"github.com/opencontainers/go-digest"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 )
@@ -72,6 +73,9 @@ type ImagePushConfig struct {
 	ConfigMediaType string
 	// LayerStores (indexed by operating system) manages layers.
 	LayerStores map[string]PushLayerProvider
+	// TrustKey is the private key for legacy signatures. This is typically
+	// an ephemeral key, since these signatures are no longer verified.
+	TrustKey libtrust.PrivateKey
 	// UploadManager dispatches uploads.
 	UploadManager *xfer.LayerUploadManager
 }

distribution/pull.go

@@ -39,12 +39,7 @@ func newPuller(endpoint registry.APIEndpoint, repoInfo *registry.RepositoryInfo,
 			repoInfo:          repoInfo,
 		}, nil
 	case registry.APIVersion1:
-		return &v1Puller{
-			v1IDService: metadata.NewV1IDService(imagePullConfig.MetadataStore),
-			endpoint:    endpoint,
-			config:      imagePullConfig,
-			repoInfo:    repoInfo,
-		}, nil
+		return nil, fmt.Errorf("protocol version %d no longer supported. Please contact admins of registry %s", endpoint.Version, endpoint.URL)
 	}
 	return nil, fmt.Errorf("unknown version %d for registry %s", endpoint.Version, endpoint.URL)
 }

distribution/pull_v1.go

@@ -1,365 +0,0 @@
-package distribution // import "github.com/docker/docker/distribution"
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net"
-	"net/url"
-	"os"
-	"strings"
-	"time"
-
-	"github.com/docker/distribution/reference"
-	"github.com/docker/distribution/registry/client/transport"
-	"github.com/docker/docker/distribution/metadata"
-	"github.com/docker/docker/distribution/xfer"
-	"github.com/docker/docker/dockerversion"
-	"github.com/docker/docker/image"
-	"github.com/docker/docker/image/v1"
-	"github.com/docker/docker/layer"
-	"github.com/docker/docker/pkg/ioutils"
-	"github.com/docker/docker/pkg/progress"
-	"github.com/docker/docker/pkg/stringid"
-	"github.com/docker/docker/registry"
-	specs "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/sirupsen/logrus"
-)
-
-type v1Puller struct {
-	v1IDService *metadata.V1IDService
-	endpoint    registry.APIEndpoint
-	config      *ImagePullConfig
-	repoInfo    *registry.RepositoryInfo
-	session     *registry.Session
-}
-
-func (p *v1Puller) Pull(ctx context.Context, ref reference.Named, _ *specs.Platform) error {
-	if _, isCanonical := ref.(reference.Canonical); isCanonical {
-		// Allowing fallback, because HTTPS v1 is before HTTP v2
-		return fallbackError{err: ErrNoSupport{Err: errors.New("Cannot pull by digest with v1 registry")}}
-	}
-
-	tlsConfig, err := p.config.RegistryService.TLSConfig(p.repoInfo.Index.Name)
-	if err != nil {
-		return err
-	}
-	// Adds Docker-specific headers as well as user-specified headers (metaHeaders)
-	tr := transport.NewTransport(
-		// TODO(tiborvass): was ReceiveTimeout
-		registry.NewTransport(tlsConfig),
-		registry.Headers(dockerversion.DockerUserAgent(ctx), p.config.MetaHeaders)...,
-	)
-	client := registry.HTTPClient(tr)
-	v1Endpoint := p.endpoint.ToV1Endpoint(dockerversion.DockerUserAgent(ctx), p.config.MetaHeaders)
-	p.session, err = registry.NewSession(client, p.config.AuthConfig, v1Endpoint)
-	if err != nil {
-		// TODO(dmcgowan): Check if should fallback
-		logrus.Debugf("Fallback from error: %s", err)
-		return fallbackError{err: err}
-	}
-	if err := p.pullRepository(ctx, ref); err != nil {
-		// TODO(dmcgowan): Check if should fallback
-		return err
-	}
-	progress.Message(p.config.ProgressOutput, "", p.repoInfo.Name.Name()+": this image was pulled from a legacy registry.  Important: This registry version will not be supported in future versions of docker.")
-
-	return nil
-}
-
-func (p *v1Puller) pullRepository(ctx context.Context, ref reference.Named) error {
-	progress.Message(p.config.ProgressOutput, "", "Pulling repository "+p.repoInfo.Name.Name())
-
-	tagged, isTagged := ref.(reference.NamedTagged)
-
-	repoData, err := p.session.GetRepositoryData(p.repoInfo.Name)
-	if err != nil {
-		if strings.Contains(err.Error(), "HTTP code: 404") {
-			if isTagged {
-				return fmt.Errorf("Error: image %s:%s not found", reference.Path(p.repoInfo.Name), tagged.Tag())
-			}
-			return fmt.Errorf("Error: image %s not found", reference.Path(p.repoInfo.Name))
-		}
-		// Unexpected HTTP error
-		return err
-	}
-
-	logrus.Debug("Retrieving the tag list")
-	var tagsList map[string]string
-	if !isTagged {
-		tagsList, err = p.session.GetRemoteTags(repoData.Endpoints, p.repoInfo.Name)
-	} else {
-		var tagID string
-		tagsList = make(map[string]string)
-		tagID, err = p.session.GetRemoteTag(repoData.Endpoints, p.repoInfo.Name, tagged.Tag())
-		if err == registry.ErrRepoNotFound {
-			return fmt.Errorf("Tag %s not found in repository %s", tagged.Tag(), p.repoInfo.Name.Name())
-		}
-		tagsList[tagged.Tag()] = tagID
-	}
-	if err != nil {
-		logrus.Errorf("unable to get remote tags: %s", err)
-		return err
-	}
-
-	for tag, id := range tagsList {
-		repoData.ImgList[id] = &registry.ImgData{
-			ID:       id,
-			Tag:      tag,
-			Checksum: "",
-		}
-	}
-
-	layersDownloaded := false
-	for _, imgData := range repoData.ImgList {
-		if isTagged && imgData.Tag != tagged.Tag() {
-			continue
-		}
-
-		err := p.downloadImage(ctx, repoData, imgData, &layersDownloaded)
-		if err != nil {
-			return err
-		}
-	}
-
-	writeStatus(reference.FamiliarString(ref), p.config.ProgressOutput, layersDownloaded)
-	return nil
-}
-
-func (p *v1Puller) downloadImage(ctx context.Context, repoData *registry.RepositoryData, img *registry.ImgData, layersDownloaded *bool) error {
-	if img.Tag == "" {
-		logrus.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
-		return nil
-	}
-
-	localNameRef, err := reference.WithTag(p.repoInfo.Name, img.Tag)
-	if err != nil {
-		retErr := fmt.Errorf("Image (id: %s) has invalid tag: %s", img.ID, img.Tag)
-		logrus.Debug(retErr.Error())
-		return retErr
-	}
-
-	if err := v1.ValidateID(img.ID); err != nil {
-		return err
-	}
-
-	progress.Updatef(p.config.ProgressOutput, stringid.TruncateID(img.ID), "Pulling image (%s) from %s", img.Tag, p.repoInfo.Name.Name())
-	success := false
-	var lastErr error
-	for _, ep := range p.repoInfo.Index.Mirrors {
-		ep += "v1/"
-		progress.Updatef(p.config.ProgressOutput, stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, p.repoInfo.Name.Name(), ep))
-		if err = p.pullImage(ctx, img.ID, ep, localNameRef, layersDownloaded); err != nil {
-			// Don't report errors when pulling from mirrors.
-			logrus.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, p.repoInfo.Name.Name(), ep, err)
-			continue
-		}
-		success = true
-		break
-	}
-	if !success {
-		for _, ep := range repoData.Endpoints {
-			progress.Updatef(p.config.ProgressOutput, stringid.TruncateID(img.ID), "Pulling image (%s) from %s, endpoint: %s", img.Tag, p.repoInfo.Name.Name(), ep)
-			if err = p.pullImage(ctx, img.ID, ep, localNameRef, layersDownloaded); err != nil {
-				// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
-				// As the error is also given to the output stream the user will see the error.
-				lastErr = err
-				progress.Updatef(p.config.ProgressOutput, stringid.TruncateID(img.ID), "Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, p.repoInfo.Name.Name(), ep, err)
-				continue
-			}
-			success = true
-			break
-		}
-	}
-	if !success {
-		err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, p.repoInfo.Name.Name(), lastErr)
-		progress.Update(p.config.ProgressOutput, stringid.TruncateID(img.ID), err.Error())
-		return err
-	}
-	return nil
-}
-
-func (p *v1Puller) pullImage(ctx context.Context, v1ID, endpoint string, localNameRef reference.Named, layersDownloaded *bool) (err error) {
-	var history []string
-	history, err = p.session.GetRemoteHistory(v1ID, endpoint)
-	if err != nil {
-		return err
-	}
-	if len(history) < 1 {
-		return fmt.Errorf("empty history for image %s", v1ID)
-	}
-	progress.Update(p.config.ProgressOutput, stringid.TruncateID(v1ID), "Pulling dependent layers")
-
-	var (
-		descriptors []xfer.DownloadDescriptor
-		newHistory  []image.History
-		imgJSON     []byte
-		imgSize     int64
-	)
-
-	// Iterate over layers, in order from bottom-most to top-most. Download
-	// config for all layers and create descriptors.
-	for i := len(history) - 1; i >= 0; i-- {
-		v1LayerID := history[i]
-		imgJSON, imgSize, err = p.downloadLayerConfig(v1LayerID, endpoint)
-		if err != nil {
-			return err
-		}
-
-		// Create a new-style config from the legacy configs
-		h, err := v1.HistoryFromConfig(imgJSON, false)
-		if err != nil {
-			return err
-		}
-		newHistory = append(newHistory, h)
-
-		layerDescriptor := &v1LayerDescriptor{
-			v1LayerID:        v1LayerID,
-			indexName:        p.repoInfo.Index.Name,
-			endpoint:         endpoint,
-			v1IDService:      p.v1IDService,
-			layersDownloaded: layersDownloaded,
-			layerSize:        imgSize,
-			session:          p.session,
-		}
-
-		descriptors = append(descriptors, layerDescriptor)
-	}
-
-	rootFS := image.NewRootFS()
-	resultRootFS, release, err := p.config.DownloadManager.Download(ctx, *rootFS, "", descriptors, p.config.ProgressOutput)
-	if err != nil {
-		return err
-	}
-	defer release()
-
-	config, err := v1.MakeConfigFromV1Config(imgJSON, &resultRootFS, newHistory)
-	if err != nil {
-		return err
-	}
-
-	imageID, err := p.config.ImageStore.Put(config)
-	if err != nil {
-		return err
-	}
-
-	if p.config.ReferenceStore != nil {
-		if err := p.config.ReferenceStore.AddTag(localNameRef, imageID, true); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (p *v1Puller) downloadLayerConfig(v1LayerID, endpoint string) (imgJSON []byte, imgSize int64, err error) {
-	progress.Update(p.config.ProgressOutput, stringid.TruncateID(v1LayerID), "Pulling metadata")
-
-	retries := 5
-	for j := 1; j <= retries; j++ {
-		imgJSON, imgSize, err := p.session.GetRemoteImageJSON(v1LayerID, endpoint)
-		if err != nil && j == retries {
-			progress.Update(p.config.ProgressOutput, stringid.TruncateID(v1LayerID), "Error pulling layer metadata")
-			return nil, 0, err
-		} else if err != nil {
-			time.Sleep(time.Duration(j) * 500 * time.Millisecond)
-			continue
-		}
-
-		return imgJSON, imgSize, nil
-	}
-
-	// not reached
-	return nil, 0, nil
-}
-
-type v1LayerDescriptor struct {
-	v1LayerID        string
-	indexName        string
-	endpoint         string
-	v1IDService      *metadata.V1IDService
-	layersDownloaded *bool
-	layerSize        int64
-	session          *registry.Session
-	tmpFile          *os.File
-}
-
-func (ld *v1LayerDescriptor) Key() string {
-	return "v1:" + ld.v1LayerID
-}
-
-func (ld *v1LayerDescriptor) ID() string {
-	return stringid.TruncateID(ld.v1LayerID)
-}
-
-func (ld *v1LayerDescriptor) DiffID() (layer.DiffID, error) {
-	return ld.v1IDService.Get(ld.v1LayerID, ld.indexName)
-}
-
-func (ld *v1LayerDescriptor) Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error) {
-	progress.Update(progressOutput, ld.ID(), "Pulling fs layer")
-	layerReader, err := ld.session.GetRemoteImageLayer(ld.v1LayerID, ld.endpoint, ld.layerSize)
-	if err != nil {
-		progress.Update(progressOutput, ld.ID(), "Error pulling dependent layers")
-		if uerr, ok := err.(*url.Error); ok {
-			err = uerr.Err
-		}
-		if terr, ok := err.(net.Error); ok && terr.Timeout() {
-			return nil, 0, err
-		}
-		return nil, 0, xfer.DoNotRetry{Err: err}
-	}
-	*ld.layersDownloaded = true
-
-	ld.tmpFile, err = ioutil.TempFile("", "GetImageBlob")
-	if err != nil {
-		layerReader.Close()
-		return nil, 0, err
-	}
-
-	reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, layerReader), progressOutput, ld.layerSize, ld.ID(), "Downloading")
-	defer reader.Close()
-
-	_, err = io.Copy(ld.tmpFile, reader)
-	if err != nil {
-		ld.Close()
-		return nil, 0, err
-	}
-
-	progress.Update(progressOutput, ld.ID(), "Download complete")
-
-	logrus.Debugf("Downloaded %s to tempfile %s", ld.ID(), ld.tmpFile.Name())
-
-	ld.tmpFile.Seek(0, 0)
-
-	// hand off the temporary file to the download manager, so it will only
-	// be closed once
-	tmpFile := ld.tmpFile
-	ld.tmpFile = nil
-
-	return ioutils.NewReadCloserWrapper(tmpFile, func() error {
-		tmpFile.Close()
-		err := os.RemoveAll(tmpFile.Name())
-		if err != nil {
-			logrus.Errorf("Failed to remove temp file: %s", tmpFile.Name())
-		}
-		return err
-	}), ld.layerSize, nil
-}
-
-func (ld *v1LayerDescriptor) Close() {
-	if ld.tmpFile != nil {
-		ld.tmpFile.Close()
-		if err := os.RemoveAll(ld.tmpFile.Name()); err != nil {
-			logrus.Errorf("Failed to remove temp file: %s", ld.tmpFile.Name())
-		}
-		ld.tmpFile = nil
-	}
-}
-
-func (ld *v1LayerDescriptor) Registered(diffID layer.DiffID) {
-	// Cache mapping from this layer's DiffID to the blobsum
-	ld.v1IDService.Set(ld.v1LayerID, ld.indexName, diffID)
-}

distribution/pull_v2.go

@@ -23,7 +23,7 @@ import (
 	"github.com/docker/docker/distribution/metadata"
 	"github.com/docker/docker/distribution/xfer"
 	"github.com/docker/docker/image"
-	"github.com/docker/docker/image/v1"
+	v1 "github.com/docker/docker/image/v1"
 	"github.com/docker/docker/layer"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/docker/docker/pkg/progress"
@@ -392,6 +392,10 @@ func (p *v2Puller) pullV2Tag(ctx context.Context, ref reference.Named, platform
 		if p.config.RequireSchema2 {
 			return false, fmt.Errorf("invalid manifest: not schema2")
 		}
+		msg := schema1DeprecationMessage(ref)
+		logrus.Warn(msg)
+		progress.Message(p.config.ProgressOutput, "", msg)
+
 		id, manifestDigest, err = p.pullSchema1(ctx, ref, v, platform)
 		if err != nil {
 			return false, err
@@ -787,6 +791,10 @@ func (p *v2Puller) pullManifestList(ctx context.Context, ref reference.Named, mf
 
 	switch v := manifest.(type) {
 	case *schema1.SignedManifest:
+		msg := schema1DeprecationMessage(ref)
+		logrus.Warn(msg)
+		progress.Message(p.config.ProgressOutput, "", msg)
+
 		platform := toOCIPlatform(manifestMatches[0].Platform)
 		id, _, err = p.pullSchema1(ctx, manifestRef, v, &platform)
 		if err != nil {

distribution/push.go

@@ -41,13 +41,7 @@ func NewPusher(ref reference.Named, endpoint registry.APIEndpoint, repoInfo *reg
 			config:            imagePushConfig,
 		}, nil
 	case registry.APIVersion1:
-		return &v1Pusher{
-			v1IDService: metadata.NewV1IDService(imagePushConfig.MetadataStore),
-			ref:         ref,
-			endpoint:    endpoint,
-			repoInfo:    repoInfo,
-			config:      imagePushConfig,
-		}, nil
+		return nil, fmt.Errorf("protocol version %d no longer supported. Please contact admins of registry %s", endpoint.Version, endpoint.URL)
 	}
 	return nil, fmt.Errorf("unknown version %d for registry %s", endpoint.Version, endpoint.URL)
 }

distribution/push_v1.go

@@ -1,457 +0,0 @@
-package distribution // import "github.com/docker/docker/distribution"
-
-import (
-	"context"
-	"fmt"
-	"sync"
-
-	"github.com/docker/distribution/reference"
-	"github.com/docker/distribution/registry/client/transport"
-	"github.com/docker/docker/distribution/metadata"
-	"github.com/docker/docker/dockerversion"
-	"github.com/docker/docker/image"
-	"github.com/docker/docker/image/v1"
-	"github.com/docker/docker/layer"
-	"github.com/docker/docker/pkg/ioutils"
-	"github.com/docker/docker/pkg/progress"
-	"github.com/docker/docker/pkg/stringid"
-	"github.com/docker/docker/pkg/system"
-	"github.com/docker/docker/registry"
-	"github.com/opencontainers/go-digest"
-	"github.com/sirupsen/logrus"
-)
-
-type v1Pusher struct {
-	v1IDService *metadata.V1IDService
-	endpoint    registry.APIEndpoint
-	ref         reference.Named
-	repoInfo    *registry.RepositoryInfo
-	config      *ImagePushConfig
-	session     *registry.Session
-}
-
-func (p *v1Pusher) Push(ctx context.Context) error {
-	tlsConfig, err := p.config.RegistryService.TLSConfig(p.repoInfo.Index.Name)
-	if err != nil {
-		return err
-	}
-	// Adds Docker-specific headers as well as user-specified headers (metaHeaders)
-	tr := transport.NewTransport(
-		// TODO(tiborvass): was NoTimeout
-		registry.NewTransport(tlsConfig),
-		registry.Headers(dockerversion.DockerUserAgent(ctx), p.config.MetaHeaders)...,
-	)
-	client := registry.HTTPClient(tr)
-	v1Endpoint := p.endpoint.ToV1Endpoint(dockerversion.DockerUserAgent(ctx), p.config.MetaHeaders)
-	p.session, err = registry.NewSession(client, p.config.AuthConfig, v1Endpoint)
-	if err != nil {
-		// TODO(dmcgowan): Check if should fallback
-		return fallbackError{err: err}
-	}
-	if err := p.pushRepository(ctx); err != nil {
-		// TODO(dmcgowan): Check if should fallback
-		return err
-	}
-	return nil
-}
-
-// v1Image exposes the configuration, filesystem layer ID, and a v1 ID for an
-// image being pushed to a v1 registry.
-type v1Image interface {
-	Config() []byte
-	Layer() layer.Layer
-	V1ID() string
-}
-
-type v1ImageCommon struct {
-	layer  layer.Layer
-	config []byte
-	v1ID   string
-}
-
-func (common *v1ImageCommon) Config() []byte {
-	return common.config
-}
-
-func (common *v1ImageCommon) V1ID() string {
-	return common.v1ID
-}
-
-func (common *v1ImageCommon) Layer() layer.Layer {
-	return common.layer
-}
-
-// v1TopImage defines a runnable (top layer) image being pushed to a v1
-// registry.
-type v1TopImage struct {
-	v1ImageCommon
-	imageID image.ID
-}
-
-func newV1TopImage(imageID image.ID, img *image.Image, l layer.Layer, parent *v1DependencyImage) (*v1TopImage, error) {
-	v1ID := imageID.Digest().Hex()
-	parentV1ID := ""
-	if parent != nil {
-		parentV1ID = parent.V1ID()
-	}
-
-	config, err := v1.MakeV1ConfigFromConfig(img, v1ID, parentV1ID, false)
-	if err != nil {
-		return nil, err
-	}
-
-	return &v1TopImage{
-		v1ImageCommon: v1ImageCommon{
-			v1ID:   v1ID,
-			config: config,
-			layer:  l,
-		},
-		imageID: imageID,
-	}, nil
-}
-
-// v1DependencyImage defines a dependency layer being pushed to a v1 registry.
-type v1DependencyImage struct {
-	v1ImageCommon
-}
-
-func newV1DependencyImage(l layer.Layer, parent *v1DependencyImage) *v1DependencyImage {
-	v1ID := digest.Digest(l.ChainID()).Hex()
-
-	var config string
-	if parent != nil {
-		config = fmt.Sprintf(`{"id":"%s","parent":"%s"}`, v1ID, parent.V1ID())
-	} else {
-		config = fmt.Sprintf(`{"id":"%s"}`, v1ID)
-	}
-	return &v1DependencyImage{
-		v1ImageCommon: v1ImageCommon{
-			v1ID:   v1ID,
-			config: []byte(config),
-			layer:  l,
-		},
-	}
-}
-
-// Retrieve the all the images to be uploaded in the correct order
-func (p *v1Pusher) getImageList() (imageList []v1Image, tagsByImage map[image.ID][]string, referencedLayers []PushLayer, err error) {
-	tagsByImage = make(map[image.ID][]string)
-
-	// Ignore digest references
-	if _, isCanonical := p.ref.(reference.Canonical); isCanonical {
-		return
-	}
-
-	tagged, isTagged := p.ref.(reference.NamedTagged)
-	if isTagged {
-		// Push a specific tag
-		var imgID image.ID
-		var dgst digest.Digest
-		dgst, err = p.config.ReferenceStore.Get(p.ref)
-		if err != nil {
-			return
-		}
-		imgID = image.IDFromDigest(dgst)
-
-		imageList, err = p.imageListForTag(imgID, nil, &referencedLayers)
-		if err != nil {
-			return
-		}
-
-		tagsByImage[imgID] = []string{tagged.Tag()}
-
-		return
-	}
-
-	imagesSeen := make(map[digest.Digest]struct{})
-	dependenciesSeen := make(map[layer.ChainID]*v1DependencyImage)
-
-	associations := p.config.ReferenceStore.ReferencesByName(p.ref)
-	for _, association := range associations {
-		if tagged, isTagged = association.Ref.(reference.NamedTagged); !isTagged {
-			// Ignore digest references.
-			continue
-		}
-
-		imgID := image.IDFromDigest(association.ID)
-		tagsByImage[imgID] = append(tagsByImage[imgID], tagged.Tag())
-
-		if _, present := imagesSeen[association.ID]; present {
-			// Skip generating image list for already-seen image
-			continue
-		}
-		imagesSeen[association.ID] = struct{}{}
-
-		imageListForThisTag, err := p.imageListForTag(imgID, dependenciesSeen, &referencedLayers)
-		if err != nil {
-			return nil, nil, nil, err
-		}
-
-		// append to main image list
-		imageList = append(imageList, imageListForThisTag...)
-	}
-	if len(imageList) == 0 {
-		return nil, nil, nil, fmt.Errorf("No images found for the requested repository / tag")
-	}
-	logrus.Debugf("Image list: %v", imageList)
-	logrus.Debugf("Tags by image: %v", tagsByImage)
-
-	return
-}
-
-func (p *v1Pusher) imageListForTag(imgID image.ID, dependenciesSeen map[layer.ChainID]*v1DependencyImage, referencedLayers *[]PushLayer) (imageListForThisTag []v1Image, err error) {
-	ics, ok := p.config.ImageStore.(*imageConfigStore)
-	if !ok {
-		return nil, fmt.Errorf("only image store images supported for v1 push")
-	}
-	img, err := ics.Store.Get(imgID)
-	if err != nil {
-		return nil, err
-	}
-
-	topLayerID := img.RootFS.ChainID()
-
-	if !system.IsOSSupported(img.OperatingSystem()) {
-		return nil, system.ErrNotSupportedOperatingSystem
-	}
-	pl, err := p.config.LayerStores[img.OperatingSystem()].Get(topLayerID)
-	*referencedLayers = append(*referencedLayers, pl)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get top layer from image: %v", err)
-	}
-
-	// V1 push is deprecated, only support existing layerstore layers
-	lsl, ok := pl.(*storeLayer)
-	if !ok {
-		return nil, fmt.Errorf("only layer store layers supported for v1 push")
-	}
-	l := lsl.Layer
-
-	dependencyImages, parent := generateDependencyImages(l.Parent(), dependenciesSeen)
-
-	topImage, err := newV1TopImage(imgID, img, l, parent)
-	if err != nil {
-		return nil, err
-	}
-
-	imageListForThisTag = append(dependencyImages, topImage)
-
-	return
-}
-
-func generateDependencyImages(l layer.Layer, dependenciesSeen map[layer.ChainID]*v1DependencyImage) (imageListForThisTag []v1Image, parent *v1DependencyImage) {
-	if l == nil {
-		return nil, nil
-	}
-
-	imageListForThisTag, parent = generateDependencyImages(l.Parent(), dependenciesSeen)
-
-	if dependenciesSeen != nil {
-		if dependencyImage, present := dependenciesSeen[l.ChainID()]; present {
-			// This layer is already on the list, we can ignore it
-			// and all its parents.
-			return imageListForThisTag, dependencyImage
-		}
-	}
-
-	dependencyImage := newV1DependencyImage(l, parent)
-	imageListForThisTag = append(imageListForThisTag, dependencyImage)
-
-	if dependenciesSeen != nil {
-		dependenciesSeen[l.ChainID()] = dependencyImage
-	}
-
-	return imageListForThisTag, dependencyImage
-}
-
-// createImageIndex returns an index of an image's layer IDs and tags.
-func createImageIndex(images []v1Image, tags map[image.ID][]string) []*registry.ImgData {
-	var imageIndex []*registry.ImgData
-	for _, img := range images {
-		v1ID := img.V1ID()
-
-		if topImage, isTopImage := img.(*v1TopImage); isTopImage {
-			if tags, hasTags := tags[topImage.imageID]; hasTags {
-				// If an image has tags you must add an entry in the image index
-				// for each tag
-				for _, tag := range tags {
-					imageIndex = append(imageIndex, &registry.ImgData{
-						ID:  v1ID,
-						Tag: tag,
-					})
-				}
-				continue
-			}
-		}
-
-		// If the image does not have a tag it still needs to be sent to the
-		// registry with an empty tag so that it is associated with the repository
-		imageIndex = append(imageIndex, &registry.ImgData{
-			ID:  v1ID,
-			Tag: "",
-		})
-	}
-	return imageIndex
-}
-
-// lookupImageOnEndpoint checks the specified endpoint to see if an image exists
-// and if it is absent then it sends the image id to the channel to be pushed.
-func (p *v1Pusher) lookupImageOnEndpoint(wg *sync.WaitGroup, endpoint string, images chan v1Image, imagesToPush chan string) {
-	defer wg.Done()
-	for image := range images {
-		v1ID := image.V1ID()
-		truncID := stringid.TruncateID(image.Layer().DiffID().String())
-		if err := p.session.LookupRemoteImage(v1ID, endpoint); err != nil {
-			logrus.Errorf("Error in LookupRemoteImage: %s", err)
-			imagesToPush <- v1ID
-			progress.Update(p.config.ProgressOutput, truncID, "Waiting")
-		} else {
-			progress.Update(p.config.ProgressOutput, truncID, "Already exists")
-		}
-	}
-}
-
-func (p *v1Pusher) pushImageToEndpoint(ctx context.Context, endpoint string, imageList []v1Image, tags map[image.ID][]string, repo *registry.RepositoryData) error {
-	workerCount := len(imageList)
-	// start a maximum of 5 workers to check if images exist on the specified endpoint.
-	if workerCount > 5 {
-		workerCount = 5
-	}
-	var (
-		wg           = &sync.WaitGroup{}
-		imageData    = make(chan v1Image, workerCount*2)
-		imagesToPush = make(chan string, workerCount*2)
-		pushes       = make(chan map[string]struct{}, 1)
-	)
-	for i := 0; i < workerCount; i++ {
-		wg.Add(1)
-		go p.lookupImageOnEndpoint(wg, endpoint, imageData, imagesToPush)
-	}
-	// start a go routine that consumes the images to push
-	go func() {
-		shouldPush := make(map[string]struct{})
-		for id := range imagesToPush {
-			shouldPush[id] = struct{}{}
-		}
-		pushes <- shouldPush
-	}()
-	for _, v1Image := range imageList {
-		imageData <- v1Image
-	}
-	// close the channel to notify the workers that there will be no more images to check.
-	close(imageData)
-	wg.Wait()
-	close(imagesToPush)
-	// wait for all the images that require pushes to be collected into a consumable map.
-	shouldPush := <-pushes
-	// finish by pushing any images and tags to the endpoint.  The order that the images are pushed
-	// is very important that is why we are still iterating over the ordered list of imageIDs.
-	for _, img := range imageList {
-		v1ID := img.V1ID()
-		if _, push := shouldPush[v1ID]; push {
-			if _, err := p.pushImage(ctx, img, endpoint); err != nil {
-				// FIXME: Continue on error?
-				return err
-			}
-		}
-		if topImage, isTopImage := img.(*v1TopImage); isTopImage {
-			for _, tag := range tags[topImage.imageID] {
-				progress.Messagef(p.config.ProgressOutput, "", "Pushing tag for rev [%s] on {%s}", stringid.TruncateID(v1ID), endpoint+"repositories/"+reference.Path(p.repoInfo.Name)+"/tags/"+tag)
-				if err := p.session.PushRegistryTag(p.repoInfo.Name, v1ID, tag, endpoint); err != nil {
-					return err
-				}
-			}
-		}
-	}
-	return nil
-}
-
-// pushRepository pushes layers that do not already exist on the registry.
-func (p *v1Pusher) pushRepository(ctx context.Context) error {
-	imgList, tags, referencedLayers, err := p.getImageList()
-	defer func() {
-		for _, l := range referencedLayers {
-			l.Release()
-		}
-	}()
-	if err != nil {
-		return err
-	}
-
-	imageIndex := createImageIndex(imgList, tags)
-	for _, data := range imageIndex {
-		logrus.Debugf("Pushing ID: %s with Tag: %s", data.ID, data.Tag)
-	}
-
-	// Register all the images in a repository with the registry
-	// If an image is not in this list it will not be associated with the repository
-	repoData, err := p.session.PushImageJSONIndex(p.repoInfo.Name, imageIndex, false, nil)
-	if err != nil {
-		return err
-	}
-	// push the repository to each of the endpoints only if it does not exist.
-	for _, endpoint := range repoData.Endpoints {
-		if err := p.pushImageToEndpoint(ctx, endpoint, imgList, tags, repoData); err != nil {
-			return err
-		}
-	}
-	_, err = p.session.PushImageJSONIndex(p.repoInfo.Name, imageIndex, true, repoData.Endpoints)
-	return err
-}
-
-func (p *v1Pusher) pushImage(ctx context.Context, v1Image v1Image, ep string) (checksum string, err error) {
-	l := v1Image.Layer()
-	v1ID := v1Image.V1ID()
-	truncID := stringid.TruncateID(l.DiffID().String())
-
-	jsonRaw := v1Image.Config()
-	progress.Update(p.config.ProgressOutput, truncID, "Pushing")
-
-	// General rule is to use ID for graph accesses and compatibilityID for
-	// calls to session.registry()
-	imgData := &registry.ImgData{
-		ID: v1ID,
-	}
-
-	// Send the json
-	if err := p.session.PushImageJSONRegistry(imgData, jsonRaw, ep); err != nil {
-		if err == registry.ErrAlreadyExists {
-			progress.Update(p.config.ProgressOutput, truncID, "Image already pushed, skipping")
-			return "", nil
-		}
-		return "", err
-	}
-
-	arch, err := l.TarStream()
-	if err != nil {
-		return "", err
-	}
-	defer arch.Close()
-
-	// don't care if this fails; best effort
-	size, _ := l.DiffSize()
-
-	// Send the layer
-	logrus.Debugf("rendered layer for %s of [%d] size", v1ID, size)
-
-	reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, arch), p.config.ProgressOutput, size, truncID, "Pushing")
-	defer reader.Close()
-
-	checksum, checksumPayload, err := p.session.PushImageLayerRegistry(v1ID, reader, ep, jsonRaw)
-	if err != nil {
-		return "", err
-	}
-	imgData.Checksum = checksum
-	imgData.ChecksumPayload = checksumPayload
-	// Send the checksum
-	if err := p.session.PushImageChecksumRegistry(imgData, ep); err != nil {
-		return "", err
-	}
-
-	if err := p.v1IDService.Set(v1ID, p.repoInfo.Index.Name, l.DiffID()); err != nil {
-		logrus.Warnf("Could not set v1 ID mapping: %v", err)
-	}
-
-	progress.Update(p.config.ProgressOutput, truncID, "Image successfully pushed")
-	return imgData.Checksum, nil
-}

distribution/push_v2.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"runtime"
 	"sort"
 	"strings"
 	"sync"
@@ -180,8 +181,30 @@ func (p *v2Pusher) pushV2Tag(ctx context.Context, ref reference.NamedTagged, id
 
 	putOptions := []distribution.ManifestServiceOption{distribution.WithTag(ref.Tag())}
 	if _, err = manSvc.Put(ctx, manifest, putOptions...); err != nil {
-		logrus.Warnf("failed to upload schema2 manifest: %v", err)
-		return err
+		if runtime.GOOS == "windows" || p.config.TrustKey == nil || p.config.RequireSchema2 {
+			logrus.Warnf("failed to upload schema2 manifest: %v", err)
+			return err
+		}
+
+		logrus.Warnf("failed to upload schema2 manifest: %v - falling back to schema1", err)
+
+		msg := schema1DeprecationMessage(ref)
+		logrus.Warn(msg)
+		progress.Message(p.config.ProgressOutput, "", msg)
+
+		manifestRef, err := reference.WithTag(p.repo.Named(), ref.Tag())
+		if err != nil {
+			return err
+		}
+		builder = schema1.NewConfigManifestBuilder(p.repo.Blobs(ctx), p.config.TrustKey, manifestRef, imgConfig)
+		manifest, err = manifestFromBuilder(ctx, builder, descriptors)
+		if err != nil {
+			return err
+		}
+
+		if _, err = manSvc.Put(ctx, manifest, putOptions...); err != nil {
+			return err
+		}
 	}
 
 	var canonicalManifest []byte

distribution/registry.go

@@ -156,3 +156,7 @@ func (th *existingTokenHandler) AuthorizeRequest(req *http.Request, params map[s
 	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", th.token))
 	return nil
 }
+
+func schema1DeprecationMessage(ref reference.Named) string {
+	return fmt.Sprintf("[DEPRECATION NOTICE] registry v2 schema1 support will be removed in an upcoming release. Please contact admins of the %s registry NOW to avoid future disruption.", reference.Domain(ref))
+}

docs/api/version-history.md

@@ -49,6 +49,11 @@ keywords: "API, Docker, rcli, REST, documentation"
 * `GET /info` now returns information about `DataPathPort` that is currently used in swarm
 * `GET /info` now returns `PidsLimit` boolean to indicate if the host kernel has
   PID limit support enabled.
+* `GET /info` now includes `name=rootless` in `SecurityOptions` when the daemon is running in
+  rootless mode.  This change is not versioned, and affects all API versions if the daemon has
+  this patch.
+* `GET /info` now returns `none` as `CgroupDriver` when the daemon is running in rootless mode.
+  This change is not versioned, and affects all API versions if the daemon has this patch.
 * `POST /containers/create` now accepts `DeviceRequests` as part of `HostConfig`.
   Can be used to set Nvidia GPUs.
 * `GET /swarm` endpoint now returns DataPathPort info
@@ -215,6 +220,7 @@ keywords: "API, Docker, rcli, REST, documentation"
 * `GET /events` now supports service, node and secret events which are emitted when users create, update and remove service, node and secret
 * `GET /events` now supports network remove event which is emitted when users remove a swarm scoped network
 * `GET /events` now supports a filter type `scope` in which supported value could be swarm and local
+* `PUT /containers/(name)/archive` now accepts a `copyUIDGID` parameter to allow copy UID/GID maps to dest file or dir.
 
 ## v1.29 API changes
 

docs/rootless.md

@@ -20,7 +20,6 @@ $ grep ^$(whoami): /etc/subgid
 penguin:231072:65536
 ```
 
-* Either [slirp4netns](https://github.com/rootless-containers/slirp4netns) (v0.3+) or [VPNKit](https://github.com/moby/vpnkit) needs to be installed. slirp4netns is preferred for the best performance.
 
 ### Distribution-specific hint
 
@@ -55,10 +54,9 @@ penguin:231072:65536
 You need to run `dockerd-rootless.sh` instead of `dockerd`.
 
 ```console
-$ dockerd-rootless.sh --experimental --userland-proxy --userland-proxy-path=$(which rootlesskit-docker-proxy)"
+$ dockerd-rootless.sh --experimental
 ```
 As Rootless mode is experimental per se, currently you always need to run `dockerd-rootless.sh` with `--experimental`.
-Also, to expose ports, you need to set `--userland-proxy-path` to the path of `rootlesskit-docker-proxy` binary.
 
 Remarks:
 * The socket path is set to `$XDG_RUNTIME_DIR/docker.sock` by default. `$XDG_RUNTIME_DIR` is typically set to `/run/user/$UID`.
@@ -66,6 +64,8 @@ Remarks:
 * The exec dir is set to `$XDG_RUNTIME_DIR/docker` by default.
 * The daemon config dir is set to `~/.config/docker` (not `~/.docker`, which is used by the client) by default.
 * The `dockerd-rootless.sh` script executes `dockerd` in its own user, mount, and network namespaces. You can enter the namespaces by running `nsenter -U --preserve-credentials -n -m -t $(cat $XDG_RUNTIME_DIR/docker.pid)`.
+* `docker info` shows `rootless` in `SecurityOptions`
+* `docker info` shows `none` as `Cgroup Driver`
 
 ### Client
 
@@ -82,3 +82,12 @@ To route ping packets, you need to set up `net.ipv4.ping_group_range` properly a
 ```console
 $ sudo sh -c "echo 0   2147483647  > /proc/sys/net/ipv4/ping_group_range"
 ```
+
+### Changing network stack
+
+`dockerd-rootless.sh` uses [slirp4netns](https://github.com/rootless-containers/slirp4netns) (if installed) or [VPNKit](https://github.com/moby/vpnkit) as the network stack by default.
+These network stacks run in userspace and might have performance overhead. See [RootlessKit documentation](https://github.com/rootless-containers/rootlesskit/tree/v0.4.0#network-drivers) for further information.
+
+Optionally, you can use `lxc-user-nic` instead for the best performance.
+To use `lxc-user-nic`, you need to edit [`/etc/lxc/lxc-usernet`](https://github.com/rootless-containers/rootlesskit/tree/v0.4.0#--netlxc-user-nic-experimental) and set `$DOCKERD_ROOTLESS_ROOTLESSKIT_NET=lxc-user-nic`.
+

hack/ci/windows.ps1

@@ -119,6 +119,7 @@ $FinallyColour="Cyan"
 #$env:INTEGRATION_IN_CONTAINER="yes"
 #$env:WINDOWS_BASE_IMAGE=""
 #$env:SKIP_COPY_GO="yes"
+#$env:INTEGRATION_TESTFLAGS="-test.v"
 
 Function Nuke-Everything {
     $ErrorActionPreference = 'SilentlyContinue'
@@ -409,7 +410,7 @@ Try {
     # Redirect to a temporary location. 
     $TEMPORIG=$env:TEMP
     $env:TEMP="$env:TESTRUN_DRIVE`:\$env:TESTRUN_SUBDIR\CI-$COMMITHASH"
-    $env:LOCALAPPDATA="$TEMP\localappdata"
+    $env:LOCALAPPDATA="$env:TEMP\localappdata"
     $errorActionPreference='Stop'
     New-Item -ItemType Directory "$env:TEMP" -ErrorAction SilentlyContinue | Out-Null
     New-Item -ItemType Directory "$env:TEMP\userprofile" -ErrorAction SilentlyContinue  | Out-Null
@@ -825,18 +826,32 @@ Try {
                                                         docker `
                                                         "`$env`:PATH`='c`:\target;'+`$env:PATH`;  `$env:DOCKER_HOST`='tcp`://'+(ipconfig | select -last 1).Substring(39)+'`:2357'; c:\target\runIntegrationCLI.ps1" | Out-Host } )
             } else  {
-                Write-Host -ForegroundColor Green "INFO: Integration tests being run from the host:"
-                Set-Location "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker\integration-cli"
                 $env:DOCKER_HOST=$DASHH_CUT  
                 $env:PATH="$env:TEMP\binary;$env:PATH;"  # Force to use the test binaries, not the host ones.
-                Write-Host -ForegroundColor Green "INFO: $c"
                 Write-Host -ForegroundColor Green "INFO: DOCKER_HOST at $DASHH_CUT"
+
+                $ErrorActionPreference = "SilentlyContinue"
+                Write-Host -ForegroundColor Cyan "INFO: Integration API tests being run from the host:"
+                if (!($env:INTEGRATION_TESTFLAGS)) {
+                    $env:INTEGRATION_TESTFLAGS = "-test.v"
+                }
+                Set-Location "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker"
+                $start=(Get-Date); Invoke-Expression ".\hack\make.ps1 -TestIntegration"; $Duration=New-Timespan -Start $start -End (Get-Date)
+                $ErrorActionPreference = "Stop"
+                if (-not($LastExitCode -eq 0)) {
+                    Throw "ERROR: Integration API tests failed at $(Get-Date). Duration`:$Duration"
+                }
+
+                $ErrorActionPreference = "SilentlyContinue"
+                Write-Host -ForegroundColor Green "INFO: Integration CLI tests being run from the host:"
+                Write-Host -ForegroundColor Green "INFO: $c"
+                Set-Location "$env:SOURCES_DRIVE`:\$env:SOURCES_SUBDIR\src\github.com\docker\docker\integration-cli"
                 # Explicit to not use measure-command otherwise don't get output as it goes
                 $start=(Get-Date); Invoke-Expression $c; $Duration=New-Timespan -Start $start -End (Get-Date)
             }
             $ErrorActionPreference = "Stop"
             if (-not($LastExitCode -eq 0)) {
-                Throw "ERROR: Integration tests failed at $(Get-Date). Duration`:$Duration"
+                Throw "ERROR: Integration CLI tests failed at $(Get-Date). Duration`:$Duration"
             }
             Write-Host  -ForegroundColor Green "INFO: Integration tests ended at $(Get-Date). Duration`:$Duration"
         } else {

hack/dockerfile/install/proxy.installer

@@ -3,7 +3,7 @@
 # LIBNETWORK_COMMIT is used to build the docker-userland-proxy binary. When
 # updating the binary version, consider updating github.com/docker/libnetwork
 # in vendor.conf accordingly
-LIBNETWORK_COMMIT=9ff9b57c344df5cd47443ad9e65702ec85c5aeb0
+LIBNETWORK_COMMIT=fc5a7d91d54cc98f64fc28f9e288b46a0bee756c
 
 install_proxy() {
 	case "$1" in

hack/dockerfile/install/rootlesskit.installer

@@ -1,7 +1,7 @@
 #!/bin/sh
 
-# v0.3.0
-ROOTLESSKIT_COMMIT=70e0502f328bc5ffb14692a7ea41abb77196043b
+# v0.4.1
+ROOTLESSKIT_COMMIT=27a0c7a2483732b33d4192c1d178c83c6b9e202d
 
 install_rootlesskit() {
 	case "$1" in

hack/dockerfile/install/runc.installer

@@ -4,7 +4,7 @@
 # The version of runc should match the version that is used by the containerd
 # version that is used. If you need to update runc, open a pull request in
 # the containerd project first, and update both after that is merged.
-RUNC_COMMIT=029124da7af7360afa781a0234d1b083550f797c # v1.0.0-rc7-6-g029124da
+RUNC_COMMIT=425e105d5a03fabd737a126ad93d62a9eeede87f # v1.0.0-rc8
 
 install_runc() {
 	# If using RHEL7 kernels (3.10.0 el7), disable kmem accounting/limiting

hack/make.ps1

@@ -60,6 +60,9 @@
 .PARAMETER TestUnit
      Runs unit tests.
 
+.PARAMETER TestIntegration
+     Runs integration tests.
+
 .PARAMETER All
      Runs everything this script knows about that can run in a container.
 
@@ -84,6 +87,7 @@ param(
     [Parameter(Mandatory=$False)][switch]$PkgImports,
     [Parameter(Mandatory=$False)][switch]$GoFormat,
     [Parameter(Mandatory=$False)][switch]$TestUnit,
+    [Parameter(Mandatory=$False)][switch]$TestIntegration,
     [Parameter(Mandatory=$False)][switch]$All
 )
 
@@ -320,6 +324,39 @@ Function Run-UnitTests() {
     if ($LASTEXITCODE -ne 0) { Throw "Unit tests failed" }
 }
 
+# Run the integration tests
+Function Run-IntegrationTests() {
+    $env:DOCKER_INTEGRATION_DAEMON_DEST = $root + "\bundles\tmp"
+    $dirs =  Get-ChildItem -Path integration -Directory -Recurse
+    $integration_api_dirs = @()
+    ForEach($dir in $dirs) {
+        $RelativePath = "." + $dir.FullName -replace "$($PWD.Path -replace "\\","\\")",""
+        If ($RelativePath -notmatch '(^.\\integration($|\\internal)|\\testdata)') {
+            $integration_api_dirs += $dir
+            Write-Host "Building test suite binary $RelativePath"
+            go test -c -o "$RelativePath\test.exe" $RelativePath
+        }
+    }
+
+    ForEach($dir in $integration_api_dirs) {
+        Set-Location $dir.FullName
+        Write-Host "Running $($PWD.Path)"
+        $pinfo = New-Object System.Diagnostics.ProcessStartInfo
+        $pinfo.FileName = "$($PWD.Path)\test.exe"
+        $pinfo.RedirectStandardError = $true
+        $pinfo.UseShellExecute = $false
+        $pinfo.Arguments = $env:INTEGRATION_TESTFLAGS
+        $p = New-Object System.Diagnostics.Process
+        $p.StartInfo = $pinfo
+        $p.Start() | Out-Null
+        $p.WaitForExit()
+        $err = $p.StandardError.ReadToEnd()
+        if (($LASTEXITCODE -ne 0) -and ($err -notlike "*warning: no tests to run*")) {
+            Throw "Integration tests failed: $err"
+        }
+    }
+}
+
 # Start of main code.
 Try {
     Write-Host -ForegroundColor Cyan "INFO: make.ps1 starting at $(Get-Date)"
@@ -331,13 +368,13 @@ Try {
     # Handle the "-All" shortcut to turn on all things we can handle.
     # Note we expressly only include the items which can run in a container - the validations tests cannot
     # as they require the .git directory which is excluded from the image by .dockerignore
-    if ($All) { $Client=$True; $Daemon=$True; $TestUnit=$True }
+    if ($All) { $Client=$True; $Daemon=$True; $TestUnit=$True; }
 
     # Handle the "-Binary" shortcut to build both client and daemon.
     if ($Binary) { $Client = $True; $Daemon = $True }
 
     # Default to building the daemon if not asked for anything explicitly.
-    if (-not($Client) -and -not($Daemon) -and -not($DCO) -and -not($PkgImports) -and -not($GoFormat) -and -not($TestUnit)) { $Daemon=$True }
+    if (-not($Client) -and -not($Daemon) -and -not($DCO) -and -not($PkgImports) -and -not($GoFormat) -and -not($TestUnit) -and -not($TestIntegration)) { $Daemon=$True }
 
     # Verify git is installed
     if ($(Get-Command git -ErrorAction SilentlyContinue) -eq $nil) { Throw "Git does not appear to be installed" }
@@ -425,6 +462,9 @@ Try {
     # Run unit tests
     if ($TestUnit) { Run-UnitTests }
 
+    # Run integration tests
+    if ($TestIntegration) { Run-IntegrationTests }
+
     # Gratuitous ASCII art.
     if ($Daemon -or $Client) {
         Write-Host

hack/validate/default

@@ -14,4 +14,4 @@ export SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 . ${SCRIPTDIR}/toml
 . ${SCRIPTDIR}/changelog-well-formed
 . ${SCRIPTDIR}/changelog-date-descending
-. ${SCRIPTDIR}/deprecate-integration-cli
+#. ${SCRIPTDIR}/deprecate-integration-cli

integration-cli/check_test.go

@@ -141,6 +141,39 @@ func (s *DockerRegistrySuite) TearDownTest(c *check.C) {
 	s.ds.TearDownTest(c)
 }
 
+func init() {
+	check.Suite(&DockerSchema1RegistrySuite{
+		ds: &DockerSuite{},
+	})
+}
+
+type DockerSchema1RegistrySuite struct {
+	ds  *DockerSuite
+	reg *registry.V2
+	d   *daemon.Daemon
+}
+
+func (s *DockerSchema1RegistrySuite) OnTimeout(c *check.C) {
+	s.d.DumpStackAndQuit()
+}
+
+func (s *DockerSchema1RegistrySuite) SetUpTest(c *check.C) {
+	testRequires(c, DaemonIsLinux, RegistryHosting, NotArm64, testEnv.IsLocalDaemon)
+	s.reg = registry.NewV2(c, registry.Schema1)
+	s.reg.WaitReady(c)
+	s.d = daemon.New(c, dockerBinary, dockerdBinary, testdaemon.WithEnvironment(testEnv.Execution))
+}
+
+func (s *DockerSchema1RegistrySuite) TearDownTest(c *check.C) {
+	if s.reg != nil {
+		s.reg.Close()
+	}
+	if s.d != nil {
+		s.d.Stop(c)
+	}
+	s.ds.TearDownTest(c)
+}
+
 func init() {
 	check.Suite(&DockerRegistryAuthHtpasswdSuite{
 		ds: &DockerSuite{},

integration-cli/docker_api_containers_test.go

@@ -623,29 +623,6 @@ func (s *DockerSuite) TestContainerAPICreateMultipleNetworksConfig(c *check.C) {
 	c.Assert(msg, checker.Contains, "net3")
 }
 
-func (s *DockerSuite) TestContainerAPICreateWithHostName(c *check.C) {
-	domainName := "test-domain"
-	hostName := "test-hostname"
-	config := containertypes.Config{
-		Image:      "busybox",
-		Hostname:   hostName,
-		Domainname: domainName,
-	}
-
-	cli, err := client.NewClientWithOpts(client.FromEnv)
-	assert.NilError(c, err)
-	defer cli.Close()
-
-	container, err := cli.ContainerCreate(context.Background(), &config, &containertypes.HostConfig{}, &networktypes.NetworkingConfig{}, "")
-	assert.NilError(c, err)
-
-	containerJSON, err := cli.ContainerInspect(context.Background(), container.ID)
-	assert.NilError(c, err)
-
-	c.Assert(containerJSON.Config.Hostname, checker.Equals, hostName, check.Commentf("Mismatched Hostname"))
-	c.Assert(containerJSON.Config.Domainname, checker.Equals, domainName, check.Commentf("Mismatched Domainname"))
-}
-
 func (s *DockerSuite) TestContainerAPICreateBridgeNetworkMode(c *check.C) {
 	// Windows does not support bridge
 	testRequires(c, DaemonIsLinux)

integration-cli/docker_cli_by_digest_test.go

@@ -3,9 +3,12 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 
+	"github.com/docker/distribution/manifest/schema1"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/integration-cli/checker"
@@ -77,6 +80,10 @@ func (s *DockerRegistrySuite) TestPullByTagDisplaysDigest(c *check.C) {
 	testPullByTagDisplaysDigest(c)
 }
 
+func (s *DockerSchema1RegistrySuite) TestPullByTagDisplaysDigest(c *check.C) {
+	testPullByTagDisplaysDigest(c)
+}
+
 func testPullByDigest(c *check.C) {
 	testRequires(c, DaemonIsLinux)
 	pushDigest, err := setupImage(c)
@@ -99,6 +106,10 @@ func (s *DockerRegistrySuite) TestPullByDigest(c *check.C) {
 	testPullByDigest(c)
 }
 
+func (s *DockerSchema1RegistrySuite) TestPullByDigest(c *check.C) {
+	testPullByDigest(c)
+}
+
 func testPullByDigestNoFallback(c *check.C) {
 	testRequires(c, DaemonIsLinux)
 	// pull from the registry using the <name>@<digest> reference
@@ -112,6 +123,10 @@ func (s *DockerRegistrySuite) TestPullByDigestNoFallback(c *check.C) {
 	testPullByDigestNoFallback(c)
 }
 
+func (s *DockerSchema1RegistrySuite) TestPullByDigestNoFallback(c *check.C) {
+	testPullByDigestNoFallback(c)
+}
+
 func (s *DockerRegistrySuite) TestCreateByDigest(c *check.C) {
 	pushDigest, err := setupImage(c)
 	assert.NilError(c, err, "error setting up image")
@@ -546,3 +561,131 @@ func (s *DockerRegistrySuite) TestPullFailsWithAlteredManifest(c *check.C) {
 	expectedErrorMsg := fmt.Sprintf("manifest verification failed for digest %s", manifestDigest)
 	assert.Assert(c, is.Contains(out, expectedErrorMsg))
 }
+
+// TestPullFailsWithAlteredManifest tests that a `docker pull` fails when
+// we have modified a manifest blob and its digest cannot be verified.
+// This is the schema1 version of the test.
+func (s *DockerSchema1RegistrySuite) TestPullFailsWithAlteredManifest(c *check.C) {
+	testRequires(c, DaemonIsLinux)
+	manifestDigest, err := setupImage(c)
+	c.Assert(err, checker.IsNil, check.Commentf("error setting up image"))
+
+	// Load the target manifest blob.
+	manifestBlob := s.reg.ReadBlobContents(c, manifestDigest)
+
+	var imgManifest schema1.Manifest
+	err = json.Unmarshal(manifestBlob, &imgManifest)
+	c.Assert(err, checker.IsNil, check.Commentf("unable to decode image manifest from blob"))
+
+	// Change a layer in the manifest.
+	imgManifest.FSLayers[0] = schema1.FSLayer{
+		BlobSum: digest.Digest("sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"),
+	}
+
+	// Move the existing data file aside, so that we can replace it with a
+	// malicious blob of data. NOTE: we defer the returned undo func.
+	undo := s.reg.TempMoveBlobData(c, manifestDigest)
+	defer undo()
+
+	alteredManifestBlob, err := json.MarshalIndent(imgManifest, "", "   ")
+	c.Assert(err, checker.IsNil, check.Commentf("unable to encode altered image manifest to JSON"))
+
+	s.reg.WriteBlobContents(c, manifestDigest, alteredManifestBlob)
+
+	// Now try pulling that image by digest. We should get an error about
+	// digest verification for the manifest digest.
+
+	// Pull from the registry using the <name>@<digest> reference.
+	imageReference := fmt.Sprintf("%s@%s", repoName, manifestDigest)
+	out, exitStatus, _ := dockerCmdWithError("pull", imageReference)
+	c.Assert(exitStatus, checker.Not(check.Equals), 0)
+
+	expectedErrorMsg := fmt.Sprintf("image verification failed for digest %s", manifestDigest)
+	c.Assert(out, checker.Contains, expectedErrorMsg)
+}
+
+// TestPullFailsWithAlteredLayer tests that a `docker pull` fails when
+// we have modified a layer blob and its digest cannot be verified.
+// This is the schema2 version of the test.
+func (s *DockerRegistrySuite) TestPullFailsWithAlteredLayer(c *check.C) {
+	testRequires(c, DaemonIsLinux)
+	manifestDigest, err := setupImage(c)
+	c.Assert(err, checker.IsNil)
+
+	// Load the target manifest blob.
+	manifestBlob := s.reg.ReadBlobContents(c, manifestDigest)
+
+	var imgManifest schema2.Manifest
+	err = json.Unmarshal(manifestBlob, &imgManifest)
+	c.Assert(err, checker.IsNil)
+
+	// Next, get the digest of one of the layers from the manifest.
+	targetLayerDigest := imgManifest.Layers[0].Digest
+
+	// Move the existing data file aside, so that we can replace it with a
+	// malicious blob of data. NOTE: we defer the returned undo func.
+	undo := s.reg.TempMoveBlobData(c, targetLayerDigest)
+	defer undo()
+
+	// Now make a fake data blob in this directory.
+	s.reg.WriteBlobContents(c, targetLayerDigest, []byte("This is not the data you are looking for."))
+
+	// Now try pulling that image by digest. We should get an error about
+	// digest verification for the target layer digest.
+
+	// Remove distribution cache to force a re-pull of the blobs
+	if err := os.RemoveAll(filepath.Join(testEnv.DaemonInfo.DockerRootDir, "image", s.d.StorageDriver(), "distribution")); err != nil {
+		c.Fatalf("error clearing distribution cache: %v", err)
+	}
+
+	// Pull from the registry using the <name>@<digest> reference.
+	imageReference := fmt.Sprintf("%s@%s", repoName, manifestDigest)
+	out, exitStatus, _ := dockerCmdWithError("pull", imageReference)
+	c.Assert(exitStatus, checker.Not(check.Equals), 0, check.Commentf("expected a non-zero exit status"))
+
+	expectedErrorMsg := fmt.Sprintf("filesystem layer verification failed for digest %s", targetLayerDigest)
+	c.Assert(out, checker.Contains, expectedErrorMsg, check.Commentf("expected error message in output: %s", out))
+}
+
+// TestPullFailsWithAlteredLayer tests that a `docker pull` fails when
+// we have modified a layer blob and its digest cannot be verified.
+// This is the schema1 version of the test.
+func (s *DockerSchema1RegistrySuite) TestPullFailsWithAlteredLayer(c *check.C) {
+	testRequires(c, DaemonIsLinux)
+	manifestDigest, err := setupImage(c)
+	c.Assert(err, checker.IsNil)
+
+	// Load the target manifest blob.
+	manifestBlob := s.reg.ReadBlobContents(c, manifestDigest)
+
+	var imgManifest schema1.Manifest
+	err = json.Unmarshal(manifestBlob, &imgManifest)
+	c.Assert(err, checker.IsNil)
+
+	// Next, get the digest of one of the layers from the manifest.
+	targetLayerDigest := imgManifest.FSLayers[0].BlobSum
+
+	// Move the existing data file aside, so that we can replace it with a
+	// malicious blob of data. NOTE: we defer the returned undo func.
+	undo := s.reg.TempMoveBlobData(c, targetLayerDigest)
+	defer undo()
+
+	// Now make a fake data blob in this directory.
+	s.reg.WriteBlobContents(c, targetLayerDigest, []byte("This is not the data you are looking for."))
+
+	// Now try pulling that image by digest. We should get an error about
+	// digest verification for the target layer digest.
+
+	// Remove distribution cache to force a re-pull of the blobs
+	if err := os.RemoveAll(filepath.Join(testEnv.DaemonInfo.DockerRootDir, "image", s.d.StorageDriver(), "distribution")); err != nil {
+		c.Fatalf("error clearing distribution cache: %v", err)
+	}
+
+	// Pull from the registry using the <name>@<digest> reference.
+	imageReference := fmt.Sprintf("%s@%s", repoName, manifestDigest)
+	out, exitStatus, _ := dockerCmdWithError("pull", imageReference)
+	c.Assert(exitStatus, checker.Not(check.Equals), 0, check.Commentf("expected a non-zero exit status"))
+
+	expectedErrorMsg := fmt.Sprintf("filesystem layer verification failed for digest %s", targetLayerDigest)
+	c.Assert(out, checker.Contains, expectedErrorMsg, check.Commentf("expected error message in output: %s", out))
+}

integration-cli/docker_cli_daemon_test.go

@@ -35,6 +35,7 @@ import (
 	"github.com/docker/docker/pkg/mount"
 	"github.com/docker/go-units"
 	"github.com/docker/libnetwork/iptables"
+	"github.com/docker/libtrust"
 	"github.com/go-check/check"
 	"github.com/kr/pty"
 	"golang.org/x/sys/unix"
@@ -550,6 +551,23 @@ func (s *DockerDaemonSuite) TestDaemonAllocatesListeningPort(c *check.C) {
 	}
 }
 
+func (s *DockerDaemonSuite) TestDaemonKeyGeneration(c *check.C) {
+	// TODO: skip or update for Windows daemon
+	os.Remove("/etc/docker/key.json")
+	s.d.Start(c)
+	s.d.Stop(c)
+
+	k, err := libtrust.LoadKeyFile("/etc/docker/key.json")
+	if err != nil {
+		c.Fatalf("Error opening key file")
+	}
+	kid := k.KeyID()
+	// Test Key ID is a valid fingerprint (e.g. QQXN:JY5W:TBXI:MK3X:GX6P:PD5D:F56N:NHCS:LVRZ:JA46:R24J:XEFF)
+	if len(kid) != 59 {
+		c.Fatalf("Bad key ID: %s", kid)
+	}
+}
+
 // GH#11320 - verify that the daemon exits on failure properly
 // Note that this explicitly tests the conflict of {-b,--bridge} and {--bip} options as the means
 // to get a daemon init failure; no other tests for -b/--bip conflict are therefore required
@@ -1174,6 +1192,59 @@ func (s *DockerDaemonSuite) TestDaemonUnixSockCleanedUp(c *check.C) {
 	}
 }
 
+func (s *DockerDaemonSuite) TestDaemonWithWrongkey(c *check.C) {
+	type Config struct {
+		Crv string `json:"crv"`
+		D   string `json:"d"`
+		Kid string `json:"kid"`
+		Kty string `json:"kty"`
+		X   string `json:"x"`
+		Y   string `json:"y"`
+	}
+
+	os.Remove("/etc/docker/key.json")
+	s.d.Start(c)
+	s.d.Stop(c)
+
+	config := &Config{}
+	bytes, err := ioutil.ReadFile("/etc/docker/key.json")
+	if err != nil {
+		c.Fatalf("Error reading key.json file: %s", err)
+	}
+
+	// byte[] to Data-Struct
+	if err := json.Unmarshal(bytes, &config); err != nil {
+		c.Fatalf("Error Unmarshal: %s", err)
+	}
+
+	//replace config.Kid with the fake value
+	config.Kid = "VSAJ:FUYR:X3H2:B2VZ:KZ6U:CJD5:K7BX:ZXHY:UZXT:P4FT:MJWG:HRJ4"
+
+	// NEW Data-Struct to byte[]
+	newBytes, err := json.Marshal(&config)
+	if err != nil {
+		c.Fatalf("Error Marshal: %s", err)
+	}
+
+	// write back
+	if err := ioutil.WriteFile("/etc/docker/key.json", newBytes, 0400); err != nil {
+		c.Fatalf("Error ioutil.WriteFile: %s", err)
+	}
+
+	defer os.Remove("/etc/docker/key.json")
+
+	if err := s.d.StartWithError(); err == nil {
+		c.Fatalf("It should not be successful to start daemon with wrong key: %v", err)
+	}
+
+	content, err := s.d.ReadLogFile()
+	c.Assert(err, checker.IsNil)
+
+	if !strings.Contains(string(content), "Public Key ID does not match") {
+		c.Fatalf("Missing KeyID message from daemon logs: %s", string(content))
+	}
+}
+
 func (s *DockerDaemonSuite) TestDaemonRestartKillWait(c *check.C) {
 	s.d.StartWithBusybox(c)
 
@@ -1715,7 +1786,7 @@ func (s *DockerDaemonSuite) TestDaemonNoSpaceLeftOnDeviceError(c *check.C) {
 	dockerCmd(c, "run", "--rm", "-v", testDir+":/test", "busybox", "sh", "-c", "dd of=/test/testfs.img bs=1M seek=3 count=0")
 	icmd.RunCommand("mkfs.ext4", "-F", filepath.Join(testDir, "testfs.img")).Assert(c, icmd.Success)
 
-	dockerCmd(c, "run", "--privileged", "--rm", "-v", testDir+":/test:shared", "busybox", "sh", "-c", "mkdir -p /test/test-mount/vfs && mount -n /test/testfs.img /test/test-mount/vfs")
+	dockerCmd(c, "run", "--privileged", "--rm", "-v", testDir+":/test:shared", "busybox", "sh", "-c", "mkdir -p /test/test-mount/vfs && mount -n -t ext4 /test/testfs.img /test/test-mount/vfs")
 	defer mount.Unmount(filepath.Join(testDir, "test-mount"))
 
 	s.d.Start(c, "--storage-driver", "vfs", "--data-root", filepath.Join(testDir, "test-mount"))

integration-cli/docker_cli_external_volume_driver_unix_test.go

@@ -558,7 +558,7 @@ func (s *DockerExternalVolumeSuite) TestExternalVolumeDriverCapabilities(c *chec
 }
 
 func (s *DockerExternalVolumeSuite) TestExternalVolumeDriverOutOfBandDelete(c *check.C) {
-	driverName := stringid.GenerateNonCryptoID()
+	driverName := stringid.GenerateRandomID()
 	p := newVolumePlugin(c, driverName)
 	defer p.Close()
 

integration-cli/docker_cli_ps_test.go

@@ -439,15 +439,11 @@ func (s *DockerSuite) TestPsListContainersFilterLabel(c *check.C) {
 func (s *DockerSuite) TestPsListContainersFilterExited(c *check.C) {
 	runSleepingContainer(c, "--name=sleep")
 
-	dockerCmd(c, "run", "--name", "zero1", "busybox", "true")
-	firstZero := getIDByName(c, "zero1")
-
-	dockerCmd(c, "run", "--name", "zero2", "busybox", "true")
-	secondZero := getIDByName(c, "zero2")
+	firstZero, _ := dockerCmd(c, "run", "-d", "busybox", "true")
+	secondZero, _ := dockerCmd(c, "run", "-d", "busybox", "true")
 
 	out, _, err := dockerCmdWithError("run", "--name", "nonzero1", "busybox", "false")
 	c.Assert(err, checker.NotNil, check.Commentf("Should fail.", out, err))
-
 	firstNonZero := getIDByName(c, "nonzero1")
 
 	out, _, err = dockerCmdWithError("run", "--name", "nonzero2", "busybox", "false")
@@ -456,17 +452,16 @@ func (s *DockerSuite) TestPsListContainersFilterExited(c *check.C) {
 
 	// filter containers by exited=0
 	out, _ = dockerCmd(c, "ps", "-a", "-q", "--no-trunc", "--filter=exited=0")
-	ids := strings.Split(strings.TrimSpace(out), "\n")
-	c.Assert(ids, checker.HasLen, 2, check.Commentf("Should be 2 zero exited containers got %d: %s", len(ids), out))
-	c.Assert(ids[0], checker.Equals, secondZero, check.Commentf("First in list should be %q, got %q", secondZero, ids[0]))
-	c.Assert(ids[1], checker.Equals, firstZero, check.Commentf("Second in list should be %q, got %q", firstZero, ids[1]))
+	c.Assert(out, checker.Contains, strings.TrimSpace(firstZero))
+	c.Assert(out, checker.Contains, strings.TrimSpace(secondZero))
+	c.Assert(out, checker.Not(checker.Contains), strings.TrimSpace(firstNonZero))
+	c.Assert(out, checker.Not(checker.Contains), strings.TrimSpace(secondNonZero))
 
 	out, _ = dockerCmd(c, "ps", "-a", "-q", "--no-trunc", "--filter=exited=1")
-	ids = strings.Split(strings.TrimSpace(out), "\n")
-	c.Assert(ids, checker.HasLen, 2, check.Commentf("Should be 2 zero exited containers got %d", len(ids)))
-	c.Assert(ids[0], checker.Equals, secondNonZero, check.Commentf("First in list should be %q, got %q", secondNonZero, ids[0]))
-	c.Assert(ids[1], checker.Equals, firstNonZero, check.Commentf("Second in list should be %q, got %q", firstNonZero, ids[1]))
-
+	c.Assert(out, checker.Contains, strings.TrimSpace(firstNonZero))
+	c.Assert(out, checker.Contains, strings.TrimSpace(secondNonZero))
+	c.Assert(out, checker.Not(checker.Contains), strings.TrimSpace(firstZero))
+	c.Assert(out, checker.Not(checker.Contains), strings.TrimSpace(secondZero))
 }
 
 func (s *DockerSuite) TestPsRightTagName(c *check.C) {

integration-cli/docker_cli_pull_local_test.go

@@ -56,6 +56,10 @@ func (s *DockerRegistrySuite) TestPullImageWithAliases(c *check.C) {
 	testPullImageWithAliases(c)
 }
 
+func (s *DockerSchema1RegistrySuite) TestPullImageWithAliases(c *check.C) {
+	testPullImageWithAliases(c)
+}
+
 // testConcurrentPullWholeRepo pulls the same repo concurrently.
 func testConcurrentPullWholeRepo(c *check.C) {
 	repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL)
@@ -108,6 +112,10 @@ func (s *DockerRegistrySuite) testConcurrentPullWholeRepo(c *check.C) {
 	testConcurrentPullWholeRepo(c)
 }
 
+func (s *DockerSchema1RegistrySuite) testConcurrentPullWholeRepo(c *check.C) {
+	testConcurrentPullWholeRepo(c)
+}
+
 // testConcurrentFailingPull tries a concurrent pull that doesn't succeed.
 func testConcurrentFailingPull(c *check.C) {
 	repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL)
@@ -135,6 +143,10 @@ func (s *DockerRegistrySuite) testConcurrentFailingPull(c *check.C) {
 	testConcurrentFailingPull(c)
 }
 
+func (s *DockerSchema1RegistrySuite) testConcurrentFailingPull(c *check.C) {
+	testConcurrentFailingPull(c)
+}
+
 // testConcurrentPullMultipleTags pulls multiple tags from the same repo
 // concurrently.
 func testConcurrentPullMultipleTags(c *check.C) {
@@ -187,6 +199,10 @@ func (s *DockerRegistrySuite) TestConcurrentPullMultipleTags(c *check.C) {
 	testConcurrentPullMultipleTags(c)
 }
 
+func (s *DockerSchema1RegistrySuite) TestConcurrentPullMultipleTags(c *check.C) {
+	testConcurrentPullMultipleTags(c)
+}
+
 // testPullIDStability verifies that pushing an image and pulling it back
 // preserves the image ID.
 func testPullIDStability(c *check.C) {
@@ -244,6 +260,10 @@ func (s *DockerRegistrySuite) TestPullIDStability(c *check.C) {
 	testPullIDStability(c)
 }
 
+func (s *DockerSchema1RegistrySuite) TestPullIDStability(c *check.C) {
+	testPullIDStability(c)
+}
+
 // #21213
 func testPullNoLayers(c *check.C) {
 	repoName := fmt.Sprintf("%v/dockercli/scratch", privateRegistryURL)
@@ -260,6 +280,10 @@ func (s *DockerRegistrySuite) TestPullNoLayers(c *check.C) {
 	testPullNoLayers(c)
 }
 
+func (s *DockerSchema1RegistrySuite) TestPullNoLayers(c *check.C) {
+	testPullNoLayers(c)
+}
+
 func (s *DockerRegistrySuite) TestPullManifestList(c *check.C) {
 	testRequires(c, NotArm)
 	pushDigest, err := setupImage(c)

integration-cli/docker_cli_push_test.go

@@ -30,6 +30,10 @@ func (s *DockerRegistrySuite) TestPushBusyboxImage(c *check.C) {
 	testPushBusyboxImage(c)
 }
 
+func (s *DockerSchema1RegistrySuite) TestPushBusyboxImage(c *check.C) {
+	testPushBusyboxImage(c)
+}
+
 // pushing an image without a prefix should throw an error
 func (s *DockerSuite) TestPushUnprefixedRepo(c *check.C) {
 	out, _, err := dockerCmdWithError("push", "busybox")
@@ -49,6 +53,10 @@ func (s *DockerRegistrySuite) TestPushUntagged(c *check.C) {
 	testPushUntagged(c)
 }
 
+func (s *DockerSchema1RegistrySuite) TestPushUntagged(c *check.C) {
+	testPushUntagged(c)
+}
+
 func testPushBadTag(c *check.C) {
 	repoName := fmt.Sprintf("%v/dockercli/busybox:latest", privateRegistryURL)
 	expected := "does not exist"
@@ -62,6 +70,10 @@ func (s *DockerRegistrySuite) TestPushBadTag(c *check.C) {
 	testPushBadTag(c)
 }
 
+func (s *DockerSchema1RegistrySuite) TestPushBadTag(c *check.C) {
+	testPushBadTag(c)
+}
+
 func testPushMultipleTags(c *check.C) {
 	repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL)
 	repoTag1 := fmt.Sprintf("%v/dockercli/busybox:t1", privateRegistryURL)
@@ -103,6 +115,10 @@ func (s *DockerRegistrySuite) TestPushMultipleTags(c *check.C) {
 	testPushMultipleTags(c)
 }
 
+func (s *DockerSchema1RegistrySuite) TestPushMultipleTags(c *check.C) {
+	testPushMultipleTags(c)
+}
+
 func testPushEmptyLayer(c *check.C) {
 	repoName := fmt.Sprintf("%v/dockercli/emptylayer", privateRegistryURL)
 	emptyTarball, err := ioutil.TempFile("", "empty_tarball")
@@ -130,6 +146,10 @@ func (s *DockerRegistrySuite) TestPushEmptyLayer(c *check.C) {
 	testPushEmptyLayer(c)
 }
 
+func (s *DockerSchema1RegistrySuite) TestPushEmptyLayer(c *check.C) {
+	testPushEmptyLayer(c)
+}
+
 // testConcurrentPush pushes multiple tags to the same repo
 // concurrently.
 func testConcurrentPush(c *check.C) {
@@ -180,6 +200,10 @@ func (s *DockerRegistrySuite) TestConcurrentPush(c *check.C) {
 	testConcurrentPush(c)
 }
 
+func (s *DockerSchema1RegistrySuite) TestConcurrentPush(c *check.C) {
+	testConcurrentPush(c)
+}
+
 func (s *DockerRegistrySuite) TestCrossRepositoryLayerPush(c *check.C) {
 	sourceRepoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL)
 	// tag the image to upload it to the private registry
@@ -222,6 +246,39 @@ func (s *DockerRegistrySuite) TestCrossRepositoryLayerPush(c *check.C) {
 	assert.Equal(c, out4, "hello world")
 }
 
+func (s *DockerSchema1RegistrySuite) TestCrossRepositoryLayerPushNotSupported(c *check.C) {
+	sourceRepoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL)
+	// tag the image to upload it to the private registry
+	dockerCmd(c, "tag", "busybox", sourceRepoName)
+	// push the image to the registry
+	out1, _, err := dockerCmdWithError("push", sourceRepoName)
+	assert.NilError(c, err, fmt.Sprintf("pushing the image to the private registry has failed: %s", out1))
+	// ensure that none of the layers were mounted from another repository during push
+	assert.Assert(c, !strings.Contains(out1, "Mounted from"))
+
+	digest1 := reference.DigestRegexp.FindString(out1)
+	assert.Assert(c, len(digest1) > 0, "no digest found for pushed manifest")
+
+	destRepoName := fmt.Sprintf("%v/dockercli/crossrepopush", privateRegistryURL)
+	// retag the image to upload the same layers to another repo in the same registry
+	dockerCmd(c, "tag", "busybox", destRepoName)
+	// push the image to the registry
+	out2, _, err := dockerCmdWithError("push", destRepoName)
+	assert.NilError(c, err, fmt.Sprintf("pushing the image to the private registry has failed: %s", out2))
+	// schema1 registry should not support cross-repo layer mounts, so ensure that this does not happen
+	assert.Assert(c, !strings.Contains(out2, "Mounted from"))
+
+	digest2 := reference.DigestRegexp.FindString(out2)
+	assert.Assert(c, len(digest2) > 0, "no digest found for pushed manifest")
+	assert.Assert(c, digest1 != digest2)
+
+	// ensure that we can pull and run the second pushed repository
+	dockerCmd(c, "rmi", destRepoName)
+	dockerCmd(c, "pull", destRepoName)
+	out3, _ := dockerCmd(c, "run", destRepoName, "echo", "-n", "hello world")
+	assert.Assert(c, out3 == "hello world")
+}
+
 func (s *DockerRegistryAuthHtpasswdSuite) TestPushNoCredentialsNoRetry(c *check.C) {
 	repoName := fmt.Sprintf("%s/busybox", privateRegistryURL)
 	dockerCmd(c, "tag", "busybox", repoName)

integration-cli/docker_cli_run_unix_test.go

@@ -1544,6 +1544,10 @@ func (s *DockerDaemonSuite) TestRunWithDaemonDefaultSeccompProfile(c *check.C) {
 		{
 			"name": "chmod",
 			"action": "SCMP_ACT_ERRNO"
+		},
+		{
+			"name": "fchmodat",
+			"action": "SCMP_ACT_ERRNO"
 		}
 	]
 }`

integration-cli/docker_cli_search_test.go

@@ -44,54 +44,6 @@ func (s *DockerSuite) TestSearchStarsOptionWithWrongParameter(c *check.C) {
 	assert.Assert(c, strings.Contains(out, "invalid syntax"), "couldn't find the invalid value warning")
 }
 
-func (s *DockerSuite) TestSearchCmdOptions(c *check.C) {
-	testRequires(c, Network, DaemonIsLinux)
-
-	out, _ := dockerCmd(c, "search", "--help")
-	assert.Assert(c, strings.Contains(out, "Usage:\tdocker search [OPTIONS] TERM"))
-
-	outSearchCmd, _ := dockerCmd(c, "search", "busybox")
-	outSearchCmdNotrunc, _ := dockerCmd(c, "search", "--no-trunc=true", "busybox")
-
-	assert.Assert(c, len(outSearchCmd) <= len(outSearchCmdNotrunc), "The no-trunc option can't take effect.")
-
-	outSearchCmdautomated, _ := dockerCmd(c, "search", "--filter", "is-automated=true", "busybox") //The busybox is a busybox base image, not an AUTOMATED image.
-	outSearchCmdautomatedSlice := strings.Split(outSearchCmdautomated, "\n")
-	for i := range outSearchCmdautomatedSlice {
-		assert.Assert(c, !strings.HasPrefix(outSearchCmdautomatedSlice[i], "busybox "), "The busybox is not an AUTOMATED image: %s", outSearchCmdautomated)
-	}
-
-	outSearchCmdNotOfficial, _ := dockerCmd(c, "search", "--filter", "is-official=false", "busybox") //The busybox is a busybox base image, official image.
-	outSearchCmdNotOfficialSlice := strings.Split(outSearchCmdNotOfficial, "\n")
-	for i := range outSearchCmdNotOfficialSlice {
-		assert.Assert(c, !strings.HasPrefix(outSearchCmdNotOfficialSlice[i], "busybox "), "The busybox is not an OFFICIAL image: %s", outSearchCmdNotOfficial)
-	}
-
-	outSearchCmdOfficial, _ := dockerCmd(c, "search", "--filter", "is-official=true", "busybox") //The busybox is a busybox base image, official image.
-	outSearchCmdOfficialSlice := strings.Split(outSearchCmdOfficial, "\n")
-	assert.Equal(c, len(outSearchCmdOfficialSlice), 3) // 1 header, 1 line, 1 carriage return
-	assert.Assert(c, strings.HasPrefix(outSearchCmdOfficialSlice[1], "busybox "), "The busybox is an OFFICIAL image: %s", outSearchCmdOfficial)
-
-	outSearchCmdStars, _ := dockerCmd(c, "search", "--filter", "stars=2", "busybox")
-	assert.Assert(c, strings.Count(outSearchCmdStars, "[OK]") <= strings.Count(outSearchCmd, "[OK]"), "The quantity of images with stars should be less than that of all images: %s", outSearchCmdStars)
-
-	dockerCmd(c, "search", "--filter", "is-automated=true", "--filter", "stars=2", "--no-trunc=true", "busybox")
-
-	// --automated deprecated since Docker 1.13
-	outSearchCmdautomated1, _ := dockerCmd(c, "search", "--automated=true", "busybox") //The busybox is a busybox base image, not an AUTOMATED image.
-	outSearchCmdautomatedSlice1 := strings.Split(outSearchCmdautomated1, "\n")
-	for i := range outSearchCmdautomatedSlice1 {
-		assert.Assert(c, !strings.HasPrefix(outSearchCmdautomatedSlice1[i], "busybox "), "The busybox is not an AUTOMATED image: %s", outSearchCmdautomated)
-	}
-
-	// -s --stars deprecated since Docker 1.13
-	outSearchCmdStars1, _ := dockerCmd(c, "search", "--stars=2", "busybox")
-	assert.Assert(c, strings.Count(outSearchCmdStars1, "[OK]") <= strings.Count(outSearchCmd, "[OK]"), "The quantity of images with stars should be less than that of all images: %s", outSearchCmdStars1)
-
-	// -s --stars deprecated since Docker 1.13
-	dockerCmd(c, "search", "--stars=2", "--automated=true", "--no-trunc=true", "busybox")
-}
-
 // search for repos which start with "ubuntu-" on the central registry
 func (s *DockerSuite) TestSearchOnCentralRegistryWithDash(c *check.C) {
 	testRequires(c, Network, DaemonIsLinux)

integration/build/build_test.go

@@ -468,6 +468,7 @@ RUN for g in $(seq 0 8); do dd if=/dev/urandom of=rnd bs=1K count=1 seek=$((1024
 }
 
 func TestBuildWithEmptyDockerfile(t *testing.T) {
+	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.40"), "broken in earlier versions")
 	ctx := context.TODO()
 	defer setupTest(t)()
 

integration/config/config_test.go

@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"io/ioutil"
+	"path/filepath"
 	"sort"
 	"testing"
 	"time"
@@ -13,6 +15,7 @@ import (
 	swarmtypes "github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/client"
 	"github.com/docker/docker/integration/internal/swarm"
+	"github.com/docker/docker/internal/test/daemon"
 	"github.com/docker/docker/pkg/stdcopy"
 	"gotest.tools/assert"
 	is "gotest.tools/assert/cmp"
@@ -417,6 +420,26 @@ func TestConfigCreateResolve(t *testing.T) {
 	assert.Assert(t, is.Equal(0, len(entries)))
 }
 
+func TestConfigDaemonLibtrustID(t *testing.T) {
+	skip.If(t, testEnv.DaemonInfo.OSType != "linux")
+	defer setupTest(t)()
+
+	d := daemon.New(t)
+	defer d.Stop(t)
+
+	trustKey := filepath.Join(d.RootDir(), "key.json")
+	err := ioutil.WriteFile(trustKey, []byte(`{"crv":"P-256","d":"dm28PH4Z4EbyUN8L0bPonAciAQa1QJmmyYd876mnypY","kid":"WTJ3:YSIP:CE2E:G6KJ:PSBD:YX2Y:WEYD:M64G:NU2V:XPZV:H2CR:VLUB","kty":"EC","x":"Mh5-JINSjaa_EZdXDttri255Z5fbCEOTQIZjAcScFTk","y":"eUyuAjfxevb07hCCpvi4Zi334Dy4GDWQvEToGEX4exQ"}`), 0644)
+	assert.NilError(t, err)
+
+	config := filepath.Join(d.RootDir(), "daemon.json")
+	err = ioutil.WriteFile(config, []byte(`{"deprecated-key-path": "`+trustKey+`"}`), 0644)
+	assert.NilError(t, err)
+
+	d.Start(t, "--config-file", config)
+	info := d.Info(t)
+	assert.Equal(t, info.ID, "WTJ3:YSIP:CE2E:G6KJ:PSBD:YX2Y:WEYD:M64G:NU2V:XPZV:H2CR:VLUB")
+}
+
 func configNamesFromList(entries []swarmtypes.Config) []string {
 	var values []string
 	for _, entry := range entries {

integration/container/copy_test.go

@@ -1,13 +1,20 @@
 package container // import "github.com/docker/docker/integration/container"
 
 import (
+	"archive/tar"
 	"context"
+	"encoding/json"
 	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
 	"testing"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/client"
 	"github.com/docker/docker/integration/internal/container"
+	"github.com/docker/docker/internal/test/fakecontext"
+	"github.com/docker/docker/pkg/jsonmessage"
 	"gotest.tools/assert"
 	is "gotest.tools/assert/cmp"
 	"gotest.tools/skip"
@@ -64,3 +71,93 @@ func TestCopyToContainerPathIsNotDir(t *testing.T) {
 	err := apiclient.CopyToContainer(ctx, cid, "/etc/passwd/", nil, types.CopyToContainerOptions{})
 	assert.Assert(t, is.ErrorContains(err, "not a directory"))
 }
+
+func TestCopyFromContainer(t *testing.T) {
+	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
+	defer setupTest(t)()
+
+	ctx := context.Background()
+	apiClient := testEnv.APIClient()
+
+	dir, err := ioutil.TempDir("", t.Name())
+	assert.NilError(t, err)
+	defer os.RemoveAll(dir)
+
+	buildCtx := fakecontext.New(t, dir, fakecontext.WithFile("foo", "hello"), fakecontext.WithFile("baz", "world"), fakecontext.WithDockerfile(`
+		FROM busybox
+		COPY foo /foo
+		COPY baz /bar/quux/baz
+		RUN ln -s notexist /bar/notarget && ln -s quux/baz /bar/filesymlink && ln -s quux /bar/dirsymlink && ln -s / /bar/root
+		CMD /fake
+	`))
+	defer buildCtx.Close()
+
+	resp, err := apiClient.ImageBuild(ctx, buildCtx.AsTarReader(t), types.ImageBuildOptions{})
+	assert.NilError(t, err)
+	defer resp.Body.Close()
+
+	var imageID string
+	err = jsonmessage.DisplayJSONMessagesStream(resp.Body, ioutil.Discard, 0, false, func(msg jsonmessage.JSONMessage) {
+		var r types.BuildResult
+		assert.NilError(t, json.Unmarshal(*msg.Aux, &r))
+		imageID = r.ID
+	})
+	assert.NilError(t, err)
+	assert.Assert(t, imageID != "")
+
+	cid := container.Create(t, ctx, apiClient, container.WithImage(imageID))
+
+	for _, x := range []struct {
+		src    string
+		expect map[string]string
+	}{
+		{"/", map[string]string{"/": "", "/foo": "hello", "/bar/quux/baz": "world", "/bar/filesymlink": "", "/bar/dirsymlink": "", "/bar/notarget": ""}},
+		{"/bar/root", map[string]string{"root": ""}},
+		{"/bar/root/", map[string]string{"root/": "", "root/foo": "hello", "root/bar/quux/baz": "world", "root/bar/filesymlink": "", "root/bar/dirsymlink": "", "root/bar/notarget": ""}},
+
+		{"bar/quux", map[string]string{"quux/": "", "quux/baz": "world"}},
+		{"bar/quux/", map[string]string{"quux/": "", "quux/baz": "world"}},
+		{"bar/quux/baz", map[string]string{"baz": "world"}},
+
+		{"bar/filesymlink", map[string]string{"filesymlink": ""}},
+		{"bar/dirsymlink", map[string]string{"dirsymlink": ""}},
+		{"bar/dirsymlink/", map[string]string{"dirsymlink/": "", "dirsymlink/baz": "world"}},
+		{"bar/notarget", map[string]string{"notarget": ""}},
+	} {
+		t.Run(x.src, func(t *testing.T) {
+			rdr, _, err := apiClient.CopyFromContainer(ctx, cid, x.src)
+			assert.NilError(t, err)
+			defer rdr.Close()
+
+			found := make(map[string]bool, len(x.expect))
+			var numFound int
+			tr := tar.NewReader(rdr)
+			for numFound < len(x.expect) {
+				h, err := tr.Next()
+				if err == io.EOF {
+					break
+				}
+				assert.NilError(t, err)
+
+				expected, exists := x.expect[h.Name]
+				if !exists {
+					// this archive will have extra stuff in it since we are copying from root
+					// and docker adds a bunch of stuff
+					continue
+				}
+
+				numFound++
+				found[h.Name] = true
+
+				buf, err := ioutil.ReadAll(tr)
+				if err == nil {
+					assert.Check(t, is.Equal(string(buf), expected))
+				}
+			}
+
+			for f := range x.expect {
+				assert.Check(t, found[f], f+" not found in archive")
+			}
+		})
+	}
+}

integration/container/ipcmode_linux_test.go

@@ -300,12 +300,14 @@ func TestDaemonIpcModeShareableFromConfig(t *testing.T) {
 // by default, even when the daemon default is private.
 func TestIpcModeOlderClient(t *testing.T) {
 	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.40"), "requires a daemon with DefaultIpcMode: private")
+	c := testEnv.APIClient()
+	skip.If(t, versions.LessThan(c.ClientVersion(), "1.40"), "requires client API >= 1.40")
+
 	t.Parallel()
 
 	ctx := context.Background()
 
 	// pre-check: default ipc mode in daemon is private
-	c := testEnv.APIClient()
 	cID := container.Create(t, ctx, c, container.WithAutoRemove)
 
 	inspect, err := c.ContainerInspect(ctx, cID)

integration/container/rename_test.go

@@ -61,7 +61,7 @@ func TestRenameStoppedContainer(t *testing.T) {
 	assert.NilError(t, err)
 	assert.Check(t, is.Equal("/"+oldName, inspect.Name))
 
-	newName := "new_name" + stringid.GenerateNonCryptoID()
+	newName := "new_name" + stringid.GenerateRandomID()
 	err = client.ContainerRename(ctx, oldName, newName)
 	assert.NilError(t, err)
 
@@ -79,7 +79,7 @@ func TestRenameRunningContainerAndReuse(t *testing.T) {
 	cID := container.Run(t, ctx, client, container.WithName(oldName))
 	poll.WaitOn(t, container.IsInState(ctx, client, cID, "running"), poll.WithDelay(100*time.Millisecond))
 
-	newName := "new_name" + stringid.GenerateNonCryptoID()
+	newName := "new_name" + stringid.GenerateRandomID()
 	err := client.ContainerRename(ctx, oldName, newName)
 	assert.NilError(t, err)
 

integration/container/run_linux_test.go

@@ -50,6 +50,10 @@ func TestKernelTCPMemory(t *testing.T) {
 }
 
 func TestNISDomainname(t *testing.T) {
+	// Older versions of the daemon would concatenate hostname and domainname,
+	// so hostname "foobar" and domainname "baz.cyphar.com" would produce
+	// `foobar.baz.cyphar.com` as hostname.
+	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.40"), "skip test from new feature")
 	skip.If(t, testEnv.DaemonInfo.OSType != "linux")
 
 	defer setupTest(t)()

integration/image/list_test.go

@@ -6,12 +6,15 @@ import (
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/versions"
 	"gotest.tools/assert"
 	is "gotest.tools/assert/cmp"
+	"gotest.tools/skip"
 )
 
 // Regression : #38171
 func TestImagesFilterMultiReference(t *testing.T) {
+	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.40"), "broken in earlier versions")
 	defer setupTest(t)()
 	client := testEnv.APIClient()
 	ctx := context.Background()

integration/internal/container/container.go

@@ -2,6 +2,7 @@ package container
 
 import (
 	"context"
+	"runtime"
 	"testing"
 
 	"github.com/docker/docker/api/types"
@@ -24,10 +25,14 @@ type TestContainerConfig struct {
 // nolint: golint
 func Create(t *testing.T, ctx context.Context, client client.APIClient, ops ...func(*TestContainerConfig)) string { // nolint: golint
 	t.Helper()
+	cmd := []string{"top"}
+	if runtime.GOOS == "windows" {
+		cmd = []string{"sleep", "240"}
+	}
 	config := &TestContainerConfig{
 		Config: &container.Config{
 			Image: "busybox",
-			Cmd:   []string{"top"},
+			Cmd:   cmd,
 		},
 		HostConfig:       &container.HostConfig{},
 		NetworkingConfig: &network.NetworkingConfig{},

integration/network/ipvlan/ipvlan_test.go

@@ -4,7 +4,10 @@ package ipvlan
 
 import (
 	"context"
+	"os"
+	"os/exec"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -21,7 +24,7 @@ func TestDockerNetworkIpvlanPersistance(t *testing.T) {
 	// verify the driver automatically provisions the 802.1q link (di-dummy0.70)
 	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
 	skip.If(t, testEnv.IsRemoteDaemon)
-	skip.If(t, !ipvlanKernelSupport(), "Kernel doesn't support ipvlan")
+	skip.If(t, !ipvlanKernelSupport(t), "Kernel doesn't support ipvlan")
 
 	d := daemon.New(t, daemon.WithExperimental)
 	d.StartWithBusybox(t)
@@ -49,7 +52,7 @@ func TestDockerNetworkIpvlanPersistance(t *testing.T) {
 func TestDockerNetworkIpvlan(t *testing.T) {
 	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
 	skip.If(t, testEnv.IsRemoteDaemon)
-	skip.If(t, !ipvlanKernelSupport(), "Kernel doesn't support ipvlan")
+	skip.If(t, !ipvlanKernelSupport(t), "Kernel doesn't support ipvlan")
 
 	for _, tc := range []struct {
 		name string
@@ -425,7 +428,23 @@ func testIpvlanAddressing(client dclient.APIClient) func(*testing.T) {
 	}
 }
 
-// ensure Kernel version is >= v4.2 for ipvlan support
-func ipvlanKernelSupport() bool {
-	return n.CheckKernelMajorVersionGreaterOrEqualThen(4, 2)
+var (
+	once            sync.Once
+	ipvlanSupported bool
+)
+
+// figure out if ipvlan is supported by the kernel
+func ipvlanKernelSupport(t *testing.T) bool {
+	once.Do(func() {
+		// this may have the side effect of enabling the ipvlan module
+		exec.Command("modprobe", "ipvlan").Run()
+		_, err := os.Stat("/sys/module/ipvlan")
+		if err == nil {
+			ipvlanSupported = true
+		} else if !os.IsNotExist(err) {
+			t.Logf("WARNING: ipvlanKernelSupport: stat failed: %v\n", err)
+		}
+	})
+
+	return ipvlanSupported
 }

integration/service/network_test.go

@@ -75,3 +75,43 @@ func TestDockerNetworkConnectAlias(t *testing.T) {
 	assert.Check(t, is.Equal(len(ng2.NetworkSettings.Networks[name].Aliases), 2))
 	assert.Check(t, is.Equal(ng2.NetworkSettings.Networks[name].Aliases[0], "bbb"))
 }
+
+func TestDockerNetworkReConnect(t *testing.T) {
+	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
+	defer setupTest(t)()
+	d := swarm.NewSwarm(t, testEnv)
+	defer d.Stop(t)
+	client := d.NewClientT(t)
+	defer client.Close()
+	ctx := context.Background()
+
+	name := t.Name() + "dummyNet"
+	net.CreateNoError(t, ctx, client, name,
+		net.WithDriver("overlay"),
+		net.WithAttachable(),
+	)
+
+	c1 := container.Create(t, ctx, client, func(c *container.TestContainerConfig) {
+		c.NetworkingConfig = &network.NetworkingConfig{
+			EndpointsConfig: map[string]*network.EndpointSettings{
+				name: {},
+			},
+		}
+	})
+
+	err := client.NetworkConnect(ctx, name, c1, &network.EndpointSettings{})
+	assert.NilError(t, err)
+
+	err = client.ContainerStart(ctx, c1, types.ContainerStartOptions{})
+	assert.NilError(t, err)
+
+	n1, err := client.ContainerInspect(ctx, c1)
+	assert.NilError(t, err)
+
+	err = client.NetworkConnect(ctx, name, c1, &network.EndpointSettings{})
+	assert.ErrorContains(t, err, "is already attached to network")
+
+	n2, err := client.ContainerInspect(ctx, c1)
+	assert.NilError(t, err)
+	assert.Check(t, is.DeepEqual(n1, n2))
+}

integration/service/update_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/docker/docker/api/types"
 	swarmtypes "github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/client"
+	"github.com/docker/docker/integration/internal/network"
 	"github.com/docker/docker/integration/internal/swarm"
 	"gotest.tools/assert"
 	is "gotest.tools/assert/cmp"
@@ -32,7 +33,7 @@ func TestServiceUpdateLabel(t *testing.T) {
 	service.Spec.Labels["foo"] = "bar"
 	_, err := cli.ServiceUpdate(ctx, serviceID, service.Version, service.Spec, types.ServiceUpdateOptions{})
 	assert.NilError(t, err)
-	poll.WaitOn(t, serviceIsUpdated(cli, serviceID), swarm.ServicePoll)
+	poll.WaitOn(t, serviceSpecIsUpdated(cli, serviceID, service.Version.Index), swarm.ServicePoll)
 	service = getService(t, cli, serviceID)
 	assert.Check(t, is.DeepEqual(service.Spec.Labels, map[string]string{"foo": "bar"}))
 
@@ -40,21 +41,21 @@ func TestServiceUpdateLabel(t *testing.T) {
 	service.Spec.Labels["foo2"] = "bar"
 	_, err = cli.ServiceUpdate(ctx, serviceID, service.Version, service.Spec, types.ServiceUpdateOptions{})
 	assert.NilError(t, err)
-	poll.WaitOn(t, serviceIsUpdated(cli, serviceID), swarm.ServicePoll)
+	poll.WaitOn(t, serviceSpecIsUpdated(cli, serviceID, service.Version.Index), swarm.ServicePoll)
 	service = getService(t, cli, serviceID)
 	assert.Check(t, is.DeepEqual(service.Spec.Labels, map[string]string{"foo": "bar", "foo2": "bar"}))
 
 	delete(service.Spec.Labels, "foo2")
 	_, err = cli.ServiceUpdate(ctx, serviceID, service.Version, service.Spec, types.ServiceUpdateOptions{})
 	assert.NilError(t, err)
-	poll.WaitOn(t, serviceIsUpdated(cli, serviceID), swarm.ServicePoll)
+	poll.WaitOn(t, serviceSpecIsUpdated(cli, serviceID, service.Version.Index), swarm.ServicePoll)
 	service = getService(t, cli, serviceID)
 	assert.Check(t, is.DeepEqual(service.Spec.Labels, map[string]string{"foo": "bar"}))
 
 	delete(service.Spec.Labels, "foo")
 	_, err = cli.ServiceUpdate(ctx, serviceID, service.Version, service.Spec, types.ServiceUpdateOptions{})
 	assert.NilError(t, err)
-	poll.WaitOn(t, serviceIsUpdated(cli, serviceID), swarm.ServicePoll)
+	poll.WaitOn(t, serviceSpecIsUpdated(cli, serviceID, service.Version.Index), swarm.ServicePoll)
 	service = getService(t, cli, serviceID)
 	assert.Check(t, is.DeepEqual(service.Spec.Labels, map[string]string{}))
 
@@ -62,7 +63,7 @@ func TestServiceUpdateLabel(t *testing.T) {
 	service.Spec.Labels["foo"] = "bar"
 	_, err = cli.ServiceUpdate(ctx, serviceID, service.Version, service.Spec, types.ServiceUpdateOptions{})
 	assert.NilError(t, err)
-	poll.WaitOn(t, serviceIsUpdated(cli, serviceID), swarm.ServicePoll)
+	poll.WaitOn(t, serviceSpecIsUpdated(cli, serviceID, service.Version.Index), swarm.ServicePoll)
 	service = getService(t, cli, serviceID)
 	assert.Check(t, is.DeepEqual(service.Spec.Labels, map[string]string{"foo": "bar"}))
 
@@ -194,6 +195,59 @@ func TestServiceUpdateConfigs(t *testing.T) {
 	assert.NilError(t, err)
 }
 
+func TestServiceUpdateNetwork(t *testing.T) {
+	skip.If(t, testEnv.DaemonInfo.OSType != "linux")
+	defer setupTest(t)()
+	d := swarm.NewSwarm(t, testEnv)
+	defer d.Stop(t)
+	cli := d.NewClientT(t)
+	defer cli.Close()
+
+	ctx := context.Background()
+
+	// Create a overlay network
+	testNet := "testNet" + t.Name()
+	overlayID := network.CreateNoError(t, ctx, cli, testNet,
+		network.WithDriver("overlay"))
+
+	var instances uint64 = 1
+	// Create service with the overlay network
+	serviceName := "TestServiceUpdateNetworkRM_" + t.Name()
+	serviceID := swarm.CreateService(t, d,
+		swarm.ServiceWithReplicas(instances),
+		swarm.ServiceWithName(serviceName),
+		swarm.ServiceWithNetwork(testNet))
+
+	poll.WaitOn(t, swarm.RunningTasksCount(cli, serviceID, instances), swarm.ServicePoll)
+	service := getService(t, cli, serviceID)
+	netInfo, err := cli.NetworkInspect(ctx, testNet, types.NetworkInspectOptions{
+		Verbose: true,
+		Scope:   "swarm",
+	})
+	assert.NilError(t, err)
+	assert.Assert(t, len(netInfo.Containers) == 2, "Expected 2 endpoints, one for container and one for LB Sandbox")
+
+	//Remove network from service
+	service.Spec.TaskTemplate.Networks = []swarmtypes.NetworkAttachmentConfig{}
+	_, err = cli.ServiceUpdate(ctx, serviceID, service.Version, service.Spec, types.ServiceUpdateOptions{})
+	assert.NilError(t, err)
+	poll.WaitOn(t, serviceIsUpdated(cli, serviceID), swarm.ServicePoll)
+
+	netInfo, err = cli.NetworkInspect(ctx, testNet, types.NetworkInspectOptions{
+		Verbose: true,
+		Scope:   "swarm",
+	})
+
+	assert.NilError(t, err)
+	assert.Assert(t, len(netInfo.Containers) == 0, "Load balancing endpoint still exists in network")
+
+	err = cli.NetworkRemove(ctx, overlayID)
+	assert.NilError(t, err)
+
+	err = cli.ServiceRemove(ctx, serviceID)
+	assert.NilError(t, err)
+}
+
 func getService(t *testing.T, cli client.ServiceAPIClient, serviceID string) swarmtypes.Service {
 	t.Helper()
 	service, _, err := cli.ServiceInspectWithRaw(context.Background(), serviceID, types.ServiceInspectOptions{})
@@ -217,3 +271,17 @@ func serviceIsUpdated(client client.ServiceAPIClient, serviceID string) func(log
 		}
 	}
 }
+
+func serviceSpecIsUpdated(client client.ServiceAPIClient, serviceID string, serviceOldVersion uint64) func(log poll.LogT) poll.Result {
+	return func(log poll.LogT) poll.Result {
+		service, _, err := client.ServiceInspectWithRaw(context.Background(), serviceID, types.ServiceInspectOptions{})
+		switch {
+		case err != nil:
+			return poll.Error(err)
+		case service.Version.Index > serviceOldVersion:
+			return poll.Success()
+		default:
+			return poll.Continue("waiting for service %s to be updated", serviceID)
+		}
+	}
+}

integration/system/ping_test.go

@@ -12,6 +12,7 @@ import (
 )
 
 func TestPingCacheHeaders(t *testing.T) {
+	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.40"), "skip test from new feature")
 	defer setupTest(t)()
 
 	res, _, err := request.Get("/_ping")

integration/system/uuid_test.go

@@ -1,20 +0,0 @@
-package system
-
-import (
-	"context"
-	"testing"
-
-	"github.com/google/uuid"
-	"gotest.tools/assert"
-)
-
-func TestUUIDGeneration(t *testing.T) {
-	defer setupTest(t)()
-
-	c := testEnv.APIClient()
-	info, err := c.Info(context.Background())
-	assert.NilError(t, err)
-
-	_, err = uuid.Parse(info.ID)
-	assert.NilError(t, err)
-}

integration/volume/volume_test.go

@@ -25,6 +25,10 @@ func TestVolumesCreateAndList(t *testing.T) {
 	ctx := context.Background()
 
 	name := t.Name()
+	// Windows file system is case insensitive
+	if testEnv.OSType == "windows" {
+		name = strings.ToLower(name)
+	}
 	vol, err := client.VolumeCreate(ctx, volumetypes.VolumeCreateBody{
 		Name: name,
 	})
@@ -40,12 +44,20 @@ func TestVolumesCreateAndList(t *testing.T) {
 	}
 	assert.Check(t, is.DeepEqual(vol, expected, cmpopts.EquateEmpty()))
 
-	volumes, err := client.VolumeList(ctx, filters.Args{})
+	volList, err := client.VolumeList(ctx, filters.Args{})
 	assert.NilError(t, err)
+	assert.Assert(t, len(volList.Volumes) > 0)
 
-	assert.Check(t, is.Equal(len(volumes.Volumes), 1))
-	assert.Check(t, volumes.Volumes[0] != nil)
-	assert.Check(t, is.DeepEqual(*volumes.Volumes[0], expected, cmpopts.EquateEmpty()))
+	volumes := volList.Volumes[:0]
+	for _, v := range volList.Volumes {
+		if v.Name == vol.Name {
+			volumes = append(volumes, v)
+		}
+	}
+
+	assert.Check(t, is.Equal(len(volumes), 1))
+	assert.Check(t, volumes[0] != nil)
+	assert.Check(t, is.DeepEqual(*volumes[0], expected, cmpopts.EquateEmpty()))
 }
 
 func TestVolumesRemove(t *testing.T) {

internal/procfs/procfs_linux.go

@@ -1,105 +0,0 @@
-package procfs
-
-/*
-Copyright 2015 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"regexp"
-	"strconv"
-	"strings"
-	"unicode"
-
-	"github.com/sirupsen/logrus"
-)
-
-// PidOf finds process(es) with a specified name (regexp match)
-// and return their pid(s)
-func PidOf(name string) ([]int, error) {
-	if len(name) == 0 {
-		return []int{}, fmt.Errorf("name should not be empty")
-	}
-	re, err := regexp.Compile("(^|/)" + name + "$")
-	if err != nil {
-		return []int{}, err
-	}
-	return getPids(re), nil
-}
-
-func getPids(re *regexp.Regexp) []int {
-	pids := []int{}
-
-	dirFD, err := os.Open("/proc")
-	if err != nil {
-		return nil
-	}
-	defer dirFD.Close()
-
-	for {
-		// Read a small number at a time in case there are many entries, we don't want to
-		// allocate a lot here.
-		ls, err := dirFD.Readdir(10)
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return nil
-		}
-
-		for _, entry := range ls {
-			if !entry.IsDir() {
-				continue
-			}
-
-			// If the directory is not a number (i.e. not a PID), skip it
-			pid, err := strconv.Atoi(entry.Name())
-			if err != nil {
-				continue
-			}
-
-			cmdline, err := ioutil.ReadFile(filepath.Join("/proc", entry.Name(), "cmdline"))
-			if err != nil {
-				logrus.Infof("Error reading file %s: %+v", filepath.Join("/proc", entry.Name(), "cmdline"), err)
-				continue
-			}
-
-			// The bytes we read have '\0' as a separator for the command line
-			parts := bytes.SplitN(cmdline, []byte{0}, 2)
-			if len(parts) == 0 {
-				continue
-			}
-			// Split the command line itself we are interested in just the first part
-			exe := strings.FieldsFunc(string(parts[0]), func(c rune) bool {
-				return unicode.IsSpace(c) || c == ':'
-			})
-			if len(exe) == 0 {
-				continue
-			}
-			// Check if the name of the executable is what we are looking for
-			if re.MatchString(exe[0]) {
-				// Grab the PID from the directory path
-				pids = append(pids, pid)
-			}
-		}
-	}
-
-	return pids
-}

internal/procfs/procfs_linux_test.go

@@ -1,36 +0,0 @@
-package procfs
-
-import (
-	"os"
-	"path/filepath"
-	"regexp"
-	"runtime"
-	"testing"
-
-	"gotest.tools/assert"
-)
-
-func TestPidOf(t *testing.T) {
-	pids, err := PidOf(filepath.Base(os.Args[0]))
-	assert.NilError(t, err)
-	assert.Check(t, len(pids) == 1)
-	assert.DeepEqual(t, pids[0], os.Getpid())
-}
-
-func BenchmarkGetPids(b *testing.B) {
-	if runtime.GOOS == "darwin" || runtime.GOOS == "windows" {
-		b.Skipf("not supported on GOOS=%s", runtime.GOOS)
-	}
-
-	re, err := regexp.Compile("(^|/)" + filepath.Base(os.Args[0]) + "$")
-	assert.Check(b, err == nil)
-
-	for i := 0; i < b.N; i++ {
-		pids := getPids(re)
-
-		b.StopTimer()
-		assert.Check(b, len(pids) > 0)
-		assert.Check(b, pids[0] == os.Getpid())
-		b.StartTimer()
-	}
-}