Merge pull request #74 from thaJeztah/18.09_backport_no_more_version_mismatch

[18.09 backport] remove version-checks for containerd and runc

Dockerfile

@@ -24,10 +24,10 @@
 # the case. Therefore, you don't have to disable it anymore.
 #
 
-FROM golang:1.10.3 AS base
+FROM golang:1.10.4 AS base
 # FIXME(vdemeester) this is kept for other script depending on it to not fail right away
 # Remove this once the other scripts uses something else to detect the version
-ENV GO_VERSION 1.10.3
+ENV GO_VERSION 1.10.4
 # allow replacing httpredir or deb mirror
 ARG APT_MIRROR=deb.debian.org
 RUN sed -ri "s/(httpredir|deb).debian.org/$APT_MIRROR/g" /etc/apt/sources.list

Dockerfile.e2e

@@ -1,5 +1,5 @@
 ## Step 1: Build tests
-FROM golang:1.10.3-alpine3.7 as builder
+FROM golang:1.10.4-alpine3.7 as builder
 
 RUN apk add --update \
     bash \

Dockerfile.simple

@@ -42,7 +42,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 #            will need updating, to avoid errors. Ping #docker-maintainers on IRC
 #            with a heads-up.
 # IMPORTANT: When updating this please note that stdlib archive/tar pkg is vendored
-ENV GO_VERSION 1.10.3
+ENV GO_VERSION 1.10.4
 RUN curl -fsSL "https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz" \
 	| tar -xzC /usr/local
 ENV PATH /go/bin:/usr/local/go/bin:$PATH

Dockerfile.windows

@@ -161,7 +161,7 @@ SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPref
 # Environment variable notes:
 #  - GO_VERSION must be consistent with 'Dockerfile' used by Linux.
 #  - FROM_DOCKERFILE is used for detection of building within a container.
-ENV GO_VERSION=1.10.3 `
+ENV GO_VERSION=1.10.4 `
     GIT_VERSION=2.11.1 `
     GOPATH=C:\go `
     FROM_DOCKERFILE=1

api/server/backend/build/backend.go

@@ -88,7 +88,7 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
 }
 
 // PruneCache removes all cached build sources
-func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport, error) {
+func (b *Backend) PruneCache(ctx context.Context, opts types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error) {
 	eg, ctx := errgroup.WithContext(ctx)
 
 	var fsCacheSize uint64
@@ -102,9 +102,10 @@ func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport,
 	})
 
 	var buildCacheSize int64
+	var cacheIDs []string
 	eg.Go(func() error {
 		var err error
-		buildCacheSize, err = b.buildkit.Prune(ctx)
+		buildCacheSize, cacheIDs, err = b.buildkit.Prune(ctx, opts)
 		if err != nil {
 			return errors.Wrap(err, "failed to prune build cache")
 		}
@@ -115,7 +116,7 @@ func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport,
 		return nil, err
 	}
 
-	return &types.BuildCachePruneReport{SpaceReclaimed: fsCacheSize + uint64(buildCacheSize)}, nil
+	return &types.BuildCachePruneReport{SpaceReclaimed: fsCacheSize + uint64(buildCacheSize), CachesDeleted: cacheIDs}, nil
 }
 
 // Cancel cancels the build by ID

api/server/router/build/backend.go

@@ -14,7 +14,7 @@ type Backend interface {
 	Build(context.Context, backend.BuildConfig) (string, error)
 
 	// Prune build cache
-	PruneCache(context.Context) (*types.BuildCachePruneReport, error)
+	PruneCache(context.Context, types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error)
 
 	Cancel(context.Context, string) error
 }

api/server/router/build/build.go

@@ -7,15 +7,19 @@ import (
 
 // buildRouter is a router to talk with the build controller
 type buildRouter struct {
-	backend        Backend
-	daemon         experimentalProvider
-	routes         []router.Route
-	builderVersion types.BuilderVersion
+	backend  Backend
+	daemon   experimentalProvider
+	routes   []router.Route
+	features *map[string]bool
 }
 
 // NewRouter initializes a new build router
-func NewRouter(b Backend, d experimentalProvider, bv types.BuilderVersion) router.Router {
-	r := &buildRouter{backend: b, daemon: d, builderVersion: bv}
+func NewRouter(b Backend, d experimentalProvider, features *map[string]bool) router.Router {
+	r := &buildRouter{
+		backend:  b,
+		daemon:   d,
+		features: features,
+	}
 	r.initRoutes()
 	return r
 }
@@ -32,3 +36,18 @@ func (r *buildRouter) initRoutes() {
 		router.NewPostRoute("/build/cancel", r.postCancel),
 	}
 }
+
+// BuilderVersion derives the default docker builder version from the config
+// Note: it is valid to have BuilderVersion unset which means it is up to the
+// client to choose which builder to use.
+func BuilderVersion(features map[string]bool) types.BuilderVersion {
+	var bv types.BuilderVersion
+	if v, ok := features["buildkit"]; ok {
+		if v {
+			bv = types.BuilderBuildKit
+		} else {
+			bv = types.BuilderV1
+		}
+	}
+	return bv
+}

api/server/router/build/build_routes.go

@@ -18,6 +18,7 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/versions"
 	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/pkg/ioutils"
@@ -161,7 +162,29 @@ func parseVersion(s string) (types.BuilderVersion, error) {
 }
 
 func (br *buildRouter) postPrune(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
-	report, err := br.backend.PruneCache(ctx)
+	if err := httputils.ParseForm(r); err != nil {
+		return err
+	}
+	filters, err := filters.FromJSON(r.Form.Get("filters"))
+	if err != nil {
+		return errors.Wrap(err, "could not parse filters")
+	}
+	ksfv := r.FormValue("keep-storage")
+	if ksfv == "" {
+		ksfv = "0"
+	}
+	ks, err := strconv.Atoi(ksfv)
+	if err != nil {
+		return errors.Wrapf(err, "keep-storage is in bytes and expects an integer, got %v", ksfv)
+	}
+
+	opts := types.BuildCachePruneOptions{
+		All:         httputils.BoolValue(r, "all"),
+		Filters:     filters,
+		KeepStorage: int64(ks),
+	}
+
+	report, err := br.backend.PruneCache(ctx, opts)
 	if err != nil {
 		return err
 	}
@@ -230,11 +253,6 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *
 		return errdefs.InvalidParameter(errors.New("squash is only supported with experimental mode"))
 	}
 
-	// check if the builder feature has been enabled from daemon as well.
-	if buildOptions.Version == types.BuilderBuildKit && br.builderVersion != "" && br.builderVersion != types.BuilderBuildKit {
-		return errdefs.InvalidParameter(errors.New("buildkit is not enabled on daemon"))
-	}
-
 	out := io.Writer(output)
 	if buildOptions.SuppressOutput {
 		out = notVerboseBuffer

api/server/router/system/system.go

@@ -2,30 +2,29 @@ package system // import "github.com/docker/docker/api/server/router/system"
 
 import (
 	"github.com/docker/docker/api/server/router"
-	"github.com/docker/docker/api/types"
-	buildkit "github.com/docker/docker/builder/builder-next"
+	"github.com/docker/docker/builder/builder-next"
 	"github.com/docker/docker/builder/fscache"
 )
 
 // systemRouter provides information about the Docker system overall.
 // It gathers information about host, daemon and container events.
 type systemRouter struct {
-	backend        Backend
-	cluster        ClusterBackend
-	routes         []router.Route
-	fscache        *fscache.FSCache // legacy
-	builder        *buildkit.Builder
-	builderVersion types.BuilderVersion
+	backend  Backend
+	cluster  ClusterBackend
+	routes   []router.Route
+	fscache  *fscache.FSCache // legacy
+	builder  *buildkit.Builder
+	features *map[string]bool
 }
 
 // NewRouter initializes a new system router
-func NewRouter(b Backend, c ClusterBackend, fscache *fscache.FSCache, builder *buildkit.Builder, bv types.BuilderVersion) router.Router {
+func NewRouter(b Backend, c ClusterBackend, fscache *fscache.FSCache, builder *buildkit.Builder, features *map[string]bool) router.Router {
 	r := &systemRouter{
-		backend:        b,
-		cluster:        c,
-		fscache:        fscache,
-		builder:        builder,
-		builderVersion: bv,
+		backend:  b,
+		cluster:  c,
+		fscache:  fscache,
+		builder:  builder,
+		features: features,
 	}
 
 	r.routes = []router.Route{

api/server/router/system/system_routes.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/server/httputils"
+	"github.com/docker/docker/api/server/router/build"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/events"
 	"github.com/docker/docker/api/types/filters"
@@ -26,7 +27,8 @@ func optionsHandler(ctx context.Context, w http.ResponseWriter, r *http.Request,
 }
 
 func (s *systemRouter) pingHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
-	if bv := s.builderVersion; bv != "" {
+	builderVersion := build.BuilderVersion(*s.features)
+	if bv := builderVersion; bv != "" {
 		w.Header().Set("Builder-Version", string(bv))
 	}
 	_, err := w.Write([]byte{'O', 'K'})

api/swagger.yaml

@@ -1513,6 +1513,31 @@ definitions:
       aux:
         $ref: "#/definitions/ImageID"
 
+  BuildCache:
+    type: "object"
+    properties:
+      ID:
+        type: "string"
+      Parent:
+        type: "string"
+      Type:
+        type: "string"
+      Description:
+        type: "string"
+      InUse:
+        type: "boolean"
+      Shared:
+        type: "boolean"
+      Size:
+        type: "integer"
+      CreatedAt:
+        type: "integer"
+      LastUsedAt:
+        type: "integer"
+        x-nullable: true
+      UsageCount:
+        type: "integer"
+
   ImageID:
     type: "object"
     description: "Image ID or Digest"
@@ -3722,18 +3747,22 @@ definitions:
         description: |
           HTTP-proxy configured for the daemon. This value is obtained from the
           [`HTTP_PROXY`](https://www.gnu.org/software/wget/manual/html_node/Proxies.html) environment variable.
+          Credentials ([user info component](https://tools.ietf.org/html/rfc3986#section-3.2.1)) in the proxy URL
+          are masked in the API response.
 
           Containers do not automatically inherit this configuration.
         type: "string"
-        example: "http://user:pass@proxy.corp.example.com:8080"
+        example: "http://xxxxx:xxxxx@proxy.corp.example.com:8080"
       HttpsProxy:
         description: |
           HTTPS-proxy configured for the daemon. This value is obtained from the
           [`HTTPS_PROXY`](https://www.gnu.org/software/wget/manual/html_node/Proxies.html) environment variable.
+          Credentials ([user info component](https://tools.ietf.org/html/rfc3986#section-3.2.1)) in the proxy URL
+          are masked in the API response.
 
           Containers do not automatically inherit this configuration.
         type: "string"
-        example: "https://user:pass@proxy.corp.example.com:4443"
+        example: "https://xxxxx:xxxxx@proxy.corp.example.com:4443"
       NoProxy:
         description: |
           Comma-separated list of domain extensions for which no proxy should be
@@ -3823,10 +3852,10 @@ definitions:
           $ref: "#/definitions/Runtime"
         default:
           runc:
-            path: "docker-runc"
+            path: "runc"
         example:
           runc:
-            path: "docker-runc"
+            path: "runc"
           runc-master:
             path: "/go/bin/runc"
           custom:
@@ -6358,6 +6387,29 @@ paths:
       produces:
         - "application/json"
       operationId: "BuildPrune"
+      parameters:
+        - name: "keep-storage"
+          in: "query"
+          description: "Amount of disk space in bytes to keep for cache"
+          type: "integer"
+          format: "int64"
+        - name: "all"
+          in: "query"
+          type: "boolean"
+          description: "Remove all types of build cache"
+        - name: "filters"
+          in: "query"
+          type: "string"
+          description: |
+            A JSON encoded value of the filters (a `map[string][]string`) to process on the list of build cache objects. Available filters:
+            - `unused-for=<duration>`: duration relative to daemon's time, during which build cache was not used, in Go's duration format (e.g., '24h')
+            - `id=<id>`
+            - `parent=<id>`
+            - `type=<string>`
+            - `description=<string>`
+            - `inuse`
+            - `shared`
+            - `private`
       responses:
         200:
           description: "No error"
@@ -6365,6 +6417,11 @@ paths:
             type: "object"
             title: "BuildPruneResponse"
             properties:
+              CachesDeleted:
+                type: "array"
+                items:
+                  description: "ID of build cache object"
+                  type: "string"
               SpaceReclaimed:
                 description: "Disk space reclaimed in bytes"
                 type: "integer"
@@ -7199,6 +7256,10 @@ paths:
                 type: "array"
                 items:
                   $ref: "#/definitions/Volume"
+              BuildCache:
+                type: "array"
+                items:
+                  $ref: "#/definitions/BuildCache"
             example:
               LayersSize: 1092588
               Images:
@@ -9284,7 +9345,10 @@ paths:
 
         - name: "version"
           in: "query"
-          description: "The version number of the service object being updated. This is required to avoid conflicting writes."
+          description: "The version number of the service object being updated.
+          This is required to avoid conflicting writes.
+          This version number should be the value as currently set on the service *before* the update.
+          You can find the current version by calling `GET /services/{id}`"
           required: true
           type: "integer"
         - name: "registryAuthFrom"

api/types/types.go

@@ -543,6 +543,7 @@ type ImagesPruneReport struct {
 // BuildCachePruneReport contains the response for Engine API:
 // POST "/build/prune"
 type BuildCachePruneReport struct {
+	CachesDeleted  []string
 	SpaceReclaimed uint64
 }
 
@@ -592,14 +593,21 @@ type BuildResult struct {
 
 // BuildCache contains information about a build cache record
 type BuildCache struct {
-	ID      string
-	Mutable bool
-	InUse   bool
-	Size    int64
-
+	ID          string
+	Parent      string
+	Type        string
+	Description string
+	InUse       bool
+	Shared      bool
+	Size        int64
 	CreatedAt   time.Time
 	LastUsedAt  *time.Time
 	UsageCount  int
-	Parent      string
-	Description string
+}
+
+// BuildCachePruneOptions hold parameters to prune the build cache
+type BuildCachePruneOptions struct {
+	All         bool
+	KeepStorage int64
+	Filters     filters.Args
 }

builder/builder-next/adapters/containerimage/pull.go

@@ -34,6 +34,7 @@ import (
 	"github.com/moby/buildkit/util/flightcontrol"
 	"github.com/moby/buildkit/util/imageutil"
 	"github.com/moby/buildkit/util/progress"
+	"github.com/moby/buildkit/util/resolver"
 	"github.com/moby/buildkit/util/tracing"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/identity"
@@ -51,6 +52,7 @@ type SourceOpt struct {
 	DownloadManager distribution.RootFSDownloadManager
 	MetadataStore   metadata.V2MetadataService
 	ImageStore      image.Store
+	ResolverOpt     resolver.ResolveOptionsFunc
 }
 
 type imageSource struct {
@@ -71,11 +73,16 @@ func (is *imageSource) ID() string {
 	return source.DockerImageScheme
 }
 
-func (is *imageSource) getResolver(ctx context.Context) remotes.Resolver {
-	return docker.NewResolver(docker.ResolverOptions{
-		Client:      tracing.DefaultClient,
-		Credentials: is.getCredentialsFromSession(ctx),
-	})
+func (is *imageSource) getResolver(ctx context.Context, rfn resolver.ResolveOptionsFunc, ref string) remotes.Resolver {
+	opt := docker.ResolverOptions{
+		Client: tracing.DefaultClient,
+	}
+	if rfn != nil {
+		opt = rfn(ref)
+	}
+	opt.Credentials = is.getCredentialsFromSession(ctx)
+	r := docker.NewResolver(opt)
+	return r
 }
 
 func (is *imageSource) getCredentialsFromSession(ctx context.Context) func(string) (string, string, error) {
@@ -118,7 +125,7 @@ func (is *imageSource) resolveRemote(ctx context.Context, ref string, platform *
 		dt   []byte
 	}
 	res, err := is.g.Do(ctx, ref, func(ctx context.Context) (interface{}, error) {
-		dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx), is.ContentStore, platform)
+		dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx, is.ResolverOpt, ref), is.ContentStore, platform)
 		if err != nil {
 			return nil, err
 		}
@@ -181,7 +188,7 @@ func (is *imageSource) Resolve(ctx context.Context, id source.Identifier) (sourc
 	p := &puller{
 		src:      imageIdentifier,
 		is:       is,
-		resolver: is.getResolver(ctx),
+		resolver: is.getResolver(ctx, is.ResolverOpt, imageIdentifier.Reference.String()),
 		platform: platform,
 	}
 	return p, nil
@@ -516,6 +523,15 @@ func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
 		return nil, err
 	}
 
+	// TODO: handle windows layers for cross platform builds
+
+	if p.src.RecordType != "" && cache.GetRecordType(ref) == "" {
+		if err := cache.SetRecordType(ref, p.src.RecordType); err != nil {
+			ref.Release(context.TODO())
+			return nil, err
+		}
+	}
+
 	return ref, nil
 }
 

builder/builder-next/adapters/snapshot/layer.go

@@ -5,10 +5,10 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/boltdb/bolt"
 	"github.com/docker/docker/layer"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
 	"golang.org/x/sync/errgroup"
 )
 

builder/builder-next/adapters/snapshot/snapshot.go

@@ -7,7 +7,6 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/boltdb/bolt"
 	"github.com/containerd/containerd/mount"
 	"github.com/containerd/containerd/snapshots"
 	"github.com/docker/docker/daemon/graphdriver"
@@ -16,6 +15,7 @@ import (
 	"github.com/moby/buildkit/snapshot"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
 )
 
 var keyParent = []byte("parent")
@@ -110,6 +110,10 @@ func (s *snapshotter) chainID(key string) (layer.ChainID, bool) {
 	return "", false
 }
 
+func (s *snapshotter) GetLayer(key string) (layer.Layer, error) {
+	return s.getLayer(key, true)
+}
+
 func (s *snapshotter) getLayer(key string, withCommitted bool) (layer.Layer, error) {
 	s.mu.Lock()
 	l, ok := s.refs[key]
@@ -422,10 +426,11 @@ func (s *snapshotter) Close() error {
 }
 
 type mountable struct {
-	mu      sync.Mutex
-	mounts  []mount.Mount
-	acquire func() ([]mount.Mount, error)
-	release func() error
+	mu       sync.Mutex
+	mounts   []mount.Mount
+	acquire  func() ([]mount.Mount, error)
+	release  func() error
+	refCount int
 }
 
 func (m *mountable) Mount() ([]mount.Mount, error) {
@@ -433,6 +438,7 @@ func (m *mountable) Mount() ([]mount.Mount, error) {
 	defer m.mu.Unlock()
 
 	if m.mounts != nil {
+		m.refCount++
 		return m.mounts, nil
 	}
 
@@ -441,6 +447,7 @@ func (m *mountable) Mount() ([]mount.Mount, error) {
 		return nil, err
 	}
 	m.mounts = mounts
+	m.refCount = 1
 
 	return m.mounts, nil
 }
@@ -448,6 +455,13 @@ func (m *mountable) Mount() ([]mount.Mount, error) {
 func (m *mountable) Release() error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
+
+	if m.refCount > 1 {
+		m.refCount--
+		return nil
+	}
+
+	m.refCount = 0
 	if m.release == nil {
 		return nil
 	}

builder/builder-next/builder.go

@@ -13,32 +13,53 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/builder"
+	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/daemon/images"
 	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/libnetwork"
 	controlapi "github.com/moby/buildkit/api/services/control"
+	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/control"
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/solver/llbsolver"
 	"github.com/moby/buildkit/util/entitlements"
+	"github.com/moby/buildkit/util/resolver"
 	"github.com/moby/buildkit/util/tracing"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
 	grpcmetadata "google.golang.org/grpc/metadata"
 )
 
+var errMultipleFilterValues = errors.New("filters expect only one value")
+
+var cacheFields = map[string]bool{
+	"id":          true,
+	"parent":      true,
+	"type":        true,
+	"description": true,
+	"inuse":       true,
+	"shared":      true,
+	"private":     true,
+	// fields from buildkit that are not exposed
+	"mutable":   false,
+	"immutable": false,
+}
+
 func init() {
 	llbsolver.AllowNetworkHostUnstable = true
 }
 
 // Opt is option struct required for creating the builder
 type Opt struct {
-	SessionManager    *session.Manager
-	Root              string
-	Dist              images.DistributionServices
-	NetworkController libnetwork.NetworkController
+	SessionManager      *session.Manager
+	Root                string
+	Dist                images.DistributionServices
+	NetworkController   libnetwork.NetworkController
+	DefaultCgroupParent string
+	ResolverOpt         resolver.ResolveOptionsFunc
+	BuilderConfig       config.BuilderConfig
 }
 
 // Builder can build using BuildKit backend
@@ -86,48 +107,69 @@ func (b *Builder) DiskUsage(ctx context.Context) ([]*types.BuildCache, error) {
 	var items []*types.BuildCache
 	for _, r := range duResp.Record {
 		items = append(items, &types.BuildCache{
-			ID:      r.ID,
-			Mutable: r.Mutable,
-			InUse:   r.InUse,
-			Size:    r.Size_,
-
+			ID:          r.ID,
+			Parent:      r.Parent,
+			Type:        r.RecordType,
+			Description: r.Description,
+			InUse:       r.InUse,
+			Shared:      r.Shared,
+			Size:        r.Size_,
 			CreatedAt:   r.CreatedAt,
 			LastUsedAt:  r.LastUsedAt,
 			UsageCount:  int(r.UsageCount),
-			Parent:      r.Parent,
-			Description: r.Description,
 		})
 	}
 	return items, nil
 }
 
 // Prune clears all reclaimable build cache
-func (b *Builder) Prune(ctx context.Context) (int64, error) {
+func (b *Builder) Prune(ctx context.Context, opts types.BuildCachePruneOptions) (int64, []string, error) {
 	ch := make(chan *controlapi.UsageRecord)
 
 	eg, ctx := errgroup.WithContext(ctx)
 
+	validFilters := make(map[string]bool, 1+len(cacheFields))
+	validFilters["unused-for"] = true
+	for k, v := range cacheFields {
+		validFilters[k] = v
+	}
+	if err := opts.Filters.Validate(validFilters); err != nil {
+		return 0, nil, err
+	}
+
+	pi, err := toBuildkitPruneInfo(opts)
+	if err != nil {
+		return 0, nil, err
+	}
+
 	eg.Go(func() error {
 		defer close(ch)
-		return b.controller.Prune(&controlapi.PruneRequest{}, &pruneProxy{
+		return b.controller.Prune(&controlapi.PruneRequest{
+			All:          pi.All,
+			KeepDuration: int64(pi.KeepDuration),
+			KeepBytes:    pi.KeepBytes,
+			Filter:       pi.Filter,
+		}, &pruneProxy{
 			streamProxy: streamProxy{ctx: ctx},
 			ch:          ch,
 		})
 	})
 
 	var size int64
+	var cacheIDs []string
 	eg.Go(func() error {
 		for r := range ch {
 			size += r.Size_
+			cacheIDs = append(cacheIDs, r.ID)
 		}
 		return nil
 	})
 
 	if err := eg.Wait(); err != nil {
-		return 0, err
+		return 0, nil, err
 	}
 
-	return size, nil
+	return size, cacheIDs, nil
 }
 
 // Build executes a build request
@@ -179,7 +221,9 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.
 
 	id := identity.NewID()
 
-	frontendAttrs := map[string]string{}
+	frontendAttrs := map[string]string{
+		"override-copy-image": "docker.io/docker/dockerfile-copy:v0.1.9@sha256:e8f159d3f00786604b93c675ee2783f8dc194bb565e61ca5788f6a6e9d304061",
+	}
 
 	if opt.Options.Target != "" {
 		frontendAttrs["target"] = opt.Options.Target
@@ -467,3 +511,47 @@ func toBuildkitExtraHosts(inp []string) (string, error) {
 	}
 	return strings.Join(hosts, ","), nil
 }
+
+func toBuildkitPruneInfo(opts types.BuildCachePruneOptions) (client.PruneInfo, error) {
+	var unusedFor time.Duration
+	unusedForValues := opts.Filters.Get("unused-for")
+
+	switch len(unusedForValues) {
+	case 0:
+
+	case 1:
+		var err error
+		unusedFor, err = time.ParseDuration(unusedForValues[0])
+		if err != nil {
+			return client.PruneInfo{}, errors.Wrap(err, "unused-for filter expects a duration (e.g., '24h')")
+		}
+
+	default:
+		return client.PruneInfo{}, errMultipleFilterValues
+	}
+
+	bkFilter := make([]string, 0, opts.Filters.Len())
+	for cacheField := range cacheFields {
+		if opts.Filters.Include(cacheField) {
+			values := opts.Filters.Get(cacheField)
+			switch len(values) {
+			case 0:
+				bkFilter = append(bkFilter, cacheField)
+			case 1:
+				if cacheField == "id" {
+					bkFilter = append(bkFilter, cacheField+"~="+values[0])
+				} else {
+					bkFilter = append(bkFilter, cacheField+"=="+values[0])
+				}
+			default:
+				return client.PruneInfo{}, errMultipleFilterValues
+			}
+		}
+	}
+	return client.PruneInfo{
+		All:          opts.All,
+		KeepDuration: unusedFor,
+		KeepBytes:    opts.KeepStorage,
+		Filter:       []string{strings.Join(bkFilter, ",")},
+	}, nil
+}

builder/builder-next/controller.go

@@ -6,14 +6,19 @@ import (
 	"path/filepath"
 
 	"github.com/containerd/containerd/content/local"
+	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/builder/builder-next/adapters/containerimage"
 	"github.com/docker/docker/builder/builder-next/adapters/snapshot"
 	containerimageexp "github.com/docker/docker/builder/builder-next/exporter"
+	"github.com/docker/docker/builder/builder-next/imagerefchecker"
 	mobyworker "github.com/docker/docker/builder/builder-next/worker"
+	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/daemon/graphdriver"
+	units "github.com/docker/go-units"
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/cache/metadata"
 	registryremotecache "github.com/moby/buildkit/cache/remotecache/registry"
+	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/control"
 	"github.com/moby/buildkit/exporter"
 	"github.com/moby/buildkit/frontend"
@@ -21,7 +26,7 @@ import (
 	"github.com/moby/buildkit/frontend/gateway"
 	"github.com/moby/buildkit/frontend/gateway/forwarder"
 	"github.com/moby/buildkit/snapshot/blobmapping"
-	"github.com/moby/buildkit/solver/boltdbcachestorage"
+	"github.com/moby/buildkit/solver/bboltcachestorage"
 	"github.com/moby/buildkit/worker"
 	"github.com/pkg/errors"
 )
@@ -69,9 +74,20 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
 		MetadataStore: md,
 	})
 
+	layerGetter, ok := sbase.(imagerefchecker.LayerGetter)
+	if !ok {
+		return nil, errors.Errorf("snapshotter does not implement layergetter")
+	}
+
+	refChecker := imagerefchecker.New(imagerefchecker.Opt{
+		ImageStore:  dist.ImageStore,
+		LayerGetter: layerGetter,
+	})
+
 	cm, err := cache.NewManager(cache.ManagerOpt{
-		Snapshotter:   snapshotter,
-		MetadataStore: md,
+		Snapshotter:     snapshotter,
+		MetadataStore:   md,
+		PruneRefChecker: refChecker,
 	})
 	if err != nil {
 		return nil, err
@@ -85,12 +101,13 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
 		MetadataStore:   dist.V2MetadataService,
 		ImageStore:      dist.ImageStore,
 		ReferenceStore:  dist.ReferenceStore,
+		ResolverOpt:     opt.ResolverOpt,
 	})
 	if err != nil {
 		return nil, err
 	}
 
-	exec, err := newExecutor(root, opt.NetworkController)
+	exec, err := newExecutor(root, opt.DefaultCgroupParent, opt.NetworkController)
 	if err != nil {
 		return nil, err
 	}
@@ -109,17 +126,23 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
 		return nil, err
 	}
 
-	cacheStorage, err := boltdbcachestorage.NewStore(filepath.Join(opt.Root, "cache.db"))
+	cacheStorage, err := bboltcachestorage.NewStore(filepath.Join(opt.Root, "cache.db"))
 	if err != nil {
 		return nil, err
 	}
 
+	gcPolicy, err := getGCPolicy(opt.BuilderConfig, root)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get builder GC policy")
+	}
+
 	wopt := mobyworker.Opt{
 		ID:                "moby",
 		SessionManager:    opt.SessionManager,
 		MetadataStore:     md,
 		ContentStore:      store,
 		CacheManager:      cm,
+		GCPolicy:          gcPolicy,
 		Snapshotter:       snapshotter,
 		Executor:          exec,
 		ImageSource:       src,
@@ -148,7 +171,48 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
 		WorkerController:         wc,
 		Frontends:                frontends,
 		CacheKeyStorage:          cacheStorage,
-		ResolveCacheImporterFunc: registryremotecache.ResolveCacheImporterFunc(opt.SessionManager),
+		ResolveCacheImporterFunc: registryremotecache.ResolveCacheImporterFunc(opt.SessionManager, opt.ResolverOpt),
 		// TODO: set ResolveCacheExporterFunc for exporting cache
 	})
 }
+
+func getGCPolicy(conf config.BuilderConfig, root string) ([]client.PruneInfo, error) {
+	var gcPolicy []client.PruneInfo
+	if conf.GC.Enabled {
+		var (
+			defaultKeepStorage int64
+			err                error
+		)
+
+		if conf.GC.DefaultKeepStorage != "" {
+			defaultKeepStorage, err = units.RAMInBytes(conf.GC.DefaultKeepStorage)
+			if err != nil {
+				return nil, errors.Wrapf(err, "could not parse '%s' as Builder.GC.DefaultKeepStorage config", conf.GC.DefaultKeepStorage)
+			}
+		}
+
+		if conf.GC.Policy == nil {
+			gcPolicy = mobyworker.DefaultGCPolicy(root, defaultKeepStorage)
+		} else {
+			gcPolicy = make([]client.PruneInfo, len(conf.GC.Policy))
+			for i, p := range conf.GC.Policy {
+				b, err := units.RAMInBytes(p.KeepStorage)
+				if err != nil {
+					return nil, err
+				}
+				if b == 0 {
+					b = defaultKeepStorage
+				}
+				gcPolicy[i], err = toBuildkitPruneInfo(types.BuildCachePruneOptions{
+					All:         p.All,
+					KeepStorage: b,
+					Filters:     p.Filter,
+				})
+				if err != nil {
+					return nil, err
+				}
+			}
+		}
+	}
+	return gcPolicy, nil
+}

builder/builder-next/executor_unix.go

@@ -3,41 +3,46 @@
 package buildkit
 
 import (
-	"fmt"
+	"os"
 	"path/filepath"
+	"strconv"
 	"sync"
 
 	"github.com/docker/libnetwork"
 	"github.com/moby/buildkit/executor"
 	"github.com/moby/buildkit/executor/runcexecutor"
 	"github.com/moby/buildkit/identity"
+	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/network"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
 
 const networkName = "bridge"
 
-func newExecutor(root string, net libnetwork.NetworkController) (executor.Executor, error) {
-	// FIXME: fix bridge networking
-	_ = bridgeProvider{}
+func newExecutor(root, cgroupParent string, net libnetwork.NetworkController) (executor.Executor, error) {
+	networkProviders := map[pb.NetMode]network.Provider{
+		pb.NetMode_UNSET: &bridgeProvider{NetworkController: net},
+		pb.NetMode_HOST:  network.NewHostProvider(),
+		pb.NetMode_NONE:  network.NewNoneProvider(),
+	}
 	return runcexecutor.New(runcexecutor.Opt{
-		Root:              filepath.Join(root, "executor"),
-		CommandCandidates: []string{"docker-runc", "runc"},
-	}, nil)
+		Root:                filepath.Join(root, "executor"),
+		CommandCandidates:   []string{"runc"},
+		DefaultCgroupParent: cgroupParent,
+	}, networkProviders)
 }
 
 type bridgeProvider struct {
 	libnetwork.NetworkController
 }
 
-func (p *bridgeProvider) NewInterface() (network.Interface, error) {
+func (p *bridgeProvider) New() (network.Namespace, error) {
 	n, err := p.NetworkByName(networkName)
 	if err != nil {
 		return nil, err
 	}
 
-	iface := &lnInterface{ready: make(chan struct{})}
+	iface := &lnInterface{ready: make(chan struct{}), provider: p}
 	iface.Once.Do(func() {
 		go iface.init(p.NetworkController, n)
 	})
@@ -45,33 +50,13 @@ func (p *bridgeProvider) NewInterface() (network.Interface, error) {
 	return iface, nil
 }
 
-func (p *bridgeProvider) Release(iface network.Interface) error {
-	go func() {
-		if err := p.release(iface); err != nil {
-			logrus.Errorf("%s", err)
-		}
-	}()
-	return nil
-}
-
-func (p *bridgeProvider) release(iface network.Interface) error {
-	li, ok := iface.(*lnInterface)
-	if !ok {
-		return errors.Errorf("invalid interface %T", iface)
-	}
-	err := li.sbx.Delete()
-	if err1 := li.ep.Delete(true); err1 != nil && err == nil {
-		err = err1
-	}
-	return err
-}
-
 type lnInterface struct {
 	ep  libnetwork.Endpoint
 	sbx libnetwork.Sandbox
 	sync.Once
-	err   error
-	ready chan struct{}
+	err      error
+	ready    chan struct{}
+	provider *bridgeProvider
 }
 
 func (iface *lnInterface) init(c libnetwork.NetworkController, n libnetwork.Network) {
@@ -99,14 +84,26 @@ func (iface *lnInterface) init(c libnetwork.NetworkController, n libnetwork.Netw
 	iface.ep = ep
 }
 
-func (iface *lnInterface) Set(pid int) error {
+func (iface *lnInterface) Set(s *specs.Spec) {
 	<-iface.ready
 	if iface.err != nil {
-		return iface.err
+		return
+	}
+	// attach netns to bridge within the container namespace, using reexec in a prestart hook
+	s.Hooks = &specs.Hooks{
+		Prestart: []specs.Hook{{
+			Path: filepath.Join("/proc", strconv.Itoa(os.Getpid()), "exe"),
+			Args: []string{"libnetwork-setkey", iface.sbx.ContainerID(), iface.provider.NetworkController.ID()},
+		}},
 	}
-	return iface.sbx.SetKey(fmt.Sprintf("/proc/%d/ns/net", pid))
 }
 
-func (iface *lnInterface) Remove(pid int) error {
-	return nil
+func (iface *lnInterface) Close() error {
+	<-iface.ready
+	err := iface.sbx.Delete()
+	if iface.err != nil {
+		// iface.err takes precedence over cleanup errors
+		return iface.err
+	}
+	return err
 }

builder/builder-next/executor_windows.go

@@ -10,7 +10,7 @@ import (
 	"github.com/moby/buildkit/executor"
 )
 
-func newExecutor(_ string, _ libnetwork.NetworkController) (executor.Executor, error) {
+func newExecutor(_, _ string, _ libnetwork.NetworkController) (executor.Executor, error) {
 	return &winExecutor{}, nil
 }
 

builder/builder-next/imagerefchecker/checker.go

@@ -0,0 +1,96 @@
+package imagerefchecker
+
+import (
+	"sync"
+
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/layer"
+	"github.com/moby/buildkit/cache"
+)
+
+// LayerGetter abstracts away the snapshotter
+type LayerGetter interface {
+	GetLayer(string) (layer.Layer, error)
+}
+
+// Opt represents the options needed to create a refchecker
+type Opt struct {
+	LayerGetter LayerGetter
+	ImageStore  image.Store
+}
+
+// New creates new image reference checker that can be used to see if a reference
+// is being used by any of the images in the image store
+func New(opt Opt) cache.ExternalRefCheckerFunc {
+	return func() (cache.ExternalRefChecker, error) {
+		return &checker{opt: opt, layers: lchain{}, cache: map[string]bool{}}, nil
+	}
+}
+
+type lchain map[layer.DiffID]lchain
+
+func (c lchain) add(ids []layer.DiffID) {
+	if len(ids) == 0 {
+		return
+	}
+	id := ids[0]
+	ch, ok := c[id]
+	if !ok {
+		ch = lchain{}
+		c[id] = ch
+	}
+	ch.add(ids[1:])
+}
+
+func (c lchain) has(ids []layer.DiffID) bool {
+	if len(ids) == 0 {
+		return true
+	}
+	ch, ok := c[ids[0]]
+	return ok && ch.has(ids[1:])
+}
+
+type checker struct {
+	opt    Opt
+	once   sync.Once
+	layers lchain
+	cache  map[string]bool
+}
+
+func (c *checker) Exists(key string) bool {
+	if c.opt.ImageStore == nil {
+		return false
+	}
+
+	c.once.Do(c.init)
+
+	if b, ok := c.cache[key]; ok {
+		return b
+	}
+
+	l, err := c.opt.LayerGetter.GetLayer(key)
+	if err != nil || l == nil {
+		c.cache[key] = false
+		return false
+	}
+
+	ok := c.layers.has(diffIDs(l))
+	c.cache[key] = ok
+	return ok
+}
+
+func (c *checker) init() {
+	imgs := c.opt.ImageStore.Map()
+
+	for _, img := range imgs {
+		c.layers.add(img.RootFS.DiffIDs)
+	}
+}
+
+func diffIDs(l layer.Layer) []layer.DiffID {
+	p := l.Parent()
+	if p == nil {
+		return []layer.DiffID{l.DiffID()}
+	}
+	return append(diffIDs(p), l.DiffID())
+}

builder/builder-next/worker/gc.go

@@ -0,0 +1,51 @@
+package worker
+
+import (
+	"math"
+
+	"github.com/moby/buildkit/client"
+)
+
+const defaultCap int64 = 2e9 // 2GB
+
+// tempCachePercent represents the percentage ratio of the cache size in bytes to temporarily keep for a short period of time (couple of days)
+// over the total cache size in bytes. Because there is no perfect value, a mathematically pleasing one was chosen.
+// The value is approximately 13.8
+const tempCachePercent = math.E * math.Pi * math.Phi
+
+// DefaultGCPolicy returns a default builder GC policy
+func DefaultGCPolicy(p string, defaultKeepBytes int64) []client.PruneInfo {
+	keep := defaultKeepBytes
+	if defaultKeepBytes == 0 {
+		keep = detectDefaultGCCap(p)
+	}
+
+	tempCacheKeepBytes := int64(math.Round(float64(keep) / 100. * float64(tempCachePercent)))
+	const minTempCacheKeepBytes = 512 * 1e6 // 512MB
+	if tempCacheKeepBytes < minTempCacheKeepBytes {
+		tempCacheKeepBytes = minTempCacheKeepBytes
+	}
+
+	return []client.PruneInfo{
+		// if build cache uses more than 512MB delete the most easily reproducible data after it has not been used for 2 days
+		{
+			Filter:       []string{"type==source.local,type==exec.cachemount,type==source.git.checkout"},
+			KeepDuration: 48 * 3600, // 48h
+			KeepBytes:    tempCacheKeepBytes,
+		},
+		// remove any data not used for 60 days
+		{
+			KeepDuration: 60 * 24 * 3600, // 60d
+			KeepBytes:    keep,
+		},
+		// keep the unshared build cache under cap
+		{
+			KeepBytes: keep,
+		},
+		// if previous policies were insufficient start deleting internal data to keep build cache under cap
+		{
+			All:       true,
+			KeepBytes: keep,
+		},
+	}
+}

builder/builder-next/worker/gc_unix.go

@@ -0,0 +1,17 @@
+// +build !windows
+
+package worker
+
+import (
+	"syscall"
+)
+
+func detectDefaultGCCap(root string) int64 {
+	var st syscall.Statfs_t
+	if err := syscall.Statfs(root, &st); err != nil {
+		return defaultCap
+	}
+	diskSize := int64(st.Bsize) * int64(st.Blocks) // nolint unconvert
+	avail := diskSize / 10
+	return (avail/(1<<30) + 1) * 1e9 // round up
+}

builder/builder-next/worker/gc_windows.go

@@ -0,0 +1,7 @@
+// +build windows
+
+package worker
+
+func detectDefaultGCCap(root string) int64 {
+	return defaultCap
+}

builder/builder-next/worker/worker.go

@@ -46,6 +46,7 @@ import (
 type Opt struct {
 	ID                string
 	Labels            map[string]string
+	GCPolicy          []client.PruneInfo
 	SessionManager    *session.Manager
 	MetadataStore     *metadata.Store
 	Executor          executor.Executor
@@ -130,9 +131,18 @@ func (w *Worker) Platforms() []ocispec.Platform {
 	return []ocispec.Platform{platforms.DefaultSpec()}
 }
 
+// GCPolicy returns automatic GC Policy
+func (w *Worker) GCPolicy() []client.PruneInfo {
+	return w.Opt.GCPolicy
+}
+
 // LoadRef loads a reference by ID
-func (w *Worker) LoadRef(id string) (cache.ImmutableRef, error) {
-	return w.CacheManager.Get(context.TODO(), id)
+func (w *Worker) LoadRef(id string, hidden bool) (cache.ImmutableRef, error) {
+	var opts []cache.RefOption
+	if hidden {
+		opts = append(opts, cache.NoUpdateLastUsed)
+	}
+	return w.CacheManager.Get(context.TODO(), id, opts...)
 }
 
 // ResolveOp converts a LLB vertex into a LLB operation
@@ -176,8 +186,8 @@ func (w *Worker) DiskUsage(ctx context.Context, opt client.DiskUsageInfo) ([]*cl
 }
 
 // Prune deletes reclaimable build cache
-func (w *Worker) Prune(ctx context.Context, ch chan client.UsageInfo, info client.PruneInfo) error {
-	return w.CacheManager.Prune(ctx, ch, info)
+func (w *Worker) Prune(ctx context.Context, ch chan client.UsageInfo, info ...client.PruneInfo) error {
+	return w.CacheManager.Prune(ctx, ch, info...)
 }
 
 // Exporter returns exporter by name

builder/fscache/fscache.go

@@ -12,7 +12,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/boltdb/bolt"
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/builder/remotecontext"
 	"github.com/docker/docker/pkg/archive"
@@ -23,6 +22,8 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/tonistiigi/fsutil"
+	fsutiltypes "github.com/tonistiigi/fsutil/types"
+	bolt "go.etcd.io/bbolt"
 	"golang.org/x/sync/singleflight"
 )
 
@@ -614,7 +615,7 @@ func (s sortableCacheSources) Swap(i, j int) {
 	s[i], s[j] = s[j], s[i]
 }
 
-func newTarsumHash(stat *fsutil.Stat) (hash.Hash, error) {
+func newTarsumHash(stat *fsutiltypes.Stat) (hash.Hash, error) {
 	fi := &fsutil.StatInfo{Stat: stat}
 	p := stat.Path
 	if fi.IsDir() {

client/build_prune.go

@@ -4,19 +4,34 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/url"
 
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/pkg/errors"
 )
 
 // BuildCachePrune requests the daemon to delete unused cache data
-func (cli *Client) BuildCachePrune(ctx context.Context) (*types.BuildCachePruneReport, error) {
+func (cli *Client) BuildCachePrune(ctx context.Context, opts types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error) {
 	if err := cli.NewVersionError("1.31", "build prune"); err != nil {
 		return nil, err
 	}
 
 	report := types.BuildCachePruneReport{}
 
-	serverResp, err := cli.post(ctx, "/build/prune", nil, nil, nil)
+	query := url.Values{}
+	if opts.All {
+		query.Set("all", "1")
+	}
+	query.Set("keep-storage", fmt.Sprintf("%d", opts.KeepStorage))
+	filters, err := filters.ToJSON(opts.Filters)
+	if err != nil {
+		return nil, errors.Wrap(err, "prune could not marshal filters option")
+	}
+	query.Set("filters", filters)
+
+	serverResp, err := cli.post(ctx, "/build/prune", query, nil, nil)
+
 	if err != nil {
 		return nil, err
 	}

client/client.go

@@ -413,7 +413,7 @@ func (cli *Client) SetCustomHTTPHeaders(headers map[string]string) {
 func (cli *Client) Dialer() func(context.Context) (net.Conn, error) {
 	return func(ctx context.Context) (net.Conn, error) {
 		if transport, ok := cli.client.Transport.(*http.Transport); ok {
-			if transport.DialContext != nil {
+			if transport.DialContext != nil && transport.TLSClientConfig == nil {
 				return transport.DialContext(ctx, cli.proto, cli.addr)
 			}
 		}

client/interface.go

@@ -86,7 +86,7 @@ type DistributionAPIClient interface {
 // ImageAPIClient defines API client methods for the images
 type ImageAPIClient interface {
 	ImageBuild(ctx context.Context, context io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error)
-	BuildCachePrune(ctx context.Context) (*types.BuildCachePruneReport, error)
+	BuildCachePrune(ctx context.Context, opts types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error)
 	BuildCancel(ctx context.Context, id string) error
 	ImageCreate(ctx context.Context, parentReference string, options types.ImageCreateOptions) (io.ReadCloser, error)
 	ImageHistory(ctx context.Context, image string) ([]image.HistoryResponseItem, error)

cmd/dockerd/daemon.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	containerddefaults "github.com/containerd/containerd/defaults"
 	"github.com/docker/distribution/uuid"
 	"github.com/docker/docker/api"
 	apiserver "github.com/docker/docker/api/server"
@@ -27,7 +28,6 @@ import (
 	swarmrouter "github.com/docker/docker/api/server/router/swarm"
 	systemrouter "github.com/docker/docker/api/server/router/system"
 	"github.com/docker/docker/api/server/router/volume"
-	"github.com/docker/docker/api/types"
 	buildkit "github.com/docker/docker/builder/builder-next"
 	"github.com/docker/docker/builder/dockerfile"
 	"github.com/docker/docker/builder/fscache"
@@ -141,22 +141,25 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
 
 	ctx, cancel := context.WithCancel(context.Background())
 	if cli.Config.ContainerdAddr == "" && runtime.GOOS != "windows" {
-		opts, err := cli.getContainerdDaemonOpts()
-		if err != nil {
-			cancel()
-			return fmt.Errorf("Failed to generate containerd options: %v", err)
+		if !systemContainerdRunning() {
+			opts, err := cli.getContainerdDaemonOpts()
+			if err != nil {
+				cancel()
+				return fmt.Errorf("Failed to generate containerd options: %v", err)
+			}
+
+			r, err := supervisor.Start(ctx, filepath.Join(cli.Config.Root, "containerd"), filepath.Join(cli.Config.ExecRoot, "containerd"), opts...)
+			if err != nil {
+				cancel()
+				return fmt.Errorf("Failed to start containerd: %v", err)
+			}
+			cli.Config.ContainerdAddr = r.Address()
+
+			// Try to wait for containerd to shutdown
+			defer r.WaitTimeout(10 * time.Second)
+		} else {
+			cli.Config.ContainerdAddr = containerddefaults.DefaultAddress
 		}
-
-		r, err := supervisor.Start(ctx, filepath.Join(cli.Config.Root, "containerd"), filepath.Join(cli.Config.ExecRoot, "containerd"), opts...)
-		if err != nil {
-			cancel()
-			return fmt.Errorf("Failed to start containerd: %v", err)
-		}
-
-		cli.Config.ContainerdAddr = r.Address()
-
-		// Try to wait for containerd to shutdown
-		defer r.WaitTimeout(10 * time.Second)
 	}
 	defer cancel()
 
@@ -253,14 +256,14 @@ type routerOptions struct {
 	sessionManager *session.Manager
 	buildBackend   *buildbackend.Backend
 	buildCache     *fscache.FSCache // legacy
+	features       *map[string]bool
 	buildkit       *buildkit.Builder
-	builderVersion types.BuilderVersion
 	daemon         *daemon.Daemon
 	api            *apiserver.Server
 	cluster        *cluster.Cluster
 }
 
-func newRouterOptions(config *config.Config, daemon *daemon.Daemon) (routerOptions, error) {
+func newRouterOptions(config *config.Config, d *daemon.Daemon) (routerOptions, error) {
 	opts := routerOptions{}
 	sm, err := session.NewManager()
 	if err != nil {
@@ -281,39 +284,35 @@ func newRouterOptions(config *config.Config, daemon *daemon.Daemon) (routerOptio
 		return opts, errors.Wrap(err, "failed to create fscache")
 	}
 
-	manager, err := dockerfile.NewBuildManager(daemon.BuilderBackend(), sm, buildCache, daemon.IdentityMapping())
+	manager, err := dockerfile.NewBuildManager(d.BuilderBackend(), sm, buildCache, d.IdentityMapping())
 	if err != nil {
 		return opts, err
 	}
+	cgroupParent := newCgroupParent(config)
 	bk, err := buildkit.New(buildkit.Opt{
-		SessionManager:    sm,
-		Root:              filepath.Join(config.Root, "buildkit"),
-		Dist:              daemon.DistributionServices(),
-		NetworkController: daemon.NetworkController(),
+		SessionManager:      sm,
+		Root:                filepath.Join(config.Root, "buildkit"),
+		Dist:                d.DistributionServices(),
+		NetworkController:   d.NetworkController(),
+		DefaultCgroupParent: cgroupParent,
+		ResolverOpt:         d.NewResolveOptionsFunc(),
+		BuilderConfig:       config.Builder,
 	})
 	if err != nil {
 		return opts, err
 	}
 
-	bb, err := buildbackend.NewBackend(daemon.ImageService(), manager, buildCache, bk)
+	bb, err := buildbackend.NewBackend(d.ImageService(), manager, buildCache, bk)
 	if err != nil {
 		return opts, errors.Wrap(err, "failed to create buildmanager")
 	}
-	var bv types.BuilderVersion
-	if v, ok := config.Features["buildkit"]; ok {
-		if v {
-			bv = types.BuilderBuildKit
-		} else {
-			bv = types.BuilderV1
-		}
-	}
 	return routerOptions{
 		sessionManager: sm,
 		buildBackend:   bb,
 		buildCache:     buildCache,
 		buildkit:       bk,
-		builderVersion: bv,
-		daemon:         daemon,
+		features:       d.Features(),
+		daemon:         d,
 	}, nil
 }
 
@@ -486,9 +485,9 @@ func initRouter(opts routerOptions) {
 		checkpointrouter.NewRouter(opts.daemon, decoder),
 		container.NewRouter(opts.daemon, decoder),
 		image.NewRouter(opts.daemon.ImageService()),
-		systemrouter.NewRouter(opts.daemon, opts.cluster, opts.buildCache, opts.buildkit, opts.builderVersion),
+		systemrouter.NewRouter(opts.daemon, opts.cluster, opts.buildCache, opts.buildkit, opts.features),
 		volume.NewRouter(opts.daemon.VolumesService()),
-		build.NewRouter(opts.buildBackend, opts.daemon, opts.builderVersion),
+		build.NewRouter(opts.buildBackend, opts.daemon, opts.features),
 		sessionrouter.NewRouter(opts.sessionManager),
 		swarmrouter.NewRouter(opts.cluster),
 		pluginrouter.NewRouter(opts.daemon.PluginManager()),
@@ -666,3 +665,8 @@ func validateAuthzPlugins(requestedPlugins []string, pg plugingetter.PluginGette
 	}
 	return nil
 }
+
+func systemContainerdRunning() bool {
+	_, err := os.Lstat(containerddefaults.DefaultAddress)
+	return err == nil
+}

cmd/dockerd/daemon_unix.go

@@ -13,6 +13,7 @@ import (
 	"github.com/containerd/containerd/runtime/v1/linux"
 	"github.com/docker/docker/cmd/dockerd/hack"
 	"github.com/docker/docker/daemon"
+	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/libcontainerd/supervisor"
 	"github.com/docker/libnetwork/portallocator"
 	"golang.org/x/sys/unix"
@@ -107,3 +108,18 @@ func wrapListeners(proto string, ls []net.Listener) []net.Listener {
 	}
 	return ls
 }
+
+func newCgroupParent(config *config.Config) string {
+	cgroupParent := "docker"
+	useSystemd := daemon.UsingSystemd(config)
+	if useSystemd {
+		cgroupParent = "system.slice"
+	}
+	if config.CgroupParent != "" {
+		cgroupParent = config.CgroupParent
+	}
+	if useSystemd {
+		cgroupParent = cgroupParent + ":" + "docker" + ":"
+	}
+	return cgroupParent
+}

cmd/dockerd/daemon_windows.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/libcontainerd/supervisor"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/windows"
@@ -83,3 +84,7 @@ func allocateDaemonPort(addr string) error {
 func wrapListeners(proto string, ls []net.Listener) []net.Listener {
 	return ls
 }
+
+func newCgroupParent(config *config.Config) string {
+	return ""
+}

contrib/docker-machine-install-bundle.sh

@@ -31,7 +31,7 @@ bundle_files(){
 	    echo $BUNDLE/binary-daemon/$f
 	fi
     done
-    for f in docker-containerd docker-containerd-ctr docker-containerd-shim docker-init docker-runc; do
+    for f in containerd ctr containerd-shim docker-init runc; do
 	echo $BUNDLE/binary-daemon/$f
     done
     if [ -d $BUNDLE/dynbinary-client ]; then

daemon/attach.go

@@ -123,7 +123,7 @@ func (daemon *Daemon) containerAttach(c *container.Container, cfg *stream.Attach
 			return logger.ErrReadLogsNotSupported{}
 		}
 		logs := cLog.ReadLogs(logger.ReadConfig{Tail: -1})
-		defer logs.Close()
+		defer logs.ConsumerGone()
 
 	LogLoop:
 		for {

daemon/cluster/cluster.go

@@ -41,6 +41,7 @@ package cluster // import "github.com/docker/docker/daemon/cluster"
 import (
 	"context"
 	"fmt"
+	"math"
 	"net"
 	"os"
 	"path/filepath"
@@ -67,9 +68,10 @@ const stateFile = "docker-state.json"
 const defaultAddr = "0.0.0.0:2377"
 
 const (
-	initialReconnectDelay = 100 * time.Millisecond
-	maxReconnectDelay     = 30 * time.Second
-	contextPrefix         = "com.docker.swarm"
+	initialReconnectDelay          = 100 * time.Millisecond
+	maxReconnectDelay              = 30 * time.Second
+	contextPrefix                  = "com.docker.swarm"
+	defaultRecvSizeForListResponse = math.MaxInt32 // the max recv limit grpc <1.4.0
 )
 
 // NetworkSubnetsProvider exposes functions for retrieving the subnets

daemon/cluster/listen_addr.go

@@ -3,6 +3,7 @@ package cluster // import "github.com/docker/docker/daemon/cluster"
 import (
 	"fmt"
 	"net"
+	"strings"
 )
 
 const (
@@ -87,6 +88,41 @@ func (c *Cluster) resolveAdvertiseAddr(advertiseAddr, listenAddrPort string) (st
 	return systemAddr.String(), listenAddrPort, nil
 }
 
+// validateDefaultAddrPool validates default address pool
+// it also strips white space from the string before validation
+func validateDefaultAddrPool(defaultAddrPool []string, size uint32) error {
+	if defaultAddrPool == nil {
+		// defaultAddrPool is not defined
+		return nil
+	}
+	//if size is not set, then we use default value 24
+	if size == 0 {
+		size = 24
+	}
+	// We allow max value as 29. We can have 8 IP addresses for max value 29
+	// If we allow 30, then we will get only 4 IP addresses. But with latest
+	// libnetwork LB scale implementation, we use total of 4 IP addresses for internal use.
+	// Hence keeping 29 as max value, we will have 8 IP addresses. This will be
+	// smallest subnet that can be used in overlay network.
+	if size > 29 {
+		return fmt.Errorf("subnet size is out of range: %d", size)
+	}
+	for i := range defaultAddrPool {
+		// trim leading and trailing white spaces
+		defaultAddrPool[i] = strings.TrimSpace(defaultAddrPool[i])
+		_, b, err := net.ParseCIDR(defaultAddrPool[i])
+		if err != nil {
+			return fmt.Errorf("invalid base pool %s: %v", defaultAddrPool[i], err)
+		}
+		ones, _ := b.Mask.Size()
+		if size < uint32(ones) {
+			return fmt.Errorf("invalid CIDR: %q. Subnet size is too small for pool: %d", defaultAddrPool[i], size)
+		}
+	}
+
+	return nil
+}
+
 func resolveDataPathAddr(dataPathAddr string) (string, error) {
 	if dataPathAddr == "" {
 		// dataPathAddr is not defined

daemon/cluster/noderunner.go

@@ -3,7 +3,6 @@ package cluster // import "github.com/docker/docker/daemon/cluster"
 import (
 	"context"
 	"fmt"
-	"net"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -14,6 +13,7 @@ import (
 	"github.com/docker/docker/daemon/cluster/executor/container"
 	lncluster "github.com/docker/libnetwork/cluster"
 	swarmapi "github.com/docker/swarmkit/api"
+	"github.com/docker/swarmkit/manager/allocator/cnmallocator"
 	swarmnode "github.com/docker/swarmkit/node"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -115,12 +115,6 @@ func (n *nodeRunner) start(conf nodeStartConfig) error {
 		joinAddr = conf.RemoteAddr
 	}
 
-	var defaultAddrPool []*net.IPNet
-	for _, address := range conf.DefaultAddressPool {
-		if _, b, err := net.ParseCIDR(address); err == nil {
-			defaultAddrPool = append(defaultAddrPool, b)
-		}
-	}
 	// Hostname is not set here. Instead, it is obtained from
 	// the node description that is reported periodically
 	swarmnodeConfig := swarmnode.Config{
@@ -128,11 +122,13 @@ func (n *nodeRunner) start(conf nodeStartConfig) error {
 		ListenControlAPI:   control,
 		ListenRemoteAPI:    conf.ListenAddr,
 		AdvertiseRemoteAPI: conf.AdvertiseAddr,
-		DefaultAddrPool:    defaultAddrPool,
-		SubnetSize:         int(conf.SubnetSize),
-		JoinAddr:           joinAddr,
-		StateDir:           n.cluster.root,
-		JoinToken:          conf.joinToken,
+		NetworkConfig: &cnmallocator.NetworkConfig{
+			DefaultAddrPool: conf.DefaultAddressPool,
+			SubnetSize:      conf.SubnetSize,
+		},
+		JoinAddr:  joinAddr,
+		StateDir:  n.cluster.root,
+		JoinToken: conf.joinToken,
 		Executor: container.NewExecutor(
 			n.cluster.config.Backend,
 			n.cluster.config.PluginBackend,

daemon/cluster/services.go

@@ -23,6 +23,7 @@ import (
 	gogotypes "github.com/gogo/protobuf/types"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"google.golang.org/grpc"
 )
 
 // GetServices returns all services of a managed swarm cluster.
@@ -67,7 +68,9 @@ func (c *Cluster) GetServices(options apitypes.ServiceListOptions) ([]types.Serv
 
 	r, err := state.controlClient.ListServices(
 		ctx,
-		&swarmapi.ListServicesRequest{Filters: filters})
+		&swarmapi.ListServicesRequest{Filters: filters},
+		grpc.MaxCallRecvMsgSize(defaultRecvSizeForListResponse),
+	)
 	if err != nil {
 		return nil, err
 	}

daemon/cluster/swarm.go

@@ -92,6 +92,10 @@ func (c *Cluster) Init(req types.InitRequest) (string, error) {
 		}
 	}
 
+	//Validate Default Address Pool input
+	if err := validateDefaultAddrPool(req.DefaultAddrPool, req.SubnetSize); err != nil {
+		return "", err
+	}
 	nr, err := c.newNodeRunner(nodeStartConfig{
 		forceNewCluster:    req.ForceNewCluster,
 		autolock:           req.AutoLockManagers,

daemon/cluster/tasks.go

@@ -8,6 +8,7 @@ import (
 	types "github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/daemon/cluster/convert"
 	swarmapi "github.com/docker/swarmkit/api"
+	"google.golang.org/grpc"
 )
 
 // GetTasks returns a list of tasks matching the filter options.
@@ -53,7 +54,9 @@ func (c *Cluster) GetTasks(options apitypes.TaskListOptions) ([]types.Task, erro
 
 		r, err = state.controlClient.ListTasks(
 			ctx,
-			&swarmapi.ListTasksRequest{Filters: filters})
+			&swarmapi.ListTasksRequest{Filters: filters},
+			grpc.MaxCallRecvMsgSize(defaultRecvSizeForListResponse),
+		)
 		return err
 	}); err != nil {
 		return nil, err

daemon/config/builder.go

@@ -0,0 +1,22 @@
+package config
+
+import "github.com/docker/docker/api/types/filters"
+
+// BuilderGCRule represents a GC rule for buildkit cache
+type BuilderGCRule struct {
+	All         bool         `json:",omitempty"`
+	Filter      filters.Args `json:",omitempty"`
+	KeepStorage string       `json:",omitempty"`
+}
+
+// BuilderGCConfig contains GC config for a buildkit builder
+type BuilderGCConfig struct {
+	Enabled            bool            `json:",omitempty"`
+	Policy             []BuilderGCRule `json:",omitempty"`
+	DefaultKeepStorage string          `json:",omitempty"`
+}
+
+// BuilderConfig contains config for the builder
+type BuilderConfig struct {
+	GC BuilderGCConfig `json:",omitempty"`
+}

daemon/config/config.go

@@ -55,6 +55,7 @@ var flatOptions = map[string]bool{
 	"runtimes":           true,
 	"default-ulimits":    true,
 	"features":           true,
+	"builder":            true,
 }
 
 // skipValidateOptions contains configuration keys
@@ -62,6 +63,17 @@ var flatOptions = map[string]bool{
 // for unknown flag validation.
 var skipValidateOptions = map[string]bool{
 	"features": true,
+	"builder":  true,
+}
+
+// skipDuplicates contains configuration keys that
+// will be skipped when checking duplicated
+// configuration field defined in both daemon
+// config file and from dockerd cli flags.
+// This allows some configurations to be merged
+// during the parsing.
+var skipDuplicates = map[string]bool{
+	"runtimes": true,
 }
 
 // LogConfig represents the default log configuration.
@@ -215,6 +227,8 @@ type CommonConfig struct {
 	// Features contains a list of feature key value pairs indicating what features are enabled or disabled.
 	// If a certain feature doesn't appear in this list then it's unset (i.e. neither true nor false).
 	Features map[string]bool `json:"features,omitempty"`
+
+	Builder BuilderConfig `json:"builder,omitempty"`
 }
 
 // IsValueSet returns true if a configuration value
@@ -491,13 +505,13 @@ func findConfigurationConflicts(config map[string]interface{}, flags *pflag.Flag
 	duplicatedConflicts := func(f *pflag.Flag) {
 		// search option name in the json configuration payload if the value is a named option
 		if namedOption, ok := f.Value.(opts.NamedOption); ok {
-			if optsValue, ok := config[namedOption.Name()]; ok {
+			if optsValue, ok := config[namedOption.Name()]; ok && !skipDuplicates[namedOption.Name()] {
 				conflicts = append(conflicts, printConflict(namedOption.Name(), f.Value.String(), optsValue))
 			}
 		} else {
 			// search flag name in the json configuration payload
 			for _, name := range []string{f.Name, f.Shorthand} {
-				if value, ok := config[name]; ok {
+				if value, ok := config[name]; ok && !skipDuplicates[name] {
 					conflicts = append(conflicts, printConflict(name, f.Value.String(), value))
 					break
 				}

daemon/daemon.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"net"
 	"os"
 	"path"
@@ -23,6 +24,8 @@ import (
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/defaults"
 	"github.com/containerd/containerd/pkg/dialer"
+	"github.com/containerd/containerd/remotes/docker"
+	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
 	containertypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/swarm"
@@ -36,6 +39,8 @@ import (
 	"github.com/docker/docker/daemon/logger"
 	"github.com/docker/docker/daemon/network"
 	"github.com/docker/docker/errdefs"
+	"github.com/moby/buildkit/util/resolver"
+	"github.com/moby/buildkit/util/tracing"
 	"github.com/sirupsen/logrus"
 	// register graph drivers
 	_ "github.com/docker/docker/daemon/graphdriver/register"
@@ -136,6 +141,62 @@ func (daemon *Daemon) HasExperimental() bool {
 	return daemon.configStore != nil && daemon.configStore.Experimental
 }
 
+// Features returns the features map from configStore
+func (daemon *Daemon) Features() *map[string]bool {
+	return &daemon.configStore.Features
+}
+
+// NewResolveOptionsFunc returns a call back function to resolve "registry-mirrors" and
+// "insecure-registries" for buildkit
+func (daemon *Daemon) NewResolveOptionsFunc() resolver.ResolveOptionsFunc {
+	return func(ref string) docker.ResolverOptions {
+		var (
+			registryKey = "docker.io"
+			mirrors     = make([]string, len(daemon.configStore.Mirrors))
+			m           = map[string]resolver.RegistryConf{}
+		)
+		// must trim "https://" or "http://" prefix
+		for i, v := range daemon.configStore.Mirrors {
+			v = strings.TrimPrefix(v, "https://")
+			v = strings.TrimPrefix(v, "http://")
+			mirrors[i] = v
+		}
+		// set "registry-mirrors"
+		m[registryKey] = resolver.RegistryConf{Mirrors: mirrors}
+		// set "insecure-registries"
+		for _, v := range daemon.configStore.InsecureRegistries {
+			v = strings.TrimPrefix(v, "http://")
+			m[v] = resolver.RegistryConf{
+				PlainHTTP: true,
+			}
+		}
+		def := docker.ResolverOptions{
+			Client: tracing.DefaultClient,
+		}
+
+		parsed, err := reference.ParseNormalizedNamed(ref)
+		if err != nil {
+			return def
+		}
+		host := reference.Domain(parsed)
+
+		c, ok := m[host]
+		if !ok {
+			return def
+		}
+
+		if len(c.Mirrors) > 0 {
+			def.Host = func(string) (string, error) {
+				return c.Mirrors[rand.Intn(len(c.Mirrors))], nil
+			}
+		}
+
+		def.PlainHTTP = c.PlainHTTP
+
+		return def
+	}
+}
+
 func (daemon *Daemon) restore() error {
 	containers := make(map[string]*container.Container)
 

daemon/daemon_unix.go

@@ -54,11 +54,11 @@ import (
 const (
 	// DefaultShimBinary is the default shim to be used by containerd if none
 	// is specified
-	DefaultShimBinary = "docker-containerd-shim"
+	DefaultShimBinary = "containerd-shim"
 
 	// DefaultRuntimeBinary is the default runtime to be used by
 	// containerd if none is specified
-	DefaultRuntimeBinary = "docker-runc"
+	DefaultRuntimeBinary = "runc"
 
 	// See https://git.kernel.org/cgit/linux/kernel/git/tip/tip.git/tree/kernel/sched/sched.h?id=8cd9234c64c584432f6992fe944ca9e46ca8ea76#n269
 	linuxMinCPUShares = 2
@@ -76,7 +76,7 @@ const (
 
 	// DefaultRuntimeName is the default runtime to be used by
 	// containerd if none is specified
-	DefaultRuntimeName = "docker-runc"
+	DefaultRuntimeName = "runc"
 )
 
 type containerGetter interface {
@@ -482,14 +482,14 @@ func verifyContainerResources(resources *containertypes.Resources, sysInfo *sysi
 	}
 	cpusAvailable, err := sysInfo.IsCpusetCpusAvailable(resources.CpusetCpus)
 	if err != nil {
-		return warnings, fmt.Errorf("Invalid value %s for cpuset cpus", resources.CpusetCpus)
+		return warnings, errors.Wrapf(err, "Invalid value %s for cpuset cpus", resources.CpusetCpus)
 	}
 	if !cpusAvailable {
 		return warnings, fmt.Errorf("Requested CPUs are not available - requested %s, available: %s", resources.CpusetCpus, sysInfo.Cpus)
 	}
 	memsAvailable, err := sysInfo.IsCpusetMemsAvailable(resources.CpusetMems)
 	if err != nil {
-		return warnings, fmt.Errorf("Invalid value %s for cpuset mems", resources.CpusetMems)
+		return warnings, errors.Wrapf(err, "Invalid value %s for cpuset mems", resources.CpusetMems)
 	}
 	if !memsAvailable {
 		return warnings, fmt.Errorf("Requested memory nodes are not available - requested %s, available: %s", resources.CpusetMems, sysInfo.Mems)

daemon/daemon_windows.go

@@ -323,7 +323,8 @@ func (daemon *Daemon) initNetworkController(config *config.Config, activeSandbox
 	// discover and add HNS networks to windows
 	// network that exist are removed and added again
 	for _, v := range hnsresponse {
-		if strings.ToLower(v.Type) == "private" {
+		networkTypeNorm := strings.ToLower(v.Type)
+		if networkTypeNorm == "private" || networkTypeNorm == "internal" {
 			continue // workaround for HNS reporting unsupported networks
 		}
 		var n libnetwork.Network

daemon/graphdriver/btrfs/btrfs.go

@@ -29,10 +29,12 @@ import (
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
+	"github.com/docker/docker/pkg/mount"
 	"github.com/docker/docker/pkg/parsers"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/go-units"
 	"github.com/opencontainers/selinux/go-selinux/label"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 )
@@ -81,6 +83,15 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap
 		return nil, err
 	}
 
+	// For some reason shared mount propagation between a container
+	// and the host does not work for btrfs, and a remedy is to bind
+	// mount graphdriver home to itself (even without changing the
+	// propagation mode).
+	err = mount.MakeMount(home)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to make %s a mount", home)
+	}
+
 	driver := &Driver{
 		home:    home,
 		uidMaps: uidMaps,
@@ -158,7 +169,19 @@ func (d *Driver) GetMetadata(id string) (map[string]string, error) {
 
 // Cleanup unmounts the home directory.
 func (d *Driver) Cleanup() error {
-	return d.subvolDisableQuota()
+	err := d.subvolDisableQuota()
+	umountErr := mount.Unmount(d.home)
+
+	// in case we have two errors, prefer the one from disableQuota()
+	if err != nil {
+		return err
+	}
+
+	if umountErr != nil {
+		return errors.Wrapf(umountErr, "error unmounting %s", d.home)
+	}
+
+	return nil
 }
 
 func free(p *C.char) {

daemon/graphdriver/devmapper/device_setup.go

@@ -27,7 +27,7 @@ type directLVMConfig struct {
 var (
 	errThinpPercentMissing = errors.New("must set both `dm.thinp_percent` and `dm.thinp_metapercent` if either is specified")
 	errThinpPercentTooBig  = errors.New("combined `dm.thinp_percent` and `dm.thinp_metapercent` must not be greater than 100")
-	errMissingSetupDevice  = errors.New("must provide device path in `dm.setup_device` in order to configure direct-lvm")
+	errMissingSetupDevice  = errors.New("must provide device path in `dm.directlvm_device` in order to configure direct-lvm")
 )
 
 func validateLVMConfig(cfg directLVMConfig) error {

daemon/images/service.go

@@ -205,8 +205,6 @@ func (i *ImageService) LayerDiskUsage(ctx context.Context) (int64, error) {
 				if err == nil {
 					if _, ok := layerRefs[l.ChainID()]; ok {
 						allLayersSize += size
-					} else {
-						logrus.Warnf("found leaked image layer %v", l.ChainID())
 					}
 				} else {
 					logrus.Warnf("failed to get diff size for layer %v", l.ChainID())

daemon/info.go

@@ -2,6 +2,7 @@ package daemon // import "github.com/docker/docker/daemon"
 
 import (
 	"fmt"
+	"net/url"
 	"os"
 	"runtime"
 	"strings"
@@ -61,8 +62,8 @@ func (daemon *Daemon) SystemInfo() (*types.Info, error) {
 		ServerVersion:      dockerversion.Version,
 		ClusterStore:       daemon.configStore.ClusterStore,
 		ClusterAdvertise:   daemon.configStore.ClusterAdvertise,
-		HTTPProxy:          sockets.GetProxyEnv("http_proxy"),
-		HTTPSProxy:         sockets.GetProxyEnv("https_proxy"),
+		HTTPProxy:          maskCredentials(sockets.GetProxyEnv("http_proxy")),
+		HTTPSProxy:         maskCredentials(sockets.GetProxyEnv("https_proxy")),
 		NoProxy:            sockets.GetProxyEnv("no_proxy"),
 		LiveRestoreEnabled: daemon.configStore.LiveRestoreEnabled,
 		Isolation:          daemon.defaultIsolation,
@@ -245,3 +246,13 @@ func operatingSystem() string {
 	}
 	return operatingSystem
 }
+
+func maskCredentials(rawURL string) string {
+	parsedURL, err := url.Parse(rawURL)
+	if err != nil || parsedURL.User == nil {
+		return rawURL
+	}
+	parsedURL.User = url.UserPassword("xxxxx", "xxxxx")
+	maskedURL := parsedURL.String()
+	return maskedURL
+}

daemon/info_test.go

@@ -0,0 +1,53 @@
+package daemon
+
+import (
+	"testing"
+
+	"gotest.tools/assert"
+)
+
+func TestMaskURLCredentials(t *testing.T) {
+	tests := []struct {
+		rawURL    string
+		maskedURL string
+	}{
+		{
+			rawURL:    "",
+			maskedURL: "",
+		}, {
+			rawURL:    "invalidURL",
+			maskedURL: "invalidURL",
+		}, {
+			rawURL:    "http://proxy.example.com:80/",
+			maskedURL: "http://proxy.example.com:80/",
+		}, {
+			rawURL:    "http://USER:PASSWORD@proxy.example.com:80/",
+			maskedURL: "http://xxxxx:xxxxx@proxy.example.com:80/",
+		}, {
+			rawURL:    "http://PASSWORD:PASSWORD@proxy.example.com:80/",
+			maskedURL: "http://xxxxx:xxxxx@proxy.example.com:80/",
+		}, {
+			rawURL:    "http://USER:@proxy.example.com:80/",
+			maskedURL: "http://xxxxx:xxxxx@proxy.example.com:80/",
+		}, {
+			rawURL:    "http://:PASSWORD@proxy.example.com:80/",
+			maskedURL: "http://xxxxx:xxxxx@proxy.example.com:80/",
+		}, {
+			rawURL:    "http://USER@docker:password@proxy.example.com:80/",
+			maskedURL: "http://xxxxx:xxxxx@proxy.example.com:80/",
+		}, {
+			rawURL:    "http://USER%40docker:password@proxy.example.com:80/",
+			maskedURL: "http://xxxxx:xxxxx@proxy.example.com:80/",
+		}, {
+			rawURL:    "http://USER%40docker:pa%3Fsword@proxy.example.com:80/",
+			maskedURL: "http://xxxxx:xxxxx@proxy.example.com:80/",
+		}, {
+			rawURL:    "http://USER%40docker:pa%3Fsword@proxy.example.com:80/hello%20world",
+			maskedURL: "http://xxxxx:xxxxx@proxy.example.com:80/hello%20world",
+		},
+	}
+	for _, test := range tests {
+		maskedURL := maskCredentials(test.rawURL)
+		assert.Equal(t, maskedURL, test.maskedURL)
+	}
+}

daemon/info_unix.go

@@ -29,7 +29,6 @@ func (daemon *Daemon) fillPlatformInfo(v *types.Info, sysInfo *sysinfo.SysInfo)
 	v.DefaultRuntime = daemon.configStore.GetDefaultRuntimeName()
 	v.InitBinary = daemon.configStore.GetInitPath()
 
-	v.RuncCommit.Expected = dockerversion.RuncCommitID
 	defaultRuntimeBinary := daemon.configStore.GetRuntime(v.DefaultRuntime).Path
 	if rv, err := exec.Command(defaultRuntimeBinary, "--version").Output(); err == nil {
 		parts := strings.Split(strings.TrimSpace(string(rv)), "\n")
@@ -49,7 +48,10 @@ func (daemon *Daemon) fillPlatformInfo(v *types.Info, sysInfo *sysinfo.SysInfo)
 		v.RuncCommit.ID = "N/A"
 	}
 
-	v.ContainerdCommit.Expected = dockerversion.ContainerdCommitID
+	// runc is now shipped as a separate package. Set "expected" to same value
+	// as "ID" to prevent clients from reporting a version-mismatch
+	v.RuncCommit.Expected = v.RuncCommit.ID
+
 	if rv, err := daemon.containerd.Version(context.Background()); err == nil {
 		v.ContainerdCommit.ID = rv.Revision
 	} else {
@@ -57,6 +59,10 @@ func (daemon *Daemon) fillPlatformInfo(v *types.Info, sysInfo *sysinfo.SysInfo)
 		v.ContainerdCommit.ID = "N/A"
 	}
 
+	// containerd is now shipped as a separate package. Set "expected" to same
+	// value as "ID" to prevent clients from reporting a version-mismatch
+	v.ContainerdCommit.Expected = v.ContainerdCommit.ID
+
 	defaultInitBinary := daemon.configStore.GetInitPath()
 	if rv, err := exec.Command(defaultInitBinary, "--version").Output(); err == nil {
 		ver, err := parseInitVersion(string(rv))

daemon/list.go

@@ -146,7 +146,8 @@ func (daemon *Daemon) filterByNameIDMatches(view container.View, ctx *listContex
 				continue
 			}
 			for _, eachName := range idNames {
-				if ctx.filters.Match("name", strings.TrimPrefix(eachName, "/")) {
+				// match both on container name with, and without slash-prefix
+				if ctx.filters.Match("name", eachName) || ctx.filters.Match("name", strings.TrimPrefix(eachName, "/")) {
 					matches[id] = true
 				}
 			}
@@ -429,7 +430,7 @@ func includeContainerInList(container *container.Snapshot, ctx *listContext) ite
 	}
 
 	// Do not include container if the name doesn't match
-	if !ctx.filters.Match("name", strings.TrimPrefix(container.Name, "/")) {
+	if !ctx.filters.Match("name", container.Name) && !ctx.filters.Match("name", strings.TrimPrefix(container.Name, "/")) {
 		return excludeContainer
 	}
 

daemon/list_test.go

@@ -4,7 +4,6 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"strings"
 	"testing"
 
 	"github.com/docker/docker/api/types"
@@ -35,6 +34,7 @@ func TestMain(m *testing.M) {
 // work against it. It takes in a pointer to Daemon so that
 // minor operations are not repeated by the caller
 func setupContainerWithName(t *testing.T, name string, daemon *Daemon) *container.Container {
+	t.Helper()
 	var (
 		id              = uuid.New()
 		computedImageID = digest.FromString(id)
@@ -46,6 +46,9 @@ func setupContainerWithName(t *testing.T, name string, daemon *Daemon) *containe
 
 	c := container.NewBaseContainer(id, cRoot)
 	// these are for passing includeContainerInList
+	if name[0] != '/' {
+		name = "/" + name
+	}
 	c.Name = name
 	c.Running = true
 	c.HostConfig = &containertypes.HostConfig{}
@@ -68,7 +71,7 @@ func setupContainerWithName(t *testing.T, name string, daemon *Daemon) *containe
 func containerListContainsName(containers []*types.Container, name string) bool {
 	for _, container := range containers {
 		for _, containerName := range container.Names {
-			if strings.TrimPrefix(containerName, "/") == name {
+			if containerName == name {
 				return true
 			}
 		}
@@ -110,16 +113,33 @@ func TestNameFilter(t *testing.T) {
 	containerList, err := d.Containers(&types.ContainerListOptions{
 		Filters: filters.NewArgs(filters.Arg("name", "^a")),
 	})
-	assert.Assert(t, err == nil)
+	assert.NilError(t, err)
 	assert.Assert(t, is.Len(containerList, 2))
 	assert.Assert(t, containerListContainsName(containerList, one.Name))
 	assert.Assert(t, containerListContainsName(containerList, two.Name))
 
+	// Same as above but with slash prefix should produce the same result
+	containerListWithPrefix, err := d.Containers(&types.ContainerListOptions{
+		Filters: filters.NewArgs(filters.Arg("name", "^/a")),
+	})
+	assert.NilError(t, err)
+	assert.Assert(t, is.Len(containerListWithPrefix, 2))
+	assert.Assert(t, containerListContainsName(containerListWithPrefix, one.Name))
+	assert.Assert(t, containerListContainsName(containerListWithPrefix, two.Name))
+
 	// Same as above but make sure it works for exact names
 	containerList, err = d.Containers(&types.ContainerListOptions{
 		Filters: filters.NewArgs(filters.Arg("name", "b1")),
 	})
-	assert.Assert(t, err == nil)
+	assert.NilError(t, err)
 	assert.Assert(t, is.Len(containerList, 1))
 	assert.Assert(t, containerListContainsName(containerList, three.Name))
+
+	// Same as above but with slash prefix should produce the same result
+	containerListWithPrefix, err = d.Containers(&types.ContainerListOptions{
+		Filters: filters.NewArgs(filters.Arg("name", "/b1")),
+	})
+	assert.NilError(t, err)
+	assert.Assert(t, is.Len(containerListWithPrefix, 1))
+	assert.Assert(t, containerListContainsName(containerListWithPrefix, three.Name))
 }

daemon/logger/adapter.go

@@ -93,21 +93,12 @@ func (a *pluginAdapterWithRead) ReadLogs(config ReadConfig) *LogWatcher {
 
 		dec := logdriver.NewLogEntryDecoder(stream)
 		for {
-			select {
-			case <-watcher.WatchClose():
-				return
-			default:
-			}
-
 			var buf logdriver.LogEntry
 			if err := dec.Decode(&buf); err != nil {
 				if err == io.EOF {
 					return
 				}
-				select {
-				case watcher.Err <- errors.Wrap(err, "error decoding log message"):
-				case <-watcher.WatchClose():
-				}
+				watcher.Err <- errors.Wrap(err, "error decoding log message")
 				return
 			}
 
@@ -125,11 +116,10 @@ func (a *pluginAdapterWithRead) ReadLogs(config ReadConfig) *LogWatcher {
 				return
 			}
 
+			// send the message unless the consumer is gone
 			select {
 			case watcher.Msg <- msg:
-			case <-watcher.WatchClose():
-				// make sure the message we consumed is sent
-				watcher.Msg <- msg
+			case <-watcher.WatchConsumerGone():
 				return
 			}
 		}

daemon/logger/adapter_test.go

@@ -174,7 +174,7 @@ func TestAdapterReadLogs(t *testing.T) {
 		t.Fatal("timeout waiting for message channel to close")
 
 	}
-	lw.Close()
+	lw.ProducerGone()
 
 	lw = lr.ReadLogs(ReadConfig{Follow: true})
 	for _, x := range testMsg {

daemon/logger/journald/read.go

@@ -165,7 +165,7 @@ func (s *journald) Close() error {
 	s.mu.Lock()
 	s.closed = true
 	for reader := range s.readers.readers {
-		reader.Close()
+		reader.ProducerGone()
 	}
 	s.mu.Unlock()
 	return nil
@@ -299,7 +299,7 @@ func (s *journald) followJournal(logWatcher *logger.LogWatcher, j *C.sd_journal,
 	// Wait until we're told to stop.
 	select {
 	case cursor = <-newCursor:
-	case <-logWatcher.WatchClose():
+	case <-logWatcher.WatchConsumerGone():
 		// Notify the other goroutine that its work is done.
 		C.close(pfd[1])
 		cursor = <-newCursor

daemon/logger/jsonfilelog/jsonfilelog.go

@@ -50,7 +50,7 @@ func New(info logger.Info) (logger.Logger, error) {
 			return nil, err
 		}
 		if capval <= 0 {
-			return nil, fmt.Errorf("max-size should be a positive numbler")
+			return nil, fmt.Errorf("max-size must be a positive number")
 		}
 	}
 	var maxFiles = 1
@@ -166,13 +166,14 @@ func ValidateLogOpt(cfg map[string]string) error {
 	return nil
 }
 
-// Close closes underlying file and signals all readers to stop.
+// Close closes underlying file and signals all the readers
+// that the logs producer is gone.
 func (l *JSONFileLogger) Close() error {
 	l.mu.Lock()
 	l.closed = true
 	err := l.writer.Close()
 	for r := range l.readers {
-		r.Close()
+		r.ProducerGone()
 		delete(l.readers, r)
 	}
 	l.mu.Unlock()

daemon/logger/jsonfilelog/read_test.go

@@ -50,11 +50,10 @@ func BenchmarkJSONFileLoggerReadLogs(b *testing.B) {
 	}()
 
 	lw := jsonlogger.(*JSONFileLogger).ReadLogs(logger.ReadConfig{Follow: true})
-	watchClose := lw.WatchClose()
 	for {
 		select {
 		case <-lw.Msg:
-		case <-watchClose:
+		case <-lw.WatchProducerGone():
 			return
 		case err := <-chError:
 			if err != nil {

daemon/logger/local/local.go

@@ -166,7 +166,7 @@ func (d *driver) Close() error {
 	d.closed = true
 	err := d.logfile.Close()
 	for r := range d.readers {
-		r.Close()
+		r.ProducerGone()
 		delete(d.readers, r)
 	}
 	d.mu.Unlock()

daemon/logger/logger.go

@@ -104,33 +104,50 @@ type LogWatcher struct {
 	// For sending log messages to a reader.
 	Msg chan *Message
 	// For sending error messages that occur while while reading logs.
-	Err           chan error
-	closeOnce     sync.Once
-	closeNotifier chan struct{}
+	Err          chan error
+	producerOnce sync.Once
+	producerGone chan struct{}
+	consumerOnce sync.Once
+	consumerGone chan struct{}
 }
 
 // NewLogWatcher returns a new LogWatcher.
 func NewLogWatcher() *LogWatcher {
 	return &LogWatcher{
-		Msg:           make(chan *Message, logWatcherBufferSize),
-		Err:           make(chan error, 1),
-		closeNotifier: make(chan struct{}),
+		Msg:          make(chan *Message, logWatcherBufferSize),
+		Err:          make(chan error, 1),
+		producerGone: make(chan struct{}),
+		consumerGone: make(chan struct{}),
 	}
 }
 
-// Close notifies the underlying log reader to stop.
-func (w *LogWatcher) Close() {
+// ProducerGone notifies the underlying log reader that
+// the logs producer (a container) is gone.
+func (w *LogWatcher) ProducerGone() {
 	// only close if not already closed
-	w.closeOnce.Do(func() {
-		close(w.closeNotifier)
+	w.producerOnce.Do(func() {
+		close(w.producerGone)
 	})
 }
 
-// WatchClose returns a channel receiver that receives notification
-// when the watcher has been closed. This should only be called from
-// one goroutine.
-func (w *LogWatcher) WatchClose() <-chan struct{} {
-	return w.closeNotifier
+// WatchProducerGone returns a channel receiver that receives notification
+// once the logs producer (a container) is gone.
+func (w *LogWatcher) WatchProducerGone() <-chan struct{} {
+	return w.producerGone
+}
+
+// ConsumerGone notifies that the logs consumer is gone.
+func (w *LogWatcher) ConsumerGone() {
+	// only close if not already closed
+	w.consumerOnce.Do(func() {
+		close(w.consumerGone)
+	})
+}
+
+// WatchConsumerGone returns a channel receiver that receives notification
+// when the log watcher consumer is gone.
+func (w *LogWatcher) WatchConsumerGone() <-chan struct{} {
+	return w.consumerGone
 }
 
 // Capability defines the list of capabilities that a driver can implement

daemon/logger/loggerutils/logfile.go

@@ -488,7 +488,7 @@ func tailFiles(files []SizeReaderAt, watcher *logger.LogWatcher, createDecoder m
 	go func() {
 		select {
 		case <-ctx.Done():
-		case <-watcher.WatchClose():
+		case <-watcher.WatchConsumerGone():
 			cancel()
 		}
 	}()
@@ -546,22 +546,9 @@ func followLogs(f *os.File, logWatcher *logger.LogWatcher, notifyRotate chan int
 	}
 	defer func() {
 		f.Close()
-		fileWatcher.Remove(name)
 		fileWatcher.Close()
 	}()
 
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	go func() {
-		select {
-		case <-logWatcher.WatchClose():
-			fileWatcher.Remove(name)
-			cancel()
-		case <-ctx.Done():
-			return
-		}
-	}()
-
 	var retries int
 	handleRotate := func() error {
 		f.Close()
@@ -596,7 +583,9 @@ func followLogs(f *os.File, logWatcher *logger.LogWatcher, notifyRotate chan int
 			case fsnotify.Rename, fsnotify.Remove:
 				select {
 				case <-notifyRotate:
-				case <-ctx.Done():
+				case <-logWatcher.WatchProducerGone():
+					return errDone
+				case <-logWatcher.WatchConsumerGone():
 					return errDone
 				}
 				if err := handleRotate(); err != nil {
@@ -618,7 +607,9 @@ func followLogs(f *os.File, logWatcher *logger.LogWatcher, notifyRotate chan int
 				return errRetry
 			}
 			return err
-		case <-ctx.Done():
+		case <-logWatcher.WatchProducerGone():
+			return errDone
+		case <-logWatcher.WatchConsumerGone():
 			return errDone
 		}
 	}
@@ -664,23 +655,11 @@ func followLogs(f *os.File, logWatcher *logger.LogWatcher, notifyRotate chan int
 		if !until.IsZero() && msg.Timestamp.After(until) {
 			return
 		}
+		// send the message, unless the consumer is gone
 		select {
 		case logWatcher.Msg <- msg:
-		case <-ctx.Done():
-			logWatcher.Msg <- msg
-			for {
-				msg, err := decodeLogLine()
-				if err != nil {
-					return
-				}
-				if !since.IsZero() && msg.Timestamp.Before(since) {
-					continue
-				}
-				if !until.IsZero() && msg.Timestamp.After(until) {
-					return
-				}
-				logWatcher.Msg <- msg
-			}
+		case <-logWatcher.WatchConsumerGone():
+			return
 		}
 	}
 }

daemon/logger/loggerutils/logfile_test.go

@@ -4,6 +4,8 @@ import (
 	"bufio"
 	"context"
 	"io"
+	"io/ioutil"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -74,3 +76,128 @@ func TestTailFiles(t *testing.T) {
 		assert.Assert(t, string(msg.Line) == "Where we're going we don't need roads.", string(msg.Line))
 	}
 }
+
+func TestFollowLogsConsumerGone(t *testing.T) {
+	lw := logger.NewLogWatcher()
+
+	f, err := ioutil.TempFile("", t.Name())
+	assert.NilError(t, err)
+	defer func() {
+		f.Close()
+		os.Remove(f.Name())
+	}()
+
+	makeDecoder := func(rdr io.Reader) func() (*logger.Message, error) {
+		return func() (*logger.Message, error) {
+			return &logger.Message{}, nil
+		}
+	}
+
+	followLogsDone := make(chan struct{})
+	var since, until time.Time
+	go func() {
+		followLogs(f, lw, make(chan interface{}), makeDecoder, since, until)
+		close(followLogsDone)
+	}()
+
+	select {
+	case <-lw.Msg:
+	case err := <-lw.Err:
+		assert.NilError(t, err)
+	case <-followLogsDone:
+		t.Fatal("follow logs finished unexpectedly")
+	case <-time.After(10 * time.Second):
+		t.Fatal("timeout waiting for log message")
+	}
+
+	lw.ConsumerGone()
+	select {
+	case <-followLogsDone:
+	case <-time.After(20 * time.Second):
+		t.Fatal("timeout waiting for followLogs() to finish")
+	}
+}
+
+func TestFollowLogsProducerGone(t *testing.T) {
+	lw := logger.NewLogWatcher()
+
+	f, err := ioutil.TempFile("", t.Name())
+	assert.NilError(t, err)
+	defer os.Remove(f.Name())
+
+	var sent, received, closed int
+	makeDecoder := func(rdr io.Reader) func() (*logger.Message, error) {
+		return func() (*logger.Message, error) {
+			if closed == 1 {
+				closed++
+				t.Logf("logDecode() closed after sending %d messages\n", sent)
+				return nil, io.EOF
+			} else if closed > 1 {
+				t.Fatal("logDecode() called after closing!")
+				return nil, io.EOF
+			}
+			sent++
+			return &logger.Message{}, nil
+		}
+	}
+	var since, until time.Time
+
+	followLogsDone := make(chan struct{})
+	go func() {
+		followLogs(f, lw, make(chan interface{}), makeDecoder, since, until)
+		close(followLogsDone)
+	}()
+
+	// read 1 message
+	select {
+	case <-lw.Msg:
+		received++
+	case err := <-lw.Err:
+		assert.NilError(t, err)
+	case <-followLogsDone:
+		t.Fatal("followLogs() finished unexpectedly")
+	case <-time.After(10 * time.Second):
+		t.Fatal("timeout waiting for log message")
+	}
+
+	// "stop" the "container"
+	closed = 1
+	lw.ProducerGone()
+
+	// should receive all the messages sent
+	readDone := make(chan struct{})
+	go func() {
+		defer close(readDone)
+		for {
+			select {
+			case <-lw.Msg:
+				received++
+				if received == sent {
+					return
+				}
+			case err := <-lw.Err:
+				assert.NilError(t, err)
+			}
+		}
+	}()
+	select {
+	case <-readDone:
+	case <-time.After(30 * time.Second):
+		t.Fatalf("timeout waiting for log messages to be read (sent: %d, received: %d", sent, received)
+	}
+
+	t.Logf("messages sent: %d, received: %d", sent, received)
+
+	// followLogs() should be done by now
+	select {
+	case <-followLogsDone:
+	case <-time.After(30 * time.Second):
+		t.Fatal("timeout waiting for followLogs() to finish")
+	}
+
+	select {
+	case <-lw.WatchConsumerGone():
+		t.Fatal("consumer should not have exited")
+	default:
+	}
+}

daemon/logs.go

@@ -110,14 +110,16 @@ func (daemon *Daemon) ContainerLogs(ctx context.Context, containerName string, c
 				}
 			}()
 		}
-		// set up some defers
-		defer logs.Close()
+		// signal that the log reader is gone
+		defer logs.ConsumerGone()
 
 		// close the messages channel. closing is the only way to signal above
 		// that we're doing with logs (other than context cancel i guess).
 		defer close(messageChan)
 
 		lg.Debug("begin logs")
+		defer lg.Debugf("end logs (%v)", ctx.Err())
+
 		for {
 			select {
 			// i do not believe as the system is currently designed any error
@@ -132,14 +134,12 @@ func (daemon *Daemon) ContainerLogs(ctx context.Context, containerName string, c
 				}
 				return
 			case <-ctx.Done():
-				lg.Debugf("logs: end stream, ctx is done: %v", ctx.Err())
 				return
 			case msg, ok := <-logs.Msg:
 				// there is some kind of pool or ring buffer in the logger that
 				// produces these messages, and a possible future optimization
 				// might be to use that pool and reuse message objects
 				if !ok {
-					lg.Debug("end logs")
 					return
 				}
 				m := msg.AsLogMessage() // just a pointer conversion, does not copy data

daemon/reload.go

@@ -45,6 +45,7 @@ func (daemon *Daemon) Reload(conf *config.Config) (err error) {
 	daemon.reloadDebug(conf, attributes)
 	daemon.reloadMaxConcurrentDownloadsAndUploads(conf, attributes)
 	daemon.reloadShutdownTimeout(conf, attributes)
+	daemon.reloadFeatures(conf, attributes)
 
 	if err := daemon.reloadClusterDiscovery(conf, attributes); err != nil {
 		return err
@@ -322,3 +323,13 @@ func (daemon *Daemon) reloadNetworkDiagnosticPort(conf *config.Config, attribute
 
 	return nil
 }
+
+// reloadFeatures updates configuration with enabled/disabled features
+func (daemon *Daemon) reloadFeatures(conf *config.Config, attributes map[string]string) {
+	// update corresponding configuration
+	// note that we allow features option to be entirely unset
+	daemon.configStore.Features = conf.Features
+
+	// prepare reload event attributes with updatable configurations
+	attributes["features"] = fmt.Sprintf("%v", daemon.configStore.Features)
+}

daemon/trustkey.go

@@ -17,7 +17,7 @@ import (
 // TODO: this should use more of libtrust.LoadOrCreateTrustKey which may need
 // a refactor or this function to be moved into libtrust
 func loadOrCreateTrustKey(trustKeyPath string) (libtrust.PrivateKey, error) {
-	err := system.MkdirAll(filepath.Dir(trustKeyPath), 0700, "")
+	err := system.MkdirAll(filepath.Dir(trustKeyPath), 0755, "")
 	if err != nil {
 		return nil, err
 	}

daemon/volumes.go

@@ -210,6 +210,8 @@ func (daemon *Daemon) registerMountPoints(container *container.Container, hostCo
 			mp.Name = v.Name
 			mp.Driver = v.Driver
 
+			// need to selinux-relabel local mounts
+			mp.Source = v.Mountpoint
 			if mp.Driver == volume.DefaultDriverName {
 				setBindModeIfNull(mp)
 			}

dockerversion/version_lib.go

@@ -10,8 +10,6 @@ const (
 	Version               = "library-import"
 	BuildTime             = "library-import"
 	IAmStatic             = "library-import"
-	ContainerdCommitID    = "library-import"
-	RuncCommitID          = "library-import"
 	InitCommitID          = "library-import"
 	PlatformName          = ""
 	ProductName           = ""

hack/dockerfile/install/containerd.installer

@@ -30,7 +30,7 @@ install_containerd() {
 
 	mkdir -p ${PREFIX}
 
-	cp bin/containerd ${PREFIX}/docker-containerd
-	cp bin/containerd-shim ${PREFIX}/docker-containerd-shim
-	cp bin/ctr ${PREFIX}/docker-containerd-ctr
+	cp bin/containerd ${PREFIX}/containerd
+	cp bin/containerd-shim ${PREFIX}/containerd-shim
+	cp bin/ctr ${PREFIX}/ctr
 }

hack/dockerfile/install/proxy.installer

@@ -3,7 +3,7 @@
 # LIBNETWORK_COMMIT is used to build the docker-userland-proxy binary. When
 # updating the binary version, consider updating github.com/docker/libnetwork
 # in vendor.conf accordingly
-LIBNETWORK_COMMIT=f30a35b091cc2a431ef9856c75c343f75bb5f2e2
+LIBNETWORK_COMMIT=6da50d1978302f04c3e2089e29112ea24812f05b
 
 install_proxy() {
 	case "$1" in

hack/dockerfile/install/runc.installer

@@ -18,5 +18,5 @@ install_runc() {
 	fi
 	make BUILDTAGS="$RUNC_BUILDTAGS" "$target"
 	mkdir -p ${PREFIX}
-	cp runc ${PREFIX}/docker-runc
+	cp runc ${PREFIX}/runc
 }

hack/make/.binary-setup

@@ -1,9 +1,9 @@
 #!/usr/bin/env bash
 
 DOCKER_DAEMON_BINARY_NAME='dockerd'
-DOCKER_RUNC_BINARY_NAME='docker-runc'
-DOCKER_CONTAINERD_BINARY_NAME='docker-containerd'
-DOCKER_CONTAINERD_CTR_BINARY_NAME='docker-containerd-ctr'
-DOCKER_CONTAINERD_SHIM_BINARY_NAME='docker-containerd-shim'
+DOCKER_RUNC_BINARY_NAME='runc'
+DOCKER_CONTAINERD_BINARY_NAME='containerd'
+DOCKER_CONTAINERD_CTR_BINARY_NAME='ctr'
+DOCKER_CONTAINERD_SHIM_BINARY_NAME='containerd-shim'
 DOCKER_PROXY_BINARY_NAME='docker-proxy'
 DOCKER_INIT_BINARY_NAME='docker-init'

hack/make/.go-autogen

@@ -19,7 +19,6 @@ const (
 	Version               string = "$VERSION"
 	BuildTime             string = "$BUILDTIME"
 	IAmStatic             string = "${IAMSTATIC:-true}"
-	ContainerdCommitID    string = "${CONTAINERD_COMMIT}"
 	PlatformName          string = "${PLATFORM}"
 	ProductName           string = "${PRODUCT}"
 	DefaultProductLicense string = "${DEFAULT_PRODUCT_LICENSE}"
@@ -37,7 +36,6 @@ package dockerversion
 // Default build-time variable for library-import.
 // This file is overridden on build with build-time informations.
 const (
-	RuncCommitID string = "${RUNC_COMMIT}"
 	InitCommitID string = "${TINI_COMMIT}"
 )
 

hack/make/.integration-test-helpers

@@ -112,7 +112,7 @@ error_on_leaked_containerd_shims() {
 	fi
 
 	leftovers=$(ps -ax -o pid,cmd |
-	            awk '$2 == "docker-containerd-shim" && $4 ~ /.*\/bundles\/.*\/test-integration/ { print $1 }')
+	            awk '$2 == "containerd-shim" && $4 ~ /.*\/bundles\/.*\/test-integration/ { print $1 }')
 	if [ -n "$leftovers" ]; then
 		ps aux
 		kill -9 $leftovers 2> /dev/null

hack/make/binary-daemon

@@ -10,14 +10,14 @@ copy_binaries() {
 	if [ "$(go env GOOS)/$(go env GOARCH)" != "$(go env GOHOSTOS)/$(go env GOHOSTARCH)" ]; then
 		return
 	fi
-	if [ ! -x /usr/local/bin/docker-runc ]; then
+	if [ ! -x /usr/local/bin/runc ]; then
 		return
 	fi
 	echo "Copying nested executables into $dir"
-	for file in containerd containerd-shim containerd-ctr runc init proxy; do
-		cp -f `which "docker-$file"` "$dir/"
+	for file in containerd containerd-shim ctr runc docker-init docker-proxy; do
+		cp -f `which "$file"` "$dir/"
 		if [ "$hash" == "hash" ]; then
-			hash_files "$dir/docker-$file"
+			hash_files "$dir/$file"
 		fi
 	done
 }

integration-cli/check_test.go

@@ -32,7 +32,7 @@ const (
 	privateRegistryURL = registry.DefaultURL
 
 	// path to containerd's ctr binary
-	ctrBinary = "docker-containerd-ctr"
+	ctrBinary = "ctr"
 
 	// the docker daemon binary to use
 	dockerdBinary = "dockerd"

integration-cli/docker_api_containers_test.go

@@ -1759,7 +1759,7 @@ func (s *DockerSuite) TestContainersAPICreateMountsValidation(c *check.C) {
 					Target: destPath}}},
 			msg: "source path does not exist",
 			// FIXME(vdemeester) fails into e2e, migrate to integration/container anyway
-			// msg: "bind mount source path does not exist: " + notExistPath,
+			// msg: "source path does not exist: " + notExistPath,
 		},
 		{
 			config: containertypes.Config{

integration-cli/docker_cli_daemon_test.go

@@ -44,6 +44,8 @@ import (
 	"gotest.tools/icmd"
 )
 
+const containerdSocket = "/var/run/docker/containerd/containerd.sock"
+
 // TestLegacyDaemonCommand test starting docker daemon using "deprecated" docker daemon
 // command. Remove this test when we remove this.
 func (s *DockerDaemonSuite) TestLegacyDaemonCommand(c *check.C) {
@@ -1449,7 +1451,7 @@ func (s *DockerDaemonSuite) TestCleanupMountsAfterDaemonAndContainerKill(c *chec
 	c.Assert(d.Kill(), check.IsNil)
 
 	// kill the container
-	icmd.RunCommand(ctrBinary, "--address", "/var/run/docker/containerd/docker-containerd.sock",
+	icmd.RunCommand(ctrBinary, "--address", containerdSocket,
 		"--namespace", moby_daemon.ContainersNamespace, "tasks", "kill", id).Assert(c, icmd.Success)
 
 	// restart daemon.
@@ -1971,7 +1973,7 @@ func (s *DockerDaemonSuite) TestDaemonRestartWithKilledRunningContainer(t *check
 	}
 
 	// kill the container
-	icmd.RunCommand(ctrBinary, "--address", "/var/run/docker/containerd/docker-containerd.sock",
+	icmd.RunCommand(ctrBinary, "--address", containerdSocket,
 		"--namespace", moby_daemon.ContainersNamespace, "tasks", "kill", cid).Assert(t, icmd.Success)
 
 	// Give time to containerd to process the command if we don't
@@ -2074,7 +2076,7 @@ func (s *DockerDaemonSuite) TestDaemonRestartWithUnpausedRunningContainer(t *che
 	// resume the container
 	result := icmd.RunCommand(
 		ctrBinary,
-		"--address", "/var/run/docker/containerd/docker-containerd.sock",
+		"--address", containerdSocket,
 		"--namespace", moby_daemon.ContainersNamespace,
 		"tasks", "resume", cid)
 	result.Assert(t, icmd.Success)
@@ -2409,7 +2411,7 @@ func (s *DockerDaemonSuite) TestRunWithRuntimeFromConfigFile(c *check.C) {
 {
     "runtimes": {
         "oci": {
-            "path": "docker-runc"
+            "path": "runc"
         },
         "vm": {
             "path": "/usr/local/bin/vm-manager",
@@ -2491,7 +2493,7 @@ func (s *DockerDaemonSuite) TestRunWithRuntimeFromConfigFile(c *check.C) {
     "default-runtime": "vm",
     "runtimes": {
         "oci": {
-            "path": "docker-runc"
+            "path": "runc"
         },
         "vm": {
             "path": "/usr/local/bin/vm-manager",
@@ -2517,7 +2519,7 @@ func (s *DockerDaemonSuite) TestRunWithRuntimeFromConfigFile(c *check.C) {
 }
 
 func (s *DockerDaemonSuite) TestRunWithRuntimeFromCommandLine(c *check.C) {
-	s.d.StartWithBusybox(c, "--add-runtime", "oci=docker-runc", "--add-runtime", "vm=/usr/local/bin/vm-manager")
+	s.d.StartWithBusybox(c, "--add-runtime", "oci=runc", "--add-runtime", "vm=/usr/local/bin/vm-manager")
 
 	// Run with default runtime
 	out, err := s.d.Cmd("run", "--rm", "busybox", "ls")
@@ -2564,7 +2566,7 @@ func (s *DockerDaemonSuite) TestRunWithRuntimeFromCommandLine(c *check.C) {
 
 	// Check that we can select a default runtime
 	s.d.Stop(c)
-	s.d.StartWithBusybox(c, "--default-runtime=vm", "--add-runtime", "oci=docker-runc", "--add-runtime", "vm=/usr/local/bin/vm-manager")
+	s.d.StartWithBusybox(c, "--default-runtime=vm", "--add-runtime", "oci=runc", "--add-runtime", "vm=/usr/local/bin/vm-manager")
 
 	out, err = s.d.Cmd("run", "--rm", "busybox", "ls")
 	c.Assert(err, check.NotNil, check.Commentf("%s", out))

integration/build/build_session_test.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/docker/docker/api/types"
 	dclient "github.com/docker/docker/client"
 	"github.com/docker/docker/internal/test/fakecontext"
 	"github.com/docker/docker/internal/test/request"
@@ -76,7 +77,7 @@ func TestBuildWithSession(t *testing.T) {
 	assert.Check(t, is.Contains(string(outBytes), "Successfully built"))
 	assert.Check(t, is.Equal(strings.Count(string(outBytes), "Using cache"), 4))
 
-	_, err = client.BuildCachePrune(context.TODO())
+	_, err = client.BuildCachePrune(context.TODO(), types.BuildCachePruneOptions{All: true})
 	assert.Check(t, err)
 
 	du, err = client.DiskUsage(context.TODO())

integration/build/build_test.go

@@ -423,6 +423,38 @@ RUN [ ! -f foo ]
 	assert.Check(t, is.Contains(out.String(), "Successfully built"))
 }
 
+// #37581
+func TestBuildWithHugeFile(t *testing.T) {
+	ctx := context.TODO()
+	defer setupTest(t)()
+
+	dockerfile := `FROM busybox
+# create a sparse file with size over 8GB
+RUN for g in $(seq 0 8); do dd if=/dev/urandom of=rnd bs=1K count=1 seek=$((1024*1024*g)) status=none; done && \
+    ls -la rnd && du -sk rnd`
+
+	buf := bytes.NewBuffer(nil)
+	w := tar.NewWriter(buf)
+	writeTarRecord(t, w, "Dockerfile", dockerfile)
+	err := w.Close()
+	assert.NilError(t, err)
+
+	apiclient := testEnv.APIClient()
+	resp, err := apiclient.ImageBuild(ctx,
+		buf,
+		types.ImageBuildOptions{
+			Remove:      true,
+			ForceRemove: true,
+		})
+
+	out := bytes.NewBuffer(nil)
+	assert.NilError(t, err)
+	_, err = io.Copy(out, resp.Body)
+	resp.Body.Close()
+	assert.NilError(t, err)
+	assert.Check(t, is.Contains(out.String(), "Successfully built"))
+}
+
 func writeTarRecord(t *testing.T, w *tar.Writer, fn, contents string) {
 	err := w.WriteHeader(&tar.Header{
 		Name:     fn,

integration/container/exec_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"io/ioutil"
 	"testing"
+	"time"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/strslice"
@@ -15,6 +16,74 @@ import (
 	"gotest.tools/skip"
 )
 
+// TestExecWithCloseStdin adds case for moby#37870 issue.
+func TestExecWithCloseStdin(t *testing.T) {
+	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.39"), "broken in earlier versions")
+	defer setupTest(t)()
+
+	ctx := context.Background()
+	client := request.NewAPIClient(t)
+
+	// run top with detached mode
+	cID := container.Run(t, ctx, client)
+
+	expected := "closeIO"
+	execResp, err := client.ContainerExecCreate(ctx, cID,
+		types.ExecConfig{
+			AttachStdin:  true,
+			AttachStdout: true,
+			Cmd:          strslice.StrSlice([]string{"sh", "-c", "cat && echo " + expected}),
+		},
+	)
+	assert.NilError(t, err)
+
+	resp, err := client.ContainerExecAttach(ctx, execResp.ID,
+		types.ExecStartCheck{
+			Detach: false,
+			Tty:    false,
+		},
+	)
+	assert.NilError(t, err)
+	defer resp.Close()
+
+	// close stdin to send EOF to cat
+	assert.NilError(t, resp.CloseWrite())
+
+	var (
+		waitCh = make(chan struct{})
+		resCh  = make(chan struct {
+			content string
+			err     error
+		})
+	)
+
+	go func() {
+		close(waitCh)
+		defer close(resCh)
+		r, err := ioutil.ReadAll(resp.Reader)
+
+		resCh <- struct {
+			content string
+			err     error
+		}{
+			content: string(r),
+			err:     err,
+		}
+	}()
+
+	<-waitCh
+	select {
+	case <-time.After(3 * time.Second):
+		t.Fatal("failed to read the content in time")
+	case got := <-resCh:
+		assert.NilError(t, got.err)
+
+		// NOTE: using Contains because no-tty's stream contains UX information
+		// like size, stream type.
+		assert.Assert(t, is.Contains(got.content, expected))
+	}
+}
+
 func TestExec(t *testing.T) {
 	skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.35"), "broken in earlier versions")
 	defer setupTest(t)()

integration/network/service_test.go

@@ -345,6 +345,8 @@ func TestServiceWithDefaultAddressPoolInit(t *testing.T) {
 
 	out, err := cli.NetworkInspect(context.Background(), name, types.NetworkInspectOptions{})
 	assert.NilError(t, err)
+	t.Logf("%s: NetworkInspect: %+v", t.Name(), out)
+	assert.Assert(t, len(out.IPAM.Config) > 0)
 	assert.Equal(t, out.IPAM.Config[0].Subnet, "20.20.0.0/24")
 
 	err = cli.ServiceRemove(context.Background(), serviceID)
@@ -353,7 +355,7 @@ func TestServiceWithDefaultAddressPoolInit(t *testing.T) {
 	d.Stop(t)
 
 	// Clean up , set it back to original one to make sure other tests don't fail
-	ipAddr = []string{"10.10.0.0/8"}
+	ipAddr = []string{"10.0.0.0/8"}
 	ops = append(ops, daemon.WithSwarmDefaultAddrPool(ipAddr))
 	ops = append(ops, daemon.WithSwarmDefaultAddrPoolSubnetSize(24))
 	d = swarm.NewSwarm(t, testEnv, ops...)

internal/test/daemon/daemon.go

@@ -38,6 +38,7 @@ type logT interface {
 }
 
 const defaultDockerdBinary = "dockerd"
+const containerdSocket = "/var/run/docker/containerd/containerd.sock"
 
 var errDaemonNotStarted = errors.New("daemon not started")
 
@@ -224,7 +225,7 @@ func (d *Daemon) StartWithLogFile(out *os.File, providedArgs ...string) error {
 		return errors.Wrapf(err, "[%s] could not find docker binary in $PATH", d.id)
 	}
 	args := append(d.GlobalFlags,
-		"--containerd", "/var/run/docker/containerd/docker-containerd.sock",
+		"--containerd", containerdSocket,
 		"--data-root", d.Root,
 		"--exec-root", d.execRoot,
 		"--pidfile", fmt.Sprintf("%s/docker.pid", d.Folder),

libcontainerd/client_daemon.go

@@ -328,6 +328,13 @@ func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin
 	return int(t.Pid()), nil
 }
 
+// Exec creates exec process.
+//
+// The containerd client calls Exec to register the exec config in the shim side.
+// When the client calls Start, the shim will create stdin fifo if needs. But
+// for the container main process, the stdin fifo will be created in Create not
+// the Start call. stdinCloseSync channel should be closed after Start exec
+// process.
 func (c *client) Exec(ctx context.Context, containerID, processID string, spec *specs.Process, withStdin bool, attachStdio StdioCallback) (int, error) {
 	ctr := c.getContainer(containerID)
 	if ctr == nil {
@@ -372,7 +379,9 @@ func (c *client) Exec(ctx context.Context, containerID, processID string, spec *
 	ctr.addProcess(processID, p)
 
 	// Signal c.createIO that it can call CloseIO
-	close(stdinCloseSync)
+	//
+	// the stdin of exec process will be created after p.Start in containerd
+	defer close(stdinCloseSync)
 
 	if err = p.Start(ctx); err != nil {
 		p.Delete(context.Background())

libcontainerd/supervisor/remote_daemon.go

@@ -27,8 +27,8 @@ const (
 	shutdownTimeout         = 15 * time.Second
 	startupTimeout          = 15 * time.Second
 	configFile              = "containerd.toml"
-	binaryName              = "docker-containerd"
-	pidFile                 = "docker-containerd.pid"
+	binaryName              = "containerd"
+	pidFile                 = "containerd.pid"
 )
 
 type pluginConfigs struct {
@@ -43,7 +43,7 @@ type remote struct {
 	logger    *logrus.Entry
 
 	daemonWaitCh  chan struct{}
-	daemonStartCh chan struct{}
+	daemonStartCh chan error
 	daemonStopCh  chan struct{}
 
 	rootDir     string
@@ -72,7 +72,7 @@ func Start(ctx context.Context, rootDir, stateDir string, opts ...DaemonOpt) (Da
 		pluginConfs:   pluginConfigs{make(map[string]interface{})},
 		daemonPid:     -1,
 		logger:        logrus.WithField("module", "libcontainerd"),
-		daemonStartCh: make(chan struct{}),
+		daemonStartCh: make(chan error, 1),
 		daemonStopCh:  make(chan struct{}),
 	}
 
@@ -92,7 +92,10 @@ func Start(ctx context.Context, rootDir, stateDir string, opts ...DaemonOpt) (Da
 	select {
 	case <-time.After(startupTimeout):
 		return nil, errors.New("timeout waiting for containerd to start")
-	case <-r.daemonStartCh:
+	case err := <-r.daemonStartCh:
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	return r, nil
@@ -245,25 +248,35 @@ func (r *remote) monitorDaemon(ctx context.Context) {
 	}()
 
 	for {
-		select {
-		case <-ctx.Done():
-			r.logger.Info("stopping healthcheck following graceful shutdown")
-			if client != nil {
-				client.Close()
+		if delay != nil {
+			select {
+			case <-ctx.Done():
+				r.logger.Info("stopping healthcheck following graceful shutdown")
+				if client != nil {
+					client.Close()
+				}
+				return
+			case <-delay:
 			}
-			return
-		case <-delay:
-		default:
 		}
 
 		if r.daemonPid == -1 {
 			if r.daemonWaitCh != nil {
-				<-r.daemonWaitCh
+				select {
+				case <-ctx.Done():
+					r.logger.Info("stopping containerd startup following graceful shutdown")
+					return
+				case <-r.daemonWaitCh:
+				}
 			}
 
 			os.RemoveAll(r.GRPC.Address)
 			if err := r.startContainerd(); err != nil {
-				r.logger.WithError(err).Error("failed starting containerd")
+				if !started {
+					r.daemonStartCh <- err
+					return
+				}
+				r.logger.WithError(err).Error("failed restarting containerd")
 				delay = time.After(50 * time.Millisecond)
 				continue
 			}
@@ -276,26 +289,28 @@ func (r *remote) monitorDaemon(ctx context.Context) {
 			}
 		}
 
-		tctx, cancel := context.WithTimeout(ctx, healthCheckTimeout)
-		_, err := client.IsServing(tctx)
-		cancel()
-		if err == nil {
-			if !started {
-				close(r.daemonStartCh)
-				started = true
+		if client != nil {
+			tctx, cancel := context.WithTimeout(ctx, healthCheckTimeout)
+			_, err := client.IsServing(tctx)
+			cancel()
+			if err == nil {
+				if !started {
+					close(r.daemonStartCh)
+					started = true
+				}
+
+				transientFailureCount = 0
+				delay = time.After(500 * time.Millisecond)
+				continue
 			}
 
-			transientFailureCount = 0
-			delay = time.After(500 * time.Millisecond)
-			continue
-		}
+			r.logger.WithError(err).WithField("binary", binaryName).Debug("daemon is not responding")
 
-		r.logger.WithError(err).WithField("binary", binaryName).Debug("daemon is not responding")
-
-		transientFailureCount++
-		if transientFailureCount < maxConnectionRetryCount || system.IsProcessAlive(r.daemonPid) {
-			delay = time.After(time.Duration(transientFailureCount) * 200 * time.Millisecond)
-			continue
+			transientFailureCount++
+			if transientFailureCount < maxConnectionRetryCount || system.IsProcessAlive(r.daemonPid) {
+				delay = time.After(time.Duration(transientFailureCount) * 200 * time.Millisecond)
+				continue
+			}
 		}
 
 		if system.IsProcessAlive(r.daemonPid) {
@@ -304,6 +319,7 @@ func (r *remote) monitorDaemon(ctx context.Context) {
 		}
 
 		client.Close()
+		client = nil
 		r.daemonPid = -1
 		delay = nil
 		transientFailureCount = 0

libcontainerd/supervisor/remote_daemon_linux.go

@@ -11,8 +11,8 @@ import (
 )
 
 const (
-	sockFile      = "docker-containerd.sock"
-	debugSockFile = "docker-containerd-debug.sock"
+	sockFile      = "containerd.sock"
+	debugSockFile = "containerd-debug.sock"
 )
 
 func (r *remote) setDefaults() {

libcontainerd/supervisor/remote_daemon_windows.go

@@ -7,8 +7,8 @@ import (
 )
 
 const (
-	grpcPipeName  = `\\.\pipe\docker-containerd-containerd`
-	debugPipeName = `\\.\pipe\docker-containerd-debug`
+	grpcPipeName  = `\\.\pipe\containerd-containerd`
+	debugPipeName = `\\.\pipe\containerd-debug`
 )
 
 func (r *remote) setDefaults() {

pkg/filenotify/poller.go

@@ -54,6 +54,7 @@ func (w *filePoller) Add(name string) error {
 	}
 	fi, err := os.Stat(name)
 	if err != nil {
+		f.Close()
 		return err
 	}
 
@@ -61,6 +62,7 @@ func (w *filePoller) Add(name string) error {
 		w.watches = make(map[string]chan struct{})
 	}
 	if _, exists := w.watches[name]; exists {
+		f.Close()
 		return fmt.Errorf("watch exists")
 	}
 	chClose := make(chan struct{})
@@ -113,11 +115,10 @@ func (w *filePoller) Close() error {
 		return nil
 	}
 
-	w.closed = true
 	for name := range w.watches {
 		w.remove(name)
-		delete(w.watches, name)
 	}
+	w.closed = true
 	return nil
 }
 
@@ -146,12 +147,11 @@ func (w *filePoller) sendErr(e error, chClose <-chan struct{}) error {
 func (w *filePoller) watch(f *os.File, lastFi os.FileInfo, chClose chan struct{}) {
 	defer f.Close()
 	for {
-		time.Sleep(watchWaitTime)
 		select {
+		case <-time.After(watchWaitTime):
 		case <-chClose:
 			logrus.Debugf("watch for %s closed", f.Name())
 			return
-		default:
 		}
 
 		fi, err := os.Stat(f.Name())

pkg/mount/sharedsubtree_linux.go

@@ -48,18 +48,22 @@ func MakeRUnbindable(mountPoint string) error {
 	return ensureMountedAs(mountPoint, "runbindable")
 }
 
-func ensureMountedAs(mountPoint, options string) error {
-	mounted, err := Mounted(mountPoint)
+// MakeMount ensures that the file or directory given is a mount point,
+// bind mounting it to itself it case it is not.
+func MakeMount(mnt string) error {
+	mounted, err := Mounted(mnt)
 	if err != nil {
 		return err
 	}
-
-	if !mounted {
-		if err := Mount(mountPoint, mountPoint, "none", "bind,rw"); err != nil {
-			return err
-		}
+	if mounted {
+		return nil
 	}
-	if _, err = Mounted(mountPoint); err != nil {
+
+	return Mount(mnt, mnt, "none", "bind")
+}
+
+func ensureMountedAs(mountPoint, options string) error {
+	if err := MakeMount(mountPoint); err != nil {
 		return err
 	}
 

pkg/parsers/parsers.go

@@ -18,6 +18,24 @@ func ParseKeyValueOpt(opt string) (string, string, error) {
 	return strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]), nil
 }
 
+// ParseUintListMaximum parses and validates the specified string as the value
+// found in some cgroup file (e.g. `cpuset.cpus`, `cpuset.mems`), which could be
+// one of the formats below. Note that duplicates are actually allowed in the
+// input string. It returns a `map[int]bool` with available elements from `val`
+// set to `true`. Values larger than `maximum` cause an error if max is non zero,
+// in order to stop the map becoming excessively large.
+// Supported formats:
+//     7
+//     1-6
+//     0,3-4,7,8-10
+//     0-0,0,1-7
+//     03,1-3      <- this is gonna get parsed as [1,2,3]
+//     3,2,1
+//     0-2,3,1
+func ParseUintListMaximum(val string, maximum int) (map[int]bool, error) {
+	return parseUintList(val, maximum)
+}
+
 // ParseUintList parses and validates the specified string as the value
 // found in some cgroup file (e.g. `cpuset.cpus`, `cpuset.mems`), which could be
 // one of the formats below. Note that duplicates are actually allowed in the
@@ -32,6 +50,10 @@ func ParseKeyValueOpt(opt string) (string, string, error) {
 //     3,2,1
 //     0-2,3,1
 func ParseUintList(val string) (map[int]bool, error) {
+	return parseUintList(val, 0)
+}
+
+func parseUintList(val string, maximum int) (map[int]bool, error) {
 	if val == "" {
 		return map[int]bool{}, nil
 	}
@@ -46,6 +68,9 @@ func ParseUintList(val string) (map[int]bool, error) {
 			if err != nil {
 				return nil, errInvalidFormat
 			}
+			if maximum != 0 && v > maximum {
+				return nil, fmt.Errorf("value of out range, maximum is %d", maximum)
+			}
 			availableInts[v] = true
 		} else {
 			split := strings.SplitN(r, "-", 2)
@@ -60,6 +85,9 @@ func ParseUintList(val string) (map[int]bool, error) {
 			if max < min {
 				return nil, errInvalidFormat
 			}
+			if maximum != 0 && max > maximum {
+				return nil, fmt.Errorf("value of out range, maximum is %d", maximum)
+			}
 			for i := min; i <= max; i++ {
 				availableInts[i] = true
 			}

pkg/parsers/parsers_test.go

@@ -68,3 +68,16 @@ func TestParseUintList(t *testing.T) {
 		}
 	}
 }
+
+func TestParseUintListMaximumLimits(t *testing.T) {
+	v := "10,1000"
+	if _, err := ParseUintListMaximum(v, 0); err != nil {
+		t.Fatalf("Expected not to fail, got %v", err)
+	}
+	if _, err := ParseUintListMaximum(v, 1000); err != nil {
+		t.Fatalf("Expected not to fail, got %v", err)
+	}
+	if out, err := ParseUintListMaximum(v, 100); err == nil {
+		t.Fatalf("Expected failure with %s but got %v", v, out)
+	}
+}

pkg/progress/progress.go

@@ -39,6 +39,10 @@ type Output interface {
 type chanOutput chan<- Progress
 
 func (out chanOutput) WriteProgress(p Progress) error {
+	// FIXME: workaround for panic in #37735
+	defer func() {
+		recover()
+	}()
 	out <- p
 	return nil
 }

pkg/sysinfo/sysinfo.go

@@ -117,11 +117,19 @@ func (c cgroupCpusetInfo) IsCpusetMemsAvailable(provided string) (bool, error) {
 }
 
 func isCpusetListAvailable(provided, available string) (bool, error) {
-	parsedProvided, err := parsers.ParseUintList(provided)
+	parsedAvailable, err := parsers.ParseUintList(available)
 	if err != nil {
 		return false, err
 	}
-	parsedAvailable, err := parsers.ParseUintList(available)
+	// 8192 is the normal maximum number of CPUs in Linux, so accept numbers up to this
+	// or more if we actually have more CPUs.
+	max := 8192
+	for m := range parsedAvailable {
+		if m > max {
+			max = m
+		}
+	}
+	parsedProvided, err := parsers.ParseUintListMaximum(provided, max)
 	if err != nil {
 		return false, err
 	}

profiles/seccomp/default.json

@@ -329,7 +329,6 @@
 				"sync_file_range",
 				"syncfs",
 				"sysinfo",
-				"syslog",
 				"tee",
 				"tgkill",
 				"time",
@@ -561,6 +560,7 @@
 				"setdomainname",
 				"sethostname",
 				"setns",
+				"syslog",
 				"umount",
 				"umount2",
 				"unshare"
@@ -762,6 +762,20 @@
 				]
 			},
 			"excludes": {}
+		},
+		{
+			"names": [
+				"syslog"
+			],
+			"action": "SCMP_ACT_ALLOW",
+			"args": [],
+			"comment": "",
+			"includes": {
+				"caps": [
+					"CAP_SYSLOG"
+				]
+			},
+			"excludes": {}
 		}
 	]
 }

profiles/seccomp/seccomp_default.go

@@ -322,7 +322,6 @@ func DefaultProfile() *types.Seccomp {
 				"sync_file_range",
 				"syncfs",
 				"sysinfo",
-				"syslog",
 				"tee",
 				"tgkill",
 				"time",
@@ -492,6 +491,7 @@ func DefaultProfile() *types.Seccomp {
 				"setdomainname",
 				"sethostname",
 				"setns",
+				"syslog",
 				"umount",
 				"umount2",
 				"unshare",
@@ -642,6 +642,16 @@ func DefaultProfile() *types.Seccomp {
 				Caps: []string{"CAP_SYS_NICE"},
 			},
 		},
+		{
+			Names: []string{
+				"syslog",
+			},
+			Action: types.ActAllow,
+			Args:   []*types.Arg{},
+			Includes: types.Filter{
+				Caps: []string{"CAP_SYSLOG"},
+			},
+		},
 	}
 
 	return &types.Seccomp{

vendor.conf

@@ -1,7 +1,7 @@
 # the following lines are in sorted order, FYI
 github.com/Azure/go-ansiterm d6e3b3328b783f23731bc4d058875b0371ff8109
-github.com/Microsoft/hcsshim v0.6.12
-github.com/Microsoft/go-winio v0.4.9
+github.com/Microsoft/hcsshim v0.7.6
+github.com/Microsoft/go-winio v0.4.11
 github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a
 github.com/go-check/check 4ed411733c5785b40214c70bce814c3a3a689609 https://github.com/cpuguy83/check.git
 github.com/golang/gddo 9b12a26f3fbd7397dee4e20939ddca719d840d2a
@@ -26,8 +26,8 @@ github.com/imdario/mergo v0.3.6
 golang.org/x/sync 1d60e4601c6fd243af51cc01ddf169918a5407ca
 
 # buildkit
-github.com/moby/buildkit 49906c62925ed429ec9174a0b6869982967f1a39
-github.com/tonistiigi/fsutil b19464cd1b6a00773b4f2eb7acf9c30426f9df42
+github.com/moby/buildkit c7bb575343df0cbfeab8b5b28149630b8153fcc6
+github.com/tonistiigi/fsutil f567071bed2416e4d87d260d3162722651182317
 github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746
 github.com/opentracing/opentracing-go 1361b9cd60be79c4c3a7fa9841b3c132e40066a7
 github.com/google/shlex 6f45313302b9c56850fc17f99e40caebce98c716
@@ -37,7 +37,7 @@ github.com/mitchellh/hashstructure 2bca23e0e452137f789efbc8610126fd8b94f73b
 #get libnetwork packages
 
 # When updating, also update LIBNETWORK_COMMIT in hack/dockerfile/install/proxy accordingly
-github.com/docker/libnetwork a79d3687931697244b8e03485bf7b2042f8ec6b6
+github.com/docker/libnetwork 6da50d1978302f04c3e2089e29112ea24812f05b
 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9
 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec
@@ -47,7 +47,7 @@ github.com/sean-/seed e2103e2c35297fb7e17febb81e49b312087a2372
 github.com/hashicorp/go-sockaddr 6d291a969b86c4b633730bfc6b8b9d64c3aafed9
 github.com/hashicorp/go-multierror fcdddc395df1ddf4247c69bd436e84cfa0733f7e
 github.com/hashicorp/serf 598c54895cc5a7b1a24a398d635e8c0ea0959870
-github.com/docker/libkv 1d8431073ae03cdaedb198a89722f3aab6d418ef
+github.com/docker/libkv 458977154600b9f23984d9f4b82e79570b5ae12b
 github.com/vishvananda/netns 604eaf189ee867d8c147fafc28def2394e878d25
 github.com/vishvananda/netlink b2de5d10e38ecce8607e6b438b6d174f389a004e
 
@@ -59,13 +59,13 @@ github.com/coreos/etcd v3.2.1
 github.com/coreos/go-semver v0.2.0
 github.com/ugorji/go f1f1a805ed361a0e078bb537e4ea78cd37dcf065
 github.com/hashicorp/consul v0.5.2
-github.com/boltdb/bolt fff57c100f4dea1905678da7e90d92429dff2904
 github.com/miekg/dns v1.0.7
 github.com/ishidawataru/sctp 07191f837fedd2f13d1ec7b5f885f0f3ec54b1cb
+go.etcd.io/bbolt v1.3.1-etcd.8
 
 # get graph and distribution packages
 github.com/docker/distribution 83389a148052d74ac602f5f1d62f86ff2f3c4aa5
-github.com/vbatts/tar-split v0.10.2
+github.com/vbatts/tar-split v0.11.0
 github.com/opencontainers/go-digest v1.0.0-rc1
 
 # get go-zfs packages
@@ -75,8 +75,8 @@ github.com/pborman/uuid v1.0
 google.golang.org/grpc v1.12.0
 
 # This does not need to match RUNC_COMMIT as it is used for helper packages but should be newer or equal
-github.com/opencontainers/runc ad0f5255060d36872be04de22f8731f38ef2d7b1
-github.com/opencontainers/runtime-spec d810dbc60d8c5aeeb3d054bd1132fab2121968ce # v1.0.1-43-gd810dbc
+github.com/opencontainers/runc 00dc70017d222b178a002ed30e9321b12647af2d
+github.com/opencontainers/runtime-spec eba862dc2470385a233c7507392675cbeadf7353 # v1.0.1-45-geba862d
 github.com/opencontainers/image-spec v1.0.1
 github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0
 
@@ -114,23 +114,24 @@ github.com/googleapis/gax-go v2.0.0
 google.golang.org/genproto 694d95ba50e67b2e363f3483057db5d4910c18f9
 
 # containerd
-github.com/containerd/containerd v1.2.0-beta.0
+github.com/containerd/containerd 0c5f8f63c3368856c320ae8a1c125e703b73b51d # v1.2.0-rc.1
 github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c
-github.com/containerd/continuity d3c23511c1bf5851696cba83143d9cbcd666869b
+github.com/containerd/continuity bd77b46c8352f74eb12c85bdc01f4b90f69d66b4
 github.com/containerd/cgroups 5e610833b72089b37d0e615de9a92dfc043757c2
-github.com/containerd/console 4d8a41f4ce5b9bae77c41786ea2458330f43f081
-github.com/containerd/go-runc edcf3de1f4971445c42d61f20d506b30612aa031
+github.com/containerd/console c12b1e7919c14469339a5d38f2f8ed9b64a9de23
+github.com/containerd/cri 9f39e3289533fc228c5e5fcac0a6dbdd60c6047b # release/1.2 branch
+github.com/containerd/go-runc 5a6d9f37cfa36b15efba46dc7ea349fa9b7143c3
 github.com/containerd/typeurl a93fcdb778cd272c6e9b3028b2f42d813e785d40
-github.com/containerd/ttrpc 94dde388801693c54f88a6596f713b51a8b30b2d
+github.com/containerd/ttrpc 2a805f71863501300ae1976d29f0454ae003e85a
 github.com/gogo/googleapis 08a7655d27152912db7aaf4f983275eaf8d128ef
 
 # cluster
-github.com/docker/swarmkit cfa742c8abe6f8e922f6e4e920153c408e7d9c3b
+github.com/docker/swarmkit 6186e40fb04a7681e25a9101dbc7418c37ef0c8b # bump_v18.09 branch
 github.com/gogo/protobuf v1.0.0
 github.com/cloudflare/cfssl 1.3.2
 github.com/fernet/fernet-go 1b2437bc582b3cfbb341ee5a29f8ef5b42912ff2
 github.com/google/certificate-transparency-go v1.0.20
-golang.org/x/crypto a2144134853fc9a27a7b1e3eb4f19f1a76df13c9
+golang.org/x/crypto 0709b304e793a5edb4a2c0145f281ecdc20838a4
 golang.org/x/time fbb02b2291d28baffd63558aa44b4b56f178d650
 github.com/hashicorp/go-memdb cb9a474f84cc5e41b273b20c6927680b2a8776ad
 github.com/hashicorp/go-immutable-radix 826af9ccf0feeee615d546d69b11f8e98da8c8f1 git://github.com/tonistiigi/go-immutable-radix.git
@@ -143,8 +144,8 @@ github.com/prometheus/client_model fa8ad6fec33561be4280a8f0514318c79d7f6cb6
 github.com/prometheus/common ebdfc6da46522d58825777cf1f90490a5b1ef1d8
 github.com/prometheus/procfs abf152e5f3e97f2fafac028d2cc06c1feb87ffa5
 github.com/matttproud/golang_protobuf_extensions v1.0.0
-github.com/pkg/errors 839d9e913e063e28dfd0e6c7b7512793e0a48be9
-github.com/grpc-ecosystem/go-grpc-prometheus 6b7015e65d366bf3f19b2b2a000a831940f0f7e0
+github.com/pkg/errors 645ef00459ed84a119197bfb8d8205042c6df63d # v0.8.0
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 
 # cli
 github.com/spf13/cobra v0.0.3
@@ -155,7 +156,7 @@ github.com/Nvveen/Gotty a8b993ba6abdb0e0c12b0125c603323a71c7790c https://github.
 # metrics
 github.com/docker/go-metrics d466d4f6fd960e01820085bd7e1a24426ee7ef18
 
-github.com/opencontainers/selinux b29023b86e4a69d1b46b7e7b4e2b6fda03f0b9cd
+github.com/opencontainers/selinux b6fa367ed7f534f9ba25391cc2d467085dbb445a
 
 
 # archive/tar (for Go 1.10, see https://github.com/golang/go/issues/24787)

vendor/github.com/Microsoft/go-winio/backuptar/tar.go

@@ -303,7 +303,7 @@ func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *win
 		if err != nil {
 			return "", 0, nil, err
 		}
-		fileInfo.FileAttributes = uintptr(attr)
+		fileInfo.FileAttributes = uint32(attr)
 	} else {
 		if hdr.Typeflag == tar.TypeDir {
 			fileInfo.FileAttributes |= syscall.FILE_ATTRIBUTE_DIRECTORY

vendor/github.com/Microsoft/go-winio/ea.go

@@ -1,137 +1,137 @@
-package winio
-
-import (
-	"bytes"
-	"encoding/binary"
-	"errors"
-)
-
-type fileFullEaInformation struct {
-	NextEntryOffset uint32
-	Flags           uint8
-	NameLength      uint8
-	ValueLength     uint16
-}
-
-var (
-	fileFullEaInformationSize = binary.Size(&fileFullEaInformation{})
-
-	errInvalidEaBuffer = errors.New("invalid extended attribute buffer")
-	errEaNameTooLarge  = errors.New("extended attribute name too large")
-	errEaValueTooLarge = errors.New("extended attribute value too large")
-)
-
-// ExtendedAttribute represents a single Windows EA.
-type ExtendedAttribute struct {
-	Name  string
-	Value []byte
-	Flags uint8
-}
-
-func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) {
-	var info fileFullEaInformation
-	err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info)
-	if err != nil {
-		err = errInvalidEaBuffer
-		return
-	}
-
-	nameOffset := fileFullEaInformationSize
-	nameLen := int(info.NameLength)
-	valueOffset := nameOffset + int(info.NameLength) + 1
-	valueLen := int(info.ValueLength)
-	nextOffset := int(info.NextEntryOffset)
-	if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) {
-		err = errInvalidEaBuffer
-		return
-	}
-
-	ea.Name = string(b[nameOffset : nameOffset+nameLen])
-	ea.Value = b[valueOffset : valueOffset+valueLen]
-	ea.Flags = info.Flags
-	if info.NextEntryOffset != 0 {
-		nb = b[info.NextEntryOffset:]
-	}
-	return
-}
-
-// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION
-// buffer retrieved from BackupRead, ZwQueryEaFile, etc.
-func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) {
-	for len(b) != 0 {
-		ea, nb, err := parseEa(b)
-		if err != nil {
-			return nil, err
-		}
-
-		eas = append(eas, ea)
-		b = nb
-	}
-	return
-}
-
-func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error {
-	if int(uint8(len(ea.Name))) != len(ea.Name) {
-		return errEaNameTooLarge
-	}
-	if int(uint16(len(ea.Value))) != len(ea.Value) {
-		return errEaValueTooLarge
-	}
-	entrySize := uint32(fileFullEaInformationSize + len(ea.Name) + 1 + len(ea.Value))
-	withPadding := (entrySize + 3) &^ 3
-	nextOffset := uint32(0)
-	if !last {
-		nextOffset = withPadding
-	}
-	info := fileFullEaInformation{
-		NextEntryOffset: nextOffset,
-		Flags:           ea.Flags,
-		NameLength:      uint8(len(ea.Name)),
-		ValueLength:     uint16(len(ea.Value)),
-	}
-
-	err := binary.Write(buf, binary.LittleEndian, &info)
-	if err != nil {
-		return err
-	}
-
-	_, err = buf.Write([]byte(ea.Name))
-	if err != nil {
-		return err
-	}
-
-	err = buf.WriteByte(0)
-	if err != nil {
-		return err
-	}
-
-	_, err = buf.Write(ea.Value)
-	if err != nil {
-		return err
-	}
-
-	_, err = buf.Write([]byte{0, 0, 0}[0 : withPadding-entrySize])
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// EncodeExtendedAttributes encodes a list of EAs into a FILE_FULL_EA_INFORMATION
-// buffer for use with BackupWrite, ZwSetEaFile, etc.
-func EncodeExtendedAttributes(eas []ExtendedAttribute) ([]byte, error) {
-	var buf bytes.Buffer
-	for i := range eas {
-		last := false
-		if i == len(eas)-1 {
-			last = true
-		}
-
-		err := writeEa(&buf, &eas[i], last)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return buf.Bytes(), nil
-}
+package winio
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+)
+
+type fileFullEaInformation struct {
+	NextEntryOffset uint32
+	Flags           uint8
+	NameLength      uint8
+	ValueLength     uint16
+}
+
+var (
+	fileFullEaInformationSize = binary.Size(&fileFullEaInformation{})
+
+	errInvalidEaBuffer = errors.New("invalid extended attribute buffer")
+	errEaNameTooLarge  = errors.New("extended attribute name too large")
+	errEaValueTooLarge = errors.New("extended attribute value too large")
+)
+
+// ExtendedAttribute represents a single Windows EA.
+type ExtendedAttribute struct {
+	Name  string
+	Value []byte
+	Flags uint8
+}
+
+func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) {
+	var info fileFullEaInformation
+	err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info)
+	if err != nil {
+		err = errInvalidEaBuffer
+		return
+	}
+
+	nameOffset := fileFullEaInformationSize
+	nameLen := int(info.NameLength)
+	valueOffset := nameOffset + int(info.NameLength) + 1
+	valueLen := int(info.ValueLength)
+	nextOffset := int(info.NextEntryOffset)
+	if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) {
+		err = errInvalidEaBuffer
+		return
+	}
+
+	ea.Name = string(b[nameOffset : nameOffset+nameLen])
+	ea.Value = b[valueOffset : valueOffset+valueLen]
+	ea.Flags = info.Flags
+	if info.NextEntryOffset != 0 {
+		nb = b[info.NextEntryOffset:]
+	}
+	return
+}
+
+// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION
+// buffer retrieved from BackupRead, ZwQueryEaFile, etc.
+func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) {
+	for len(b) != 0 {
+		ea, nb, err := parseEa(b)
+		if err != nil {
+			return nil, err
+		}
+
+		eas = append(eas, ea)
+		b = nb
+	}
+	return
+}
+
+func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error {
+	if int(uint8(len(ea.Name))) != len(ea.Name) {
+		return errEaNameTooLarge
+	}
+	if int(uint16(len(ea.Value))) != len(ea.Value) {
+		return errEaValueTooLarge
+	}
+	entrySize := uint32(fileFullEaInformationSize + len(ea.Name) + 1 + len(ea.Value))
+	withPadding := (entrySize + 3) &^ 3
+	nextOffset := uint32(0)
+	if !last {
+		nextOffset = withPadding
+	}
+	info := fileFullEaInformation{
+		NextEntryOffset: nextOffset,
+		Flags:           ea.Flags,
+		NameLength:      uint8(len(ea.Name)),
+		ValueLength:     uint16(len(ea.Value)),
+	}
+
+	err := binary.Write(buf, binary.LittleEndian, &info)
+	if err != nil {
+		return err
+	}
+
+	_, err = buf.Write([]byte(ea.Name))
+	if err != nil {
+		return err
+	}
+
+	err = buf.WriteByte(0)
+	if err != nil {
+		return err
+	}
+
+	_, err = buf.Write(ea.Value)
+	if err != nil {
+		return err
+	}
+
+	_, err = buf.Write([]byte{0, 0, 0}[0 : withPadding-entrySize])
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// EncodeExtendedAttributes encodes a list of EAs into a FILE_FULL_EA_INFORMATION
+// buffer for use with BackupWrite, ZwSetEaFile, etc.
+func EncodeExtendedAttributes(eas []ExtendedAttribute) ([]byte, error) {
+	var buf bytes.Buffer
+	for i := range eas {
+		last := false
+		if i == len(eas)-1 {
+			last = true
+		}
+
+		err := writeEa(&buf, &eas[i], last)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return buf.Bytes(), nil
+}

vendor/github.com/Microsoft/go-winio/fileinfo.go

@@ -20,7 +20,8 @@ const (
 // FileBasicInfo contains file access time and file attributes information.
 type FileBasicInfo struct {
 	CreationTime, LastAccessTime, LastWriteTime, ChangeTime syscall.Filetime
-	FileAttributes                                          uintptr // includes padding
+	FileAttributes                                          uint32
+	pad                                                     uint32 // padding
 }
 
 // GetFileBasicInfo retrieves times and attributes for a file.

vendor/github.com/Microsoft/go-winio/vhd/vhd.go

@@ -1,82 +1,108 @@
-// +build windows
-
-package vhd
-
-import "syscall"
-
-//go:generate go run mksyscall_windows.go -output zvhd.go vhd.go
-
-//sys createVirtualDisk(virtualStorageType *virtualStorageType, path string, virtualDiskAccessMask uint32, securityDescriptor *uintptr, flags uint32, providerSpecificFlags uint32, parameters *createVirtualDiskParameters, o *syscall.Overlapped, handle *syscall.Handle) (err error) [failretval != 0] = VirtDisk.CreateVirtualDisk
-
-type virtualStorageType struct {
-	DeviceID uint32
-	VendorID [16]byte
-}
-
-const virtualDiskAccessNONE uint32 = 0
-const virtualDiskAccessATTACHRO uint32 = 65536
-const virtualDiskAccessATTACHRW uint32 = 131072
-const virtualDiskAccessDETACH uint32 = 262144
-const virtualDiskAccessGETINFO uint32 = 524288
-const virtualDiskAccessCREATE uint32 = 1048576
-const virtualDiskAccessMETAOPS uint32 = 2097152
-const virtualDiskAccessREAD uint32 = 851968
-const virtualDiskAccessALL uint32 = 4128768
-const virtualDiskAccessWRITABLE uint32 = 3276800
-
-const createVirtualDiskFlagNone uint32 = 0
-const createVirtualDiskFlagFullPhysicalAllocation uint32 = 1
-const createVirtualDiskFlagPreventWritesToSourceDisk uint32 = 2
-const createVirtualDiskFlagDoNotCopyMetadataFromParent uint32 = 4
-
-type version2 struct {
-	UniqueID                 [16]byte // GUID
-	MaximumSize              uint64
-	BlockSizeInBytes         uint32
-	SectorSizeInBytes        uint32
-	ParentPath               *uint16 // string
-	SourcePath               *uint16 // string
-	OpenFlags                uint32
-	ParentVirtualStorageType virtualStorageType
-	SourceVirtualStorageType virtualStorageType
-	ResiliencyGUID           [16]byte // GUID
-}
-
-type createVirtualDiskParameters struct {
-	Version  uint32 // Must always be set to 2
-	Version2 version2
-}
-
-// CreateVhdx will create a simple vhdx file at the given path using default values.
-func CreateVhdx(path string, maxSizeInGb, blockSizeInMb uint32) error {
-	var defaultType virtualStorageType
-
-	parameters := createVirtualDiskParameters{
-		Version: 2,
-		Version2: version2{
-			MaximumSize:      uint64(maxSizeInGb) * 1024 * 1024 * 1024,
-			BlockSizeInBytes: blockSizeInMb * 1024 * 1024,
-		},
-	}
-
-	var handle syscall.Handle
-
-	if err := createVirtualDisk(
-		&defaultType,
-		path,
-		virtualDiskAccessNONE,
-		nil,
-		createVirtualDiskFlagNone,
-		0,
-		&parameters,
-		nil,
-		&handle); err != nil {
-		return err
-	}
-
-	if err := syscall.CloseHandle(handle); err != nil {
-		return err
-	}
-
-	return nil
-}
+// +build windows
+
+package vhd
+
+import "syscall"
+
+//go:generate go run mksyscall_windows.go -output zvhd.go vhd.go
+
+//sys createVirtualDisk(virtualStorageType *virtualStorageType, path string, virtualDiskAccessMask uint32, securityDescriptor *uintptr, flags uint32, providerSpecificFlags uint32, parameters *createVirtualDiskParameters, o *syscall.Overlapped, handle *syscall.Handle) (err error) [failretval != 0] = VirtDisk.CreateVirtualDisk
+//sys openVirtualDisk(virtualStorageType *virtualStorageType, path string, virtualDiskAccessMask uint32, flags uint32, parameters *uintptr, handle *syscall.Handle) (err error) [failretval != 0] = VirtDisk.OpenVirtualDisk
+//sys detachVirtualDisk(handle syscall.Handle, flags uint32, providerSpecificFlags uint32) (err error) [failretval != 0] = VirtDisk.DetachVirtualDisk
+
+type virtualStorageType struct {
+	DeviceID uint32
+	VendorID [16]byte
+}
+
+const virtualDiskAccessNONE uint32 = 0
+const virtualDiskAccessATTACHRO uint32 = 65536
+const virtualDiskAccessATTACHRW uint32 = 131072
+const virtualDiskAccessDETACH uint32 = 262144
+const virtualDiskAccessGETINFO uint32 = 524288
+const virtualDiskAccessCREATE uint32 = 1048576
+const virtualDiskAccessMETAOPS uint32 = 2097152
+const virtualDiskAccessREAD uint32 = 851968
+const virtualDiskAccessALL uint32 = 4128768
+const virtualDiskAccessWRITABLE uint32 = 3276800
+
+const createVirtualDiskFlagNone uint32 = 0
+const createVirtualDiskFlagFullPhysicalAllocation uint32 = 1
+const createVirtualDiskFlagPreventWritesToSourceDisk uint32 = 2
+const createVirtualDiskFlagDoNotCopyMetadataFromParent uint32 = 4
+
+type version2 struct {
+	UniqueID                 [16]byte // GUID
+	MaximumSize              uint64
+	BlockSizeInBytes         uint32
+	SectorSizeInBytes        uint32
+	ParentPath               *uint16 // string
+	SourcePath               *uint16 // string
+	OpenFlags                uint32
+	ParentVirtualStorageType virtualStorageType
+	SourceVirtualStorageType virtualStorageType
+	ResiliencyGUID           [16]byte // GUID
+}
+
+type createVirtualDiskParameters struct {
+	Version  uint32 // Must always be set to 2
+	Version2 version2
+}
+
+// CreateVhdx will create a simple vhdx file at the given path using default values.
+func CreateVhdx(path string, maxSizeInGb, blockSizeInMb uint32) error {
+	var defaultType virtualStorageType
+
+	parameters := createVirtualDiskParameters{
+		Version: 2,
+		Version2: version2{
+			MaximumSize:      uint64(maxSizeInGb) * 1024 * 1024 * 1024,
+			BlockSizeInBytes: blockSizeInMb * 1024 * 1024,
+		},
+	}
+
+	var handle syscall.Handle
+
+	if err := createVirtualDisk(
+		&defaultType,
+		path,
+		virtualDiskAccessNONE,
+		nil,
+		createVirtualDiskFlagNone,
+		0,
+		&parameters,
+		nil,
+		&handle); err != nil {
+		return err
+	}
+
+	if err := syscall.CloseHandle(handle); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// DetachVhd detaches a VHD attached at the given path.
+func DetachVhd(path string) error {
+	var (
+		defaultType virtualStorageType
+		handle      syscall.Handle
+	)
+
+	if err := openVirtualDisk(
+		&defaultType,
+		path,
+		virtualDiskAccessDETACH,
+		0,
+		nil,
+		&handle); err != nil {
+		return err
+	}
+	defer syscall.CloseHandle(handle)
+
+	if err := detachVirtualDisk(handle, 0, 0); err != nil {
+		return err
+	}
+	return nil
+}

vendor/github.com/Microsoft/go-winio/vhd/zvhd.go

@@ -40,6 +40,8 @@ var (
 	modVirtDisk = windows.NewLazySystemDLL("VirtDisk.dll")
 
 	procCreateVirtualDisk = modVirtDisk.NewProc("CreateVirtualDisk")
+	procOpenVirtualDisk   = modVirtDisk.NewProc("OpenVirtualDisk")
+	procDetachVirtualDisk = modVirtDisk.NewProc("DetachVirtualDisk")
 )
 
 func createVirtualDisk(virtualStorageType *virtualStorageType, path string, virtualDiskAccessMask uint32, securityDescriptor *uintptr, flags uint32, providerSpecificFlags uint32, parameters *createVirtualDiskParameters, o *syscall.Overlapped, handle *syscall.Handle) (err error) {
@@ -62,3 +64,36 @@ func _createVirtualDisk(virtualStorageType *virtualStorageType, path *uint16, vi
 	}
 	return
 }
+
+func openVirtualDisk(virtualStorageType *virtualStorageType, path string, virtualDiskAccessMask uint32, flags uint32, parameters *uintptr, handle *syscall.Handle) (err error) {
+	var _p0 *uint16
+	_p0, err = syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return
+	}
+	return _openVirtualDisk(virtualStorageType, _p0, virtualDiskAccessMask, flags, parameters, handle)
+}
+
+func _openVirtualDisk(virtualStorageType *virtualStorageType, path *uint16, virtualDiskAccessMask uint32, flags uint32, parameters *uintptr, handle *syscall.Handle) (err error) {
+	r1, _, e1 := syscall.Syscall6(procOpenVirtualDisk.Addr(), 6, uintptr(unsafe.Pointer(virtualStorageType)), uintptr(unsafe.Pointer(path)), uintptr(virtualDiskAccessMask), uintptr(flags), uintptr(unsafe.Pointer(parameters)), uintptr(unsafe.Pointer(handle)))
+	if r1 != 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}
+
+func detachVirtualDisk(handle syscall.Handle, flags uint32, providerSpecificFlags uint32) (err error) {
+	r1, _, e1 := syscall.Syscall(procDetachVirtualDisk.Addr(), 3, uintptr(handle), uintptr(flags), uintptr(providerSpecificFlags))
+	if r1 != 0 {
+		if e1 != 0 {
+			err = errnoErr(e1)
+		} else {
+			err = syscall.EINVAL
+		}
+	}
+	return
+}

vendor/github.com/Microsoft/hcsshim/README.md

@@ -1,12 +1,13 @@
 # hcsshim
 
-This package supports launching Windows Server containers from Go. It is
-primarily used in the [Docker Engine](https://github.com/docker/docker) project,
-but it can be freely used by other projects as well.
+[![Build status](https://ci.appveyor.com/api/projects/status/nbcw28mnkqml0loa/branch/master?svg=true)](https://ci.appveyor.com/project/WindowsVirtualization/hcsshim/branch/master)
 
+This package contains the Golang interface for using the Windows [Host Compute Service](https://blogs.technet.microsoft.com/virtualization/2017/01/27/introducing-the-host-compute-service-hcs/) (HCS) to launch and manage [Windows Containers](https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/). It also contains other helpers and functions for managing Windows Containers such as the Golang interface for the Host Network Service (HNS).
+
+It is primarily used in the [Moby Project](https://github.com/moby/moby), but it can be freely used by other projects as well.
 
 ## Contributing
----------------
+
 This project welcomes contributions and suggestions.  Most contributions require you to agree to a
 Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
 the rights to use your contribution. For details, visit https://cla.microsoft.com.
@@ -19,6 +20,11 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope
 For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
 contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
 
+## Dependencies
+
+This project requires Golang 1.9 or newer to build.
+
+For system requirements to run this project, see the Microsoft docs on [Windows Container requirements](https://docs.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/system-requirements).
 
 ## Reporting Security Issues
 
@@ -29,5 +35,7 @@ email to ensure we received your original message. Further information, includin
 [MSRC PGP](https://technet.microsoft.com/en-us/security/dn606155) key, can be found in
 the [Security TechCenter](https://technet.microsoft.com/en-us/security/default).
 
--------------------------------------------
+For additional details, see [Report a Computer Security Vulnerability](https://technet.microsoft.com/en-us/security/ff852094.aspx) on Technet
+
+---------------
 Copyright (c) 2018 Microsoft Corp.  All rights reserved.