bump to 17.04.0 GA

Signed-off-by: Victor Vieux <victorvieux@gmail.com>

CHANGELOG.md

@@ -5,6 +5,91 @@ information on the list of deprecated flags and APIs please have a look at
 https://docs.docker.com/engine/deprecated/ where target removal dates can also
 be found.
 
+## 17.04.0-ce (2017-04-05)
+
+### Builder
+
+* Disable container logging for build containers [#29552](https://github.com/docker/docker/pull/29552)
+* Fix use of `**/` in `.dockerignore` [#29043](https://github.com/docker/docker/pull/29043)
+
+### Client
+
++ Sort `docker stack ls` by name [#31085](https://github.com/docker/docker/pull/31085)
++ Flags for specifying bind mount consistency [#31047](https://github.com/docker/docker/pull/31047)
+* Output of docker CLI --help is now wrapped to the terminal width [#28751](https://github.com/docker/docker/pull/28751)
+* Suppress image digest in docker ps [#30848](https://github.com/docker/docker/pull/30848)
+* Hide command options that are related to Windows [#30788](https://github.com/docker/docker/pull/30788)
+* Fix `docker plugin install` prompt to accept "enter" for the "N" default [#30769](https://github.com/docker/docker/pull/30769)
++ Add `truncate` function for Go templates [#30484](https://github.com/docker/docker/pull/30484)
+* Support expanded syntax of ports in `stack deploy` [#30476](https://github.com/docker/docker/pull/30476)
+* Support expanded syntax of mounts in `stack deploy` [#30597](https://github.com/docker/docker/pull/30597) [#31795](https://github.com/docker/docker/pull/31795)
++ Add `--add-host` for docker build [#30383](https://github.com/docker/docker/pull/30383)
++ Add `.CreatedAt` placeholder for `docker network ls --format` [#29900](https://github.com/docker/docker/pull/29900)
+* Update order of `--secret-rm` and `--secret-add` [#29802](https://github.com/docker/docker/pull/29802)
++ Add `--filter enabled=true` for `docker plugin ls` [#28627](https://github.com/docker/docker/pull/28627)
++ Add `--format` to `docker service ls` [#28199](https://github.com/docker/docker/pull/28199)
++ Add `publish` and `expose` filter for `docker ps --filter` [#27557](https://github.com/docker/docker/pull/27557)
+* Support multiple service IDs on `docker service ps` [#25234](https://github.com/docker/docker/pull/25234)
++ Allow swarm join with `--availability=drain` [#24993](https://github.com/docker/docker/pull/24993)
+* Docker inspect now shows "docker-default" when AppArmor is enabled and no other profile was defined [#27083](https://github.com/docker/docker/pull/27083)
+
+### Logging
+
++ Implement optional ring buffer for container logs [#28762](https://github.com/docker/docker/pull/28762)
++ Add `--log-opt awslogs-create-group=<true|false>` for awslogs (CloudWatch) to support creation of log groups as needed [#29504](https://github.com/docker/docker/pull/29504)
+- Fix segfault when using the gcplogs logging driver with a "static" binary [#29478](https://github.com/docker/docker/pull/29478)
+
+
+### Networking
+
+* Check parameter `--ip`, `--ip6` and `--link-local-ip` in `docker network connect` [#30807](https://github.com/docker/docker/pull/30807)
++ Added support for `dns-search` [#30117](https://github.com/docker/docker/pull/30117)
++ Added --verbose option for docker network inspect to show task details from all swarm nodes [#31710](https://github.com/docker/docker/pull/31710)
+* Clear stale datapath encryption states when joining the cluster [docker/libnetwork#1354](https://github.com/docker/libnetwork/pull/1354)
++ Ensure iptables initialization only happens once [docker/libnetwork#1676](https://github.com/docker/libnetwork/pull/1676)
+* Fix bad order of iptables filter rules [docker/libnetwork#961](https://github.com/docker/libnetwork/pull/961)
++ Add anonymous container alias to service record on attachable network [docker/libnetwork#1651](https://github.com/docker/libnetwork/pull/1651)
++ Support for `com.docker.network.container_interface_prefix` driver label [docker/libnetwork#1667](https://github.com/docker/libnetwork/pull/1667)
++ Improve network list performance by omitting network details that are not used [#30673](https://github.com/docker/docker/pull/30673)
+
+### Runtime
+
+* Handle paused container when restoring without live-restore set [#31704](https://github.com/docker/docker/pull/31704)
+- Do not allow sub second in healthcheck options in Dockerfile [#31177](https://github.com/docker/docker/pull/31177)
+* Support name and id prefix in  `secret update` [#30856](https://github.com/docker/docker/pull/30856)
+* Use binary frame for websocket attach endpoint [#30460](https://github.com/docker/docker/pull/30460)
+* Fix linux mount calls not applying propagation type changes [#30416](https://github.com/docker/docker/pull/30416)
+* Fix ExecIds leak on failed `exec -i` [#30340](https://github.com/docker/docker/pull/30340)
+* Prune named but untagged images if `danglingOnly=true` [#30330](https://github.com/docker/docker/pull/30330)
++ Add daemon flag to set `no_new_priv` as default for unprivileged containers [#29984](https://github.com/docker/docker/pull/29984)
++ Add daemon option `--default-shm-size` [#29692](https://github.com/docker/docker/pull/29692)
++ Support registry mirror config reload [#29650](https://github.com/docker/docker/pull/29650)
+- Ignore the daemon log config when building images [#29552](https://github.com/docker/docker/pull/29552)
+* Move secret name or ID prefix resolving from client to daemon [#29218](https://github.com/docker/docker/pull/29218)
++ Allow adding rules to `cgroup devices.allow` on container create/run [#22563](https://github.com/docker/docker/pull/22563)
+- Fix `cpu.cfs_quota_us` being reset when running `systemd daemon-reload` [#31736](https://github.com/docker/docker/pull/31736)
+
+### Swarm Mode
+
++ Topology-aware scheduling [#30725](https://github.com/docker/docker/pull/30725)
++ Automatic service rollback on failure [#31108](https://github.com/docker/docker/pull/31108)
++ Worker and manager on the same node are now connected through a UNIX socket [docker/swarmkit#1828](https://github.com/docker/swarmkit/pull/1828), [docker/swarmkit#1850](https://github.com/docker/swarmkit/pull/1850), [docker/swarmkit#1851](https://github.com/docker/swarmkit/pull/1851)
+* Improve raft transport package [docker/swarmkit#1748](https://github.com/docker/swarmkit/pull/1748)
+* No automatic manager shutdown on demotion/removal [docker/swarmkit#1829](https://github.com/docker/swarmkit/pull/1829)
+* Use TransferLeadership to make leader demotion safer [docker/swarmkit#1939](https://github.com/docker/swarmkit/pull/1939)
+* Decrease default monitoring period [docker/swarmkit#1967](https://github.com/docker/swarmkit/pull/1967)
++ Add Service logs formatting [#31672](https://github.com/docker/docker/pull/31672)
+* Fix service logs API to be able to specify stream [#31313](https://github.com/docker/docker/pull/31313)
++ Add `--stop-signal` for `service create` and `service update` [#30754](https://github.com/docker/docker/pull/30754)
++ Add `--read-only` for `service create` and `service update` [#30162](https://github.com/docker/docker/pull/30162)
++ Renew the context after communicating with the registry [#31586](https://github.com/docker/docker/pull/31586)
++ (experimental) Add `--tail` and `--since` options to `docker service logs` [#31500](https://github.com/docker/docker/pull/31500)
++ (experimental) Add `--no-task-ids` and `--no-trunc` options to `docker service logs` [#31672](https://github.com/docker/docker/pull/31672)
+
+### Windows
+
+* Block pulling Windows images on non-Windows daemons [#29001](https://github.com/docker/docker/pull/29001)
+
 ## 17.03.0-ce (2017-03-01)
 
 **IMPORTANT**: Starting with this release, Docker is on a monthly release cycle and uses a
@@ -49,6 +134,7 @@ Upgrading from Docker 1.13.1 to 17.03.0 is expected to be simple and low-risk.
 * Fix extraneous task creations for global services that led to confusing replica counts in `docker service ls` [docker/swarmkit#1957](https://github.com/docker/swarmkit/pull/1957)
 * Fix problem that made rolling updates slow when `task-history-limit` was set to 1 [docker/swarmkit#1948](https://github.com/docker/swarmkit/pull/1948)
 * Restart tasks elsewhere, if appropriate, when they are shut down as a result of nodes no longer satisfying constraints [docker/swarmkit#1958](https://github.com/docker/swarmkit/pull/1958)
+* (experimental)
 
 ## 1.13.1 (2017-02-08)
 
@@ -319,7 +405,7 @@ To manually remove all plugins and resolve this problem, take the following step
 - Fix selinux labeling of volumes shared in a container [#23024](https://github.com/docker/docker/pull/23024)
 - Prohibit `/sys/firmware/**` from being accessed with apparmor [#26618](https://github.com/docker/docker/pull/26618)
 
-### DEPRECATION
+### Deprecation
 
 - Marked the `docker daemon` command as deprecated. The daemon is moved to a separate binary (`dockerd`), and should be used instead [#26834](https://github.com/docker/docker/pull/26834)
 - Deprecate unversioned API endpoints [#28208](https://github.com/docker/docker/pull/28208)
@@ -942,7 +1028,8 @@ installing docker, please make sure to update them accordingly.
 - Windows: Enable auto-creation of host-path to match Linux [#22094](https://github.com/docker/docker/pull/22094)
 
 
-### DEPRECATION
+### Deprecation
+
 * Environment variables `DOCKER_CONTENT_TRUST_OFFLINE_PASSPHRASE` and `DOCKER_CONTENT_TRUST_TAGGING_PASSPHRASE` have been renamed
   to `DOCKER_CONTENT_TRUST_ROOT_PASSPHRASE` and `DOCKER_CONTENT_TRUST_REPOSITORY_PASSPHRASE` respectively [#22574](https://github.com/docker/docker/pull/22574)
 * Remove deprecated `syslog-tag`, `gelf-tag`, `fluentd-tag` log option in favor of the more generic `tag` one [#22620](https://github.com/docker/docker/pull/22620)

Makefile

@@ -141,6 +141,9 @@ run: build ## run the docker daemon in a container
 shell: build ## start a shell inside the build env
 	$(DOCKER_RUN_DOCKER) bash
 
+yaml-docs-gen: build ## generate documentation YAML files consumed by docs repo
+	$(DOCKER_RUN_DOCKER) sh -c 'hack/make.sh yaml-docs-generator && ( cd bundles/latest/yaml-docs-generator; mkdir docs; ./yaml-docs-generator --target $$(pwd)/docs )'
+
 test: build ## run the unit, integration and docker-py tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary cross test-unit test-integration-cli test-docker-py
 

VERSION

@@ -1 +1 @@
-17.04.0-dev
+17.04.0-ce

api/server/router/image/image_routes.go

@@ -253,9 +253,9 @@ func (s *imageRouter) getImagesJSON(ctx context.Context, w http.ResponseWriter,
 		return err
 	}
 
-	version := httputils.VersionFromContext(ctx)
 	filterParam := r.Form.Get("filter")
-	if versions.LessThanOrEqualTo(version, "1.28") && filterParam != "" {
+	// FIXME(vdemeester) This has been deprecated in 1.13, and is target for removal for v17.12
+	if filterParam != "" {
 		imageFilters.Add("reference", filterParam)
 	}
 

api/swagger.yaml

@@ -486,10 +486,10 @@ definitions:
         description: |
           The test to perform. Possible values are:
 
-          - `{}` inherit healthcheck from image or parent image
-          - `{"NONE"}` disable healthcheck
-          - `{"CMD", args...}` exec arguments directly
-          - `{"CMD-SHELL", command}` run command with system's default shell
+          - `[]` inherit healthcheck from image or parent image
+          - `["NONE"]` disable healthcheck
+          - `["CMD", args...]` exec arguments directly
+          - `["CMD-SHELL", command]` run command with system's default shell
         type: "array"
         items:
           type: "string"

cli/command/cli.go

@@ -250,8 +250,9 @@ func newHTTPClient(host string, tlsOptions *tlsconfig.Options) (*http.Client, er
 		// let the api client configure the default transport.
 		return nil, nil
 	}
-
-	config, err := tlsconfig.Client(*tlsOptions)
+	opts := *tlsOptions
+	opts.ExclusiveRootPools = true
+	config, err := tlsconfig.Client(opts)
 	if err != nil {
 		return nil, err
 	}

cli/command/image/build.go

@@ -305,6 +305,7 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
 			}
 			return cli.StatusError{Status: jerr.Message, StatusCode: jerr.Code}
 		}
+		return err
 	}
 
 	// Windows: show error message about modified file permissions if the

cli/command/registry/login.go

@@ -35,9 +35,14 @@ func NewLoginCommand(dockerCli *command.DockerCli) *cobra.Command {
 	}
 
 	flags := cmd.Flags()
+
 	flags.StringVarP(&opts.user, "username", "u", "", "Username")
 	flags.StringVarP(&opts.password, "password", "p", "", "Password")
 
+	// Deprecated in 1.11: Should be removed in docker 17.06
+	flags.StringVarP(&opts.email, "email", "e", "", "Email")
+	flags.MarkDeprecated("email", "will be removed in 17.06.")
+
 	return cmd
 }
 

cli/command/stack/deploy_bundlefile.go

@@ -46,7 +46,7 @@ func deployBundle(ctx context.Context, dockerCli *command.DockerCli, opts deploy
 		for _, networkName := range service.Networks {
 			nets = append(nets, swarm.NetworkAttachmentConfig{
 				Target:  namespace.Scope(networkName),
-				Aliases: []string{networkName},
+				Aliases: []string{internalName},
 			})
 		}
 

cli/command/stack/deploy_composefile.go

@@ -15,6 +15,7 @@ import (
 	composetypes "github.com/docker/docker/cli/compose/types"
 	apiclient "github.com/docker/docker/client"
 	dockerclient "github.com/docker/docker/client"
+	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 )
 
@@ -115,9 +116,26 @@ func getConfigDetails(opts deployOptions) (composetypes.ConfigDetails, error) {
 	}
 	// TODO: support multiple files
 	details.ConfigFiles = []composetypes.ConfigFile{*configFile}
+	details.Environment, err = buildEnvironment(os.Environ())
+	if err != nil {
+		return details, err
+	}
 	return details, nil
 }
 
+func buildEnvironment(env []string) (map[string]string, error) {
+	result := make(map[string]string, len(env))
+	for _, s := range env {
+		// if value is empty, s is like "K=", not "K".
+		if !strings.Contains(s, "=") {
+			return result, errors.Errorf("unexpected environment %q", s)
+		}
+		kv := strings.SplitN(s, "=", 2)
+		result[kv[0]] = kv[1]
+	}
+	return result, nil
+}
+
 func getConfigFile(filename string) (*composetypes.ConfigFile, error) {
 	bytes, err := ioutil.ReadFile(filename)
 	if err != nil {

cli/compose/convert/service.go

@@ -393,11 +393,16 @@ func convertEndpointSpec(endpointMode string, source []composetypes.ServicePortC
 	}, nil
 }
 
-func convertEnvironment(source map[string]string) []string {
+func convertEnvironment(source map[string]*string) []string {
 	var output []string
 
 	for name, value := range source {
-		output = append(output, fmt.Sprintf("%s=%s", name, value))
+		switch value {
+		case nil:
+			output = append(output, name)
+		default:
+			output = append(output, fmt.Sprintf("%s=%s", name, *value))
+		}
 	}
 
 	return output

cli/compose/convert/service_test.go

@@ -43,10 +43,14 @@ func TestConvertRestartPolicyFromFailure(t *testing.T) {
 	assert.DeepEqual(t, policy, expected)
 }
 
+func strPtr(val string) *string {
+	return &val
+}
+
 func TestConvertEnvironment(t *testing.T) {
-	source := map[string]string{
-		"foo": "bar",
-		"key": "value",
+	source := map[string]*string{
+		"foo": strPtr("bar"),
+		"key": strPtr("value"),
 	}
 	env := convertEnvironment(source)
 	sort.Strings(env)

cli/compose/loader/example1.env

@@ -1,8 +1,8 @@
 # passed through
-FOO=1
+FOO=foo_from_env_file
 
 # overridden in example2.env
-BAR=1
+BAR=bar_from_env_file
 
 # overridden in full-example.yml
-BAZ=1
+BAZ=baz_from_env_file

cli/compose/loader/example2.env

@@ -1 +1,4 @@
-BAR=2
+BAR=bar_from_env_file_2
+
+# overridden in configDetails.Environment
+QUX=quz_from_env_file_2

cli/compose/loader/full-example.yml

@@ -77,10 +77,8 @@ services:
     # Mapping values can be strings, numbers or null
     # Booleans are not allowed - must be quoted
     environment:
-      RACK_ENV: development
-      SHOW: 'true'
-      SESSION_SECRET:
-      BAZ: 3
+      BAZ: baz_from_service_def
+      QUX:
     # environment:
     #   - RACK_ENV=development
     #   - SHOW=true

cli/compose/loader/loader.go

@@ -2,15 +2,16 @@ package loader
 
 import (
 	"fmt"
-	"os"
 	"path"
 	"reflect"
 	"regexp"
 	"sort"
 	"strings"
 
+	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/cli/compose/interpolation"
 	"github.com/docker/docker/cli/compose/schema"
+	"github.com/docker/docker/cli/compose/template"
 	"github.com/docker/docker/cli/compose/types"
 	"github.com/docker/docker/opts"
 	runconfigopts "github.com/docker/docker/runconfig/opts"
@@ -69,13 +70,17 @@ func Load(configDetails types.ConfigDetails) (*types.Config, error) {
 	}
 
 	cfg := types.Config{}
+	lookupEnv := func(k string) (string, bool) {
+		v, ok := configDetails.Environment[k]
+		return v, ok
+	}
 	if services, ok := configDict["services"]; ok {
-		servicesConfig, err := interpolation.Interpolate(services.(types.Dict), "service", os.LookupEnv)
+		servicesConfig, err := interpolation.Interpolate(services.(types.Dict), "service", lookupEnv)
 		if err != nil {
 			return nil, err
 		}
 
-		servicesList, err := LoadServices(servicesConfig, configDetails.WorkingDir)
+		servicesList, err := LoadServices(servicesConfig, configDetails.WorkingDir, lookupEnv)
 		if err != nil {
 			return nil, err
 		}
@@ -84,7 +89,7 @@ func Load(configDetails types.ConfigDetails) (*types.Config, error) {
 	}
 
 	if networks, ok := configDict["networks"]; ok {
-		networksConfig, err := interpolation.Interpolate(networks.(types.Dict), "network", os.LookupEnv)
+		networksConfig, err := interpolation.Interpolate(networks.(types.Dict), "network", lookupEnv)
 		if err != nil {
 			return nil, err
 		}
@@ -98,7 +103,7 @@ func Load(configDetails types.ConfigDetails) (*types.Config, error) {
 	}
 
 	if volumes, ok := configDict["volumes"]; ok {
-		volumesConfig, err := interpolation.Interpolate(volumes.(types.Dict), "volume", os.LookupEnv)
+		volumesConfig, err := interpolation.Interpolate(volumes.(types.Dict), "volume", lookupEnv)
 		if err != nil {
 			return nil, err
 		}
@@ -112,7 +117,7 @@ func Load(configDetails types.ConfigDetails) (*types.Config, error) {
 	}
 
 	if secrets, ok := configDict["secrets"]; ok {
-		secretsConfig, err := interpolation.Interpolate(secrets.(types.Dict), "secret", os.LookupEnv)
+		secretsConfig, err := interpolation.Interpolate(secrets.(types.Dict), "secret", lookupEnv)
 		if err != nil {
 			return nil, err
 		}
@@ -248,9 +253,11 @@ func transformHook(
 	case reflect.TypeOf(map[string]*types.ServiceNetworkConfig{}):
 		return transformServiceNetworkMap(data)
 	case reflect.TypeOf(types.MappingWithEquals{}):
-		return transformMappingOrList(data, "="), nil
+		return transformMappingOrList(data, "=", true), nil
+	case reflect.TypeOf(types.Labels{}):
+		return transformMappingOrList(data, "=", false), nil
 	case reflect.TypeOf(types.MappingWithColon{}):
-		return transformMappingOrList(data, ":"), nil
+		return transformMappingOrList(data, ":", false), nil
 	case reflect.TypeOf(types.ServiceVolumeConfig{}):
 		return transformServiceVolumeConfig(data)
 	}
@@ -308,11 +315,11 @@ func formatInvalidKeyError(keyPrefix string, key interface{}) error {
 
 // LoadServices produces a ServiceConfig map from a compose file Dict
 // the servicesDict is not validated if directly used. Use Load() to enable validation
-func LoadServices(servicesDict types.Dict, workingDir string) ([]types.ServiceConfig, error) {
+func LoadServices(servicesDict types.Dict, workingDir string, lookupEnv template.Mapping) ([]types.ServiceConfig, error) {
 	var services []types.ServiceConfig
 
 	for name, serviceDef := range servicesDict {
-		serviceConfig, err := LoadService(name, serviceDef.(types.Dict), workingDir)
+		serviceConfig, err := LoadService(name, serviceDef.(types.Dict), workingDir, lookupEnv)
 		if err != nil {
 			return nil, err
 		}
@@ -324,23 +331,35 @@ func LoadServices(servicesDict types.Dict, workingDir string) ([]types.ServiceCo
 
 // LoadService produces a single ServiceConfig from a compose file Dict
 // the serviceDict is not validated if directly used. Use Load() to enable validation
-func LoadService(name string, serviceDict types.Dict, workingDir string) (*types.ServiceConfig, error) {
+func LoadService(name string, serviceDict types.Dict, workingDir string, lookupEnv template.Mapping) (*types.ServiceConfig, error) {
 	serviceConfig := &types.ServiceConfig{}
 	if err := transform(serviceDict, serviceConfig); err != nil {
 		return nil, err
 	}
 	serviceConfig.Name = name
 
-	if err := resolveEnvironment(serviceConfig, workingDir); err != nil {
+	if err := resolveEnvironment(serviceConfig, workingDir, lookupEnv); err != nil {
 		return nil, err
 	}
 
-	resolveVolumePaths(serviceConfig.Volumes, workingDir)
+	resolveVolumePaths(serviceConfig.Volumes, workingDir, lookupEnv)
 	return serviceConfig, nil
 }
 
-func resolveEnvironment(serviceConfig *types.ServiceConfig, workingDir string) error {
-	environment := make(map[string]string)
+func updateEnvironment(environment map[string]*string, vars map[string]*string, lookupEnv template.Mapping) {
+	for k, v := range vars {
+		interpolatedV, ok := lookupEnv(k)
+		if (v == nil || *v == "") && ok {
+			// lookupEnv is prioritized over vars
+			environment[k] = &interpolatedV
+		} else {
+			environment[k] = v
+		}
+	}
+}
+
+func resolveEnvironment(serviceConfig *types.ServiceConfig, workingDir string, lookupEnv template.Mapping) error {
+	environment := make(map[string]*string)
 
 	if len(serviceConfig.EnvFile) > 0 {
 		var envVars []string
@@ -353,36 +372,35 @@ func resolveEnvironment(serviceConfig *types.ServiceConfig, workingDir string) e
 			}
 			envVars = append(envVars, fileVars...)
 		}
-
-		for k, v := range runconfigopts.ConvertKVStringsToMap(envVars) {
-			environment[k] = v
-		}
-	}
-
-	for k, v := range serviceConfig.Environment {
-		environment[k] = v
+		updateEnvironment(environment,
+			runconfigopts.ConvertKVStringsToMapWithNil(envVars), lookupEnv)
 	}
 
+	updateEnvironment(environment, serviceConfig.Environment, lookupEnv)
 	serviceConfig.Environment = environment
-
 	return nil
 }
 
-func resolveVolumePaths(volumes []types.ServiceVolumeConfig, workingDir string) {
+func resolveVolumePaths(volumes []types.ServiceVolumeConfig, workingDir string, lookupEnv template.Mapping) {
 	for i, volume := range volumes {
 		if volume.Type != "bind" {
 			continue
 		}
 
-		volume.Source = absPath(workingDir, expandUser(volume.Source))
+		volume.Source = absPath(workingDir, expandUser(volume.Source, lookupEnv))
 		volumes[i] = volume
 	}
 }
 
 // TODO: make this more robust
-func expandUser(path string) string {
+func expandUser(path string, lookupEnv template.Mapping) string {
 	if strings.HasPrefix(path, "~") {
-		return strings.Replace(path, "~", os.Getenv("HOME"), 1)
+		home, ok := lookupEnv("HOME")
+		if !ok {
+			logrus.Warn("cannot expand '~', because the environment lacks HOME")
+			return path
+		}
+		return strings.Replace(path, "~", home, 1)
 	}
 	return path
 }
@@ -476,9 +494,9 @@ func absPath(workingDir string, filepath string) string {
 func transformMapStringString(data interface{}) (interface{}, error) {
 	switch value := data.(type) {
 	case map[string]interface{}:
-		return toMapStringString(value), nil
+		return toMapStringString(value, false), nil
 	case types.Dict:
-		return toMapStringString(value), nil
+		return toMapStringString(value, false), nil
 	case map[string]string:
 		return value, nil
 	default:
@@ -592,23 +610,27 @@ func transformStringList(data interface{}) (interface{}, error) {
 	}
 }
 
-func transformMappingOrList(mappingOrList interface{}, sep string) map[string]string {
-	if mapping, ok := mappingOrList.(types.Dict); ok {
-		return toMapStringString(mapping)
-	}
-	if list, ok := mappingOrList.([]interface{}); ok {
-		result := make(map[string]string)
-		for _, value := range list {
+func transformMappingOrList(mappingOrList interface{}, sep string, allowNil bool) interface{} {
+	switch value := mappingOrList.(type) {
+	case types.Dict:
+		return toMapStringString(value, allowNil)
+	case ([]interface{}):
+		result := make(map[string]interface{})
+		for _, value := range value {
 			parts := strings.SplitN(value.(string), sep, 2)
-			if len(parts) == 1 {
-				result[parts[0]] = ""
-			} else {
-				result[parts[0]] = parts[1]
+			key := parts[0]
+			switch {
+			case len(parts) == 1 && allowNil:
+				result[key] = nil
+			case len(parts) == 1 && !allowNil:
+				result[key] = ""
+			default:
+				result[key] = parts[1]
 			}
 		}
 		return result
 	}
-	panic(fmt.Errorf("expected a map or a slice, got: %#v", mappingOrList))
+	panic(fmt.Errorf("expected a map or a list, got %T: %#v", mappingOrList, mappingOrList))
 }
 
 func transformShellCommand(value interface{}) (interface{}, error) {
@@ -672,17 +694,21 @@ func toServicePortConfigs(value string) ([]interface{}, error) {
 	return portConfigs, nil
 }
 
-func toMapStringString(value map[string]interface{}) map[string]string {
-	output := make(map[string]string)
+func toMapStringString(value map[string]interface{}, allowNil bool) map[string]interface{} {
+	output := make(map[string]interface{})
 	for key, value := range value {
-		output[key] = toString(value)
+		output[key] = toString(value, allowNil)
 	}
 	return output
 }
 
-func toString(value interface{}) string {
-	if value == nil {
+func toString(value interface{}, allowNil bool) interface{} {
+	switch {
+	case value != nil:
+		return fmt.Sprint(value)
+	case allowNil:
+		return nil
+	default:
 		return ""
 	}
-	return fmt.Sprint(value)
 }

cli/compose/loader/loader_test.go

@@ -12,7 +12,7 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func buildConfigDetails(source types.Dict) types.ConfigDetails {
+func buildConfigDetails(source types.Dict, env map[string]string) types.ConfigDetails {
 	workingDir, err := os.Getwd()
 	if err != nil {
 		panic(err)
@@ -23,10 +23,23 @@ func buildConfigDetails(source types.Dict) types.ConfigDetails {
 		ConfigFiles: []types.ConfigFile{
 			{Filename: "filename.yml", Config: source},
 		},
-		Environment: nil,
+		Environment: env,
 	}
 }
 
+func loadYAML(yaml string) (*types.Config, error) {
+	return loadYAMLWithEnv(yaml, nil)
+}
+
+func loadYAMLWithEnv(yaml string, env map[string]string) (*types.Config, error) {
+	dict, err := ParseYAML([]byte(yaml))
+	if err != nil {
+		return nil, err
+	}
+
+	return Load(buildConfigDetails(dict, env))
+}
+
 var sampleYAML = `
 version: "3"
 services:
@@ -98,12 +111,16 @@ var sampleDict = types.Dict{
 	},
 }
 
+func strPtr(val string) *string {
+	return &val
+}
+
 var sampleConfig = types.Config{
 	Services: []types.ServiceConfig{
 		{
 			Name:        "foo",
 			Image:       "busybox",
-			Environment: map[string]string{},
+			Environment: map[string]*string{},
 			Networks: map[string]*types.ServiceNetworkConfig{
 				"with_me": nil,
 			},
@@ -111,7 +128,7 @@ var sampleConfig = types.Config{
 		{
 			Name:        "bar",
 			Image:       "busybox",
-			Environment: map[string]string{"FOO": "1"},
+			Environment: map[string]*string{"FOO": strPtr("1")},
 			Networks: map[string]*types.ServiceNetworkConfig{
 				"with_ipam": nil,
 			},
@@ -154,7 +171,7 @@ func TestParseYAML(t *testing.T) {
 }
 
 func TestLoad(t *testing.T) {
-	actual, err := Load(buildConfigDetails(sampleDict))
+	actual, err := Load(buildConfigDetails(sampleDict, nil))
 	if !assert.NoError(t, err) {
 		return
 	}
@@ -373,8 +390,8 @@ services:
 	assert.Contains(t, err.Error(), "services.foo.image must be a string")
 }
 
-func TestValidEnvironment(t *testing.T) {
-	config, err := loadYAML(`
+func TestLoadWithEnvironment(t *testing.T) {
+	config, err := loadYAMLWithEnv(`
 version: "3"
 services:
   dict-env:
@@ -383,6 +400,7 @@ services:
       FOO: "1"
       BAR: 2
       BAZ: 2.5
+      QUX:
       QUUX:
   list-env:
     image: busybox
@@ -390,15 +408,17 @@ services:
       - FOO=1
       - BAR=2
       - BAZ=2.5
-      - QUUX=
-`)
+      - QUX=
+      - QUUX
+`, map[string]string{"QUX": "qux"})
 	assert.NoError(t, err)
 
 	expected := types.MappingWithEquals{
-		"FOO":  "1",
-		"BAR":  "2",
-		"BAZ":  "2.5",
-		"QUUX": "",
+		"FOO":  strPtr("1"),
+		"BAR":  strPtr("2"),
+		"BAZ":  strPtr("2.5"),
+		"QUX":  strPtr("qux"),
+		"QUUX": nil,
 	}
 
 	assert.Equal(t, 2, len(config.Services))
@@ -434,7 +454,8 @@ services:
 }
 
 func TestEnvironmentInterpolation(t *testing.T) {
-	config, err := loadYAML(`
+	home := "/home/foo"
+	config, err := loadYAMLWithEnv(`
 version: "3"
 services:
   test:
@@ -450,13 +471,14 @@ networks:
 volumes:
   test:
     driver: $HOME
-`)
+`, map[string]string{
+		"HOME": home,
+		"FOO":  "foo",
+	})
 
 	assert.NoError(t, err)
 
-	home := os.Getenv("HOME")
-
-	expectedLabels := types.MappingWithEquals{
+	expectedLabels := types.Labels{
 		"home1":       home,
 		"home2":       home,
 		"nonexistent": "",
@@ -483,7 +505,7 @@ services:
 `))
 	assert.NoError(t, err)
 
-	configDetails := buildConfigDetails(dict)
+	configDetails := buildConfigDetails(dict, nil)
 
 	_, err = Load(configDetails)
 	assert.NoError(t, err)
@@ -506,7 +528,7 @@ services:
 `))
 	assert.NoError(t, err)
 
-	configDetails := buildConfigDetails(dict)
+	configDetails := buildConfigDetails(dict, nil)
 
 	_, err = Load(configDetails)
 	assert.NoError(t, err)
@@ -601,7 +623,9 @@ func TestFullExample(t *testing.T) {
 	bytes, err := ioutil.ReadFile("full-example.yml")
 	assert.NoError(t, err)
 
-	config, err := loadYAML(string(bytes))
+	homeDir := "/home/foo"
+	env := map[string]string{"HOME": homeDir, "QUX": "qux_from_environment"}
+	config, err := loadYAMLWithEnv(string(bytes), env)
 	if !assert.NoError(t, err) {
 		return
 	}
@@ -609,7 +633,6 @@ func TestFullExample(t *testing.T) {
 	workingDir, err := os.Getwd()
 	assert.NoError(t, err)
 
-	homeDir := os.Getenv("HOME")
 	stopGracePeriod := time.Duration(20 * time.Second)
 
 	expectedServiceConfig := types.ServiceConfig{
@@ -657,13 +680,11 @@ func TestFullExample(t *testing.T) {
 		DNSSearch:  []string{"dc1.example.com", "dc2.example.com"},
 		DomainName: "foo.com",
 		Entrypoint: []string{"/code/entrypoint.sh", "-p", "3000"},
-		Environment: map[string]string{
-			"RACK_ENV":       "development",
-			"SHOW":           "true",
-			"SESSION_SECRET": "",
-			"FOO":            "1",
-			"BAR":            "2",
-			"BAZ":            "3",
+		Environment: map[string]*string{
+			"FOO": strPtr("foo_from_env_file"),
+			"BAR": strPtr("bar_from_env_file_2"),
+			"BAZ": strPtr("baz_from_service_def"),
+			"QUX": strPtr("qux_from_environment"),
 		},
 		EnvFile: []string{
 			"./example1.env",
@@ -955,15 +976,6 @@ func TestFullExample(t *testing.T) {
 	assert.Equal(t, expectedVolumeConfig, config.Volumes)
 }
 
-func loadYAML(yaml string) (*types.Config, error) {
-	dict, err := ParseYAML([]byte(yaml))
-	if err != nil {
-		return nil, err
-	}
-
-	return Load(buildConfigDetails(dict))
-}
-
 func serviceSort(services []types.ServiceConfig) []types.ServiceConfig {
 	sort.Sort(servicesByName(services))
 	return services

cli/compose/schema/data/config_schema_v3.2.json

@@ -1,6 +1,6 @@
 {
   "$schema": "http://json-schema.org/draft-04/schema#",
-  "id": "config_schema_v3.1.json",
+  "id": "config_schema_v3.2.json",
   "type": "object",
   "required": ["version"],
 

cli/compose/types/types.go

@@ -99,7 +99,7 @@ type ServiceConfig struct {
 	HealthCheck     *HealthCheckConfig
 	Image           string
 	Ipc             string
-	Labels          MappingWithEquals
+	Labels          Labels
 	Links           []string
 	Logging         *LoggingConfig
 	MacAddress      string `mapstructure:"mac_address"`
@@ -134,8 +134,13 @@ type StringList []string
 type StringOrNumberList []string
 
 // MappingWithEquals is a mapping type that can be converted from a list of
-// key=value strings
-type MappingWithEquals map[string]string
+// key[=value] strings.
+// For the key with an empty value (`key=`), the mapped value is set to a pointer to `""`.
+// For the key without value (`key`), the mapped value is set to nil.
+type MappingWithEquals map[string]*string
+
+// Labels is a mapping type for labels
+type Labels map[string]string
 
 // MappingWithColon is a mapping type that can be converted from a list of
 // 'key: value' strings
@@ -151,7 +156,7 @@ type LoggingConfig struct {
 type DeployConfig struct {
 	Mode          string
 	Replicas      *uint64
-	Labels        MappingWithEquals
+	Labels        Labels
 	UpdateConfig  *UpdateConfig `mapstructure:"update_config"`
 	Resources     Resources
 	RestartPolicy *RestartPolicy `mapstructure:"restart_policy"`
@@ -268,7 +273,7 @@ type NetworkConfig struct {
 	External   External
 	Internal   bool
 	Attachable bool
-	Labels     MappingWithEquals
+	Labels     Labels
 }
 
 // IPAMConfig for a network
@@ -287,7 +292,7 @@ type VolumeConfig struct {
 	Driver     string
 	DriverOpts map[string]string `mapstructure:"driver_opts"`
 	External   External
-	Labels     MappingWithEquals
+	Labels     Labels
 }
 
 // External identifies a Volume or Network as a reference to a resource that is
@@ -301,5 +306,5 @@ type External struct {
 type SecretConfig struct {
 	File     string
 	External External
-	Labels   MappingWithEquals
+	Labels   Labels
 }

cmd/docker/daemon_unix.go

@@ -24,7 +24,7 @@ func newDaemonCommand() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return runDaemon()
 		},
-		Deprecated: "and will be removed in Docker 1.16. Please run `dockerd` directly.",
+		Deprecated: "and will be removed in Docker 17.12. Please run `dockerd` directly.",
 	}
 	cmd.SetHelpFunc(helpFunc)
 	return cmd

contrib/completion/zsh/_docker

@@ -978,6 +978,7 @@ __docker_image_subcommand() {
         (build)
             _arguments $(__docker_arguments) \
                 $opts_help \
+                "($help)*--add-host=[Add a custom host-to-IP mapping]:host\:ip mapping: " \
                 "($help)*--build-arg=[Build-time variables]:<varname>=<value>: " \
                 "($help)*--cache-from=[Images to consider as cache sources]: :__docker_complete_repositories_with_tags" \
                 "($help -c --cpu-shares)"{-c=,--cpu-shares=}"[CPU shares (relative weight)]:CPU shares:(0 10 100 200 500 800 1000)" \

daemon/attach.go

@@ -36,7 +36,7 @@ func (daemon *Daemon) ContainerAttach(prefixOrName string, c *backend.ContainerA
 	}
 
 	cfg := stream.AttachConfig{
-		UseStdin:   c.UseStdin && container.Config.OpenStdin,
+		UseStdin:   c.UseStdin,
 		UseStdout:  c.UseStdout,
 		UseStderr:  c.UseStderr,
 		TTY:        container.Config.Tty,
@@ -79,7 +79,7 @@ func (daemon *Daemon) ContainerAttachRaw(prefixOrName string, stdin io.ReadClose
 		return err
 	}
 	cfg := stream.AttachConfig{
-		UseStdin:   stdin != nil && container.Config.OpenStdin,
+		UseStdin:   stdin != nil,
 		UseStdout:  stdout != nil,
 		UseStderr:  stderr != nil,
 		TTY:        container.Config.Tty,
@@ -147,6 +147,10 @@ func (daemon *Daemon) containerAttach(c *container.Container, cfg *stream.Attach
 		cfg.Stdin = r
 	}
 
+	if !c.Config.OpenStdin {
+		cfg.Stdin = nil
+	}
+
 	waitChan := make(chan struct{})
 	if c.Config.StdinOnce && !c.Config.Tty {
 		defer func() {

daemon/daemon_windows.go

@@ -6,7 +6,6 @@ import (
 	"path/filepath"
 	"strings"
 	"syscall"
-	"unsafe"
 
 	"github.com/Microsoft/hcsshim"
 	"github.com/Sirupsen/logrus"
@@ -245,7 +244,7 @@ func checkSystem() error {
 		return fmt.Errorf("Failed to load vmcompute.dll. Ensure that the Containers role is installed.")
 	}
 
-	return waitOOBEComplete()
+	return nil
 }
 
 // configureKernelSecuritySupport configures and validate security support for the kernel
@@ -617,35 +616,3 @@ func (daemon *Daemon) verifyVolumesInfo(container *container.Container) error {
 func (daemon *Daemon) setupSeccompProfile() error {
 	return nil
 }
-
-func waitOOBEComplete() error {
-	kernel32 := windows.NewLazySystemDLL("kernel32.dll")
-	registerWaitUntilOOBECompleted := kernel32.NewProc("RegisterWaitUntilOOBECompleted")
-	unregisterWaitUntilOOBECompleted := kernel32.NewProc("UnregisterWaitUntilOOBECompleted")
-
-	callbackChan := make(chan struct{})
-	callbackFunc := func(uintptr) uintptr {
-		close(callbackChan)
-		return 0
-	}
-	callbackFuncPtr := syscall.NewCallback(callbackFunc)
-
-	var callbackHandle syscall.Handle
-	ret, _, err := registerWaitUntilOOBECompleted.Call(callbackFuncPtr, 0, uintptr(unsafe.Pointer(&callbackHandle)))
-	if ret == 0 {
-		if err == errInvalidState {
-			return nil
-		}
-		return fmt.Errorf("failed to register OOBEComplete callback. Error: %v", err)
-	}
-
-	// Wait for the callback when OOBE is finished
-	<-callbackChan
-
-	ret, _, err = unregisterWaitUntilOOBECompleted.Call(uintptr(callbackHandle))
-	if ret == 0 {
-		return fmt.Errorf("failed to unregister OOBEComplete callback. Error: %v", err)
-	}
-
-	return nil
-}

docs/deprecated.md

@@ -266,9 +266,9 @@ of the `--changes` flag that allows to pass `Dockerfile` commands.
 
 ### Interacting with V1 registries
 
-**Disabled By Default In Release: v17.04**
+**Disabled By Default In Release: v17.06**
 
-**Target For Removal In Release: v17.10**
+**Target For Removal In Release: v17.12**
 
 Version 1.9 adds a flag (`--disable-legacy-registry=false`) which prevents the
 docker daemon from `pull`, `push`, and `login` operations against v1

docs/reference/commandline/container.md

@@ -0,0 +1,61 @@
+
+---
+title: "container"
+description: "The container command description and usage"
+keywords: "container"
+---
+
+<!-- This file is maintained within the docker/docker Github
+     repository at https://github.com/docker/docker/. Make all
+     pull requests against that repo. If you see this file in
+     another repository, consider it read-only there, as it will
+     periodically be overwritten by the definitive file. Pull
+     requests which include edits to this file in other repositories
+     will be rejected.
+-->
+
+# container
+
+```markdown
+Usage:  docker container COMMAND
+
+Manage containers
+
+Options:
+      --help   Print usage
+
+Commands:
+  attach      Attach to a running container
+  commit      Create a new image from a container's changes
+  cp          Copy files/folders between a container and the local filesystem
+  create      Create a new container
+  diff        Inspect changes to files or directories on a container's filesystem
+  exec        Run a command in a running container
+  export      Export a container's filesystem as a tar archive
+  inspect     Display detailed information on one or more containers
+  kill        Kill one or more running containers
+  logs        Fetch the logs of a container
+  ls          List containers
+  pause       Pause all processes within one or more containers
+  port        List port mappings or a specific mapping for the container
+  prune       Remove all stopped containers
+  rename      Rename a container
+  restart     Restart one or more containers
+  rm          Remove one or more containers
+  run         Run a command in a new container
+  start       Start one or more stopped containers
+  stats       Display a live stream of container(s) resource usage statistics
+  stop        Stop one or more running containers
+  top         Display the running processes of a container
+  unpause     Unpause all processes within one or more containers
+  update      Update configuration of one or more containers
+  wait        Block until one or more containers stop, then print their exit codes
+
+Run 'docker container COMMAND --help' for more information on a command.
+
+```
+
+## Description
+
+Manage containers.
+

docs/reference/commandline/image.md

@@ -0,0 +1,47 @@
+
+---
+title: "image"
+description: "The image command description and usage"
+keywords: "image"
+---
+
+<!-- This file is maintained within the docker/docker Github
+     repository at https://github.com/docker/docker/. Make all
+     pull requests against that repo. If you see this file in
+     another repository, consider it read-only there, as it will
+     periodically be overwritten by the definitive file. Pull
+     requests which include edits to this file in other repositories
+     will be rejected.
+-->
+
+# image
+
+```markdown
+Usage:  docker image COMMAND
+
+Manage images
+
+Options:
+      --help   Print usage
+
+Commands:
+  build       Build an image from a Dockerfile
+  history     Show the history of an image
+  import      Import the contents from a tarball to create a filesystem image
+  inspect     Display detailed information on one or more images
+  load        Load an image from a tar archive or STDIN
+  ls          List images
+  prune       Remove unused images
+  pull        Pull an image or a repository from a registry
+  push        Push an image or a repository to a registry
+  rm          Remove one or more images
+  save        Save one or more images to a tar archive (streamed to STDOUT by default)
+  tag         Create a tag TARGET_IMAGE that refers to SOURCE_IMAGE
+
+Run 'docker image COMMAND --help' for more information on a command.
+
+```
+
+## Description
+
+Manage images.

docs/reference/commandline/load.md

@@ -26,7 +26,7 @@ Options:
                        The tarball may be compressed with gzip, bzip, or xz
   -q, --quiet          Suppress the load output but still outputs the imported images
 ```
-## Descriptino
+## Description
 
 `docker load` loads a tarred repository from a file or the standard input stream.
 It restores both images and tags.

docs/reference/commandline/network.md

@@ -0,0 +1,49 @@
+---
+title: "network"
+description: "The network command description and usage"
+keywords: "network"
+---
+
+<!-- This file is maintained within the docker/docker Github
+     repository at https://github.com/docker/docker/. Make all
+     pull requests against that repo. If you see this file in
+     another repository, consider it read-only there, as it will
+     periodically be overwritten by the definitive file. Pull
+     requests which include edits to this file in other repositories
+     will be rejected.
+-->
+
+# network
+
+```markdown
+Usage:  docker network COMMAND
+
+Manage networks
+
+Options:
+      --help   Print usage
+
+Commands:
+  connect     Connect a container to a network
+  create      Create a network
+  disconnect  Disconnect a container from a network
+  inspect     Display detailed information on one or more networks
+  ls          List networks
+  prune       Remove all unused networks
+  rm          Remove one or more networks
+
+Run 'docker network COMMAND --help' for more information on a command.
+```
+
+## Description
+
+Manage networks. You can use subcommand to create, list, inspect, remove,
+connect and disconnect networks.
+
+## Related commands
+
+* [network create](network_create.md)
+* [network inspect](network_inspect.md)
+* [network list](network_list.md)
+* [network rm](network_rm.md)
+* [network prune](network_prune.md)

docs/reference/commandline/node.md

@@ -0,0 +1,42 @@
+
+---
+title: "node"
+description: "The node command description and usage"
+keywords: "node"
+---
+
+<!-- This file is maintained within the docker/docker Github
+     repository at https://github.com/docker/docker/. Make all
+     pull requests against that repo. If you see this file in
+     another repository, consider it read-only there, as it will
+     periodically be overwritten by the definitive file. Pull
+     requests which include edits to this file in other repositories
+     will be rejected.
+-->
+
+# node
+
+```markdown
+Usage:  docker node COMMAND
+
+Manage Swarm nodes
+
+Options:
+      --help   Print usage
+
+Commands:
+  demote      Demote one or more nodes from manager in the swarm
+  inspect     Display detailed information on one or more nodes
+  ls          List nodes in the swarm
+  promote     Promote one or more nodes to manager in the swarm
+  ps          List tasks running on one or more nodes, defaults to current node
+  rm          Remove one or more nodes from the swarm
+  update      Update a node
+
+Run 'docker node COMMAND --help' for more information on a command.
+```
+
+## Description
+
+Manage nodes.
+

docs/reference/commandline/plugin.md

@@ -0,0 +1,44 @@
+---
+title: "plugin"
+description: "The plugin command description and usage"
+keywords: "plugin"
+---
+
+<!-- This file is maintained within the docker/docker Github
+     repository at https://github.com/docker/docker/. Make all
+     pull requests against that repo. If you see this file in
+     another repository, consider it read-only there, as it will
+     periodically be overwritten by the definitive file. Pull
+     requests which include edits to this file in other repositories
+     will be rejected.
+-->
+
+# plugin
+
+```markdown
+Usage:  docker plugin COMMAND
+
+Manage plugins
+
+Options:
+      --help   Print usage
+
+Commands:
+  create      Create a plugin from a rootfs and configuration. Plugin data directory must contain config.json and rootfs directory.
+  disable     Disable a plugin
+  enable      Enable a plugin
+  inspect     Display detailed information on one or more plugins
+  install     Install a plugin
+  ls          List plugins
+  push        Push a plugin to a registry
+  rm          Remove one or more plugins
+  set         Change settings for a plugin
+  upgrade     Upgrade an existing plugin
+
+Run 'docker plugin COMMAND --help' for more information on a command.
+
+```
+
+## Description
+
+Manage plugins.

docs/reference/commandline/secret.md

@@ -0,0 +1,45 @@
+---
+title: "secret"
+description: "The secret command description and usage"
+keywords: "secret"
+---
+
+<!-- This file is maintained within the docker/docker Github
+     repository at https://github.com/docker/docker/. Make all
+     pull requests against that repo. If you see this file in
+     another repository, consider it read-only there, as it will
+     periodically be overwritten by the definitive file. Pull
+     requests which include edits to this file in other repositories
+     will be rejected.
+-->
+
+# secret
+
+```markdown
+Usage:  docker secret COMMAND
+
+Manage Docker secrets
+
+Options:
+      --help   Print usage
+
+Commands:
+  create      Create a secret from a file or STDIN as content
+  inspect     Display detailed information on one or more secrets
+  ls          List secrets
+  rm          Remove one or more secrets
+
+Run 'docker secret COMMAND --help' for more information on a command.
+
+```
+
+## Description
+
+Manage secrets.
+
+## Related commands
+
+* [secret create](secret_create.md)
+* [secret inspect](secret_inspect.md)
+* [secret list](secret_list.md)
+* [secret rm](secret_rm.md)

docs/reference/commandline/service.md

@@ -0,0 +1,42 @@
+---
+title: "service"
+description: "The service command description and usage"
+keywords: "service"
+---
+
+<!-- This file is maintained within the docker/docker Github
+     repository at https://github.com/docker/docker/. Make all
+     pull requests against that repo. If you see this file in
+     another repository, consider it read-only there, as it will
+     periodically be overwritten by the definitive file. Pull
+     requests which include edits to this file in other repositories
+     will be rejected.
+-->
+
+# service
+
+```markdown
+Usage:  docker service COMMAND
+
+Manage services
+
+Options:
+      --help   Print usage
+
+Commands:
+  create      Create a new service
+  inspect     Display detailed information on one or more services
+  logs        Fetch the logs of a service
+  ls          List services
+  ps          List the tasks of a service
+  rm          Remove one or more services
+  scale       Scale one or multiple replicated services
+  update      Update a service
+
+Run 'docker service COMMAND --help' for more information on a command.
+```
+
+## Description
+
+Manage services.
+

docs/reference/commandline/stack.md

@@ -0,0 +1,39 @@
+---
+title: "stack"
+description: "The stack command description and usage"
+keywords: "stack"
+---
+
+<!-- This file is maintained within the docker/docker Github
+     repository at https://github.com/docker/docker/. Make all
+     pull requests against that repo. If you see this file in
+     another repository, consider it read-only there, as it will
+     periodically be overwritten by the definitive file. Pull
+     requests which include edits to this file in other repositories
+     will be rejected.
+-->
+
+# stack
+
+```markdown
+Usage:  docker stack COMMAND
+
+Manage Docker stacks
+
+Options:
+      --help   Print usage
+
+Commands:
+  deploy      Deploy a new stack or update an existing stack
+  ls          List stacks
+  ps          List the tasks in the stack
+  rm          Remove the stack
+  services    List the services in the stack
+
+Run 'docker stack COMMAND --help' for more information on a command.
+```
+
+## Description
+
+Manage stacks.
+

docs/reference/commandline/stack_ls.md

@@ -27,7 +27,7 @@ Options:
       --help   Print usage
 ```
 
-## Descriptino
+## Description
 
 Lists the stacks.
 

docs/reference/commandline/swarm.md

@@ -0,0 +1,40 @@
+---
+title: "swarm"
+description: "The swarm command description and usage"
+keywords: "swarm"
+---
+
+<!-- This file is maintained within the docker/docker Github
+     repository at https://github.com/docker/docker/. Make all
+     pull requests against that repo. If you see this file in
+     another repository, consider it read-only there, as it will
+     periodically be overwritten by the definitive file. Pull
+     requests which include edits to this file in other repositories
+     will be rejected.
+-->
+
+# swarm
+
+```markdown
+Usage:  docker swarm COMMAND
+
+Manage Swarm
+
+Options:
+      --help   Print usage
+
+Commands:
+  init        Initialize a swarm
+  join        Join a swarm as a node and/or manager
+  join-token  Manage join tokens
+  leave       Leave the swarm
+  unlock      Unlock swarm
+  unlock-key  Manage the unlock key
+  update      Update the swarm
+
+Run 'docker swarm COMMAND --help' for more information on a command.
+```
+
+## Description
+
+Manage the swarm.

docs/reference/commandline/system.md

@@ -0,0 +1,37 @@
+---
+title: "system"
+description: "The system command description and usage"
+keywords: "system"
+---
+
+<!-- This file is maintained within the docker/docker Github
+     repository at https://github.com/docker/docker/. Make all
+     pull requests against that repo. If you see this file in
+     another repository, consider it read-only there, as it will
+     periodically be overwritten by the definitive file. Pull
+     requests which include edits to this file in other repositories
+     will be rejected.
+-->
+
+# system
+
+```markdown
+Usage:  docker system COMMAND
+
+Manage Docker
+
+Options:
+      --help   Print usage
+
+Commands:
+  df          Show docker disk usage
+  events      Get real time events from the server
+  info        Display system-wide information
+  prune       Remove unused data
+
+Run 'docker system COMMAND --help' for more information on a command.
+```
+
+## Description
+
+Manage docker.

docs/reference/commandline/volume.md

@@ -0,0 +1,48 @@
+---
+title: "volume"
+description: "The volume command description and usage"
+keywords: "volume"
+---
+
+<!-- This file is maintained within the docker/docker Github
+     repository at https://github.com/docker/docker/. Make all
+     pull requests against that repo. If you see this file in
+     another repository, consider it read-only there, as it will
+     periodically be overwritten by the definitive file. Pull
+     requests which include edits to this file in other repositories
+     will be rejected.
+-->
+
+# volume
+
+```markdown
+Usage:  docker volume COMMAND
+
+Manage volumes
+
+Options:
+      --help   Print usage
+
+Commands:
+  create      Create a volume
+  inspect     Display detailed information on one or more volumes
+  ls          List volumes
+  prune       Remove all unused volumes
+  rm          Remove one or more volumes
+
+Run 'docker volume COMMAND --help' for more information on a command.
+```
+
+## Description
+
+Manage volumes. You can use subcommand to create, list, inspect, remove
+volumes.
+
+## Related commands
+
+* [volume create](volume_create.md)
+* [volume inspect](volume_inspect.md)
+* [volume list](volume_list.md)
+* [volume rm](volume_rm.md)
+* [volume prune](volume_prune.md)
+* [Understand Data Volumes](https://docs.docker.com/engine/tutorials/dockervolumes/)

docs/yaml/Dockerfile

@@ -0,0 +1,4 @@
+FROM scratch
+COPY docs /docs
+# CMD cannot be nil so we set it to empty string
+CMD  [""]

docs/yaml/generate.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/docker/docker/cli/command"
+	"github.com/docker/docker/cli/command/commands"
+	"github.com/docker/docker/pkg/term"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+)
+
+const descriptionSourcePath = "docs/reference/commandline/"
+
+func generateCliYaml(opts *options) error {
+	stdin, stdout, stderr := term.StdStreams()
+	dockerCli := command.NewDockerCli(stdin, stdout, stderr)
+	cmd := &cobra.Command{Use: "docker"}
+	commands.AddCommands(cmd, dockerCli)
+	source := filepath.Join(opts.source, descriptionSourcePath)
+	if err := loadLongDescription(cmd, source); err != nil {
+		return err
+	}
+
+	cmd.DisableAutoGenTag = true
+	return GenYamlTree(cmd, opts.target)
+}
+
+func loadLongDescription(cmd *cobra.Command, path ...string) error {
+	for _, cmd := range cmd.Commands() {
+		if cmd.Name() == "" {
+			continue
+		}
+		fullpath := filepath.Join(path[0], strings.Join(append(path[1:], cmd.Name()), "_")+".md")
+
+		if cmd.HasSubCommands() {
+			loadLongDescription(cmd, path[0], cmd.Name())
+		}
+
+		if _, err := os.Stat(fullpath); err != nil {
+			log.Printf("WARN: %s does not exist, skipping\n", fullpath)
+			continue
+		}
+
+		content, err := ioutil.ReadFile(fullpath)
+		if err != nil {
+			return err
+		}
+		description, examples := parseMDContent(string(content))
+		cmd.Long = description
+		cmd.Example = examples
+	}
+	return nil
+}
+
+type options struct {
+	source string
+	target string
+}
+
+func parseArgs() (*options, error) {
+	opts := &options{}
+	cwd, _ := os.Getwd()
+	flags := pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError)
+	flags.StringVar(&opts.source, "root", cwd, "Path to project root")
+	flags.StringVar(&opts.target, "target", "/tmp", "Target path for generated yaml files")
+	err := flags.Parse(os.Args[1:])
+	return opts, err
+}
+
+func main() {
+	opts, err := parseArgs()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+	}
+	fmt.Printf("Project root: %s\n", opts.source)
+	fmt.Printf("Generating yaml files into %s\n", opts.target)
+	if err := generateCliYaml(opts); err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to generate yaml files: %s\n", err.Error())
+	}
+}

docs/yaml/yaml.go

@@ -0,0 +1,212 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"gopkg.in/yaml.v2"
+)
+
+type cmdOption struct {
+	Option       string
+	Shorthand    string `yaml:",omitempty"`
+	DefaultValue string `yaml:"default_value,omitempty"`
+	Description  string `yaml:",omitempty"`
+}
+
+type cmdDoc struct {
+	Name             string      `yaml:"command"`
+	SeeAlso          []string    `yaml:"parent,omitempty"`
+	Version          string      `yaml:"engine_version,omitempty"`
+	Aliases          string      `yaml:",omitempty"`
+	Short            string      `yaml:",omitempty"`
+	Long             string      `yaml:",omitempty"`
+	Usage            string      `yaml:",omitempty"`
+	Pname            string      `yaml:",omitempty"`
+	Plink            string      `yaml:",omitempty"`
+	Cname            []string    `yaml:",omitempty"`
+	Clink            []string    `yaml:",omitempty"`
+	Options          []cmdOption `yaml:",omitempty"`
+	InheritedOptions []cmdOption `yaml:"inherited_options,omitempty"`
+	Example          string      `yaml:"examples,omitempty"`
+}
+
+// GenYamlTree creates yaml structured ref files
+func GenYamlTree(cmd *cobra.Command, dir string) error {
+	identity := func(s string) string { return s }
+	emptyStr := func(s string) string { return "" }
+	return GenYamlTreeCustom(cmd, dir, emptyStr, identity)
+}
+
+// GenYamlTreeCustom creates yaml structured ref files
+func GenYamlTreeCustom(cmd *cobra.Command, dir string, filePrepender, linkHandler func(string) string) error {
+	for _, c := range cmd.Commands() {
+		if !c.IsAvailableCommand() || c.IsHelpCommand() {
+			continue
+		}
+		if err := GenYamlTreeCustom(c, dir, filePrepender, linkHandler); err != nil {
+			return err
+		}
+	}
+
+	basename := strings.Replace(cmd.CommandPath(), " ", "_", -1) + ".yaml"
+	filename := filepath.Join(dir, basename)
+	f, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	if _, err := io.WriteString(f, filePrepender(filename)); err != nil {
+		return err
+	}
+	if err := GenYamlCustom(cmd, f, linkHandler); err != nil {
+		return err
+	}
+	return nil
+}
+
+// GenYamlCustom creates custom yaml output
+func GenYamlCustom(cmd *cobra.Command, w io.Writer, linkHandler func(string) string) error {
+	cliDoc := cmdDoc{}
+	cliDoc.Name = cmd.CommandPath()
+
+	// Check experimental: ok := cmd.Tags["experimental"]
+
+	cliDoc.Aliases = strings.Join(cmd.Aliases, ", ")
+	cliDoc.Short = cmd.Short
+	cliDoc.Long = cmd.Long
+	if len(cliDoc.Long) == 0 {
+		cliDoc.Long = cliDoc.Short
+	}
+
+	if cmd.Runnable() {
+		cliDoc.Usage = cmd.UseLine()
+	}
+
+	if len(cmd.Example) > 0 {
+		cliDoc.Example = cmd.Example
+	}
+
+	flags := cmd.NonInheritedFlags()
+	if flags.HasFlags() {
+		cliDoc.Options = genFlagResult(flags)
+	}
+	flags = cmd.InheritedFlags()
+	if flags.HasFlags() {
+		cliDoc.InheritedOptions = genFlagResult(flags)
+	}
+
+	if hasSeeAlso(cmd) {
+		if cmd.HasParent() {
+			parent := cmd.Parent()
+			cliDoc.Pname = parent.CommandPath()
+			link := cliDoc.Pname + ".yaml"
+			cliDoc.Plink = strings.Replace(link, " ", "_", -1)
+			cmd.VisitParents(func(c *cobra.Command) {
+				if c.DisableAutoGenTag {
+					cmd.DisableAutoGenTag = c.DisableAutoGenTag
+				}
+			})
+		}
+
+		children := cmd.Commands()
+		sort.Sort(byName(children))
+
+		for _, child := range children {
+			if !child.IsAvailableCommand() || child.IsHelpCommand() {
+				continue
+			}
+			currentChild := cliDoc.Name + " " + child.Name()
+			cliDoc.Cname = append(cliDoc.Cname, cliDoc.Name+" "+child.Name())
+			link := currentChild + ".yaml"
+			cliDoc.Clink = append(cliDoc.Clink, strings.Replace(link, " ", "_", -1))
+		}
+	}
+
+	final, err := yaml.Marshal(&cliDoc)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	if _, err := fmt.Fprintln(w, string(final)); err != nil {
+		return err
+	}
+	return nil
+}
+
+func genFlagResult(flags *pflag.FlagSet) []cmdOption {
+	var result []cmdOption
+
+	flags.VisitAll(func(flag *pflag.Flag) {
+		// Todo, when we mark a shorthand is deprecated, but specify an empty message.
+		// The flag.ShorthandDeprecated is empty as the shorthand is deprecated.
+		// Using len(flag.ShorthandDeprecated) > 0 can't handle this, others are ok.
+		if !(len(flag.ShorthandDeprecated) > 0) && len(flag.Shorthand) > 0 {
+			opt := cmdOption{
+				Option:       flag.Name,
+				Shorthand:    flag.Shorthand,
+				DefaultValue: flag.DefValue,
+				Description:  forceMultiLine(flag.Usage),
+			}
+			result = append(result, opt)
+		} else {
+			opt := cmdOption{
+				Option:       flag.Name,
+				DefaultValue: forceMultiLine(flag.DefValue),
+				Description:  forceMultiLine(flag.Usage),
+			}
+			result = append(result, opt)
+		}
+	})
+
+	return result
+}
+
+// Temporary workaround for yaml lib generating incorrect yaml with long strings
+// that do not contain \n.
+func forceMultiLine(s string) string {
+	if len(s) > 60 && !strings.Contains(s, "\n") {
+		s = s + "\n"
+	}
+	return s
+}
+
+// Small duplication for cobra utils
+func hasSeeAlso(cmd *cobra.Command) bool {
+	if cmd.HasParent() {
+		return true
+	}
+	for _, c := range cmd.Commands() {
+		if !c.IsAvailableCommand() || c.IsHelpCommand() {
+			continue
+		}
+		return true
+	}
+	return false
+}
+
+func parseMDContent(mdString string) (description string, examples string) {
+	parsedContent := strings.Split(mdString, "\n## ")
+	for _, s := range parsedContent {
+		if strings.Index(s, "Description") == 0 {
+			description = strings.Trim(s, "Description\n")
+		}
+		if strings.Index(s, "Examples") == 0 {
+			examples = strings.Trim(s, "Examples\n")
+		}
+	}
+	return
+}
+
+type byName []*cobra.Command
+
+func (s byName) Len() int           { return len(s) }
+func (s byName) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
+func (s byName) Less(i, j int) bool { return s[i].Name() < s[j].Name() }

hack/dockerfile/binaries-commits

@@ -4,7 +4,7 @@ TOMLV_COMMIT=9baf8a8a9f2ed20a8e54160840c492f937eeaf9a
 
 # When updating RUNC_COMMIT, also update runc in vendor.conf accordingly
 RUNC_COMMIT=9c2d8d184e5da67c95d601382adf14862e4f2228
-CONTAINERD_COMMIT=7c92ec529830300eae0f524a45f7b7394104d4f5
+CONTAINERD_COMMIT=422e31ce907fd9c3833a38d7b8fdd023e5a76e73
 TINI_COMMIT=949e6facb77383876aeff8a6944dde66b3089574
 LIBNETWORK_COMMIT=7b2b1feb1de4817d522cc372af149ff48d25028e
 VNDR_COMMIT=19220953c4a45310a4c404b7905154e29120249e

hack/install.sh

@@ -366,6 +366,9 @@ do_install() {
 		debian|raspbian)
 			dist_version="$(cat /etc/debian_version | sed 's/\/.*//' | sed 's/\..*//')"
 			case "$dist_version" in
+				9)
+					dist_version="stretch"
+				;;
 				8)
 					dist_version="jessie"
 				;;

hack/make/yaml-docs-generator

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -e
+
+[ -z "$KEEPDEST" ] && \
+	rm -rf "$DEST"
+
+(
+	source "${MAKEDIR}/.binary-setup"
+	export BINARY_SHORT_NAME="yaml-docs-generator"
+	export GO_PACKAGE='github.com/docker/docker/docs/yaml'
+	source "${MAKEDIR}/.binary"
+)

hooks/post_build

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+if [ -n "${BUILD_DOCS}" ]; then
+	set -e
+	DOCS_IMAGE=${DOCS_IMAGE:-${IMAGE_NAME}-docs}
+	docker run \
+		--entrypoint '' \
+		--privileged \
+		-e DOCKER_GITCOMMIT=$(git rev-parse --short HEAD) \
+		-v $(pwd)/docs/yaml/docs:/docs \
+		"${IMAGE_NAME}" \
+		sh -c 'hack/make.sh yaml-docs-generator && bundles/latest/yaml-docs-generator/yaml-docs-generator --target /docs'
+
+	(
+		cd docs/yaml
+		docker build -t ${DOCS_IMAGE} .
+		docker push ${DOCS_IMAGE}
+	)
+fi

integration-cli/docker_cli_login_test.go

@@ -28,3 +28,17 @@ func (s *DockerRegistryAuthHtpasswdSuite) TestLoginToPrivateRegistry(c *check.C)
 	// now it's fine
 	dockerCmd(c, "login", "-u", s.reg.Username(), "-p", s.reg.Password(), privateRegistryURL)
 }
+
+func (s *DockerRegistryAuthHtpasswdSuite) TestLoginToPrivateRegistryDeprecatedEmailFlag(c *check.C) {
+	// Test to make sure login still works with the deprecated -e and --email flags
+	// wrong credentials
+	out, _, err := dockerCmdWithError("login", "-u", s.reg.Username(), "-p", "WRONGPASSWORD", "-e", s.reg.Email(), privateRegistryURL)
+	c.Assert(err, checker.NotNil, check.Commentf(out))
+	c.Assert(out, checker.Contains, "401 Unauthorized")
+
+	// now it's fine
+	// -e flag
+	dockerCmd(c, "login", "-u", s.reg.Username(), "-p", s.reg.Password(), "-e", s.reg.Email(), privateRegistryURL)
+	// --email flag
+	dockerCmd(c, "login", "-u", s.reg.Username(), "-p", s.reg.Password(), "--email", s.reg.Email(), privateRegistryURL)
+}

integration-cli/docker_cli_registry_user_agent_test.go

@@ -107,7 +107,7 @@ func (s *DockerRegistrySuite) TestUserAgentPassThrough(c *check.C) {
 	s.d.Cmd("build", "--file", dockerfileName, ".")
 	regexpCheckUA(c, buildUA)
 
-	s.d.Cmd("login", "-u", "richard", "-p", "testtest", loginReg.URL())
+	s.d.Cmd("login", "-u", "richard", "-p", "testtest", "-e", "testuser@testdomain.com", loginReg.URL())
 	regexpCheckUA(c, loginUA)
 
 	s.d.Cmd("pull", pullRepoName)

integration-cli/registry/registry.go

@@ -203,6 +203,11 @@ func (r *V2) Password() string {
 	return r.password
 }
 
+// Email returns the configured email of the server
+func (r *V2) Email() string {
+	return r.email
+}
+
 // Path returns the path where the registry write data
 func (r *V2) Path() string {
 	return filepath.Join(r.dir, "docker", "registry", "v2")

vendor.conf

@@ -1,7 +1,8 @@
 # the following lines are in sorted order, FYI
 github.com/Azure/go-ansiterm 388960b655244e76e24c75f48631564eaefade62
 github.com/Microsoft/hcsshim v0.5.12
-github.com/Microsoft/go-winio v0.3.8
+# TODO: get rid of this fork once PR https://github.com/Microsoft/go-winio/pull/43 is merged
+github.com/Microsoft/go-winio 7c7d6b461cb10872c1138a0d7f3acf9a41b5c353 https://github.com/dgageot/go-winio.git
 github.com/Sirupsen/logrus v0.11.0
 github.com/davecgh/go-spew 6d212800a42e8ab5c146b8ace3490ee17e5225f9
 github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a
@@ -16,7 +17,7 @@ github.com/vdemeester/shakers 24d7f1d6a71aa5d9cbe7390e4afb66b7eef9e1b3
 golang.org/x/net c427ad74c6d7a814201695e9ffde0c5d400a7674
 golang.org/x/sys 8f0908ab3b2457e2e15403d3697c9ef5cb4b57a9
 github.com/docker/go-units 9e638d38cf6977a37a8ea0078f3ee75a7cdb2dd1
-github.com/docker/go-connections 7da10c8c50cad14494ec818dcdfb6506265c0086
+github.com/docker/go-connections d217f8e36aba4dbc397981e692a65d3f13b9a46d
 golang.org/x/text f72d8390a633d5dfb0cc84043294db9f6c935756
 
 github.com/RackSec/srslog 456df3a81436d29ba874f3590eeeee25d666f8a5
@@ -100,11 +101,11 @@ google.golang.org/genproto b3e7c2fb04031add52c4817f53f43757ccbf9c18
 github.com/docker/docker-credential-helpers v0.5.0
 
 # containerd
-github.com/docker/containerd 7c92ec529830300eae0f524a45f7b7394104d4f5
+github.com/docker/containerd 422e31ce907fd9c3833a38d7b8fdd023e5a76e73
 github.com/tonistiigi/fifo 1405643975692217d6720f8b54aeee1bf2cd5cf4
 
 # cluster
-github.com/docker/swarmkit 0e2d9ebcea9d5bbd4a06b3b964fb96356801f880
+github.com/docker/swarmkit d316a73f803e9eb75e3daa7e0f846017b0c9a145
 github.com/golang/mock bd3c8e81be01eef76d4b503f5e687d2d1354d2d9
 github.com/gogo/protobuf 8d70fb3182befc465c4a1eac8ad4d38ff49778e2
 github.com/cloudflare/cfssl 7fb22c8cba7ecaf98e4082d22d65800cf45e042a

vendor/github.com/Microsoft/go-winio/file.go

@@ -219,3 +219,7 @@ func (f *win32File) SetWriteDeadline(t time.Time) error {
 	f.writeDeadline = t
 	return nil
 }
+
+func (f *win32File) Flush() error {
+	return syscall.FlushFileBuffers(f.handle)
+}

vendor/github.com/Microsoft/go-winio/pipe.go

@@ -87,7 +87,11 @@ func (f *win32MessageBytePipe) CloseWrite() error {
 	if f.writeClosed {
 		return errPipeWriteClosed
 	}
-	_, err := f.win32File.Write(nil)
+	err := f.win32File.Flush()
+	if err != nil {
+		return err
+	}
+	_, err = f.win32File.Write(nil)
 	if err != nil {
 		return err
 	}

vendor/github.com/docker/go-connections/tlsconfig/config.go

@@ -29,6 +29,11 @@ type Options struct {
 	InsecureSkipVerify bool
 	// server-only option
 	ClientAuth tls.ClientAuthType
+
+	// If ExclusiveRootPools is set, then if a CA file is provided, the root pool used for TLS
+	// creds will include exclusively the roots in that CA file.  If no CA file is provided,
+	// the system pool will be used.
+	ExclusiveRootPools bool
 }
 
 // Extra (server-side) accepted CBC cipher suites - will phase out in the future
@@ -66,11 +71,19 @@ func ClientDefault() *tls.Config {
 }
 
 // certPool returns an X.509 certificate pool from `caFile`, the certificate file.
-func certPool(caFile string) (*x509.CertPool, error) {
+func certPool(caFile string, exclusivePool bool) (*x509.CertPool, error) {
 	// If we should verify the server, we need to load a trusted ca
-	certPool, err := SystemCertPool()
-	if err != nil {
-		return nil, fmt.Errorf("failed to read system certificates: %v", err)
+	var (
+		certPool *x509.CertPool
+		err      error
+	)
+	if exclusivePool {
+		certPool = x509.NewCertPool()
+	} else {
+		certPool, err = SystemCertPool()
+		if err != nil {
+			return nil, fmt.Errorf("failed to read system certificates: %v", err)
+		}
 	}
 	pem, err := ioutil.ReadFile(caFile)
 	if err != nil {
@@ -88,7 +101,7 @@ func Client(options Options) (*tls.Config, error) {
 	tlsConfig := ClientDefault()
 	tlsConfig.InsecureSkipVerify = options.InsecureSkipVerify
 	if !options.InsecureSkipVerify && options.CAFile != "" {
-		CAs, err := certPool(options.CAFile)
+		CAs, err := certPool(options.CAFile, options.ExclusiveRootPools)
 		if err != nil {
 			return nil, err
 		}
@@ -119,7 +132,7 @@ func Server(options Options) (*tls.Config, error) {
 	}
 	tlsConfig.Certificates = []tls.Certificate{tlsCert}
 	if options.ClientAuth >= tls.VerifyClientCertIfGiven && options.CAFile != "" {
-		CAs, err := certPool(options.CAFile)
+		CAs, err := certPool(options.CAFile, options.ExclusiveRootPools)
 		if err != nil {
 			return nil, err
 		}

vendor/github.com/docker/swarmkit/manager/scheduler/nodeset.go

@@ -111,6 +111,10 @@ func (ns *nodeSet) tree(serviceID string, preferences []*api.PlacementPreference
 			tree = next
 		}
 
+		if node.ActiveTasksCountByService != nil {
+			tree.tasks += node.ActiveTasksCountByService[serviceID]
+		}
+
 		if tree.nodeHeap.lessFunc == nil {
 			tree.nodeHeap.lessFunc = nodeLess
 		}

vendor/github.com/docker/swarmkit/manager/scheduler/scheduler.go

@@ -1,7 +1,6 @@
 package scheduler
 
 import (
-	"container/list"
 	"time"
 
 	"github.com/docker/swarmkit/api"
@@ -30,7 +29,7 @@ type schedulingDecision struct {
 // Scheduler assigns tasks to nodes.
 type Scheduler struct {
 	store           *store.MemoryStore
-	unassignedTasks *list.List
+	unassignedTasks map[string]*api.Task
 	// preassignedTasks already have NodeID, need resource validation
 	preassignedTasks map[string]*api.Task
 	nodeSet          nodeSet
@@ -47,7 +46,7 @@ type Scheduler struct {
 func New(store *store.MemoryStore) *Scheduler {
 	return &Scheduler{
 		store:            store,
-		unassignedTasks:  list.New(),
+		unassignedTasks:  make(map[string]*api.Task),
 		preassignedTasks: make(map[string]*api.Task),
 		allTasks:         make(map[string]*api.Task),
 		stopChan:         make(chan struct{}),
@@ -191,7 +190,7 @@ func (s *Scheduler) Stop() {
 
 // enqueue queues a task for scheduling.
 func (s *Scheduler) enqueue(t *api.Task) {
-	s.unassignedTasks.PushBack(t)
+	s.unassignedTasks[t.ID] = t
 }
 
 func (s *Scheduler) createTask(ctx context.Context, t *api.Task) int {
@@ -333,15 +332,12 @@ func (s *Scheduler) processPreassignedTasks(ctx context.Context) {
 // tick attempts to schedule the queue.
 func (s *Scheduler) tick(ctx context.Context) {
 	tasksByCommonSpec := make(map[string]map[string]*api.Task)
-	schedulingDecisions := make(map[string]schedulingDecision, s.unassignedTasks.Len())
+	schedulingDecisions := make(map[string]schedulingDecision, len(s.unassignedTasks))
 
-	var next *list.Element
-	for e := s.unassignedTasks.Front(); e != nil; e = next {
-		next = e.Next()
-		t := s.allTasks[e.Value.(*api.Task).ID]
+	for taskID, t := range s.unassignedTasks {
 		if t == nil || t.NodeID != "" {
 			// task deleted or already assigned
-			s.unassignedTasks.Remove(e)
+			delete(s.unassignedTasks, taskID)
 			continue
 		}
 
@@ -362,8 +358,8 @@ func (s *Scheduler) tick(ctx context.Context) {
 		if tasksByCommonSpec[taskGroupKey] == nil {
 			tasksByCommonSpec[taskGroupKey] = make(map[string]*api.Task)
 		}
-		tasksByCommonSpec[taskGroupKey][t.ID] = t
-		s.unassignedTasks.Remove(e)
+		tasksByCommonSpec[taskGroupKey][taskID] = t
+		delete(s.unassignedTasks, taskID)
 	}
 
 	for _, taskGroup := range tasksByCommonSpec {
@@ -602,6 +598,12 @@ func (s *Scheduler) scheduleNTasksOnNodes(ctx context.Context, n int, taskGroup
 	nodeIter := 0
 	nodeCount := len(nodes)
 	for taskID, t := range taskGroup {
+		// Skip tasks which were already scheduled because they ended
+		// up in two groups at once.
+		if _, exists := schedulingDecisions[taskID]; exists {
+			continue
+		}
+
 		node := &nodes[nodeIter%nodeCount]
 
 		log.G(ctx).WithField("task.id", t.ID).Debugf("assigning to node %s", node.ID)