Windows: Run containerd as managed process

Signed-off-by: Charity Kathure <ckathure@microsoft.com>
Co-authored-by: Olli Janatuinen <olli.janatuinen@gmail.com>

.github/workflows/.windows.yml

@@ -349,33 +349,12 @@ jobs:
             $ErrorActionPreference = "Stop"
             Write-Host "Service removed"
           }
-      -
-        name: Starting containerd
-        if: matrix.runtime == 'containerd'
-        run: |
-          Write-Host "Generating config"
-          & "${{ env.BIN_OUT }}\containerd.exe" config default | Out-File "$env:TEMP\ctn.toml" -Encoding ascii
-          Write-Host "Creating service"
-          New-Item -ItemType Directory "$env:TEMP\ctn-root" -ErrorAction SilentlyContinue | Out-Null
-          New-Item -ItemType Directory "$env:TEMP\ctn-state" -ErrorAction SilentlyContinue | Out-Null
-          Start-Process -Wait "${{ env.BIN_OUT }}\containerd.exe" `
-            -ArgumentList "--log-level=debug", `
-              "--config=$env:TEMP\ctn.toml", `
-              "--address=\\.\pipe\containerd-containerd", `
-              "--root=$env:TEMP\ctn-root", `
-              "--state=$env:TEMP\ctn-state", `
-              "--log-file=$env:TEMP\ctn.log", `
-              "--register-service"
-          Write-Host "Starting service"
-          Start-Service -Name containerd
-          Start-Sleep -Seconds 5
-          Write-Host "Service started successfully!"
       -
         name: Starting test daemon
         run: |
           Write-Host "Creating service"
           If ("${{ matrix.runtime }}" -eq "containerd") {
-            $runtimeArg="--containerd=\\.\pipe\containerd-containerd"
+            $runtimeArg="--default-runtime=io.containerd.runhcs.v1"
             echo "DOCKER_WINDOWS_CONTAINERD_RUNTIME=1" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
           }
           New-Item -ItemType Directory "$env:TEMP\moby-root" -ErrorAction SilentlyContinue | Out-Null
@@ -415,6 +394,17 @@ jobs:
             Start-Sleep -Seconds 1
           }
           Write-Host "Test daemon started and replied!"
+          If ("${{ matrix.runtime }}" -eq "containerd") {
+            $containerdProcesses = Get-Process -Name containerd -ErrorAction:SilentlyContinue
+            If (-not $containerdProcesses) {
+              Throw "containerd process is not running"
+            } else {
+              foreach ($process in $containerdProcesses) {
+                $processPath = (Get-Process -Id $process.Id -FileVersionInfo).FileName
+                Write-Output "Running containerd instance binary Path: $($processPath)"
+              }
+            }
+          }
         env:
           DOCKER_HOST: npipe:////./pipe/docker_engine
       -
@@ -479,19 +469,6 @@ jobs:
           & "${{ env.BIN_OUT }}\docker" info
         env:
           DOCKER_HOST: npipe:////./pipe/docker_engine
-      -
-        name: Stop containerd
-        if: always() && matrix.runtime == 'containerd'
-        run: |
-          $ErrorActionPreference = "SilentlyContinue"
-          Stop-Service -Force -Name containerd
-          $ErrorActionPreference = "Stop"
-      -
-        name: Containerd logs
-        if: always() && matrix.runtime == 'containerd'
-        run: |
-          Copy-Item "$env:TEMP\ctn.log" -Destination ".\bundles\containerd.log"
-          Get-Content "$env:TEMP\ctn.log" | Out-Host
       -
         name: Stop daemon
         if: always()

cmd/dockerd/daemon.go

@@ -784,6 +784,10 @@ func (cli *daemonCLI) getContainerdDaemonOpts() ([]supervisor.DaemonOpt, error)
 		opts = append(opts, supervisor.WithCRIDisabled())
 	}
 
+	if runtime.GOOS == "windows" {
+		opts = append(opts, supervisor.WithDetectLocalBinary())
+	}
+
 	return opts, nil
 }
 
@@ -1024,3 +1028,30 @@ func overrideProxyEnv(name, val string) {
 	}
 	_ = os.Setenv(name, val)
 }
+
+func (cli *daemonCLI) initializeContainerd(ctx context.Context) (func(time.Duration) error, error) {
+	systemContainerdAddr, ok, err := systemContainerdRunning(honorXDG)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not determine whether the system containerd is running")
+	}
+	if ok {
+		// detected a system containerd at the given address.
+		cli.ContainerdAddr = systemContainerdAddr
+		return nil, nil
+	}
+
+	log.G(ctx).Info("containerd not running, starting managed containerd")
+	opts, err := cli.getContainerdDaemonOpts()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate containerd options")
+	}
+
+	r, err := supervisor.Start(ctx, filepath.Join(cli.Root, "containerd"), filepath.Join(cli.ExecRoot, "containerd"), opts...)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to start containerd")
+	}
+	cli.ContainerdAddr = r.Address()
+
+	// Try to wait for containerd to shutdown
+	return r.WaitTimeout, nil
+}

cmd/dockerd/daemon_unix.go

@@ -11,10 +11,8 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/containerd/log"
 	"github.com/docker/docker/daemon"
 	"github.com/docker/docker/daemon/config"
-	"github.com/docker/docker/libcontainerd/supervisor"
 	"github.com/docker/docker/libnetwork/portallocator"
 	"github.com/docker/docker/pkg/homedir"
 	"github.com/pkg/errors"
@@ -122,28 +120,5 @@ func (cli *daemonCLI) initContainerd(ctx context.Context) (func(time.Duration) e
 		return nil, nil
 	}
 
-	systemContainerdAddr, ok, err := systemContainerdRunning(honorXDG)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not determine whether the system containerd is running")
-	}
-	if ok {
-		// detected a system containerd at the given address.
-		cli.ContainerdAddr = systemContainerdAddr
-		return nil, nil
-	}
-
-	log.G(ctx).Info("containerd not running, starting managed containerd")
-	opts, err := cli.getContainerdDaemonOpts()
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to generate containerd options")
-	}
-
-	r, err := supervisor.Start(ctx, filepath.Join(cli.Root, "containerd"), filepath.Join(cli.ExecRoot, "containerd"), opts...)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to start containerd")
-	}
-	cli.ContainerdAddr = r.Address()
-
-	// Try to wait for containerd to shutdown
-	return r.WaitTimeout, nil
+	return cli.initializeContainerd(ctx)
 }

cmd/dockerd/daemon_windows.go

@@ -100,9 +100,18 @@ func newCgroupParent(config *config.Config) string {
 	return ""
 }
 
-func (cli *daemonCLI) initContainerd(_ context.Context) (func(time.Duration) error, error) {
-	system.InitContainerdRuntime(cli.ContainerdAddr)
-	return nil, nil
+func (cli *daemonCLI) initContainerd(ctx context.Context) (func(time.Duration) error, error) {
+	defer func() { system.EnableContainerdRuntime(cli.ContainerdAddr) }()
+
+	if cli.ContainerdAddr != "" {
+		return nil, nil
+	}
+
+	if cli.DefaultRuntime != config.WindowsV2RuntimeName {
+		return nil, nil
+	}
+
+	return cli.initializeContainerd(ctx)
 }
 
 func validateCPURealtimeOptions(_ *config.Config) error {

daemon/config/config_windows.go

@@ -13,8 +13,16 @@ const (
 	// default value. On Windows keep this empty so the value is auto-detected
 	// based on other options.
 	StockRuntimeName = ""
+
+	WindowsV1RuntimeName = "com.docker.hcsshim.v1"
+	WindowsV2RuntimeName = "io.containerd.runhcs.v1"
 )
 
+var builtinRuntimes = map[string]bool{
+	WindowsV1RuntimeName: true,
+	WindowsV2RuntimeName: true,
+}
+
 // BridgeConfig is meant to store all the parameters for both the bridge driver and the default bridge network. On
 // Windows: 1. "bridge" in this context reference the nat driver and the default nat network; 2. the nat driver has no
 // specific parameters, so this struct effectively just stores parameters for the default nat network.

daemon/daemon_windows.go

@@ -40,9 +40,6 @@ const (
 	windowsMaxCPUShares  = 10000
 	windowsMinCPUPercent = 1
 	windowsMaxCPUPercent = 100
-
-	windowsV1RuntimeName = "com.docker.hcsshim.v1"
-	windowsV2RuntimeName = "io.containerd.runhcs.v1"
 )
 
 // Windows containers are much larger than Linux containers and each of them
@@ -563,14 +560,14 @@ func (daemon *Daemon) initLibcontainerd(ctx context.Context, cfg *config.Config)
 	rt := cfg.DefaultRuntime
 	if rt == "" {
 		if cfg.ContainerdAddr == "" {
-			rt = windowsV1RuntimeName
+			rt = config.WindowsV1RuntimeName
 		} else {
-			rt = windowsV2RuntimeName
+			rt = config.WindowsV2RuntimeName
 		}
 	}
 
 	switch rt {
-	case windowsV1RuntimeName:
+	case config.WindowsV1RuntimeName:
 		daemon.containerd, err = local.NewClient(
 			ctx,
 			daemon.containerdClient,
@@ -578,7 +575,7 @@ func (daemon *Daemon) initLibcontainerd(ctx context.Context, cfg *config.Config)
 			cfg.ContainerdNamespace,
 			daemon,
 		)
-	case windowsV2RuntimeName:
+	case config.WindowsV2RuntimeName:
 		if cfg.ContainerdAddr == "" {
 			return fmt.Errorf("cannot use the specified runtime %q without containerd", rt)
 		}

daemon/start_windows.go

@@ -3,13 +3,14 @@ package daemon // import "github.com/docker/docker/daemon"
 import (
 	"github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options"
 	"github.com/docker/docker/container"
+	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/pkg/system"
 )
 
 func (daemon *Daemon) getLibcontainerdCreateOptions(*configStore, *container.Container) (string, interface{}, error) {
 	if system.ContainerdRuntimeSupported() {
 		opts := &options.Options{}
-		return "io.containerd.runhcs.v1", opts, nil
+		return config.WindowsV2RuntimeName, opts, nil
 	}
 	return "", nil, nil
 }

libcontainerd/supervisor/remote_daemon.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/defaults"
+	"github.com/containerd/containerd/pkg/dialer"
 	"github.com/containerd/containerd/services/server/config"
 	"github.com/containerd/log"
 	"github.com/docker/docker/pkg/pidfile"
@@ -29,7 +30,6 @@ const (
 	shutdownTimeout         = 15 * time.Second
 	startupTimeout          = 15 * time.Second
 	configFile              = "containerd.toml"
-	binaryName              = "containerd"
 	pidFile                 = "containerd.pid"
 )
 
@@ -40,9 +40,13 @@ type remote struct {
 	// file is saved.
 	configFile string
 
-	daemonPid int
-	pidFile   string
-	logger    *log.Entry
+	// daemonPath is the binary to execute, and can be either a basename (to use
+	// a binary installed in the system's $PATH), or the full path to the binary
+	// to use.
+	daemonPath string
+	daemonPid  int
+	pidFile    string
+	logger     *log.Entry
 
 	daemonWaitCh  chan struct{}
 	daemonStartCh chan error
@@ -75,6 +79,7 @@ func Start(ctx context.Context, rootDir, stateDir string, opts ...DaemonOpt) (Da
 			},
 		},
 		configFile:    filepath.Join(stateDir, configFile),
+		daemonPath:    binaryName,
 		daemonPid:     -1,
 		pidFile:       filepath.Join(stateDir, pidFile),
 		logger:        log.G(ctx).WithField("module", "libcontainerd"),
@@ -156,7 +161,8 @@ func (r *remote) startContainerd() error {
 		return err
 	}
 
-	cmd := exec.Command(binaryName, "--config", cfgFile)
+	r.logger.WithField("binary", r.daemonPath).Debug("starting containerd binary")
+	cmd := exec.Command(r.daemonPath, "--config", cfgFile)
 	// redirect containerd logs to docker logs
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
@@ -262,7 +268,9 @@ func (r *remote) monitorDaemon(ctx context.Context) {
 				}
 			}
 
-			os.RemoveAll(r.GRPC.Address)
+			if err := os.RemoveAll(r.GRPC.Address); err != nil {
+				r.logger.WithError(err).Error("failed to remove old gRPC address")
+			}
 			if err := r.startContainerd(); err != nil {
 				if !started {
 					r.daemonStartCh <- err
@@ -273,16 +281,19 @@ func (r *remote) monitorDaemon(ctx context.Context) {
 				continue
 			}
 
+			gopts := []grpc.DialOption{
+				grpc.WithTransportCredentials(insecure.NewCredentials()),
+				grpc.WithContextDialer(dialer.ContextDialer),
+				grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)),
+				grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
+				grpc.WithUnaryInterceptor(grpcerrors.UnaryClientInterceptor),
+				grpc.WithStreamInterceptor(grpcerrors.StreamClientInterceptor),
+			}
+
 			client, err = containerd.New(
 				r.GRPC.Address,
 				containerd.WithTimeout(60*time.Second),
-				containerd.WithDialOpts([]grpc.DialOption{
-					grpc.WithUnaryInterceptor(grpcerrors.UnaryClientInterceptor),
-					grpc.WithStreamInterceptor(grpcerrors.StreamClientInterceptor),
-					grpc.WithTransportCredentials(insecure.NewCredentials()),
-					grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)),
-					grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
-				}),
+				containerd.WithDialOpts(gopts),
 			)
 			if err != nil {
 				r.logger.WithError(err).Error("failed connecting to containerd")

libcontainerd/supervisor/remote_daemon_linux.go

@@ -10,6 +10,7 @@ import (
 )
 
 const (
+	binaryName    = "containerd"
 	sockFile      = "containerd.sock"
 	debugSockFile = "containerd-debug.sock"
 )

libcontainerd/supervisor/remote_daemon_options.go

@@ -1,7 +1,11 @@
 package supervisor // import "github.com/docker/docker/libcontainerd/supervisor"
 
 import (
+	"os"
+	"path/filepath"
+
 	"github.com/containerd/log"
+	"github.com/pkg/errors"
 )
 
 // WithLogLevel defines which log level to start containerd with.
@@ -33,3 +37,31 @@ func WithCRIDisabled() DaemonOpt {
 		return nil
 	}
 }
+
+// WithDetectLocalBinary checks if a containerd binary is present in the same
+// directory as the dockerd binary, and overrides the path of the containerd
+// binary to start if found. If no binary is found, no changes are made.
+func WithDetectLocalBinary() DaemonOpt {
+	return func(r *remote) error {
+		dockerdPath, err := os.Executable()
+		if err != nil {
+			return errors.Wrap(err, "looking up binary path")
+		}
+
+		localBinary := filepath.Join(filepath.Dir(dockerdPath), binaryName)
+		fi, err := os.Stat(localBinary)
+		if err != nil {
+			if !errors.Is(err, os.ErrNotExist) {
+				return err
+			}
+			return nil
+		}
+		if fi.IsDir() {
+			return errors.Errorf("local containerd path found (%s), but is a directory", localBinary)
+		}
+		r.daemonPath = localBinary
+		r.logger.WithField("daemon path", r.daemonPath).Debug("Local containerd daemon found.")
+
+		return nil
+	}
+}

libcontainerd/supervisor/remote_daemon_windows.go

@@ -7,8 +7,9 @@ import (
 )
 
 const (
-	grpcPipeName  = `\\.\pipe\containerd-containerd`
-	debugPipeName = `\\.\pipe\containerd-debug`
+	binaryName    = "containerd.exe"
+	grpcPipeName  = `\\.\pipe\docker-containerd`
+	debugPipeName = `\\.\pipe\docker-containerd-debug`
 )
 
 func defaultGRPCAddress(stateDir string) string {

pkg/system/init_windows.go

@@ -3,8 +3,8 @@ package system // import "github.com/docker/docker/pkg/system"
 // containerdRuntimeSupported determines if containerd should be the runtime.
 var containerdRuntimeSupported = false
 
-// InitContainerdRuntime sets whether to use containerd for runtime on Windows.
-func InitContainerdRuntime(cdPath string) {
+// EnableContainerdRuntime sets whether to use containerd for runtime on Windows.
+func EnableContainerdRuntime(cdPath string) {
 	if len(cdPath) > 0 {
 		containerdRuntimeSupported = true
 	}