From 84965c075215de0add8968848e7f93219155be85 Mon Sep 17 00:00:00 2001 From: Charity Kathure Date: Tue, 11 Jun 2024 23:04:20 +0300 Subject: [PATCH] Windows: Run containerd as managed process Signed-off-by: Charity Kathure Co-authored-by: Olli Janatuinen --- .github/workflows/.windows.yml | 47 +++++-------------- cmd/dockerd/daemon.go | 31 ++++++++++++ cmd/dockerd/daemon_unix.go | 27 +---------- cmd/dockerd/daemon_windows.go | 15 ++++-- daemon/config/config_windows.go | 8 ++++ daemon/daemon_windows.go | 11 ++--- daemon/start_windows.go | 3 +- libcontainerd/supervisor/remote_daemon.go | 37 ++++++++++----- .../supervisor/remote_daemon_linux.go | 1 + .../supervisor/remote_daemon_options.go | 32 +++++++++++++ .../supervisor/remote_daemon_windows.go | 5 +- pkg/system/init_windows.go | 4 +- 12 files changed, 132 insertions(+), 89 deletions(-) diff --git a/.github/workflows/.windows.yml b/.github/workflows/.windows.yml index f6ce773da7..7734acab8c 100644 --- a/.github/workflows/.windows.yml +++ b/.github/workflows/.windows.yml @@ -349,33 +349,12 @@ jobs: $ErrorActionPreference = "Stop" Write-Host "Service removed" } - - - name: Starting containerd - if: matrix.runtime == 'containerd' - run: | - Write-Host "Generating config" - & "${{ env.BIN_OUT }}\containerd.exe" config default | Out-File "$env:TEMP\ctn.toml" -Encoding ascii - Write-Host "Creating service" - New-Item -ItemType Directory "$env:TEMP\ctn-root" -ErrorAction SilentlyContinue | Out-Null - New-Item -ItemType Directory "$env:TEMP\ctn-state" -ErrorAction SilentlyContinue | Out-Null - Start-Process -Wait "${{ env.BIN_OUT }}\containerd.exe" ` - -ArgumentList "--log-level=debug", ` - "--config=$env:TEMP\ctn.toml", ` - "--address=\\.\pipe\containerd-containerd", ` - "--root=$env:TEMP\ctn-root", ` - "--state=$env:TEMP\ctn-state", ` - "--log-file=$env:TEMP\ctn.log", ` - "--register-service" - Write-Host "Starting service" - Start-Service -Name containerd - Start-Sleep -Seconds 5 - Write-Host "Service started successfully!" - name: Starting test daemon run: | Write-Host "Creating service" If ("${{ matrix.runtime }}" -eq "containerd") { - $runtimeArg="--containerd=\\.\pipe\containerd-containerd" + $runtimeArg="--default-runtime=io.containerd.runhcs.v1" echo "DOCKER_WINDOWS_CONTAINERD_RUNTIME=1" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append } New-Item -ItemType Directory "$env:TEMP\moby-root" -ErrorAction SilentlyContinue | Out-Null @@ -415,6 +394,17 @@ jobs: Start-Sleep -Seconds 1 } Write-Host "Test daemon started and replied!" + If ("${{ matrix.runtime }}" -eq "containerd") { + $containerdProcesses = Get-Process -Name containerd -ErrorAction:SilentlyContinue + If (-not $containerdProcesses) { + Throw "containerd process is not running" + } else { + foreach ($process in $containerdProcesses) { + $processPath = (Get-Process -Id $process.Id -FileVersionInfo).FileName + Write-Output "Running containerd instance binary Path: $($processPath)" + } + } + } env: DOCKER_HOST: npipe:////./pipe/docker_engine - @@ -479,19 +469,6 @@ jobs: & "${{ env.BIN_OUT }}\docker" info env: DOCKER_HOST: npipe:////./pipe/docker_engine - - - name: Stop containerd - if: always() && matrix.runtime == 'containerd' - run: | - $ErrorActionPreference = "SilentlyContinue" - Stop-Service -Force -Name containerd - $ErrorActionPreference = "Stop" - - - name: Containerd logs - if: always() && matrix.runtime == 'containerd' - run: | - Copy-Item "$env:TEMP\ctn.log" -Destination ".\bundles\containerd.log" - Get-Content "$env:TEMP\ctn.log" | Out-Host - name: Stop daemon if: always() diff --git a/cmd/dockerd/daemon.go b/cmd/dockerd/daemon.go index 3d4184b171..f67aea60cc 100644 --- a/cmd/dockerd/daemon.go +++ b/cmd/dockerd/daemon.go @@ -784,6 +784,10 @@ func (cli *daemonCLI) getContainerdDaemonOpts() ([]supervisor.DaemonOpt, error) opts = append(opts, supervisor.WithCRIDisabled()) } + if runtime.GOOS == "windows" { + opts = append(opts, supervisor.WithDetectLocalBinary()) + } + return opts, nil } @@ -1024,3 +1028,30 @@ func overrideProxyEnv(name, val string) { } _ = os.Setenv(name, val) } + +func (cli *daemonCLI) initializeContainerd(ctx context.Context) (func(time.Duration) error, error) { + systemContainerdAddr, ok, err := systemContainerdRunning(honorXDG) + if err != nil { + return nil, errors.Wrap(err, "could not determine whether the system containerd is running") + } + if ok { + // detected a system containerd at the given address. + cli.ContainerdAddr = systemContainerdAddr + return nil, nil + } + + log.G(ctx).Info("containerd not running, starting managed containerd") + opts, err := cli.getContainerdDaemonOpts() + if err != nil { + return nil, errors.Wrap(err, "failed to generate containerd options") + } + + r, err := supervisor.Start(ctx, filepath.Join(cli.Root, "containerd"), filepath.Join(cli.ExecRoot, "containerd"), opts...) + if err != nil { + return nil, errors.Wrap(err, "failed to start containerd") + } + cli.ContainerdAddr = r.Address() + + // Try to wait for containerd to shutdown + return r.WaitTimeout, nil +} diff --git a/cmd/dockerd/daemon_unix.go b/cmd/dockerd/daemon_unix.go index c6f7d7b4a1..d8b69fa28b 100644 --- a/cmd/dockerd/daemon_unix.go +++ b/cmd/dockerd/daemon_unix.go @@ -11,10 +11,8 @@ import ( "strconv" "time" - "github.com/containerd/log" "github.com/docker/docker/daemon" "github.com/docker/docker/daemon/config" - "github.com/docker/docker/libcontainerd/supervisor" "github.com/docker/docker/libnetwork/portallocator" "github.com/docker/docker/pkg/homedir" "github.com/pkg/errors" @@ -122,28 +120,5 @@ func (cli *daemonCLI) initContainerd(ctx context.Context) (func(time.Duration) e return nil, nil } - systemContainerdAddr, ok, err := systemContainerdRunning(honorXDG) - if err != nil { - return nil, errors.Wrap(err, "could not determine whether the system containerd is running") - } - if ok { - // detected a system containerd at the given address. - cli.ContainerdAddr = systemContainerdAddr - return nil, nil - } - - log.G(ctx).Info("containerd not running, starting managed containerd") - opts, err := cli.getContainerdDaemonOpts() - if err != nil { - return nil, errors.Wrap(err, "failed to generate containerd options") - } - - r, err := supervisor.Start(ctx, filepath.Join(cli.Root, "containerd"), filepath.Join(cli.ExecRoot, "containerd"), opts...) - if err != nil { - return nil, errors.Wrap(err, "failed to start containerd") - } - cli.ContainerdAddr = r.Address() - - // Try to wait for containerd to shutdown - return r.WaitTimeout, nil + return cli.initializeContainerd(ctx) } diff --git a/cmd/dockerd/daemon_windows.go b/cmd/dockerd/daemon_windows.go index 8bbbae8457..6d417b89f5 100644 --- a/cmd/dockerd/daemon_windows.go +++ b/cmd/dockerd/daemon_windows.go @@ -100,9 +100,18 @@ func newCgroupParent(config *config.Config) string { return "" } -func (cli *daemonCLI) initContainerd(_ context.Context) (func(time.Duration) error, error) { - system.InitContainerdRuntime(cli.ContainerdAddr) - return nil, nil +func (cli *daemonCLI) initContainerd(ctx context.Context) (func(time.Duration) error, error) { + defer func() { system.EnableContainerdRuntime(cli.ContainerdAddr) }() + + if cli.ContainerdAddr != "" { + return nil, nil + } + + if cli.DefaultRuntime != config.WindowsV2RuntimeName { + return nil, nil + } + + return cli.initializeContainerd(ctx) } func validateCPURealtimeOptions(_ *config.Config) error { diff --git a/daemon/config/config_windows.go b/daemon/config/config_windows.go index 05e185da88..c0593c0fa8 100644 --- a/daemon/config/config_windows.go +++ b/daemon/config/config_windows.go @@ -13,8 +13,16 @@ const ( // default value. On Windows keep this empty so the value is auto-detected // based on other options. StockRuntimeName = "" + + WindowsV1RuntimeName = "com.docker.hcsshim.v1" + WindowsV2RuntimeName = "io.containerd.runhcs.v1" ) +var builtinRuntimes = map[string]bool{ + WindowsV1RuntimeName: true, + WindowsV2RuntimeName: true, +} + // BridgeConfig is meant to store all the parameters for both the bridge driver and the default bridge network. On // Windows: 1. "bridge" in this context reference the nat driver and the default nat network; 2. the nat driver has no // specific parameters, so this struct effectively just stores parameters for the default nat network. diff --git a/daemon/daemon_windows.go b/daemon/daemon_windows.go index 81894cca73..515cb4924d 100644 --- a/daemon/daemon_windows.go +++ b/daemon/daemon_windows.go @@ -40,9 +40,6 @@ const ( windowsMaxCPUShares = 10000 windowsMinCPUPercent = 1 windowsMaxCPUPercent = 100 - - windowsV1RuntimeName = "com.docker.hcsshim.v1" - windowsV2RuntimeName = "io.containerd.runhcs.v1" ) // Windows containers are much larger than Linux containers and each of them @@ -563,14 +560,14 @@ func (daemon *Daemon) initLibcontainerd(ctx context.Context, cfg *config.Config) rt := cfg.DefaultRuntime if rt == "" { if cfg.ContainerdAddr == "" { - rt = windowsV1RuntimeName + rt = config.WindowsV1RuntimeName } else { - rt = windowsV2RuntimeName + rt = config.WindowsV2RuntimeName } } switch rt { - case windowsV1RuntimeName: + case config.WindowsV1RuntimeName: daemon.containerd, err = local.NewClient( ctx, daemon.containerdClient, @@ -578,7 +575,7 @@ func (daemon *Daemon) initLibcontainerd(ctx context.Context, cfg *config.Config) cfg.ContainerdNamespace, daemon, ) - case windowsV2RuntimeName: + case config.WindowsV2RuntimeName: if cfg.ContainerdAddr == "" { return fmt.Errorf("cannot use the specified runtime %q without containerd", rt) } diff --git a/daemon/start_windows.go b/daemon/start_windows.go index 3c87f004c8..3d01843a02 100644 --- a/daemon/start_windows.go +++ b/daemon/start_windows.go @@ -3,13 +3,14 @@ package daemon // import "github.com/docker/docker/daemon" import ( "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options" "github.com/docker/docker/container" + "github.com/docker/docker/daemon/config" "github.com/docker/docker/pkg/system" ) func (daemon *Daemon) getLibcontainerdCreateOptions(*configStore, *container.Container) (string, interface{}, error) { if system.ContainerdRuntimeSupported() { opts := &options.Options{} - return "io.containerd.runhcs.v1", opts, nil + return config.WindowsV2RuntimeName, opts, nil } return "", nil, nil } diff --git a/libcontainerd/supervisor/remote_daemon.go b/libcontainerd/supervisor/remote_daemon.go index 1d80e06fec..8998f36f7f 100644 --- a/libcontainerd/supervisor/remote_daemon.go +++ b/libcontainerd/supervisor/remote_daemon.go @@ -11,6 +11,7 @@ import ( "github.com/containerd/containerd" "github.com/containerd/containerd/defaults" + "github.com/containerd/containerd/pkg/dialer" "github.com/containerd/containerd/services/server/config" "github.com/containerd/log" "github.com/docker/docker/pkg/pidfile" @@ -29,7 +30,6 @@ const ( shutdownTimeout = 15 * time.Second startupTimeout = 15 * time.Second configFile = "containerd.toml" - binaryName = "containerd" pidFile = "containerd.pid" ) @@ -40,9 +40,13 @@ type remote struct { // file is saved. configFile string - daemonPid int - pidFile string - logger *log.Entry + // daemonPath is the binary to execute, and can be either a basename (to use + // a binary installed in the system's $PATH), or the full path to the binary + // to use. + daemonPath string + daemonPid int + pidFile string + logger *log.Entry daemonWaitCh chan struct{} daemonStartCh chan error @@ -75,6 +79,7 @@ func Start(ctx context.Context, rootDir, stateDir string, opts ...DaemonOpt) (Da }, }, configFile: filepath.Join(stateDir, configFile), + daemonPath: binaryName, daemonPid: -1, pidFile: filepath.Join(stateDir, pidFile), logger: log.G(ctx).WithField("module", "libcontainerd"), @@ -156,7 +161,8 @@ func (r *remote) startContainerd() error { return err } - cmd := exec.Command(binaryName, "--config", cfgFile) + r.logger.WithField("binary", r.daemonPath).Debug("starting containerd binary") + cmd := exec.Command(r.daemonPath, "--config", cfgFile) // redirect containerd logs to docker logs cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -262,7 +268,9 @@ func (r *remote) monitorDaemon(ctx context.Context) { } } - os.RemoveAll(r.GRPC.Address) + if err := os.RemoveAll(r.GRPC.Address); err != nil { + r.logger.WithError(err).Error("failed to remove old gRPC address") + } if err := r.startContainerd(); err != nil { if !started { r.daemonStartCh <- err @@ -273,16 +281,19 @@ func (r *remote) monitorDaemon(ctx context.Context) { continue } + gopts := []grpc.DialOption{ + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(dialer.ContextDialer), + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)), + grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)), + grpc.WithUnaryInterceptor(grpcerrors.UnaryClientInterceptor), + grpc.WithStreamInterceptor(grpcerrors.StreamClientInterceptor), + } + client, err = containerd.New( r.GRPC.Address, containerd.WithTimeout(60*time.Second), - containerd.WithDialOpts([]grpc.DialOption{ - grpc.WithUnaryInterceptor(grpcerrors.UnaryClientInterceptor), - grpc.WithStreamInterceptor(grpcerrors.StreamClientInterceptor), - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)), - grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)), - }), + containerd.WithDialOpts(gopts), ) if err != nil { r.logger.WithError(err).Error("failed connecting to containerd") diff --git a/libcontainerd/supervisor/remote_daemon_linux.go b/libcontainerd/supervisor/remote_daemon_linux.go index fa80a74317..1ae9efce67 100644 --- a/libcontainerd/supervisor/remote_daemon_linux.go +++ b/libcontainerd/supervisor/remote_daemon_linux.go @@ -10,6 +10,7 @@ import ( ) const ( + binaryName = "containerd" sockFile = "containerd.sock" debugSockFile = "containerd-debug.sock" ) diff --git a/libcontainerd/supervisor/remote_daemon_options.go b/libcontainerd/supervisor/remote_daemon_options.go index 6ad2b79fea..09043d2a8e 100644 --- a/libcontainerd/supervisor/remote_daemon_options.go +++ b/libcontainerd/supervisor/remote_daemon_options.go @@ -1,7 +1,11 @@ package supervisor // import "github.com/docker/docker/libcontainerd/supervisor" import ( + "os" + "path/filepath" + "github.com/containerd/log" + "github.com/pkg/errors" ) // WithLogLevel defines which log level to start containerd with. @@ -33,3 +37,31 @@ func WithCRIDisabled() DaemonOpt { return nil } } + +// WithDetectLocalBinary checks if a containerd binary is present in the same +// directory as the dockerd binary, and overrides the path of the containerd +// binary to start if found. If no binary is found, no changes are made. +func WithDetectLocalBinary() DaemonOpt { + return func(r *remote) error { + dockerdPath, err := os.Executable() + if err != nil { + return errors.Wrap(err, "looking up binary path") + } + + localBinary := filepath.Join(filepath.Dir(dockerdPath), binaryName) + fi, err := os.Stat(localBinary) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return err + } + return nil + } + if fi.IsDir() { + return errors.Errorf("local containerd path found (%s), but is a directory", localBinary) + } + r.daemonPath = localBinary + r.logger.WithField("daemon path", r.daemonPath).Debug("Local containerd daemon found.") + + return nil + } +} diff --git a/libcontainerd/supervisor/remote_daemon_windows.go b/libcontainerd/supervisor/remote_daemon_windows.go index ddb31de8c3..297f56565c 100644 --- a/libcontainerd/supervisor/remote_daemon_windows.go +++ b/libcontainerd/supervisor/remote_daemon_windows.go @@ -7,8 +7,9 @@ import ( ) const ( - grpcPipeName = `\\.\pipe\containerd-containerd` - debugPipeName = `\\.\pipe\containerd-debug` + binaryName = "containerd.exe" + grpcPipeName = `\\.\pipe\docker-containerd` + debugPipeName = `\\.\pipe\docker-containerd-debug` ) func defaultGRPCAddress(stateDir string) string { diff --git a/pkg/system/init_windows.go b/pkg/system/init_windows.go index 7603efbbd8..51e332c4b7 100644 --- a/pkg/system/init_windows.go +++ b/pkg/system/init_windows.go @@ -3,8 +3,8 @@ package system // import "github.com/docker/docker/pkg/system" // containerdRuntimeSupported determines if containerd should be the runtime. var containerdRuntimeSupported = false -// InitContainerdRuntime sets whether to use containerd for runtime on Windows. -func InitContainerdRuntime(cdPath string) { +// EnableContainerdRuntime sets whether to use containerd for runtime on Windows. +func EnableContainerdRuntime(cdPath string) { if len(cdPath) > 0 { containerdRuntimeSupported = true }