Add support for CDI devices to docker daemon under linux

These changes add basic CDI integration to the docker daemon.

A cdi driver is added to handle cdi device requests. This
is gated by an experimental feature flag and is only supported on linux

This change also adds a CDISpecDirs (cdi-spec-dirs) option to the config.
This allows the default values of `/etc/cdi`, /var/run/cdi` to be overridden
which is useful for testing.

Signed-off-by: Evan Lezar <elezar@nvidia.com>

cmd/dockerd/config.go

@@ -61,6 +61,8 @@ func installCommonConfigFlags(conf *config.Config, flags *pflag.FlagSet) error {
 	flags.StringVar(&conf.HTTPSProxy, "https-proxy", "", "HTTPS proxy URL to use for outgoing traffic")
 	flags.StringVar(&conf.NoProxy, "no-proxy", "", "Comma-separated list of hosts or IP addresses for which the proxy is skipped")
 
+	flags.Var(opts.NewNamedListOptsRef("cdi-spec-dirs", &conf.CDISpecDirs, nil), "cdi-spec-dir", "CDI specification directories to use")
+
 	// Deprecated flags / options
 
 	flags.BoolVarP(&conf.AutoRestart, "restart", "r", true, "--restart on the daemon has been deprecated in favor of --restart policies on docker run")

cmd/dockerd/daemon.go

@@ -14,6 +14,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	containerddefaults "github.com/containerd/containerd/defaults"
 	"github.com/docker/docker/api"
 	apiserver "github.com/docker/docker/api/server"
@@ -241,6 +242,18 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
 		return errors.Wrap(err, "failed to validate authorization plugin")
 	}
 
+	// Note that CDI is not inherrently linux-specific, there are some linux-specific assumptions / implementations in the code that
+	// queries the properties of device on the host as wel as performs the injection of device nodes and their access permissions into the OCI spec.
+	//
+	// In order to lift this restriction the following would have to be addressed:
+	// - Support needs to be added to the cdi package for injecting Windows devices: https://github.com/container-orchestrated-devices/container-device-interface/issues/28
+	// - The DeviceRequests API must be extended to non-linux platforms.
+	if runtime.GOOS == "linux" && cli.Config.Experimental {
+		daemon.RegisterCDIDriver(
+			cdi.WithSpecDirs(cli.Config.CDISpecDirs...),
+		)
+	}
+
 	cli.d = d
 
 	if err := startMetricsServer(cli.Config.MetricsAddress); err != nil {

daemon/cdi.go

@@ -0,0 +1,90 @@
+package daemon
+
+import (
+	"fmt"
+
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/docker/docker/errdefs"
+	"github.com/docker/docker/pkg/capabilities"
+	"github.com/hashicorp/go-multierror"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+type cdiHandler struct {
+	registry *cdi.Cache
+}
+
+// RegisterCDIDriver registers the CDI device driver.
+// The driver injects CDI devices into an incoming OCI spec and is called for DeviceRequests associated with CDI devices.
+func RegisterCDIDriver(opts ...cdi.Option) {
+	cache, err := cdi.NewCache(opts...)
+	if err != nil {
+		logrus.WithError(err).Error("CDI registry initialization failed")
+		// We create a spec updater that always returns an error.
+		// This error will be returned only when a CDI device is requested.
+		// This ensures that daemon startup is not blocked by a CDI registry initialization failure.
+		errorOnUpdateSpec := func(s *specs.Spec, dev *deviceInstance) error {
+			return fmt.Errorf("CDI device injection failed due to registry initialization failure: %w", err)
+		}
+		driver := &deviceDriver{
+			capset:     capabilities.Set{"cdi": struct{}{}},
+			updateSpec: errorOnUpdateSpec,
+		}
+		registerDeviceDriver("cdi", driver)
+		return
+	}
+
+	// We construct a spec updates that injects CDI devices into the OCI spec using the initialized registry.
+	c := &cdiHandler{
+		registry: cache,
+	}
+
+	driver := &deviceDriver{
+		capset:     capabilities.Set{"cdi": struct{}{}},
+		updateSpec: c.injectCDIDevices,
+	}
+
+	registerDeviceDriver("cdi", driver)
+}
+
+// injectCDIDevices injects a set of CDI devices into the specified OCI specification.
+func (c *cdiHandler) injectCDIDevices(s *specs.Spec, dev *deviceInstance) error {
+	if dev.req.Count != 0 {
+		return errdefs.InvalidParameter(errors.New("unexpected count in CDI device request"))
+	}
+	if len(dev.req.Options) > 0 {
+		return errdefs.InvalidParameter(errors.New("unexpected options in CDI device request"))
+	}
+
+	cdiDeviceNames := dev.req.DeviceIDs
+	if len(cdiDeviceNames) == 0 {
+		return nil
+	}
+
+	_, err := c.registry.InjectDevices(s, cdiDeviceNames...)
+	if err != nil {
+		if rerrs := c.getErrors(); rerrs != nil {
+			// We log the errors that may have been generated while refreshing the CDI registry.
+			// These may be due to malformed specifications or device name conflicts that could be
+			// the cause of an injection failure.
+			logrus.WithError(rerrs).Warning("Refreshing the CDI registry generated errors")
+		}
+
+		return fmt.Errorf("CDI device injection failed: %w", err)
+	}
+
+	return nil
+}
+
+// getErrors returns a single error representation of errors that may have occurred while refreshing the CDI registry.
+func (c *cdiHandler) getErrors() error {
+	errors := c.registry.GetErrors()
+
+	var err *multierror.Error
+	for _, errs := range errors {
+		err = multierror.Append(err, errs...)
+	}
+	return err.ErrorOrNil()
+}

daemon/config/config.go

@@ -15,6 +15,7 @@ import (
 	"golang.org/x/text/encoding/unicode"
 	"golang.org/x/text/transform"
 
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
 	"github.com/containerd/containerd/runtime/v2/shim"
 	"github.com/docker/docker/opts"
 	"github.com/docker/docker/registry"
@@ -256,6 +257,9 @@ type CommonConfig struct {
 	ContainerdPluginNamespace string `json:"containerd-plugin-namespace,omitempty"`
 
 	DefaultRuntime string `json:"default-runtime,omitempty"`
+
+	// CDISpecDirs is a list of directories in which CDI specifications can be found.
+	CDISpecDirs []string `json:"cdi-spec-dirs,omitempty"`
 }
 
 // Proxies holds the proxies that are configured for the daemon.
@@ -295,6 +299,7 @@ func New() (*Config, error) {
 			ContainerdNamespace:       DefaultContainersNamespace,
 			ContainerdPluginNamespace: DefaultPluginNamespace,
 			DefaultRuntime:            StockRuntimeName,
+			CDISpecDirs:               append([]string(nil), cdi.DefaultSpecDirs...),
 		},
 	}
 

integration/container/cdi_test.go

@@ -0,0 +1,65 @@
+package container // import "github.com/docker/docker/integration/container"
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/docker/docker/api/types"
+	containertypes "github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/integration/internal/container"
+	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/docker/docker/testutil/daemon"
+	"gotest.tools/v3/assert"
+	is "gotest.tools/v3/assert/cmp"
+	"gotest.tools/v3/skip"
+)
+
+func TestCreateWithCDIDevices(t *testing.T) {
+	skip.If(t, testEnv.OSType != "linux", "CDI devices are only supported on Linux")
+	skip.If(t, testEnv.IsRemoteDaemon, "cannot run cdi tests with a remote daemon")
+
+	cwd, err := os.Getwd()
+	assert.NilError(t, err)
+	d := daemon.New(t, daemon.WithExperimental())
+	d.StartWithBusybox(t, "--cdi-spec-dir="+filepath.Join(cwd, "testdata", "cdi"))
+	defer d.Stop(t)
+
+	client := d.NewClientT(t)
+
+	ctx := context.Background()
+	id := container.Run(ctx, t, client,
+		container.WithCmd("/bin/sh", "-c", "env"),
+		container.WithCDIDevices("vendor1.com/device=foo"),
+	)
+	defer client.ContainerRemove(ctx, id, types.ContainerRemoveOptions{Force: true})
+
+	inspect, err := client.ContainerInspect(ctx, id)
+	assert.NilError(t, err)
+
+	expectedRequests := []containertypes.DeviceRequest{
+		{
+			Driver:       "cdi",
+			DeviceIDs:    []string{"vendor1.com/device=foo"},
+			Capabilities: [][]string{{"cdi"}},
+		},
+	}
+	assert.Check(t, is.DeepEqual(inspect.HostConfig.DeviceRequests, expectedRequests))
+
+	reader, err := client.ContainerLogs(ctx, id, types.ContainerLogsOptions{
+		ShowStdout: true,
+	})
+	assert.NilError(t, err)
+
+	actualStdout := new(bytes.Buffer)
+	actualStderr := io.Discard
+	_, err = stdcopy.StdCopy(actualStdout, actualStderr, reader)
+	assert.NilError(t, err)
+
+	outlines := strings.Split(actualStdout.String(), "\n")
+	assert.Assert(t, is.Contains(outlines, "FOO=injected"))
+}

integration/container/testdata/cdi/vendor1.yaml

@@ -0,0 +1,7 @@
+cdiVersion: "0.3.0"
+kind: "vendor1.com/device"
+devices:
+- name: foo
+  containerEdits:
+    env:
+    - FOO=injected

integration/internal/container/ops.go

@@ -237,3 +237,15 @@ func WithRuntime(name string) func(*TestContainerConfig) {
 		c.HostConfig.Runtime = name
 	}
 }
+
+// WithCDIDevices sets the CDI devices to use to start the container
+func WithCDIDevices(cdiDeviceNames ...string) func(*TestContainerConfig) {
+	return func(c *TestContainerConfig) {
+		request := containertypes.DeviceRequest{
+			Driver:       "cdi",
+			Capabilities: [][]string{{"cdi"}},
+			DeviceIDs:    cdiDeviceNames,
+		}
+		c.HostConfig.DeviceRequests = append(c.HostConfig.DeviceRequests, request)
+	}
+}