From 9f5f4f5a4273e920d5d77c1e73db8bebe65982bb Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 9 Oct 2024 15:19:32 -0700 Subject: [PATCH 01/12] Add containerd migration to daemon startup Add layer migration on startup Use image size threshold rather than image count Add daemon integration test Add test for migrating to containerd snapshotters Add vfs migration Add tar export for containerd migration Add containerd migration test with save and load Signed-off-by: Derek McGowan --- daemon/containerd/migration/migration.go | 316 +++++++++++++++++++++++ daemon/daemon.go | 210 +++++++++++---- daemon/graphdriver/vfs/driver.go | 11 +- integration/daemon/migration_test.go | 159 ++++++++++++ 4 files changed, 645 insertions(+), 51 deletions(-) create mode 100644 daemon/containerd/migration/migration.go create mode 100644 integration/daemon/migration_test.go diff --git a/daemon/containerd/migration/migration.go b/daemon/containerd/migration/migration.go new file mode 100644 index 0000000000..fc380a2a7e --- /dev/null +++ b/daemon/containerd/migration/migration.go @@ -0,0 +1,316 @@ +package migration + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "strings" + "sync" + "time" + + "github.com/containerd/containerd/v2/core/content" + "github.com/containerd/containerd/v2/core/images" + "github.com/containerd/containerd/v2/core/leases" + "github.com/containerd/containerd/v2/core/mount" + "github.com/containerd/containerd/v2/core/snapshots" + "github.com/containerd/containerd/v2/pkg/archive/compression" + "github.com/containerd/continuity/fs" + cerrdefs "github.com/containerd/errdefs" + "github.com/containerd/log" + "github.com/moby/moby/v2/daemon/internal/image" + "github.com/moby/moby/v2/daemon/internal/layer" + refstore "github.com/moby/moby/v2/daemon/internal/refstore" + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/identity" + "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "golang.org/x/sync/errgroup" +) + +type LayerMigrator struct { + layers layer.Store + refs refstore.Store + dis image.Store + leases leases.Manager + content content.Store + cis images.Store +} + +type Config struct { + LayerStore layer.Store + ReferenceStore refstore.Store + DockerImageStore image.Store + Leases leases.Manager + Content content.Store + ImageStore images.Store +} + +func NewLayerMigrator(config Config) *LayerMigrator { + return &LayerMigrator{ + layers: config.LayerStore, + refs: config.ReferenceStore, + dis: config.DockerImageStore, + leases: config.Leases, + content: config.Content, + cis: config.ImageStore, + } +} + +// MigrateTocontainerd migrates containers from overlay2 to overlayfs or vfs to native +func (lm *LayerMigrator) MigrateTocontainerd(ctx context.Context, snKey string, sn snapshots.Snapshotter) error { + if sn == nil { + return fmt.Errorf("no snapshotter to migrate to: %w", cerrdefs.ErrNotImplemented) + } + + switch driver := lm.layers.DriverName(); driver { + case "overlay2": + case "vfs": + default: + return fmt.Errorf("%q not supported for migration: %w", driver, cerrdefs.ErrNotImplemented) + } + + var ( + // Zstd makes migration 10x faster + // TODO: make configurable + layerMediaType = ocispec.MediaTypeImageLayerZstd + layerCompression = compression.Zstd + ) + + l, err := lm.leases.Create(ctx, leases.WithRandomID(), leases.WithExpiration(24*time.Hour)) + if err != nil { + return err + } + defer func() { + lm.leases.Delete(ctx, l) + }() + ctx = leases.WithLease(ctx, l.ID) + + for imgID, img := range lm.dis.Heads() { + diffids := img.RootFS.DiffIDs + if len(diffids) == 0 { + continue + } + var ( + parent string + manifest = ocispec.Manifest{ + MediaType: ocispec.MediaTypeImageManifest, + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + Layers: make([]ocispec.Descriptor, len(diffids)), + } + ml sync.Mutex + eg, egctx = errgroup.WithContext(ctx) + ) + for i := range diffids { + chainID := identity.ChainID(diffids[:i+1]) + l, err := lm.layers.Get(chainID) + if err != nil { + return fmt.Errorf("failed to get layer [%d] %q: %w", i, chainID, err) + } + layerIndex := i + eg.Go(func() error { + ctx := egctx + t1 := time.Now() + ts, err := l.TarStream() + if err != nil { + return err + } + + desc := ocispec.Descriptor{ + MediaType: layerMediaType, + } + + cw, err := lm.content.Writer(ctx, + content.WithRef(fmt.Sprintf("ingest-%s", chainID)), + content.WithDescriptor(desc)) + if err != nil { + return fmt.Errorf("failed to get content writer: %w", err) + } + + dgstr := digest.Canonical.Digester() + cs, _ := compression.CompressStream(io.MultiWriter(cw, dgstr.Hash()), layerCompression) + _, err = io.Copy(cs, ts) + if err != nil { + return fmt.Errorf("failed to copy to compressed stream: %w", err) + } + cs.Close() + + status, err := cw.Status() + if err != nil { + return err + } + + desc.Size = status.Offset + desc.Digest = dgstr.Digest() + + if err := cw.Commit(ctx, desc.Size, desc.Digest); err != nil && !cerrdefs.IsAlreadyExists(err) { + return err + } + + log.G(ctx).WithFields(log.Fields{ + "t": time.Since(t1), + "size": desc.Size, + "digest": desc.Digest, + }).Debug("Converted layer to content tar") + + ml.Lock() + manifest.Layers[layerIndex] = desc + ml.Unlock() + return nil + }) + + metadata, err := l.Metadata() + if err != nil { + return err + } + src, ok := metadata["UpperDir"] + if !ok { + src, ok = metadata["SourceDir"] + if !ok { + log.G(ctx).WithField("metadata", metadata).WithField("driver", lm.layers.DriverName()).Debug("no source directory metadata") + return fmt.Errorf("graphdriver not supported: %w", cerrdefs.ErrNotImplemented) + } + } + log.G(ctx).WithField("metadata", metadata).Debugf("migrating %s from %s", chainID, src) + + active := fmt.Sprintf("migration-%s", chainID) + + key := chainID.String() + + snapshotLabels := map[string]string{ + "containerd.io/snapshot.ref": key, + } + mounts, err := sn.Prepare(ctx, active, parent, snapshots.WithLabels(snapshotLabels)) + parent = key + if err != nil { + if cerrdefs.IsAlreadyExists(err) { + continue + } + return err + } + + dst, err := extractSource(mounts) + if err != nil { + return err + } + + t1 := time.Now() + if err := fs.CopyDir(dst, src); err != nil { + return err + } + log.G(ctx).WithFields(log.Fields{ + "t": time.Since(t1), + "key": key, + }).Debug("Copied layer to snapshot") + + if err := sn.Commit(ctx, key, active); err != nil && !cerrdefs.IsAlreadyExists(err) { + return err + } + } + + configBytes := img.RawJSON() + digest.FromBytes(configBytes) + manifest.Config = ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageConfig, + Digest: digest.FromBytes(configBytes), + Size: int64(len(configBytes)), + } + + configLabels := map[string]string{ + fmt.Sprintf("containerd.io/gc.ref.snapshot.%s", snKey): parent, + } + if err = content.WriteBlob(ctx, lm.content, "config"+manifest.Config.Digest.String(), bytes.NewReader(configBytes), manifest.Config, content.WithLabels(configLabels)); err != nil && !cerrdefs.IsAlreadyExists(err) { + return err + } + + if err := eg.Wait(); err != nil { + return err + } + + manifestBytes, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return err + } + + manifestDesc := ocispec.Descriptor{ + MediaType: manifest.MediaType, + Digest: digest.FromBytes(manifestBytes), + Size: int64(len(manifestBytes)), + } + + manifestLabels := map[string]string{ + "containerd.io/gc.ref.content.config": manifest.Config.Digest.String(), + } + for i := range manifest.Layers { + manifestLabels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = manifest.Layers[i].Digest.String() + } + + if err = content.WriteBlob(ctx, lm.content, "manifest"+manifestDesc.Digest.String(), bytes.NewReader(manifestBytes), manifestDesc, content.WithLabels(manifestLabels)); err != nil && !cerrdefs.IsAlreadyExists(err) { + return err + } + + childrenHandler := images.ChildrenHandler(lm.content) + childrenHandler = images.SetChildrenMappedLabels(lm.content, childrenHandler, nil) + if err = images.Walk(ctx, childrenHandler, manifestDesc); err != nil { + return err + } + + var added bool + for _, named := range lm.refs.References(digest.Digest(imgID)) { + img := images.Image{ + Name: named.String(), + Target: manifestDesc, + // TODO: Any labels? + } + img, err = lm.cis.Create(ctx, img) + if err != nil && !cerrdefs.IsAlreadyExists(err) { + return err + } else if err != nil { + log.G(ctx).Infof("Tag already exists: %s", named) + continue + } + + log.G(ctx).Infof("Migrated image %s to %s", img.Name, img.Target.Digest) + added = true + } + + if !added { + img := images.Image{ + Name: "moby-dangling@" + manifestDesc.Digest.String(), + Target: manifestDesc, + // TODO: Any labels? + } + img, err = lm.cis.Create(ctx, img) + if err != nil && !cerrdefs.IsAlreadyExists(err) { + return err + } else if err == nil { + log.G(ctx).Infof("Migrated image %s to %s", img.Name, img.Target.Digest) + } + } + } + + return nil +} + +func extractSource(mounts []mount.Mount) (string, error) { + if len(mounts) != 1 { + return "", fmt.Errorf("cannot support snapshotters with multiple mount sources: %w", cerrdefs.ErrNotImplemented) + } + switch mounts[0].Type { + case "bind": + return mounts[0].Source, nil + case "overlay": + for _, option := range mounts[0].Options { + if strings.HasPrefix(option, "upperdir=") { + return option[9:], nil + } + } + default: + return "", fmt.Errorf("mount type %q not supported: %w", mounts[0].Type, cerrdefs.ErrNotImplemented) + } + + return "", fmt.Errorf("mount is missing upper option: %w", cerrdefs.ErrNotImplemented) +} diff --git a/daemon/daemon.go b/daemon/daemon.go index 4f7ad50aa6..ee332935d9 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -10,6 +10,7 @@ import ( "crypto/sha256" "encoding/binary" "fmt" + "math" "net" "net/netip" "os" @@ -42,6 +43,7 @@ import ( "github.com/moby/moby/v2/daemon/config" "github.com/moby/moby/v2/daemon/container" ctrd "github.com/moby/moby/v2/daemon/containerd" + "github.com/moby/moby/v2/daemon/containerd/migration" "github.com/moby/moby/v2/daemon/events" _ "github.com/moby/moby/v2/daemon/graphdriver/register" // register graph drivers "github.com/moby/moby/v2/daemon/images" @@ -202,15 +204,15 @@ func (daemon *Daemon) UsesSnapshotter() bool { return daemon.usesSnapshotter } -func (daemon *Daemon) restore(cfg *configStore) error { +func (daemon *Daemon) loadContainers() (map[string]map[string]*container.Container, error) { var mapLock sync.Mutex - containers := make(map[string]*container.Container) + driverContainers := make(map[string]map[string]*container.Container) log.G(context.TODO()).Info("Loading containers: start.") dir, err := os.ReadDir(daemon.repository) if err != nil { - return err + return nil, err } // parallelLimit is the maximum number of parallel startup jobs that we @@ -238,29 +240,39 @@ func (daemon *Daemon) restore(cfg *configStore) error { logger.WithError(err).Error("failed to load container") return } - if c.Driver != daemon.imageService.StorageDriver() { - // Ignore the container if it wasn't created with the current storage-driver - logger.Debugf("not restoring container because it was created with another storage driver (%s)", c.Driver) - return - } - rwlayer, err := daemon.imageService.GetLayerByID(c.ID) - if err != nil { - logger.WithError(err).Error("failed to load container mount") - return - } - c.RWLayer = rwlayer - logger.WithFields(log.Fields{ - "running": c.IsRunning(), - "paused": c.IsPaused(), - }).Debug("loaded container") mapLock.Lock() - containers[c.ID] = c + if containers, ok := driverContainers[c.Driver]; !ok { + driverContainers[c.Driver] = map[string]*container.Container{ + c.ID: c, + } + } else { + containers[c.ID] = c + } mapLock.Unlock() }(v.Name()) } group.Wait() + return driverContainers, nil +} + +func (daemon *Daemon) restore(cfg *configStore, containers map[string]*container.Container) error { + var mapLock sync.Mutex + + log.G(context.TODO()).Info("Restoring containers: start.") + + // parallelLimit is the maximum number of parallel startup jobs that we + // allow (this is the limited used for all startup semaphores). The multipler + // (128) was chosen after some fairly significant benchmarking -- don't change + // it unless you've tested it significantly (this value is adjusted if + // RLIMIT_NOFILE is small to avoid EMFILE). + parallelLimit := adjustParallelLimit(len(containers), 128*runtime.NumCPU()) + + // Re-used for all parallel startup jobs. + var group sync.WaitGroup + sem := semaphore.NewWeighted(int64(parallelLimit)) + removeContainers := make(map[string]*container.Container) restartContainers := make(map[*container.Container]chan struct{}) activeSandboxes := make(map[string]any) @@ -274,6 +286,17 @@ func (daemon *Daemon) restore(cfg *configStore) error { logger := log.G(context.TODO()).WithField("container", c.ID) + rwlayer, err := daemon.imageService.GetLayerByID(c.ID) + if err != nil { + logger.WithError(err).Error("failed to load container mount") + return + } + c.RWLayer = rwlayer + logger.WithFields(log.Fields{ + "running": c.IsRunning(), + "paused": c.IsPaused(), + }).Debug("loaded container") + if err := daemon.registerName(c); err != nil { logger.WithError(err).Errorf("failed to register container name: %s", c.Name) mapLock.Lock() @@ -523,7 +546,7 @@ func (daemon *Daemon) restore(cfg *configStore) error { // // Note that we cannot initialize the network controller earlier, as it // needs to know if there's active sandboxes (running containers). - if err = daemon.initNetworkController(&cfg.Config, activeSandboxes); err != nil { + if err := daemon.initNetworkController(&cfg.Config, activeSandboxes); err != nil { return fmt.Errorf("Error initializing network controller: %v", err) } @@ -833,10 +856,17 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S d.configStore.Store(cfgStore) // TEST_INTEGRATION_USE_SNAPSHOTTER is used for integration tests only. + migrationThreshold := int64(-1) if os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "" { d.usesSnapshotter = true } else { - d.usesSnapshotter = config.Features["containerd-snapshotter"] + log.G(ctx).WithField("features", config.Features).Debug("Checking features for migration") + if config.Features["containerd-migration"] { + // TODO: Allow setting the threshold + migrationThreshold = math.MaxInt64 + } else { + d.usesSnapshotter = config.Features["containerd-snapshotter"] + } } // Ensure the daemon is properly shutdown if there is a failure during @@ -1033,12 +1063,17 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S d.linkIndex = newLinkIndex() + containers, err := d.loadContainers() + if err != nil { + return nil, err + } + // On Windows we don't support the environment variable, or a user supplied graphdriver // Unix platforms however run a single graphdriver for all containers, and it can // be set through an environment variable, a daemon start parameter, or chosen through // initialization of the layerstore through driver priority order for example. driverName := os.Getenv("DOCKER_DRIVER") - if isWindows && d.UsesSnapshotter() { + if isWindows && d.usesSnapshotter { // Containerd WCOW snapshotter driverName = "windows" } else if isWindows { @@ -1050,33 +1085,8 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S driverName = cfgStore.GraphDriver } - if d.UsesSnapshotter() { - if os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "" { - log.G(ctx).Warn("Enabling containerd snapshotter through the $TEST_INTEGRATION_USE_SNAPSHOTTER environment variable. This should only be used for testing.") - } - log.G(ctx).Info("Starting daemon with containerd snapshotter integration enabled") - - // FIXME(thaJeztah): implement automatic snapshotter-selection similar to graph-driver selection; see https://github.com/moby/moby/issues/44076 - if driverName == "" { - driverName = defaults.DefaultSnapshotter - } - - // Configure and validate the kernels security support. Note this is a Linux/FreeBSD - // operation only, so it is safe to pass *just* the runtime OS graphdriver. - if err := configureKernelSecuritySupport(&cfgStore.Config, driverName); err != nil { - return nil, err - } - d.imageService = ctrd.NewService(ctrd.ImageServiceConfig{ - Client: d.containerdClient, - Containers: d.containers, - Snapshotter: driverName, - RegistryHosts: d.RegistryHosts, - Registry: d.registryService, - EventsService: d.EventsService, - IDMapping: idMapping, - RefCountMounter: snapshotter.NewMounter(config.Root, driverName, idMapping), - }) - } else { + var migrationConfig migration.Config + if !d.usesSnapshotter { layerStore, err := layer.NewStoreFromOptions(layer.StoreOptions{ Root: cfgStore.Root, GraphDriver: driverName, @@ -1161,6 +1171,93 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S log.G(ctx).Debugf("Max Concurrent Downloads: %d", imgSvcConfig.MaxConcurrentDownloads) log.G(ctx).Debugf("Max Concurrent Uploads: %d", imgSvcConfig.MaxConcurrentUploads) log.G(ctx).Debugf("Max Download Attempts: %d", imgSvcConfig.MaxDownloadAttempts) + + // If no containers are running, check whether can migrate image service + if drv := d.imageService.StorageDriver(); len(containers[drv]) == 0 && migrationThreshold >= 0 { + switch drv { + case "overlay2": + driverName = "overlayfs" + case "windowsfilter": + driverName = "windows" + migrationThreshold = 0 + case "vfs": + driverName = "native" + default: + migrationThreshold = -1 + log.G(ctx).Infof("Not migrating to containerd snapshotter, no migration defined for graph driver %q", drv) + } + + var totalSize int64 + ic := d.imageService.CountImages(ctx) + if migrationThreshold >= 0 && ic > 0 { + sum, err := d.imageService.Images(ctx, imagetypes.ListOptions{All: true}) + if err != nil { + return nil, err + } + for _, s := range sum { + // Just add the size, don't consider shared size since this + // represents a maximum size + totalSize += s.Size + } + + } + + if totalSize <= migrationThreshold { + log.G(ctx).WithField("total", totalSize).Infof("Enabling containerd snapshotter because migration set with no containers and %d images in graph driver", ic) + d.usesSnapshotter = true + migrationConfig.LayerStore = imgSvcConfig.LayerStore + migrationConfig.DockerImageStore = imgSvcConfig.ImageStore + migrationConfig.ReferenceStore = imgSvcConfig.ReferenceStore + } else if migrationThreshold >= 0 { + log.G(ctx).Warnf("Not migrating to containerd snapshotter because still have %d images in graph driver", ic) + } + } else { + log.G(ctx).Debugf("Not attempting migration with %d containers and %d image threshold", len(containers[d.imageService.StorageDriver()]), migrationThreshold) + } + } + + if d.usesSnapshotter { + if os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "" { + log.G(ctx).Warn("Enabling containerd snapshotter through the $TEST_INTEGRATION_USE_SNAPSHOTTER environment variable. This should only be used for testing.") + } + log.G(ctx).Info("Starting daemon with containerd snapshotter integration enabled") + + // FIXME(thaJeztah): implement automatic snapshotter-selection similar to graph-driver selection; see https://github.com/moby/moby/issues/44076 + if driverName == "" { + driverName = defaults.DefaultSnapshotter + } + + // Configure and validate the kernels security support. Note this is a Linux/FreeBSD + // operation only, so it is safe to pass *just* the runtime OS graphdriver. + if err := configureKernelSecuritySupport(&cfgStore.Config, driverName); err != nil { + return nil, err + } + oldImageService := d.imageService + d.imageService = ctrd.NewService(ctrd.ImageServiceConfig{ + Client: d.containerdClient, + Containers: d.containers, + Snapshotter: driverName, + RegistryHosts: d.RegistryHosts, + Registry: d.registryService, + EventsService: d.EventsService, + IDMapping: idMapping, + RefCountMounter: snapshotter.NewMounter(config.Root, driverName, idMapping), + }) + + if oldImageService != nil { + if count := oldImageService.CountImages(ctx); count > 0 { + migrationConfig.Leases = d.containerdClient.LeasesService() + migrationConfig.Content = d.containerdClient.ContentStore() + migrationConfig.ImageStore = d.containerdClient.ImageService() + m := migration.NewLayerMigrator(migrationConfig) + err := m.MigrateTocontainerd(ctx, driverName, d.containerdClient.SnapshotService(driverName)) + if err != nil { + log.G(ctx).WithError(err).Errorf("Failed to migrate images to containerd, images in graph driver %q are no longer visible", oldImageService.StorageDriver()) + } else { + log.G(ctx).WithField("image_count", count).Infof("Successfully migrated images from %q to containerd", oldImageService.StorageDriver()) + } + } + } } go d.execCommandGC() @@ -1169,9 +1266,22 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S return nil, err } - if err := d.restore(cfgStore); err != nil { + driverContainers, ok := containers[driverName] + // Log containers which are not loaded with current driver + if (!ok && len(containers) > 0) || len(containers) > 1 { + for driver, all := range containers { + if driver == driverName { + continue + } + for id := range all { + log.G(ctx).WithField("container", id).Debugf("not restoring container because it was created with another storage driver (%s)", driver) + } + } + } + if err := d.restore(cfgStore, driverContainers); err != nil { return nil, err } + // Wait for migration to complete close(d.startupDone) info, err := d.SystemInfo(ctx) diff --git a/daemon/graphdriver/vfs/driver.go b/daemon/graphdriver/vfs/driver.go index 48940a5ef3..f416620e3a 100644 --- a/daemon/graphdriver/vfs/driver.go +++ b/daemon/graphdriver/vfs/driver.go @@ -86,7 +86,16 @@ func (d *Driver) Status() [][2]string { // GetMetadata is used for implementing the graphdriver.ProtoDriver interface. VFS does not currently have any meta data. func (d *Driver) GetMetadata(id string) (map[string]string, error) { - return nil, nil + dir := d.dir(id) + if _, err := os.Stat(dir); err != nil { + return nil, err + } + + metadata := map[string]string{ + "SourceDir": dir, + } + + return metadata, nil } // Cleanup is used to implement graphdriver.ProtoDriver. There is no cleanup required for this driver. diff --git a/integration/daemon/migration_test.go b/integration/daemon/migration_test.go new file mode 100644 index 0000000000..5949c7855c --- /dev/null +++ b/integration/daemon/migration_test.go @@ -0,0 +1,159 @@ +package daemon // import "github.com/docker/docker/integration/daemon" + +import ( + "bytes" + "io" + "os" + "runtime" + "testing" + + containertypes "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/image" + "github.com/moby/moby/v2/integration/internal/container" + "github.com/moby/moby/v2/testutil" + "github.com/moby/moby/v2/testutil/daemon" + "github.com/moby/moby/v2/testutil/fixtures/load" + "gotest.tools/v3/assert" + "gotest.tools/v3/skip" +) + +func TestMigrateOverlaySnapshotter(t *testing.T) { + testMigrateSnapshotter(t, "overlay2", "overlayfs") +} + +func TestMigrateNativeSnapshotter(t *testing.T) { + testMigrateSnapshotter(t, "vfs", "native") +} + +func testMigrateSnapshotter(t *testing.T, graphdriver, snapshotter string) { + skip.If(t, runtime.GOOS != "linux") + skip.If(t, os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "") + + ctx := testutil.StartSpan(baseContext, t) + + d := daemon.New(t) + defer d.Stop(t) + + d.Start(t, "--iptables=false", "--ip6tables=false", "-s", graphdriver) + info := d.Info(t) + id := info.ID + assert.Check(t, id != "") + assert.Equal(t, info.Containers, 0) + assert.Equal(t, info.Images, 0) + assert.Equal(t, info.Driver, graphdriver) + + load.FrozenImagesLinux(ctx, d.NewClientT(t), "busybox:latest") + + info = d.Info(t) + allImages := info.Images + assert.Check(t, allImages > 0) + + apiClient := d.NewClientT(t) + + containerID := container.Run(ctx, t, apiClient, func(c *container.TestContainerConfig) { + c.Name = "Migration-1-" + snapshotter + c.Config.Image = "busybox:latest" + c.Config.Cmd = []string{"top"} + }) + + d.Stop(t) + + // Start with migration feature but with a container which will prevent migration + d.Start(t, "--iptables=false", "--ip6tables=false", "-s", graphdriver, "--feature", "containerd-migration") + info = d.Info(t) + assert.Equal(t, info.ID, id) + assert.Equal(t, info.Driver, graphdriver) + assert.Equal(t, info.Containers, 1) + assert.Equal(t, info.Images, allImages) + container.Remove(ctx, t, apiClient, containerID, containertypes.RemoveOptions{ + Force: true, + }) + + d.Stop(t) + + d.Start(t, "--iptables=false", "--ip6tables=false", "-s", graphdriver, "--feature", "containerd-migration") + info = d.Info(t) + assert.Equal(t, info.ID, id) + assert.Equal(t, info.Containers, 0) + assert.Equal(t, info.Driver, snapshotter, "expected migrate to switch from %s to %s", graphdriver, snapshotter) + assert.Equal(t, info.Images, allImages) + + result := container.RunAttach(ctx, t, apiClient, func(c *container.TestContainerConfig) { + c.Name = "Migration-2-" + snapshotter + c.Config.Image = "busybox:latest" + c.Config.Cmd = []string{"echo", "hello"} + }) + assert.Equal(t, result.ExitCode, 0) + container.Remove(ctx, t, apiClient, result.ContainerID, containertypes.RemoveOptions{}) +} + +func TestMigrateSaveLoad(t *testing.T) { + skip.If(t, runtime.GOOS != "linux") + skip.If(t, os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "") + + var ( + ctx = testutil.StartSpan(baseContext, t) + d = daemon.New(t) + graphdriver = "overlay2" + snapshotter = "overlayfs" + ) + defer d.Stop(t) + + d.Start(t, "--iptables=false", "--ip6tables=false", "-s", graphdriver) + info := d.Info(t) + id := info.ID + assert.Check(t, id != "") + assert.Equal(t, info.Containers, 0) + assert.Equal(t, info.Images, 0) + assert.Equal(t, info.Driver, graphdriver) + + load.FrozenImagesLinux(ctx, d.NewClientT(t), "busybox:latest") + + info = d.Info(t) + allImages := info.Images + assert.Check(t, allImages > 0) + + d.Stop(t) + + d.Start(t, "--iptables=false", "--ip6tables=false", "-s", graphdriver, "--feature", "containerd-migration") + info = d.Info(t) + assert.Equal(t, info.ID, id) + assert.Equal(t, info.Containers, 0) + assert.Equal(t, info.Driver, snapshotter, "expected migrate to switch from %s to %s", graphdriver, snapshotter) + assert.Equal(t, info.Images, allImages) + + apiClient := d.NewClientT(t) + + // Save image to buffer + rdr, err := apiClient.ImageSave(ctx, []string{"busybox:latest"}, image.SaveOptions{}) + assert.NilError(t, err) + buf := bytes.NewBuffer(nil) + io.Copy(buf, rdr) + rdr.Close() + + // Delete all images + list, err := apiClient.ImageList(ctx, image.ListOptions{}) + assert.NilError(t, err) + for _, i := range list { + _, err = apiClient.ImageRemove(ctx, i.ID, image.RemoveOptions{}) + assert.NilError(t, err) + } + + // Check zero images + info = d.Info(t) + assert.Equal(t, info.Images, 0) + + // Import + lr, err := apiClient.ImageLoad(ctx, bytes.NewReader(buf.Bytes()), image.LoadOptions{Quiet: true}) + assert.NilError(t, err) + io.Copy(io.Discard, lr.Body) + lr.Body.Close() + + result := container.RunAttach(ctx, t, apiClient, func(c *container.TestContainerConfig) { + c.Name = "Migration-save-load-" + snapshotter + c.Config.Image = "busybox:latest" + c.Config.Cmd = []string{"echo", "hello"} + }) + assert.Equal(t, result.ExitCode, 0) + container.Remove(ctx, t, apiClient, result.ContainerID, containertypes.RemoveOptions{}) +} From 632de98f75dc87e0e1900097ee1177aa64c8c45d Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 9 Jul 2025 16:21:57 -0700 Subject: [PATCH 02/12] Enable containerd snapshotters by default Signed-off-by: Derek McGowan --- daemon/containerd/migration/migration.go | 1 + daemon/daemon.go | 136 +++++++++++++---------- daemon/graphdriver/driver.go | 6 + daemon/images/service.go | 6 +- daemon/reload_test.go | 6 +- 5 files changed, 91 insertions(+), 64 deletions(-) diff --git a/daemon/containerd/migration/migration.go b/daemon/containerd/migration/migration.go index fc380a2a7e..3af016f7c1 100644 --- a/daemon/containerd/migration/migration.go +++ b/daemon/containerd/migration/migration.go @@ -39,6 +39,7 @@ type LayerMigrator struct { } type Config struct { + ImageCount int LayerStore layer.Store ReferenceStore refstore.Store DockerImageStore image.Store diff --git a/daemon/daemon.go b/daemon/daemon.go index ee332935d9..79a9fe4a84 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -45,6 +45,7 @@ import ( ctrd "github.com/moby/moby/v2/daemon/containerd" "github.com/moby/moby/v2/daemon/containerd/migration" "github.com/moby/moby/v2/daemon/events" + "github.com/moby/moby/v2/daemon/graphdriver" _ "github.com/moby/moby/v2/daemon/graphdriver/register" // register graph drivers "github.com/moby/moby/v2/daemon/images" "github.com/moby/moby/v2/daemon/internal/distribution" @@ -855,19 +856,25 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S } d.configStore.Store(cfgStore) - // TEST_INTEGRATION_USE_SNAPSHOTTER is used for integration tests only. migrationThreshold := int64(-1) - if os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "" { - d.usesSnapshotter = true - } else { - log.G(ctx).WithField("features", config.Features).Debug("Checking features for migration") - if config.Features["containerd-migration"] { - // TODO: Allow setting the threshold - migrationThreshold = math.MaxInt64 - } else { - d.usesSnapshotter = config.Features["containerd-snapshotter"] + tryGraphDriver := graphdriver.IsRegistered + if os.Getenv("TEST_INTEGRATION_USE_GRAPHDRIVER") != "" { + tryGraphDriver = func(driver string) bool { + if driver == "" || graphdriver.IsRegistered(driver) { + return true + } + log.G(ctx).WithField("driver", driver).Warn("TEST_INTEGRATION_USE_GRAPHDRIVER is set but graphdriver is not registered") + return false } } + if config.Features["containerd-migration"] { + // TODO: Allow setting the threshold + migrationThreshold = math.MaxInt64 + log.G(ctx).WithField("max_size", migrationThreshold).Info("(Experimental) Migration to containerd is enabled, driver will be switched to snapshotter after migration is complete") + } + if config.Features["containerd-snapshotter"] { + log.G(ctx).Warn(`"containerd-snapshotter" is now the default and no longer needed to be set`) + } // Ensure the daemon is properly shutdown if there is a failure during // initialization @@ -1068,17 +1075,29 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S return nil, err } - // On Windows we don't support the environment variable, or a user supplied graphdriver + // On Windows we don't support the environment variable, or a user supplied graphdriver, + // but it is allowed when using snapshotters. // Unix platforms however run a single graphdriver for all containers, and it can // be set through an environment variable, a daemon start parameter, or chosen through // initialization of the layerstore through driver priority order for example. driverName := os.Getenv("DOCKER_DRIVER") - if isWindows && d.usesSnapshotter { - // Containerd WCOW snapshotter - driverName = "windows" - } else if isWindows { - // Docker WCOW graphdriver - driverName = "windowsfilter" + if isWindows { + if driverName == "" { + driverName = cfgStore.GraphDriver + } + switch driverName { + case "windows": + // Docker WCOW snapshotters + case "windowsfilter": + // Docker WCOW graphdriver + case "": + // Use graph driver but enable migration + driverName = "windowsfilter" + migrationThreshold = 0 + default: + log.G(ctx).Infof("Using non-default snapshotter %s", driverName) + + } } else if driverName != "" { log.G(ctx).Infof("Setting the storage driver from the $DOCKER_DRIVER environment variable (%s)", driverName) } else { @@ -1086,7 +1105,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S } var migrationConfig migration.Config - if !d.usesSnapshotter { + if tryGraphDriver(driverName) { layerStore, err := layer.NewStoreFromOptions(layer.StoreOptions{ Root: cfgStore.Root, GraphDriver: driverName, @@ -1163,23 +1182,13 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S } } - // TODO: imageStore, distributionMetadataStore, and ReferenceStore are only - // used above to run migration. They could be initialized in ImageService - // if migration is called from daemon/images. layerStore might move as well. - d.imageService = images.NewImageService(imgSvcConfig) - - log.G(ctx).Debugf("Max Concurrent Downloads: %d", imgSvcConfig.MaxConcurrentDownloads) - log.G(ctx).Debugf("Max Concurrent Uploads: %d", imgSvcConfig.MaxConcurrentUploads) - log.G(ctx).Debugf("Max Download Attempts: %d", imgSvcConfig.MaxDownloadAttempts) - // If no containers are running, check whether can migrate image service - if drv := d.imageService.StorageDriver(); len(containers[drv]) == 0 && migrationThreshold >= 0 { + if drv := layerStore.DriverName(); len(containers[drv]) == 0 && migrationThreshold >= 0 { switch drv { case "overlay2": driverName = "overlayfs" case "windowsfilter": driverName = "windows" - migrationThreshold = 0 case "vfs": driverName = "native" default: @@ -1188,38 +1197,45 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S } var totalSize int64 - ic := d.imageService.CountImages(ctx) + ic := imgSvcConfig.ImageStore.Len() if migrationThreshold >= 0 && ic > 0 { - sum, err := d.imageService.Images(ctx, imagetypes.ListOptions{All: true}) - if err != nil { - return nil, err - } - for _, s := range sum { - // Just add the size, don't consider shared size since this - // represents a maximum size - totalSize += s.Size + for _, img := range imgSvcConfig.ImageStore.Map() { + if layerID := img.RootFS.ChainID(); layerID != "" { + l, err := imgSvcConfig.LayerStore.Get(layerID) + if err != nil { + if errors.Is(err, layer.ErrLayerDoesNotExist) { + continue + } + return nil, err + } + + // Just look at layer sizze for considering maximum size + totalSize += l.Size() + layer.ReleaseAndLog(imgSvcConfig.LayerStore, l) + } } } if totalSize <= migrationThreshold { log.G(ctx).WithField("total", totalSize).Infof("Enabling containerd snapshotter because migration set with no containers and %d images in graph driver", ic) - d.usesSnapshotter = true - migrationConfig.LayerStore = imgSvcConfig.LayerStore - migrationConfig.DockerImageStore = imgSvcConfig.ImageStore - migrationConfig.ReferenceStore = imgSvcConfig.ReferenceStore + migrationConfig = migration.Config{ + ImageCount: ic, + LayerStore: imgSvcConfig.LayerStore, + DockerImageStore: imgSvcConfig.ImageStore, + ReferenceStore: imgSvcConfig.ReferenceStore, + } } else if migrationThreshold >= 0 { log.G(ctx).Warnf("Not migrating to containerd snapshotter because still have %d images in graph driver", ic) + d.imageService = images.NewImageService(ctx, imgSvcConfig) } } else { - log.G(ctx).Debugf("Not attempting migration with %d containers and %d image threshold", len(containers[d.imageService.StorageDriver()]), migrationThreshold) + log.G(ctx).Debugf("Not attempting migration with %d containers and %d image threshold", len(containers[drv]), migrationThreshold) + d.imageService = images.NewImageService(ctx, imgSvcConfig) } } - if d.usesSnapshotter { - if os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "" { - log.G(ctx).Warn("Enabling containerd snapshotter through the $TEST_INTEGRATION_USE_SNAPSHOTTER environment variable. This should only be used for testing.") - } + if d.imageService == nil { log.G(ctx).Info("Starting daemon with containerd snapshotter integration enabled") // FIXME(thaJeztah): implement automatic snapshotter-selection similar to graph-driver selection; see https://github.com/moby/moby/issues/44076 @@ -1227,12 +1243,14 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S driverName = defaults.DefaultSnapshotter } + // TODO: Load containerd drivers and check if the driver is initialized + // Configure and validate the kernels security support. Note this is a Linux/FreeBSD // operation only, so it is safe to pass *just* the runtime OS graphdriver. if err := configureKernelSecuritySupport(&cfgStore.Config, driverName); err != nil { return nil, err } - oldImageService := d.imageService + d.usesSnapshotter = true d.imageService = ctrd.NewService(ctrd.ImageServiceConfig{ Client: d.containerdClient, Containers: d.containers, @@ -1244,18 +1262,16 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S RefCountMounter: snapshotter.NewMounter(config.Root, driverName, idMapping), }) - if oldImageService != nil { - if count := oldImageService.CountImages(ctx); count > 0 { - migrationConfig.Leases = d.containerdClient.LeasesService() - migrationConfig.Content = d.containerdClient.ContentStore() - migrationConfig.ImageStore = d.containerdClient.ImageService() - m := migration.NewLayerMigrator(migrationConfig) - err := m.MigrateTocontainerd(ctx, driverName, d.containerdClient.SnapshotService(driverName)) - if err != nil { - log.G(ctx).WithError(err).Errorf("Failed to migrate images to containerd, images in graph driver %q are no longer visible", oldImageService.StorageDriver()) - } else { - log.G(ctx).WithField("image_count", count).Infof("Successfully migrated images from %q to containerd", oldImageService.StorageDriver()) - } + if migrationConfig.ImageCount > 0 { + migrationConfig.Leases = d.containerdClient.LeasesService() + migrationConfig.Content = d.containerdClient.ContentStore() + migrationConfig.ImageStore = d.containerdClient.ImageService() + m := migration.NewLayerMigrator(migrationConfig) + err := m.MigrateTocontainerd(ctx, driverName, d.containerdClient.SnapshotService(driverName)) + if err != nil { + log.G(ctx).WithError(err).Errorf("Failed to migrate images to containerd, images in graph driver %q are no longer visible", migrationConfig.LayerStore.DriverName()) + } else { + log.G(ctx).WithField("image_count", migrationConfig.ImageCount).Infof("Successfully migrated images from %q to containerd", migrationConfig.LayerStore.DriverName()) } } } diff --git a/daemon/graphdriver/driver.go b/daemon/graphdriver/driver.go index ba138b2778..039777547e 100644 --- a/daemon/graphdriver/driver.go +++ b/daemon/graphdriver/driver.go @@ -121,6 +121,12 @@ func Register(name string, initFunc InitFunc) error { return nil } +// IsRegistered checks to see if the drive with the given name is registered +func IsRegistered(name string) bool { + _, exists := drivers[name] + return exists +} + // getDriver initializes and returns the registered driver. func getDriver(name string, config Options) (Driver, error) { if initFunc, exists := drivers[name]; exists { diff --git a/daemon/images/service.go b/daemon/images/service.go index f02d41ea42..a2235c8f64 100644 --- a/daemon/images/service.go +++ b/daemon/images/service.go @@ -49,7 +49,11 @@ type ImageServiceConfig struct { } // NewImageService returns a new ImageService from a configuration -func NewImageService(config ImageServiceConfig) *ImageService { +func NewImageService(ctx context.Context, config ImageServiceConfig) *ImageService { + log.G(ctx).Debugf("Max Concurrent Downloads: %d", config.MaxConcurrentDownloads) + log.G(ctx).Debugf("Max Concurrent Uploads: %d", config.MaxConcurrentUploads) + log.G(ctx).Debugf("Max Download Attempts: %d", config.MaxDownloadAttempts) + return &ImageService{ containers: config.ContainerStore, distributionMetadataStore: config.DistributionMetadataStore, diff --git a/daemon/reload_test.go b/daemon/reload_test.go index 9d970cd593..a2e98bd6a9 100644 --- a/daemon/reload_test.go +++ b/daemon/reload_test.go @@ -25,7 +25,7 @@ func muteLogs(t *testing.T) { func newDaemonForReloadT(t *testing.T, cfg *config.Config) *Daemon { t.Helper() daemon := &Daemon{ - imageService: images.NewImageService(images.ImageServiceConfig{}), + imageService: images.NewImageService(context.TODO(), images.ImageServiceConfig{}), } var err error daemon.registryService, err = registry.NewService(registry.ServiceOptions{}) @@ -63,7 +63,7 @@ func TestDaemonReloadLabels(t *testing.T) { func TestDaemonReloadMirrors(t *testing.T) { daemon := &Daemon{ - imageService: images.NewImageService(images.ImageServiceConfig{}), + imageService: images.NewImageService(context.TODO(), images.ImageServiceConfig{}), } muteLogs(t) @@ -162,7 +162,7 @@ func TestDaemonReloadMirrors(t *testing.T) { func TestDaemonReloadInsecureRegistries(t *testing.T) { daemon := &Daemon{ - imageService: images.NewImageService(images.ImageServiceConfig{}), + imageService: images.NewImageService(context.TODO(), images.ImageServiceConfig{}), } muteLogs(t) From 7f87cf9d8ae565829c15742493c7713c22cad40d Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 9 Jul 2025 19:19:36 -0700 Subject: [PATCH 03/12] Check for snapshotter plugin availability Signed-off-by: Derek McGowan --- daemon/daemon.go | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/daemon/daemon.go b/daemon/daemon.go index 79a9fe4a84..65a2bfdf96 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -10,6 +10,7 @@ import ( "crypto/sha256" "encoding/binary" "fmt" + "maps" "math" "net" "net/netip" @@ -1238,12 +1239,37 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S if d.imageService == nil { log.G(ctx).Info("Starting daemon with containerd snapshotter integration enabled") - // FIXME(thaJeztah): implement automatic snapshotter-selection similar to graph-driver selection; see https://github.com/moby/moby/issues/44076 - if driverName == "" { - driverName = defaults.DefaultSnapshotter + resp, err := d.containerdClient.IntrospectionService().Plugins(ctx, `type=="io.containerd.snapshotter.v1"`) + if err != nil { + return nil, fmt.Errorf("failed to get containerd plugins: %w", err) + } + if resp == nil || len(resp.Plugins) == 0 { + return nil, fmt.Errorf("failed to get containerd plugins response: %w", cerrdefs.ErrUnavailable) + } + availableDrivers := map[string]struct{}{} + for _, p := range resp.Plugins { + if p == nil || p.Type != "io.containerd.snapshotter.v1" { + continue + } + if p.InitErr == nil { + availableDrivers[p.ID] = struct{}{} + } else if (p.ID == driverName) || (driverName == "" && p.ID == defaults.DefaultSnapshotter) { + log.G(ctx).WithField("message", p.InitErr.Message).Warn("Preferred snapshotter not available in containerd") + } } - // TODO: Load containerd drivers and check if the driver is initialized + if driverName == "" { + if _, ok := availableDrivers[defaults.DefaultSnapshotter]; ok { + driverName = defaults.DefaultSnapshotter + } else if _, ok := availableDrivers["native"]; ok { + driverName = "native" + } else { + log.G(ctx).WithField("available", maps.Keys(availableDrivers)).Debug("Preferred snapshotter not available in containerd") + return nil, fmt.Errorf("snapshotter selection failed, no drivers available: %w", cerrdefs.ErrUnavailable) + } + } else if _, ok := availableDrivers[driverName]; !ok { + return nil, fmt.Errorf("configured driver %q not available: %w", driverName, cerrdefs.ErrUnavailable) + } // Configure and validate the kernels security support. Note this is a Linux/FreeBSD // operation only, so it is safe to pass *just* the runtime OS graphdriver. From 00463b92166f1fd2960f27aefb821ea046ee32c7 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 9 Jul 2025 19:13:32 -0700 Subject: [PATCH 04/12] Fix containerd image count Ensure image count returned by containerd image service only includes the count of unique images. Signed-off-by: Derek McGowan --- daemon/containerd/service.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/daemon/containerd/service.go b/daemon/containerd/service.go index 7a7a4b0932..10e56b1c7c 100644 --- a/daemon/containerd/service.go +++ b/daemon/containerd/service.go @@ -98,7 +98,15 @@ func (i *ImageService) CountImages(ctx context.Context) int { return 0 } - return len(imgs) + uniqueImages := map[digest.Digest]struct{}{} + for _, i := range imgs { + dgst := i.Target().Digest + if _, ok := uniqueImages[dgst]; !ok { + uniqueImages[dgst] = struct{}{} + } + } + + return len(uniqueImages) } // LayerStoreStatus returns the status for each layer store From 8700bca2bf164ff417c56af64d2110b1b5ff36e1 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 9 Jul 2025 19:14:24 -0700 Subject: [PATCH 05/12] Update migration test to use graphdriver env Signed-off-by: Derek McGowan --- .github/workflows/.test.yml | 2 +- .github/workflows/.windows.yml | 6 +++--- Makefile | 2 +- hack/make/.integration-test-helpers | 2 +- hack/make/test-docker-py | 2 +- integration/daemon/migration_test.go | 11 ++++++----- testutil/environment/environment.go | 5 ++--- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/.test.yml b/.github/workflows/.test.yml index dfd93f11db..8178ac9675 100644 --- a/.github/workflows/.test.yml +++ b/.github/workflows/.test.yml @@ -27,7 +27,7 @@ env: ITG_CLI_MATRIX_SIZE: 6 DOCKER_EXPERIMENTAL: 1 DOCKER_GRAPHDRIVER: ${{ inputs.storage == 'snapshotter' && 'overlayfs' || 'overlay2' }} - TEST_INTEGRATION_USE_SNAPSHOTTER: ${{ inputs.storage == 'snapshotter' && '1' || '' }} + TEST_INTEGRATION_USE_GRAPHDRIVER: ${{ inputs.storage == 'graphdriver' && '1' || '' }} SETUP_BUILDX_VERSION: edge SETUP_BUILDKIT_IMAGE: moby/buildkit:latest diff --git a/.github/workflows/.windows.yml b/.github/workflows/.windows.yml index f476a7100d..1f8f968110 100644 --- a/.github/workflows/.windows.yml +++ b/.github/workflows/.windows.yml @@ -364,10 +364,10 @@ jobs: "--exec-root=$env:TEMP\moby-exec", ` "--pidfile=$env:TEMP\docker.pid", ` "--register-service" - If ("${{ inputs.storage }}" -eq "snapshotter") { + If ("${{ inputs.storage }}" -eq "graphdriver") { # Make the env-var visible to the service-managed dockerd, as there's no CLI flag for this option. - & reg add "HKLM\SYSTEM\CurrentControlSet\Services\docker" /v Environment /t REG_MULTI_SZ /s '@' /d TEST_INTEGRATION_USE_SNAPSHOTTER=1 - echo "TEST_INTEGRATION_USE_SNAPSHOTTER=1" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append + & reg add "HKLM\SYSTEM\CurrentControlSet\Services\docker" /v Environment /t REG_MULTI_SZ /s '@' /d TEST_INTEGRATION_USE_GRAPHDRIVER=1 + echo "TEST_INTEGRATION_USE_GRAPHDRIVER=1" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append } Write-Host "Starting service" Start-Service -Name docker diff --git a/Makefile b/Makefile index 87f2dfef43..26e453c642 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ DOCKER_ENVS := \ -e GITHUB_ACTIONS \ -e TEST_FORCE_VALIDATE \ -e TEST_INTEGRATION_DIR \ - -e TEST_INTEGRATION_USE_SNAPSHOTTER \ + -e TEST_INTEGRATION_USE_GRAPHDRIVER \ -e TEST_INTEGRATION_FAIL_FAST \ -e TEST_SKIP_INTEGRATION \ -e TEST_SKIP_INTEGRATION_CLI \ diff --git a/hack/make/.integration-test-helpers b/hack/make/.integration-test-helpers index 3c7d7265b7..334d5a76bf 100644 --- a/hack/make/.integration-test-helpers +++ b/hack/make/.integration-test-helpers @@ -210,7 +210,7 @@ test_env() { PATH="$PATH" \ TEMP="$TEMP" \ TEST_CLIENT_BINARY="$TEST_CLIENT_BINARY" \ - TEST_INTEGRATION_USE_SNAPSHOTTER="$TEST_INTEGRATION_USE_SNAPSHOTTER" \ + TEST_INTEGRATION_USE_GRAPHDRIVER="$TEST_INTEGRATION_USE_GRAPHDRIVER" \ OTEL_EXPORTER_OTLP_ENDPOINT="$OTEL_EXPORTER_OTLP_ENDPOINT" \ OTEL_SERVICE_NAME="$OTEL_SERVICE_NAME" \ "$@" diff --git a/hack/make/test-docker-py b/hack/make/test-docker-py index 764ccfc08e..4d3f98cbef 100644 --- a/hack/make/test-docker-py +++ b/hack/make/test-docker-py @@ -15,7 +15,7 @@ source hack/make/.integration-test-helpers : "${PY_TEST_OPTIONS:=--junitxml=${DEST}/junit-report.xml}" # build --squash is not supported with containerd integration. -if [ -n "$TEST_INTEGRATION_USE_SNAPSHOTTER" ]; then +if [ -z "$TEST_INTEGRATION_USE_GRAPHDRIVER" ]; then PY_TEST_OPTIONS="$PY_TEST_OPTIONS --deselect=tests/integration/api_build_test.py::BuildTest::test_build_squash" fi diff --git a/integration/daemon/migration_test.go b/integration/daemon/migration_test.go index 5949c7855c..d56cfac05c 100644 --- a/integration/daemon/migration_test.go +++ b/integration/daemon/migration_test.go @@ -9,6 +9,7 @@ import ( containertypes "github.com/moby/moby/api/types/container" "github.com/moby/moby/api/types/image" + "github.com/moby/moby/client" "github.com/moby/moby/v2/integration/internal/container" "github.com/moby/moby/v2/testutil" "github.com/moby/moby/v2/testutil/daemon" @@ -27,7 +28,7 @@ func TestMigrateNativeSnapshotter(t *testing.T) { func testMigrateSnapshotter(t *testing.T, graphdriver, snapshotter string) { skip.If(t, runtime.GOOS != "linux") - skip.If(t, os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "") + skip.If(t, os.Getenv("TEST_INTEGRATION_USE_GRAPHDRIVER") == "") ctx := testutil.StartSpan(baseContext, t) @@ -89,7 +90,7 @@ func testMigrateSnapshotter(t *testing.T, graphdriver, snapshotter string) { func TestMigrateSaveLoad(t *testing.T) { skip.If(t, runtime.GOOS != "linux") - skip.If(t, os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "") + skip.If(t, os.Getenv("TEST_INTEGRATION_USE_GRAPHDRIVER") == "") var ( ctx = testutil.StartSpan(baseContext, t) @@ -125,7 +126,7 @@ func TestMigrateSaveLoad(t *testing.T) { apiClient := d.NewClientT(t) // Save image to buffer - rdr, err := apiClient.ImageSave(ctx, []string{"busybox:latest"}, image.SaveOptions{}) + rdr, err := apiClient.ImageSave(ctx, []string{"busybox:latest"}) assert.NilError(t, err) buf := bytes.NewBuffer(nil) io.Copy(buf, rdr) @@ -135,7 +136,7 @@ func TestMigrateSaveLoad(t *testing.T) { list, err := apiClient.ImageList(ctx, image.ListOptions{}) assert.NilError(t, err) for _, i := range list { - _, err = apiClient.ImageRemove(ctx, i.ID, image.RemoveOptions{}) + _, err = apiClient.ImageRemove(ctx, i.ID, image.RemoveOptions{Force: true}) assert.NilError(t, err) } @@ -144,7 +145,7 @@ func TestMigrateSaveLoad(t *testing.T) { assert.Equal(t, info.Images, 0) // Import - lr, err := apiClient.ImageLoad(ctx, bytes.NewReader(buf.Bytes()), image.LoadOptions{Quiet: true}) + lr, err := apiClient.ImageLoad(ctx, bytes.NewReader(buf.Bytes()), client.ImageLoadWithQuiet(true)) assert.NilError(t, err) io.Copy(io.Discard, lr.Body) lr.Body.Close() diff --git a/testutil/environment/environment.go b/testutil/environment/environment.go index d89b4cc3d4..2ab1f6433e 100644 --- a/testutil/environment/environment.go +++ b/testutil/environment/environment.go @@ -189,10 +189,9 @@ func (e *Execution) IsUserNamespaceInKernel() bool { } // UsingSnapshotter returns whether containerd snapshotters are used for the -// tests by checking if the "TEST_INTEGRATION_USE_SNAPSHOTTER" is set to a -// non-empty value. +// tests by checking if the "TEST_INTEGRATION_USE_GRAPHDRIVER" is empty func (e *Execution) UsingSnapshotter() bool { - return os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "" + return os.Getenv("TEST_INTEGRATION_USE_GRAPHDRIVER") == "" } // HasExistingImage checks whether there is an image with the given reference. From b41babafaad73e8706e089c98cf4e207ca4c9069 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 9 Jul 2025 19:51:05 -0700 Subject: [PATCH 06/12] Fix windows test graphdriver setting Signed-off-by: Derek McGowan --- daemon/daemon.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/daemon/daemon.go b/daemon/daemon.go index 65a2bfdf96..797ab9c310 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -1094,7 +1094,10 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S case "": // Use graph driver but enable migration driverName = "windowsfilter" - migrationThreshold = 0 + if os.Getenv("TEST_INTEGRATION_USE_GRAPHDRIVER") == "" { + // Don't force migration if graph driver is explicit + migrationThreshold = 0 + } default: log.G(ctx).Infof("Using non-default snapshotter %s", driverName) From 632fb0c89a9bcedc2d68cb758ad10b6d8b26ae10 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 23 Jul 2025 08:32:00 -0700 Subject: [PATCH 07/12] Update graphdriver check logic to account for disabling of snapshotter Ensure migration is never attempted multiple times. Signed-off-by: Derek McGowan --- daemon/daemon.go | 43 ++++++++++++++++---------- hack/dockerfile/etc/docker/daemon.json | 3 -- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/daemon/daemon.go b/daemon/daemon.go index 797ab9c310..b23e937c15 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -858,14 +858,15 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S d.configStore.Store(cfgStore) migrationThreshold := int64(-1) - tryGraphDriver := graphdriver.IsRegistered - if os.Getenv("TEST_INTEGRATION_USE_GRAPHDRIVER") != "" { - tryGraphDriver = func(driver string) bool { + isGraphDriver := func(driver string) (bool, error) { + return graphdriver.IsRegistered(driver), nil + } + if enabled, ok := config.Features["containerd-snapshotter"]; (ok && !enabled) || os.Getenv("TEST_INTEGRATION_USE_GRAPHDRIVER") != "" { + isGraphDriver = func(driver string) (bool, error) { if driver == "" || graphdriver.IsRegistered(driver) { - return true + return true, nil } - log.G(ctx).WithField("driver", driver).Warn("TEST_INTEGRATION_USE_GRAPHDRIVER is set but graphdriver is not registered") - return false + return false, fmt.Errorf("graphdriver is explicitly enabled but %q is not registered", driver) } } if config.Features["containerd-migration"] { @@ -1109,7 +1110,11 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S } var migrationConfig migration.Config - if tryGraphDriver(driverName) { + tryGraphDriver, err := isGraphDriver(driverName) + if err != nil { + return nil, err + } + if tryGraphDriver { layerStore, err := layer.NewStoreFromOptions(layer.StoreOptions{ Root: cfgStore.Root, GraphDriver: driverName, @@ -1186,7 +1191,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S } } - // If no containers are running, check whether can migrate image service + // If no containers are present, check whether can migrate image service if drv := layerStore.DriverName(); len(containers[drv]) == 0 && migrationThreshold >= 0 { switch drv { case "overlay2": @@ -1232,6 +1237,8 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S } else if migrationThreshold >= 0 { log.G(ctx).Warnf("Not migrating to containerd snapshotter because still have %d images in graph driver", ic) d.imageService = images.NewImageService(ctx, imgSvcConfig) + } else { + d.imageService = images.NewImageService(ctx, imgSvcConfig) } } else { log.G(ctx).Debugf("Not attempting migration with %d containers and %d image threshold", len(containers[drv]), migrationThreshold) @@ -1292,15 +1299,19 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S }) if migrationConfig.ImageCount > 0 { - migrationConfig.Leases = d.containerdClient.LeasesService() - migrationConfig.Content = d.containerdClient.ContentStore() - migrationConfig.ImageStore = d.containerdClient.ImageService() - m := migration.NewLayerMigrator(migrationConfig) - err := m.MigrateTocontainerd(ctx, driverName, d.containerdClient.SnapshotService(driverName)) - if err != nil { - log.G(ctx).WithError(err).Errorf("Failed to migrate images to containerd, images in graph driver %q are no longer visible", migrationConfig.LayerStore.DriverName()) + if d.imageService.CountImages(ctx) > 0 { + log.G(ctx).WithField("image_count", migrationConfig.ImageCount).Warnf("Images not migrated because images already exist in containerd %q", migrationConfig.LayerStore.DriverName()) } else { - log.G(ctx).WithField("image_count", migrationConfig.ImageCount).Infof("Successfully migrated images from %q to containerd", migrationConfig.LayerStore.DriverName()) + migrationConfig.Leases = d.containerdClient.LeasesService() + migrationConfig.Content = d.containerdClient.ContentStore() + migrationConfig.ImageStore = d.containerdClient.ImageService() + m := migration.NewLayerMigrator(migrationConfig) + err := m.MigrateTocontainerd(ctx, driverName, d.containerdClient.SnapshotService(driverName)) + if err != nil { + log.G(ctx).WithError(err).Errorf("Failed to migrate images to containerd, images in graph driver %q are no longer visible", migrationConfig.LayerStore.DriverName()) + } else { + log.G(ctx).WithField("image_count", migrationConfig.ImageCount).Infof("Successfully migrated images from %q to containerd", migrationConfig.LayerStore.DriverName()) + } } } } diff --git a/hack/dockerfile/etc/docker/daemon.json b/hack/dockerfile/etc/docker/daemon.json index 2319c13a67..ff9fe36aa6 100644 --- a/hack/dockerfile/etc/docker/daemon.json +++ b/hack/dockerfile/etc/docker/daemon.json @@ -3,8 +3,5 @@ "crun": { "path": "/usr/local/bin/crun" } - }, - "features": { - "containerd-snapshotter": false } } From 4816383c0b7f655abedcbe785ae6d2cd2f2627e8 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 24 Jul 2025 12:05:55 -0700 Subject: [PATCH 08/12] Add environment variable to define the threshold Signed-off-by: Derek McGowan --- daemon/daemon.go | 57 ++++++++++++++++++---------- integration/daemon/migration_test.go | 9 +++-- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/daemon/daemon.go b/daemon/daemon.go index b23e937c15..da3e1a2a95 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -11,7 +11,6 @@ import ( "encoding/binary" "fmt" "maps" - "math" "net" "net/netip" "os" @@ -30,6 +29,7 @@ import ( "github.com/containerd/log" "github.com/distribution/reference" dist "github.com/docker/distribution" + "github.com/docker/go-units" "github.com/moby/buildkit/util/grpcerrors" "github.com/moby/buildkit/util/tracing" "github.com/moby/locker" @@ -39,6 +39,19 @@ import ( registrytypes "github.com/moby/moby/api/types/registry" "github.com/moby/moby/api/types/swarm" volumetypes "github.com/moby/moby/api/types/volume" + "github.com/moby/sys/user" + "github.com/moby/sys/userns" + "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "go.opentelemetry.io/otel" + "golang.org/x/sync/semaphore" + "google.golang.org/grpc" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/credentials/insecure" + "resenje.org/singleflight" + "tags.cncf.io/container-device-interface/pkg/cdi" + "github.com/moby/moby/v2/daemon/builder" executorpkg "github.com/moby/moby/v2/daemon/cluster/executor" "github.com/moby/moby/v2/daemon/config" @@ -75,18 +88,6 @@ import ( "github.com/moby/moby/v2/pkg/authorization" "github.com/moby/moby/v2/pkg/plugingetter" "github.com/moby/moby/v2/pkg/sysinfo" - "github.com/moby/sys/user" - "github.com/moby/sys/userns" - "github.com/pkg/errors" - bolt "go.etcd.io/bbolt" - "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" - "go.opentelemetry.io/otel" - "golang.org/x/sync/semaphore" - "google.golang.org/grpc" - "google.golang.org/grpc/backoff" - "google.golang.org/grpc/credentials/insecure" - "resenje.org/singleflight" - "tags.cncf.io/container-device-interface/pkg/cdi" ) type configStore struct { @@ -866,13 +867,27 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S if driver == "" || graphdriver.IsRegistered(driver) { return true, nil } - return false, fmt.Errorf("graphdriver is explicitly enabled but %q is not registered", driver) + return false, fmt.Errorf("graphdriver is explicitly enabled but %q is not registered, %v %v", driver, config.Features, os.Getenv("TEST_INTEGRATION_USE_GRAPHDRIVER")) } } if config.Features["containerd-migration"] { - // TODO: Allow setting the threshold - migrationThreshold = math.MaxInt64 - log.G(ctx).WithField("max_size", migrationThreshold).Info("(Experimental) Migration to containerd is enabled, driver will be switched to snapshotter after migration is complete") + if ts := os.Getenv("DOCKER_MIGRATE_SNAPSHOTTER_THRESHOLD"); ts != "" { + v, err := units.FromHumanSize(ts) + if err == nil { + migrationThreshold = v + } else { + log.G(ctx).WithError(err).WithField("size", ts).Warn("Invalid migration threshold value, defaulting to 0") + migrationThreshold = 0 + } + + } else { + migrationThreshold = 0 + } + if migrationThreshold > 0 { + log.G(ctx).WithField("max_size", migrationThreshold).Info("(Experimental) Migration to containerd is enabled, driver will be switched to snapshotter after migration is complete") + } else { + log.G(ctx).WithField("env", os.Environ()).Info("Migration to containerd is enabled, driver will be switched to snapshotter if there are no images or containers") + } } if config.Features["containerd-snapshotter"] { log.G(ctx).Warn(`"containerd-snapshotter" is now the default and no longer needed to be set`) @@ -1082,7 +1097,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S // Unix platforms however run a single graphdriver for all containers, and it can // be set through an environment variable, a daemon start parameter, or chosen through // initialization of the layerstore through driver priority order for example. - driverName := os.Getenv("DOCKER_DRIVER") + driverName := os.Getenv("DOCKER_GRAPHDRIVER") if isWindows { if driverName == "" { driverName = cfgStore.GraphDriver @@ -1104,7 +1119,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S } } else if driverName != "" { - log.G(ctx).Infof("Setting the storage driver from the $DOCKER_DRIVER environment variable (%s)", driverName) + log.G(ctx).Infof("Setting the storage driver from the $DOCKER_GRAPHDRIVER environment variable (%s)", driverName) } else { driverName = cfgStore.GraphDriver } @@ -1218,7 +1233,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S return nil, err } - // Just look at layer sizze for considering maximum size + // Just look at layer size for considering maximum size totalSize += l.Size() layer.ReleaseAndLog(imgSvcConfig.LayerStore, l) } @@ -1235,7 +1250,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S ReferenceStore: imgSvcConfig.ReferenceStore, } } else if migrationThreshold >= 0 { - log.G(ctx).Warnf("Not migrating to containerd snapshotter because still have %d images in graph driver", ic) + log.G(ctx).WithField("total", totalSize).Warnf("Not migrating to containerd snapshotter because still have %d images in graph driver", ic) d.imageService = images.NewImageService(ctx, imgSvcConfig) } else { d.imageService = images.NewImageService(ctx, imgSvcConfig) diff --git a/integration/daemon/migration_test.go b/integration/daemon/migration_test.go index d56cfac05c..b7850030b8 100644 --- a/integration/daemon/migration_test.go +++ b/integration/daemon/migration_test.go @@ -3,7 +3,6 @@ package daemon // import "github.com/docker/docker/integration/daemon" import ( "bytes" "io" - "os" "runtime" "testing" @@ -28,7 +27,9 @@ func TestMigrateNativeSnapshotter(t *testing.T) { func testMigrateSnapshotter(t *testing.T, graphdriver, snapshotter string) { skip.If(t, runtime.GOOS != "linux") - skip.If(t, os.Getenv("TEST_INTEGRATION_USE_GRAPHDRIVER") == "") + + t.Setenv("DOCKER_MIGRATE_SNAPSHOTTER_THRESHOLD", "200M") + t.Setenv("DOCKER_GRAPHDRIVER", "") ctx := testutil.StartSpan(baseContext, t) @@ -90,7 +91,9 @@ func testMigrateSnapshotter(t *testing.T, graphdriver, snapshotter string) { func TestMigrateSaveLoad(t *testing.T) { skip.If(t, runtime.GOOS != "linux") - skip.If(t, os.Getenv("TEST_INTEGRATION_USE_GRAPHDRIVER") == "") + + t.Setenv("DOCKER_MIGRATE_SNAPSHOTTER_THRESHOLD", "200M") + t.Setenv("DOCKER_GRAPHDRIVER", "") var ( ctx = testutil.StartSpan(baseContext, t) From 99181f56ce2017b38fa7836627b15da773143da8 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 30 Jul 2025 19:27:51 -0700 Subject: [PATCH 09/12] Fix symlink evaluation to a directory that may not exist During the arm64 tests, the rootfs directory does not seem to exist when this test is run and will cause a failure when using snapshotter. Signed-off-by: Derek McGowan --- integration/volume/mount_test.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/integration/volume/mount_test.go b/integration/volume/mount_test.go index 5e82f750bf..e4d6641f32 100644 --- a/integration/volume/mount_test.go +++ b/integration/volume/mount_test.go @@ -147,7 +147,7 @@ func TestRunMountImage(t *testing.T) { {name: "image_remove_force", cmd: []string{"cat", "/image/foo"}, expected: "barbar"}, } { t.Run(tc.name, func(t *testing.T) { - testImage := setupTestImage(t, ctx, apiClient, tc.name, testEnv.UsingSnapshotter()) + testImage := setupTestImage(t, ctx, apiClient, tc.name) if testImage != "" { defer apiClient.ImageRemove(ctx, testImage, image.RemoveOptions{Force: true}) } @@ -292,26 +292,19 @@ func setupTestVolume(t *testing.T, apiClient client.APIClient) string { return volumeName } -func setupTestImage(t *testing.T, ctx context.Context, apiClient client.APIClient, test string, snapshotter bool) string { +func setupTestImage(t *testing.T, ctx context.Context, apiClient client.APIClient, test string) string { imgName := "test-image" if test == "image_tag" { imgName += ":foo" } - var symlink string - if snapshotter { - symlink = "../../../../rootfs" - } else { - symlink = "../../../../../docker" - } - //nolint:dupword // ignore "Duplicate words (subdir) found (dupword)" dockerfile := ` FROM busybox as symlink RUN mkdir /hack \ && ln -s "../subdir" /hack/good \ - && ln -s "` + symlink + `" /hack/bad + && ln -s "../../../../../docker" /hack/bad #-- FROM scratch COPY foo / From ead007f1f1e632e3eab99caee444b99696fefcda Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 31 Jul 2025 17:07:50 -0700 Subject: [PATCH 10/12] Use native snapshotter for integration tests and run Signed-off-by: Derek McGowan --- hack/make/.integration-daemon-start | 2 +- hack/make/run | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/make/.integration-daemon-start b/hack/make/.integration-daemon-start index 78f056356f..06c2485b58 100644 --- a/hack/make/.integration-daemon-start +++ b/hack/make/.integration-daemon-start @@ -40,7 +40,7 @@ fi # intentionally open a couple bogus file descriptors to help test that they get scrubbed in containers exec 41>&1 42>&2 -export DOCKER_GRAPHDRIVER=${DOCKER_GRAPHDRIVER:-vfs} +export DOCKER_GRAPHDRIVER=${DOCKER_GRAPHDRIVER:-native} export DOCKER_USERLANDPROXY=${DOCKER_USERLANDPROXY:-true} # example usage: DOCKER_STORAGE_OPTS="dm.basesize=20G,dm.loopdatasize=200G" diff --git a/hack/make/run b/hack/make/run index a528d18c07..3702e8c9cf 100644 --- a/hack/make/run +++ b/hack/make/run @@ -10,7 +10,7 @@ fi DOCKER_COMMAND="$(command -v dockerd)" -DOCKER_GRAPHDRIVER=${DOCKER_GRAPHDRIVER:-vfs} +DOCKER_GRAPHDRIVER=${DOCKER_GRAPHDRIVER:-native} DOCKER_USERLANDPROXY=${DOCKER_USERLANDPROXY:-true} # example usage: DOCKER_STORAGE_OPTS="dm.basesize=20G,dm.loopdatasize=200G" From 85b79f83f435c92b09c74e1c315f91d665498e7b Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 8 Aug 2025 11:42:58 -0700 Subject: [PATCH 11/12] Fix hardlink handling in containerd snapshot remap When files are hardlinked, the inodes only need to be chowned once. Signed-off-by: Derek McGowan --- daemon/containerd/image_snapshot_unix.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/daemon/containerd/image_snapshot_unix.go b/daemon/containerd/image_snapshot_unix.go index 5188a65e51..f5985da351 100644 --- a/daemon/containerd/image_snapshot_unix.go +++ b/daemon/containerd/image_snapshot_unix.go @@ -71,6 +71,7 @@ func (i *ImageService) copyAndUnremapRootFS(ctx context.Context, dst, src []moun return fmt.Errorf("failed to copy: %w", err) } + inos := make(map[uint64]struct{}) return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { if err != nil { return err @@ -80,11 +81,16 @@ func (i *ImageService) copyAndUnremapRootFS(ctx context.Context, dst, src []moun if stat == nil { return fmt.Errorf("cannot get underlying data for %s", path) } + if _, ok := inos[stat.Ino]; ok { + // Inode already processed, skip + return nil + } uid, gid, err := i.idMapping.ToContainer(int(stat.Uid), int(stat.Gid)) if err != nil { return err } + inos[stat.Ino] = struct{}{} return chownWithCaps(path, uid, gid) }) From a4fbbc15706767dffd1ec64b5b50b0310d2c3b9f Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 8 Aug 2025 11:59:54 -0700 Subject: [PATCH 12/12] Add context to restore and load containers Signed-off-by: Derek McGowan --- daemon/daemon.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/daemon/daemon.go b/daemon/daemon.go index da3e1a2a95..654ac941d8 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -207,11 +207,11 @@ func (daemon *Daemon) UsesSnapshotter() bool { return daemon.usesSnapshotter } -func (daemon *Daemon) loadContainers() (map[string]map[string]*container.Container, error) { +func (daemon *Daemon) loadContainers(ctx context.Context) (map[string]map[string]*container.Container, error) { var mapLock sync.Mutex driverContainers := make(map[string]map[string]*container.Container) - log.G(context.TODO()).Info("Loading containers: start.") + log.G(ctx).Info("Loading containers: start.") dir, err := os.ReadDir(daemon.repository) if err != nil { @@ -236,7 +236,7 @@ func (daemon *Daemon) loadContainers() (map[string]map[string]*container.Contain _ = sem.Acquire(context.Background(), 1) defer sem.Release(1) - logger := log.G(context.TODO()).WithField("container", id) + logger := log.G(ctx).WithField("container", id) c, err := daemon.load(id) if err != nil { @@ -260,10 +260,10 @@ func (daemon *Daemon) loadContainers() (map[string]map[string]*container.Contain return driverContainers, nil } -func (daemon *Daemon) restore(cfg *configStore, containers map[string]*container.Container) error { +func (daemon *Daemon) restore(ctx context.Context, cfg *configStore, containers map[string]*container.Container) error { var mapLock sync.Mutex - log.G(context.TODO()).Info("Restoring containers: start.") + log.G(ctx).Info("Restoring containers: start.") // parallelLimit is the maximum number of parallel startup jobs that we // allow (this is the limited used for all startup semaphores). The multipler @@ -287,7 +287,7 @@ func (daemon *Daemon) restore(cfg *configStore, containers map[string]*container _ = sem.Acquire(context.Background(), 1) defer sem.Release(1) - logger := log.G(context.TODO()).WithField("container", c.ID) + logger := log.G(ctx).WithField("container", c.ID) rwlayer, err := daemon.imageService.GetLayerByID(c.ID) if err != nil { @@ -301,14 +301,14 @@ func (daemon *Daemon) restore(cfg *configStore, containers map[string]*container }).Debug("loaded container") if err := daemon.registerName(c); err != nil { - logger.WithError(err).Errorf("failed to register container name: %s", c.Name) + log.G(ctx).WithError(err).Errorf("failed to register container name: %s", c.Name) mapLock.Lock() delete(containers, c.ID) mapLock.Unlock() return } if err := daemon.register(context.TODO(), c); err != nil { - logger.WithError(err).Error("failed to register container") + log.G(ctx).WithError(err).Error("failed to register container") mapLock.Lock() delete(containers, c.ID) mapLock.Unlock() @@ -325,7 +325,7 @@ func (daemon *Daemon) restore(cfg *configStore, containers map[string]*container _ = sem.Acquire(context.Background(), 1) defer sem.Release(1) - baseLogger := log.G(context.TODO()).WithField("container", c.ID) + baseLogger := log.G(ctx).WithField("container", c.ID) if c.HostConfig != nil { // Migrate containers that don't have the default ("no") restart-policy set. @@ -578,7 +578,7 @@ func (daemon *Daemon) restore(cfg *configStore, containers map[string]*container _ = sem.Acquire(context.Background(), 1) if err := daemon.registerLinks(c, c.HostConfig); err != nil { - log.G(context.TODO()).WithField("container", c.ID).WithError(err).Error("failed to register link for container") + log.G(ctx).WithField("container", c.ID).WithError(err).Error("failed to register link for container") } sem.Release(1) @@ -592,7 +592,7 @@ func (daemon *Daemon) restore(cfg *configStore, containers map[string]*container go func(c *container.Container, chNotify chan struct{}) { _ = sem.Acquire(context.Background(), 1) - logger := log.G(context.TODO()).WithField("container", c.ID) + logger := log.G(ctx).WithField("container", c.ID) logger.Debug("starting container") @@ -632,7 +632,7 @@ func (daemon *Daemon) restore(cfg *configStore, containers map[string]*container _ = sem.Acquire(context.Background(), 1) if err := daemon.containerRm(&cfg.Config, cid, &backend.ContainerRmConfig{ForceRemove: true, RemoveVolume: true}); err != nil { - log.G(context.TODO()).WithField("container", cid).WithError(err).Error("failed to remove container") + log.G(ctx).WithField("container", cid).WithError(err).Error("failed to remove container") } sem.Release(1) @@ -663,7 +663,7 @@ func (daemon *Daemon) restore(cfg *configStore, containers map[string]*container _ = sem.Acquire(context.Background(), 1) if err := daemon.prepareMountPoints(c); err != nil { - log.G(context.TODO()).WithField("container", c.ID).WithError(err).Error("failed to prepare mountpoints for container") + log.G(ctx).WithField("container", c.ID).WithError(err).Error("failed to prepare mountpoints for container") } sem.Release(1) @@ -672,7 +672,7 @@ func (daemon *Daemon) restore(cfg *configStore, containers map[string]*container } group.Wait() - log.G(context.TODO()).Info("Loading containers: done.") + log.G(ctx).Info("Loading containers: done.") return nil } @@ -1087,7 +1087,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S d.linkIndex = newLinkIndex() - containers, err := d.loadContainers() + containers, err := d.loadContainers(ctx) if err != nil { return nil, err } @@ -1349,7 +1349,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S } } } - if err := d.restore(cfgStore, driverContainers); err != nil { + if err := d.restore(ctx, cfgStore, driverContainers); err != nil { return nil, err } // Wait for migration to complete