diff --git a/.github/workflows/.test.yml b/.github/workflows/.test.yml index dfd93f11db..8178ac9675 100644 --- a/.github/workflows/.test.yml +++ b/.github/workflows/.test.yml @@ -27,7 +27,7 @@ env: ITG_CLI_MATRIX_SIZE: 6 DOCKER_EXPERIMENTAL: 1 DOCKER_GRAPHDRIVER: ${{ inputs.storage == 'snapshotter' && 'overlayfs' || 'overlay2' }} - TEST_INTEGRATION_USE_SNAPSHOTTER: ${{ inputs.storage == 'snapshotter' && '1' || '' }} + TEST_INTEGRATION_USE_GRAPHDRIVER: ${{ inputs.storage == 'graphdriver' && '1' || '' }} SETUP_BUILDX_VERSION: edge SETUP_BUILDKIT_IMAGE: moby/buildkit:latest diff --git a/.github/workflows/.windows.yml b/.github/workflows/.windows.yml index f476a7100d..1f8f968110 100644 --- a/.github/workflows/.windows.yml +++ b/.github/workflows/.windows.yml @@ -364,10 +364,10 @@ jobs: "--exec-root=$env:TEMP\moby-exec", ` "--pidfile=$env:TEMP\docker.pid", ` "--register-service" - If ("${{ inputs.storage }}" -eq "snapshotter") { + If ("${{ inputs.storage }}" -eq "graphdriver") { # Make the env-var visible to the service-managed dockerd, as there's no CLI flag for this option. - & reg add "HKLM\SYSTEM\CurrentControlSet\Services\docker" /v Environment /t REG_MULTI_SZ /s '@' /d TEST_INTEGRATION_USE_SNAPSHOTTER=1 - echo "TEST_INTEGRATION_USE_SNAPSHOTTER=1" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append + & reg add "HKLM\SYSTEM\CurrentControlSet\Services\docker" /v Environment /t REG_MULTI_SZ /s '@' /d TEST_INTEGRATION_USE_GRAPHDRIVER=1 + echo "TEST_INTEGRATION_USE_GRAPHDRIVER=1" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append } Write-Host "Starting service" Start-Service -Name docker diff --git a/Makefile b/Makefile index 87f2dfef43..26e453c642 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ DOCKER_ENVS := \ -e GITHUB_ACTIONS \ -e TEST_FORCE_VALIDATE \ -e TEST_INTEGRATION_DIR \ - -e TEST_INTEGRATION_USE_SNAPSHOTTER \ + -e TEST_INTEGRATION_USE_GRAPHDRIVER \ -e TEST_INTEGRATION_FAIL_FAST \ -e TEST_SKIP_INTEGRATION \ -e TEST_SKIP_INTEGRATION_CLI \ diff --git a/daemon/containerd/image_snapshot_unix.go b/daemon/containerd/image_snapshot_unix.go index 5188a65e51..f5985da351 100644 --- a/daemon/containerd/image_snapshot_unix.go +++ b/daemon/containerd/image_snapshot_unix.go @@ -71,6 +71,7 @@ func (i *ImageService) copyAndUnremapRootFS(ctx context.Context, dst, src []moun return fmt.Errorf("failed to copy: %w", err) } + inos := make(map[uint64]struct{}) return filepath.Walk(root, func(path string, info os.FileInfo, err error) error { if err != nil { return err @@ -80,11 +81,16 @@ func (i *ImageService) copyAndUnremapRootFS(ctx context.Context, dst, src []moun if stat == nil { return fmt.Errorf("cannot get underlying data for %s", path) } + if _, ok := inos[stat.Ino]; ok { + // Inode already processed, skip + return nil + } uid, gid, err := i.idMapping.ToContainer(int(stat.Uid), int(stat.Gid)) if err != nil { return err } + inos[stat.Ino] = struct{}{} return chownWithCaps(path, uid, gid) }) diff --git a/daemon/containerd/migration/migration.go b/daemon/containerd/migration/migration.go new file mode 100644 index 0000000000..3af016f7c1 --- /dev/null +++ b/daemon/containerd/migration/migration.go @@ -0,0 +1,317 @@ +package migration + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "strings" + "sync" + "time" + + "github.com/containerd/containerd/v2/core/content" + "github.com/containerd/containerd/v2/core/images" + "github.com/containerd/containerd/v2/core/leases" + "github.com/containerd/containerd/v2/core/mount" + "github.com/containerd/containerd/v2/core/snapshots" + "github.com/containerd/containerd/v2/pkg/archive/compression" + "github.com/containerd/continuity/fs" + cerrdefs "github.com/containerd/errdefs" + "github.com/containerd/log" + "github.com/moby/moby/v2/daemon/internal/image" + "github.com/moby/moby/v2/daemon/internal/layer" + refstore "github.com/moby/moby/v2/daemon/internal/refstore" + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/identity" + "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "golang.org/x/sync/errgroup" +) + +type LayerMigrator struct { + layers layer.Store + refs refstore.Store + dis image.Store + leases leases.Manager + content content.Store + cis images.Store +} + +type Config struct { + ImageCount int + LayerStore layer.Store + ReferenceStore refstore.Store + DockerImageStore image.Store + Leases leases.Manager + Content content.Store + ImageStore images.Store +} + +func NewLayerMigrator(config Config) *LayerMigrator { + return &LayerMigrator{ + layers: config.LayerStore, + refs: config.ReferenceStore, + dis: config.DockerImageStore, + leases: config.Leases, + content: config.Content, + cis: config.ImageStore, + } +} + +// MigrateTocontainerd migrates containers from overlay2 to overlayfs or vfs to native +func (lm *LayerMigrator) MigrateTocontainerd(ctx context.Context, snKey string, sn snapshots.Snapshotter) error { + if sn == nil { + return fmt.Errorf("no snapshotter to migrate to: %w", cerrdefs.ErrNotImplemented) + } + + switch driver := lm.layers.DriverName(); driver { + case "overlay2": + case "vfs": + default: + return fmt.Errorf("%q not supported for migration: %w", driver, cerrdefs.ErrNotImplemented) + } + + var ( + // Zstd makes migration 10x faster + // TODO: make configurable + layerMediaType = ocispec.MediaTypeImageLayerZstd + layerCompression = compression.Zstd + ) + + l, err := lm.leases.Create(ctx, leases.WithRandomID(), leases.WithExpiration(24*time.Hour)) + if err != nil { + return err + } + defer func() { + lm.leases.Delete(ctx, l) + }() + ctx = leases.WithLease(ctx, l.ID) + + for imgID, img := range lm.dis.Heads() { + diffids := img.RootFS.DiffIDs + if len(diffids) == 0 { + continue + } + var ( + parent string + manifest = ocispec.Manifest{ + MediaType: ocispec.MediaTypeImageManifest, + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + Layers: make([]ocispec.Descriptor, len(diffids)), + } + ml sync.Mutex + eg, egctx = errgroup.WithContext(ctx) + ) + for i := range diffids { + chainID := identity.ChainID(diffids[:i+1]) + l, err := lm.layers.Get(chainID) + if err != nil { + return fmt.Errorf("failed to get layer [%d] %q: %w", i, chainID, err) + } + layerIndex := i + eg.Go(func() error { + ctx := egctx + t1 := time.Now() + ts, err := l.TarStream() + if err != nil { + return err + } + + desc := ocispec.Descriptor{ + MediaType: layerMediaType, + } + + cw, err := lm.content.Writer(ctx, + content.WithRef(fmt.Sprintf("ingest-%s", chainID)), + content.WithDescriptor(desc)) + if err != nil { + return fmt.Errorf("failed to get content writer: %w", err) + } + + dgstr := digest.Canonical.Digester() + cs, _ := compression.CompressStream(io.MultiWriter(cw, dgstr.Hash()), layerCompression) + _, err = io.Copy(cs, ts) + if err != nil { + return fmt.Errorf("failed to copy to compressed stream: %w", err) + } + cs.Close() + + status, err := cw.Status() + if err != nil { + return err + } + + desc.Size = status.Offset + desc.Digest = dgstr.Digest() + + if err := cw.Commit(ctx, desc.Size, desc.Digest); err != nil && !cerrdefs.IsAlreadyExists(err) { + return err + } + + log.G(ctx).WithFields(log.Fields{ + "t": time.Since(t1), + "size": desc.Size, + "digest": desc.Digest, + }).Debug("Converted layer to content tar") + + ml.Lock() + manifest.Layers[layerIndex] = desc + ml.Unlock() + return nil + }) + + metadata, err := l.Metadata() + if err != nil { + return err + } + src, ok := metadata["UpperDir"] + if !ok { + src, ok = metadata["SourceDir"] + if !ok { + log.G(ctx).WithField("metadata", metadata).WithField("driver", lm.layers.DriverName()).Debug("no source directory metadata") + return fmt.Errorf("graphdriver not supported: %w", cerrdefs.ErrNotImplemented) + } + } + log.G(ctx).WithField("metadata", metadata).Debugf("migrating %s from %s", chainID, src) + + active := fmt.Sprintf("migration-%s", chainID) + + key := chainID.String() + + snapshotLabels := map[string]string{ + "containerd.io/snapshot.ref": key, + } + mounts, err := sn.Prepare(ctx, active, parent, snapshots.WithLabels(snapshotLabels)) + parent = key + if err != nil { + if cerrdefs.IsAlreadyExists(err) { + continue + } + return err + } + + dst, err := extractSource(mounts) + if err != nil { + return err + } + + t1 := time.Now() + if err := fs.CopyDir(dst, src); err != nil { + return err + } + log.G(ctx).WithFields(log.Fields{ + "t": time.Since(t1), + "key": key, + }).Debug("Copied layer to snapshot") + + if err := sn.Commit(ctx, key, active); err != nil && !cerrdefs.IsAlreadyExists(err) { + return err + } + } + + configBytes := img.RawJSON() + digest.FromBytes(configBytes) + manifest.Config = ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageConfig, + Digest: digest.FromBytes(configBytes), + Size: int64(len(configBytes)), + } + + configLabels := map[string]string{ + fmt.Sprintf("containerd.io/gc.ref.snapshot.%s", snKey): parent, + } + if err = content.WriteBlob(ctx, lm.content, "config"+manifest.Config.Digest.String(), bytes.NewReader(configBytes), manifest.Config, content.WithLabels(configLabels)); err != nil && !cerrdefs.IsAlreadyExists(err) { + return err + } + + if err := eg.Wait(); err != nil { + return err + } + + manifestBytes, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return err + } + + manifestDesc := ocispec.Descriptor{ + MediaType: manifest.MediaType, + Digest: digest.FromBytes(manifestBytes), + Size: int64(len(manifestBytes)), + } + + manifestLabels := map[string]string{ + "containerd.io/gc.ref.content.config": manifest.Config.Digest.String(), + } + for i := range manifest.Layers { + manifestLabels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = manifest.Layers[i].Digest.String() + } + + if err = content.WriteBlob(ctx, lm.content, "manifest"+manifestDesc.Digest.String(), bytes.NewReader(manifestBytes), manifestDesc, content.WithLabels(manifestLabels)); err != nil && !cerrdefs.IsAlreadyExists(err) { + return err + } + + childrenHandler := images.ChildrenHandler(lm.content) + childrenHandler = images.SetChildrenMappedLabels(lm.content, childrenHandler, nil) + if err = images.Walk(ctx, childrenHandler, manifestDesc); err != nil { + return err + } + + var added bool + for _, named := range lm.refs.References(digest.Digest(imgID)) { + img := images.Image{ + Name: named.String(), + Target: manifestDesc, + // TODO: Any labels? + } + img, err = lm.cis.Create(ctx, img) + if err != nil && !cerrdefs.IsAlreadyExists(err) { + return err + } else if err != nil { + log.G(ctx).Infof("Tag already exists: %s", named) + continue + } + + log.G(ctx).Infof("Migrated image %s to %s", img.Name, img.Target.Digest) + added = true + } + + if !added { + img := images.Image{ + Name: "moby-dangling@" + manifestDesc.Digest.String(), + Target: manifestDesc, + // TODO: Any labels? + } + img, err = lm.cis.Create(ctx, img) + if err != nil && !cerrdefs.IsAlreadyExists(err) { + return err + } else if err == nil { + log.G(ctx).Infof("Migrated image %s to %s", img.Name, img.Target.Digest) + } + } + } + + return nil +} + +func extractSource(mounts []mount.Mount) (string, error) { + if len(mounts) != 1 { + return "", fmt.Errorf("cannot support snapshotters with multiple mount sources: %w", cerrdefs.ErrNotImplemented) + } + switch mounts[0].Type { + case "bind": + return mounts[0].Source, nil + case "overlay": + for _, option := range mounts[0].Options { + if strings.HasPrefix(option, "upperdir=") { + return option[9:], nil + } + } + default: + return "", fmt.Errorf("mount type %q not supported: %w", mounts[0].Type, cerrdefs.ErrNotImplemented) + } + + return "", fmt.Errorf("mount is missing upper option: %w", cerrdefs.ErrNotImplemented) +} diff --git a/daemon/containerd/service.go b/daemon/containerd/service.go index 7a7a4b0932..10e56b1c7c 100644 --- a/daemon/containerd/service.go +++ b/daemon/containerd/service.go @@ -98,7 +98,15 @@ func (i *ImageService) CountImages(ctx context.Context) int { return 0 } - return len(imgs) + uniqueImages := map[digest.Digest]struct{}{} + for _, i := range imgs { + dgst := i.Target().Digest + if _, ok := uniqueImages[dgst]; !ok { + uniqueImages[dgst] = struct{}{} + } + } + + return len(uniqueImages) } // LayerStoreStatus returns the status for each layer store diff --git a/daemon/daemon.go b/daemon/daemon.go index 6865ecef5a..0369222474 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -10,6 +10,7 @@ import ( "crypto/sha256" "encoding/binary" "fmt" + "maps" "net" "net/netip" "os" @@ -28,6 +29,7 @@ import ( "github.com/containerd/log" "github.com/distribution/reference" dist "github.com/docker/distribution" + "github.com/docker/go-units" "github.com/moby/buildkit/util/grpcerrors" "github.com/moby/buildkit/util/tracing" "github.com/moby/locker" @@ -37,12 +39,27 @@ import ( registrytypes "github.com/moby/moby/api/types/registry" "github.com/moby/moby/api/types/swarm" volumetypes "github.com/moby/moby/api/types/volume" + "github.com/moby/sys/user" + "github.com/moby/sys/userns" + "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + "go.opentelemetry.io/otel" + "golang.org/x/sync/semaphore" + "google.golang.org/grpc" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/credentials/insecure" + "resenje.org/singleflight" + "tags.cncf.io/container-device-interface/pkg/cdi" + "github.com/moby/moby/v2/daemon/builder" executorpkg "github.com/moby/moby/v2/daemon/cluster/executor" "github.com/moby/moby/v2/daemon/config" "github.com/moby/moby/v2/daemon/container" ctrd "github.com/moby/moby/v2/daemon/containerd" + "github.com/moby/moby/v2/daemon/containerd/migration" "github.com/moby/moby/v2/daemon/events" + "github.com/moby/moby/v2/daemon/graphdriver" _ "github.com/moby/moby/v2/daemon/graphdriver/register" // register graph drivers "github.com/moby/moby/v2/daemon/images" "github.com/moby/moby/v2/daemon/internal/distribution" @@ -72,18 +89,6 @@ import ( "github.com/moby/moby/v2/pkg/authorization" "github.com/moby/moby/v2/pkg/plugingetter" "github.com/moby/moby/v2/pkg/sysinfo" - "github.com/moby/sys/user" - "github.com/moby/sys/userns" - "github.com/pkg/errors" - bolt "go.etcd.io/bbolt" - "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" - "go.opentelemetry.io/otel" - "golang.org/x/sync/semaphore" - "google.golang.org/grpc" - "google.golang.org/grpc/backoff" - "google.golang.org/grpc/credentials/insecure" - "resenje.org/singleflight" - "tags.cncf.io/container-device-interface/pkg/cdi" ) type configStore struct { @@ -203,15 +208,15 @@ func (daemon *Daemon) UsesSnapshotter() bool { return daemon.usesSnapshotter } -func (daemon *Daemon) restore(cfg *configStore) error { +func (daemon *Daemon) loadContainers(ctx context.Context) (map[string]map[string]*container.Container, error) { var mapLock sync.Mutex - containers := make(map[string]*container.Container) + driverContainers := make(map[string]map[string]*container.Container) - log.G(context.TODO()).Info("Loading containers: start.") + log.G(ctx).Info("Loading containers: start.") dir, err := os.ReadDir(daemon.repository) if err != nil { - return err + return nil, err } // parallelLimit is the maximum number of parallel startup jobs that we @@ -232,36 +237,46 @@ func (daemon *Daemon) restore(cfg *configStore) error { _ = sem.Acquire(context.Background(), 1) defer sem.Release(1) - logger := log.G(context.TODO()).WithField("container", id) + logger := log.G(ctx).WithField("container", id) c, err := daemon.load(id) if err != nil { logger.WithError(err).Error("failed to load container") return } - if c.Driver != daemon.imageService.StorageDriver() { - // Ignore the container if it wasn't created with the current storage-driver - logger.Debugf("not restoring container because it was created with another storage driver (%s)", c.Driver) - return - } - rwlayer, err := daemon.imageService.GetLayerByID(c.ID) - if err != nil { - logger.WithError(err).Error("failed to load container mount") - return - } - c.RWLayer = rwlayer - logger.WithFields(log.Fields{ - "running": c.IsRunning(), - "paused": c.IsPaused(), - }).Debug("loaded container") mapLock.Lock() - containers[c.ID] = c + if containers, ok := driverContainers[c.Driver]; !ok { + driverContainers[c.Driver] = map[string]*container.Container{ + c.ID: c, + } + } else { + containers[c.ID] = c + } mapLock.Unlock() }(v.Name()) } group.Wait() + return driverContainers, nil +} + +func (daemon *Daemon) restore(ctx context.Context, cfg *configStore, containers map[string]*container.Container) error { + var mapLock sync.Mutex + + log.G(ctx).Info("Restoring containers: start.") + + // parallelLimit is the maximum number of parallel startup jobs that we + // allow (this is the limited used for all startup semaphores). The multipler + // (128) was chosen after some fairly significant benchmarking -- don't change + // it unless you've tested it significantly (this value is adjusted if + // RLIMIT_NOFILE is small to avoid EMFILE). + parallelLimit := adjustParallelLimit(len(containers), 128*runtime.NumCPU()) + + // Re-used for all parallel startup jobs. + var group sync.WaitGroup + sem := semaphore.NewWeighted(int64(parallelLimit)) + removeContainers := make(map[string]*container.Container) restartContainers := make(map[*container.Container]chan struct{}) activeSandboxes := make(map[string]any) @@ -273,17 +288,28 @@ func (daemon *Daemon) restore(cfg *configStore) error { _ = sem.Acquire(context.Background(), 1) defer sem.Release(1) - logger := log.G(context.TODO()).WithField("container", c.ID) + logger := log.G(ctx).WithField("container", c.ID) + + rwlayer, err := daemon.imageService.GetLayerByID(c.ID) + if err != nil { + logger.WithError(err).Error("failed to load container mount") + return + } + c.RWLayer = rwlayer + logger.WithFields(log.Fields{ + "running": c.IsRunning(), + "paused": c.IsPaused(), + }).Debug("loaded container") if err := daemon.registerName(c); err != nil { - logger.WithError(err).Errorf("failed to register container name: %s", c.Name) + log.G(ctx).WithError(err).Errorf("failed to register container name: %s", c.Name) mapLock.Lock() delete(containers, c.ID) mapLock.Unlock() return } if err := daemon.register(context.TODO(), c); err != nil { - logger.WithError(err).Error("failed to register container") + log.G(ctx).WithError(err).Error("failed to register container") mapLock.Lock() delete(containers, c.ID) mapLock.Unlock() @@ -300,7 +326,7 @@ func (daemon *Daemon) restore(cfg *configStore) error { _ = sem.Acquire(context.Background(), 1) defer sem.Release(1) - baseLogger := log.G(context.TODO()).WithField("container", c.ID) + baseLogger := log.G(ctx).WithField("container", c.ID) if c.HostConfig != nil { // Migrate containers that don't have the default ("no") restart-policy set. @@ -524,7 +550,7 @@ func (daemon *Daemon) restore(cfg *configStore) error { // // Note that we cannot initialize the network controller earlier, as it // needs to know if there's active sandboxes (running containers). - if err = daemon.initNetworkController(&cfg.Config, activeSandboxes); err != nil { + if err := daemon.initNetworkController(&cfg.Config, activeSandboxes); err != nil { return fmt.Errorf("Error initializing network controller: %v", err) } @@ -553,7 +579,7 @@ func (daemon *Daemon) restore(cfg *configStore) error { _ = sem.Acquire(context.Background(), 1) if err := daemon.registerLinks(c, c.HostConfig); err != nil { - log.G(context.TODO()).WithField("container", c.ID).WithError(err).Error("failed to register link for container") + log.G(ctx).WithField("container", c.ID).WithError(err).Error("failed to register link for container") } sem.Release(1) @@ -567,7 +593,7 @@ func (daemon *Daemon) restore(cfg *configStore) error { go func(c *container.Container, chNotify chan struct{}) { _ = sem.Acquire(context.Background(), 1) - logger := log.G(context.TODO()).WithField("container", c.ID) + logger := log.G(ctx).WithField("container", c.ID) logger.Debug("starting container") @@ -607,7 +633,7 @@ func (daemon *Daemon) restore(cfg *configStore) error { _ = sem.Acquire(context.Background(), 1) if err := daemon.containerRm(&cfg.Config, cid, &backend.ContainerRmConfig{ForceRemove: true, RemoveVolume: true}); err != nil { - log.G(context.TODO()).WithField("container", cid).WithError(err).Error("failed to remove container") + log.G(ctx).WithField("container", cid).WithError(err).Error("failed to remove container") } sem.Release(1) @@ -638,7 +664,7 @@ func (daemon *Daemon) restore(cfg *configStore) error { _ = sem.Acquire(context.Background(), 1) if err := daemon.prepareMountPoints(c); err != nil { - log.G(context.TODO()).WithField("container", c.ID).WithError(err).Error("failed to prepare mountpoints for container") + log.G(ctx).WithField("container", c.ID).WithError(err).Error("failed to prepare mountpoints for container") } sem.Release(1) @@ -647,7 +673,7 @@ func (daemon *Daemon) restore(cfg *configStore) error { } group.Wait() - log.G(context.TODO()).Info("Loading containers: done.") + log.G(ctx).Info("Loading containers: done.") return nil } @@ -833,11 +859,39 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S } d.configStore.Store(cfgStore) - // TEST_INTEGRATION_USE_SNAPSHOTTER is used for integration tests only. - if os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "" { - d.usesSnapshotter = true - } else { - d.usesSnapshotter = config.Features["containerd-snapshotter"] + migrationThreshold := int64(-1) + isGraphDriver := func(driver string) (bool, error) { + return graphdriver.IsRegistered(driver), nil + } + if enabled, ok := config.Features["containerd-snapshotter"]; (ok && !enabled) || os.Getenv("TEST_INTEGRATION_USE_GRAPHDRIVER") != "" { + isGraphDriver = func(driver string) (bool, error) { + if driver == "" || graphdriver.IsRegistered(driver) { + return true, nil + } + return false, fmt.Errorf("graphdriver is explicitly enabled but %q is not registered, %v %v", driver, config.Features, os.Getenv("TEST_INTEGRATION_USE_GRAPHDRIVER")) + } + } + if config.Features["containerd-migration"] { + if ts := os.Getenv("DOCKER_MIGRATE_SNAPSHOTTER_THRESHOLD"); ts != "" { + v, err := units.FromHumanSize(ts) + if err == nil { + migrationThreshold = v + } else { + log.G(ctx).WithError(err).WithField("size", ts).Warn("Invalid migration threshold value, defaulting to 0") + migrationThreshold = 0 + } + + } else { + migrationThreshold = 0 + } + if migrationThreshold > 0 { + log.G(ctx).WithField("max_size", migrationThreshold).Info("(Experimental) Migration to containerd is enabled, driver will be switched to snapshotter after migration is complete") + } else { + log.G(ctx).WithField("env", os.Environ()).Info("Migration to containerd is enabled, driver will be switched to snapshotter if there are no images or containers") + } + } + if config.Features["containerd-snapshotter"] { + log.G(ctx).Warn(`"containerd-snapshotter" is now the default and no longer needed to be set`) } // Ensure the daemon is properly shutdown if there is a failure during @@ -1034,50 +1088,49 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S d.linkIndex = newLinkIndex() - // On Windows we don't support the environment variable, or a user supplied graphdriver + containers, err := d.loadContainers(ctx) + if err != nil { + return nil, err + } + + // On Windows we don't support the environment variable, or a user supplied graphdriver, + // but it is allowed when using snapshotters. // Unix platforms however run a single graphdriver for all containers, and it can // be set through an environment variable, a daemon start parameter, or chosen through // initialization of the layerstore through driver priority order for example. - driverName := os.Getenv("DOCKER_DRIVER") - if isWindows && d.UsesSnapshotter() { - // Containerd WCOW snapshotter - driverName = "windows" - } else if isWindows { - // Docker WCOW graphdriver - driverName = "windowsfilter" + driverName := os.Getenv("DOCKER_GRAPHDRIVER") + if isWindows { + if driverName == "" { + driverName = cfgStore.GraphDriver + } + switch driverName { + case "windows": + // Docker WCOW snapshotters + case "windowsfilter": + // Docker WCOW graphdriver + case "": + // Use graph driver but enable migration + driverName = "windowsfilter" + if os.Getenv("TEST_INTEGRATION_USE_GRAPHDRIVER") == "" { + // Don't force migration if graph driver is explicit + migrationThreshold = 0 + } + default: + log.G(ctx).Infof("Using non-default snapshotter %s", driverName) + + } } else if driverName != "" { - log.G(ctx).Infof("Setting the storage driver from the $DOCKER_DRIVER environment variable (%s)", driverName) + log.G(ctx).Infof("Setting the storage driver from the $DOCKER_GRAPHDRIVER environment variable (%s)", driverName) } else { driverName = cfgStore.GraphDriver } - if d.UsesSnapshotter() { - if os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "" { - log.G(ctx).Warn("Enabling containerd snapshotter through the $TEST_INTEGRATION_USE_SNAPSHOTTER environment variable. This should only be used for testing.") - } - log.G(ctx).Info("Starting daemon with containerd snapshotter integration enabled") - - // FIXME(thaJeztah): implement automatic snapshotter-selection similar to graph-driver selection; see https://github.com/moby/moby/issues/44076 - if driverName == "" { - driverName = defaults.DefaultSnapshotter - } - - // Configure and validate the kernels security support. Note this is a Linux/FreeBSD - // operation only, so it is safe to pass *just* the runtime OS graphdriver. - if err := configureKernelSecuritySupport(&cfgStore.Config, driverName); err != nil { - return nil, err - } - d.imageService = ctrd.NewService(ctrd.ImageServiceConfig{ - Client: d.containerdClient, - Containers: d.containers, - Snapshotter: driverName, - RegistryHosts: d.RegistryHosts, - Registry: d.registryService, - EventsService: d.EventsService, - IDMapping: idMapping, - RefCountMounter: snapshotter.NewMounter(config.Root, driverName, idMapping), - }) - } else { + var migrationConfig migration.Config + tryGraphDriver, err := isGraphDriver(driverName) + if err != nil { + return nil, err + } + if tryGraphDriver { layerStore, err := layer.NewStoreFromOptions(layer.StoreOptions{ Root: cfgStore.Root, GraphDriver: driverName, @@ -1154,14 +1207,129 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S } } - // TODO: imageStore, distributionMetadataStore, and ReferenceStore are only - // used above to run migration. They could be initialized in ImageService - // if migration is called from daemon/images. layerStore might move as well. - d.imageService = images.NewImageService(imgSvcConfig) + // If no containers are present, check whether can migrate image service + if drv := layerStore.DriverName(); len(containers[drv]) == 0 && migrationThreshold >= 0 { + switch drv { + case "overlay2": + driverName = "overlayfs" + case "windowsfilter": + driverName = "windows" + case "vfs": + driverName = "native" + default: + migrationThreshold = -1 + log.G(ctx).Infof("Not migrating to containerd snapshotter, no migration defined for graph driver %q", drv) + } - log.G(ctx).Debugf("Max Concurrent Downloads: %d", imgSvcConfig.MaxConcurrentDownloads) - log.G(ctx).Debugf("Max Concurrent Uploads: %d", imgSvcConfig.MaxConcurrentUploads) - log.G(ctx).Debugf("Max Download Attempts: %d", imgSvcConfig.MaxDownloadAttempts) + var totalSize int64 + ic := imgSvcConfig.ImageStore.Len() + if migrationThreshold >= 0 && ic > 0 { + for _, img := range imgSvcConfig.ImageStore.Map() { + if layerID := img.RootFS.ChainID(); layerID != "" { + l, err := imgSvcConfig.LayerStore.Get(layerID) + if err != nil { + if errors.Is(err, layer.ErrLayerDoesNotExist) { + continue + } + return nil, err + } + + // Just look at layer size for considering maximum size + totalSize += l.Size() + layer.ReleaseAndLog(imgSvcConfig.LayerStore, l) + } + } + + } + + if totalSize <= migrationThreshold { + log.G(ctx).WithField("total", totalSize).Infof("Enabling containerd snapshotter because migration set with no containers and %d images in graph driver", ic) + migrationConfig = migration.Config{ + ImageCount: ic, + LayerStore: imgSvcConfig.LayerStore, + DockerImageStore: imgSvcConfig.ImageStore, + ReferenceStore: imgSvcConfig.ReferenceStore, + } + } else if migrationThreshold >= 0 { + log.G(ctx).WithField("total", totalSize).Warnf("Not migrating to containerd snapshotter because still have %d images in graph driver", ic) + d.imageService = images.NewImageService(ctx, imgSvcConfig) + } else { + d.imageService = images.NewImageService(ctx, imgSvcConfig) + } + } else { + log.G(ctx).Debugf("Not attempting migration with %d containers and %d image threshold", len(containers[drv]), migrationThreshold) + d.imageService = images.NewImageService(ctx, imgSvcConfig) + } + } + + if d.imageService == nil { + log.G(ctx).Info("Starting daemon with containerd snapshotter integration enabled") + + resp, err := d.containerdClient.IntrospectionService().Plugins(ctx, `type=="io.containerd.snapshotter.v1"`) + if err != nil { + return nil, fmt.Errorf("failed to get containerd plugins: %w", err) + } + if resp == nil || len(resp.Plugins) == 0 { + return nil, fmt.Errorf("failed to get containerd plugins response: %w", cerrdefs.ErrUnavailable) + } + availableDrivers := map[string]struct{}{} + for _, p := range resp.Plugins { + if p == nil || p.Type != "io.containerd.snapshotter.v1" { + continue + } + if p.InitErr == nil { + availableDrivers[p.ID] = struct{}{} + } else if (p.ID == driverName) || (driverName == "" && p.ID == defaults.DefaultSnapshotter) { + log.G(ctx).WithField("message", p.InitErr.Message).Warn("Preferred snapshotter not available in containerd") + } + } + + if driverName == "" { + if _, ok := availableDrivers[defaults.DefaultSnapshotter]; ok { + driverName = defaults.DefaultSnapshotter + } else if _, ok := availableDrivers["native"]; ok { + driverName = "native" + } else { + log.G(ctx).WithField("available", maps.Keys(availableDrivers)).Debug("Preferred snapshotter not available in containerd") + return nil, fmt.Errorf("snapshotter selection failed, no drivers available: %w", cerrdefs.ErrUnavailable) + } + } else if _, ok := availableDrivers[driverName]; !ok { + return nil, fmt.Errorf("configured driver %q not available: %w", driverName, cerrdefs.ErrUnavailable) + } + + // Configure and validate the kernels security support. Note this is a Linux/FreeBSD + // operation only, so it is safe to pass *just* the runtime OS graphdriver. + if err := configureKernelSecuritySupport(&cfgStore.Config, driverName); err != nil { + return nil, err + } + d.usesSnapshotter = true + d.imageService = ctrd.NewService(ctrd.ImageServiceConfig{ + Client: d.containerdClient, + Containers: d.containers, + Snapshotter: driverName, + RegistryHosts: d.RegistryHosts, + Registry: d.registryService, + EventsService: d.EventsService, + IDMapping: idMapping, + RefCountMounter: snapshotter.NewMounter(config.Root, driverName, idMapping), + }) + + if migrationConfig.ImageCount > 0 { + if d.imageService.CountImages(ctx) > 0 { + log.G(ctx).WithField("image_count", migrationConfig.ImageCount).Warnf("Images not migrated because images already exist in containerd %q", migrationConfig.LayerStore.DriverName()) + } else { + migrationConfig.Leases = d.containerdClient.LeasesService() + migrationConfig.Content = d.containerdClient.ContentStore() + migrationConfig.ImageStore = d.containerdClient.ImageService() + m := migration.NewLayerMigrator(migrationConfig) + err := m.MigrateTocontainerd(ctx, driverName, d.containerdClient.SnapshotService(driverName)) + if err != nil { + log.G(ctx).WithError(err).Errorf("Failed to migrate images to containerd, images in graph driver %q are no longer visible", migrationConfig.LayerStore.DriverName()) + } else { + log.G(ctx).WithField("image_count", migrationConfig.ImageCount).Infof("Successfully migrated images from %q to containerd", migrationConfig.LayerStore.DriverName()) + } + } + } } go d.execCommandGC() @@ -1171,9 +1339,22 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S return nil, err } - if err := d.restore(cfgStore); err != nil { + driverContainers, ok := containers[driverName] + // Log containers which are not loaded with current driver + if (!ok && len(containers) > 0) || len(containers) > 1 { + for driver, all := range containers { + if driver == driverName { + continue + } + for id := range all { + log.G(ctx).WithField("container", id).Debugf("not restoring container because it was created with another storage driver (%s)", driver) + } + } + } + if err := d.restore(ctx, cfgStore, driverContainers); err != nil { return nil, err } + // Wait for migration to complete close(d.startupDone) info, err := d.SystemInfo(ctx) diff --git a/daemon/graphdriver/driver.go b/daemon/graphdriver/driver.go index ba138b2778..039777547e 100644 --- a/daemon/graphdriver/driver.go +++ b/daemon/graphdriver/driver.go @@ -121,6 +121,12 @@ func Register(name string, initFunc InitFunc) error { return nil } +// IsRegistered checks to see if the drive with the given name is registered +func IsRegistered(name string) bool { + _, exists := drivers[name] + return exists +} + // getDriver initializes and returns the registered driver. func getDriver(name string, config Options) (Driver, error) { if initFunc, exists := drivers[name]; exists { diff --git a/daemon/graphdriver/vfs/driver.go b/daemon/graphdriver/vfs/driver.go index 48940a5ef3..f416620e3a 100644 --- a/daemon/graphdriver/vfs/driver.go +++ b/daemon/graphdriver/vfs/driver.go @@ -86,7 +86,16 @@ func (d *Driver) Status() [][2]string { // GetMetadata is used for implementing the graphdriver.ProtoDriver interface. VFS does not currently have any meta data. func (d *Driver) GetMetadata(id string) (map[string]string, error) { - return nil, nil + dir := d.dir(id) + if _, err := os.Stat(dir); err != nil { + return nil, err + } + + metadata := map[string]string{ + "SourceDir": dir, + } + + return metadata, nil } // Cleanup is used to implement graphdriver.ProtoDriver. There is no cleanup required for this driver. diff --git a/daemon/images/service.go b/daemon/images/service.go index f02d41ea42..a2235c8f64 100644 --- a/daemon/images/service.go +++ b/daemon/images/service.go @@ -49,7 +49,11 @@ type ImageServiceConfig struct { } // NewImageService returns a new ImageService from a configuration -func NewImageService(config ImageServiceConfig) *ImageService { +func NewImageService(ctx context.Context, config ImageServiceConfig) *ImageService { + log.G(ctx).Debugf("Max Concurrent Downloads: %d", config.MaxConcurrentDownloads) + log.G(ctx).Debugf("Max Concurrent Uploads: %d", config.MaxConcurrentUploads) + log.G(ctx).Debugf("Max Download Attempts: %d", config.MaxDownloadAttempts) + return &ImageService{ containers: config.ContainerStore, distributionMetadataStore: config.DistributionMetadataStore, diff --git a/daemon/reload_test.go b/daemon/reload_test.go index 9d970cd593..a2e98bd6a9 100644 --- a/daemon/reload_test.go +++ b/daemon/reload_test.go @@ -25,7 +25,7 @@ func muteLogs(t *testing.T) { func newDaemonForReloadT(t *testing.T, cfg *config.Config) *Daemon { t.Helper() daemon := &Daemon{ - imageService: images.NewImageService(images.ImageServiceConfig{}), + imageService: images.NewImageService(context.TODO(), images.ImageServiceConfig{}), } var err error daemon.registryService, err = registry.NewService(registry.ServiceOptions{}) @@ -63,7 +63,7 @@ func TestDaemonReloadLabels(t *testing.T) { func TestDaemonReloadMirrors(t *testing.T) { daemon := &Daemon{ - imageService: images.NewImageService(images.ImageServiceConfig{}), + imageService: images.NewImageService(context.TODO(), images.ImageServiceConfig{}), } muteLogs(t) @@ -162,7 +162,7 @@ func TestDaemonReloadMirrors(t *testing.T) { func TestDaemonReloadInsecureRegistries(t *testing.T) { daemon := &Daemon{ - imageService: images.NewImageService(images.ImageServiceConfig{}), + imageService: images.NewImageService(context.TODO(), images.ImageServiceConfig{}), } muteLogs(t) diff --git a/hack/dockerfile/etc/docker/daemon.json b/hack/dockerfile/etc/docker/daemon.json index 2319c13a67..ff9fe36aa6 100644 --- a/hack/dockerfile/etc/docker/daemon.json +++ b/hack/dockerfile/etc/docker/daemon.json @@ -3,8 +3,5 @@ "crun": { "path": "/usr/local/bin/crun" } - }, - "features": { - "containerd-snapshotter": false } } diff --git a/hack/make/.integration-daemon-start b/hack/make/.integration-daemon-start index 76d2efecdf..94b3e97f8c 100644 --- a/hack/make/.integration-daemon-start +++ b/hack/make/.integration-daemon-start @@ -40,7 +40,7 @@ fi # intentionally open a couple bogus file descriptors to help test that they get scrubbed in containers exec 41>&1 42>&2 -export DOCKER_GRAPHDRIVER=${DOCKER_GRAPHDRIVER:-vfs} +export DOCKER_GRAPHDRIVER=${DOCKER_GRAPHDRIVER:-native} export DOCKER_USERLANDPROXY=${DOCKER_USERLANDPROXY:-true} # example usage: DOCKER_STORAGE_OPTS="dm.basesize=20G,dm.loopdatasize=200G" diff --git a/hack/make/.integration-test-helpers b/hack/make/.integration-test-helpers index 3c7d7265b7..334d5a76bf 100644 --- a/hack/make/.integration-test-helpers +++ b/hack/make/.integration-test-helpers @@ -210,7 +210,7 @@ test_env() { PATH="$PATH" \ TEMP="$TEMP" \ TEST_CLIENT_BINARY="$TEST_CLIENT_BINARY" \ - TEST_INTEGRATION_USE_SNAPSHOTTER="$TEST_INTEGRATION_USE_SNAPSHOTTER" \ + TEST_INTEGRATION_USE_GRAPHDRIVER="$TEST_INTEGRATION_USE_GRAPHDRIVER" \ OTEL_EXPORTER_OTLP_ENDPOINT="$OTEL_EXPORTER_OTLP_ENDPOINT" \ OTEL_SERVICE_NAME="$OTEL_SERVICE_NAME" \ "$@" diff --git a/hack/make/run b/hack/make/run index a528d18c07..3702e8c9cf 100644 --- a/hack/make/run +++ b/hack/make/run @@ -10,7 +10,7 @@ fi DOCKER_COMMAND="$(command -v dockerd)" -DOCKER_GRAPHDRIVER=${DOCKER_GRAPHDRIVER:-vfs} +DOCKER_GRAPHDRIVER=${DOCKER_GRAPHDRIVER:-native} DOCKER_USERLANDPROXY=${DOCKER_USERLANDPROXY:-true} # example usage: DOCKER_STORAGE_OPTS="dm.basesize=20G,dm.loopdatasize=200G" diff --git a/hack/make/test-docker-py b/hack/make/test-docker-py index 764ccfc08e..4d3f98cbef 100644 --- a/hack/make/test-docker-py +++ b/hack/make/test-docker-py @@ -15,7 +15,7 @@ source hack/make/.integration-test-helpers : "${PY_TEST_OPTIONS:=--junitxml=${DEST}/junit-report.xml}" # build --squash is not supported with containerd integration. -if [ -n "$TEST_INTEGRATION_USE_SNAPSHOTTER" ]; then +if [ -z "$TEST_INTEGRATION_USE_GRAPHDRIVER" ]; then PY_TEST_OPTIONS="$PY_TEST_OPTIONS --deselect=tests/integration/api_build_test.py::BuildTest::test_build_squash" fi diff --git a/integration/daemon/migration_test.go b/integration/daemon/migration_test.go new file mode 100644 index 0000000000..b7850030b8 --- /dev/null +++ b/integration/daemon/migration_test.go @@ -0,0 +1,163 @@ +package daemon // import "github.com/docker/docker/integration/daemon" + +import ( + "bytes" + "io" + "runtime" + "testing" + + containertypes "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/image" + "github.com/moby/moby/client" + "github.com/moby/moby/v2/integration/internal/container" + "github.com/moby/moby/v2/testutil" + "github.com/moby/moby/v2/testutil/daemon" + "github.com/moby/moby/v2/testutil/fixtures/load" + "gotest.tools/v3/assert" + "gotest.tools/v3/skip" +) + +func TestMigrateOverlaySnapshotter(t *testing.T) { + testMigrateSnapshotter(t, "overlay2", "overlayfs") +} + +func TestMigrateNativeSnapshotter(t *testing.T) { + testMigrateSnapshotter(t, "vfs", "native") +} + +func testMigrateSnapshotter(t *testing.T, graphdriver, snapshotter string) { + skip.If(t, runtime.GOOS != "linux") + + t.Setenv("DOCKER_MIGRATE_SNAPSHOTTER_THRESHOLD", "200M") + t.Setenv("DOCKER_GRAPHDRIVER", "") + + ctx := testutil.StartSpan(baseContext, t) + + d := daemon.New(t) + defer d.Stop(t) + + d.Start(t, "--iptables=false", "--ip6tables=false", "-s", graphdriver) + info := d.Info(t) + id := info.ID + assert.Check(t, id != "") + assert.Equal(t, info.Containers, 0) + assert.Equal(t, info.Images, 0) + assert.Equal(t, info.Driver, graphdriver) + + load.FrozenImagesLinux(ctx, d.NewClientT(t), "busybox:latest") + + info = d.Info(t) + allImages := info.Images + assert.Check(t, allImages > 0) + + apiClient := d.NewClientT(t) + + containerID := container.Run(ctx, t, apiClient, func(c *container.TestContainerConfig) { + c.Name = "Migration-1-" + snapshotter + c.Config.Image = "busybox:latest" + c.Config.Cmd = []string{"top"} + }) + + d.Stop(t) + + // Start with migration feature but with a container which will prevent migration + d.Start(t, "--iptables=false", "--ip6tables=false", "-s", graphdriver, "--feature", "containerd-migration") + info = d.Info(t) + assert.Equal(t, info.ID, id) + assert.Equal(t, info.Driver, graphdriver) + assert.Equal(t, info.Containers, 1) + assert.Equal(t, info.Images, allImages) + container.Remove(ctx, t, apiClient, containerID, containertypes.RemoveOptions{ + Force: true, + }) + + d.Stop(t) + + d.Start(t, "--iptables=false", "--ip6tables=false", "-s", graphdriver, "--feature", "containerd-migration") + info = d.Info(t) + assert.Equal(t, info.ID, id) + assert.Equal(t, info.Containers, 0) + assert.Equal(t, info.Driver, snapshotter, "expected migrate to switch from %s to %s", graphdriver, snapshotter) + assert.Equal(t, info.Images, allImages) + + result := container.RunAttach(ctx, t, apiClient, func(c *container.TestContainerConfig) { + c.Name = "Migration-2-" + snapshotter + c.Config.Image = "busybox:latest" + c.Config.Cmd = []string{"echo", "hello"} + }) + assert.Equal(t, result.ExitCode, 0) + container.Remove(ctx, t, apiClient, result.ContainerID, containertypes.RemoveOptions{}) +} + +func TestMigrateSaveLoad(t *testing.T) { + skip.If(t, runtime.GOOS != "linux") + + t.Setenv("DOCKER_MIGRATE_SNAPSHOTTER_THRESHOLD", "200M") + t.Setenv("DOCKER_GRAPHDRIVER", "") + + var ( + ctx = testutil.StartSpan(baseContext, t) + d = daemon.New(t) + graphdriver = "overlay2" + snapshotter = "overlayfs" + ) + defer d.Stop(t) + + d.Start(t, "--iptables=false", "--ip6tables=false", "-s", graphdriver) + info := d.Info(t) + id := info.ID + assert.Check(t, id != "") + assert.Equal(t, info.Containers, 0) + assert.Equal(t, info.Images, 0) + assert.Equal(t, info.Driver, graphdriver) + + load.FrozenImagesLinux(ctx, d.NewClientT(t), "busybox:latest") + + info = d.Info(t) + allImages := info.Images + assert.Check(t, allImages > 0) + + d.Stop(t) + + d.Start(t, "--iptables=false", "--ip6tables=false", "-s", graphdriver, "--feature", "containerd-migration") + info = d.Info(t) + assert.Equal(t, info.ID, id) + assert.Equal(t, info.Containers, 0) + assert.Equal(t, info.Driver, snapshotter, "expected migrate to switch from %s to %s", graphdriver, snapshotter) + assert.Equal(t, info.Images, allImages) + + apiClient := d.NewClientT(t) + + // Save image to buffer + rdr, err := apiClient.ImageSave(ctx, []string{"busybox:latest"}) + assert.NilError(t, err) + buf := bytes.NewBuffer(nil) + io.Copy(buf, rdr) + rdr.Close() + + // Delete all images + list, err := apiClient.ImageList(ctx, image.ListOptions{}) + assert.NilError(t, err) + for _, i := range list { + _, err = apiClient.ImageRemove(ctx, i.ID, image.RemoveOptions{Force: true}) + assert.NilError(t, err) + } + + // Check zero images + info = d.Info(t) + assert.Equal(t, info.Images, 0) + + // Import + lr, err := apiClient.ImageLoad(ctx, bytes.NewReader(buf.Bytes()), client.ImageLoadWithQuiet(true)) + assert.NilError(t, err) + io.Copy(io.Discard, lr.Body) + lr.Body.Close() + + result := container.RunAttach(ctx, t, apiClient, func(c *container.TestContainerConfig) { + c.Name = "Migration-save-load-" + snapshotter + c.Config.Image = "busybox:latest" + c.Config.Cmd = []string{"echo", "hello"} + }) + assert.Equal(t, result.ExitCode, 0) + container.Remove(ctx, t, apiClient, result.ContainerID, containertypes.RemoveOptions{}) +} diff --git a/integration/volume/mount_test.go b/integration/volume/mount_test.go index 5e82f750bf..e4d6641f32 100644 --- a/integration/volume/mount_test.go +++ b/integration/volume/mount_test.go @@ -147,7 +147,7 @@ func TestRunMountImage(t *testing.T) { {name: "image_remove_force", cmd: []string{"cat", "/image/foo"}, expected: "barbar"}, } { t.Run(tc.name, func(t *testing.T) { - testImage := setupTestImage(t, ctx, apiClient, tc.name, testEnv.UsingSnapshotter()) + testImage := setupTestImage(t, ctx, apiClient, tc.name) if testImage != "" { defer apiClient.ImageRemove(ctx, testImage, image.RemoveOptions{Force: true}) } @@ -292,26 +292,19 @@ func setupTestVolume(t *testing.T, apiClient client.APIClient) string { return volumeName } -func setupTestImage(t *testing.T, ctx context.Context, apiClient client.APIClient, test string, snapshotter bool) string { +func setupTestImage(t *testing.T, ctx context.Context, apiClient client.APIClient, test string) string { imgName := "test-image" if test == "image_tag" { imgName += ":foo" } - var symlink string - if snapshotter { - symlink = "../../../../rootfs" - } else { - symlink = "../../../../../docker" - } - //nolint:dupword // ignore "Duplicate words (subdir) found (dupword)" dockerfile := ` FROM busybox as symlink RUN mkdir /hack \ && ln -s "../subdir" /hack/good \ - && ln -s "` + symlink + `" /hack/bad + && ln -s "../../../../../docker" /hack/bad #-- FROM scratch COPY foo / diff --git a/testutil/environment/environment.go b/testutil/environment/environment.go index d89b4cc3d4..2ab1f6433e 100644 --- a/testutil/environment/environment.go +++ b/testutil/environment/environment.go @@ -189,10 +189,9 @@ func (e *Execution) IsUserNamespaceInKernel() bool { } // UsingSnapshotter returns whether containerd snapshotters are used for the -// tests by checking if the "TEST_INTEGRATION_USE_SNAPSHOTTER" is set to a -// non-empty value. +// tests by checking if the "TEST_INTEGRATION_USE_GRAPHDRIVER" is empty func (e *Execution) UsingSnapshotter() bool { - return os.Getenv("TEST_INTEGRATION_USE_SNAPSHOTTER") != "" + return os.Getenv("TEST_INTEGRATION_USE_GRAPHDRIVER") == "" } // HasExistingImage checks whether there is an image with the given reference.