diff --git a/contrib/dockerd-rootless.sh b/contrib/dockerd-rootless.sh
index a5d13d02e3..3bc12e72b0 100755
--- a/contrib/dockerd-rootless.sh
+++ b/contrib/dockerd-rootless.sh
@@ -190,24 +190,6 @@ if [ -z "$_DOCKERD_ROOTLESS_CHILD" ]; then
 else
 	[ "$_DOCKERD_ROOTLESS_CHILD" = 1 ]
 
-	# The Container Device Interface (CDI) specs can be found by default
-	# under {/etc,/var/run}/cdi. More information at:
-	# https://github.com/cncf-tags/container-device-interface
-	#
-	# In order to use the Container Device Interface (CDI) integration,
-	# the CDI paths need to exist before the Docker daemon is started in
-	# order for it to read the CDI specification files. Otherwise, a
-	# Docker daemon restart will be required for the daemon to discover
-	# them.
-	#
-	# If another set of CDI paths (other than the default /etc/cdi and
-	# /var/run/cdi) are configured through the Docker configuration file
-	# (using "cdi-spec-dirs"), they need to be bind mounted in rootless
-	# mode; otherwise the Docker daemon won't have access to the CDI
-	# specification files.
-	mount_directory /etc/cdi
-	mount_directory /var/run/cdi
-
 	# remove the symlinks for the existing files in the parent namespace if any,
 	# so that we can create our own files in our mount namespace.
 	rm -f /run/docker /run/containerd /run/xtables.lock
diff --git a/daemon/cdi.go b/daemon/cdi.go
index e4074e2130..df31dcb4a1 100644
--- a/daemon/cdi.go
+++ b/daemon/cdi.go
@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path/filepath"
 
 	"github.com/containerd/log"
 	"github.com/moby/moby/api/types/system"
@@ -22,6 +23,14 @@ type cdiHandler struct {
 // The driver injects CDI devices into an incoming OCI spec and is called for DeviceRequests associated with CDI devices.
 // If the list of CDI spec directories is empty, the driver is not registered.
 func RegisterCDIDriver(cdiSpecDirs ...string) *cdi.Cache {
+	for i, dir := range cdiSpecDirs {
+		if _, err := os.Stat(dir); !errors.Is(err, os.ErrNotExist) {
+			cdiSpecDirs[i], err = filepath.EvalSymlinks(dir)
+			if err != nil {
+				log.L.WithField("dir", dir).WithError(err).Warn("Failed to evaluate symlinks for CDI spec directory")
+			}
+		}
+	}
 	driver, cache := newCDIDeviceDriver(cdiSpecDirs...)
 	registerDeviceDriver("cdi", driver)
 	return cache
diff --git a/daemon/command/daemon.go b/daemon/command/daemon.go
index cc913474be..688ddf085d 100644
--- a/daemon/command/daemon.go
+++ b/daemon/command/daemon.go
@@ -57,6 +57,7 @@ import (
 	"github.com/moby/moby/v2/pkg/homedir"
 	"github.com/moby/moby/v2/pkg/pidfile"
 	"github.com/moby/moby/v2/pkg/plugingetter"
+	"github.com/moby/sys/userns"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
@@ -666,7 +667,35 @@ func loadDaemonCliConfig(opts *daemonOptions) (*config.Config, error) {
 	if conf.CDISpecDirs == nil {
 		// If the CDISpecDirs is not set at this stage, we set it to the default.
 		conf.CDISpecDirs = append([]string(nil), cdi.DefaultSpecDirs...)
-	} else if len(conf.CDISpecDirs) == 1 && conf.CDISpecDirs[0] == "" {
+		if rootless.RunningWithRootlessKit() {
+			// In rootless mode, we add the user-specific CDI spec directory.
+			xch, err := homedir.GetConfigHome()
+			if err != nil {
+				return nil, err
+			}
+			xrd, err := homedir.GetRuntimeDir()
+			if err != nil {
+				return nil, err
+			}
+			conf.CDISpecDirs = append(conf.CDISpecDirs, filepath.Join(xch, "cdi"), filepath.Join(xrd, "cdi"))
+		}
+	}
+	// Filter out CDI spec directories that are not readable, and log appropriately
+	var cdiSpecDirs []string
+	for _, dir := range conf.CDISpecDirs {
+		// Non-existing directories are not filtered out here, as CDI spec directories are allowed to not exist.
+		if _, err := os.ReadDir(dir); err == nil || errors.Is(err, os.ErrNotExist) {
+			cdiSpecDirs = append(cdiSpecDirs, dir)
+		} else {
+			logLevel := log.ErrorLevel
+			if userns.RunningInUserNS() && errors.Is(err, os.ErrPermission) {
+				logLevel = log.DebugLevel
+			}
+			log.L.WithField("dir", dir).WithError(err).Log(logLevel, "CDI spec directory cannot be accessed, skipping")
+		}
+	}
+	conf.CDISpecDirs = cdiSpecDirs
+	if len(conf.CDISpecDirs) == 1 && conf.CDISpecDirs[0] == "" {
 		// If CDISpecDirs is set to an empty string, we clear it to ensure that CDI is disabled.
 		conf.CDISpecDirs = nil
 	}
diff --git a/integration/container/cdi_test.go b/integration/container/cdi_test.go
index a9318d7f49..07473d7614 100644
--- a/integration/container/cdi_test.go
+++ b/integration/container/cdi_test.go
@@ -2,6 +2,7 @@ package container
 
 import (
 	"bytes"
+	"errors"
 	"io"
 	"os"
 	"path/filepath"
@@ -203,3 +204,64 @@ func TestCDIInfoDiscoveredDevices(t *testing.T) {
 	assert.Check(t, is.Equal(len(info.DiscoveredDevices), 1), "Expected one discovered device")
 	assert.Check(t, is.DeepEqual(info.DiscoveredDevices, []system.DeviceInfo{expectedDevice}))
 }
+
+// TestEtcCDI verifies that the daemon picks up CDI specs from /etc/cdi, even in rootless mode.
+// Added for https://github.com/moby/moby/pull/51624
+func TestEtcCDI(t *testing.T) {
+	skip.If(t, testEnv.IsRemoteDaemon, "cannot run daemon when remote daemon")
+	skip.If(t, testEnv.DaemonInfo.OSType == "windows", "CDI not supported on Windows")
+
+	ctx := testutil.StartSpan(baseContext, t)
+
+	// Create a sample CDI spec file
+	specContent := `{
+		"cdiVersion": "0.5.0",
+		"kind": "test.mobyproject.org/device",
+		"devices": [
+			{
+				"name": "mygpu0",
+				"containerEdits": {
+					"deviceNodes": [
+						{"path": "/dev/null"}
+					]
+				}
+			}
+		]
+	}`
+
+	cdiDir := "/etc/cdi"
+	_, err := os.Stat(cdiDir)
+	cdiDirExisted := !errors.Is(err, os.ErrNotExist)
+	err = os.MkdirAll(cdiDir, 0o755)
+	assert.NilError(t, err, "Failed to create /etc/cdi directory")
+
+	t.Cleanup(func() {
+		if !cdiDirExisted {
+			rmErr := os.Remove(cdiDir)
+			assert.NilError(t, rmErr, "Failed to remove /etc/cdi directory")
+		}
+	})
+
+	specFilePath := filepath.Join(cdiDir, "moby-integration-test-device.json")
+	err = os.WriteFile(specFilePath, []byte(specContent), 0o644)
+	assert.NilError(t, err, "Failed to write sample CDI spec file")
+	t.Cleanup(func() {
+		rmErr := os.RemoveAll(specFilePath)
+		assert.NilError(t, rmErr, "Failed to remove sample CDI spec file")
+	})
+
+	d := daemon.New(t)
+	d.Start(t, "--feature", "cdi")
+	defer d.Stop(t)
+
+	c := d.NewClientT(t)
+	result, err := c.Info(ctx, client.InfoOptions{})
+	assert.NilError(t, err)
+	info := result.Info
+
+	expectedDevice := system.DeviceInfo{
+		Source: "cdi",
+		ID:     "test.mobyproject.org/device=mygpu0",
+	}
+	assert.Check(t, is.Contains(info.DiscoveredDevices, expectedDevice))
+}