diff --git a/daemon/command/daemon.go b/daemon/command/daemon.go
index 7e0430ab25..67a11ad9f3 100644
--- a/daemon/command/daemon.go
+++ b/daemon/command/daemon.go
@@ -656,6 +656,18 @@ func loadDaemonCliConfig(opts *daemonOptions) (*config.Config, error) {
 	if conf.CDISpecDirs == nil {
 		// If the CDISpecDirs is not set at this stage, we set it to the default.
 		conf.CDISpecDirs = append([]string(nil), cdi.DefaultSpecDirs...)
+		if rootless.RunningWithRootlessKit() {
+			// In rootless mode, we add the user-specific CDI spec directory.
+			xch, err := homedir.GetConfigHome()
+			if err != nil {
+				return nil, err
+			}
+			xrd, err := homedir.GetRuntimeDir()
+			if err != nil {
+				return nil, err
+			}
+			conf.CDISpecDirs = append(conf.CDISpecDirs, filepath.Join(xch, "cdi"), filepath.Join(xrd, "cdi"))
+		}
 	} else if len(conf.CDISpecDirs) == 1 && conf.CDISpecDirs[0] == "" {
 		// If CDISpecDirs is set to an empty string, we clear it to ensure that CDI is disabled.
 		conf.CDISpecDirs = nil