Use ioctl to try to trigger kernel module loads

An ioctl() call to get the "interface index" for a kernel module triggers
the kernel to try to load the module, if the process is running with
CAP_SYS_MODULE. This tends to be more reliable than "modprobe" for
docker-in-docker.

If the ioctl() method fails, fall back to trying "modprobe".

Signed-off-by: Rob Murray <rob.murray@docker.com>

internal/modprobe/modprobe_linux.go

@@ -0,0 +1,110 @@
+// Package modprobe attempts to load kernel modules. It may have more success
+// than simply running "modprobe", particularly for docker-in-docker.
+package modprobe
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"github.com/containerd/log"
+	"golang.org/x/sys/unix"
+)
+
+// LoadModules attempts to load kernel modules, if necessary.
+//
+// isLoaded must be a function that checks whether the modules are loaded. It may
+// be called multiple times. isLoaded must return an error to indicate that the
+// modules still need to be loaded, otherwise nil.
+//
+// For each method of loading modules, LoadModules will attempt the load for each
+// of modNames, then it will call isLoaded to check the result - moving on to try
+// the next method if needed, and there is one.
+//
+// The returned error is the result of the final call to isLoaded.
+func LoadModules(ctx context.Context, isLoaded func() error, modNames ...string) error {
+	if isLoaded() == nil {
+		log.G(ctx).WithFields(log.Fields{
+			"modules": modNames,
+		}).Debug("Modules already loaded")
+		return nil
+	}
+
+	if err := tryLoad(ctx, isLoaded, modNames, ioctlLoader{}); err != nil {
+		return tryLoad(ctx, isLoaded, modNames, modprobeLoader{})
+	}
+	return nil
+}
+
+type loader interface {
+	name() string
+	load(modName string) error
+}
+
+func tryLoad(ctx context.Context, isLoaded func() error, modNames []string, loader loader) error {
+	var loadErrs []error
+	for _, modName := range modNames {
+		if err := loader.load(modName); err != nil {
+			loadErrs = append(loadErrs, err)
+		}
+	}
+
+	if checkResult := isLoaded(); checkResult != nil {
+		log.G(ctx).WithFields(log.Fields{
+			"loader":      loader.name(),
+			"modules":     modNames,
+			"loadErrors":  errors.Join(loadErrs...),
+			"checkResult": checkResult,
+		}).Debug("Modules not loaded")
+		return checkResult
+	}
+
+	log.G(ctx).WithFields(log.Fields{
+		"loader":     loader.name(),
+		"modules":    modNames,
+		"loadErrors": errors.Join(loadErrs...),
+	}).Debug("Modules loaded")
+	return nil
+}
+
+// ioctlLoader attempts to load the module using an ioctl() to get the interface index
+// of a module - it won't have one, but the kernel may load the module. This tends to
+// work in docker-in-docker, where the inner-docker may not have "modprobe" or access
+// to modules in the host's filesystem.
+type ioctlLoader struct{}
+
+func (il ioctlLoader) name() string { return "ioctl" }
+
+func (il ioctlLoader) load(modName string) error {
+	sd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0)
+	if err != nil {
+		return fmt.Errorf("creating socket for ioctl load of %s: %w", modName, err)
+	}
+	defer unix.Close(sd)
+
+	// This tends to work, if running with CAP_SYS_MODULE, because...
+	//   https://github.com/torvalds/linux/blob/6f7da290413ba713f0cdd9ff1a2a9bb129ef4f6c/net/core/dev_ioctl.c#L457
+	//   https://github.com/torvalds/linux/blob/6f7da290413ba713f0cdd9ff1a2a9bb129ef4f6c/net/core/dev_ioctl.c#L371-L372
+	ifreq, err := unix.NewIfreq(modName)
+	if err != nil {
+		return fmt.Errorf("creating ifreq for %s: %w", modName, err)
+	}
+	// An error is returned even if the module load is successful. So, ignore it.
+	_ = unix.IoctlIfreq(sd, unix.SIOCGIFINDEX, ifreq)
+	return nil
+}
+
+// modprobeLoader attempts to load a kernel module using modprobe.
+type modprobeLoader struct{}
+
+func (ml modprobeLoader) name() string { return "modprobe" }
+
+func (ml modprobeLoader) load(modName string) error {
+	out, err := exec.Command("modprobe", "-va", modName).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("modprobe %s failed with message: %q, error: %w", modName, strings.TrimSpace(string(out)), err)
+	}
+	return nil
+}

libnetwork/drivers/bridge/setup_bridgenetfiltering.go

@@ -7,10 +7,10 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"os/exec"
 	"syscall"
 
 	"github.com/containerd/log"
+	"github.com/docker/docker/internal/modprobe"
 )
 
 // setupIPv4BridgeNetFiltering checks whether IPv4 forwarding is enabled and, if
@@ -46,21 +46,17 @@ func setupIPv6BridgeNetFiltering(config *networkConfiguration, _ *bridgeInterfac
 }
 
 func loadBridgeNetFilterModule(fullPath string) error {
-	// br_netfilter implictly loads bridge module upon modprobe
-	modName := "br_netfilter"
-	if _, err := os.Stat(fullPath); err != nil {
-		if out, err := exec.Command("modprobe", "-va", modName).CombinedOutput(); err != nil {
-			log.G(context.TODO()).WithError(err).Errorf("Running modprobe %s failed with message: %s", modName, out)
-			return fmt.Errorf("cannot restrict inter-container communication: modprobe %s failed: %w", modName, err)
-		}
-	}
-	return nil
+	// br_netfilter implicitly loads bridge module upon modprobe
+	return modprobe.LoadModules(context.TODO(), func() error {
+		_, err := os.Stat(fullPath)
+		return err
+	}, "br_netfilter")
 }
 
 // Enable bridge net filtering if not already enabled. See GitHub issue #11404
 func enableBridgeNetFiltering(nfParam string) error {
 	if err := loadBridgeNetFilterModule(nfParam); err != nil {
-		return fmt.Errorf("loadBridgeNetFilterModule failed: %s", err)
+		return fmt.Errorf("cannot restrict inter-container communication or run without the userland proxy: %w", err)
 	}
 	enabled, err := getKernelBoolParam(nfParam)
 	if err != nil {

libnetwork/ns/init_linux.go

@@ -2,14 +2,12 @@ package ns
 
 import (
 	"context"
-	"fmt"
-	"os/exec"
-	"strings"
 	"sync"
 	"syscall"
 	"time"
 
 	"github.com/containerd/log"
+	"github.com/docker/docker/internal/modprobe"
 	"github.com/docker/docker/internal/nlwrap"
 	"github.com/vishvananda/netns"
 )
@@ -65,12 +63,8 @@ func getSupportedNlFamilies() []int {
 		fams = append(fams, syscall.NETLINK_XFRM)
 	}
 	// NETLINK_NETFILTER test
-	if err := loadNfConntrackModules(); err != nil {
-		if checkNfSocket() != nil {
-			log.G(context.TODO()).Warnf("Could not load necessary modules for Conntrack: %v", err)
-		} else {
-			fams = append(fams, syscall.NETLINK_NETFILTER)
-		}
+	if err := modprobe.LoadModules(context.TODO(), checkNfSocket, "nf_conntrack", "nf_conntrack_netlink"); err != nil {
+		log.G(context.TODO()).Warnf("Could not load necessary modules for Conntrack: %v", err)
 	} else {
 		fams = append(fams, syscall.NETLINK_NETFILTER)
 	}
@@ -88,16 +82,6 @@ func checkXfrmSocket() error {
 	return nil
 }
 
-func loadNfConntrackModules() error {
-	if out, err := exec.Command("modprobe", "-va", "nf_conntrack").CombinedOutput(); err != nil {
-		return fmt.Errorf("Running modprobe nf_conntrack failed with message: `%s`, error: %v", strings.TrimSpace(string(out)), err)
-	}
-	if out, err := exec.Command("modprobe", "-va", "nf_conntrack_netlink").CombinedOutput(); err != nil {
-		return fmt.Errorf("Running modprobe nf_conntrack_netlink failed with message: `%s`, error: %v", strings.TrimSpace(string(out)), err)
-	}
-	return nil
-}
-
 // API check on required nf_conntrack* modules (nf_conntrack, nf_conntrack_netlink)
 func checkNfSocket() error {
 	fd, err := syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_NETFILTER)