Allow separate IPv4/IPv6 gateway endpoints.

A dual-stack endpoint still has priority when selecting a gateway
Endpoint for a Sandbox. But, now there are IPv6-only networks, it
is possible to have a Sandbox with only IPv4-only and IPv6-only
endpoints. This change means they are both gateway endpoints.

Tell the network driver it mustn't proxy host-IPv6 to endpoint-IPv4
when there's an IPv6 gateway endpoint (which may belong to a different
net driver). Update that when networks are connected/disconnected.

Signed-off-by: Rob Murray <rob.murray@docker.com>

integration/networking/bridge_linux_test.go

@@ -2,9 +2,11 @@ package networking
 
 import (
 	"context"
+	"flag"
 	"fmt"
 	"os/exec"
 	"regexp"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -14,10 +16,12 @@ import (
 	"github.com/docker/docker/client"
 	"github.com/docker/docker/integration/internal/container"
 	"github.com/docker/docker/integration/internal/network"
+	n "github.com/docker/docker/integration/network"
 	"github.com/docker/docker/libnetwork/drivers/bridge"
 	"github.com/docker/docker/libnetwork/netlabel"
 	"github.com/docker/docker/testutil"
 	"github.com/docker/docker/testutil/daemon"
+	"github.com/docker/go-connections/nat"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"gotest.tools/v3/assert"
 	is "gotest.tools/v3/assert/cmp"
@@ -951,3 +955,148 @@ func TestContainerDisabledIPv6(t *testing.T) {
 	assert.Check(t, is.Equal(res.Stdout(), ""))
 	assert.Check(t, is.Contains(res.Stderr(), "bad address"))
 }
+
+type expProxyCfg struct {
+	proto      string
+	hostIP     string
+	hostPort   string
+	ctrName    string
+	ctrNetName string
+	ctrIPv4    bool
+	ctrPort    string
+}
+
+func TestGatewaySelection(t *testing.T) {
+	skip.If(t, testEnv.IsRootless, "proxies run in child namespace")
+
+	ctx := setupTest(t)
+	d := daemon.New(t, daemon.WithExperimental())
+	d.StartWithBusybox(ctx, t)
+	defer d.Stop(t)
+	c := d.NewClientT(t)
+	defer c.Close()
+
+	const netName4 = "net4"
+	network.CreateNoError(ctx, t, c, netName4)
+	defer network.RemoveNoError(ctx, t, c, netName4)
+
+	const netName6 = "net6"
+	netId6 := network.CreateNoError(ctx, t, c, netName6, network.WithIPv6(), network.WithIPv4(false))
+	defer network.RemoveNoError(ctx, t, c, netName6)
+
+	const netName46 = "net46"
+	netId46 := network.CreateNoError(ctx, t, c, netName46, network.WithIPv6())
+	defer network.RemoveNoError(ctx, t, c, netName46)
+
+	master := "dm-dummy0"
+	n.CreateMasterDummy(ctx, t, master)
+	defer n.DeleteInterface(ctx, t, master)
+	const netNameIpvlan6 = "ipvlan6"
+	netIdIpvlan6 := network.CreateNoError(ctx, t, c, netNameIpvlan6,
+		network.WithIPvlan("dm-dummy0", "l2"),
+		network.WithIPv4(false),
+		network.WithIPv6(),
+	)
+	defer network.RemoveNoError(ctx, t, c, netNameIpvlan6)
+
+	const ctrName = "ctr"
+	ctrId := container.Run(ctx, t, c,
+		container.WithName(ctrName),
+		container.WithNetworkMode(netName4),
+		container.WithExposedPorts("80"),
+		container.WithPortMap(nat.PortMap{"80": {{HostPort: "8080"}}}),
+		container.WithCmd("httpd", "-f"),
+	)
+	defer c.ContainerRemove(ctx, ctrId, containertypes.RemoveOptions{Force: true})
+
+	// The container only has an IPv4 endpoint, it should be the gateway, and
+	// the host-IPv6 should be proxied to container-IPv4.
+	checkProxies(ctx, t, c, d.Pid(), []expProxyCfg{
+		{"tcp", "0.0.0.0", "8080", ctrName, netName4, true, "80"},
+		{"tcp", "::", "8080", ctrName, netName4, true, "80"},
+	})
+
+	// Connect the IPv6-only network. The IPv6 endpoint should become the
+	// gateway for IPv6, the IPv4 endpoint should be reconfigured as the
+	// gateway for IPv4 only.
+	err := c.NetworkConnect(ctx, netId6, ctrId, nil)
+	assert.NilError(t, err)
+	checkProxies(ctx, t, c, d.Pid(), []expProxyCfg{
+		{"tcp", "0.0.0.0", "8080", ctrName, netName4, true, "80"},
+		{"tcp", "::", "8080", ctrName, netName6, false, "80"},
+	})
+
+	// Disconnect the IPv6-only network, the IPv4 should get back the mapping
+	// from host-IPv6.
+	err = c.NetworkDisconnect(ctx, netId6, ctrId, false)
+	assert.NilError(t, err)
+	checkProxies(ctx, t, c, d.Pid(), []expProxyCfg{
+		{"tcp", "0.0.0.0", "8080", ctrName, netName4, true, "80"},
+		{"tcp", "::", "8080", ctrName, netName4, true, "80"},
+	})
+
+	// Connect the dual-stack network, it should become the gateway for v6 and v4.
+	err = c.NetworkConnect(ctx, netId46, ctrId, nil)
+	assert.NilError(t, err)
+	checkProxies(ctx, t, c, d.Pid(), []expProxyCfg{
+		{"tcp", "0.0.0.0", "8080", ctrName, netName46, true, "80"},
+		{"tcp", "::", "8080", ctrName, netName46, false, "80"},
+	})
+
+	// Go back to the IPv4-only gateway, with proxy from host IPv6.
+	err = c.NetworkDisconnect(ctx, netId46, ctrId, false)
+	assert.NilError(t, err)
+	checkProxies(ctx, t, c, d.Pid(), []expProxyCfg{
+		{"tcp", "0.0.0.0", "8080", ctrName, netName4, true, "80"},
+		{"tcp", "::", "8080", ctrName, netName4, true, "80"},
+	})
+
+	// Connect the IPv6-only ipvlan network, its new Endpoint should become the IPv6
+	// gateway, so the IPv4-only bridge is expected to drop its mapping from host IPv6.
+	err = c.NetworkConnect(ctx, netIdIpvlan6, ctrId, nil)
+	assert.NilError(t, err)
+	checkProxies(ctx, t, c, d.Pid(), []expProxyCfg{
+		{"tcp", "0.0.0.0", "8080", ctrName, netName4, true, "80"},
+	})
+}
+
+func checkProxies(ctx context.Context, t *testing.T, c *client.Client, daemonPid int, exp []expProxyCfg) {
+	t.Helper()
+	makeExpStr := func(proto, hostIP, hostPort, ctrIP, ctrPort string) string {
+		return fmt.Sprintf("%s:%s/%s <-> %s:%s", hostIP, hostPort, proto, ctrIP, ctrPort)
+	}
+
+	wantProxies := make([]string, len(exp))
+	for _, e := range exp {
+		inspect := container.Inspect(ctx, t, c, e.ctrName)
+		nw := inspect.NetworkSettings.Networks[e.ctrNetName]
+		ctrIP := nw.GlobalIPv6Address
+		if e.ctrIPv4 {
+			ctrIP = nw.IPAddress
+		}
+		wantProxies = append(wantProxies, makeExpStr(e.proto, e.hostIP, e.hostPort, ctrIP, e.ctrPort))
+	}
+
+	gotProxies := make([]string, len(exp))
+	res, err := exec.Command("ps", "-f", "--ppid", strconv.Itoa(daemonPid)).CombinedOutput()
+	assert.NilError(t, err)
+	for _, line := range strings.Split(string(res), "\n") {
+		_, args, ok := strings.Cut(line, "docker-proxy")
+		if !ok {
+			continue
+		}
+		var proto, hostIP, hostPort, ctrIP, ctrPort string
+		var useListenFd bool
+		fs := flag.NewFlagSet("docker-proxy", flag.ContinueOnError)
+		fs.StringVar(&proto, "proto", "", "Protocol")
+		fs.StringVar(&hostIP, "host-ip", "", "Host IP")
+		fs.StringVar(&hostPort, "host-port", "", "Host Port")
+		fs.StringVar(&ctrIP, "container-ip", "", "Container IP")
+		fs.StringVar(&ctrPort, "container-port", "", "Container Port")
+		fs.BoolVar(&useListenFd, "use-listen-fd", false, "Use listen fd")
+		fs.Parse(strings.Split(strings.TrimSpace(args), " "))
+		gotProxies = append(gotProxies, makeExpStr(proto, hostIP, hostPort, ctrIP, ctrPort))
+	}
+
+	assert.DeepEqual(t, gotProxies, wantProxies)
+}

libnetwork/default_gateway.go

@@ -175,15 +175,31 @@ func (c *Controller) defaultGwNetwork() (*Network, error) {
 	return n, err
 }
 
-// Returns the endpoint which is providing external connectivity to the sandbox
-func (sb *Sandbox) getGatewayEndpoint() *Endpoint {
-	for _, ep := range sb.Endpoints() {
+// getGatewayEndpoint returns the endpoints providing external connectivity to
+// the sandbox. If the gateway is dual-stack, ep4 and ep6 will point at the same
+// endpoint. If there is no IPv4/IPv6 connectivity, nil pointers will be returned.
+func (sb *Sandbox) getGatewayEndpoint() (ep4, ep6 *Endpoint) {
+	return selectGatewayEndpoint(sb.Endpoints())
+}
+
+// selectGatewayEndpoint is like getGatewayEndpoint, but selects only from
+// endpoints.
+func selectGatewayEndpoint(endpoints []*Endpoint) (ep4, ep6 *Endpoint) {
+	for _, ep := range endpoints {
 		if ep.getNetwork().Type() == "null" || ep.getNetwork().Type() == "host" {
 			continue
 		}
-		if len(ep.Gateway()) != 0 {
-			return ep
+		gw4 := len(ep.Gateway()) != 0
+		gw6 := len(ep.GatewayIPv6()) != 0
+		if gw4 && gw6 {
+			return ep, ep
+		}
+		if gw4 && ep4 == nil {
+			ep4 = ep
+		}
+		if gw6 && ep6 == nil {
+			ep6 = ep
 		}
 	}
-	return nil
+	return ep4, ep6
 }

libnetwork/endpoint.go

@@ -557,8 +557,8 @@ func (ep *Endpoint) sbJoin(ctx context.Context, sb *Sandbox, options ...Endpoint
 		return err
 	}
 
-	// Current endpoint providing external connectivity for the sandbox
-	extEp := sb.getGatewayEndpoint()
+	// Current endpoint(s) providing external connectivity for the sandbox
+	gwepBefore4, gwepBefore6 := sb.getGatewayEndpoint()
 
 	sb.addEndpoint(ep)
 	defer func() {
@@ -606,36 +606,103 @@ func (ep *Endpoint) sbJoin(ctx context.Context, sb *Sandbox, options ...Endpoint
 		sb.resolver.SetForwardingPolicy(sb.hasExternalAccess())
 	}
 
-	currentExtEp := sb.getGatewayEndpoint()
-	moveExtConn := currentExtEp != extEp
-	if moveExtConn {
-		if extEp != nil {
-			log.G(ctx).Debugf("Revoking external connectivity on endpoint %s (%s)", extEp.Name(), extEp.ID())
-			extN, err := extEp.getNetworkFromStore()
-			if err != nil {
-				return fmt.Errorf("failed to get network from store for revoking external connectivity during join: %v", err)
+	gwepAfter4, gwepAfter6 := sb.getGatewayEndpoint()
+	if ep == gwepAfter4 || ep == gwepAfter6 {
+		// When the driver programs external connectivity for a Sandbox to use an
+		// IPv4-only Endpoint, it may choose to map ports from host IPv6 addresses (as
+		// well as host IPv4) to the Endpoint's IPv4 address. But, it must not do that if
+		// there is an IPv6-only Endpoint acting as the IPv6 gateway.
+		//
+		// So, for an IPv4-only Endpoint acting as a gateway, "noProxy6To4=true" must be
+		// set if the Sandbox has a different Endpoint acting as IPv6 gateway. And, if ep
+		// becoming the gateway changes noProxy6To4, the IPv4 gateway must be reset.
+		//
+		// This happens naturally in most cases. For example, if ep is dual-stack, sb had
+		// an IPv4-only gateway and no IPv6 gateway - connectivity will be revoked from
+		// the original IPv4-only gateway Endpoint and given to ep. So the old gateway
+		// won't be mapping from the host-IPv6 address.
+		//
+		// But, if ep is IPv6 only, an existing IPv4 only gateway may be proxying 6To4.
+		// So, its connectivity needs to be revoked and re-added with noProxy6To4 set.
+		// Similarly, when an IPv6-only gateway is disconnected from the Sandbox,
+		// gwepAfter6 will become nil and noProxy6To4 needs to be cleared in the
+		// configuration of an IPv4-only gateway.
+		//
+		// Note that revoking/restoring external connectivity will result in the bridge
+		// driver assigning new host ports for port mappings where the host port is not
+		// specified.
+		noProxy6To4Before := gwepBefore4 != nil && gwepBefore6 != nil && gwepBefore4 != gwepBefore6
+		noProxy6To4After := gwepAfter4 != nil && gwepAfter6 != nil && gwepAfter4 != gwepAfter6
+		restartGw4 := ep != gwepAfter4 && noProxy6To4Before != noProxy6To4After
+
+		// If ep is the new IPv4 gateway, remove the old IPv4 gateway.
+		if gwepBefore4 != nil && (ep == gwepAfter4 || restartGw4) {
+			role := "IPv4"
+			if gwepAfter6 == gwepAfter4 {
+				role = "dual-stack"
 			}
-			extD, err := extN.driver(true)
+			log.G(ctx).Debugf("Revoking %s external connectivity on endpoint %s (%s), NoProxy6To4:%v",
+				role, ep.Name(), ep.ID(), noProxy6To4Before)
+			undoFunc, err := gwepBefore4.revokeExternalConnectivity()
 			if err != nil {
-				return fmt.Errorf("failed to get driver for revoking external connectivity during join: %v", err)
+				return err
 			}
-			if err := extD.RevokeExternalConnectivity(extEp.network.ID(), extEp.ID()); err != nil {
-				return types.InternalErrorf(
-					"driver failed revoking external connectivity on endpoint %s (%s): %v",
-					extEp.Name(), extEp.ID(), err)
+			if restartGw4 {
+				// The IPv4 gateway hasn't changed, but its noProxy6To4 setting has. So,
+				// restore it as the gateway with that new setting.
+				log.G(ctx).Debugf("Programming IPv4 gateway endpoint %s (%s) NoProxy6To4:%v",
+					ep.Name(), ep.ID(), noProxy6To4After)
+				labelsAfter := sb.Labels()
+				labelsAfter[netlabel.NoProxy6To4] = noProxy6To4After
+				if err := undoFunc(ctx, labelsAfter); err != nil {
+					log.G(ctx).WithFields(log.Fields{
+						"endpointName": ep.Name(),
+						"endpointId":   ep.ID(),
+						"error":        err,
+					}).Warn("Failed to restore IPv4 connectivity")
+				}
+			} else {
+				defer func() {
+					if retErr != nil {
+						labelsBefore := sb.Labels()
+						labelsBefore[netlabel.NoProxy6To4] = noProxy6To4Before
+						if err := undoFunc(ctx, labelsBefore); err != nil {
+							log.G(ctx).WithFields(log.Fields{
+								"endpointName": ep.Name(),
+								"endpointId":   ep.ID(),
+								"role":         role,
+								"error":        err,
+							}).Warn("Failed to restore connectivity during rollback")
+						}
+					}
+				}()
+			}
+		}
+		// If ep is the new IPv6 gateway, there's an old IPv6 gateway to remove, and it
+		// wasn't also the IPv4 gateway (removed already) - remove the old gateway.
+		if ep == gwepAfter6 && gwepBefore6 != nil && gwepBefore6 != gwepBefore4 {
+			log.G(ctx).Debugf("Programming IPv6 gateway endpoint %s (%s)", ep.Name(), ep.ID())
+			undoFunc, err := gwepBefore6.revokeExternalConnectivity()
+			if err != nil {
+				return err
 			}
 			defer func() {
 				if retErr != nil {
-					if e := extD.ProgramExternalConnectivity(context.WithoutCancel(ctx), extEp.network.ID(), extEp.ID(), sb.Labels()); e != nil {
-						log.G(ctx).Warnf("Failed to roll-back external connectivity on endpoint %s (%s): %v",
-							extEp.Name(), extEp.ID(), e)
+					if err := undoFunc(ctx, sb.Labels()); err != nil {
+						log.G(ctx).WithFields(log.Fields{
+							"endpointName": ep.Name(),
+							"endpointId":   ep.ID(),
+							"error":        err,
+						}).Warn("Failed to restore IPv6 connectivity during rollback")
 					}
 				}
 			}()
 		}
 		if !n.internal {
 			log.G(ctx).Debugf("Programming external connectivity on endpoint %s (%s)", ep.Name(), ep.ID())
-			if err = d.ProgramExternalConnectivity(ctx, n.ID(), ep.ID(), sb.Labels()); err != nil {
+			labels := sb.Labels()
+			labels[netlabel.NoProxy6To4] = noProxy6To4After
+			if err := d.ProgramExternalConnectivity(ctx, n.ID(), ep.ID(), labels); err != nil {
 				return errdefs.System(fmt.Errorf(
 					"driver failed programming external connectivity on endpoint %s (%s): %v",
 					ep.Name(), ep.ID(), err))
@@ -653,6 +720,42 @@ func (ep *Endpoint) sbJoin(ctx context.Context, sb *Sandbox, options ...Endpoint
 	return nil
 }
 
+func (ep *Endpoint) programExternalConnectivity(ctx context.Context, labels map[string]interface{}) error {
+	log.G(ctx).Debugf("Programming external connectivity on endpoint %s (%s)", ep.Name(), ep.ID())
+	extN, err := ep.getNetworkFromStore()
+	if err != nil {
+		return types.InternalErrorf("failed to get network from store for programming external connectivity: %v", err)
+	}
+	extD, err := extN.driver(true)
+	if err != nil {
+		return types.InternalErrorf("failed to get driver for programming external connectivity: %v", err)
+	}
+	if err := extD.ProgramExternalConnectivity(context.WithoutCancel(ctx), ep.network.ID(), ep.ID(), labels); err != nil {
+		return types.InternalErrorf("driver failed programming external connectivity on endpoint %s (%s): %v",
+			ep.Name(), ep.ID(), err)
+	}
+	return nil
+}
+
+func (ep *Endpoint) revokeExternalConnectivity() (func(context.Context, map[string]interface{}) error, error) {
+	extN, err := ep.getNetworkFromStore()
+	if err != nil {
+		return nil, types.InternalErrorf("failed to get network from store for revoking external connectivity: %v", err)
+	}
+	extD, err := extN.driver(true)
+	if err != nil {
+		return nil, types.InternalErrorf("failed to get driver for revoking external connectivity: %v", err)
+	}
+	if err = extD.RevokeExternalConnectivity(ep.network.ID(), ep.ID()); err != nil {
+		return nil, types.InternalErrorf(
+			"driver failed revoking external connectivity on endpoint %s (%s): %v",
+			ep.Name(), ep.ID(), err)
+	}
+	return func(ctx context.Context, labels map[string]interface{}) error {
+		return extD.ProgramExternalConnectivity(context.WithoutCancel(ctx), ep.network.ID(), ep.ID(), labels)
+	}, nil
+}
+
 func (ep *Endpoint) rename(name string) error {
 	ep.mu.Lock()
 	ep.name = name
@@ -753,12 +856,13 @@ func (ep *Endpoint) sbLeave(ctx context.Context, sb *Sandbox, force bool) error
 	ep.network = n
 	ep.mu.Unlock()
 
-	// Current endpoint providing external connectivity to the sandbox
-	extEp := sb.getGatewayEndpoint()
-	moveExtConn := extEp != nil && (extEp.ID() == ep.ID())
+	// Current endpoint(s) providing external connectivity to the sandbox
+	gwepBefore4, gwepBefore6 := sb.getGatewayEndpoint()
+	moveExtConn4 := gwepBefore4 != nil && gwepBefore4.ID() == ep.ID()
+	moveExtConn6 := gwepBefore6 != nil && gwepBefore6.ID() == ep.ID()
 
 	if d != nil {
-		if moveExtConn {
+		if moveExtConn4 || moveExtConn6 {
 			log.G(ctx).Debugf("Revoking external connectivity on endpoint %s (%s)", ep.Name(), ep.ID())
 			if err := d.RevokeExternalConnectivity(n.id, ep.id); err != nil {
 				log.G(ctx).Warnf("driver failed revoking external connectivity on endpoint %s (%s): %v",
@@ -808,21 +912,48 @@ func (ep *Endpoint) sbLeave(ctx context.Context, sb *Sandbox, force bool) error
 		sb.resolver.SetForwardingPolicy(sb.hasExternalAccess())
 	}
 
-	// New endpoint providing external connectivity for the sandbox
-	extEp = sb.getGatewayEndpoint()
-	if moveExtConn && extEp != nil {
-		log.G(ctx).Debugf("Programming external connectivity on endpoint %s (%s)", extEp.Name(), extEp.ID())
-		extN, err := extEp.getNetworkFromStore()
-		if err != nil {
-			return fmt.Errorf("failed to get network from store for programming external connectivity during leave: %v", err)
+	// New endpoint(s) providing external connectivity for the sandbox
+	if moveExtConn4 || moveExtConn6 {
+		gwepAfter4, gwepAfter6 := sb.getGatewayEndpoint()
+		if gwepAfter4 != nil {
+			// If the IPv4 gateway hasn't changed, and there was no IPv6 gateway before but
+			// there is now, the driver for the IPv4 gateway must not proxy host-IPv6 to
+			// container-IPv4 (6To4). Conversely, if there was an IPv6 gateway before but
+			// there isn't one now, the driver must now be told it can proxy 6To4.
+			//
+			// Note that revoking/restoring external connectivity will result in the bridge
+			// driver assigning new host ports for port mappings where the host port is not
+			// specified.
+			restartGw4 := gwepBefore4 == gwepAfter4 && ((gwepBefore6 == nil) != (gwepAfter6 == nil))
+			noProxy6To4 := gwepAfter6 != nil && gwepAfter6 != gwepAfter4
+			labels := sb.Labels()
+			labels[netlabel.NoProxy6To4] = noProxy6To4
+			if restartGw4 {
+				log.G(ctx).Debugf("Resetting IPv4 endpoint %s (%s) NoProxy6To4:%v",
+					ep.Name(), ep.ID(), noProxy6To4)
+				if undoFunc, err := gwepBefore4.revokeExternalConnectivity(); err != nil {
+					log.G(ctx).WithError(err).Error("Failed to restart IPv4 gateway")
+				} else if err := undoFunc(ctx, labels); err != nil {
+					log.G(ctx).WithError(err).Error("Failed to restore IPv4 gateway")
+				}
+			} else if moveExtConn4 {
+				log.G(ctx).Debugf("Programming IPv6 gateway endpoint %s (%s)", ep.Name(), ep.ID())
+				if err := gwepAfter4.programExternalConnectivity(ctx, labels); err != nil {
+					role := "IPv4"
+					if gwepAfter6 == gwepAfter4 {
+						role = "dual-stack"
+					}
+					log.G(ctx).WithFields(log.Fields{
+						"role":  role,
+						"error": err,
+					}).Error("Failed to set gateway")
+				}
+			}
 		}
-		extD, err := extN.driver(true)
-		if err != nil {
-			return fmt.Errorf("failed to get driver for programming external connectivity during leave: %v", err)
-		}
-		if err := extD.ProgramExternalConnectivity(context.WithoutCancel(ctx), extEp.network.ID(), extEp.ID(), sb.Labels()); err != nil {
-			log.G(ctx).Warnf("driver failed programming external connectivity on endpoint %s: (%s) %v",
-				extEp.Name(), extEp.ID(), err)
+		if gwepAfter6 != nil && moveExtConn6 && gwepAfter6 != gwepAfter4 {
+			if err := gwepAfter6.programExternalConnectivity(ctx, sb.Labels()); err != nil {
+				log.G(ctx).WithError(err).Error("Failed to set IPv6 gateway")
+			}
 		}
 	}
 

libnetwork/sandbox.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"slices"
 	"sort"
 	"strings"
 	"sync"
@@ -584,40 +585,21 @@ func (sb *Sandbox) clearNetworkResources(origEp *Endpoint) error {
 		return nil
 	}
 
-	var (
-		gwepBefore, gwepAfter *Endpoint
-		index                 = -1
-	)
-	for i, e := range sb.endpoints {
-		if e == ep {
-			index = i
-		}
-		if len(e.Gateway()) > 0 && gwepBefore == nil {
-			gwepBefore = e
-		}
-		if index != -1 && gwepBefore != nil {
-			break
-		}
-	}
-
-	if index == -1 {
+	if !slices.Contains(sb.endpoints, ep) {
 		log.G(context.TODO()).Warnf("Endpoint %s has already been deleted", ep.Name())
 		sb.mu.Unlock()
 		return nil
 	}
 
+	gwepBefore4, gwepBefore6 := selectGatewayEndpoint(sb.endpoints)
 	sb.removeEndpointRaw(ep)
-	for _, e := range sb.endpoints {
-		if len(e.Gateway()) > 0 {
-			gwepAfter = e
-			break
-		}
-	}
+	gwepAfter4, gwepAfter6 := selectGatewayEndpoint(sb.endpoints)
 	delete(sb.epPriority, ep.ID())
+
 	sb.mu.Unlock()
 
-	if gwepAfter != nil && gwepBefore != gwepAfter {
-		if err := sb.updateGateway(gwepAfter); err != nil {
+	if (gwepAfter4 != nil && gwepBefore4 != gwepAfter4) || (gwepAfter6 != nil && gwepBefore6 != gwepAfter6) {
+		if err := sb.updateGateway(gwepAfter4, gwepAfter6); err != nil {
 			return err
 		}
 	}

libnetwork/sandbox_linux.go

@@ -74,9 +74,16 @@ func (sb *Sandbox) Statistics() (map[string]*types.InterfaceStatistics, error) {
 	return m, nil
 }
 
-func (sb *Sandbox) updateGateway(ep *Endpoint) error {
+func (sb *Sandbox) updateGateway(ep4, ep6 *Endpoint) error {
+	var populated4, populated6 bool
 	sb.mu.Lock()
 	osSbox := sb.osSbox
+	if ep4 != nil {
+		_, populated4 = sb.populatedEndpoints[ep4.ID()]
+	}
+	if ep6 != nil {
+		_, populated6 = sb.populatedEndpoints[ep6.ID()]
+	}
 	sb.mu.Unlock()
 	if osSbox == nil {
 		return nil
@@ -84,20 +91,24 @@ func (sb *Sandbox) updateGateway(ep *Endpoint) error {
 	osSbox.UnsetGateway()     //nolint:errcheck
 	osSbox.UnsetGatewayIPv6() //nolint:errcheck
 
-	if ep == nil {
-		return nil
+	if populated4 {
+		ep4.mu.Lock()
+		joinInfo := ep4.joinInfo
+		ep4.mu.Unlock()
+
+		if err := osSbox.SetGateway(joinInfo.gw); err != nil {
+			return fmt.Errorf("failed to set gateway while updating gateway: %v", err)
+		}
 	}
 
-	ep.mu.Lock()
-	joinInfo := ep.joinInfo
-	ep.mu.Unlock()
+	if populated6 {
+		ep6.mu.Lock()
+		joinInfo := ep6.joinInfo
+		ep6.mu.Unlock()
 
-	if err := osSbox.SetGateway(joinInfo.gw); err != nil {
-		return fmt.Errorf("failed to set gateway while updating gateway: %v", err)
-	}
-
-	if err := osSbox.SetGatewayIPv6(joinInfo.gw6); err != nil {
-		return fmt.Errorf("failed to set IPv6 gateway while updating gateway: %v", err)
+		if err := osSbox.SetGatewayIPv6(joinInfo.gw6); err != nil {
+			return fmt.Errorf("failed to set IPv6 gateway while updating gateway: %v", err)
+		}
 	}
 
 	return nil
@@ -254,13 +265,19 @@ func (sb *Sandbox) restoreOslSandbox() error {
 		}
 	}
 
-	gwep := sb.getGatewayEndpoint()
-	if gwep == nil {
-		return nil
+	gwep4, gwep6 := sb.getGatewayEndpoint()
+	if gwep4 != nil {
+		if err := sb.osSbox.Restore(interfaces, routes, gwep4.joinInfo.gw, gwep4.joinInfo.gw6); err != nil {
+			return err
+		}
+	}
+	if gwep6 != nil {
+		if err := sb.osSbox.Restore(interfaces, routes, gwep6.joinInfo.gw, gwep6.joinInfo.gw6); err != nil {
+			return err
+		}
 	}
 
-	// restore osl sandbox
-	return sb.osSbox.Restore(interfaces, routes, gwep.joinInfo.gw, gwep.joinInfo.gw6)
+	return nil
 }
 
 func (sb *Sandbox) populateNetworkResources(ctx context.Context, ep *Endpoint) error {
@@ -329,18 +346,18 @@ func (sb *Sandbox) populateNetworkResources(ctx context.Context, ep *Endpoint) e
 		}
 	}
 
-	if ep == sb.getGatewayEndpoint() {
-		if err := sb.updateGateway(ep); err != nil {
-			return err
-		}
-	}
-
 	// Make sure to add the endpoint to the populated endpoint set
-	// before populating loadbalancers.
+	// before updating gateways or populating loadbalancers.
 	sb.mu.Lock()
 	sb.populatedEndpoints[ep.ID()] = struct{}{}
 	sb.mu.Unlock()
 
+	if gw4, gw6 := sb.getGatewayEndpoint(); ep == gw4 || ep == gw6 {
+		if err := sb.updateGateway(gw4, gw6); err != nil {
+			return err
+		}
+	}
+
 	// Populate load balancer only after updating all the other
 	// information including gateway and other routes so that
 	// loadbalancers are populated all the network state is in

libnetwork/sandbox_windows.go

@@ -8,7 +8,7 @@ import (
 
 func releaseOSSboxResources(*osl.Namespace, *Endpoint) {}
 
-func (sb *Sandbox) updateGateway(*Endpoint) error {
+func (sb *Sandbox) updateGateway(_, _ *Endpoint) error {
 	// not implemented on Windows (Sandbox.osSbox is always nil)
 	return nil
 }

libnetwork/service_linux.go

@@ -122,7 +122,7 @@ func (n *Network) addLBBackend(ip net.IP, lb *loadBalancer) {
 
 		if sb.ingress {
 			var gwIP net.IP
-			if ep := sb.getGatewayEndpoint(); ep != nil {
+			if ep, _ := sb.getGatewayEndpoint(); ep != nil {
 				gwIP = ep.Iface().Address().IP
 			}
 			if err := programIngress(gwIP, lb.service.ingressPorts, false); err != nil {
@@ -223,7 +223,7 @@ func (n *Network) rmLBBackend(ip net.IP, lb *loadBalancer, rmService bool, fullR
 
 		if sb.ingress {
 			var gwIP net.IP
-			if gwEP := sb.getGatewayEndpoint(); gwEP != nil {
+			if gwEP, _ := sb.getGatewayEndpoint(); gwEP != nil {
 				gwIP = gwEP.Iface().Address().IP
 			}
 			if err := programIngress(gwIP, lb.service.ingressPorts, true); err != nil {