libnetwork/d/overlay: filter local peers explicitly

The overlay driver's checkEncryption function configures the IPSec
parameters for the VXLAN tunnels to peer nodes. When called with
isLocal=true, it configures encryption for all peer nodes with at least
one peerDB entry. Since the local peers are also included in the peerDB,
it needs to filter those entries out. It does so by filtering out any
peer entries whose VTEP address is equal to the current local advertise
address. Trouble is, the local advertise address is not necessarily
constant. The driver tries to handle this case by calling
peerDBUpdateSelf() when the advertise address changes. This function
iterates through the peerDB and tries to update the VTEP address for all
local peer entries, but it does not actually do anything: it mutates a
temporary copy of the entry which is not persisted back into the peerDB.
(It used to be functional, but was broken when the peerDB was extended
to use SetMatrix.) So there may be cases where local peer entries are
not filtered out properly, resulting in spurious encryption parameters
being programmed into the kernel.

Filter out local peers when walking the peerDB by filtering on whether
the entry has the isLocal flag set. Remove the no-op code which attempts
to update local entries in the peerDB. No other code takes any interest
in the VTEP value for isLocal peer entries.

Signed-off-by: Cory Snider <csnider@mirantis.com>

libnetwork/drivers/overlay/encryption.go

@@ -132,7 +132,7 @@ func (d *driver) checkEncryption(nid string, rIP netip.Addr, isLocal, add bool)
 	switch {
 	case isLocal:
 		if err := d.peerDbNetworkWalk(nid, func(_ netip.Addr, _ net.HardwareAddr, pEntry *peerEntry) bool {
-			if aIP != pEntry.vtep {
+			if !pEntry.isLocal {
 				nodes[pEntry.vtep] = struct{}{}
 			}
 			return false

libnetwork/drivers/overlay/overlay.go

@@ -30,14 +30,13 @@ var _ discoverapi.Discover = (*driver)(nil)
 type driver struct {
 	bindAddress, advertiseAddress netip.Addr
 
-	config        map[string]interface{}
-	peerDb        peerNetworkMap
-	secMap        *encrMap
-	networks      networkTable
-	initOS        sync.Once
-	localJoinOnce sync.Once
-	keys          []*key
-	peerOpMu      sync.Mutex
+	config   map[string]interface{}
+	peerDb   peerNetworkMap
+	secMap   *encrMap
+	networks networkTable
+	initOS   sync.Once
+	keys     []*key
+	peerOpMu sync.Mutex
 	sync.Mutex
 }
 
@@ -95,12 +94,6 @@ func (d *driver) nodeJoin(data discoverapi.NodeDiscoveryData) error {
 		d.advertiseAddress = advAddr
 		d.bindAddress = bindAddr
 		d.Unlock()
-
-		// If containers are already running on this network update the
-		// advertise address in the peerDB
-		d.localJoinOnce.Do(func() {
-			d.peerDBUpdateSelf()
-		})
 	}
 	return nil
 }

libnetwork/drivers/overlay/peerdb.go

@@ -37,22 +37,6 @@ type peerNetworkMap struct {
 	sync.Mutex
 }
 
-func (d *driver) peerDbWalk(f func(string, netip.Addr, net.HardwareAddr, *peerEntry) bool) error {
-	d.peerDb.Lock()
-	nids := []string{}
-	for nid := range d.peerDb.mp {
-		nids = append(nids, nid)
-	}
-	d.peerDb.Unlock()
-
-	for _, nid := range nids {
-		d.peerDbNetworkWalk(nid, func(peerIP netip.Addr, peerMac net.HardwareAddr, pEntry *peerEntry) bool {
-			return f(nid, peerIP, peerMac, pEntry)
-		})
-	}
-	return nil
-}
-
 func (d *driver) peerDbNetworkWalk(nid string, f func(netip.Addr, net.HardwareAddr, *peerEntry) bool) error {
 	d.peerDb.Lock()
 	pMap, ok := d.peerDb.mp[nid]
@@ -359,12 +343,3 @@ func (d *driver) peerFlushOp(nid string) error {
 	delete(d.peerDb.mp, nid)
 	return nil
 }
-
-func (d *driver) peerDBUpdateSelf() {
-	d.peerDbWalk(func(nid string, _ netip.Addr, _ net.HardwareAddr, pEntry *peerEntry) bool {
-		if pEntry.isLocal {
-			pEntry.vtep = d.advertiseAddress
-		}
-		return false
-	})
-}