moby/daemon/libnetwork/drivers/overlay/peerdb.go

//go:build linux

package overlay

import (
	"context"
	"errors"
	"fmt"
	"net/netip"
	"syscall"

	"github.com/containerd/log"
	"github.com/moby/moby/v2/daemon/libnetwork/internal/hashable"
	"github.com/moby/moby/v2/daemon/libnetwork/internal/setmatrix"
	"github.com/moby/moby/v2/daemon/libnetwork/osl"
)

type peerEntry struct {
	eid  string
	mac  hashable.MACAddr
	vtep netip.Addr
}

func (p *peerEntry) isLocal() bool {
	return !p.vtep.IsValid()
}

type peerMap struct {
	mp setmatrix.SetMatrix[netip.Prefix, peerEntry]
}

func (pm *peerMap) Walk(f func(netip.Prefix, peerEntry)) {
	for _, peerAddr := range pm.mp.Keys() {
		entry, ok := pm.Get(peerAddr)
		if ok {
			f(peerAddr, entry)
		}
	}
}

func (pm *peerMap) Get(peerIP netip.Prefix) (peerEntry, bool) {
	c, _ := pm.mp.Get(peerIP)
	if len(c) == 0 {
		return peerEntry{}, false
	}
	return c[0], true
}

func (pm *peerMap) Add(eid string, peerIP netip.Prefix, peerMac hashable.MACAddr, vtep netip.Addr) (bool, int) {
	pEntry := peerEntry{
		eid:  eid,
		mac:  peerMac,
		vtep: vtep,
	}
	b, i := pm.mp.Insert(peerIP, pEntry)
	if i != 1 {
		// Transient case, there is more than one endpoint that is using the same IP
		s, _ := pm.mp.String(peerIP)
		log.G(context.TODO()).Warnf("peerDbAdd transient condition - Key:%s cardinality:%d db state:%s", peerIP, i, s)
	}
	return b, i
}

func (pm *peerMap) Delete(eid string, peerIP netip.Prefix, peerMac hashable.MACAddr, vtep netip.Addr) (bool, int) {
	pEntry := peerEntry{
		eid:  eid,
		mac:  peerMac,
		vtep: vtep,
	}

	b, i := pm.mp.Remove(peerIP, pEntry)
	if i != 0 {
		// Transient case, there is more than one endpoint that is using the same IP
		s, _ := pm.mp.String(peerIP)
		log.G(context.TODO()).Warnf("peerDbDelete transient condition - Key:%s cardinality:%d db state:%s", peerIP, i, s)
	}
	return b, i
}

// The overlay uses a lazy initialization approach, this means that when a network is created
// and the driver registered the overlay does not allocate resources till the moment that a
// sandbox is actually created.
// At the moment of this call, that happens when a sandbox is initialized, is possible that
// networkDB has already delivered some events of peers already available on remote nodes,
// these peers are saved into the peerDB and this function is used to properly configure
// the network sandbox with all those peers that got previously notified.
//
// The caller is responsible for ensuring that peerAdd and peerDelete are not
// called concurrently with this function to guarantee consistency.
func (n *network) initSandboxPeerDB() error {
	var errs []error
	n.peerdb.Walk(func(peerIP netip.Prefix, pEntry peerEntry) {
		if !pEntry.isLocal() {
			if err := n.addNeighbor(peerIP, pEntry.mac, pEntry.vtep); err != nil {
				errs = append(errs, fmt.Errorf("failed to add neighbor entries for %s: %w", peerIP, err))
			}
		}
	})
	return errors.Join(errs...)
}

// peerAdd adds a new entry to the peer database.
//
// Local peers are signified by an invalid vtep (i.e. netip.Addr{}).
func (n *network) peerAdd(eid string, peerIP netip.Prefix, peerMac hashable.MACAddr, vtep netip.Addr) error {
	if eid == "" {
		return errors.New("invalid endpoint id")
	}

	inserted, dbEntries := n.peerdb.Add(eid, peerIP, peerMac, vtep)
	if !inserted {
		log.G(context.TODO()).Warnf("Entry already present in db: nid:%s eid:%s peerIP:%v peerMac:%v vtep:%v",
			n.id, eid, peerIP, peerMac, vtep)
	}
	if vtep.IsValid() {
		err := n.addNeighbor(peerIP, peerMac, vtep)
		if err != nil {
			if dbEntries > 1 && errors.As(err, &osl.NeighborSearchError{}) {
				// Conflicting neighbor entries are already programmed into the kernel and we are in the transient case.
				// Upon deletion if the active configuration is deleted the next one from the database will be restored.
				return nil
			}
			return fmt.Errorf("peer add operation failed: %w", err)
		}
	}
	return nil
}

// addNeighbor programs the kernel so the given peer is reachable through the VXLAN tunnel.
func (n *network) addNeighbor(peerIP netip.Prefix, peerMac hashable.MACAddr, vtep netip.Addr) error {
	if n.sbox == nil {
		// We are hitting this case for all the events that are arriving before that the sandbox
		// is being created. The peer got already added into the database and the sandbox init will
		// call the peerDbUpdateSandbox that will configure all these peers from the database
		return nil
	}

	s := n.getSubnetforIP(peerIP)
	if s == nil {
		return fmt.Errorf("couldn't find the subnet %q in network %q", peerIP.String(), n.id)
	}

	if err := n.joinSandbox(s, false); err != nil {
		return fmt.Errorf("subnet sandbox join failed for %q: %v", s.subnetIP.String(), err)
	}

	if n.secure {
		if err := n.driver.setupEncryption(vtep); err != nil {
			log.G(context.TODO()).Warn(err)
		}
	}

	// Add neighbor entry for the peer IP
	if err := n.sbox.AddNeighbor(peerIP.Addr().AsSlice(), peerMac.AsSlice(), osl.WithLinkName(s.vxlanName)); err != nil {
		return fmt.Errorf("could not add neighbor entry into the sandbox: %w", err)
	}

	// Add fdb entry to the bridge for the peer mac
	if n.fdbCnt.Add(hashable.IPMACFrom(vtep, peerMac), 1) == 1 {
		if err := n.sbox.AddNeighbor(vtep.AsSlice(), peerMac.AsSlice(), osl.WithLinkName(s.vxlanName), osl.WithFamily(syscall.AF_BRIDGE)); err != nil {
			return fmt.Errorf("could not add fdb entry into the sandbox: %w", err)
		}
	}

	return nil
}

// peerDelete removes an entry from the peer database.
//
// Local peers are signified by an invalid vtep (i.e. netip.Addr{}).
func (n *network) peerDelete(eid string, peerIP netip.Prefix, peerMac hashable.MACAddr, vtep netip.Addr) error {
	if eid == "" {
		return errors.New("invalid endpoint id")
	}

	logger := log.G(context.TODO()).WithFields(log.Fields{
		"nid":  n.id,
		"eid":  eid,
		"ip":   peerIP,
		"mac":  peerMac,
		"vtep": vtep,
	})
	deleted, dbEntries := n.peerdb.Delete(eid, peerIP, peerMac, vtep)
	if !deleted {
		logger.Warn("Peer entry was not in db")
	}
	if vtep.IsValid() {
		err := n.deleteNeighbor(peerIP, peerMac, vtep)
		if err != nil {
			if dbEntries > 0 && errors.As(err, &osl.NeighborSearchError{}) {
				// We fall in here if there is a transient state and if the neighbor that is being deleted
				// was never been configured into the kernel (we allow only 1 configuration at the time per <ip,mac> mapping)
				return nil
			}
			logger.WithError(err).Warn("Peer delete operation failed")
		}
	}

	if dbEntries > 0 {
		// If there is still an entry into the database and the deletion went through without errors means that there is now no
		// configuration active in the kernel.
		// Restore one configuration for the ip directly from the database, note that is guaranteed that there is one
		peerEntry, ok := n.peerdb.Get(peerIP)
		if !ok {
			return fmt.Errorf("peerDelete: unable to restore a configuration: no entry for %v found in the database", peerIP)
		}
		err := n.addNeighbor(peerIP, peerEntry.mac, peerEntry.vtep)
		if err != nil {
			return fmt.Errorf("peer delete operation failed: %w", err)
		}
	}
	return nil
}

// deleteNeighbor removes programming from the kernel for the given peer to be
// reachable through the VXLAN tunnel. It is the inverse of [driver.addNeighbor].
func (n *network) deleteNeighbor(peerIP netip.Prefix, peerMac hashable.MACAddr, vtep netip.Addr) error {
	if n.sbox == nil {
		return nil
	}

	if n.secure {
		if err := n.driver.removeEncryption(vtep); err != nil {
			log.G(context.TODO()).Warn(err)
		}
	}

	s := n.getSubnetforIP(peerIP)
	if s == nil {
		return fmt.Errorf("could not find the subnet %q in network %q", peerIP.String(), n.id)
	}
	// Remove fdb entry to the bridge for the peer mac
	if n.fdbCnt.Add(hashable.IPMACFrom(vtep, peerMac), -1) == 0 {
		if err := n.sbox.DeleteNeighbor(vtep.AsSlice(), peerMac.AsSlice(), osl.WithLinkName(s.vxlanName), osl.WithFamily(syscall.AF_BRIDGE)); err != nil {
			return fmt.Errorf("could not delete fdb entry in the sandbox: %w", err)
		}
	}

	// Delete neighbor entry for the peer IP
	if err := n.sbox.DeleteNeighbor(peerIP.Addr().AsSlice(), peerMac.AsSlice(), osl.WithLinkName(s.vxlanName)); err != nil {
		return fmt.Errorf("could not delete neighbor entry in the sandbox:%v", err)
	}

	return nil
}