libnetwork/d/overlay: ref-count encryption params

The IPsec encryption parameters (Security Association Database and
Security Policy Database entries) for a particular overlay network peer
(VTEP) are shared global state as they have to be programmed into the
root network namespace. The same parameters are used when encrypting
VXLAN traffic to a particular VTEP for all overlay networks. Deleting
the entries for a VTEP will break encryption to that VTEP across all
encrypted overlay networks, therefore the decision of when to delete the
entries must take the state of all overlay networks into account.
Unfortunately this is not the case.

The overlay driver uses local per-network state to decide when to
program and delete the parameters for a VTEP. In practice, the
parameters for all VTEPs participating in an encrypted overlay network
are deleted when the network is deleted. Encryption to that VTEP over
all other active encrypted overlay networks would be broken until some
other incidental peerDB event triggered a re-programming of the
parameters for that VTEP.

Change the setupEncryption and removeEncryption functions to be
reference-counted. The removeEncryption function needs to be called the
same number of times as addEncryption before the parameters are deleted
from the kernel.

Signed-off-by: Cory Snider <csnider@mirantis.com>

libnetwork/drivers/overlay/encryption.go

@@ -91,8 +91,13 @@ func (s *spi) String() string {
 	return fmt.Sprintf("SPI(FWD: 0x%x, REV: 0x%x)", uint32(s.forward), uint32(s.reverse))
 }
 
+type encrNode struct {
+	spi   []spi
+	count int
+}
+
 type encrMap struct {
-	nodes map[netip.Addr][]*spi
+	nodes map[netip.Addr]encrNode
 	sync.Mutex
 }
 
@@ -105,7 +110,7 @@ func (e *encrMap) String() string {
 		b.WriteString(k.String())
 		b.WriteString(":")
 		b.WriteString("[")
-		for _, s := range v {
+		for _, s := range v.spi {
 			b.WriteString(s.String())
 			b.WriteString(",")
 		}
@@ -126,10 +131,10 @@ func (d *driver) setupEncryption(remoteIP netip.Addr) error {
 	}
 	log.G(context.TODO()).Debugf("Programming encryption between %s and %s", localIP, remoteIP)
 
-	indices := make([]*spi, 0, len(keys))
+	indices := make([]spi, 0, len(keys))
 
 	for i, k := range keys {
-		spis := &spi{buildSPI(advIP.AsSlice(), remoteIP.AsSlice(), k.tag), buildSPI(remoteIP.AsSlice(), advIP.AsSlice(), k.tag)}
+		spis := spi{buildSPI(advIP.AsSlice(), remoteIP.AsSlice(), k.tag), buildSPI(remoteIP.AsSlice(), advIP.AsSlice(), k.tag)}
 		dir := reverse
 		if i == 0 {
 			dir = bidir
@@ -149,7 +154,10 @@ func (d *driver) setupEncryption(remoteIP netip.Addr) error {
 	}
 
 	d.secMap.Lock()
-	d.secMap.nodes[remoteIP] = indices
+	node := d.secMap.nodes[remoteIP]
+	node.spi = indices
+	node.count++
+	d.secMap.nodes[remoteIP] = node
 	d.secMap.Unlock()
 
 	return nil
@@ -158,13 +166,20 @@ func (d *driver) setupEncryption(remoteIP netip.Addr) error {
 func (d *driver) removeEncryption(remoteIP netip.Addr) error {
 	log.G(context.TODO()).Debugf("removeEncryption(%s)", remoteIP)
 
-	d.secMap.Lock()
-	indices, ok := d.secMap.nodes[remoteIP]
-	d.secMap.Unlock()
-	if !ok {
+	spi := func() []spi {
+		d.secMap.Lock()
+		defer d.secMap.Unlock()
+		node := d.secMap.nodes[remoteIP]
+		if node.count == 1 {
+			delete(d.secMap.nodes, remoteIP)
+			return node.spi
+		}
+		node.count--
+		d.secMap.nodes[remoteIP] = node
 		return nil
-	}
-	for i, idxs := range indices {
+	}()
+
+	for i, idxs := range spi {
 		dir := reverse
 		if i == 0 {
 			dir = bidir
@@ -263,7 +278,7 @@ func (d *driver) programInput(vni uint32, add bool) error {
 	return nil
 }
 
-func programSA(localIP, remoteIP net.IP, spi *spi, k *key, dir int, add bool) (fSA *netlink.XfrmState, rSA *netlink.XfrmState, lastErr error) {
+func programSA(localIP, remoteIP net.IP, spi spi, k *key, dir int, add bool) (fSA *netlink.XfrmState, rSA *netlink.XfrmState, lastErr error) {
 	var (
 		action      = "Removing"
 		xfrmProgram = ns.NlHandle().XfrmStateDel
@@ -436,12 +451,12 @@ func buildAeadAlgo(k *key, s int) *netlink.XfrmStateAlgo {
 	}
 }
 
-func (d *driver) secMapWalk(f func(netip.Addr, []*spi) ([]*spi, bool)) error {
+func (d *driver) secMapWalk(f func(netip.Addr, []spi) ([]spi, bool)) error {
 	d.secMap.Lock()
-	for node, indices := range d.secMap.nodes {
-		idxs, stop := f(node, indices)
+	for rIP, node := range d.secMap.nodes {
+		idxs, stop := f(rIP, node.spi)
 		if idxs != nil {
-			d.secMap.nodes[node] = idxs
+			d.secMap.nodes[rIP] = encrNode{idxs, node.count}
 		}
 		if stop {
 			break
@@ -457,7 +472,7 @@ func (d *driver) setKeys(keys []*key) error {
 	// Accept the encryption keys and clear any stale encryption map
 	d.Lock()
 	d.keys = keys
-	d.secMap = &encrMap{nodes: map[netip.Addr][]*spi{}}
+	d.secMap = &encrMap{nodes: map[netip.Addr]encrNode{}}
 	d.Unlock()
 	log.G(context.TODO()).Debugf("Initial encryption keys: %v", keys)
 	return nil
@@ -506,7 +521,7 @@ func (d *driver) updateKeys(newKey, primary, pruneKey *key) error {
 		return types.InvalidParameterErrorf("attempting to both make a key (index %d) primary and delete it", priIdx)
 	}
 
-	d.secMapWalk(func(rIP netip.Addr, spis []*spi) ([]*spi, bool) {
+	d.secMapWalk(func(rIP netip.Addr, spis []spi) ([]spi, bool) {
 		return updateNodeKey(lIP.AsSlice(), aIP.AsSlice(), rIP.AsSlice(), spis, d.keys, newIdx, priIdx, delIdx), false
 	})
 
@@ -534,7 +549,7 @@ func (d *driver) updateKeys(newKey, primary, pruneKey *key) error {
  *********************************************************/
 
 // Spis and keys are sorted in such away the one in position 0 is the primary
-func updateNodeKey(lIP, aIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, priIdx, delIdx int) []*spi {
+func updateNodeKey(lIP, aIP, rIP net.IP, idxs []spi, curKeys []*key, newIdx, priIdx, delIdx int) []spi {
 	log.G(context.TODO()).Debugf("Updating keys for node: %s (%d,%d,%d)", rIP, newIdx, priIdx, delIdx)
 
 	spis := idxs
@@ -542,7 +557,7 @@ func updateNodeKey(lIP, aIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, pr
 
 	// add new
 	if newIdx != -1 {
-		spis = append(spis, &spi{
+		spis = append(spis, spi{
 			forward: buildSPI(aIP, rIP, curKeys[newIdx].tag),
 			reverse: buildSPI(rIP, aIP, curKeys[newIdx].tag),
 		})

libnetwork/drivers/overlay/overlay.go

@@ -47,7 +47,7 @@ func Register(r driverapi.Registerer, config map[string]interface{}) error {
 		peerDb: peerNetworkMap{
 			mp: map[string]*peerMap{},
 		},
-		secMap: &encrMap{nodes: map[netip.Addr][]*spi{}},
+		secMap: &encrMap{nodes: map[netip.Addr]encrNode{}},
 		config: config,
 	}
 	return r.RegisterDriver(NetworkType, d, driverapi.Capability{

libnetwork/drivers/overlay/peerdb.go

@@ -212,7 +212,7 @@ func (d *driver) addNeighbor(nid string, peerIP netip.Prefix, peerMac net.Hardwa
 		return fmt.Errorf("subnet sandbox join failed for %q: %v", s.subnetIP.String(), err)
 	}
 
-	if n.secure && len(n.endpoints) > 0 {
+	if n.secure {
 		if err := d.setupEncryption(vtep); err != nil {
 			log.G(context.TODO()).Warn(err)
 		}
@@ -307,7 +307,7 @@ func (d *driver) deleteNeighbor(nid string, peerIP netip.Prefix, peerMac net.Har
 		return nil
 	}
 
-	if n.secure && len(n.endpoints) == 0 {
+	if n.secure {
 		if err := d.removeEncryption(vtep); err != nil {
 			log.G(context.TODO()).Warn(err)
 		}