libnet/drvapi: make NetworkAllocate optional

This method is only used by the cnmallocator to allocate Swarm-scoped
network resources. Its only concrete implementation is in the ovmanager.
Other network drivers are implementing it too to adhere to the
driverapi.Driver interface, but they all return a 'not implemented'
error.

Extract this method into a separate interface, and add a dedicated
RegisterNetworkAllocator to the driver registry. Update the cnmallocator
to load 'network allocators' instead of 'drivers'.

Signed-off-by: Albin Kerouanton <albinker@gmail.com>

daemon/libnetwork/cnmallocator/networkallocator.go

@@ -15,7 +15,6 @@ import (
 	"github.com/moby/moby/v2/daemon/libnetwork/ipams/defaultipam"
 	remoteipam "github.com/moby/moby/v2/daemon/libnetwork/ipams/remote"
 	"github.com/moby/moby/v2/daemon/libnetwork/netlabel"
-	"github.com/moby/moby/v2/daemon/libnetwork/scope"
 	"github.com/moby/moby/v2/pkg/plugingetter"
 	"github.com/moby/swarmkit/v2/api"
 	"github.com/moby/swarmkit/v2/manager/allocator/networkallocator"
@@ -81,7 +80,7 @@ type network struct {
 }
 
 type networkDriver struct {
-	driver driverapi.Driver // driver is nil when isNodeLocal == true
+	driver driverapi.NetworkAllocator // driver is nil when isNodeLocal == true
 	name   string
 	// isNodeLocal indicates whether that driver is locally-managed or requires
 	// global resources allocation.
@@ -786,24 +785,25 @@ func (na *cnmNetworkAllocator) resolveDriver(n *api.Network) (*networkDriver, er
 		return &networkDriver{name: dName, isNodeLocal: true}, nil
 	}
 
-	d, drvCap := na.networkRegistry.Driver(dName)
-	if d == nil {
-		err := na.loadDriver(dName)
-		if err != nil {
-			return nil, err
-		}
-
-		d, drvCap = na.networkRegistry.Driver(dName)
-		if d == nil {
-			return nil, fmt.Errorf("could not resolve network driver %s", dName)
+	if na.networkRegistry.HasDriverOrNwAllocator(dName) {
+		if nwAlloc := na.networkRegistry.NetworkAllocator(dName); nwAlloc != nil {
+			return &networkDriver{driver: nwAlloc, name: dName}, nil
 		}
+		return &networkDriver{name: dName, isNodeLocal: true}, nil
 	}
 
-	return &networkDriver{
-		driver:      d,
-		name:        dName,
-		isNodeLocal: drvCap.DataScope == scope.Local,
-	}, nil
+	if err := na.loadDriver(dName); err != nil {
+		return nil, err
+	}
+
+	if na.networkRegistry.HasDriverOrNwAllocator(dName) {
+		if nwAlloc := na.networkRegistry.NetworkAllocator(dName); nwAlloc != nil {
+			return &networkDriver{driver: nwAlloc, name: dName}, nil
+		}
+		return &networkDriver{name: dName, isNodeLocal: true}, nil
+	}
+
+	return nil, fmt.Errorf("network driver %s not found", dName)
 }
 
 func (na *cnmNetworkAllocator) loadDriver(name string) error {

daemon/libnetwork/controller.go

@@ -512,6 +512,10 @@ func (c *Controller) RegisterDriver(networkType string, driver driverapi.Driver,
 	return nil
 }
 
+func (c *Controller) RegisterNetworkAllocator(_ string, _ driverapi.NetworkAllocator) error {
+	return nil
+}
+
 // XXX  This should be made driver agnostic.  See comment below.
 const overlayDSROptionString = "dsr"
 

daemon/libnetwork/driverapi/driverapi.go

@@ -13,16 +13,6 @@ const NetworkPluginEndpointType = "NetworkDriver"
 
 // Driver is an interface that every plugin driver needs to implement.
 type Driver interface {
-	// NetworkAllocate invokes the driver method to allocate network
-	// specific resources passing network id and network specific config.
-	// It returns a key,value pair of network specific driver allocations
-	// to the caller.
-	NetworkAllocate(nid string, options map[string]string, ipV4Data, ipV6Data []IPAMData) (map[string]string, error)
-
-	// NetworkFree invokes the driver method to free network specific resources
-	// associated with a given network id.
-	NetworkFree(nid string) error
-
 	// CreateNetwork invokes the driver method to create a network
 	// passing the network id and network specific config. The
 	// config mechanism will eventually be replaced with labels
@@ -64,6 +54,23 @@ type Driver interface {
 	IsBuiltIn() bool
 }
 
+// NetworkAllocator is a special kind of network driver used by cnmallocator to
+// allocate resources inside a Swarm cluster.
+type NetworkAllocator interface {
+	// NetworkAllocate invokes the driver method to allocate network
+	// specific resources passing network id and network specific config.
+	// It returns a key,value pair of network specific driver allocations
+	// to the caller.
+	NetworkAllocate(nid string, options map[string]string, ipV4Data, ipV6Data []IPAMData) (map[string]string, error)
+
+	// NetworkFree invokes the driver method to free network specific resources
+	// associated with a given network id.
+	NetworkFree(nid string) error
+
+	// IsBuiltIn returns true if it is a built-in driver
+	IsBuiltIn() bool
+}
+
 // TableWatcher is an optional interface for a network driver.
 type TableWatcher interface {
 	// EventNotify notifies the driver when a CRUD operation has
@@ -187,6 +194,7 @@ type JoinInfo interface {
 // Registerer provides a way for network drivers to be dynamically registered.
 type Registerer interface {
 	RegisterDriver(name string, driver Driver, capability Capability) error
+	RegisterNetworkAllocator(name string, driver NetworkAllocator) error
 }
 
 // Capability represents the high level capabilities of the drivers which libnetwork can make use of

daemon/libnetwork/drivers/bridge/bridge_linux.go

@@ -689,14 +689,6 @@ func (d *driver) getNetworks() []*bridgeNetwork {
 	return ls
 }
 
-func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
-	return nil, types.NotImplementedErrorf("not implemented")
-}
-
-func (d *driver) NetworkFree(id string) error {
-	return types.NotImplementedErrorf("not implemented")
-}
-
 func (d *driver) GetSkipGwAlloc(opts options.Generic) (ipv4, ipv6 bool, _ error) {
 	// The network doesn't exist yet, so use a dummy id that's long enough to be
 	// truncated to a short-id (12 characters) and used in the bridge device name.

daemon/libnetwork/drivers/host/host.go

@@ -23,14 +23,6 @@ func Register(r driverapi.Registerer) error {
 	})
 }
 
-func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
-	return nil, types.NotImplementedErrorf("not implemented")
-}
-
-func (d *driver) NetworkFree(id string) error {
-	return types.NotImplementedErrorf("not implemented")
-}
-
 func (d *driver) CreateNetwork(ctx context.Context, id string, option map[string]any, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error {
 	d.Lock()
 	defer d.Unlock()

daemon/libnetwork/drivers/ipvlan/ipvlan.go

@@ -9,7 +9,6 @@ import (
 	"github.com/moby/moby/v2/daemon/libnetwork/datastore"
 	"github.com/moby/moby/v2/daemon/libnetwork/driverapi"
 	"github.com/moby/moby/v2/daemon/libnetwork/scope"
-	"github.com/moby/moby/v2/daemon/libnetwork/types"
 )
 
 const (
@@ -76,14 +75,6 @@ func Register(r driverapi.Registerer, store *datastore.Store, config map[string]
 	})
 }
 
-func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
-	return nil, types.NotImplementedErrorf("not implemented")
-}
-
-func (d *driver) NetworkFree(id string) error {
-	return types.NotImplementedErrorf("not implemented")
-}
-
 func (d *driver) EndpointOperInfo(nid, eid string) (map[string]any, error) {
 	return make(map[string]any), nil
 }

daemon/libnetwork/drivers/ipvlan/ipvlan_test.go

@@ -31,6 +31,11 @@ func (dt *driverTester) RegisterDriver(name string, drv driverapi.Driver, capabi
 	return nil
 }
 
+func (dt *driverTester) RegisterNetworkAllocator(name string, _ driverapi.NetworkAllocator) error {
+	dt.t.Fatalf("Unexpected call to RegisterNetworkAllocator for %q", name)
+	return nil
+}
+
 func TestIpvlanRegister(t *testing.T) {
 	if err := Register(&driverTester{t: t}, storeutils.NewTempStore(t), nil); err != nil {
 		t.Fatal(err)

daemon/libnetwork/drivers/macvlan/macvlan.go

@@ -9,7 +9,6 @@ import (
 	"github.com/moby/moby/v2/daemon/libnetwork/datastore"
 	"github.com/moby/moby/v2/daemon/libnetwork/driverapi"
 	"github.com/moby/moby/v2/daemon/libnetwork/scope"
-	"github.com/moby/moby/v2/daemon/libnetwork/types"
 )
 
 const (
@@ -70,14 +69,6 @@ func Register(r driverapi.Registerer, store *datastore.Store, _ map[string]any)
 	})
 }
 
-func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
-	return nil, types.NotImplementedErrorf("not implemented")
-}
-
-func (d *driver) NetworkFree(id string) error {
-	return types.NotImplementedErrorf("not implemented")
-}
-
 func (d *driver) EndpointOperInfo(nid, eid string) (map[string]any, error) {
 	return make(map[string]any), nil
 }

daemon/libnetwork/drivers/macvlan/macvlan_test.go

@@ -31,6 +31,11 @@ func (dt *driverTester) RegisterDriver(name string, drv driverapi.Driver, capabi
 	return nil
 }
 
+func (dt *driverTester) RegisterNetworkAllocator(name string, _ driverapi.NetworkAllocator) error {
+	dt.t.Fatalf("Unexpected call to RegisterNetworkAllocator for %q", name)
+	return nil
+}
+
 func TestMacvlanRegister(t *testing.T) {
 	if err := Register(&driverTester{t: t}, storeutils.NewTempStore(t), nil); err != nil {
 		t.Fatal(err)

daemon/libnetwork/drivers/null/null.go

@@ -23,14 +23,6 @@ func Register(r driverapi.Registerer) error {
 	})
 }
 
-func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
-	return nil, types.NotImplementedErrorf("not implemented")
-}
-
-func (d *driver) NetworkFree(id string) error {
-	return types.NotImplementedErrorf("not implemented")
-}
-
 func (d *driver) CreateNetwork(ctx context.Context, id string, option map[string]any, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error {
 	d.Lock()
 	defer d.Unlock()

daemon/libnetwork/drivers/overlay/ov_network.go

@@ -82,14 +82,6 @@ func init() {
 	runtime.LockOSThread()
 }
 
-func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
-	return nil, types.NotImplementedErrorf("not implemented")
-}
-
-func (d *driver) NetworkFree(id string) error {
-	return types.NotImplementedErrorf("not implemented")
-}
-
 func (d *driver) CreateNetwork(ctx context.Context, id string, option map[string]any, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error {
 	if id == "" {
 		return errors.New("invalid network id")

daemon/libnetwork/drivers/overlay/overlay_test.go

@@ -35,6 +35,11 @@ func (dt *driverTester) RegisterDriver(name string, drv driverapi.Driver, capabi
 	return nil
 }
 
+func (dt *driverTester) RegisterNetworkAllocator(name string, _ driverapi.NetworkAllocator) error {
+	dt.t.Fatalf("Unexpected call to RegisterNetworkAllocator for %q", name)
+	return nil
+}
+
 func TestOverlayInit(t *testing.T) {
 	if err := Register(&driverTester{t: t}, nil); err != nil {
 		t.Fatal(err)

daemon/libnetwork/drivers/overlay/ovmanager/ovmanager.go

@@ -13,8 +13,6 @@ import (
 	"github.com/moby/moby/v2/daemon/libnetwork/driverapi"
 	"github.com/moby/moby/v2/daemon/libnetwork/drivers/overlay/overlayutils"
 	"github.com/moby/moby/v2/daemon/libnetwork/netlabel"
-	"github.com/moby/moby/v2/daemon/libnetwork/scope"
-	"github.com/moby/moby/v2/daemon/libnetwork/types"
 )
 
 const (
@@ -48,10 +46,7 @@ type network struct {
 
 // Register registers a new instance of the overlay driver.
 func Register(r driverapi.Registerer) error {
-	return r.RegisterDriver(networkType, newDriver(), driverapi.Capability{
-		DataScope:         scope.Global,
-		ConnectivityScope: scope.Global,
-	})
+	return r.RegisterNetworkAllocator(networkType, newDriver())
 }
 
 func newDriver() *driver {
@@ -163,40 +158,6 @@ func (n *network) releaseVxlanID() {
 	}
 }
 
-func (d *driver) CreateNetwork(ctx context.Context, id string, option map[string]any, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error {
-	return types.NotImplementedErrorf("not implemented")
-}
-
-func (d *driver) DeleteNetwork(nid string) error {
-	return types.NotImplementedErrorf("not implemented")
-}
-
-func (d *driver) CreateEndpoint(_ context.Context, nid, eid string, ifInfo driverapi.InterfaceInfo, epOptions map[string]any) error {
-	return types.NotImplementedErrorf("not implemented")
-}
-
-func (d *driver) DeleteEndpoint(nid, eid string) error {
-	return types.NotImplementedErrorf("not implemented")
-}
-
-func (d *driver) EndpointOperInfo(nid, eid string) (map[string]any, error) {
-	return nil, types.NotImplementedErrorf("not implemented")
-}
-
-// Join method is invoked when a Sandbox is attached to an endpoint.
-func (d *driver) Join(_ context.Context, nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, _, _ map[string]any) error {
-	return types.NotImplementedErrorf("not implemented")
-}
-
-// Leave method is invoked when a Sandbox detaches from an endpoint.
-func (d *driver) Leave(nid, eid string) error {
-	return types.NotImplementedErrorf("not implemented")
-}
-
-func (d *driver) Type() string {
-	return networkType
-}
-
 func (d *driver) IsBuiltIn() bool {
 	return true
 }

daemon/libnetwork/drivers/remote/driver.go

@@ -64,6 +64,11 @@ func Register(r driverapi.Registerer, pg plugingetter.PluginGetter) error {
 		if err = r.RegisterDriver(name, d, *c); err != nil {
 			log.G(context.TODO()).Errorf("error registering driver for %s due to %v", name, err)
 		}
+		if c.DataScope == scope.Global {
+			if err := r.RegisterNetworkAllocator(name, d); err != nil {
+				log.G(context.TODO()).Errorf("error registering network allocator for %s due to %v", name, err)
+			}
+		}
 	}
 
 	// Unit test code is unaware of a true PluginStore. So we fall back to v1 plugins.

daemon/libnetwork/drivers/windows/overlay/ov_network_windows.go

@@ -54,14 +54,6 @@ type network struct {
 	sync.Mutex
 }
 
-func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
-	return nil, types.NotImplementedErrorf("not implemented")
-}
-
-func (d *driver) NetworkFree(id string) error {
-	return types.NotImplementedErrorf("not implemented")
-}
-
 func (d *driver) CreateNetwork(ctx context.Context, id string, option map[string]any, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error {
 	var (
 		networkName   string

daemon/libnetwork/drivers/windows/windows.go

@@ -943,14 +943,6 @@ func (d *driver) Leave(nid, eid string) error {
 	return nil
 }
 
-func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
-	return nil, types.NotImplementedErrorf("not implemented")
-}
-
-func (d *driver) NetworkFree(id string) error {
-	return types.NotImplementedErrorf("not implemented")
-}
-
 func (d *driver) Type() string {
 	return d.name
 }

daemon/libnetwork/drvregistry/networks.go

@@ -22,8 +22,9 @@ type Networks struct {
 	// Notify is called whenever a network driver is registered.
 	Notify driverapi.Registerer
 
-	mu      sync.Mutex
-	drivers map[string]driverData
+	mu       sync.Mutex
+	drivers  map[string]driverData
+	nwAllocs map[string]driverapi.NetworkAllocator
 }
 
 var _ driverapi.Registerer = (*Networks)(nil)
@@ -88,3 +89,51 @@ func (nr *Networks) RegisterDriver(ntype string, driver driverapi.Driver, capabi
 
 	return nil
 }
+
+// NetworkAllocator returns the NetworkAllocator registered under name, and its capability.
+func (nr *Networks) NetworkAllocator(name string) driverapi.NetworkAllocator {
+	nr.mu.Lock()
+	defer nr.mu.Unlock()
+
+	d := nr.nwAllocs[name]
+	return d
+}
+
+func (nr *Networks) RegisterNetworkAllocator(ntype string, nwAlloc driverapi.NetworkAllocator) error {
+	if strings.TrimSpace(ntype) == "" {
+		return errors.New("network type string cannot be empty")
+	}
+
+	nr.mu.Lock()
+	dd, ok := nr.nwAllocs[ntype]
+	nr.mu.Unlock()
+
+	if ok && dd.IsBuiltIn() {
+		return driverapi.ErrActiveRegistration(ntype)
+	}
+
+	if nr.Notify != nil {
+		if err := nr.Notify.RegisterNetworkAllocator(ntype, nwAlloc); err != nil {
+			return err
+		}
+	}
+
+	nr.mu.Lock()
+	defer nr.mu.Unlock()
+
+	if nr.nwAllocs == nil {
+		nr.nwAllocs = make(map[string]driverapi.NetworkAllocator)
+	}
+	nr.nwAllocs[ntype] = nwAlloc
+
+	return nil
+}
+
+func (nr *Networks) HasDriverOrNwAllocator(ntype string) bool {
+	nr.mu.Lock()
+	defer nr.mu.Unlock()
+
+	_, hasDriver := nr.drivers[ntype]
+	_, hasNwAlloc := nr.nwAllocs[ntype]
+	return hasDriver || hasNwAlloc
+}

daemon/libnetwork/libnetwork_internal_test.go

@@ -823,11 +823,3 @@ func (b *badDriver) Type() string {
 func (b *badDriver) IsBuiltIn() bool {
 	return false
 }
-
-func (b *badDriver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
-	return nil, types.NotImplementedErrorf("not implemented")
-}
-
-func (b *badDriver) NetworkFree(id string) error {
-	return types.NotImplementedErrorf("not implemented")
-}