diff --git a/daemon/libnetwork/cnmallocator/networkallocator.go b/daemon/libnetwork/cnmallocator/networkallocator.go index 6865d842dc..c48a36289c 100644 --- a/daemon/libnetwork/cnmallocator/networkallocator.go +++ b/daemon/libnetwork/cnmallocator/networkallocator.go @@ -15,7 +15,6 @@ import ( "github.com/moby/moby/v2/daemon/libnetwork/ipams/defaultipam" remoteipam "github.com/moby/moby/v2/daemon/libnetwork/ipams/remote" "github.com/moby/moby/v2/daemon/libnetwork/netlabel" - "github.com/moby/moby/v2/daemon/libnetwork/scope" "github.com/moby/moby/v2/pkg/plugingetter" "github.com/moby/swarmkit/v2/api" "github.com/moby/swarmkit/v2/manager/allocator/networkallocator" @@ -81,7 +80,7 @@ type network struct { } type networkDriver struct { - driver driverapi.Driver // driver is nil when isNodeLocal == true + driver driverapi.NetworkAllocator // driver is nil when isNodeLocal == true name string // isNodeLocal indicates whether that driver is locally-managed or requires // global resources allocation. @@ -786,24 +785,25 @@ func (na *cnmNetworkAllocator) resolveDriver(n *api.Network) (*networkDriver, er return &networkDriver{name: dName, isNodeLocal: true}, nil } - d, drvCap := na.networkRegistry.Driver(dName) - if d == nil { - err := na.loadDriver(dName) - if err != nil { - return nil, err - } - - d, drvCap = na.networkRegistry.Driver(dName) - if d == nil { - return nil, fmt.Errorf("could not resolve network driver %s", dName) + if na.networkRegistry.HasDriverOrNwAllocator(dName) { + if nwAlloc := na.networkRegistry.NetworkAllocator(dName); nwAlloc != nil { + return &networkDriver{driver: nwAlloc, name: dName}, nil } + return &networkDriver{name: dName, isNodeLocal: true}, nil } - return &networkDriver{ - driver: d, - name: dName, - isNodeLocal: drvCap.DataScope == scope.Local, - }, nil + if err := na.loadDriver(dName); err != nil { + return nil, err + } + + if na.networkRegistry.HasDriverOrNwAllocator(dName) { + if nwAlloc := na.networkRegistry.NetworkAllocator(dName); nwAlloc != nil { + return &networkDriver{driver: nwAlloc, name: dName}, nil + } + return &networkDriver{name: dName, isNodeLocal: true}, nil + } + + return nil, fmt.Errorf("network driver %s not found", dName) } func (na *cnmNetworkAllocator) loadDriver(name string) error { diff --git a/daemon/libnetwork/controller.go b/daemon/libnetwork/controller.go index 22011cd24e..24cf1d3b12 100644 --- a/daemon/libnetwork/controller.go +++ b/daemon/libnetwork/controller.go @@ -512,6 +512,10 @@ func (c *Controller) RegisterDriver(networkType string, driver driverapi.Driver, return nil } +func (c *Controller) RegisterNetworkAllocator(_ string, _ driverapi.NetworkAllocator) error { + return nil +} + // XXX This should be made driver agnostic. See comment below. const overlayDSROptionString = "dsr" diff --git a/daemon/libnetwork/driverapi/driverapi.go b/daemon/libnetwork/driverapi/driverapi.go index 35db522425..216afd1fa6 100644 --- a/daemon/libnetwork/driverapi/driverapi.go +++ b/daemon/libnetwork/driverapi/driverapi.go @@ -13,16 +13,6 @@ const NetworkPluginEndpointType = "NetworkDriver" // Driver is an interface that every plugin driver needs to implement. type Driver interface { - // NetworkAllocate invokes the driver method to allocate network - // specific resources passing network id and network specific config. - // It returns a key,value pair of network specific driver allocations - // to the caller. - NetworkAllocate(nid string, options map[string]string, ipV4Data, ipV6Data []IPAMData) (map[string]string, error) - - // NetworkFree invokes the driver method to free network specific resources - // associated with a given network id. - NetworkFree(nid string) error - // CreateNetwork invokes the driver method to create a network // passing the network id and network specific config. The // config mechanism will eventually be replaced with labels @@ -64,6 +54,23 @@ type Driver interface { IsBuiltIn() bool } +// NetworkAllocator is a special kind of network driver used by cnmallocator to +// allocate resources inside a Swarm cluster. +type NetworkAllocator interface { + // NetworkAllocate invokes the driver method to allocate network + // specific resources passing network id and network specific config. + // It returns a key,value pair of network specific driver allocations + // to the caller. + NetworkAllocate(nid string, options map[string]string, ipV4Data, ipV6Data []IPAMData) (map[string]string, error) + + // NetworkFree invokes the driver method to free network specific resources + // associated with a given network id. + NetworkFree(nid string) error + + // IsBuiltIn returns true if it is a built-in driver + IsBuiltIn() bool +} + // TableWatcher is an optional interface for a network driver. type TableWatcher interface { // EventNotify notifies the driver when a CRUD operation has @@ -187,6 +194,7 @@ type JoinInfo interface { // Registerer provides a way for network drivers to be dynamically registered. type Registerer interface { RegisterDriver(name string, driver Driver, capability Capability) error + RegisterNetworkAllocator(name string, driver NetworkAllocator) error } // Capability represents the high level capabilities of the drivers which libnetwork can make use of diff --git a/daemon/libnetwork/drivers/bridge/bridge_linux.go b/daemon/libnetwork/drivers/bridge/bridge_linux.go index 7e193e41a8..ff0cae7d6d 100644 --- a/daemon/libnetwork/drivers/bridge/bridge_linux.go +++ b/daemon/libnetwork/drivers/bridge/bridge_linux.go @@ -689,14 +689,6 @@ func (d *driver) getNetworks() []*bridgeNetwork { return ls } -func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { - return nil, types.NotImplementedErrorf("not implemented") -} - -func (d *driver) NetworkFree(id string) error { - return types.NotImplementedErrorf("not implemented") -} - func (d *driver) GetSkipGwAlloc(opts options.Generic) (ipv4, ipv6 bool, _ error) { // The network doesn't exist yet, so use a dummy id that's long enough to be // truncated to a short-id (12 characters) and used in the bridge device name. diff --git a/daemon/libnetwork/drivers/host/host.go b/daemon/libnetwork/drivers/host/host.go index 4c81e2ac44..6297e841e6 100644 --- a/daemon/libnetwork/drivers/host/host.go +++ b/daemon/libnetwork/drivers/host/host.go @@ -23,14 +23,6 @@ func Register(r driverapi.Registerer) error { }) } -func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { - return nil, types.NotImplementedErrorf("not implemented") -} - -func (d *driver) NetworkFree(id string) error { - return types.NotImplementedErrorf("not implemented") -} - func (d *driver) CreateNetwork(ctx context.Context, id string, option map[string]any, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error { d.Lock() defer d.Unlock() diff --git a/daemon/libnetwork/drivers/ipvlan/ipvlan.go b/daemon/libnetwork/drivers/ipvlan/ipvlan.go index 38f8ef7894..8b16521afe 100644 --- a/daemon/libnetwork/drivers/ipvlan/ipvlan.go +++ b/daemon/libnetwork/drivers/ipvlan/ipvlan.go @@ -9,7 +9,6 @@ import ( "github.com/moby/moby/v2/daemon/libnetwork/datastore" "github.com/moby/moby/v2/daemon/libnetwork/driverapi" "github.com/moby/moby/v2/daemon/libnetwork/scope" - "github.com/moby/moby/v2/daemon/libnetwork/types" ) const ( @@ -76,14 +75,6 @@ func Register(r driverapi.Registerer, store *datastore.Store, config map[string] }) } -func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { - return nil, types.NotImplementedErrorf("not implemented") -} - -func (d *driver) NetworkFree(id string) error { - return types.NotImplementedErrorf("not implemented") -} - func (d *driver) EndpointOperInfo(nid, eid string) (map[string]any, error) { return make(map[string]any), nil } diff --git a/daemon/libnetwork/drivers/ipvlan/ipvlan_test.go b/daemon/libnetwork/drivers/ipvlan/ipvlan_test.go index b9116332ee..e6249ab76b 100644 --- a/daemon/libnetwork/drivers/ipvlan/ipvlan_test.go +++ b/daemon/libnetwork/drivers/ipvlan/ipvlan_test.go @@ -31,6 +31,11 @@ func (dt *driverTester) RegisterDriver(name string, drv driverapi.Driver, capabi return nil } +func (dt *driverTester) RegisterNetworkAllocator(name string, _ driverapi.NetworkAllocator) error { + dt.t.Fatalf("Unexpected call to RegisterNetworkAllocator for %q", name) + return nil +} + func TestIpvlanRegister(t *testing.T) { if err := Register(&driverTester{t: t}, storeutils.NewTempStore(t), nil); err != nil { t.Fatal(err) diff --git a/daemon/libnetwork/drivers/macvlan/macvlan.go b/daemon/libnetwork/drivers/macvlan/macvlan.go index 907df4b08e..a1eebf19da 100644 --- a/daemon/libnetwork/drivers/macvlan/macvlan.go +++ b/daemon/libnetwork/drivers/macvlan/macvlan.go @@ -9,7 +9,6 @@ import ( "github.com/moby/moby/v2/daemon/libnetwork/datastore" "github.com/moby/moby/v2/daemon/libnetwork/driverapi" "github.com/moby/moby/v2/daemon/libnetwork/scope" - "github.com/moby/moby/v2/daemon/libnetwork/types" ) const ( @@ -70,14 +69,6 @@ func Register(r driverapi.Registerer, store *datastore.Store, _ map[string]any) }) } -func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { - return nil, types.NotImplementedErrorf("not implemented") -} - -func (d *driver) NetworkFree(id string) error { - return types.NotImplementedErrorf("not implemented") -} - func (d *driver) EndpointOperInfo(nid, eid string) (map[string]any, error) { return make(map[string]any), nil } diff --git a/daemon/libnetwork/drivers/macvlan/macvlan_test.go b/daemon/libnetwork/drivers/macvlan/macvlan_test.go index 6fc23de52a..6c6d4ff11b 100644 --- a/daemon/libnetwork/drivers/macvlan/macvlan_test.go +++ b/daemon/libnetwork/drivers/macvlan/macvlan_test.go @@ -31,6 +31,11 @@ func (dt *driverTester) RegisterDriver(name string, drv driverapi.Driver, capabi return nil } +func (dt *driverTester) RegisterNetworkAllocator(name string, _ driverapi.NetworkAllocator) error { + dt.t.Fatalf("Unexpected call to RegisterNetworkAllocator for %q", name) + return nil +} + func TestMacvlanRegister(t *testing.T) { if err := Register(&driverTester{t: t}, storeutils.NewTempStore(t), nil); err != nil { t.Fatal(err) diff --git a/daemon/libnetwork/drivers/null/null.go b/daemon/libnetwork/drivers/null/null.go index 948dcfeaae..3742504073 100644 --- a/daemon/libnetwork/drivers/null/null.go +++ b/daemon/libnetwork/drivers/null/null.go @@ -23,14 +23,6 @@ func Register(r driverapi.Registerer) error { }) } -func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { - return nil, types.NotImplementedErrorf("not implemented") -} - -func (d *driver) NetworkFree(id string) error { - return types.NotImplementedErrorf("not implemented") -} - func (d *driver) CreateNetwork(ctx context.Context, id string, option map[string]any, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error { d.Lock() defer d.Unlock() diff --git a/daemon/libnetwork/drivers/overlay/ov_network.go b/daemon/libnetwork/drivers/overlay/ov_network.go index ca6adb50fd..7cb49c3d8a 100644 --- a/daemon/libnetwork/drivers/overlay/ov_network.go +++ b/daemon/libnetwork/drivers/overlay/ov_network.go @@ -82,14 +82,6 @@ func init() { runtime.LockOSThread() } -func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { - return nil, types.NotImplementedErrorf("not implemented") -} - -func (d *driver) NetworkFree(id string) error { - return types.NotImplementedErrorf("not implemented") -} - func (d *driver) CreateNetwork(ctx context.Context, id string, option map[string]any, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error { if id == "" { return errors.New("invalid network id") diff --git a/daemon/libnetwork/drivers/overlay/overlay_test.go b/daemon/libnetwork/drivers/overlay/overlay_test.go index 12334f1f72..92c337d68c 100644 --- a/daemon/libnetwork/drivers/overlay/overlay_test.go +++ b/daemon/libnetwork/drivers/overlay/overlay_test.go @@ -35,6 +35,11 @@ func (dt *driverTester) RegisterDriver(name string, drv driverapi.Driver, capabi return nil } +func (dt *driverTester) RegisterNetworkAllocator(name string, _ driverapi.NetworkAllocator) error { + dt.t.Fatalf("Unexpected call to RegisterNetworkAllocator for %q", name) + return nil +} + func TestOverlayInit(t *testing.T) { if err := Register(&driverTester{t: t}, nil); err != nil { t.Fatal(err) diff --git a/daemon/libnetwork/drivers/overlay/ovmanager/ovmanager.go b/daemon/libnetwork/drivers/overlay/ovmanager/ovmanager.go index 62abcd7e56..0a6ccde0cf 100644 --- a/daemon/libnetwork/drivers/overlay/ovmanager/ovmanager.go +++ b/daemon/libnetwork/drivers/overlay/ovmanager/ovmanager.go @@ -13,8 +13,6 @@ import ( "github.com/moby/moby/v2/daemon/libnetwork/driverapi" "github.com/moby/moby/v2/daemon/libnetwork/drivers/overlay/overlayutils" "github.com/moby/moby/v2/daemon/libnetwork/netlabel" - "github.com/moby/moby/v2/daemon/libnetwork/scope" - "github.com/moby/moby/v2/daemon/libnetwork/types" ) const ( @@ -48,10 +46,7 @@ type network struct { // Register registers a new instance of the overlay driver. func Register(r driverapi.Registerer) error { - return r.RegisterDriver(networkType, newDriver(), driverapi.Capability{ - DataScope: scope.Global, - ConnectivityScope: scope.Global, - }) + return r.RegisterNetworkAllocator(networkType, newDriver()) } func newDriver() *driver { @@ -163,40 +158,6 @@ func (n *network) releaseVxlanID() { } } -func (d *driver) CreateNetwork(ctx context.Context, id string, option map[string]any, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error { - return types.NotImplementedErrorf("not implemented") -} - -func (d *driver) DeleteNetwork(nid string) error { - return types.NotImplementedErrorf("not implemented") -} - -func (d *driver) CreateEndpoint(_ context.Context, nid, eid string, ifInfo driverapi.InterfaceInfo, epOptions map[string]any) error { - return types.NotImplementedErrorf("not implemented") -} - -func (d *driver) DeleteEndpoint(nid, eid string) error { - return types.NotImplementedErrorf("not implemented") -} - -func (d *driver) EndpointOperInfo(nid, eid string) (map[string]any, error) { - return nil, types.NotImplementedErrorf("not implemented") -} - -// Join method is invoked when a Sandbox is attached to an endpoint. -func (d *driver) Join(_ context.Context, nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, _, _ map[string]any) error { - return types.NotImplementedErrorf("not implemented") -} - -// Leave method is invoked when a Sandbox detaches from an endpoint. -func (d *driver) Leave(nid, eid string) error { - return types.NotImplementedErrorf("not implemented") -} - -func (d *driver) Type() string { - return networkType -} - func (d *driver) IsBuiltIn() bool { return true } diff --git a/daemon/libnetwork/drivers/remote/driver.go b/daemon/libnetwork/drivers/remote/driver.go index ea358cdeff..5064062c0b 100644 --- a/daemon/libnetwork/drivers/remote/driver.go +++ b/daemon/libnetwork/drivers/remote/driver.go @@ -64,6 +64,11 @@ func Register(r driverapi.Registerer, pg plugingetter.PluginGetter) error { if err = r.RegisterDriver(name, d, *c); err != nil { log.G(context.TODO()).Errorf("error registering driver for %s due to %v", name, err) } + if c.DataScope == scope.Global { + if err := r.RegisterNetworkAllocator(name, d); err != nil { + log.G(context.TODO()).Errorf("error registering network allocator for %s due to %v", name, err) + } + } } // Unit test code is unaware of a true PluginStore. So we fall back to v1 plugins. diff --git a/daemon/libnetwork/drivers/windows/overlay/ov_network_windows.go b/daemon/libnetwork/drivers/windows/overlay/ov_network_windows.go index 8474db7130..f0ac35732b 100644 --- a/daemon/libnetwork/drivers/windows/overlay/ov_network_windows.go +++ b/daemon/libnetwork/drivers/windows/overlay/ov_network_windows.go @@ -54,14 +54,6 @@ type network struct { sync.Mutex } -func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { - return nil, types.NotImplementedErrorf("not implemented") -} - -func (d *driver) NetworkFree(id string) error { - return types.NotImplementedErrorf("not implemented") -} - func (d *driver) CreateNetwork(ctx context.Context, id string, option map[string]any, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error { var ( networkName string diff --git a/daemon/libnetwork/drivers/windows/windows.go b/daemon/libnetwork/drivers/windows/windows.go index 3c30481c1d..da7292b2a7 100644 --- a/daemon/libnetwork/drivers/windows/windows.go +++ b/daemon/libnetwork/drivers/windows/windows.go @@ -943,14 +943,6 @@ func (d *driver) Leave(nid, eid string) error { return nil } -func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { - return nil, types.NotImplementedErrorf("not implemented") -} - -func (d *driver) NetworkFree(id string) error { - return types.NotImplementedErrorf("not implemented") -} - func (d *driver) Type() string { return d.name } diff --git a/daemon/libnetwork/drvregistry/networks.go b/daemon/libnetwork/drvregistry/networks.go index 96736731b2..b48c70b263 100644 --- a/daemon/libnetwork/drvregistry/networks.go +++ b/daemon/libnetwork/drvregistry/networks.go @@ -22,8 +22,9 @@ type Networks struct { // Notify is called whenever a network driver is registered. Notify driverapi.Registerer - mu sync.Mutex - drivers map[string]driverData + mu sync.Mutex + drivers map[string]driverData + nwAllocs map[string]driverapi.NetworkAllocator } var _ driverapi.Registerer = (*Networks)(nil) @@ -88,3 +89,51 @@ func (nr *Networks) RegisterDriver(ntype string, driver driverapi.Driver, capabi return nil } + +// NetworkAllocator returns the NetworkAllocator registered under name, and its capability. +func (nr *Networks) NetworkAllocator(name string) driverapi.NetworkAllocator { + nr.mu.Lock() + defer nr.mu.Unlock() + + d := nr.nwAllocs[name] + return d +} + +func (nr *Networks) RegisterNetworkAllocator(ntype string, nwAlloc driverapi.NetworkAllocator) error { + if strings.TrimSpace(ntype) == "" { + return errors.New("network type string cannot be empty") + } + + nr.mu.Lock() + dd, ok := nr.nwAllocs[ntype] + nr.mu.Unlock() + + if ok && dd.IsBuiltIn() { + return driverapi.ErrActiveRegistration(ntype) + } + + if nr.Notify != nil { + if err := nr.Notify.RegisterNetworkAllocator(ntype, nwAlloc); err != nil { + return err + } + } + + nr.mu.Lock() + defer nr.mu.Unlock() + + if nr.nwAllocs == nil { + nr.nwAllocs = make(map[string]driverapi.NetworkAllocator) + } + nr.nwAllocs[ntype] = nwAlloc + + return nil +} + +func (nr *Networks) HasDriverOrNwAllocator(ntype string) bool { + nr.mu.Lock() + defer nr.mu.Unlock() + + _, hasDriver := nr.drivers[ntype] + _, hasNwAlloc := nr.nwAllocs[ntype] + return hasDriver || hasNwAlloc +} diff --git a/daemon/libnetwork/libnetwork_internal_test.go b/daemon/libnetwork/libnetwork_internal_test.go index 29109565d4..771950c27f 100644 --- a/daemon/libnetwork/libnetwork_internal_test.go +++ b/daemon/libnetwork/libnetwork_internal_test.go @@ -823,11 +823,3 @@ func (b *badDriver) Type() string { func (b *badDriver) IsBuiltIn() bool { return false } - -func (b *badDriver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { - return nil, types.NotImplementedErrorf("not implemented") -} - -func (b *badDriver) NetworkFree(id string) error { - return types.NotImplementedErrorf("not implemented") -}