From 5b752fab327c0c9871d3c7df369ccbaed7ad62db Mon Sep 17 00:00:00 2001 From: Albin Kerouanton Date: Thu, 13 Jul 2023 10:40:03 +0200 Subject: [PATCH] api: add Priority field to EndpointSettings This new field is used by libnetwork to determine which endpoint provides the default gateway for a container. Signed-off-by: Albin Kerouanton --- .../router/container/container_routes.go | 8 + api/swagger.yaml | 11 ++ api/types/network/endpoint.go | 1 + container/view.go | 1 + daemon/network.go | 5 +- docs/api/version-history.md | 8 + integration/network/network_linux_test.go | 175 ++++++++++++++++++ 7 files changed, 208 insertions(+), 1 deletion(-) diff --git a/api/server/router/container/container_routes.go b/api/server/router/container/container_routes.go index b6a78480fd..011808b721 100644 --- a/api/server/router/container/container_routes.go +++ b/api/server/router/container/container_routes.go @@ -643,6 +643,14 @@ func (c *containerRouter) postContainersCreate(ctx context.Context, w http.Respo } } + if versions.LessThan(version, "1.48") { + for _, epConfig := range networkingConfig.EndpointsConfig { + // Before 1.48, all endpoints had the same priority, so + // reinitialize this field. + epConfig.GwPriority = 0 + } + } + var warnings []string if warn := handleVolumeDriverBC(version, hostConfig); warn != "" { warnings = append(warnings, warn) diff --git a/api/swagger.yaml b/api/swagger.yaml index a237f374aa..b9f805a52f 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -2927,6 +2927,16 @@ definitions: example: com.example.some-label: "some-value" com.example.some-other-label: "some-other-value" + GwPriority: + description: | + This property determines which endpoint will provide the default + gateway for a container. The endpoint with the highest priority will + be used. If multiple endpoints have the same priority, endpoints are + lexicographically sorted based on their network name, and the one + that sorts first is picked. + type: "number" + example: + - 10 # Operational data NetworkID: @@ -10910,6 +10920,7 @@ paths: IPv4Address: "172.24.56.89" IPv6Address: "2001:db8::5689" MacAddress: "02:42:ac:12:05:02" + Priority: 100 tags: ["Network"] /networks/{id}/disconnect: diff --git a/api/types/network/endpoint.go b/api/types/network/endpoint.go index 0fbb40b351..d724ea02a2 100644 --- a/api/types/network/endpoint.go +++ b/api/types/network/endpoint.go @@ -19,6 +19,7 @@ type EndpointSettings struct { // generated address). MacAddress string DriverOpts map[string]string + GwPriority int // Operational data NetworkID string EndpointID string diff --git a/container/view.go b/container/view.go index eef5ec8a74..ba3c03de14 100644 --- a/container/view.go +++ b/container/view.go @@ -377,6 +377,7 @@ func (v *View) transform(ctr *Container) *Snapshot { GlobalIPv6PrefixLen: netw.GlobalIPv6PrefixLen, MacAddress: netw.MacAddress, NetworkID: netw.NetworkID, + GwPriority: netw.GwPriority, } if netw.IPAMConfig != nil { networks[name].IPAMConfig = &network.EndpointIPAMConfig{ diff --git a/daemon/network.go b/daemon/network.go index eb88467c78..dd2667abc0 100644 --- a/daemon/network.go +++ b/daemon/network.go @@ -1120,7 +1120,10 @@ func buildJoinOptions(settings *network.Settings, n interface{ Name() string }) return []libnetwork.EndpointOption{}, nil } - var joinOptions []libnetwork.EndpointOption + joinOptions := []libnetwork.EndpointOption{ + libnetwork.JoinOptionPriority(epConfig.GwPriority), + } + for _, str := range epConfig.Links { name, alias, err := opts.ParseLink(str) if err != nil { diff --git a/docs/api/version-history.md b/docs/api/version-history.md index 15ba0941d4..5921d63b4f 100644 --- a/docs/api/version-history.md +++ b/docs/api/version-history.md @@ -50,6 +50,14 @@ keywords: "API, Docker, rcli, REST, documentation" daemon has experimental features enabled. * `GET /networks/{id}` now returns an `EnableIPv4` field showing whether the network has IPv4 IPAM enabled. +* `POST /networks/{id}/connect` and `POST /containers/create` now accept a + `GwPriority` field in `EndpointsConfig`. This value is used to determine which + network endpoint provides the default gateway for the container. The endpoint + with the highest priority is selected. If multiple endpoints have the same + priority, endpoints are sorted lexicographically by their network name, and + the one that sorts first is picked. +* `GET /containers/json` now returns a `GwPriority` field in `NetworkSettings` + for each network endpoint. ## v1.47 API changes diff --git a/integration/network/network_linux_test.go b/integration/network/network_linux_test.go index 5300b5908f..55b8f40b64 100644 --- a/integration/network/network_linux_test.go +++ b/integration/network/network_linux_test.go @@ -2,13 +2,19 @@ package network // import "github.com/docker/docker/integration/network" import ( "bytes" + "context" "fmt" "os/exec" + "slices" "strings" + "syscall" "testing" + "time" containertypes "github.com/docker/docker/api/types/container" networktypes "github.com/docker/docker/api/types/network" + "github.com/docker/docker/api/types/versions" + "github.com/docker/docker/client" "github.com/docker/docker/integration/internal/container" "github.com/docker/docker/integration/internal/network" "github.com/docker/docker/internal/testutils/networking" @@ -218,3 +224,172 @@ func TestHostGatewayFromDocker0(t *testing.T) { assert.Check(t, is.Contains(res.Stdout.String(), "192.168.50.1\thg")) assert.Check(t, is.Contains(res.Stdout.String(), "fddd:6ff4:6e08::1\thg")) } + +func TestCreateWithPriority(t *testing.T) { + // This feature should work on Windows, but the test is skipped because: + // 1. Linux-specific tools are used here; 2. 'windows' IPAM driver doesn't + // support static allocations. + skip.If(t, testEnv.DaemonInfo.OSType == "windows") + skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.48"), "requires API v1.48") + + ctx := setupTest(t) + apiClient := testEnv.APIClient() + + network.CreateNoError(ctx, t, apiClient, "testnet1", + network.WithIPv6(), + network.WithIPAM("10.100.20.0/24", "10.100.20.1"), + network.WithIPAM("fd54:7a1b:8269::/64", "fd54:7a1b:8269::1")) + defer network.RemoveNoError(ctx, t, apiClient, "testnet1") + + network.CreateNoError(ctx, t, apiClient, "testnet2", + network.WithIPv6(), + network.WithIPAM("10.100.30.0/24", "10.100.30.1"), + network.WithIPAM("fdff:6dfe:37d2::/64", "fdff:6dfe:37d2::1")) + defer network.RemoveNoError(ctx, t, apiClient, "testnet2") + + ctrID := container.Run(ctx, t, apiClient, + container.WithCmd("sleep", "infinity"), + container.WithNetworkMode("testnet1"), + container.WithEndpointSettings("testnet1", &networktypes.EndpointSettings{GwPriority: 10}), + container.WithEndpointSettings("testnet2", &networktypes.EndpointSettings{GwPriority: 100})) + defer container.Remove(ctx, t, apiClient, ctrID, containertypes.RemoveOptions{Force: true}) + + checkCtrRoutes(t, ctx, apiClient, ctrID, syscall.AF_INET, 3, "default via 10.100.30.1 dev") + // IPv6 routing table will contain for each interface, one route for the LL + // address, one for the ULA, and one multicast. + checkCtrRoutes(t, ctx, apiClient, ctrID, syscall.AF_INET6, 7, "default via fdff:6dfe:37d2::1 dev") +} + +func TestConnectWithPriority(t *testing.T) { + // This feature should work on Windows, but the test is skipped because: + // 1. Linux-specific tools are used here; 2. 'windows' IPAM driver doesn't + // support static allocations. + skip.If(t, testEnv.DaemonInfo.OSType == "windows") + skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.48"), "requires API v1.48") + + ctx := setupTest(t) + apiClient := testEnv.APIClient() + + network.CreateNoError(ctx, t, apiClient, "testnet1", + network.WithIPv6(), + network.WithIPAM("10.100.10.0/24", "10.100.10.1"), + network.WithIPAM("fddd:4901:f594::/64", "fddd:4901:f594::1")) + defer network.RemoveNoError(ctx, t, apiClient, "testnet1") + + network.CreateNoError(ctx, t, apiClient, "testnet2", + network.WithIPv6(), + network.WithIPAM("10.100.20.0/24", "10.100.20.1"), + network.WithIPAM("fd83:7683:7008::/64", "fd83:7683:7008::1")) + defer network.RemoveNoError(ctx, t, apiClient, "testnet2") + + network.CreateNoError(ctx, t, apiClient, "testnet3", + network.WithDriver("bridge"), + network.WithIPv6(), + network.WithIPAM("10.100.30.0/24", "10.100.30.1"), + network.WithIPAM("fd72:de0:adad::/64", "fd72:de0:adad::1")) + defer network.RemoveNoError(ctx, t, apiClient, "testnet3") + + network.CreateNoError(ctx, t, apiClient, "testnet4", + network.WithIPv6(), + network.WithIPAM("10.100.40.0/24", "10.100.40.1"), + network.WithIPAM("fd4c:c927:7d90::/64", "fd4c:c927:7d90::1")) + defer network.RemoveNoError(ctx, t, apiClient, "testnet4") + + network.CreateNoError(ctx, t, apiClient, "testnet5", + network.WithIPv6(), + network.WithIPAM("10.100.50.0/24", "10.100.50.1"), + network.WithIPAM("fd4c:364b:1110::/64", "fd4c:364b:1110::1")) + defer network.RemoveNoError(ctx, t, apiClient, "testnet5") + + ctrID := container.Run(ctx, t, apiClient, + container.WithCmd("sleep", "infinity"), + container.WithNetworkMode("testnet1"), + container.WithEndpointSettings("testnet1", &networktypes.EndpointSettings{})) + defer container.Remove(ctx, t, apiClient, ctrID, containertypes.RemoveOptions{Force: true}) + + checkCtrRoutes(t, ctx, apiClient, ctrID, syscall.AF_INET, 2, "default via 10.100.10.1 dev eth0") + checkCtrRoutes(t, ctx, apiClient, ctrID, syscall.AF_INET6, 4, "default via fddd:4901:f594::1 dev eth0") + + // testnet5 has a negative priority -- the default gateway should not change. + err := apiClient.NetworkConnect(ctx, "testnet5", ctrID, &networktypes.EndpointSettings{GwPriority: -100}) + assert.NilError(t, err) + checkCtrRoutes(t, ctx, apiClient, ctrID, syscall.AF_INET, 3, "default via 10.100.10.1 dev eth0") + checkCtrRoutes(t, ctx, apiClient, ctrID, syscall.AF_INET6, 7, "default via fddd:4901:f594::1 dev eth0") + + // testnet2 has a higher priority. It should now provide the default gateway. + err = apiClient.NetworkConnect(ctx, "testnet2", ctrID, &networktypes.EndpointSettings{GwPriority: 100}) + assert.NilError(t, err) + checkCtrRoutes(t, ctx, apiClient, ctrID, syscall.AF_INET, 4, "default via 10.100.20.1 dev eth2") + checkCtrRoutes(t, ctx, apiClient, ctrID, syscall.AF_INET6, 10, "default via fd83:7683:7008::1 dev eth2") + + // testnet3 has a lower priority, so testnet2 should still provide the default gateway. + err = apiClient.NetworkConnect(ctx, "testnet3", ctrID, &networktypes.EndpointSettings{GwPriority: 10}) + assert.NilError(t, err) + checkCtrRoutes(t, ctx, apiClient, ctrID, syscall.AF_INET, 5, "default via 10.100.20.1 dev eth2") + checkCtrRoutes(t, ctx, apiClient, ctrID, syscall.AF_INET6, 13, "default via fd83:7683:7008::1 dev eth2") + + // testnet4 has the same priority as testnet3, but it sorts after in + // lexicographic order. For now, testnet2 stays the default gateway. + err = apiClient.NetworkConnect(ctx, "testnet4", ctrID, &networktypes.EndpointSettings{GwPriority: 10}) + assert.NilError(t, err) + checkCtrRoutes(t, ctx, apiClient, ctrID, syscall.AF_INET, 6, "default via 10.100.20.1 dev eth2") + checkCtrRoutes(t, ctx, apiClient, ctrID, syscall.AF_INET6, 16, "default via fd83:7683:7008::1 dev eth2") + + inspect := container.Inspect(ctx, t, apiClient, ctrID) + assert.Equal(t, inspect.NetworkSettings.Networks["testnet1"].GwPriority, 0) + assert.Equal(t, inspect.NetworkSettings.Networks["testnet2"].GwPriority, 100) + assert.Equal(t, inspect.NetworkSettings.Networks["testnet3"].GwPriority, 10) + assert.Equal(t, inspect.NetworkSettings.Networks["testnet4"].GwPriority, 10) + assert.Equal(t, inspect.NetworkSettings.Networks["testnet5"].GwPriority, -100) + + // Disconnect testnet2, so testnet3 should now provide the default gateway. + // When two endpoints have the same priority (eg. testnet3 vs testnet4), + // the one that sorts first in lexicographic order is picked. + err = apiClient.NetworkDisconnect(ctx, "testnet2", ctrID, true) + assert.NilError(t, err) + checkCtrRoutes(t, ctx, apiClient, ctrID, syscall.AF_INET, 5, "default via 10.100.30.1 dev eth3") + checkCtrRoutes(t, ctx, apiClient, ctrID, syscall.AF_INET6, 13, "default via fd72:de0:adad::1 dev eth3") + + // Disconnect testnet3, so testnet4 should now provide the default gateway. + err = apiClient.NetworkDisconnect(ctx, "testnet3", ctrID, true) + assert.NilError(t, err) + checkCtrRoutes(t, ctx, apiClient, ctrID, syscall.AF_INET, 4, "default via 10.100.40.1 dev eth4") + checkCtrRoutes(t, ctx, apiClient, ctrID, syscall.AF_INET6, 10, "default via fd4c:c927:7d90::1 dev eth4") + + // Disconnect testnet4, so testnet1 should now provide the default gateway. + err = apiClient.NetworkDisconnect(ctx, "testnet4", ctrID, true) + assert.NilError(t, err) + checkCtrRoutes(t, ctx, apiClient, ctrID, syscall.AF_INET, 3, "default via 10.100.10.1 dev eth0") + checkCtrRoutes(t, ctx, apiClient, ctrID, syscall.AF_INET6, 7, "default via fddd:4901:f594::1 dev eth0") +} + +// checkCtrRoutes execute 'ip route show' in a container, and check that the +// number of routes matches expRoutes. It also checks that the default route +// matches expDefRoute. A substring match is used to avoid issues with +// non-stable interface names. +func checkCtrRoutes(t *testing.T, ctx context.Context, apiClient client.APIClient, ctrID string, af, expRoutes int, expDefRoute string) { + t.Helper() + + fam := "-4" + if af == syscall.AF_INET6 { + fam = "-6" + } + + ctx, cancel := context.WithTimeout(ctx, 1*time.Second) + defer cancel() + res, err := container.Exec(ctx, apiClient, ctrID, []string{"ip", "-o", fam, "route", "show"}) + assert.NilError(t, err) + + assert.Equal(t, res.ExitCode, 0) + assert.Equal(t, res.Stderr(), "") + + routes := slices.DeleteFunc(strings.Split(res.Stdout(), "\n"), func(s string) bool { + return s == "" + }) + + assert.Equal(t, len(routes), expRoutes, "expected %d routes, got %d:\n%s", expRoutes, len(routes), strings.Join(routes, "\n")) + defFound := slices.ContainsFunc(routes, func(s string) bool { + return strings.Contains(s, expDefRoute) + }) + assert.Assert(t, defFound, "default route %q not found:\n%s", expDefRoute, strings.Join(routes, "\n")) +}