Merge pull request #50599 from thaJeztah/cleanup_registry

daemon/pkg/registry: some cleanups to start separating "search"
This commit is contained in:
Sebastiaan van Stijn
2025-08-05 13:51:17 +02:00
committed by GitHub
9 changed files with 305 additions and 321 deletions

View File

@@ -145,9 +145,9 @@ func ConvertToHostname(maybeURL string) string {
return stripped
}
// ResolveAuthConfig matches an auth configuration to a server address or a URL
func ResolveAuthConfig(authConfigs map[string]registry.AuthConfig, index *registry.IndexInfo) registry.AuthConfig {
configKey := GetAuthConfigKey(index)
// resolveAuthConfig matches an auth configuration to a server address or a URL
func resolveAuthConfig(authConfigs map[string]registry.AuthConfig, index *registry.IndexInfo) registry.AuthConfig {
configKey := getAuthConfigKey(index)
// First try the happy case
if c, found := authConfigs[configKey]; found || index.Official {
return c

View File

@@ -31,10 +31,10 @@ func TestResolveAuthConfigIndexServer(t *testing.T) {
Official: false,
}
resolved := ResolveAuthConfig(authConfigs, officialIndex)
resolved := resolveAuthConfig(authConfigs, officialIndex)
assert.Equal(t, resolved, indexConfig, "Expected ResolveAuthConfig to return IndexServer")
resolved = ResolveAuthConfig(authConfigs, privateIndex)
resolved = resolveAuthConfig(authConfigs, privateIndex)
assert.Check(t, resolved != indexConfig, "Expected ResolveAuthConfig to not return IndexServer")
}
@@ -92,12 +92,12 @@ func TestResolveAuthConfigFullURL(t *testing.T) {
}
for _, reg := range registries {
authConfigs[reg] = configured
resolved := ResolveAuthConfig(authConfigs, index)
resolved := resolveAuthConfig(authConfigs, index)
if resolved.Username != configured.Username || resolved.Password != configured.Password {
t.Errorf("%s -> %v != %v\n", reg, resolved, configured)
}
delete(authConfigs, reg)
resolved = ResolveAuthConfig(authConfigs, index)
resolved = resolveAuthConfig(authConfigs, index)
if resolved.Username == configured.Username || resolved.Password == configured.Password {
t.Errorf("%s -> %v == %v\n", reg, resolved, configured)
}

View File

@@ -342,26 +342,9 @@ func validateHostPort(s string) error {
return nil
}
// newIndexInfo returns IndexInfo configuration from indexName
func newIndexInfo(config *serviceConfig, indexName string) *registry.IndexInfo {
indexName = normalizeIndexName(indexName)
// Return any configured index info, first.
if index, ok := config.IndexConfigs[indexName]; ok {
return index
}
// Construct a non-configured index info.
return &registry.IndexInfo{
Name: indexName,
Mirrors: []string{},
Secure: config.isSecureIndex(indexName),
}
}
// GetAuthConfigKey special-cases using the full index address of the official
// getAuthConfigKey special-cases using the full index address of the official
// index as the AuthConfig key, and uses the (host)name[:port] for private indexes.
func GetAuthConfigKey(index *registry.IndexInfo) string {
func getAuthConfigKey(index *registry.IndexInfo) string {
if index.Official {
return IndexServer
}

View File

@@ -6,9 +6,7 @@ import (
"testing"
"github.com/distribution/reference"
"github.com/moby/moby/api/types/registry"
"gotest.tools/v3/assert"
is "gotest.tools/v3/assert/cmp"
)
// overrideLookupIP overrides net.LookupIP for testing.
@@ -34,233 +32,6 @@ func overrideLookupIP(t *testing.T) {
})
}
func TestNewIndexInfo(t *testing.T) {
overrideLookupIP(t)
// ipv6Loopback is the CIDR for the IPv6 loopback address ("::1"); "::1/128"
ipv6Loopback := &net.IPNet{
IP: net.IPv6loopback,
Mask: net.CIDRMask(128, 128),
}
// ipv4Loopback is the CIDR for IPv4 loopback addresses ("127.0.0.0/8")
ipv4Loopback := &net.IPNet{
IP: net.IPv4(127, 0, 0, 0),
Mask: net.CIDRMask(8, 32),
}
// emptyServiceConfig is a default service-config for situations where
// no config-file is available (e.g. when used in the CLI). It won't
// have mirrors configured, but does have the default insecure registry
// CIDRs for loopback interfaces configured.
emptyServiceConfig := &serviceConfig{
IndexConfigs: map[string]*registry.IndexInfo{
IndexName: {
Name: IndexName,
Mirrors: []string{},
Secure: true,
Official: true,
},
},
InsecureRegistryCIDRs: []*registry.NetIPNet{
(*registry.NetIPNet)(ipv6Loopback),
(*registry.NetIPNet)(ipv4Loopback),
},
}
expectedIndexInfos := map[string]*registry.IndexInfo{
IndexName: {
Name: IndexName,
Official: true,
Secure: true,
Mirrors: []string{},
},
"index." + IndexName: {
Name: IndexName,
Official: true,
Secure: true,
Mirrors: []string{},
},
"example.com": {
Name: "example.com",
Official: false,
Secure: true,
Mirrors: []string{},
},
"127.0.0.1:5000": {
Name: "127.0.0.1:5000",
Official: false,
Secure: false,
Mirrors: []string{},
},
}
t.Run("no mirrors", func(t *testing.T) {
for indexName, expected := range expectedIndexInfos {
t.Run(indexName, func(t *testing.T) {
actual := newIndexInfo(emptyServiceConfig, indexName)
assert.Check(t, is.DeepEqual(actual, expected))
})
}
})
expectedIndexInfos = map[string]*registry.IndexInfo{
IndexName: {
Name: IndexName,
Official: true,
Secure: true,
Mirrors: []string{"http://mirror1.local/", "http://mirror2.local/"},
},
"index." + IndexName: {
Name: IndexName,
Official: true,
Secure: true,
Mirrors: []string{"http://mirror1.local/", "http://mirror2.local/"},
},
"example.com": {
Name: "example.com",
Official: false,
Secure: false,
Mirrors: []string{},
},
"example.com:5000": {
Name: "example.com:5000",
Official: false,
Secure: true,
Mirrors: []string{},
},
"127.0.0.1": {
Name: "127.0.0.1",
Official: false,
Secure: false,
Mirrors: []string{},
},
"127.0.0.1:5000": {
Name: "127.0.0.1:5000",
Official: false,
Secure: false,
Mirrors: []string{},
},
"127.255.255.255": {
Name: "127.255.255.255",
Official: false,
Secure: false,
Mirrors: []string{},
},
"127.255.255.255:5000": {
Name: "127.255.255.255:5000",
Official: false,
Secure: false,
Mirrors: []string{},
},
"::1": {
Name: "::1",
Official: false,
Secure: false,
Mirrors: []string{},
},
"[::1]:5000": {
Name: "[::1]:5000",
Official: false,
Secure: false,
Mirrors: []string{},
},
// IPv6 only has a single loopback address, so ::2 is not a loopback,
// hence not marked "insecure".
"::2": {
Name: "::2",
Official: false,
Secure: true,
Mirrors: []string{},
},
// IPv6 only has a single loopback address, so ::2 is not a loopback,
// hence not marked "insecure".
"[::2]:5000": {
Name: "[::2]:5000",
Official: false,
Secure: true,
Mirrors: []string{},
},
"other.com": {
Name: "other.com",
Official: false,
Secure: true,
Mirrors: []string{},
},
}
t.Run("mirrors", func(t *testing.T) {
// Note that newServiceConfig calls ValidateMirror internally, which normalizes
// mirror-URLs to have a trailing slash.
config, err := newServiceConfig(ServiceOptions{
Mirrors: []string{"http://mirror1.local", "http://mirror2.local"},
InsecureRegistries: []string{"example.com"},
})
assert.NilError(t, err)
for indexName, expected := range expectedIndexInfos {
t.Run(indexName, func(t *testing.T) {
actual := newIndexInfo(config, indexName)
assert.Check(t, is.DeepEqual(actual, expected))
})
}
})
expectedIndexInfos = map[string]*registry.IndexInfo{
"example.com": {
Name: "example.com",
Official: false,
Secure: false,
Mirrors: []string{},
},
"example.com:5000": {
Name: "example.com:5000",
Official: false,
Secure: false,
Mirrors: []string{},
},
"127.0.0.1": {
Name: "127.0.0.1",
Official: false,
Secure: false,
Mirrors: []string{},
},
"127.0.0.1:5000": {
Name: "127.0.0.1:5000",
Official: false,
Secure: false,
Mirrors: []string{},
},
"42.42.0.1:5000": {
Name: "42.42.0.1:5000",
Official: false,
Secure: false,
Mirrors: []string{},
},
"42.43.0.1:5000": {
Name: "42.43.0.1:5000",
Official: false,
Secure: true,
Mirrors: []string{},
},
"other.com": {
Name: "other.com",
Official: false,
Secure: true,
Mirrors: []string{},
},
}
t.Run("custom insecure", func(t *testing.T) {
config, err := newServiceConfig(ServiceOptions{
InsecureRegistries: []string{"42.42.0.0/16"},
})
assert.NilError(t, err)
for indexName, expected := range expectedIndexInfos {
t.Run(indexName, func(t *testing.T) {
actual := newIndexInfo(config, indexName)
assert.Check(t, is.DeepEqual(actual, expected))
})
}
})
}
func TestMirrorEndpointLookup(t *testing.T) {
containsMirror := func(endpoints []APIEndpoint) bool {
for _, pe := range endpoints {

View File

@@ -2,7 +2,10 @@ package registry
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
@@ -129,7 +132,7 @@ func (s *Service) searchUnfiltered(ctx context.Context, term string, limit int,
}
}
return newSession(client, endpoint).searchRepositories(ctx, remoteName, limit)
return searchRepositories(ctx, client, endpoint, remoteName, limit)
}
// splitReposSearchTerm breaks a search term into an index name and remote name
@@ -143,3 +146,58 @@ func splitReposSearchTerm(reposName string) (string, string) {
}
return nameParts[0], nameParts[1]
}
// newIndexInfo returns IndexInfo configuration from indexName
func newIndexInfo(config *serviceConfig, indexName string) *registry.IndexInfo {
indexName = normalizeIndexName(indexName)
// Return any configured index info, first.
if index, ok := config.IndexConfigs[indexName]; ok {
return index
}
// Construct a non-configured index info.
return &registry.IndexInfo{
Name: indexName,
Mirrors: []string{},
Secure: config.isSecureIndex(indexName),
}
}
// defaultSearchLimit is the default value for maximum number of returned search results.
const defaultSearchLimit = 25
// searchRepositories performs a search against the remote repository
func searchRepositories(ctx context.Context, client *http.Client, ep *v1Endpoint, term string, limit int) (*registry.SearchResults, error) {
if limit == 0 {
limit = defaultSearchLimit
}
if limit < 1 || limit > 100 {
return nil, invalidParamf("limit %d is outside the range of [1, 100]", limit)
}
u := ep.String() + "search?q=" + url.QueryEscape(term) + "&n=" + url.QueryEscape(strconv.Itoa(limit))
log.G(ctx).WithField("url", u).Debug("searchRepositories")
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, http.NoBody)
if err != nil {
return nil, invalidParamWrapf(err, "error building request")
}
// Have the AuthTransport send authentication, when logged in.
req.Header.Set("X-Docker-Token", "true")
res, err := client.Do(req)
if err != nil {
return nil, systemErr{err}
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
// TODO(thaJeztah): return upstream response body for errors (see https://github.com/moby/moby/issues/27286).
// TODO(thaJeztah): handle other status-codes to return correct error-type
return nil, errUnknown{fmt.Errorf("unexpected status code %d", res.StatusCode)}
}
result := &registry.SearchResults{}
err = json.NewDecoder(res.Body).Decode(result)
if err != nil {
return nil, systemErr{errors.Wrap(err, "error decoding registry search results")}
}
return result, nil
}

View File

@@ -39,7 +39,7 @@ func newV1Endpoint(ctx context.Context, index *registry.IndexInfo, headers http.
return nil, err
}
endpoint, err := newV1EndpointFromStr(GetAuthConfigKey(index), tlsConfig, headers)
endpoint, err := newV1EndpointFromStr(getAuthConfigKey(index), tlsConfig, headers)
if err != nil {
return nil, err
}

View File

@@ -4,13 +4,9 @@ import (
// this is required for some certificates
"context"
_ "crypto/sha512"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/cookiejar"
"net/url"
"strconv"
"strings"
"sync"
@@ -19,12 +15,6 @@ import (
"github.com/pkg/errors"
)
// A session is used to communicate with a V1 registry
type session struct {
indexEndpoint *v1Endpoint
client *http.Client
}
type authTransport struct {
base http.RoundTripper
authConfig *registry.AuthConfig
@@ -201,48 +191,3 @@ func authorizeClient(ctx context.Context, client *http.Client, authConfig *regis
return nil
}
func newSession(client *http.Client, endpoint *v1Endpoint) *session {
return &session{
client: client,
indexEndpoint: endpoint,
}
}
// defaultSearchLimit is the default value for maximum number of returned search results.
const defaultSearchLimit = 25
// searchRepositories performs a search against the remote repository
func (r *session) searchRepositories(ctx context.Context, term string, limit int) (*registry.SearchResults, error) {
if limit == 0 {
limit = defaultSearchLimit
}
if limit < 1 || limit > 100 {
return nil, invalidParamf("limit %d is outside the range of [1, 100]", limit)
}
u := r.indexEndpoint.String() + "search?q=" + url.QueryEscape(term) + "&n=" + url.QueryEscape(strconv.Itoa(limit))
log.G(ctx).WithField("url", u).Debug("searchRepositories")
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, http.NoBody)
if err != nil {
return nil, invalidParamWrapf(err, "error building request")
}
// Have the AuthTransport send authentication, when logged in.
req.Header.Set("X-Docker-Token", "true")
res, err := r.client.Do(req)
if err != nil {
return nil, systemErr{err}
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
// TODO(thaJeztah): return upstream response body for errors (see https://github.com/moby/moby/issues/27286).
// TODO(thaJeztah): handle other status-codes to return correct error-type
return nil, errUnknown{fmt.Errorf("unexpected status code %d", res.StatusCode)}
}
result := &registry.SearchResults{}
err = json.NewDecoder(res.Body).Decode(result)
if err != nil {
return nil, systemErr{errors.Wrap(err, "error decoding registry search results")}
}
return result, nil
}

View File

@@ -3,6 +3,7 @@ package registry
import (
"context"
"encoding/json"
"net"
"net/http"
"net/http/httptest"
"net/http/httputil"
@@ -13,15 +14,16 @@ import (
"github.com/moby/moby/api/types/filters"
"github.com/moby/moby/api/types/registry"
"gotest.tools/v3/assert"
is "gotest.tools/v3/assert/cmp"
)
func spawnTestRegistrySession(t *testing.T) *session {
func spawnTestRegistrySession(t *testing.T) (*http.Client, *v1Endpoint) {
t.Helper()
authConfig := &registry.AuthConfig{}
endpoint, err := newV1Endpoint(context.Background(), makeIndex("/v1/"), nil)
if err != nil {
t.Fatal(err)
}
authConfig := &registry.AuthConfig{}
userAgent := "docker test client"
var tr http.RoundTripper = debugTransport{newTransport(nil), t.Log}
tr = transport.NewTransport(newAuthTransport(tr, authConfig, false), Headers(userAgent, nil)...)
@@ -30,8 +32,6 @@ func spawnTestRegistrySession(t *testing.T) *session {
if err := authorizeClient(context.Background(), client, authConfig, endpoint); err != nil {
t.Fatal(err)
}
r := newSession(client, endpoint)
// In a normal scenario for the v1 registry, the client should send a `X-Docker-Token: true`
// header while authenticating, in order to retrieve a token that can be later used to
// perform authenticated actions.
@@ -42,8 +42,8 @@ func spawnTestRegistrySession(t *testing.T) *session {
// Because we know that the client's transport is an `*authTransport` we simply cast it,
// in order to set the internal cached token to the fake token, and thus send that fake token
// upon every subsequent requests.
r.client.Transport.(*authTransport).token = []string{"fake-token"}
return r
client.Transport.(*authTransport).token = []string{"fake-token"}
return client, endpoint
}
type debugTransport struct {
@@ -70,8 +70,8 @@ func (tr debugTransport) RoundTrip(req *http.Request) (*http.Response, error) {
}
func TestSearchRepositories(t *testing.T) {
r := spawnTestRegistrySession(t)
results, err := r.searchRepositories(context.Background(), "fakequery", 25)
client, ep := spawnTestRegistrySession(t)
results, err := searchRepositories(context.Background(), client, ep, "fakequery", 25)
if err != nil {
t.Fatal(err)
}
@@ -416,3 +416,230 @@ func TestSearch(t *testing.T) {
})
}
}
func TestNewIndexInfo(t *testing.T) {
overrideLookupIP(t)
// ipv6Loopback is the CIDR for the IPv6 loopback address ("::1"); "::1/128"
ipv6Loopback := &net.IPNet{
IP: net.IPv6loopback,
Mask: net.CIDRMask(128, 128),
}
// ipv4Loopback is the CIDR for IPv4 loopback addresses ("127.0.0.0/8")
ipv4Loopback := &net.IPNet{
IP: net.IPv4(127, 0, 0, 0),
Mask: net.CIDRMask(8, 32),
}
// emptyServiceConfig is a default service-config for situations where
// no config-file is available (e.g. when used in the CLI). It won't
// have mirrors configured, but does have the default insecure registry
// CIDRs for loopback interfaces configured.
emptyServiceConfig := &serviceConfig{
IndexConfigs: map[string]*registry.IndexInfo{
IndexName: {
Name: IndexName,
Mirrors: []string{},
Secure: true,
Official: true,
},
},
InsecureRegistryCIDRs: []*registry.NetIPNet{
(*registry.NetIPNet)(ipv6Loopback),
(*registry.NetIPNet)(ipv4Loopback),
},
}
expectedIndexInfos := map[string]*registry.IndexInfo{
IndexName: {
Name: IndexName,
Official: true,
Secure: true,
Mirrors: []string{},
},
"index." + IndexName: {
Name: IndexName,
Official: true,
Secure: true,
Mirrors: []string{},
},
"example.com": {
Name: "example.com",
Official: false,
Secure: true,
Mirrors: []string{},
},
"127.0.0.1:5000": {
Name: "127.0.0.1:5000",
Official: false,
Secure: false,
Mirrors: []string{},
},
}
t.Run("no mirrors", func(t *testing.T) {
for indexName, expected := range expectedIndexInfos {
t.Run(indexName, func(t *testing.T) {
actual := newIndexInfo(emptyServiceConfig, indexName)
assert.Check(t, is.DeepEqual(actual, expected))
})
}
})
expectedIndexInfos = map[string]*registry.IndexInfo{
IndexName: {
Name: IndexName,
Official: true,
Secure: true,
Mirrors: []string{"http://mirror1.local/", "http://mirror2.local/"},
},
"index." + IndexName: {
Name: IndexName,
Official: true,
Secure: true,
Mirrors: []string{"http://mirror1.local/", "http://mirror2.local/"},
},
"example.com": {
Name: "example.com",
Official: false,
Secure: false,
Mirrors: []string{},
},
"example.com:5000": {
Name: "example.com:5000",
Official: false,
Secure: true,
Mirrors: []string{},
},
"127.0.0.1": {
Name: "127.0.0.1",
Official: false,
Secure: false,
Mirrors: []string{},
},
"127.0.0.1:5000": {
Name: "127.0.0.1:5000",
Official: false,
Secure: false,
Mirrors: []string{},
},
"127.255.255.255": {
Name: "127.255.255.255",
Official: false,
Secure: false,
Mirrors: []string{},
},
"127.255.255.255:5000": {
Name: "127.255.255.255:5000",
Official: false,
Secure: false,
Mirrors: []string{},
},
"::1": {
Name: "::1",
Official: false,
Secure: false,
Mirrors: []string{},
},
"[::1]:5000": {
Name: "[::1]:5000",
Official: false,
Secure: false,
Mirrors: []string{},
},
// IPv6 only has a single loopback address, so ::2 is not a loopback,
// hence not marked "insecure".
"::2": {
Name: "::2",
Official: false,
Secure: true,
Mirrors: []string{},
},
// IPv6 only has a single loopback address, so ::2 is not a loopback,
// hence not marked "insecure".
"[::2]:5000": {
Name: "[::2]:5000",
Official: false,
Secure: true,
Mirrors: []string{},
},
"other.com": {
Name: "other.com",
Official: false,
Secure: true,
Mirrors: []string{},
},
}
t.Run("mirrors", func(t *testing.T) {
// Note that newServiceConfig calls ValidateMirror internally, which normalizes
// mirror-URLs to have a trailing slash.
config, err := newServiceConfig(ServiceOptions{
Mirrors: []string{"http://mirror1.local", "http://mirror2.local"},
InsecureRegistries: []string{"example.com"},
})
assert.NilError(t, err)
for indexName, expected := range expectedIndexInfos {
t.Run(indexName, func(t *testing.T) {
actual := newIndexInfo(config, indexName)
assert.Check(t, is.DeepEqual(actual, expected))
})
}
})
expectedIndexInfos = map[string]*registry.IndexInfo{
"example.com": {
Name: "example.com",
Official: false,
Secure: false,
Mirrors: []string{},
},
"example.com:5000": {
Name: "example.com:5000",
Official: false,
Secure: false,
Mirrors: []string{},
},
"127.0.0.1": {
Name: "127.0.0.1",
Official: false,
Secure: false,
Mirrors: []string{},
},
"127.0.0.1:5000": {
Name: "127.0.0.1:5000",
Official: false,
Secure: false,
Mirrors: []string{},
},
"42.42.0.1:5000": {
Name: "42.42.0.1:5000",
Official: false,
Secure: false,
Mirrors: []string{},
},
"42.43.0.1:5000": {
Name: "42.43.0.1:5000",
Official: false,
Secure: true,
Mirrors: []string{},
},
"other.com": {
Name: "other.com",
Official: false,
Secure: true,
Mirrors: []string{},
},
}
t.Run("custom insecure", func(t *testing.T) {
config, err := newServiceConfig(ServiceOptions{
InsecureRegistries: []string{"42.42.0.0/16"},
})
assert.NilError(t, err)
for indexName, expected := range expectedIndexInfos {
t.Run(indexName, func(t *testing.T) {
actual := newIndexInfo(config, indexName)
assert.Check(t, is.DeepEqual(actual, expected))
})
}
})
}

View File

@@ -122,7 +122,7 @@ func (s *Service) ResolveAuthConfig(authConfigs map[string]registry.AuthConfig,
if !ok {
registryInfo = &registry.IndexInfo{Name: indexName}
}
return ResolveAuthConfig(authConfigs, registryInfo)
return resolveAuthConfig(authConfigs, registryInfo)
}
// APIEndpoint represents a remote API endpoint