diff --git a/api/server/router/build/build_routes.go b/api/server/router/build/build_routes.go index 2508a2b3b5..b1b4821f7a 100644 --- a/api/server/router/build/build_routes.go +++ b/api/server/router/build/build_routes.go @@ -177,19 +177,55 @@ func (br *buildRouter) postPrune(ctx context.Context, w http.ResponseWriter, r * if err != nil { return err } - ksfv := r.FormValue("keep-storage") - if ksfv == "" { - ksfv = "0" - } - ks, err := strconv.Atoi(ksfv) - if err != nil { - return invalidParam{errors.Wrapf(err, "keep-storage is in bytes and expects an integer, got %v", ksfv)} - } opts := types.BuildCachePruneOptions{ - All: httputils.BoolValue(r, "all"), - Filters: fltrs, - KeepStorage: int64(ks), + All: httputils.BoolValue(r, "all"), + Filters: fltrs, + } + + parseBytesFromFormValue := func(name string) (int64, error) { + if fv := r.FormValue(name); fv != "" { + bs, err := strconv.Atoi(fv) + if err != nil { + return 0, invalidParam{errors.Wrapf(err, "%s is in bytes and expects an integer, got %v", name, fv)} + } + return int64(bs), nil + } + return 0, nil + } + + version := httputils.VersionFromContext(ctx) + if versions.GreaterThanOrEqualTo(version, "1.48") { + bs, err := parseBytesFromFormValue("reserved-space") + if err != nil { + return err + } else if bs == 0 { + // Deprecated parameter. Only checked if reserved-space is not used. + bs, err = parseBytesFromFormValue("keep-storage") + if err != nil { + return err + } + } + opts.ReservedSpace = bs + + if bs, err := parseBytesFromFormValue("max-used-space"); err != nil { + return err + } else { + opts.MaxUsedSpace = bs + } + + if bs, err := parseBytesFromFormValue("min-free-space"); err != nil { + return err + } else { + opts.MinFreeSpace = bs + } + } else { + // Only keep-storage was valid in pre-1.48 versions. + bs, err := parseBytesFromFormValue("keep-storage") + if err != nil { + return err + } + opts.ReservedSpace = bs } report, err := br.backend.PruneCache(ctx, opts) diff --git a/api/swagger.yaml b/api/swagger.yaml index ee559ff035..4f2192dc22 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -8994,10 +8994,29 @@ paths: operationId: "BuildPrune" parameters: - name: "keep-storage" + in: "query" + description: | + Amount of disk space in bytes to keep for cache + + > **Deprecated**: This parameter is deprecated and has been renamed to "reserved-space". + > It is kept for backward compatibility and will be removed in API v1.49. + type: "integer" + format: "int64" + - name: "reserved-space" in: "query" description: "Amount of disk space in bytes to keep for cache" type: "integer" format: "int64" + - name: "max-used-space" + in: "query" + description: "Maximum amount of disk space allowed to keep for cache" + type: "integer" + format: "int64" + - name: "min-free-space" + in: "query" + description: "Target amount of free disk space after pruning" + type: "integer" + format: "int64" - name: "all" in: "query" type: "boolean" diff --git a/api/types/types.go b/api/types/types.go index eb6831c5f3..82ae339c31 100644 --- a/api/types/types.go +++ b/api/types/types.go @@ -169,9 +169,11 @@ type BuildCache struct { // BuildCachePruneOptions hold parameters to prune the build cache type BuildCachePruneOptions struct { - All bool - KeepStorage int64 - Filters filters.Args + All bool + ReservedSpace int64 + MaxUsedSpace int64 + MinFreeSpace int64 + Filters filters.Args - // FIXME(thaJeztah): add new options; see https://github.com/moby/moby/issues/48639 + KeepStorage int64 // Deprecated: deprecated in API 1.48. } diff --git a/builder/builder-next/builder.go b/builder/builder-next/builder.go index e147340acb..d77b87c8e1 100644 --- a/builder/builder-next/builder.go +++ b/builder/builder-next/builder.go @@ -185,8 +185,6 @@ func (b *Builder) DiskUsage(ctx context.Context) ([]*types.BuildCache, error) { } // Prune clears all reclaimable build cache. -// -// FIXME(thaJeztah): wire up new options https://github.com/moby/moby/issues/48639 func (b *Builder) Prune(ctx context.Context, opts types.BuildCachePruneOptions) (int64, []string, error) { ch := make(chan *controlapi.UsageRecord) @@ -215,6 +213,8 @@ func (b *Builder) Prune(ctx context.Context, opts types.BuildCachePruneOptions) All: pi.All, KeepDuration: int64(pi.KeepDuration), ReservedSpace: pi.ReservedSpace, + MaxUsedSpace: pi.MaxUsedSpace, + MinFreeSpace: pi.MinFreeSpace, Filter: pi.Filter, }, &pruneProxy{ streamProxy: streamProxy{ctx: ctx}, @@ -638,7 +638,6 @@ func toBuildkitUlimits(inp []*container.Ulimit) (string, error) { return strings.Join(ulimits, ","), nil } -// FIXME(thaJeztah): wire-up new fields; see https://github.com/moby/moby/issues/48639 func toBuildkitPruneInfo(opts types.BuildCachePruneOptions) (client.PruneInfo, error) { var until time.Duration untilValues := opts.Filters.Get("until") // canonical @@ -693,10 +692,17 @@ func toBuildkitPruneInfo(opts types.BuildCachePruneOptions) (client.PruneInfo, e } } } + + if opts.ReservedSpace == 0 && opts.KeepStorage != 0 { + opts.ReservedSpace = opts.KeepStorage + } + return client.PruneInfo{ All: opts.All, KeepDuration: until, - ReservedSpace: opts.KeepStorage, + ReservedSpace: opts.ReservedSpace, + MaxUsedSpace: opts.MaxUsedSpace, + MinFreeSpace: opts.MinFreeSpace, Filter: []string{strings.Join(bkFilter, ",")}, }, nil } diff --git a/builder/builder-next/controller.go b/builder/builder-next/controller.go index 3a3fdaecd6..103df435d3 100644 --- a/builder/builder-next/controller.go +++ b/builder/builder-next/controller.go @@ -2,10 +2,12 @@ package buildkit import ( "context" + "fmt" "net/http" "os" "path/filepath" "runtime" + "strings" "time" ctd "github.com/containerd/containerd/v2/client" @@ -430,37 +432,29 @@ func getGCPolicy(conf config.BuilderConfig, root string) ([]client.PruneInfo, er var gcPolicy []client.PruneInfo if conf.GC.Enabled { if conf.GC.Policy == nil { - var defaultKeepStorage int64 - if conf.GC.DefaultKeepStorage != "" { - b, err := units.RAMInBytes(conf.GC.DefaultKeepStorage) - if err != nil { - return nil, errors.Wrapf(err, "failed to parse defaultKeepStorage") - } - defaultKeepStorage = b + reservedSpace, maxUsedSpace, minFreeSpace, err := parseGCPolicy(config.BuilderGCRule{ + ReservedSpace: conf.GC.DefaultReservedSpace, + MaxUsedSpace: conf.GC.DefaultMaxUsedSpace, + MinFreeSpace: conf.GC.DefaultMinFreeSpace, + }, "default") + if err != nil { + return nil, err } - gcPolicy = mobyworker.DefaultGCPolicy(root, defaultKeepStorage) + gcPolicy = mobyworker.DefaultGCPolicy(root, reservedSpace, maxUsedSpace, minFreeSpace) } else { gcPolicy = make([]client.PruneInfo, len(conf.GC.Policy)) for i, p := range conf.GC.Policy { - var keepStorage int64 - if p.KeepStorage != "" { - b, err := units.RAMInBytes(p.KeepStorage) - if err != nil { - return nil, errors.Wrapf(err, "failed to parse keepStorage") - } - // don't set a default here, zero is a valid value when - // specified by the user, as the gc-policy may be determined - // through other filters; - // https://github.com/moby/moby/pull/49062#issuecomment-2554981829 - keepStorage = b + reservedSpace, maxUsedSpace, minFreeSpace, err := parseGCPolicy(p, "") + if err != nil { + return nil, err } - // FIXME(thaJeztah): wire up new options https://github.com/moby/moby/issues/48639 - var err error gcPolicy[i], err = toBuildkitPruneInfo(types.BuildCachePruneOptions{ - All: p.All, - KeepStorage: keepStorage, - Filters: filters.Args(p.Filter), + All: p.All, + ReservedSpace: reservedSpace, + MaxUsedSpace: maxUsedSpace, + MinFreeSpace: minFreeSpace, + Filters: filters.Args(p.Filter), }) if err != nil { return nil, err @@ -471,6 +465,41 @@ func getGCPolicy(conf config.BuilderConfig, root string) ([]client.PruneInfo, er return gcPolicy, nil } +func parseGCPolicy(p config.BuilderGCRule, prefix string) (reservedSpace, maxUsedSpace, minFreeSpace int64, err error) { + errorString := func(key string) string { + if prefix != "" { + key = prefix + strings.ToTitle(key) + } + return fmt.Sprintf("failed to parse %s", key) + } + + if p.ReservedSpace != "" { + b, err := units.RAMInBytes(p.ReservedSpace) + if err != nil { + return 0, 0, 0, errors.Wrap(err, errorString("reservedSpace")) + } + reservedSpace = b + } + + if p.MaxUsedSpace != "" { + b, err := units.RAMInBytes(p.MaxUsedSpace) + if err != nil { + return 0, 0, 0, errors.Wrap(err, errorString("maxUsedSpace")) + } + maxUsedSpace = b + } + + if p.MinFreeSpace != "" { + b, err := units.RAMInBytes(p.MinFreeSpace) + if err != nil { + return 0, 0, 0, errors.Wrap(err, errorString("minFreeSpace")) + } + minFreeSpace = b + } + + return reservedSpace, maxUsedSpace, minFreeSpace, nil +} + func getEntitlements(conf config.BuilderConfig) []string { var ents []string // Incase of no config settings, NetworkHost should be enabled & SecurityInsecure must be disabled. diff --git a/builder/builder-next/worker/gc.go b/builder/builder-next/worker/gc.go index d05c637a5c..49b7ce6f04 100644 --- a/builder/builder-next/worker/gc.go +++ b/builder/builder-next/worker/gc.go @@ -5,9 +5,15 @@ import ( "time" "github.com/moby/buildkit/client" + "github.com/moby/buildkit/util/disk" ) -const defaultCap int64 = 2e9 // 2GB +const ( + defaultReservedSpaceBytes int64 = 2e9 // 2GB + defaultReservedSpacePercentage int64 = 10 + defaultMaxUsedPercentage int64 = 80 + defaultMinFreePercentage int64 = 20 +) // tempCachePercent represents the percentage ratio of the cache size in bytes to temporarily keep for a short period of time (couple of days) // over the total cache size in bytes. Because there is no perfect value, a mathematically pleasing one was chosen. @@ -15,39 +21,57 @@ const defaultCap int64 = 2e9 // 2GB const tempCachePercent = math.E * math.Pi * math.Phi // DefaultGCPolicy returns a default builder GC policy -func DefaultGCPolicy(p string, defaultKeepBytes int64) []client.PruneInfo { - keep := defaultKeepBytes - if defaultKeepBytes == 0 { - keep = detectDefaultGCCap(p) +func DefaultGCPolicy(p string, reservedSpace, maxUsedSpace, minFreeSpace int64) []client.PruneInfo { + if reservedSpace == 0 && maxUsedSpace == 0 && minFreeSpace == 0 { + // Only check the disk if we need to fill in an inferred value. + if dstat, err := disk.GetDiskStat(p); err == nil { + // Fill in default values only if we can read the disk. + reservedSpace = diskPercentage(dstat, defaultReservedSpacePercentage) + maxUsedSpace = diskPercentage(dstat, defaultMaxUsedPercentage) + minFreeSpace = diskPercentage(dstat, defaultMinFreePercentage) + } else { + // Fill in only reserved space if we cannot read the disk. + reservedSpace = defaultReservedSpaceBytes + } } - tempCacheKeepBytes := int64(math.Round(float64(keep) / 100. * float64(tempCachePercent))) - const minTempCacheKeepBytes = 512 * 1e6 // 512MB - if tempCacheKeepBytes < minTempCacheKeepBytes { - tempCacheKeepBytes = minTempCacheKeepBytes + tempCacheReservedSpace := int64(math.Round(float64(reservedSpace) / 100. * float64(tempCachePercent))) + const minTempCacheReservedSpace = 512 * 1e6 // 512MB + if tempCacheReservedSpace < minTempCacheReservedSpace { + tempCacheReservedSpace = minTempCacheReservedSpace } - // FIXME(thaJeztah): wire up new options https://github.com/moby/moby/issues/48639 return []client.PruneInfo{ // if build cache uses more than 512MB delete the most easily reproducible data after it has not been used for 2 days { - Filter: []string{"type==source.local,type==exec.cachemount,type==source.git.checkout"}, - KeepDuration: 48 * time.Hour, - ReservedSpace: tempCacheKeepBytes, + Filter: []string{"type==source.local,type==exec.cachemount,type==source.git.checkout"}, + KeepDuration: 48 * time.Hour, + MaxUsedSpace: tempCacheReservedSpace, }, // remove any data not used for 60 days { KeepDuration: 60 * 24 * time.Hour, - ReservedSpace: keep, + ReservedSpace: reservedSpace, + MaxUsedSpace: maxUsedSpace, + MinFreeSpace: minFreeSpace, }, // keep the unshared build cache under cap { - ReservedSpace: keep, + ReservedSpace: reservedSpace, + MaxUsedSpace: maxUsedSpace, + MinFreeSpace: minFreeSpace, }, // if previous policies were insufficient start deleting internal data to keep build cache under cap { All: true, - ReservedSpace: keep, + ReservedSpace: reservedSpace, + MaxUsedSpace: maxUsedSpace, + MinFreeSpace: minFreeSpace, }, } } + +func diskPercentage(dstat disk.DiskStat, percentage int64) int64 { + avail := dstat.Total / percentage + return (avail/(1<<30) + 1) * 1e9 // round up +} diff --git a/builder/builder-next/worker/gc_unix.go b/builder/builder-next/worker/gc_unix.go deleted file mode 100644 index 41a2c181b6..0000000000 --- a/builder/builder-next/worker/gc_unix.go +++ /dev/null @@ -1,17 +0,0 @@ -//go:build !windows - -package worker - -import ( - "syscall" -) - -func detectDefaultGCCap(root string) int64 { - var st syscall.Statfs_t - if err := syscall.Statfs(root, &st); err != nil { - return defaultCap - } - diskSize := int64(st.Bsize) * int64(st.Blocks) //nolint unconvert - avail := diskSize / 10 - return (avail/(1<<30) + 1) * 1e9 // round up -} diff --git a/builder/builder-next/worker/gc_windows.go b/builder/builder-next/worker/gc_windows.go deleted file mode 100644 index 3141c9ee18..0000000000 --- a/builder/builder-next/worker/gc_windows.go +++ /dev/null @@ -1,7 +0,0 @@ -//go:build windows - -package worker - -func detectDefaultGCCap(root string) int64 { - return defaultCap -} diff --git a/client/build_prune.go b/client/build_prune.go index f732852964..9a99d097f4 100644 --- a/client/build_prune.go +++ b/client/build_prune.go @@ -21,7 +21,19 @@ func (cli *Client) BuildCachePrune(ctx context.Context, opts types.BuildCachePru if opts.All { query.Set("all", "1") } - query.Set("keep-storage", strconv.Itoa(int(opts.KeepStorage))) + + if opts.KeepStorage != 0 { + query.Set("keep-storage", strconv.Itoa(int(opts.KeepStorage))) + } + if opts.ReservedSpace != 0 { + query.Set("reserved-space", strconv.Itoa(int(opts.ReservedSpace))) + } + if opts.MaxUsedSpace != 0 { + query.Set("max-used-space", strconv.Itoa(int(opts.MaxUsedSpace))) + } + if opts.MinFreeSpace != 0 { + query.Set("min-free-space", strconv.Itoa(int(opts.MinFreeSpace))) + } f, err := filters.ToJSON(opts.Filters) if err != nil { return nil, errors.Wrap(err, "prune could not marshal filters option") diff --git a/daemon/config/builder.go b/daemon/config/builder.go index 8801ba20cb..457f89efc3 100644 --- a/daemon/config/builder.go +++ b/daemon/config/builder.go @@ -11,9 +11,37 @@ import ( // BuilderGCRule represents a GC rule for buildkit cache type BuilderGCRule struct { - All bool `json:",omitempty"` - Filter BuilderGCFilter `json:",omitempty"` - KeepStorage string `json:",omitempty"` + All bool `json:",omitempty"` + Filter BuilderGCFilter `json:",omitempty"` + ReservedSpace string `json:",omitempty"` + MaxUsedSpace string `json:",omitempty"` + MinFreeSpace string `json:",omitempty"` +} + +func (x *BuilderGCRule) UnmarshalJSON(data []byte) error { + var xx struct { + All bool `json:",omitempty"` + Filter BuilderGCFilter `json:",omitempty"` + ReservedSpace string `json:",omitempty"` + MaxUsedSpace string `json:",omitempty"` + MinFreeSpace string `json:",omitempty"` + + // Deprecated option is now equivalent to ReservedSpace. + KeepStorage string `json:",omitempty"` + } + if err := json.Unmarshal(data, &xx); err != nil { + return err + } + + x.All = xx.All + x.Filter = xx.Filter + x.ReservedSpace = xx.ReservedSpace + x.MaxUsedSpace = xx.MaxUsedSpace + x.MinFreeSpace = xx.MinFreeSpace + if x.ReservedSpace == "" { + x.ReservedSpace = xx.KeepStorage + } + return nil } // BuilderGCFilter contains garbage-collection filter rules for a BuildKit builder @@ -56,9 +84,38 @@ func (x *BuilderGCFilter) UnmarshalJSON(data []byte) error { // BuilderGCConfig contains GC config for a buildkit builder type BuilderGCConfig struct { - Enabled bool `json:",omitempty"` - Policy []BuilderGCRule `json:",omitempty"` - DefaultKeepStorage string `json:",omitempty"` + Enabled bool `json:",omitempty"` + Policy []BuilderGCRule `json:",omitempty"` + DefaultReservedSpace string `json:",omitempty"` + DefaultMaxUsedSpace string `json:",omitempty"` + DefaultMinFreeSpace string `json:",omitempty"` +} + +func (x *BuilderGCConfig) UnmarshalJSON(data []byte) error { + var xx struct { + Enabled bool `json:",omitempty"` + Policy []BuilderGCRule `json:",omitempty"` + DefaultReservedSpace string `json:",omitempty"` + DefaultMaxUsedSpace string `json:",omitempty"` + DefaultMinFreeSpace string `json:",omitempty"` + + // Deprecated option is now equivalent to DefaultReservedSpace. + DefaultKeepStorage string `json:",omitempty"` + } + + if err := json.Unmarshal(data, &xx); err != nil { + return err + } + + x.Enabled = xx.Enabled + x.Policy = xx.Policy + x.DefaultReservedSpace = xx.DefaultReservedSpace + x.DefaultMaxUsedSpace = xx.DefaultMaxUsedSpace + x.DefaultMinFreeSpace = xx.DefaultMinFreeSpace + if x.DefaultReservedSpace == "" { + x.DefaultReservedSpace = xx.DefaultKeepStorage + } + return nil } // BuilderHistoryConfig contains history config for a buildkit builder diff --git a/daemon/config/builder_test.go b/daemon/config/builder_test.go index 0cb08619e1..eb74269257 100644 --- a/daemon/config/builder_test.go +++ b/daemon/config/builder_test.go @@ -12,6 +12,40 @@ import ( func TestBuilderGC(t *testing.T) { tempFile := fs.NewFile(t, "config", fs.WithContent(`{ + "builder": { + "gc": { + "enabled": true, + "policy": [ + {"reservedSpace": "10GB", "filter": ["unused-for=2200h"]}, + {"reservedSpace": "50GB", "filter": {"unused-for": {"3300h": true}}}, + {"reservedSpace": "100GB", "minFreeSpace": "10GB", "maxUsedSpace": "200GB", "all": true} + ] + } + } +}`)) + defer tempFile.Remove() + configFile := tempFile.Path() + + cfg, err := MergeDaemonConfigurations(&Config{}, nil, configFile) + assert.NilError(t, err) + assert.Assert(t, cfg.Builder.GC.Enabled) + f1 := filters.NewArgs() + f1.Add("unused-for", "2200h") + f2 := filters.NewArgs() + f2.Add("unused-for", "3300h") + expectedPolicy := []BuilderGCRule{ + {ReservedSpace: "10GB", Filter: BuilderGCFilter(f1)}, + {ReservedSpace: "50GB", Filter: BuilderGCFilter(f2)}, /* parsed from deprecated form */ + {ReservedSpace: "100GB", MinFreeSpace: "10GB", MaxUsedSpace: "200GB", All: true}, + } + assert.DeepEqual(t, cfg.Builder.GC.Policy, expectedPolicy, cmp.AllowUnexported(BuilderGCFilter{})) + // double check to please the skeptics + assert.Assert(t, filters.Args(cfg.Builder.GC.Policy[0].Filter).UniqueExactMatch("unused-for", "2200h")) + assert.Assert(t, filters.Args(cfg.Builder.GC.Policy[1].Filter).UniqueExactMatch("unused-for", "3300h")) +} + +func TestBuilderGC_DeprecatedKeepStorage(t *testing.T) { + tempFile := fs.NewFile(t, "config", fs.WithContent(`{ "builder": { "gc": { "enabled": true, @@ -34,9 +68,9 @@ func TestBuilderGC(t *testing.T) { f2 := filters.NewArgs() f2.Add("unused-for", "3300h") expectedPolicy := []BuilderGCRule{ - {KeepStorage: "10GB", Filter: BuilderGCFilter(f1)}, - {KeepStorage: "50GB", Filter: BuilderGCFilter(f2)}, /* parsed from deprecated form */ - {KeepStorage: "100GB", All: true}, + {ReservedSpace: "10GB", Filter: BuilderGCFilter(f1)}, + {ReservedSpace: "50GB", Filter: BuilderGCFilter(f2)}, /* parsed from deprecated form */ + {ReservedSpace: "100GB", All: true}, } assert.DeepEqual(t, cfg.Builder.GC.Policy, expectedPolicy, cmp.AllowUnexported(BuilderGCFilter{})) // double check to please the skeptics @@ -49,10 +83,10 @@ func TestBuilderGC(t *testing.T) { // missing a "=" separator). resulted in a panic during unmarshal. func TestBuilderGCFilterUnmarshal(t *testing.T) { var cfg BuilderGCConfig - err := json.Unmarshal([]byte(`{"poliCy": [{"keepStorage": "10GB", "filter": ["unused-for2200h"]}]}`), &cfg) + err := json.Unmarshal([]byte(`{"poliCy": [{"reservedSpace": "10GB", "filter": ["unused-for2200h"]}]}`), &cfg) assert.Check(t, err) expectedPolicy := []BuilderGCRule{{ - KeepStorage: "10GB", Filter: BuilderGCFilter(filters.NewArgs(filters.Arg("unused-for2200h", ""))), + ReservedSpace: "10GB", Filter: BuilderGCFilter(filters.NewArgs(filters.Arg("unused-for2200h", ""))), }} assert.DeepEqual(t, cfg.Policy, expectedPolicy, cmp.AllowUnexported(BuilderGCFilter{})) } diff --git a/docs/api/version-history.md b/docs/api/version-history.md index b0b02482a6..7f7aa13c58 100644 --- a/docs/api/version-history.md +++ b/docs/api/version-history.md @@ -71,6 +71,8 @@ keywords: "API, Docker, rcli, REST, documentation" `GET /debug/pprof/profile`, `GET /debug/pprof/symbol`, `GET /debug/pprof/trace`, `GET /debug/pprof/{name}`) are now also accessible through the versioned-API paths (`/v/`). +* `POST /build/prune` renames `keep-bytes` to `reserved-space` and now supports + additional prune parameters `max-used-space` and `min-free-space`. ## v1.47 API changes