builder: wire up new gc types for buildkit prune functionality

This wires up the new gc types that buildkit exposes in version 0.17.
The previous flag, `KeepBytes`, was renamed to `ReservedBytes` and two
new options, `MaxUsed` and `MinFree` were added.

`MaxUsed` corresponds to the maximum amount of space that buildkit will
use for the build cache and `MinFree` amount of free disk space for the
system to prevent the cache from using that space. This allows greater
configuration of the cache storage usage when used in situations where
docker is not the only service on the system using disk space.

Signed-off-by: Jonathan A. Sternberg <jonathan.sternberg@docker.com>
This commit is contained in:
Jonathan A. Sternberg
2025-01-30 09:54:12 -06:00
parent 1153242d3a
commit 8e529682af
12 changed files with 292 additions and 95 deletions

View File

@@ -177,19 +177,55 @@ func (br *buildRouter) postPrune(ctx context.Context, w http.ResponseWriter, r *
if err != nil {
return err
}
ksfv := r.FormValue("keep-storage")
if ksfv == "" {
ksfv = "0"
}
ks, err := strconv.Atoi(ksfv)
if err != nil {
return invalidParam{errors.Wrapf(err, "keep-storage is in bytes and expects an integer, got %v", ksfv)}
}
opts := types.BuildCachePruneOptions{
All: httputils.BoolValue(r, "all"),
Filters: fltrs,
KeepStorage: int64(ks),
All: httputils.BoolValue(r, "all"),
Filters: fltrs,
}
parseBytesFromFormValue := func(name string) (int64, error) {
if fv := r.FormValue(name); fv != "" {
bs, err := strconv.Atoi(fv)
if err != nil {
return 0, invalidParam{errors.Wrapf(err, "%s is in bytes and expects an integer, got %v", name, fv)}
}
return int64(bs), nil
}
return 0, nil
}
version := httputils.VersionFromContext(ctx)
if versions.GreaterThanOrEqualTo(version, "1.48") {
bs, err := parseBytesFromFormValue("reserved-space")
if err != nil {
return err
} else if bs == 0 {
// Deprecated parameter. Only checked if reserved-space is not used.
bs, err = parseBytesFromFormValue("keep-storage")
if err != nil {
return err
}
}
opts.ReservedSpace = bs
if bs, err := parseBytesFromFormValue("max-used-space"); err != nil {
return err
} else {
opts.MaxUsedSpace = bs
}
if bs, err := parseBytesFromFormValue("min-free-space"); err != nil {
return err
} else {
opts.MinFreeSpace = bs
}
} else {
// Only keep-storage was valid in pre-1.48 versions.
bs, err := parseBytesFromFormValue("keep-storage")
if err != nil {
return err
}
opts.ReservedSpace = bs
}
report, err := br.backend.PruneCache(ctx, opts)

View File

@@ -8994,10 +8994,29 @@ paths:
operationId: "BuildPrune"
parameters:
- name: "keep-storage"
in: "query"
description: |
Amount of disk space in bytes to keep for cache
> **Deprecated**: This parameter is deprecated and has been renamed to "reserved-space".
> It is kept for backward compatibility and will be removed in API v1.49.
type: "integer"
format: "int64"
- name: "reserved-space"
in: "query"
description: "Amount of disk space in bytes to keep for cache"
type: "integer"
format: "int64"
- name: "max-used-space"
in: "query"
description: "Maximum amount of disk space allowed to keep for cache"
type: "integer"
format: "int64"
- name: "min-free-space"
in: "query"
description: "Target amount of free disk space after pruning"
type: "integer"
format: "int64"
- name: "all"
in: "query"
type: "boolean"

View File

@@ -169,9 +169,11 @@ type BuildCache struct {
// BuildCachePruneOptions hold parameters to prune the build cache
type BuildCachePruneOptions struct {
All bool
KeepStorage int64
Filters filters.Args
All bool
ReservedSpace int64
MaxUsedSpace int64
MinFreeSpace int64
Filters filters.Args
// FIXME(thaJeztah): add new options; see https://github.com/moby/moby/issues/48639
KeepStorage int64 // Deprecated: deprecated in API 1.48.
}

View File

@@ -185,8 +185,6 @@ func (b *Builder) DiskUsage(ctx context.Context) ([]*types.BuildCache, error) {
}
// Prune clears all reclaimable build cache.
//
// FIXME(thaJeztah): wire up new options https://github.com/moby/moby/issues/48639
func (b *Builder) Prune(ctx context.Context, opts types.BuildCachePruneOptions) (int64, []string, error) {
ch := make(chan *controlapi.UsageRecord)
@@ -215,6 +213,8 @@ func (b *Builder) Prune(ctx context.Context, opts types.BuildCachePruneOptions)
All: pi.All,
KeepDuration: int64(pi.KeepDuration),
ReservedSpace: pi.ReservedSpace,
MaxUsedSpace: pi.MaxUsedSpace,
MinFreeSpace: pi.MinFreeSpace,
Filter: pi.Filter,
}, &pruneProxy{
streamProxy: streamProxy{ctx: ctx},
@@ -638,7 +638,6 @@ func toBuildkitUlimits(inp []*container.Ulimit) (string, error) {
return strings.Join(ulimits, ","), nil
}
// FIXME(thaJeztah): wire-up new fields; see https://github.com/moby/moby/issues/48639
func toBuildkitPruneInfo(opts types.BuildCachePruneOptions) (client.PruneInfo, error) {
var until time.Duration
untilValues := opts.Filters.Get("until") // canonical
@@ -693,10 +692,17 @@ func toBuildkitPruneInfo(opts types.BuildCachePruneOptions) (client.PruneInfo, e
}
}
}
if opts.ReservedSpace == 0 && opts.KeepStorage != 0 {
opts.ReservedSpace = opts.KeepStorage
}
return client.PruneInfo{
All: opts.All,
KeepDuration: until,
ReservedSpace: opts.KeepStorage,
ReservedSpace: opts.ReservedSpace,
MaxUsedSpace: opts.MaxUsedSpace,
MinFreeSpace: opts.MinFreeSpace,
Filter: []string{strings.Join(bkFilter, ",")},
}, nil
}

View File

@@ -2,10 +2,12 @@ package buildkit
import (
"context"
"fmt"
"net/http"
"os"
"path/filepath"
"runtime"
"strings"
"time"
ctd "github.com/containerd/containerd/v2/client"
@@ -430,37 +432,29 @@ func getGCPolicy(conf config.BuilderConfig, root string) ([]client.PruneInfo, er
var gcPolicy []client.PruneInfo
if conf.GC.Enabled {
if conf.GC.Policy == nil {
var defaultKeepStorage int64
if conf.GC.DefaultKeepStorage != "" {
b, err := units.RAMInBytes(conf.GC.DefaultKeepStorage)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse defaultKeepStorage")
}
defaultKeepStorage = b
reservedSpace, maxUsedSpace, minFreeSpace, err := parseGCPolicy(config.BuilderGCRule{
ReservedSpace: conf.GC.DefaultReservedSpace,
MaxUsedSpace: conf.GC.DefaultMaxUsedSpace,
MinFreeSpace: conf.GC.DefaultMinFreeSpace,
}, "default")
if err != nil {
return nil, err
}
gcPolicy = mobyworker.DefaultGCPolicy(root, defaultKeepStorage)
gcPolicy = mobyworker.DefaultGCPolicy(root, reservedSpace, maxUsedSpace, minFreeSpace)
} else {
gcPolicy = make([]client.PruneInfo, len(conf.GC.Policy))
for i, p := range conf.GC.Policy {
var keepStorage int64
if p.KeepStorage != "" {
b, err := units.RAMInBytes(p.KeepStorage)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse keepStorage")
}
// don't set a default here, zero is a valid value when
// specified by the user, as the gc-policy may be determined
// through other filters;
// https://github.com/moby/moby/pull/49062#issuecomment-2554981829
keepStorage = b
reservedSpace, maxUsedSpace, minFreeSpace, err := parseGCPolicy(p, "")
if err != nil {
return nil, err
}
// FIXME(thaJeztah): wire up new options https://github.com/moby/moby/issues/48639
var err error
gcPolicy[i], err = toBuildkitPruneInfo(types.BuildCachePruneOptions{
All: p.All,
KeepStorage: keepStorage,
Filters: filters.Args(p.Filter),
All: p.All,
ReservedSpace: reservedSpace,
MaxUsedSpace: maxUsedSpace,
MinFreeSpace: minFreeSpace,
Filters: filters.Args(p.Filter),
})
if err != nil {
return nil, err
@@ -471,6 +465,41 @@ func getGCPolicy(conf config.BuilderConfig, root string) ([]client.PruneInfo, er
return gcPolicy, nil
}
func parseGCPolicy(p config.BuilderGCRule, prefix string) (reservedSpace, maxUsedSpace, minFreeSpace int64, err error) {
errorString := func(key string) string {
if prefix != "" {
key = prefix + strings.ToTitle(key)
}
return fmt.Sprintf("failed to parse %s", key)
}
if p.ReservedSpace != "" {
b, err := units.RAMInBytes(p.ReservedSpace)
if err != nil {
return 0, 0, 0, errors.Wrap(err, errorString("reservedSpace"))
}
reservedSpace = b
}
if p.MaxUsedSpace != "" {
b, err := units.RAMInBytes(p.MaxUsedSpace)
if err != nil {
return 0, 0, 0, errors.Wrap(err, errorString("maxUsedSpace"))
}
maxUsedSpace = b
}
if p.MinFreeSpace != "" {
b, err := units.RAMInBytes(p.MinFreeSpace)
if err != nil {
return 0, 0, 0, errors.Wrap(err, errorString("minFreeSpace"))
}
minFreeSpace = b
}
return reservedSpace, maxUsedSpace, minFreeSpace, nil
}
func getEntitlements(conf config.BuilderConfig) []string {
var ents []string
// Incase of no config settings, NetworkHost should be enabled & SecurityInsecure must be disabled.

View File

@@ -5,9 +5,15 @@ import (
"time"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/util/disk"
)
const defaultCap int64 = 2e9 // 2GB
const (
defaultReservedSpaceBytes int64 = 2e9 // 2GB
defaultReservedSpacePercentage int64 = 10
defaultMaxUsedPercentage int64 = 80
defaultMinFreePercentage int64 = 20
)
// tempCachePercent represents the percentage ratio of the cache size in bytes to temporarily keep for a short period of time (couple of days)
// over the total cache size in bytes. Because there is no perfect value, a mathematically pleasing one was chosen.
@@ -15,39 +21,57 @@ const defaultCap int64 = 2e9 // 2GB
const tempCachePercent = math.E * math.Pi * math.Phi
// DefaultGCPolicy returns a default builder GC policy
func DefaultGCPolicy(p string, defaultKeepBytes int64) []client.PruneInfo {
keep := defaultKeepBytes
if defaultKeepBytes == 0 {
keep = detectDefaultGCCap(p)
func DefaultGCPolicy(p string, reservedSpace, maxUsedSpace, minFreeSpace int64) []client.PruneInfo {
if reservedSpace == 0 && maxUsedSpace == 0 && minFreeSpace == 0 {
// Only check the disk if we need to fill in an inferred value.
if dstat, err := disk.GetDiskStat(p); err == nil {
// Fill in default values only if we can read the disk.
reservedSpace = diskPercentage(dstat, defaultReservedSpacePercentage)
maxUsedSpace = diskPercentage(dstat, defaultMaxUsedPercentage)
minFreeSpace = diskPercentage(dstat, defaultMinFreePercentage)
} else {
// Fill in only reserved space if we cannot read the disk.
reservedSpace = defaultReservedSpaceBytes
}
}
tempCacheKeepBytes := int64(math.Round(float64(keep) / 100. * float64(tempCachePercent)))
const minTempCacheKeepBytes = 512 * 1e6 // 512MB
if tempCacheKeepBytes < minTempCacheKeepBytes {
tempCacheKeepBytes = minTempCacheKeepBytes
tempCacheReservedSpace := int64(math.Round(float64(reservedSpace) / 100. * float64(tempCachePercent)))
const minTempCacheReservedSpace = 512 * 1e6 // 512MB
if tempCacheReservedSpace < minTempCacheReservedSpace {
tempCacheReservedSpace = minTempCacheReservedSpace
}
// FIXME(thaJeztah): wire up new options https://github.com/moby/moby/issues/48639
return []client.PruneInfo{
// if build cache uses more than 512MB delete the most easily reproducible data after it has not been used for 2 days
{
Filter: []string{"type==source.local,type==exec.cachemount,type==source.git.checkout"},
KeepDuration: 48 * time.Hour,
ReservedSpace: tempCacheKeepBytes,
Filter: []string{"type==source.local,type==exec.cachemount,type==source.git.checkout"},
KeepDuration: 48 * time.Hour,
MaxUsedSpace: tempCacheReservedSpace,
},
// remove any data not used for 60 days
{
KeepDuration: 60 * 24 * time.Hour,
ReservedSpace: keep,
ReservedSpace: reservedSpace,
MaxUsedSpace: maxUsedSpace,
MinFreeSpace: minFreeSpace,
},
// keep the unshared build cache under cap
{
ReservedSpace: keep,
ReservedSpace: reservedSpace,
MaxUsedSpace: maxUsedSpace,
MinFreeSpace: minFreeSpace,
},
// if previous policies were insufficient start deleting internal data to keep build cache under cap
{
All: true,
ReservedSpace: keep,
ReservedSpace: reservedSpace,
MaxUsedSpace: maxUsedSpace,
MinFreeSpace: minFreeSpace,
},
}
}
func diskPercentage(dstat disk.DiskStat, percentage int64) int64 {
avail := dstat.Total / percentage
return (avail/(1<<30) + 1) * 1e9 // round up
}

View File

@@ -1,17 +0,0 @@
//go:build !windows
package worker
import (
"syscall"
)
func detectDefaultGCCap(root string) int64 {
var st syscall.Statfs_t
if err := syscall.Statfs(root, &st); err != nil {
return defaultCap
}
diskSize := int64(st.Bsize) * int64(st.Blocks) //nolint unconvert
avail := diskSize / 10
return (avail/(1<<30) + 1) * 1e9 // round up
}

View File

@@ -1,7 +0,0 @@
//go:build windows
package worker
func detectDefaultGCCap(root string) int64 {
return defaultCap
}

View File

@@ -21,7 +21,19 @@ func (cli *Client) BuildCachePrune(ctx context.Context, opts types.BuildCachePru
if opts.All {
query.Set("all", "1")
}
query.Set("keep-storage", strconv.Itoa(int(opts.KeepStorage)))
if opts.KeepStorage != 0 {
query.Set("keep-storage", strconv.Itoa(int(opts.KeepStorage)))
}
if opts.ReservedSpace != 0 {
query.Set("reserved-space", strconv.Itoa(int(opts.ReservedSpace)))
}
if opts.MaxUsedSpace != 0 {
query.Set("max-used-space", strconv.Itoa(int(opts.MaxUsedSpace)))
}
if opts.MinFreeSpace != 0 {
query.Set("min-free-space", strconv.Itoa(int(opts.MinFreeSpace)))
}
f, err := filters.ToJSON(opts.Filters)
if err != nil {
return nil, errors.Wrap(err, "prune could not marshal filters option")

View File

@@ -11,9 +11,37 @@ import (
// BuilderGCRule represents a GC rule for buildkit cache
type BuilderGCRule struct {
All bool `json:",omitempty"`
Filter BuilderGCFilter `json:",omitempty"`
KeepStorage string `json:",omitempty"`
All bool `json:",omitempty"`
Filter BuilderGCFilter `json:",omitempty"`
ReservedSpace string `json:",omitempty"`
MaxUsedSpace string `json:",omitempty"`
MinFreeSpace string `json:",omitempty"`
}
func (x *BuilderGCRule) UnmarshalJSON(data []byte) error {
var xx struct {
All bool `json:",omitempty"`
Filter BuilderGCFilter `json:",omitempty"`
ReservedSpace string `json:",omitempty"`
MaxUsedSpace string `json:",omitempty"`
MinFreeSpace string `json:",omitempty"`
// Deprecated option is now equivalent to ReservedSpace.
KeepStorage string `json:",omitempty"`
}
if err := json.Unmarshal(data, &xx); err != nil {
return err
}
x.All = xx.All
x.Filter = xx.Filter
x.ReservedSpace = xx.ReservedSpace
x.MaxUsedSpace = xx.MaxUsedSpace
x.MinFreeSpace = xx.MinFreeSpace
if x.ReservedSpace == "" {
x.ReservedSpace = xx.KeepStorage
}
return nil
}
// BuilderGCFilter contains garbage-collection filter rules for a BuildKit builder
@@ -56,9 +84,38 @@ func (x *BuilderGCFilter) UnmarshalJSON(data []byte) error {
// BuilderGCConfig contains GC config for a buildkit builder
type BuilderGCConfig struct {
Enabled bool `json:",omitempty"`
Policy []BuilderGCRule `json:",omitempty"`
DefaultKeepStorage string `json:",omitempty"`
Enabled bool `json:",omitempty"`
Policy []BuilderGCRule `json:",omitempty"`
DefaultReservedSpace string `json:",omitempty"`
DefaultMaxUsedSpace string `json:",omitempty"`
DefaultMinFreeSpace string `json:",omitempty"`
}
func (x *BuilderGCConfig) UnmarshalJSON(data []byte) error {
var xx struct {
Enabled bool `json:",omitempty"`
Policy []BuilderGCRule `json:",omitempty"`
DefaultReservedSpace string `json:",omitempty"`
DefaultMaxUsedSpace string `json:",omitempty"`
DefaultMinFreeSpace string `json:",omitempty"`
// Deprecated option is now equivalent to DefaultReservedSpace.
DefaultKeepStorage string `json:",omitempty"`
}
if err := json.Unmarshal(data, &xx); err != nil {
return err
}
x.Enabled = xx.Enabled
x.Policy = xx.Policy
x.DefaultReservedSpace = xx.DefaultReservedSpace
x.DefaultMaxUsedSpace = xx.DefaultMaxUsedSpace
x.DefaultMinFreeSpace = xx.DefaultMinFreeSpace
if x.DefaultReservedSpace == "" {
x.DefaultReservedSpace = xx.DefaultKeepStorage
}
return nil
}
// BuilderHistoryConfig contains history config for a buildkit builder

View File

@@ -12,6 +12,40 @@ import (
func TestBuilderGC(t *testing.T) {
tempFile := fs.NewFile(t, "config", fs.WithContent(`{
"builder": {
"gc": {
"enabled": true,
"policy": [
{"reservedSpace": "10GB", "filter": ["unused-for=2200h"]},
{"reservedSpace": "50GB", "filter": {"unused-for": {"3300h": true}}},
{"reservedSpace": "100GB", "minFreeSpace": "10GB", "maxUsedSpace": "200GB", "all": true}
]
}
}
}`))
defer tempFile.Remove()
configFile := tempFile.Path()
cfg, err := MergeDaemonConfigurations(&Config{}, nil, configFile)
assert.NilError(t, err)
assert.Assert(t, cfg.Builder.GC.Enabled)
f1 := filters.NewArgs()
f1.Add("unused-for", "2200h")
f2 := filters.NewArgs()
f2.Add("unused-for", "3300h")
expectedPolicy := []BuilderGCRule{
{ReservedSpace: "10GB", Filter: BuilderGCFilter(f1)},
{ReservedSpace: "50GB", Filter: BuilderGCFilter(f2)}, /* parsed from deprecated form */
{ReservedSpace: "100GB", MinFreeSpace: "10GB", MaxUsedSpace: "200GB", All: true},
}
assert.DeepEqual(t, cfg.Builder.GC.Policy, expectedPolicy, cmp.AllowUnexported(BuilderGCFilter{}))
// double check to please the skeptics
assert.Assert(t, filters.Args(cfg.Builder.GC.Policy[0].Filter).UniqueExactMatch("unused-for", "2200h"))
assert.Assert(t, filters.Args(cfg.Builder.GC.Policy[1].Filter).UniqueExactMatch("unused-for", "3300h"))
}
func TestBuilderGC_DeprecatedKeepStorage(t *testing.T) {
tempFile := fs.NewFile(t, "config", fs.WithContent(`{
"builder": {
"gc": {
"enabled": true,
@@ -34,9 +68,9 @@ func TestBuilderGC(t *testing.T) {
f2 := filters.NewArgs()
f2.Add("unused-for", "3300h")
expectedPolicy := []BuilderGCRule{
{KeepStorage: "10GB", Filter: BuilderGCFilter(f1)},
{KeepStorage: "50GB", Filter: BuilderGCFilter(f2)}, /* parsed from deprecated form */
{KeepStorage: "100GB", All: true},
{ReservedSpace: "10GB", Filter: BuilderGCFilter(f1)},
{ReservedSpace: "50GB", Filter: BuilderGCFilter(f2)}, /* parsed from deprecated form */
{ReservedSpace: "100GB", All: true},
}
assert.DeepEqual(t, cfg.Builder.GC.Policy, expectedPolicy, cmp.AllowUnexported(BuilderGCFilter{}))
// double check to please the skeptics
@@ -49,10 +83,10 @@ func TestBuilderGC(t *testing.T) {
// missing a "=" separator). resulted in a panic during unmarshal.
func TestBuilderGCFilterUnmarshal(t *testing.T) {
var cfg BuilderGCConfig
err := json.Unmarshal([]byte(`{"poliCy": [{"keepStorage": "10GB", "filter": ["unused-for2200h"]}]}`), &cfg)
err := json.Unmarshal([]byte(`{"poliCy": [{"reservedSpace": "10GB", "filter": ["unused-for2200h"]}]}`), &cfg)
assert.Check(t, err)
expectedPolicy := []BuilderGCRule{{
KeepStorage: "10GB", Filter: BuilderGCFilter(filters.NewArgs(filters.Arg("unused-for2200h", ""))),
ReservedSpace: "10GB", Filter: BuilderGCFilter(filters.NewArgs(filters.Arg("unused-for2200h", ""))),
}}
assert.DeepEqual(t, cfg.Policy, expectedPolicy, cmp.AllowUnexported(BuilderGCFilter{}))
}

View File

@@ -71,6 +71,8 @@ keywords: "API, Docker, rcli, REST, documentation"
`GET /debug/pprof/profile`, `GET /debug/pprof/symbol`, `GET /debug/pprof/trace`,
`GET /debug/pprof/{name}`) are now also accessible through the versioned-API
paths (`/v<API-version>/<endpoint>`).
* `POST /build/prune` renames `keep-bytes` to `reserved-space` and now supports
additional prune parameters `max-used-space` and `min-free-space`.
## v1.47 API changes