builder-next: add buildkit executor for wcow

WCOW support on Buildkit is now coming to maturity. As part
of making this generally available, integrating it in
Docker Engine is critical for it's adoption.

This commit adds the buildkit execuitor for WCOW as the
next-builder (backend) for building Windows containers.

This will be an opt-in feature, with the end users setting
DOCKER_BUILDKIT=1 environment variable to use it.

The integration tests bit has also been handled.
https://github.com/moby/buildkit/pull/5956,
BUILDKIT_REF has been set to `master` for now, so
that the tests can run successfully. On the next
release, we will revert this back to using releases.

Signed-off-by: Anthony Nandaa <profnandaa@gmail.com>
This commit is contained in:
Anthony Nandaa
2025-04-15 10:31:03 +03:00
parent 7937f0846c
commit a9ec07a005
7 changed files with 476 additions and 113 deletions

View File

@@ -32,7 +32,7 @@ jobs:
validate-dco:
uses: ./.github/workflows/.dco.yml
build:
build-linux:
runs-on: ubuntu-24.04
timeout-minutes: 120 # guardrails timeout for the whole job
needs:
@@ -59,11 +59,11 @@ jobs:
if-no-files-found: error
retention-days: 1
test:
test-linux:
runs-on: ubuntu-24.04
timeout-minutes: 120 # guardrails timeout for the whole job
needs:
- build
- build-linux
env:
TEST_IMAGE_BUILD: "0"
TEST_IMAGE_ID: "buildkit-tests"
@@ -162,3 +162,213 @@ jobs:
TESTPKGS: "./${{ matrix.pkg }}"
TESTFLAGS: "-v --parallel=1 --timeout=30m --run=//worker=${{ matrix.worker }}$"
working-directory: buildkit
build-windows:
runs-on: windows-2022
timeout-minutes: 120
needs:
- validate-dco
env:
GOPATH: ${{ github.workspace }}\go
GOBIN: ${{ github.workspace }}\go\bin
BIN_OUT: ${{ github.workspace }}\out
WINDOWS_BASE_IMAGE: mcr.microsoft.com/windows/servercore
WINDOWS_BASE_TAG_2022: ltsc2022
TEST_IMAGE_NAME: moby:test
TEST_CTN_NAME: moby
defaults:
run:
working-directory: ${{ env.GOPATH }}/src/github.com/docker/docker
steps:
- name: Checkout
uses: actions/checkout@v4
with:
path: ${{ env.GOPATH }}/src/github.com/docker/docker
- name: Env
run: |
Get-ChildItem Env: | Out-String
- name: Moby - Init
run: |
New-Item -ItemType "directory" -Path "${{ github.workspace }}\go-build"
New-Item -ItemType "directory" -Path "${{ github.workspace }}\go\pkg\mod"
echo "WINDOWS_BASE_IMAGE_TAG=${{ env.WINDOWS_BASE_TAG_2022 }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache-dependency-path: vendor.sum
- name: Cache
uses: actions/cache@v4
with:
path: |
~\AppData\Local\go-build
~\go\pkg\mod
${{ github.workspace }}\go-build
${{ env.GOPATH }}\pkg\mod
key: ${{ inputs.os }}-${{ github.job }}-${{ hashFiles('**/vendor.sum') }}
restore-keys: |
${{ inputs.os }}-${{ github.job }}-
- name: Docker info
run: |
docker info
- name: Build base image
run: |
& docker build `
--build-arg WINDOWS_BASE_IMAGE `
--build-arg WINDOWS_BASE_IMAGE_TAG `
--build-arg GO_VERSION `
-t ${{ env.TEST_IMAGE_NAME }} `
-f Dockerfile.windows .
- name: Build binaries
run: |
& docker run --name ${{ env.TEST_CTN_NAME }} -e "DOCKER_GITCOMMIT=${{ github.sha }}" `
-v "${{ github.workspace }}\go-build:C:\Users\ContainerAdministrator\AppData\Local\go-build" `
-v "${{ github.workspace }}\go\pkg\mod:C:\gopath\pkg\mod" `
${{ env.TEST_IMAGE_NAME }} hack\make.ps1 -Daemon -Client
go install github.com/distribution/distribution/v3/cmd/registry@latest
- name: Checkout BuildKit
uses: actions/checkout@v4
with:
repository: moby/buildkit
ref: master
path: buildkit
- name: Add buildctl to binaries
run: |
go install ./cmd/buildctl
working-directory: buildkit
- name: Copy artifacts
run: |
New-Item -ItemType "directory" -Path "${{ env.BIN_OUT }}"
docker cp "${{ env.TEST_CTN_NAME }}`:c`:\gopath\src\github.com\docker\docker\bundles\docker.exe" ${{ env.BIN_OUT }}\
docker cp "${{ env.TEST_CTN_NAME }}`:c`:\gopath\src\github.com\docker\docker\bundles\dockerd.exe" ${{ env.BIN_OUT }}\
docker cp "${{ env.TEST_CTN_NAME }}`:c`:\gopath\bin\gotestsum.exe" ${{ env.BIN_OUT }}\
docker cp "${{ env.TEST_CTN_NAME }}`:c`:\containerd\bin\containerd.exe" ${{ env.BIN_OUT }}\
docker cp "${{ env.TEST_CTN_NAME }}`:c`:\containerd\bin\containerd-shim-runhcs-v1.exe" ${{ env.BIN_OUT }}\
cp ${{ env.GOPATH }}\bin\registry.exe ${{ env.BIN_OUT }}
cp ${{ env.GOPATH }}\bin\buildctl.exe ${{ env.BIN_OUT }}
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: build-windows
path: ${{ env.BIN_OUT }}/*
if-no-files-found: error
retention-days: 2
test-windows:
runs-on: windows-2022
timeout-minutes: 120 # guardrails timeout for the whole job
needs:
- build-windows
env:
TEST_IMAGE_BUILD: "0"
TEST_IMAGE_ID: "buildkit-tests"
GOPATH: ${{ github.workspace }}\go
GOBIN: ${{ github.workspace }}\go\bin
BIN_OUT: ${{ github.workspace }}\out
TESTFLAGS: "-v --timeout=90m"
TEST_DOCKERD: "1"
strategy:
fail-fast: false
matrix:
worker:
- dockerd-containerd
pkg:
- ./client#1-4
- ./client#2-4
- ./client#3-4
- ./client#4-4
- ./cmd/buildctl
- ./frontend
- ./frontend/dockerfile#1-12
- ./frontend/dockerfile#2-12
- ./frontend/dockerfile#3-12
- ./frontend/dockerfile#4-12
- ./frontend/dockerfile#5-12
- ./frontend/dockerfile#6-12
- ./frontend/dockerfile#7-12
- ./frontend/dockerfile#8-12
- ./frontend/dockerfile#9-12
- ./frontend/dockerfile#10-12
- ./frontend/dockerfile#11-12
- ./frontend/dockerfile#12-12
steps:
- name: Prepare
shell: bash
run: |
disabledFeatures="cache_backend_azblob,cache_backend_s3"
if [ "${{ matrix.worker }}" = "dockerd" ]; then
disabledFeatures="${disabledFeatures},merge_diff"
fi
echo "BUILDKIT_TEST_DISABLE_FEATURES=${disabledFeatures}" >> $GITHUB_ENV
- name: Expose GitHub Runtime
uses: crazy-max/ghaction-github-runtime@v3
- name: Checkout
uses: actions/checkout@v4
with:
path: moby
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION }}
cache-dependency-path: vendor.sum
- name: BuildKit ref
shell: bash
run: |
echo "$(./hack/buildkit-ref)" >> $GITHUB_ENV
working-directory: moby
- name: Checkout BuildKit ${{ env.BUILDKIT_REF }}
uses: actions/checkout@v4
with:
repository: ${{ env.BUILDKIT_REPO }}
ref: ${{ env.BUILDKIT_REF }}
path: buildkit
- name: Download Moby artifacts
uses: actions/download-artifact@v4
with:
name: build-windows
path: ${{ env.BIN_OUT }}
- name: Add binaries to PATH
run: |
ls ${{ env.BIN_OUT }}
Write-Output "${{ env.BIN_OUT }}" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
- name: Test Prep
shell: bash
run: |
TESTPKG=$(echo "${{ matrix.pkg }}" | awk '-F#' '{print $1}')
echo "TESTPKG=$TESTPKG" >> $GITHUB_ENV
echo "TEST_REPORT_NAME=${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
testFlags="${{ env.TESTFLAGS }}"
testSlice=$(echo "${{ matrix.pkg }}" | awk '-F#' '{print $2}')
testSliceOffset=""
if [ -n "$testSlice" ]; then
testSliceOffset="slice=$testSlice/"
fi
if [ -n "${{ matrix.worker }}" ]; then
testFlags="${testFlags} --run=TestIntegration/$testSliceOffset.*/worker=${{ matrix.worker }}"
fi
echo "TESTFLAGS=${testFlags}" >> $GITHUB_ENV
- name: Test
shell: bash
run: |
mkdir -p ./bin/testreports
gotestsum \
--jsonfile="./bin/testreports/go-test-report-${{ env.TEST_REPORT_NAME }}.json" \
--junitfile="./bin/testreports/junit-report-${{ env.TEST_REPORT_NAME }}.xml" \
--packages="${{ env.TESTPKG }}" \
-- \
"-mod=vendor" \
"-coverprofile" "./bin/testreports/coverage-${{ env.TEST_REPORT_NAME }}.txt" \
"-covermode" "atomic" ${{ env.TESTFLAGS }}
working-directory: buildkit

View File

@@ -152,7 +152,18 @@ func newSnapshotterController(ctx context.Context, rt http.RoundTripper, opt Opt
wo.RegistryHosts = opt.RegistryHosts
wo.Labels = getLabels(opt, wo.Labels)
exec, err := newExecutor(opt.Root, opt.DefaultCgroupParent, opt.NetworkController, dns, opt.Rootless, opt.IdentityMapping, opt.ApparmorProfile, cdiManager)
exec, err := newExecutor(
opt.Root,
opt.DefaultCgroupParent,
opt.NetworkController,
dns,
opt.Rootless,
opt.IdentityMapping,
opt.ApparmorProfile,
cdiManager,
opt.ContainerdAddress,
opt.ContainerdNamespace,
)
if err != nil {
return nil, err
}
@@ -335,7 +346,18 @@ func newGraphDriverController(ctx context.Context, rt http.RoundTripper, opt Opt
return nil, err
}
exec, err := newExecutor(root, opt.DefaultCgroupParent, opt.NetworkController, dns, opt.Rootless, opt.IdentityMapping, opt.ApparmorProfile, cdiManager)
exec, err := newExecutorGD(
root,
opt.DefaultCgroupParent,
opt.NetworkController,
dns,
opt.Rootless,
opt.IdentityMapping,
opt.ApparmorProfile,
cdiManager,
opt.ContainerdAddress,
opt.ContainerdNamespace,
)
if err != nil {
return nil, err
}

View File

@@ -0,0 +1,119 @@
package buildkit
import (
"context"
"net"
"os"
"path/filepath"
"sync"
"github.com/containerd/log"
"github.com/docker/docker/daemon/config"
"github.com/docker/docker/libnetwork"
"github.com/moby/buildkit/executor/oci"
resourcestypes "github.com/moby/buildkit/executor/resources/types"
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/util/network"
)
type bridgeProvider struct {
*libnetwork.Controller
Root string
}
type lnInterface struct {
ep *libnetwork.Endpoint
sbx *libnetwork.Sandbox
sync.Once
err error
ready chan struct{}
provider *bridgeProvider
}
func (p *bridgeProvider) New(_ context.Context, _ string) (network.Namespace, error) {
n, err := p.NetworkByName(networkName)
if err != nil {
return nil, err
}
iface := &lnInterface{ready: make(chan struct{}), provider: p}
iface.Once.Do(func() {
go iface.init(p.Controller, n)
})
return iface, nil
}
func (p *bridgeProvider) Close() error {
return nil
}
func (iface *lnInterface) init(c *libnetwork.Controller, n *libnetwork.Network) {
defer close(iface.ready)
id := identity.NewID()
ep, err := n.CreateEndpoint(context.TODO(), id, libnetwork.CreateOptionDisableResolution())
if err != nil {
iface.err = err
return
}
sbx, err := c.NewSandbox(
context.TODO(),
id,
libnetwork.OptionUseExternalKey(),
libnetwork.OptionHostsPath(filepath.Join(iface.provider.Root, id, "hosts")),
libnetwork.OptionResolvConfPath(filepath.Join(iface.provider.Root, id, "resolv.conf")),
)
if err != nil {
iface.err = err
return
}
if err := ep.Join(context.TODO(), sbx); err != nil {
iface.err = err
return
}
iface.sbx = sbx
iface.ep = ep
}
// TODO(neersighted): Unstub Sample(), and collect data from the libnetwork Endpoint.
func (iface *lnInterface) Sample() (*resourcestypes.NetworkSample, error) {
return &resourcestypes.NetworkSample{}, nil
}
func (iface *lnInterface) Close() error {
<-iface.ready
if iface.sbx != nil {
go func() {
if err := iface.sbx.Delete(context.TODO()); err != nil {
log.G(context.TODO()).WithError(err).Errorf("failed to delete builder network sandbox")
}
if err := os.RemoveAll(filepath.Join(iface.provider.Root, iface.sbx.ContainerID())); err != nil {
log.G(context.TODO()).WithError(err).Errorf("failed to delete builder sandbox directory")
}
}()
}
return iface.err
}
func getDNSConfig(cfg config.DNSConfig) *oci.DNSConfig {
if cfg.DNS != nil || cfg.DNSSearch != nil || cfg.DNSOptions != nil {
return &oci.DNSConfig{
Nameservers: ipAddresses(cfg.DNS),
SearchDomains: cfg.DNSSearch,
Options: cfg.DNSOptions,
}
}
return nil
}
func ipAddresses(ips []net.IP) []string {
var addrs []string
for _, ip := range ips {
addrs = append(addrs, ip.String())
}
return addrs
}

View File

@@ -2,22 +2,17 @@ package buildkit
import (
"context"
"net"
"os"
"path/filepath"
"strconv"
"sync"
"github.com/containerd/log"
"github.com/docker/docker/daemon/config"
"github.com/docker/docker/libnetwork"
"github.com/docker/docker/pkg/stringid"
"github.com/moby/buildkit/executor"
"github.com/moby/buildkit/executor/oci"
"github.com/moby/buildkit/executor/resources"
resourcestypes "github.com/moby/buildkit/executor/resources/types"
"github.com/moby/buildkit/executor/runcexecutor"
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/solver/llbsolver/cdidevices"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/network"
@@ -27,7 +22,7 @@ import (
const networkName = "bridge"
func newExecutor(root, cgroupParent string, net *libnetwork.Controller, dnsConfig *oci.DNSConfig, rootless bool, idmap user.IdentityMapping, apparmorProfile string, cdiManager *cdidevices.Manager) (executor.Executor, error) {
func newExecutor(root, cgroupParent string, net *libnetwork.Controller, dnsConfig *oci.DNSConfig, rootless bool, idmap user.IdentityMapping, apparmorProfile string, cdiManager *cdidevices.Manager, _, _ string) (executor.Executor, error) {
netRoot := filepath.Join(root, "net")
networkProviders := map[pb.NetMode]network.Provider{
pb.NetMode_UNSET: &bridgeProvider{Controller: net, Root: netRoot},
@@ -79,67 +74,21 @@ func newExecutor(root, cgroupParent string, net *libnetwork.Controller, dnsConfi
}, networkProviders)
}
type bridgeProvider struct {
*libnetwork.Controller
Root string
}
func (p *bridgeProvider) New(ctx context.Context, hostname string) (network.Namespace, error) {
n, err := p.NetworkByName(networkName)
if err != nil {
return nil, err
}
iface := &lnInterface{ready: make(chan struct{}), provider: p}
iface.Once.Do(func() {
go iface.init(p.Controller, n)
})
return iface, nil
}
func (p *bridgeProvider) Close() error {
return nil
}
type lnInterface struct {
ep *libnetwork.Endpoint
sbx *libnetwork.Sandbox
sync.Once
err error
ready chan struct{}
provider *bridgeProvider
}
func (iface *lnInterface) init(c *libnetwork.Controller, n *libnetwork.Network) {
defer close(iface.ready)
id := identity.NewID()
ep, err := n.CreateEndpoint(context.TODO(), id, libnetwork.CreateOptionDisableResolution())
if err != nil {
iface.err = err
return
}
sbx, err := c.NewSandbox(context.TODO(), id, libnetwork.OptionUseExternalKey(), libnetwork.OptionHostsPath(filepath.Join(iface.provider.Root, id, "hosts")),
libnetwork.OptionResolvConfPath(filepath.Join(iface.provider.Root, id, "resolv.conf")))
if err != nil {
iface.err = err
return
}
if err := ep.Join(context.TODO(), sbx); err != nil {
iface.err = err
return
}
iface.sbx = sbx
iface.ep = ep
}
// TODO(neersighted): Unstub Sample(), and collect data from the libnetwork Endpoint.
func (iface *lnInterface) Sample() (*resourcestypes.NetworkSample, error) {
return &resourcestypes.NetworkSample{}, nil
// newExecutorGD calls newExecutor() on Linux.
// Created for symmetry with the non-linux platforms, esp. Windows.
func newExecutorGD(root, cgroupParent string, net *libnetwork.Controller, dnsConfig *oci.DNSConfig, rootless bool, idmap user.IdentityMapping, apparmorProfile string, cdiManager *cdidevices.Manager, _, _ string) (executor.Executor, error) {
return newExecutor(
root,
cgroupParent,
net,
dnsConfig,
rootless,
idmap,
apparmorProfile,
cdiManager,
"",
"",
)
}
func (iface *lnInterface) Set(s *specs.Spec) error {
@@ -158,37 +107,3 @@ func (iface *lnInterface) Set(s *specs.Spec) error {
}
return nil
}
func (iface *lnInterface) Close() error {
<-iface.ready
if iface.sbx != nil {
go func() {
if err := iface.sbx.Delete(context.TODO()); err != nil {
log.G(context.TODO()).WithError(err).Errorf("failed to delete builder network sandbox")
}
if err := os.RemoveAll(filepath.Join(iface.provider.Root, iface.sbx.ContainerID())); err != nil {
log.G(context.TODO()).WithError(err).Errorf("failed to delete builder sandbox directory")
}
}()
}
return iface.err
}
func getDNSConfig(cfg config.DNSConfig) *oci.DNSConfig {
if cfg.DNS != nil || cfg.DNSSearch != nil || cfg.DNSOptions != nil {
return &oci.DNSConfig{
Nameservers: ipAddresses(cfg.DNS),
SearchDomains: cfg.DNSSearch,
Options: cfg.DNSOptions,
}
}
return nil
}
func ipAddresses(ips []net.IP) []string {
var addrs []string
for _, ip := range ips {
addrs = append(addrs, ip.String())
}
return addrs
}

View File

@@ -7,7 +7,6 @@ import (
"errors"
"runtime"
"github.com/docker/docker/daemon/config"
"github.com/docker/docker/libnetwork"
"github.com/moby/buildkit/executor"
"github.com/moby/buildkit/executor/oci"
@@ -16,10 +15,6 @@ import (
"github.com/moby/sys/user"
)
func newExecutor(_, _ string, _ *libnetwork.Controller, _ *oci.DNSConfig, _ bool, _ user.IdentityMapping, _ string, _ *cdidevices.Manager) (executor.Executor, error) {
return &stubExecutor{}, nil
}
type stubExecutor struct{}
func (w *stubExecutor) Run(ctx context.Context, id string, root executor.Mount, mounts []executor.Mount, process executor.ProcessInfo, started chan<- struct{}) (resourcetypes.Recorder, error) {
@@ -30,6 +25,7 @@ func (w *stubExecutor) Exec(ctx context.Context, id string, process executor.Pro
return errors.New("buildkit executor not implemented for " + runtime.GOOS)
}
func getDNSConfig(config.DNSConfig) *oci.DNSConfig {
return nil
// function stub created for GraphDriver
func newExecutorGD(_, _ string, _ *libnetwork.Controller, _ *oci.DNSConfig, _ bool, _ user.IdentityMapping, _ string, _ *cdidevices.Manager, _, _ string) (executor.Executor, error) {
return &stubExecutor{}, nil
}

View File

@@ -0,0 +1,7 @@
//go:build !linux && !windows
package buildkit
func newExecutor(_, _ string, _ *libnetwork.Controller, _ *oci.DNSConfig, _ bool, _ user.IdentityMapping, _ string, _ *cdidevices.Manager, _, _ string) (executor.Executor, error) {
return &stubExecutor{}, nil
}

View File

@@ -0,0 +1,94 @@
package buildkit
import (
"context"
"encoding/json"
"path/filepath"
ctd "github.com/containerd/containerd/v2/client"
"github.com/containerd/log"
"github.com/docker/docker/libnetwork"
"github.com/moby/buildkit/executor"
"github.com/moby/buildkit/executor/containerdexecutor"
"github.com/moby/buildkit/executor/oci"
"github.com/moby/buildkit/solver/llbsolver/cdidevices"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/network"
"github.com/moby/sys/user"
"github.com/opencontainers/runtime-spec/specs-go"
)
const networkName = "nat"
func newExecutor(
root string,
_ string,
net *libnetwork.Controller,
dns *oci.DNSConfig,
_ bool,
_ user.IdentityMapping,
_ string,
cdiManager *cdidevices.Manager,
containerdAddr string,
containerdNamespace string,
) (executor.Executor, error) {
netRoot := filepath.Join(root, "net")
np := map[pb.NetMode]network.Provider{
pb.NetMode_UNSET: &bridgeProvider{Controller: net, Root: netRoot},
pb.NetMode_NONE: network.NewNoneProvider(),
}
opt := ctd.WithDefaultNamespace(containerdNamespace)
client, err := ctd.New(containerdAddr, opt)
if err != nil {
return nil, err
}
executorOpts := containerdexecutor.ExecutorOptions{
Client: client,
Root: root,
DNSConfig: dns,
CDIManager: cdiManager,
NetworkProviders: np,
}
return containerdexecutor.New(executorOpts), nil
}
func (iface *lnInterface) Set(s *specs.Spec) error {
<-iface.ready
if iface.err != nil {
log.G(context.TODO()).WithError(iface.err).Error("failed to set networking spec")
return iface.err
}
allowUnqualifiedDNSQuery := false
var epList []string
for _, ep := range iface.sbx.Endpoints() {
data, err := ep.DriverInfo()
if err != nil {
continue
}
if data["hnsid"] != nil {
epList = append(epList, data["hnsid"].(string))
}
if data["AllowUnqualifiedDNSQuery"] != nil {
allowUnqualifiedDNSQuery = true
}
}
if s.Windows == nil {
s.Windows = &specs.Windows{}
}
if s.Windows.Network == nil {
s.Windows.Network = &specs.WindowsNetwork{}
}
s.Windows.Network.EndpointList = epList
s.Windows.Network.AllowUnqualifiedDNSQuery = allowUnqualifiedDNSQuery
if b, err := json.Marshal(s); err == nil {
log.G(context.TODO()).Debugf("Generated spec: %s", string(b))
}
return nil
}