api/pkg/progress: move to client and daemon/internal

Move the progress package up into the client as a temporary shared location for
common clients like CLI and compose.

The progress package is used by the daemon to write progress updates to
some sink, typically a streamformatter. This package is of little use to
API clients as this package does not provide any facilities to consume
the progress updates.

Co-authored-by: Cory Snider <csnider@mirantis.com>
Signed-off-by: Austin Vazquez <austin.vazquez@docker.com>

api/go.mod

@@ -7,7 +7,6 @@ require (
 	github.com/moby/docker-image-spec v1.3.1
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.1
-	golang.org/x/time v0.11.0
 	gotest.tools/v3 v3.5.2
 	pgregory.net/rapid v1.2.0
 )

api/go.sum

@@ -8,8 +8,6 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
 github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
-golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
-golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
 gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
 pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk=

client/go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/opencontainers/image-spec v1.1.1
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0
 	go.opentelemetry.io/otel/trace v1.35.0
+	golang.org/x/time v0.11.0
 	gotest.tools/v3 v3.5.2
 )
 
@@ -30,7 +31,6 @@ require (
 	go.opentelemetry.io/otel v1.35.0 // indirect
 	go.opentelemetry.io/otel/metric v1.35.0 // indirect
 	golang.org/x/sys v0.33.0 // indirect
-	golang.org/x/time v0.11.0 // indirect
 )
 
 replace github.com/moby/moby/api => ../api

client/pkg/streamformatter/streamformatter.go

@@ -10,8 +10,8 @@ import (
 	"time"
 
 	"github.com/docker/go-units"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/api/types/jsonstream"
+	"github.com/moby/moby/client/pkg/progress"
 )
 
 // jsonMessage defines a message struct. It describes

daemon/builder/dockerfile/copy.go

@@ -16,10 +16,10 @@ import (
 	"github.com/containerd/log"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 	"github.com/moby/go-archive"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/v2/daemon/builder"
 	"github.com/moby/moby/v2/daemon/builder/remotecontext"
 	"github.com/moby/moby/v2/daemon/builder/remotecontext/urlutil"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/daemon/internal/system"
 	"github.com/moby/moby/v2/pkg/longpath"

daemon/containerd/image_builder.go

@@ -23,13 +23,13 @@ import (
 	"github.com/distribution/reference"
 	dockerspec "github.com/moby/docker-image-spec/specs-go/v1"
 	"github.com/moby/go-archive"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/api/types/container"
 	"github.com/moby/moby/api/types/events"
 	"github.com/moby/moby/api/types/registry"
 	"github.com/moby/moby/v2/daemon/builder"
 	"github.com/moby/moby/v2/daemon/internal/image"
 	"github.com/moby/moby/v2/daemon/internal/layer"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/daemon/internal/stringid"
 	"github.com/moby/moby/v2/daemon/server/buildbackend"

daemon/containerd/image_pull.go

@@ -15,11 +15,11 @@ import (
 	"github.com/containerd/log"
 	"github.com/containerd/platforms"
 	"github.com/distribution/reference"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/api/types/events"
 	registrytypes "github.com/moby/moby/api/types/registry"
 	"github.com/moby/moby/v2/daemon/internal/distribution"
 	"github.com/moby/moby/v2/daemon/internal/metrics"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/daemon/internal/stringid"
 	"github.com/moby/moby/v2/daemon/server/imagebackend"

daemon/containerd/image_push.go

@@ -16,11 +16,11 @@ import (
 	"github.com/containerd/log"
 	"github.com/containerd/platforms"
 	"github.com/distribution/reference"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/api/types/auxprogress"
 	"github.com/moby/moby/api/types/events"
 	"github.com/moby/moby/api/types/registry"
 	"github.com/moby/moby/v2/daemon/internal/metrics"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/daemon/server/imagebackend"
 	"github.com/moby/moby/v2/errdefs"

daemon/containerd/progress.go

@@ -17,7 +17,7 @@ import (
 	cerrdefs "github.com/containerd/errdefs"
 	"github.com/containerd/log"
 	"github.com/distribution/reference"
-	"github.com/moby/moby/api/pkg/progress"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	"github.com/moby/moby/v2/daemon/internal/stringid"
 	"github.com/moby/moby/v2/errdefs"
 	"github.com/opencontainers/go-digest"

daemon/images/image_builder.go

@@ -9,11 +9,11 @@ import (
 	"github.com/containerd/log"
 	"github.com/containerd/platforms"
 	"github.com/distribution/reference"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/api/types/registry"
 	"github.com/moby/moby/v2/daemon/builder"
 	"github.com/moby/moby/v2/daemon/internal/image"
 	"github.com/moby/moby/v2/daemon/internal/layer"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/daemon/internal/stringid"
 	"github.com/moby/moby/v2/daemon/server/buildbackend"

daemon/images/image_pull.go

@@ -10,11 +10,11 @@ import (
 	cerrdefs "github.com/containerd/errdefs"
 	"github.com/containerd/log"
 	"github.com/distribution/reference"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/api/types/registry"
 	"github.com/moby/moby/v2/daemon/internal/distribution"
 	progressutils "github.com/moby/moby/v2/daemon/internal/distribution/utils"
 	"github.com/moby/moby/v2/daemon/internal/metrics"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/daemon/server/imagebackend"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"

daemon/images/image_push.go

@@ -7,10 +7,10 @@ import (
 	cerrdefs "github.com/containerd/errdefs"
 	"github.com/distribution/reference"
 	"github.com/docker/distribution/manifest/schema2"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/v2/daemon/internal/distribution"
 	progressutils "github.com/moby/moby/v2/daemon/internal/distribution/utils"
 	"github.com/moby/moby/v2/daemon/internal/metrics"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	"github.com/moby/moby/v2/daemon/server/imagebackend"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )

daemon/internal/builder-next/adapters/containerimage/pull.go

@@ -38,12 +38,12 @@ import (
 	"github.com/moby/buildkit/util/leaseutil"
 	"github.com/moby/buildkit/util/progress"
 	"github.com/moby/buildkit/util/resolver"
-	pkgprogress "github.com/moby/moby/api/pkg/progress"
 	dimages "github.com/moby/moby/v2/daemon/images"
 	"github.com/moby/moby/v2/daemon/internal/distribution/metadata"
 	"github.com/moby/moby/v2/daemon/internal/distribution/xfer"
 	"github.com/moby/moby/v2/daemon/internal/image"
 	"github.com/moby/moby/v2/daemon/internal/layer"
+	pkgprogress "github.com/moby/moby/v2/daemon/internal/progress"
 	refstore "github.com/moby/moby/v2/daemon/internal/refstore"
 	"github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/identity"

daemon/internal/builder-next/worker/worker.go

@@ -41,13 +41,13 @@ import (
 	"github.com/moby/buildkit/util/leaseutil"
 	"github.com/moby/buildkit/util/progress"
 	"github.com/moby/buildkit/version"
-	pkgprogress "github.com/moby/moby/api/pkg/progress"
 	imageadapter "github.com/moby/moby/v2/daemon/internal/builder-next/adapters/containerimage"
 	mobyexporter "github.com/moby/moby/v2/daemon/internal/builder-next/exporter"
 	"github.com/moby/moby/v2/daemon/internal/builder-next/worker/mod"
 	distmetadata "github.com/moby/moby/v2/daemon/internal/distribution/metadata"
 	"github.com/moby/moby/v2/daemon/internal/distribution/xfer"
 	"github.com/moby/moby/v2/daemon/internal/layer"
+	pkgprogress "github.com/moby/moby/v2/daemon/internal/progress"
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"

daemon/internal/distribution/config.go

@@ -9,13 +9,13 @@ import (
 	"github.com/distribution/reference"
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/manifest/schema2"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/api/types/events"
 	"github.com/moby/moby/api/types/registry"
 	"github.com/moby/moby/v2/daemon/internal/distribution/metadata"
 	"github.com/moby/moby/v2/daemon/internal/distribution/xfer"
 	"github.com/moby/moby/v2/daemon/internal/image"
 	"github.com/moby/moby/v2/daemon/internal/layer"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	refstore "github.com/moby/moby/v2/daemon/internal/refstore"
 	registrypkg "github.com/moby/moby/v2/daemon/pkg/registry"
 	"github.com/opencontainers/go-digest"

daemon/internal/distribution/pull_v2.go

@@ -17,11 +17,11 @@ import (
 	"github.com/docker/distribution/manifest/ocischema"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/docker/distribution/registry/client/transport"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/v2/daemon/internal/distribution/metadata"
 	"github.com/moby/moby/v2/daemon/internal/distribution/xfer"
 	"github.com/moby/moby/v2/daemon/internal/image"
 	"github.com/moby/moby/v2/daemon/internal/layer"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	refstore "github.com/moby/moby/v2/daemon/internal/refstore"
 	"github.com/moby/moby/v2/daemon/internal/stringid"
 	"github.com/moby/moby/v2/daemon/pkg/registry"

daemon/internal/distribution/push.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/containerd/log"
 	"github.com/distribution/reference"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/api/types/events"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 )
 
 const compressionBufSize = 32768

daemon/internal/distribution/push_v2.go

@@ -14,11 +14,11 @@ import (
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/docker/distribution/registry/api/errcode"
 	"github.com/docker/distribution/registry/client"
-	"github.com/moby/moby/api/pkg/progress"
 	apitypes "github.com/moby/moby/api/types"
 	"github.com/moby/moby/v2/daemon/internal/distribution/metadata"
 	"github.com/moby/moby/v2/daemon/internal/distribution/xfer"
 	"github.com/moby/moby/v2/daemon/internal/layer"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	"github.com/moby/moby/v2/daemon/internal/stringid"
 	"github.com/moby/moby/v2/daemon/pkg/registry"
 	"github.com/moby/moby/v2/pkg/ioutils"

daemon/internal/distribution/push_v2_test.go

@@ -10,10 +10,10 @@ import (
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/docker/distribution/registry/api/errcode"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/api/types/registry"
 	"github.com/moby/moby/v2/daemon/internal/distribution/metadata"
 	"github.com/moby/moby/v2/daemon/internal/layer"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	refstore "github.com/moby/moby/v2/daemon/internal/refstore"
 	registrypkg "github.com/moby/moby/v2/daemon/pkg/registry"
 	"github.com/opencontainers/go-digest"

daemon/internal/distribution/utils/progress.go

@@ -7,7 +7,7 @@ import (
 	"syscall"
 
 	"github.com/containerd/log"
-	"github.com/moby/moby/api/pkg/progress"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 )
 

daemon/internal/distribution/xfer/download.go

@@ -10,9 +10,9 @@ import (
 	"github.com/containerd/log"
 	"github.com/docker/distribution"
 	"github.com/moby/go-archive/compression"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/v2/daemon/internal/image"
 	"github.com/moby/moby/v2/daemon/internal/layer"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	"github.com/moby/moby/v2/pkg/ioutils"
 )
 

daemon/internal/distribution/xfer/download_test.go

@@ -12,8 +12,8 @@ import (
 	"time"
 
 	"github.com/docker/distribution"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/v2/daemon/internal/layer"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	"github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/identity"
 	"gotest.tools/v3/assert"

daemon/internal/distribution/xfer/transfer.go

@@ -5,7 +5,7 @@ import (
 	"runtime"
 	"sync"
 
-	"github.com/moby/moby/api/pkg/progress"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	"github.com/pkg/errors"
 )
 

daemon/internal/distribution/xfer/transfer_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/moby/moby/api/pkg/progress"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 )
 
 func TestTransfer(t *testing.T) {

daemon/internal/distribution/xfer/upload.go

@@ -7,8 +7,8 @@ import (
 
 	"github.com/containerd/log"
 	"github.com/docker/distribution"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/v2/daemon/internal/layer"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 )
 
 const maxUploadAttempts = 5

daemon/internal/distribution/xfer/upload_test.go

@@ -8,8 +8,8 @@ import (
 	"time"
 
 	"github.com/docker/distribution"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/v2/daemon/internal/layer"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 )
 
 const maxUploadConcurrency = 3

daemon/internal/image/tarexport/load.go

@@ -17,11 +17,11 @@ import (
 	"github.com/docker/distribution"
 	"github.com/moby/go-archive/chrootarchive"
 	"github.com/moby/go-archive/compression"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/api/types/events"
 	"github.com/moby/moby/v2/daemon/internal/image"
 	"github.com/moby/moby/v2/daemon/internal/ioutils"
 	"github.com/moby/moby/v2/daemon/internal/layer"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/daemon/internal/stringid"
 	"github.com/moby/sys/sequential"

daemon/internal/progress/progressreader_test.go

@@ -0,0 +1,74 @@
+package progress
+
+import (
+	"bytes"
+	"io"
+	"testing"
+)
+
+func TestOutputOnPrematureClose(t *testing.T) {
+	content := []byte("TESTING")
+	reader := io.NopCloser(bytes.NewReader(content))
+	progressChan := make(chan Progress, 10)
+
+	pr := NewProgressReader(reader, ChanOutput(progressChan), int64(len(content)), "Test", "Read")
+
+	part := make([]byte, 4)
+	_, err := io.ReadFull(pr, part)
+	if err != nil {
+		pr.Close()
+		t.Fatal(err)
+	}
+
+drainLoop:
+	for {
+		select {
+		case <-progressChan:
+		default:
+			break drainLoop
+		}
+	}
+
+	pr.Close()
+
+	select {
+	case <-progressChan:
+	default:
+		t.Fatalf("Expected some output when closing prematurely")
+	}
+}
+
+func TestCompleteSilently(t *testing.T) {
+	content := []byte("TESTING")
+	reader := io.NopCloser(bytes.NewReader(content))
+	progressChan := make(chan Progress, 10)
+
+	pr := NewProgressReader(reader, ChanOutput(progressChan), int64(len(content)), "Test", "Read")
+
+	out, err := io.ReadAll(pr)
+	if err != nil {
+		pr.Close()
+		t.Fatal(err)
+	}
+	if string(out) != "TESTING" {
+		pr.Close()
+		t.Fatalf("Unexpected output %q from reader", string(out))
+	}
+
+drainLoop:
+	for {
+		select {
+		case <-progressChan:
+		default:
+			break drainLoop
+		}
+	}
+
+	pr.Close()
+
+	select {
+	case <-progressChan:
+		t.Fatalf("Should have closed silently when read is complete")
+	default:
+	}
+}

daemon/internal/streamformatter/streamformatter.go

@@ -7,8 +7,8 @@ import (
 	"io"
 	"sync"
 
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/api/types/jsonstream"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 )
 
 // jsonMessage defines a message struct. It describes

daemon/pkg/plugin/backend_linux.go

@@ -22,13 +22,13 @@ import (
 	"github.com/containerd/platforms"
 	"github.com/distribution/reference"
 	"github.com/moby/go-archive/chrootarchive"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/api/types"
 	"github.com/moby/moby/api/types/events"
 	"github.com/moby/moby/api/types/plugin"
 	"github.com/moby/moby/api/types/registry"
 	"github.com/moby/moby/v2/daemon/internal/containerfs"
 	"github.com/moby/moby/v2/daemon/internal/filters"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	"github.com/moby/moby/v2/daemon/internal/stringid"
 	v2 "github.com/moby/moby/v2/daemon/pkg/plugin/v2"
 	"github.com/moby/moby/v2/daemon/server/backend"

daemon/pkg/plugin/fetch_linux.go

@@ -14,9 +14,9 @@ import (
 	"github.com/containerd/log"
 	"github.com/distribution/reference"
 	"github.com/moby/go-archive/chrootarchive"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/api/types/registry"
 	progressutils "github.com/moby/moby/v2/daemon/internal/distribution/utils"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	"github.com/moby/moby/v2/daemon/internal/stringid"
 	"github.com/moby/moby/v2/pkg/ioutils"
 	"github.com/opencontainers/go-digest"

daemon/server/router/build/build_routes.go

@@ -16,11 +16,11 @@ import (
 	"syscall"
 
 	"github.com/containerd/log"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/api/types/build"
 	"github.com/moby/moby/api/types/container"
 	"github.com/moby/moby/api/types/registry"
 	"github.com/moby/moby/v2/daemon/internal/filters"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/daemon/internal/versions"
 	"github.com/moby/moby/v2/daemon/server/buildbackend"

daemon/server/router/image/image_routes.go

@@ -13,12 +13,12 @@ import (
 	"github.com/containerd/platforms"
 	"github.com/distribution/reference"
 	"github.com/moby/moby/api/pkg/authconfig"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/api/types/registry"
 	"github.com/moby/moby/v2/daemon/builder/remotecontext"
 	"github.com/moby/moby/v2/daemon/internal/compat"
 	"github.com/moby/moby/v2/daemon/internal/filters"
 	"github.com/moby/moby/v2/daemon/internal/image"
+	"github.com/moby/moby/v2/daemon/internal/progress"
 	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/daemon/internal/versions"
 	"github.com/moby/moby/v2/daemon/server/httputils"

vendor/modules.txt

@@ -941,7 +941,6 @@ github.com/moby/locker
 # github.com/moby/moby/api v1.52.0-beta.2 => ./api
 ## explicit; go 1.23.0
 github.com/moby/moby/api/pkg/authconfig
-github.com/moby/moby/api/pkg/progress
 github.com/moby/moby/api/pkg/stdcopy
 github.com/moby/moby/api/types
 github.com/moby/moby/api/types/auxprogress