api/pkg/progress: move to client and daemon/internal

Move the progress package up into the client as a temporary shared location for
common clients like CLI and compose.

The progress package is used by the daemon to write progress updates to
some sink, typically a streamformatter. This package is of little use to
API clients as this package does not provide any facilities to consume
the progress updates.

Co-authored-by: Cory Snider <csnider@mirantis.com>
Signed-off-by: Austin Vazquez <austin.vazquez@docker.com>

client/pkg/progress/progress.go

@@ -0,0 +1,93 @@
+package progress
+
+import (
+	"fmt"
+)
+
+// Progress represents the progress of a transfer.
+type Progress struct {
+	ID string
+
+	// Progress contains a Message or...
+	Message string
+
+	// ...progress of an action
+	Action  string
+	Current int64
+	Total   int64
+
+	// If true, don't show xB/yB
+	HideCounts bool
+	// If not empty, use units instead of bytes for counts
+	Units string
+
+	// Aux contains extra information not presented to the user, such as
+	// digests for push signing.
+	Aux any
+
+	LastUpdate bool
+}
+
+// Output is an interface for writing progress information. It's
+// like a writer for progress, but we don't call it Writer because
+// that would be confusing next to ProgressReader (also, because it
+// doesn't implement the io.Writer interface).
+type Output interface {
+	WriteProgress(Progress) error
+}
+
+type chanOutput chan<- Progress
+
+func (out chanOutput) WriteProgress(p Progress) error {
+	// FIXME: workaround for panic in #37735
+	defer func() {
+		recover()
+	}()
+	out <- p
+	return nil
+}
+
+// ChanOutput returns an Output that writes progress updates to the
+// supplied channel.
+func ChanOutput(progressChan chan<- Progress) Output {
+	return chanOutput(progressChan)
+}
+
+type discardOutput struct{}
+
+func (discardOutput) WriteProgress(Progress) error {
+	return nil
+}
+
+// DiscardOutput returns an Output that discards progress
+func DiscardOutput() Output {
+	return discardOutput{}
+}
+
+// Update is a convenience function to write a progress update to the channel.
+func Update(out Output, id, action string) {
+	out.WriteProgress(Progress{ID: id, Action: action})
+}
+
+// Updatef is a convenience function to write a printf-formatted progress update
+// to the channel.
+func Updatef(out Output, id, format string, a ...any) {
+	Update(out, id, fmt.Sprintf(format, a...))
+}
+
+// Message is a convenience function to write a progress message to the channel.
+func Message(out Output, id, message string) {
+	out.WriteProgress(Progress{ID: id, Message: message})
+}
+
+// Messagef is a convenience function to write a printf-formatted progress
+// message to the channel.
+func Messagef(out Output, id, format string, a ...any) {
+	Message(out, id, fmt.Sprintf(format, a...))
+}
+
+// Aux sends auxiliary information over a progress interface, which will not be
+// formatted for the UI. This is used for things such as push signing.
+func Aux(out Output, a any) {
+	out.WriteProgress(Progress{Aux: a})
+}

client/pkg/progress/progressreader.go

@@ -0,0 +1,66 @@
+package progress
+
+import (
+	"io"
+	"time"
+
+	"golang.org/x/time/rate"
+)
+
+// Reader is a Reader with progress bar.
+type Reader struct {
+	in          io.ReadCloser // Stream to read from
+	out         Output        // Where to send progress bar to
+	size        int64
+	current     int64
+	lastUpdate  int64
+	id          string
+	action      string
+	rateLimiter *rate.Limiter
+}
+
+// NewProgressReader creates a new ProgressReader.
+func NewProgressReader(in io.ReadCloser, out Output, size int64, id, action string) *Reader {
+	return &Reader{
+		in:          in,
+		out:         out,
+		size:        size,
+		id:          id,
+		action:      action,
+		rateLimiter: rate.NewLimiter(rate.Every(100*time.Millisecond), 1),
+	}
+}
+
+func (p *Reader) Read(buf []byte) (int, error) {
+	read, err := p.in.Read(buf)
+	p.current += int64(read)
+	updateEvery := int64(1024 * 512) // 512kB
+	if p.size > 0 {
+		// Update progress for every 1% read if 1% < 512kB
+		if increment := int64(0.01 * float64(p.size)); increment < updateEvery {
+			updateEvery = increment
+		}
+	}
+	if p.current-p.lastUpdate > updateEvery || err != nil {
+		p.updateProgress(err != nil && read == 0)
+		p.lastUpdate = p.current
+	}
+
+	return read, err
+}
+
+// Close closes the progress reader and its underlying reader.
+func (p *Reader) Close() error {
+	if p.current < p.size {
+		// print a full progress bar when closing prematurely
+		p.current = p.size
+		p.updateProgress(false)
+	}
+	return p.in.Close()
+}
+
+func (p *Reader) updateProgress(last bool) {
+	if last || p.current == p.size || p.rateLimiter.Allow() {
+		p.out.WriteProgress(Progress{ID: p.id, Action: p.action, Current: p.current, Total: p.size, LastUpdate: last})
+	}
+}

client/pkg/progress/progressreader_test.go

@@ -0,0 +1,74 @@
+package progress
+
+import (
+	"bytes"
+	"io"
+	"testing"
+)
+
+func TestOutputOnPrematureClose(t *testing.T) {
+	content := []byte("TESTING")
+	reader := io.NopCloser(bytes.NewReader(content))
+	progressChan := make(chan Progress, 10)
+
+	pr := NewProgressReader(reader, ChanOutput(progressChan), int64(len(content)), "Test", "Read")
+
+	part := make([]byte, 4)
+	_, err := io.ReadFull(pr, part)
+	if err != nil {
+		pr.Close()
+		t.Fatal(err)
+	}
+
+drainLoop:
+	for {
+		select {
+		case <-progressChan:
+		default:
+			break drainLoop
+		}
+	}
+
+	pr.Close()
+
+	select {
+	case <-progressChan:
+	default:
+		t.Fatalf("Expected some output when closing prematurely")
+	}
+}
+
+func TestCompleteSilently(t *testing.T) {
+	content := []byte("TESTING")
+	reader := io.NopCloser(bytes.NewReader(content))
+	progressChan := make(chan Progress, 10)
+
+	pr := NewProgressReader(reader, ChanOutput(progressChan), int64(len(content)), "Test", "Read")
+
+	out, err := io.ReadAll(pr)
+	if err != nil {
+		pr.Close()
+		t.Fatal(err)
+	}
+	if string(out) != "TESTING" {
+		pr.Close()
+		t.Fatalf("Unexpected output %q from reader", string(out))
+	}
+
+drainLoop:
+	for {
+		select {
+		case <-progressChan:
+		default:
+			break drainLoop
+		}
+	}
+
+	pr.Close()
+
+	select {
+	case <-progressChan:
+		t.Fatalf("Should have closed silently when read is complete")
+	default:
+	}
+}

client/pkg/streamformatter/streamformatter.go

@@ -10,8 +10,8 @@ import (
 	"time"
 
 	"github.com/docker/go-units"
-	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/api/types/jsonstream"
+	"github.com/moby/moby/client/pkg/progress"
 )
 
 // jsonMessage defines a message struct. It describes