api/pkg/streamformatter: move to client and daemon/internal

Move the streamformatter package up into the client for a temporary
shared location between common clients like CLI and compose.

The streamformatter package is used by the daemon to write streams of
status and progress messages to API clients. It is completely out of
scope of the api module and not used outside the daemon. Remove the
unused rawSteamFormatter, whose purpose is to render the progress as a
TUI.

Co-authored-by: Cory Snider <csnider@mirantis.com>
Signed-off-by: Austin Vazquez <austin.vazquez@docker.com>

api/go.mod

@@ -4,7 +4,6 @@ go 1.23.0
 
 require (
 	github.com/docker/go-units v0.5.0
-	github.com/google/go-cmp v0.7.0
 	github.com/moby/docker-image-spec v1.3.1
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.1
@@ -12,3 +11,5 @@ require (
 	gotest.tools/v3 v3.5.2
 	pgregory.net/rapid v1.2.0
 )
+
+require github.com/google/go-cmp v0.7.0 // indirect

client/go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/distribution/reference v0.6.0
 	github.com/docker/go-connections v0.6.0
 	github.com/docker/go-units v0.5.0
+	github.com/google/go-cmp v0.7.0
 	github.com/moby/moby/api v1.52.0-beta.2
 	github.com/moby/term v0.5.2
 	github.com/opencontainers/go-digest v1.0.0
@@ -24,12 +25,12 @@ require (
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/otel v1.35.0 // indirect
 	go.opentelemetry.io/otel/metric v1.35.0 // indirect
 	golang.org/x/sys v0.33.0 // indirect
+	golang.org/x/time v0.11.0 // indirect
 )
 
 replace github.com/moby/moby/api => ../api

client/go.sum

@@ -56,6 +56,8 @@ go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
 golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
+golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=

daemon/builder/dockerfile/copy.go

@@ -17,10 +17,10 @@ import (
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 	"github.com/moby/go-archive"
 	"github.com/moby/moby/api/pkg/progress"
-	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/v2/daemon/builder"
 	"github.com/moby/moby/v2/daemon/builder/remotecontext"
 	"github.com/moby/moby/v2/daemon/builder/remotecontext/urlutil"
+	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/daemon/internal/system"
 	"github.com/moby/moby/v2/pkg/longpath"
 	"github.com/moby/sys/symlink"

daemon/containerd/image_builder.go

@@ -24,13 +24,13 @@ import (
 	dockerspec "github.com/moby/docker-image-spec/specs-go/v1"
 	"github.com/moby/go-archive"
 	"github.com/moby/moby/api/pkg/progress"
-	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/container"
 	"github.com/moby/moby/api/types/events"
 	"github.com/moby/moby/api/types/registry"
 	"github.com/moby/moby/v2/daemon/builder"
 	"github.com/moby/moby/v2/daemon/internal/image"
 	"github.com/moby/moby/v2/daemon/internal/layer"
+	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/daemon/internal/stringid"
 	"github.com/moby/moby/v2/daemon/server/buildbackend"
 	"github.com/moby/moby/v2/daemon/server/imagebackend"

daemon/containerd/image_exporter.go

@@ -16,9 +16,9 @@ import (
 	"github.com/containerd/platforms"
 	"github.com/distribution/reference"
 	"github.com/moby/go-archive/compression"
-	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/events"
 	"github.com/moby/moby/v2/daemon/images"
+	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/errdefs"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"

daemon/containerd/image_pull.go

@@ -16,11 +16,11 @@ import (
 	"github.com/containerd/platforms"
 	"github.com/distribution/reference"
 	"github.com/moby/moby/api/pkg/progress"
-	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/events"
 	registrytypes "github.com/moby/moby/api/types/registry"
 	"github.com/moby/moby/v2/daemon/internal/distribution"
 	"github.com/moby/moby/v2/daemon/internal/metrics"
+	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/daemon/internal/stringid"
 	"github.com/moby/moby/v2/daemon/server/imagebackend"
 	"github.com/moby/moby/v2/errdefs"

daemon/containerd/image_push.go

@@ -17,11 +17,11 @@ import (
 	"github.com/containerd/platforms"
 	"github.com/distribution/reference"
 	"github.com/moby/moby/api/pkg/progress"
-	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/auxprogress"
 	"github.com/moby/moby/api/types/events"
 	"github.com/moby/moby/api/types/registry"
 	"github.com/moby/moby/v2/daemon/internal/metrics"
+	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/daemon/server/imagebackend"
 	"github.com/moby/moby/v2/errdefs"
 	"github.com/opencontainers/go-digest"

daemon/images/image_builder.go

@@ -10,11 +10,11 @@ import (
 	"github.com/containerd/platforms"
 	"github.com/distribution/reference"
 	"github.com/moby/moby/api/pkg/progress"
-	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/registry"
 	"github.com/moby/moby/v2/daemon/builder"
 	"github.com/moby/moby/v2/daemon/internal/image"
 	"github.com/moby/moby/v2/daemon/internal/layer"
+	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/daemon/internal/stringid"
 	"github.com/moby/moby/v2/daemon/server/buildbackend"
 	"github.com/moby/moby/v2/daemon/server/imagebackend"

daemon/images/image_pull.go

@@ -11,11 +11,11 @@ import (
 	"github.com/containerd/log"
 	"github.com/distribution/reference"
 	"github.com/moby/moby/api/pkg/progress"
-	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/registry"
 	"github.com/moby/moby/v2/daemon/internal/distribution"
 	progressutils "github.com/moby/moby/v2/daemon/internal/distribution/utils"
 	"github.com/moby/moby/v2/daemon/internal/metrics"
+	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/daemon/server/imagebackend"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"

daemon/internal/builder-next/builder.go

@@ -19,7 +19,6 @@ import (
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/util/entitlements"
 	"github.com/moby/buildkit/util/tracing"
-	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/build"
 	"github.com/moby/moby/api/types/container"
 	"github.com/moby/moby/api/types/network"
@@ -29,6 +28,7 @@ import (
 	"github.com/moby/moby/v2/daemon/internal/builder-next/exporter"
 	"github.com/moby/moby/v2/daemon/internal/builder-next/exporter/mobyexporter"
 	"github.com/moby/moby/v2/daemon/internal/builder-next/exporter/overrides"
+	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/daemon/internal/timestamp"
 	"github.com/moby/moby/v2/daemon/libnetwork"
 	"github.com/moby/moby/v2/daemon/pkg/opts"

daemon/internal/distribution/utils/progress.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/containerd/log"
 	"github.com/moby/moby/api/pkg/progress"
-	"github.com/moby/moby/api/pkg/streamformatter"
+	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 )
 
 // WriteDistributionProgress is a helper for writing progress from chan to JSON

daemon/internal/image/tarexport/load.go

@@ -18,11 +18,11 @@ import (
 	"github.com/moby/go-archive/chrootarchive"
 	"github.com/moby/go-archive/compression"
 	"github.com/moby/moby/api/pkg/progress"
-	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/events"
 	"github.com/moby/moby/v2/daemon/internal/image"
 	"github.com/moby/moby/v2/daemon/internal/ioutils"
 	"github.com/moby/moby/v2/daemon/internal/layer"
+	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/daemon/internal/stringid"
 	"github.com/moby/sys/sequential"
 	"github.com/moby/sys/symlink"

daemon/internal/streamformatter/streamformatter.go

@@ -5,11 +5,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"strings"
 	"sync"
-	"time"
 
-	"github.com/docker/go-units"
 	"github.com/moby/moby/api/pkg/progress"
 	"github.com/moby/moby/api/types/jsonstream"
 )
@@ -93,86 +90,6 @@ func (sf *jsonProgressFormatter) formatProgress(id, action string, progress *jso
 	return appendNewline(b)
 }
 
-type rawProgressFormatter struct{}
-
-func (sf *rawProgressFormatter) formatStatus(id, format string, a ...any) []byte {
-	return []byte(fmt.Sprintf(format, a...) + streamNewline)
-}
-
-func rawProgressString(p *jsonstream.Progress) string {
-	if p == nil || (p.Current <= 0 && p.Total <= 0) {
-		return ""
-	}
-	if p.Total <= 0 {
-		switch p.Units {
-		case "":
-			return fmt.Sprintf("%8v", units.HumanSize(float64(p.Current)))
-		default:
-			return fmt.Sprintf("%d %s", p.Current, p.Units)
-		}
-	}
-
-	percentage := int(float64(p.Current)/float64(p.Total)*100) / 2
-	if percentage > 50 {
-		percentage = 50
-	}
-
-	numSpaces := 0
-	if 50-percentage > 0 {
-		numSpaces = 50 - percentage
-	}
-	pbBox := fmt.Sprintf("[%s>%s] ", strings.Repeat("=", percentage), strings.Repeat(" ", numSpaces))
-
-	var numbersBox string
-	switch {
-	case p.HideCounts:
-	case p.Units == "": // no units, use bytes
-		current := units.HumanSize(float64(p.Current))
-		total := units.HumanSize(float64(p.Total))
-
-		numbersBox = fmt.Sprintf("%8v/%v", current, total)
-
-		if p.Current > p.Total {
-			// remove total display if the reported current is wonky.
-			numbersBox = fmt.Sprintf("%8v", current)
-		}
-	default:
-		numbersBox = fmt.Sprintf("%d/%d %s", p.Current, p.Total, p.Units)
-
-		if p.Current > p.Total {
-			// remove total display if the reported current is wonky.
-			numbersBox = fmt.Sprintf("%d %s", p.Current, p.Units)
-		}
-	}
-
-	var timeLeftBox string
-	if p.Current > 0 && p.Start > 0 && percentage < 50 {
-		fromStart := time.Since(time.Unix(p.Start, 0))
-		perEntry := fromStart / time.Duration(p.Current)
-		left := time.Duration(p.Total-p.Current) * perEntry
-		timeLeftBox = " " + left.Round(time.Second).String()
-	}
-	return pbBox + numbersBox + timeLeftBox
-}
-
-func (sf *rawProgressFormatter) formatProgress(id, action string, progress *jsonstream.Progress, aux any) []byte {
-	if progress == nil {
-		progress = &jsonstream.Progress{}
-	}
-	endl := "\r"
-	out := rawProgressString(progress)
-	if out == "" {
-		endl += "\n"
-	}
-	return []byte(action + " " + out + endl)
-}
-
-// NewProgressOutput returns a progress.Output object that can be passed to
-// progress.NewProgressReader.
-func NewProgressOutput(out io.Writer) progress.Output {
-	return &progressOutput{sf: &rawProgressFormatter{}, out: out, newLines: true}
-}
-
 // NewJSONProgressOutput returns a progress.Output that formats output
 // using JSON objects
 func NewJSONProgressOutput(out io.Writer, newLines bool) progress.Output {

daemon/internal/streamformatter/streamformatter_test.go

@@ -0,0 +1,89 @@
+package streamformatter
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/moby/moby/api/types/jsonstream"
+	"gotest.tools/v3/assert"
+	is "gotest.tools/v3/assert/cmp"
+)
+
+func TestFormatStatus(t *testing.T) {
+	res := FormatStatus("ID", "%s%d", "a", 1)
+	expected := `{"status":"a1","id":"ID"}` + streamNewline
+	assert.Check(t, is.Equal(expected, string(res)))
+}
+
+func TestFormatError(t *testing.T) {
+	res := FormatError(errors.New("Error for formatter"))
+	expected := `{"errorDetail":{"message":"Error for formatter"},"error":"Error for formatter"}` + "\r\n"
+	assert.Check(t, is.Equal(expected, string(res)))
+}
+
+func TestFormatJSONError(t *testing.T) {
+	err := &jsonstream.Error{Code: 50, Message: "Json error"}
+	res := FormatError(err)
+	expected := `{"errorDetail":{"code":50,"message":"Json error"},"error":"Json error"}` + streamNewline
+	assert.Check(t, is.Equal(expected, string(res)))
+}
+
+func TestJsonProgressFormatterFormatProgress(t *testing.T) {
+	sf := &jsonProgressFormatter{}
+	jsonProgress := &jsonstream.Progress{
+		Current: 15,
+		Total:   30,
+		Start:   1,
+	}
+	aux := "aux message"
+	res := sf.formatProgress("id", "action", jsonProgress, aux)
+	msg := &jsonMessage{}
+
+	assert.NilError(t, json.Unmarshal(res, msg))
+
+	rawAux := json.RawMessage(`"` + aux + `"`)
+	expected := &jsonMessage{
+		ID:       "id",
+		Status:   "action",
+		Aux:      &rawAux,
+		Progress: jsonProgress,
+	}
+	assert.DeepEqual(t, msg, expected, cmpJSONMessageOpt())
+}
+
+func cmpJSONMessageOpt() cmp.Option {
+	progressMessagePath := func(path cmp.Path) bool {
+		return path.String() == "ProgressMessage"
+	}
+	return cmp.Options{
+		// Ignore deprecated property that is a derivative of Progress
+		cmp.FilterPath(progressMessagePath, cmp.Ignore()),
+	}
+}
+
+func TestJsonProgressFormatterFormatStatus(t *testing.T) {
+	sf := jsonProgressFormatter{}
+	res := sf.formatStatus("ID", "%s%d", "a", 1)
+	assert.Check(t, is.Equal(`{"status":"a1","id":"ID"}`+streamNewline, string(res)))
+}
+
+func TestNewJSONProgressOutput(t *testing.T) {
+	b := bytes.Buffer{}
+	b.Write(FormatStatus("id", "Downloading"))
+	_ = NewJSONProgressOutput(&b, false)
+	assert.Check(t, is.Equal(`{"status":"Downloading","id":"id"}`+streamNewline, b.String()))
+}
+
+func TestAuxFormatterEmit(t *testing.T) {
+	b := bytes.Buffer{}
+	aux := &AuxFormatter{Writer: &b}
+	sampleAux := &struct {
+		Data string
+	}{"Additional data"}
+	err := aux.Emit("", sampleAux)
+	assert.NilError(t, err)
+	assert.Check(t, is.Equal(`{"aux":{"Data":"Additional data"}}`+streamNewline, b.String()))
+}

daemon/internal/streamformatter/streamwriter_test.go

@@ -0,0 +1,35 @@
+package streamformatter
+
+import (
+	"bytes"
+	"testing"
+
+	"gotest.tools/v3/assert"
+	is "gotest.tools/v3/assert/cmp"
+)
+
+func TestStreamWriterStdout(t *testing.T) {
+	buffer := &bytes.Buffer{}
+	content := "content"
+	sw := NewStdoutWriter(buffer)
+	size, err := sw.Write([]byte(content))
+
+	assert.NilError(t, err)
+	assert.Check(t, is.Equal(len(content), size))
+
+	expected := `{"stream":"content"}` + streamNewline
+	assert.Check(t, is.Equal(expected, buffer.String()))
+}
+
+func TestStreamWriterStderr(t *testing.T) {
+	buffer := &bytes.Buffer{}
+	content := "content"
+	sw := NewStderrWriter(buffer)
+	size, err := sw.Write([]byte(content))
+
+	assert.NilError(t, err)
+	assert.Check(t, is.Equal(len(content), size))
+
+	expected := `{"stream":"\u001b[91mcontent\u001b[0m"}` + streamNewline
+	assert.Check(t, is.Equal(expected, buffer.String()))
+}

daemon/server/router/build/build_routes.go

@@ -17,11 +17,11 @@ import (
 
 	"github.com/containerd/log"
 	"github.com/moby/moby/api/pkg/progress"
-	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/build"
 	"github.com/moby/moby/api/types/container"
 	"github.com/moby/moby/api/types/registry"
 	"github.com/moby/moby/v2/daemon/internal/filters"
+	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/daemon/internal/versions"
 	"github.com/moby/moby/v2/daemon/server/buildbackend"
 	"github.com/moby/moby/v2/daemon/server/httputils"

daemon/server/router/image/image_routes.go

@@ -14,12 +14,12 @@ import (
 	"github.com/distribution/reference"
 	"github.com/moby/moby/api/pkg/authconfig"
 	"github.com/moby/moby/api/pkg/progress"
-	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/registry"
 	"github.com/moby/moby/v2/daemon/builder/remotecontext"
 	"github.com/moby/moby/v2/daemon/internal/compat"
 	"github.com/moby/moby/v2/daemon/internal/filters"
 	"github.com/moby/moby/v2/daemon/internal/image"
+	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/daemon/internal/versions"
 	"github.com/moby/moby/v2/daemon/server/httputils"
 	"github.com/moby/moby/v2/daemon/server/imagebackend"

daemon/server/router/plugin/plugin_routes.go

@@ -8,10 +8,10 @@ import (
 
 	"github.com/distribution/reference"
 	"github.com/moby/moby/api/pkg/authconfig"
-	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/plugin"
 	"github.com/moby/moby/api/types/registry"
 	"github.com/moby/moby/v2/daemon/internal/filters"
+	"github.com/moby/moby/v2/daemon/internal/streamformatter"
 	"github.com/moby/moby/v2/daemon/server/backend"
 	"github.com/moby/moby/v2/daemon/server/httputils"
 	"github.com/moby/moby/v2/pkg/ioutils"

vendor/modules.txt

@@ -943,7 +943,6 @@ github.com/moby/locker
 github.com/moby/moby/api/pkg/authconfig
 github.com/moby/moby/api/pkg/progress
 github.com/moby/moby/api/pkg/stdcopy
-github.com/moby/moby/api/pkg/streamformatter
 github.com/moby/moby/api/types
 github.com/moby/moby/api/types/auxprogress
 github.com/moby/moby/api/types/blkiodev