Move pkg/streamformatter to api/pkg/streamformatter

Signed-off-by: Derek McGowan <derek@mcg.dev>
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

api/go.mod

@@ -5,11 +5,10 @@ go 1.23.0
 require (
 	github.com/docker/go-connections v0.5.0
 	github.com/docker/go-units v0.5.0
+	github.com/google/go-cmp v0.5.9
 	github.com/moby/docker-image-spec v1.3.1
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.1.1
 	golang.org/x/time v0.11.0
 	gotest.tools/v3 v3.5.2
 )
-
-require github.com/google/go-cmp v0.5.9 // indirect

daemon/builder/dockerfile/copy.go

@@ -19,10 +19,10 @@ import (
 	"github.com/docker/docker/daemon/builder/remotecontext/urlutil"
 	"github.com/docker/docker/daemon/internal/system"
 	"github.com/docker/docker/pkg/longpath"
-	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 	"github.com/moby/go-archive"
 	"github.com/moby/moby/api/pkg/progress"
+	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/sys/symlink"
 	"github.com/moby/sys/user"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"

daemon/containerd/image_builder.go

@@ -27,10 +27,10 @@ import (
 	"github.com/docker/docker/daemon/internal/stringid"
 	"github.com/docker/docker/daemon/server/backend"
 	"github.com/docker/docker/errdefs"
-	"github.com/docker/docker/pkg/streamformatter"
 	imagespec "github.com/moby/docker-image-spec/specs-go/v1"
 	"github.com/moby/go-archive"
 	"github.com/moby/moby/api/pkg/progress"
+	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/container"
 	"github.com/moby/moby/api/types/events"
 	"github.com/moby/moby/api/types/registry"

daemon/containerd/image_exporter.go

@@ -17,8 +17,8 @@ import (
 	"github.com/distribution/reference"
 	"github.com/docker/docker/daemon/images"
 	"github.com/docker/docker/errdefs"
-	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/moby/go-archive/compression"
+	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/events"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"

daemon/containerd/image_pull.go

@@ -20,8 +20,8 @@ import (
 	"github.com/docker/docker/daemon/internal/metrics"
 	"github.com/docker/docker/daemon/internal/stringid"
 	"github.com/docker/docker/errdefs"
-	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/moby/moby/api/pkg/progress"
+	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/events"
 	registrytypes "github.com/moby/moby/api/types/registry"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"

daemon/containerd/image_push.go

@@ -19,8 +19,8 @@ import (
 	"github.com/distribution/reference"
 	"github.com/docker/docker/daemon/internal/metrics"
 	"github.com/docker/docker/errdefs"
-	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/moby/moby/api/pkg/progress"
+	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/auxprogress"
 	"github.com/moby/moby/api/types/events"
 	"github.com/moby/moby/api/types/registry"

daemon/images/image_builder.go

@@ -14,8 +14,8 @@ import (
 	"github.com/docker/docker/daemon/internal/layer"
 	"github.com/docker/docker/daemon/internal/stringid"
 	"github.com/docker/docker/daemon/server/backend"
-	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/moby/moby/api/pkg/progress"
+	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/registry"
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"

daemon/images/image_pull.go

@@ -14,8 +14,8 @@ import (
 	progressutils "github.com/docker/docker/daemon/internal/distribution/utils"
 	"github.com/docker/docker/daemon/internal/metrics"
 	"github.com/docker/docker/daemon/server/backend"
-	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/moby/moby/api/pkg/progress"
+	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/registry"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"

daemon/internal/builder-next/builder.go

@@ -22,7 +22,6 @@ import (
 	"github.com/docker/docker/daemon/pkg/opts"
 	"github.com/docker/docker/daemon/server/backend"
 	"github.com/docker/docker/errdefs"
-	"github.com/docker/docker/pkg/streamformatter"
 	controlapi "github.com/moby/buildkit/api/services/control"
 	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/control"
@@ -30,6 +29,7 @@ import (
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/util/entitlements"
 	"github.com/moby/buildkit/util/tracing"
+	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/build"
 	"github.com/moby/moby/api/types/container"
 	"github.com/moby/moby/api/types/network"

daemon/internal/distribution/utils/progress.go

@@ -7,8 +7,8 @@ import (
 	"syscall"
 
 	"github.com/containerd/log"
-	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/moby/moby/api/pkg/progress"
+	"github.com/moby/moby/api/pkg/streamformatter"
 )
 
 // WriteDistributionProgress is a helper for writing progress from chan to JSON

daemon/internal/image/tarexport/load.go

@@ -19,10 +19,10 @@ import (
 	"github.com/docker/docker/daemon/internal/ioutils"
 	"github.com/docker/docker/daemon/internal/layer"
 	"github.com/docker/docker/daemon/internal/stringid"
-	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/moby/go-archive/chrootarchive"
 	"github.com/moby/go-archive/compression"
 	"github.com/moby/moby/api/pkg/progress"
+	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/events"
 	"github.com/moby/sys/sequential"
 	"github.com/moby/sys/symlink"

daemon/server/router/build/build_routes.go

@@ -19,8 +19,8 @@ import (
 	"github.com/docker/docker/daemon/server/backend"
 	"github.com/docker/docker/daemon/server/httputils"
 	"github.com/docker/docker/pkg/ioutils"
-	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/moby/moby/api/pkg/progress"
+	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/build"
 	"github.com/moby/moby/api/types/container"
 	"github.com/moby/moby/api/types/filters"

daemon/server/router/image/image_routes.go

@@ -19,8 +19,8 @@ import (
 	"github.com/docker/docker/dockerversion"
 	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/pkg/ioutils"
-	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/moby/moby/api/pkg/progress"
+	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types/filters"
 	imagetypes "github.com/moby/moby/api/types/image"
 	"github.com/moby/moby/api/types/registry"

daemon/server/router/plugin/plugin_routes.go

@@ -10,7 +10,7 @@ import (
 	"github.com/docker/docker/daemon/server/backend"
 	"github.com/docker/docker/daemon/server/httputils"
 	"github.com/docker/docker/pkg/ioutils"
-	"github.com/docker/docker/pkg/streamformatter"
+	"github.com/moby/moby/api/pkg/streamformatter"
 	"github.com/moby/moby/api/types"
 	"github.com/moby/moby/api/types/filters"
 	"github.com/moby/moby/api/types/registry"

vendor/github.com/moby/moby/api/pkg/streamformatter/streamformatter.go

@@ -0,0 +1,247 @@
+// Package streamformatter provides helper functions to format a stream.
+package streamformatter
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/docker/go-units"
+	"github.com/moby/moby/api/pkg/progress"
+	"github.com/moby/moby/api/types/jsonstream"
+)
+
+// jsonMessage defines a message struct. It describes
+// the created time, where it from, status, ID of the
+// message. It's used for docker events.
+//
+// It is a reduced set of [jsonmessage.JSONMessage].
+type jsonMessage struct {
+	Stream   string               `json:"stream,omitempty"`
+	Status   string               `json:"status,omitempty"`
+	Progress *jsonstream.Progress `json:"progressDetail,omitempty"`
+	ID       string               `json:"id,omitempty"`
+	Error    *jsonstream.Error    `json:"errorDetail,omitempty"`
+	Aux      *json.RawMessage     `json:"aux,omitempty"` // Aux contains out-of-band data, such as digests for push signing and image id after building.
+
+	// ErrorMessage contains errors encountered during the operation.
+	//
+	// Deprecated: this field is deprecated since docker v0.6.0 / API v1.4. Use [Error.Message] instead. This field will be omitted in a future release.
+	ErrorMessage string `json:"error,omitempty"` // deprecated
+}
+
+const streamNewline = "\r\n"
+
+type jsonProgressFormatter struct{}
+
+func appendNewline(source []byte) []byte {
+	return append(source, []byte(streamNewline)...)
+}
+
+// FormatStatus formats the specified objects according to the specified format (and id).
+func FormatStatus(id, format string, a ...interface{}) []byte {
+	str := fmt.Sprintf(format, a...)
+	b, err := json.Marshal(&jsonMessage{ID: id, Status: str})
+	if err != nil {
+		return FormatError(err)
+	}
+	return appendNewline(b)
+}
+
+// FormatError formats the error as a JSON object
+func FormatError(err error) []byte {
+	jsonError, ok := err.(*jsonstream.Error)
+	if !ok {
+		jsonError = &jsonstream.Error{Message: err.Error()}
+	}
+	if b, err := json.Marshal(&jsonMessage{Error: jsonError, ErrorMessage: err.Error()}); err == nil {
+		return appendNewline(b)
+	}
+	return []byte(`{"error":"format error"}` + streamNewline)
+}
+
+func (sf *jsonProgressFormatter) formatStatus(id, format string, a ...interface{}) []byte {
+	return FormatStatus(id, format, a...)
+}
+
+// formatProgress formats the progress information for a specified action.
+func (sf *jsonProgressFormatter) formatProgress(id, action string, progress *jsonstream.Progress, aux interface{}) []byte {
+	if progress == nil {
+		progress = &jsonstream.Progress{}
+	}
+	var auxJSON *json.RawMessage
+	if aux != nil {
+		auxJSONBytes, err := json.Marshal(aux)
+		if err != nil {
+			return nil
+		}
+		auxJSON = new(json.RawMessage)
+		*auxJSON = auxJSONBytes
+	}
+	b, err := json.Marshal(&jsonMessage{
+		Status:   action,
+		Progress: progress,
+		ID:       id,
+		Aux:      auxJSON,
+	})
+	if err != nil {
+		return nil
+	}
+	return appendNewline(b)
+}
+
+type rawProgressFormatter struct{}
+
+func (sf *rawProgressFormatter) formatStatus(id, format string, a ...interface{}) []byte {
+	return []byte(fmt.Sprintf(format, a...) + streamNewline)
+}
+
+func rawProgressString(p *jsonstream.Progress) string {
+	if p == nil || (p.Current <= 0 && p.Total <= 0) {
+		return ""
+	}
+	if p.Total <= 0 {
+		switch p.Units {
+		case "":
+			return fmt.Sprintf("%8v", units.HumanSize(float64(p.Current)))
+		default:
+			return fmt.Sprintf("%d %s", p.Current, p.Units)
+		}
+	}
+
+	percentage := int(float64(p.Current)/float64(p.Total)*100) / 2
+	if percentage > 50 {
+		percentage = 50
+	}
+
+	numSpaces := 0
+	if 50-percentage > 0 {
+		numSpaces = 50 - percentage
+	}
+	pbBox := fmt.Sprintf("[%s>%s] ", strings.Repeat("=", percentage), strings.Repeat(" ", numSpaces))
+
+	var numbersBox string
+	switch {
+	case p.HideCounts:
+	case p.Units == "": // no units, use bytes
+		current := units.HumanSize(float64(p.Current))
+		total := units.HumanSize(float64(p.Total))
+
+		numbersBox = fmt.Sprintf("%8v/%v", current, total)
+
+		if p.Current > p.Total {
+			// remove total display if the reported current is wonky.
+			numbersBox = fmt.Sprintf("%8v", current)
+		}
+	default:
+		numbersBox = fmt.Sprintf("%d/%d %s", p.Current, p.Total, p.Units)
+
+		if p.Current > p.Total {
+			// remove total display if the reported current is wonky.
+			numbersBox = fmt.Sprintf("%d %s", p.Current, p.Units)
+		}
+	}
+
+	var timeLeftBox string
+	if p.Current > 0 && p.Start > 0 && percentage < 50 {
+		fromStart := time.Since(time.Unix(p.Start, 0))
+		perEntry := fromStart / time.Duration(p.Current)
+		left := time.Duration(p.Total-p.Current) * perEntry
+		timeLeftBox = " " + left.Round(time.Second).String()
+	}
+	return pbBox + numbersBox + timeLeftBox
+}
+
+func (sf *rawProgressFormatter) formatProgress(id, action string, progress *jsonstream.Progress, aux interface{}) []byte {
+	if progress == nil {
+		progress = &jsonstream.Progress{}
+	}
+	endl := "\r"
+	out := rawProgressString(progress)
+	if out == "" {
+		endl += "\n"
+	}
+	return []byte(action + " " + out + endl)
+}
+
+// NewProgressOutput returns a progress.Output object that can be passed to
+// progress.NewProgressReader.
+func NewProgressOutput(out io.Writer) progress.Output {
+	return &progressOutput{sf: &rawProgressFormatter{}, out: out, newLines: true}
+}
+
+// NewJSONProgressOutput returns a progress.Output that formats output
+// using JSON objects
+func NewJSONProgressOutput(out io.Writer, newLines bool) progress.Output {
+	return &progressOutput{sf: &jsonProgressFormatter{}, out: out, newLines: newLines}
+}
+
+type formatProgress interface {
+	formatStatus(id, format string, a ...interface{}) []byte
+	formatProgress(id, action string, progress *jsonstream.Progress, aux interface{}) []byte
+}
+
+type progressOutput struct {
+	sf       formatProgress
+	out      io.Writer
+	newLines bool
+	mu       sync.Mutex
+}
+
+// WriteProgress formats progress information from a ProgressReader.
+func (out *progressOutput) WriteProgress(prog progress.Progress) error {
+	var formatted []byte
+	if prog.Message != "" {
+		formatted = out.sf.formatStatus(prog.ID, prog.Message)
+	} else {
+		jsonProgress := jsonstream.Progress{
+			Current:    prog.Current,
+			Total:      prog.Total,
+			HideCounts: prog.HideCounts,
+			Units:      prog.Units,
+		}
+		formatted = out.sf.formatProgress(prog.ID, prog.Action, &jsonProgress, prog.Aux)
+	}
+
+	out.mu.Lock()
+	defer out.mu.Unlock()
+	_, err := out.out.Write(formatted)
+	if err != nil {
+		return err
+	}
+
+	if out.newLines && prog.LastUpdate {
+		_, err = out.out.Write(out.sf.formatStatus("", ""))
+		return err
+	}
+
+	return nil
+}
+
+// AuxFormatter is a streamFormatter that writes aux progress messages
+type AuxFormatter struct {
+	io.Writer
+}
+
+// Emit emits the given interface as an aux progress message
+func (sf *AuxFormatter) Emit(id string, aux interface{}) error {
+	auxJSONBytes, err := json.Marshal(aux)
+	if err != nil {
+		return err
+	}
+	auxJSON := new(json.RawMessage)
+	*auxJSON = auxJSONBytes
+	msgJSON, err := json.Marshal(&jsonMessage{ID: id, Aux: auxJSON})
+	if err != nil {
+		return err
+	}
+	msgJSON = appendNewline(msgJSON)
+	n, err := sf.Writer.Write(msgJSON)
+	if n != len(msgJSON) {
+		return io.ErrShortWrite
+	}
+	return err
+}

vendor/github.com/moby/moby/api/pkg/streamformatter/streamwriter.go

@@ -0,0 +1,45 @@
+package streamformatter
+
+import (
+	"encoding/json"
+	"io"
+)
+
+type streamWriter struct {
+	io.Writer
+	lineFormat func([]byte) string
+}
+
+func (sw *streamWriter) Write(buf []byte) (int, error) {
+	formattedBuf := sw.format(buf)
+	n, err := sw.Writer.Write(formattedBuf)
+	if n != len(formattedBuf) {
+		return n, io.ErrShortWrite
+	}
+	return len(buf), err
+}
+
+func (sw *streamWriter) format(buf []byte) []byte {
+	msg := &jsonMessage{Stream: sw.lineFormat(buf)}
+	b, err := json.Marshal(msg)
+	if err != nil {
+		return FormatError(err)
+	}
+	return appendNewline(b)
+}
+
+// NewStdoutWriter returns a writer which formats the output as json message
+// representing stdout lines
+func NewStdoutWriter(out io.Writer) io.Writer {
+	return &streamWriter{Writer: out, lineFormat: func(buf []byte) string {
+		return string(buf)
+	}}
+}
+
+// NewStderrWriter returns a writer which formats the output as json message
+// representing stderr lines
+func NewStderrWriter(out io.Writer) io.Writer {
+	return &streamWriter{Writer: out, lineFormat: func(buf []byte) string {
+		return "\033[91m" + string(buf) + "\033[0m"
+	}}
+}

vendor/modules.txt

@@ -941,6 +941,7 @@ github.com/moby/locker
 github.com/moby/moby/api
 github.com/moby/moby/api/pkg/progress
 github.com/moby/moby/api/pkg/stdcopy
+github.com/moby/moby/api/pkg/streamformatter
 github.com/moby/moby/api/types
 github.com/moby/moby/api/types/auxprogress
 github.com/moby/moby/api/types/blkiodev