From d00ecdc479caaaac9497503ed99e6cf12abc4450 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 29 Jul 2025 13:28:31 -0700 Subject: [PATCH] Move pkg/streamformatter to api/pkg/streamformatter Signed-off-by: Derek McGowan Signed-off-by: Sebastiaan van Stijn --- api/go.mod | 3 +- .../pkg}/streamformatter/streamformatter.go | 0 .../streamformatter/streamformatter_test.go | 0 .../pkg}/streamformatter/streamwriter.go | 0 .../pkg}/streamformatter/streamwriter_test.go | 0 daemon/builder/dockerfile/copy.go | 2 +- daemon/containerd/image_builder.go | 2 +- daemon/containerd/image_exporter.go | 2 +- daemon/containerd/image_pull.go | 2 +- daemon/containerd/image_push.go | 2 +- daemon/images/image_builder.go | 2 +- daemon/images/image_pull.go | 2 +- daemon/internal/builder-next/builder.go | 2 +- .../internal/distribution/utils/progress.go | 2 +- daemon/internal/image/tarexport/load.go | 2 +- daemon/server/router/build/build_routes.go | 2 +- daemon/server/router/image/image_routes.go | 2 +- daemon/server/router/plugin/plugin_routes.go | 2 +- .../pkg/streamformatter/streamformatter.go | 247 ++++++++++++++++++ .../api/pkg/streamformatter/streamwriter.go | 45 ++++ vendor/modules.txt | 1 + 21 files changed, 307 insertions(+), 15 deletions(-) rename {pkg => api/pkg}/streamformatter/streamformatter.go (100%) rename {pkg => api/pkg}/streamformatter/streamformatter_test.go (100%) rename {pkg => api/pkg}/streamformatter/streamwriter.go (100%) rename {pkg => api/pkg}/streamformatter/streamwriter_test.go (100%) create mode 100644 vendor/github.com/moby/moby/api/pkg/streamformatter/streamformatter.go create mode 100644 vendor/github.com/moby/moby/api/pkg/streamformatter/streamwriter.go diff --git a/api/go.mod b/api/go.mod index 6c878534e3..cc260147c9 100644 --- a/api/go.mod +++ b/api/go.mod @@ -5,11 +5,10 @@ go 1.23.0 require ( github.com/docker/go-connections v0.5.0 github.com/docker/go-units v0.5.0 + github.com/google/go-cmp v0.5.9 github.com/moby/docker-image-spec v1.3.1 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.1 golang.org/x/time v0.11.0 gotest.tools/v3 v3.5.2 ) - -require github.com/google/go-cmp v0.5.9 // indirect diff --git a/pkg/streamformatter/streamformatter.go b/api/pkg/streamformatter/streamformatter.go similarity index 100% rename from pkg/streamformatter/streamformatter.go rename to api/pkg/streamformatter/streamformatter.go diff --git a/pkg/streamformatter/streamformatter_test.go b/api/pkg/streamformatter/streamformatter_test.go similarity index 100% rename from pkg/streamformatter/streamformatter_test.go rename to api/pkg/streamformatter/streamformatter_test.go diff --git a/pkg/streamformatter/streamwriter.go b/api/pkg/streamformatter/streamwriter.go similarity index 100% rename from pkg/streamformatter/streamwriter.go rename to api/pkg/streamformatter/streamwriter.go diff --git a/pkg/streamformatter/streamwriter_test.go b/api/pkg/streamformatter/streamwriter_test.go similarity index 100% rename from pkg/streamformatter/streamwriter_test.go rename to api/pkg/streamformatter/streamwriter_test.go diff --git a/daemon/builder/dockerfile/copy.go b/daemon/builder/dockerfile/copy.go index a93ced8587..e46d25dca3 100644 --- a/daemon/builder/dockerfile/copy.go +++ b/daemon/builder/dockerfile/copy.go @@ -19,10 +19,10 @@ import ( "github.com/docker/docker/daemon/builder/remotecontext/urlutil" "github.com/docker/docker/daemon/internal/system" "github.com/docker/docker/pkg/longpath" - "github.com/docker/docker/pkg/streamformatter" "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/moby/go-archive" "github.com/moby/moby/api/pkg/progress" + "github.com/moby/moby/api/pkg/streamformatter" "github.com/moby/sys/symlink" "github.com/moby/sys/user" ocispec "github.com/opencontainers/image-spec/specs-go/v1" diff --git a/daemon/containerd/image_builder.go b/daemon/containerd/image_builder.go index 6d35941ca7..f1a01fbca5 100644 --- a/daemon/containerd/image_builder.go +++ b/daemon/containerd/image_builder.go @@ -27,10 +27,10 @@ import ( "github.com/docker/docker/daemon/internal/stringid" "github.com/docker/docker/daemon/server/backend" "github.com/docker/docker/errdefs" - "github.com/docker/docker/pkg/streamformatter" imagespec "github.com/moby/docker-image-spec/specs-go/v1" "github.com/moby/go-archive" "github.com/moby/moby/api/pkg/progress" + "github.com/moby/moby/api/pkg/streamformatter" "github.com/moby/moby/api/types/container" "github.com/moby/moby/api/types/events" "github.com/moby/moby/api/types/registry" diff --git a/daemon/containerd/image_exporter.go b/daemon/containerd/image_exporter.go index 964e28553e..16014cf04d 100644 --- a/daemon/containerd/image_exporter.go +++ b/daemon/containerd/image_exporter.go @@ -17,8 +17,8 @@ import ( "github.com/distribution/reference" "github.com/docker/docker/daemon/images" "github.com/docker/docker/errdefs" - "github.com/docker/docker/pkg/streamformatter" "github.com/moby/go-archive/compression" + "github.com/moby/moby/api/pkg/streamformatter" "github.com/moby/moby/api/types/events" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" diff --git a/daemon/containerd/image_pull.go b/daemon/containerd/image_pull.go index 240a11818e..a6509441d3 100644 --- a/daemon/containerd/image_pull.go +++ b/daemon/containerd/image_pull.go @@ -20,8 +20,8 @@ import ( "github.com/docker/docker/daemon/internal/metrics" "github.com/docker/docker/daemon/internal/stringid" "github.com/docker/docker/errdefs" - "github.com/docker/docker/pkg/streamformatter" "github.com/moby/moby/api/pkg/progress" + "github.com/moby/moby/api/pkg/streamformatter" "github.com/moby/moby/api/types/events" registrytypes "github.com/moby/moby/api/types/registry" ocispec "github.com/opencontainers/image-spec/specs-go/v1" diff --git a/daemon/containerd/image_push.go b/daemon/containerd/image_push.go index 42cf0c0d47..494a372fe4 100644 --- a/daemon/containerd/image_push.go +++ b/daemon/containerd/image_push.go @@ -19,8 +19,8 @@ import ( "github.com/distribution/reference" "github.com/docker/docker/daemon/internal/metrics" "github.com/docker/docker/errdefs" - "github.com/docker/docker/pkg/streamformatter" "github.com/moby/moby/api/pkg/progress" + "github.com/moby/moby/api/pkg/streamformatter" "github.com/moby/moby/api/types/auxprogress" "github.com/moby/moby/api/types/events" "github.com/moby/moby/api/types/registry" diff --git a/daemon/images/image_builder.go b/daemon/images/image_builder.go index 2730fa7d21..fcc4c6a3f2 100644 --- a/daemon/images/image_builder.go +++ b/daemon/images/image_builder.go @@ -14,8 +14,8 @@ import ( "github.com/docker/docker/daemon/internal/layer" "github.com/docker/docker/daemon/internal/stringid" "github.com/docker/docker/daemon/server/backend" - "github.com/docker/docker/pkg/streamformatter" "github.com/moby/moby/api/pkg/progress" + "github.com/moby/moby/api/pkg/streamformatter" "github.com/moby/moby/api/types/registry" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" diff --git a/daemon/images/image_pull.go b/daemon/images/image_pull.go index 9b34982cee..23697f085a 100644 --- a/daemon/images/image_pull.go +++ b/daemon/images/image_pull.go @@ -14,8 +14,8 @@ import ( progressutils "github.com/docker/docker/daemon/internal/distribution/utils" "github.com/docker/docker/daemon/internal/metrics" "github.com/docker/docker/daemon/server/backend" - "github.com/docker/docker/pkg/streamformatter" "github.com/moby/moby/api/pkg/progress" + "github.com/moby/moby/api/pkg/streamformatter" "github.com/moby/moby/api/types/registry" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" diff --git a/daemon/internal/builder-next/builder.go b/daemon/internal/builder-next/builder.go index be747c6687..18f0fd8bd2 100644 --- a/daemon/internal/builder-next/builder.go +++ b/daemon/internal/builder-next/builder.go @@ -22,7 +22,6 @@ import ( "github.com/docker/docker/daemon/pkg/opts" "github.com/docker/docker/daemon/server/backend" "github.com/docker/docker/errdefs" - "github.com/docker/docker/pkg/streamformatter" controlapi "github.com/moby/buildkit/api/services/control" "github.com/moby/buildkit/client" "github.com/moby/buildkit/control" @@ -30,6 +29,7 @@ import ( "github.com/moby/buildkit/session" "github.com/moby/buildkit/util/entitlements" "github.com/moby/buildkit/util/tracing" + "github.com/moby/moby/api/pkg/streamformatter" "github.com/moby/moby/api/types/build" "github.com/moby/moby/api/types/container" "github.com/moby/moby/api/types/network" diff --git a/daemon/internal/distribution/utils/progress.go b/daemon/internal/distribution/utils/progress.go index 9189d08bb0..89486d124a 100644 --- a/daemon/internal/distribution/utils/progress.go +++ b/daemon/internal/distribution/utils/progress.go @@ -7,8 +7,8 @@ import ( "syscall" "github.com/containerd/log" - "github.com/docker/docker/pkg/streamformatter" "github.com/moby/moby/api/pkg/progress" + "github.com/moby/moby/api/pkg/streamformatter" ) // WriteDistributionProgress is a helper for writing progress from chan to JSON diff --git a/daemon/internal/image/tarexport/load.go b/daemon/internal/image/tarexport/load.go index f717dc7f0d..1d5f28bb3d 100644 --- a/daemon/internal/image/tarexport/load.go +++ b/daemon/internal/image/tarexport/load.go @@ -19,10 +19,10 @@ import ( "github.com/docker/docker/daemon/internal/ioutils" "github.com/docker/docker/daemon/internal/layer" "github.com/docker/docker/daemon/internal/stringid" - "github.com/docker/docker/pkg/streamformatter" "github.com/moby/go-archive/chrootarchive" "github.com/moby/go-archive/compression" "github.com/moby/moby/api/pkg/progress" + "github.com/moby/moby/api/pkg/streamformatter" "github.com/moby/moby/api/types/events" "github.com/moby/sys/sequential" "github.com/moby/sys/symlink" diff --git a/daemon/server/router/build/build_routes.go b/daemon/server/router/build/build_routes.go index 0118b5b4a2..55254bb906 100644 --- a/daemon/server/router/build/build_routes.go +++ b/daemon/server/router/build/build_routes.go @@ -19,8 +19,8 @@ import ( "github.com/docker/docker/daemon/server/backend" "github.com/docker/docker/daemon/server/httputils" "github.com/docker/docker/pkg/ioutils" - "github.com/docker/docker/pkg/streamformatter" "github.com/moby/moby/api/pkg/progress" + "github.com/moby/moby/api/pkg/streamformatter" "github.com/moby/moby/api/types/build" "github.com/moby/moby/api/types/container" "github.com/moby/moby/api/types/filters" diff --git a/daemon/server/router/image/image_routes.go b/daemon/server/router/image/image_routes.go index dd3d1cba88..1b1f4235f4 100644 --- a/daemon/server/router/image/image_routes.go +++ b/daemon/server/router/image/image_routes.go @@ -19,8 +19,8 @@ import ( "github.com/docker/docker/dockerversion" "github.com/docker/docker/errdefs" "github.com/docker/docker/pkg/ioutils" - "github.com/docker/docker/pkg/streamformatter" "github.com/moby/moby/api/pkg/progress" + "github.com/moby/moby/api/pkg/streamformatter" "github.com/moby/moby/api/types/filters" imagetypes "github.com/moby/moby/api/types/image" "github.com/moby/moby/api/types/registry" diff --git a/daemon/server/router/plugin/plugin_routes.go b/daemon/server/router/plugin/plugin_routes.go index 8948d62a30..6dc21baf59 100644 --- a/daemon/server/router/plugin/plugin_routes.go +++ b/daemon/server/router/plugin/plugin_routes.go @@ -10,7 +10,7 @@ import ( "github.com/docker/docker/daemon/server/backend" "github.com/docker/docker/daemon/server/httputils" "github.com/docker/docker/pkg/ioutils" - "github.com/docker/docker/pkg/streamformatter" + "github.com/moby/moby/api/pkg/streamformatter" "github.com/moby/moby/api/types" "github.com/moby/moby/api/types/filters" "github.com/moby/moby/api/types/registry" diff --git a/vendor/github.com/moby/moby/api/pkg/streamformatter/streamformatter.go b/vendor/github.com/moby/moby/api/pkg/streamformatter/streamformatter.go new file mode 100644 index 0000000000..5913b568a4 --- /dev/null +++ b/vendor/github.com/moby/moby/api/pkg/streamformatter/streamformatter.go @@ -0,0 +1,247 @@ +// Package streamformatter provides helper functions to format a stream. +package streamformatter + +import ( + "encoding/json" + "fmt" + "io" + "strings" + "sync" + "time" + + "github.com/docker/go-units" + "github.com/moby/moby/api/pkg/progress" + "github.com/moby/moby/api/types/jsonstream" +) + +// jsonMessage defines a message struct. It describes +// the created time, where it from, status, ID of the +// message. It's used for docker events. +// +// It is a reduced set of [jsonmessage.JSONMessage]. +type jsonMessage struct { + Stream string `json:"stream,omitempty"` + Status string `json:"status,omitempty"` + Progress *jsonstream.Progress `json:"progressDetail,omitempty"` + ID string `json:"id,omitempty"` + Error *jsonstream.Error `json:"errorDetail,omitempty"` + Aux *json.RawMessage `json:"aux,omitempty"` // Aux contains out-of-band data, such as digests for push signing and image id after building. + + // ErrorMessage contains errors encountered during the operation. + // + // Deprecated: this field is deprecated since docker v0.6.0 / API v1.4. Use [Error.Message] instead. This field will be omitted in a future release. + ErrorMessage string `json:"error,omitempty"` // deprecated +} + +const streamNewline = "\r\n" + +type jsonProgressFormatter struct{} + +func appendNewline(source []byte) []byte { + return append(source, []byte(streamNewline)...) +} + +// FormatStatus formats the specified objects according to the specified format (and id). +func FormatStatus(id, format string, a ...interface{}) []byte { + str := fmt.Sprintf(format, a...) + b, err := json.Marshal(&jsonMessage{ID: id, Status: str}) + if err != nil { + return FormatError(err) + } + return appendNewline(b) +} + +// FormatError formats the error as a JSON object +func FormatError(err error) []byte { + jsonError, ok := err.(*jsonstream.Error) + if !ok { + jsonError = &jsonstream.Error{Message: err.Error()} + } + if b, err := json.Marshal(&jsonMessage{Error: jsonError, ErrorMessage: err.Error()}); err == nil { + return appendNewline(b) + } + return []byte(`{"error":"format error"}` + streamNewline) +} + +func (sf *jsonProgressFormatter) formatStatus(id, format string, a ...interface{}) []byte { + return FormatStatus(id, format, a...) +} + +// formatProgress formats the progress information for a specified action. +func (sf *jsonProgressFormatter) formatProgress(id, action string, progress *jsonstream.Progress, aux interface{}) []byte { + if progress == nil { + progress = &jsonstream.Progress{} + } + var auxJSON *json.RawMessage + if aux != nil { + auxJSONBytes, err := json.Marshal(aux) + if err != nil { + return nil + } + auxJSON = new(json.RawMessage) + *auxJSON = auxJSONBytes + } + b, err := json.Marshal(&jsonMessage{ + Status: action, + Progress: progress, + ID: id, + Aux: auxJSON, + }) + if err != nil { + return nil + } + return appendNewline(b) +} + +type rawProgressFormatter struct{} + +func (sf *rawProgressFormatter) formatStatus(id, format string, a ...interface{}) []byte { + return []byte(fmt.Sprintf(format, a...) + streamNewline) +} + +func rawProgressString(p *jsonstream.Progress) string { + if p == nil || (p.Current <= 0 && p.Total <= 0) { + return "" + } + if p.Total <= 0 { + switch p.Units { + case "": + return fmt.Sprintf("%8v", units.HumanSize(float64(p.Current))) + default: + return fmt.Sprintf("%d %s", p.Current, p.Units) + } + } + + percentage := int(float64(p.Current)/float64(p.Total)*100) / 2 + if percentage > 50 { + percentage = 50 + } + + numSpaces := 0 + if 50-percentage > 0 { + numSpaces = 50 - percentage + } + pbBox := fmt.Sprintf("[%s>%s] ", strings.Repeat("=", percentage), strings.Repeat(" ", numSpaces)) + + var numbersBox string + switch { + case p.HideCounts: + case p.Units == "": // no units, use bytes + current := units.HumanSize(float64(p.Current)) + total := units.HumanSize(float64(p.Total)) + + numbersBox = fmt.Sprintf("%8v/%v", current, total) + + if p.Current > p.Total { + // remove total display if the reported current is wonky. + numbersBox = fmt.Sprintf("%8v", current) + } + default: + numbersBox = fmt.Sprintf("%d/%d %s", p.Current, p.Total, p.Units) + + if p.Current > p.Total { + // remove total display if the reported current is wonky. + numbersBox = fmt.Sprintf("%d %s", p.Current, p.Units) + } + } + + var timeLeftBox string + if p.Current > 0 && p.Start > 0 && percentage < 50 { + fromStart := time.Since(time.Unix(p.Start, 0)) + perEntry := fromStart / time.Duration(p.Current) + left := time.Duration(p.Total-p.Current) * perEntry + timeLeftBox = " " + left.Round(time.Second).String() + } + return pbBox + numbersBox + timeLeftBox +} + +func (sf *rawProgressFormatter) formatProgress(id, action string, progress *jsonstream.Progress, aux interface{}) []byte { + if progress == nil { + progress = &jsonstream.Progress{} + } + endl := "\r" + out := rawProgressString(progress) + if out == "" { + endl += "\n" + } + return []byte(action + " " + out + endl) +} + +// NewProgressOutput returns a progress.Output object that can be passed to +// progress.NewProgressReader. +func NewProgressOutput(out io.Writer) progress.Output { + return &progressOutput{sf: &rawProgressFormatter{}, out: out, newLines: true} +} + +// NewJSONProgressOutput returns a progress.Output that formats output +// using JSON objects +func NewJSONProgressOutput(out io.Writer, newLines bool) progress.Output { + return &progressOutput{sf: &jsonProgressFormatter{}, out: out, newLines: newLines} +} + +type formatProgress interface { + formatStatus(id, format string, a ...interface{}) []byte + formatProgress(id, action string, progress *jsonstream.Progress, aux interface{}) []byte +} + +type progressOutput struct { + sf formatProgress + out io.Writer + newLines bool + mu sync.Mutex +} + +// WriteProgress formats progress information from a ProgressReader. +func (out *progressOutput) WriteProgress(prog progress.Progress) error { + var formatted []byte + if prog.Message != "" { + formatted = out.sf.formatStatus(prog.ID, prog.Message) + } else { + jsonProgress := jsonstream.Progress{ + Current: prog.Current, + Total: prog.Total, + HideCounts: prog.HideCounts, + Units: prog.Units, + } + formatted = out.sf.formatProgress(prog.ID, prog.Action, &jsonProgress, prog.Aux) + } + + out.mu.Lock() + defer out.mu.Unlock() + _, err := out.out.Write(formatted) + if err != nil { + return err + } + + if out.newLines && prog.LastUpdate { + _, err = out.out.Write(out.sf.formatStatus("", "")) + return err + } + + return nil +} + +// AuxFormatter is a streamFormatter that writes aux progress messages +type AuxFormatter struct { + io.Writer +} + +// Emit emits the given interface as an aux progress message +func (sf *AuxFormatter) Emit(id string, aux interface{}) error { + auxJSONBytes, err := json.Marshal(aux) + if err != nil { + return err + } + auxJSON := new(json.RawMessage) + *auxJSON = auxJSONBytes + msgJSON, err := json.Marshal(&jsonMessage{ID: id, Aux: auxJSON}) + if err != nil { + return err + } + msgJSON = appendNewline(msgJSON) + n, err := sf.Writer.Write(msgJSON) + if n != len(msgJSON) { + return io.ErrShortWrite + } + return err +} diff --git a/vendor/github.com/moby/moby/api/pkg/streamformatter/streamwriter.go b/vendor/github.com/moby/moby/api/pkg/streamformatter/streamwriter.go new file mode 100644 index 0000000000..a5f26d565c --- /dev/null +++ b/vendor/github.com/moby/moby/api/pkg/streamformatter/streamwriter.go @@ -0,0 +1,45 @@ +package streamformatter + +import ( + "encoding/json" + "io" +) + +type streamWriter struct { + io.Writer + lineFormat func([]byte) string +} + +func (sw *streamWriter) Write(buf []byte) (int, error) { + formattedBuf := sw.format(buf) + n, err := sw.Writer.Write(formattedBuf) + if n != len(formattedBuf) { + return n, io.ErrShortWrite + } + return len(buf), err +} + +func (sw *streamWriter) format(buf []byte) []byte { + msg := &jsonMessage{Stream: sw.lineFormat(buf)} + b, err := json.Marshal(msg) + if err != nil { + return FormatError(err) + } + return appendNewline(b) +} + +// NewStdoutWriter returns a writer which formats the output as json message +// representing stdout lines +func NewStdoutWriter(out io.Writer) io.Writer { + return &streamWriter{Writer: out, lineFormat: func(buf []byte) string { + return string(buf) + }} +} + +// NewStderrWriter returns a writer which formats the output as json message +// representing stderr lines +func NewStderrWriter(out io.Writer) io.Writer { + return &streamWriter{Writer: out, lineFormat: func(buf []byte) string { + return "\033[91m" + string(buf) + "\033[0m" + }} +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 8e6377721e..07891dd0f5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -941,6 +941,7 @@ github.com/moby/locker github.com/moby/moby/api github.com/moby/moby/api/pkg/progress github.com/moby/moby/api/pkg/stdcopy +github.com/moby/moby/api/pkg/streamformatter github.com/moby/moby/api/types github.com/moby/moby/api/types/auxprogress github.com/moby/moby/api/types/blkiodev