Merge pull request #50462 from thaJeztah/move_stdcopy

deprecate pkg/stdcopy, move to api/stdcopy

client/container_attach.go

@@ -31,7 +31,7 @@ import (
 // SIZE1, SIZE2, SIZE3, and SIZE4 are four bytes of uint32 encoded as big endian.
 // This is the size of OUTPUT.
 //
-// You can use github.com/docker/docker/pkg/stdcopy.StdCopy to demultiplex this
+// You can use github.com/moby/moby/api/stdcopy.StdCopy to demultiplex this
 // stream.
 func (cli *Client) ContainerAttach(ctx context.Context, containerID string, options container.AttachOptions) (types.HijackedResponse, error) {
 	containerID, err := trimID("container", containerID)

client/container_exec.go

@@ -68,7 +68,7 @@ func (cli *Client) ContainerExecStart(ctx context.Context, execID string, config
 //   - If the container is *not* using a TTY, streams for stdout and stderr are
 //     multiplexed.
 //
-// You can use [github.com/docker/docker/pkg/stdcopy.StdCopy] to demultiplex this
+// You can use [github.com/moby/moby/api/stdcopy.StdCopy] to demultiplex this
 // stream. Refer to [Client.ContainerAttach] for details about the multiplexed
 // stream.
 func (cli *Client) ContainerExecAttach(ctx context.Context, execID string, config container.ExecAttachOptions) (types.HijackedResponse, error) {

client/container_logs.go

@@ -31,7 +31,7 @@ import (
 // SIZE1, SIZE2, SIZE3, and SIZE4 are four bytes of uint32 encoded as big endian.
 // This is the size of OUTPUT.
 //
-// You can use github.com/docker/docker/pkg/stdcopy.StdCopy to demultiplex this
+// You can use github.com/moby/moby/api/stdcopy.StdCopy to demultiplex this
 // stream.
 func (cli *Client) ContainerLogs(ctx context.Context, containerID string, options container.LogsOptions) (io.ReadCloser, error) {
 	containerID, err := trimID("container", containerID)

daemon/attach.go

@@ -10,7 +10,7 @@ import (
 	"github.com/docker/docker/daemon/internal/stream"
 	"github.com/docker/docker/daemon/logger"
 	"github.com/docker/docker/errdefs"
-	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/moby/moby/api/stdcopy"
 	"github.com/moby/moby/api/types/backend"
 	containertypes "github.com/moby/moby/api/types/container"
 	"github.com/moby/moby/api/types/events"

daemon/server/httputils/write_log_stream.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/docker/docker/pkg/jsonmessage"
-	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/moby/moby/api/stdcopy"
 	"github.com/moby/moby/api/types/backend"
 	"github.com/moby/moby/api/types/container"
 )

daemon/server/router/container/exec.go

@@ -9,7 +9,7 @@ import (
 	"github.com/containerd/log"
 	"github.com/docker/docker/daemon/server/httputils"
 	"github.com/docker/docker/errdefs"
-	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/moby/moby/api/stdcopy"
 	"github.com/moby/moby/api/types"
 	"github.com/moby/moby/api/types/backend"
 	"github.com/moby/moby/api/types/container"

hack/make.ps1

@@ -256,6 +256,15 @@ Function Validate-PkgImports($headCommit, $upstreamCommit) {
     $files=@(); $files = Invoke-Expression "git diff $upstreamCommit...$headCommit --diff-filter=ACMR --name-only -- `'pkg\*.go`'"
     $badFiles=@(); $files | ForEach-Object{
         $file=$_
+
+        if ($file -like "pkg\stdcopy\*") {
+            # Temporarily allow pkg/stdcopy to import "github.com/moby/moby/api/stdcopy",
+            # because it's an alias for backward-compatibility.
+            #
+            # TODO(thaJeztah): remove once "github.com/docker/docker/pkg/stdcopy" is removed.
+            return
+        }
+
         # For the current changed file, get its list of dependencies, sorted and uniqued.
         $imports = Invoke-Expression "go list -e -f `'{{ .Deps }}`' $file"
         if ($LASTEXITCODE -ne 0) { Throw "Failed go list for dependencies on $file" }

hack/validate/pkg-imports

@@ -10,6 +10,16 @@ unset IFS
 
 badFiles=()
 for f in "${files[@]}"; do
+	case "$f" in
+		pkg/stdcopy/*)
+			# Temporarily allow pkg/stdcopy to import "github.com/moby/moby/api/stdcopy",
+			# because it's an alias for backward-compatibility.
+			#
+			# TODO(thaJeztah): remove once "github.com/docker/docker/pkg/stdcopy" is removed.
+			continue
+			;;
+	esac
+
 	IFS=$'\n'
 	badImports=($(go list -e -f '{{ join .Deps "\n" }}' "$f" | sort -u \
 		| grep -vE '^github.com/docker/docker/pkg/' \

integration-cli/docker_api_attach_test.go

@@ -11,10 +11,10 @@ import (
 	"time"
 
 	"github.com/docker/docker/integration-cli/cli"
-	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/docker/docker/testutil"
 	"github.com/docker/docker/testutil/request"
 	"github.com/docker/go-connections/sockets"
+	"github.com/moby/moby/api/stdcopy"
 	"github.com/moby/moby/api/types"
 	"github.com/moby/moby/api/types/container"
 	"github.com/moby/moby/client"

integration-cli/docker_api_logs_test.go

@@ -12,9 +12,9 @@ import (
 	"time"
 
 	"github.com/docker/docker/integration-cli/cli"
-	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/docker/docker/testutil"
 	"github.com/docker/docker/testutil/request"
+	"github.com/moby/moby/api/stdcopy"
 	"github.com/moby/moby/api/types/container"
 	"github.com/moby/moby/client"
 	"gotest.tools/v3/assert"

integration/build/build_squash_test.go

@@ -7,10 +7,10 @@ import (
 	"testing"
 
 	"github.com/docker/docker/integration/internal/container"
-	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/docker/docker/testutil"
 	"github.com/docker/docker/testutil/daemon"
 	"github.com/docker/docker/testutil/fakecontext"
+	"github.com/moby/moby/api/stdcopy"
 	"github.com/moby/moby/api/types/build"
 	containertypes "github.com/moby/moby/api/types/container"
 	dclient "github.com/moby/moby/client"

integration/build/build_userns_linux_test.go

@@ -11,11 +11,11 @@ import (
 
 	"github.com/docker/docker/integration/internal/container"
 	"github.com/docker/docker/pkg/jsonmessage"
-	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/docker/docker/testutil"
 	"github.com/docker/docker/testutil/daemon"
 	"github.com/docker/docker/testutil/fakecontext"
 	"github.com/docker/docker/testutil/fixtures/load"
+	"github.com/moby/moby/api/stdcopy"
 	"github.com/moby/moby/api/types/build"
 	containertypes "github.com/moby/moby/api/types/container"
 	"gotest.tools/v3/assert"

integration/capabilities/capabilities_linux_test.go

@@ -7,12 +7,11 @@ import (
 	"testing"
 
 	"github.com/docker/docker/integration/internal/container"
-	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/docker/docker/testutil"
 	"github.com/docker/docker/testutil/fakecontext"
+	"github.com/moby/moby/api/stdcopy"
 	"github.com/moby/moby/api/types/build"
 	containertypes "github.com/moby/moby/api/types/container"
-
 	"gotest.tools/v3/assert"
 	"gotest.tools/v3/poll"
 )

integration/config/config_test.go

@@ -10,8 +10,8 @@ import (
 
 	cerrdefs "github.com/containerd/errdefs"
 	"github.com/docker/docker/integration/internal/swarm"
-	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/docker/docker/testutil"
+	"github.com/moby/moby/api/stdcopy"
 	"github.com/moby/moby/api/types/container"
 	"github.com/moby/moby/api/types/filters"
 	swarmtypes "github.com/moby/moby/api/types/swarm"

integration/container/cdi_test.go

@@ -9,9 +9,9 @@ import (
 	"testing"
 
 	"github.com/docker/docker/integration/internal/container"
-	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/docker/docker/testutil"
 	"github.com/docker/docker/testutil/daemon"
+	"github.com/moby/moby/api/stdcopy"
 	containertypes "github.com/moby/moby/api/types/container"
 	"github.com/moby/moby/api/types/system"
 	"gotest.tools/v3/assert"
@@ -179,7 +179,7 @@ func TestCDIInfoDiscoveredDevices(t *testing.T) {
 	cdiDir := testutil.TempDir(t)
 	specFilePath := filepath.Join(cdiDir, "test-device.json")
 
-	err := os.WriteFile(specFilePath, []byte(specContent), 0644)
+	err := os.WriteFile(specFilePath, []byte(specContent), 0o644)
 	assert.NilError(t, err, "Failed to write sample CDI spec file")
 
 	d := daemon.New(t)

integration/container/logs_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/docker/docker/daemon/logger/local"
 	"github.com/docker/docker/integration/internal/container"
 	"github.com/docker/docker/integration/internal/termtest"
-	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/moby/moby/api/stdcopy"
 	containertypes "github.com/moby/moby/api/types/container"
 	"gotest.tools/v3/assert"
 	is "gotest.tools/v3/assert/cmp"

integration/container/run_linux_test.go

@@ -11,9 +11,9 @@ import (
 
 	"github.com/docker/docker/integration/internal/container"
 	net "github.com/docker/docker/integration/internal/network"
-	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/docker/docker/testutil"
 	"github.com/docker/docker/testutil/daemon"
+	"github.com/moby/moby/api/stdcopy"
 	containertypes "github.com/moby/moby/api/types/container"
 	"github.com/moby/moby/api/types/versions"
 	"github.com/moby/moby/client"

integration/container/stop_linux_test.go

@@ -10,7 +10,7 @@ import (
 
 	cerrdefs "github.com/containerd/errdefs"
 	"github.com/docker/docker/integration/internal/container"
-	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/moby/moby/api/stdcopy"
 	containertypes "github.com/moby/moby/api/types/container"
 	"github.com/moby/moby/client"
 	"gotest.tools/v3/assert"

integration/daemon/daemon_test.go

@@ -18,9 +18,9 @@ import (
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/integration/internal/container"
 	"github.com/docker/docker/integration/internal/process"
-	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/docker/docker/testutil"
 	"github.com/docker/docker/testutil/daemon"
+	"github.com/moby/moby/api/stdcopy"
 	containertypes "github.com/moby/moby/api/types/container"
 	"github.com/moby/moby/api/types/image"
 	"github.com/moby/moby/api/types/mount"

integration/internal/container/container.go

@@ -8,7 +8,7 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/moby/moby/api/stdcopy"
 	"github.com/moby/moby/api/types"
 	"github.com/moby/moby/api/types/container"
 	"github.com/moby/moby/api/types/network"

integration/networking/port_mapping_linux_test.go

@@ -19,10 +19,10 @@ import (
 	"github.com/docker/docker/integration/internal/container"
 	"github.com/docker/docker/integration/internal/network"
 	"github.com/docker/docker/internal/testutils/networking"
-	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/docker/docker/testutil"
 	"github.com/docker/docker/testutil/daemon"
 	"github.com/docker/go-connections/nat"
+	"github.com/moby/moby/api/stdcopy"
 	containertypes "github.com/moby/moby/api/types/container"
 	networktypes "github.com/moby/moby/api/types/network"
 	"github.com/moby/moby/client"

integration/plugin/logging/read_test.go

@@ -8,9 +8,9 @@ import (
 	"time"
 
 	testContainer "github.com/docker/docker/integration/internal/container"
-	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/docker/docker/testutil"
 	"github.com/docker/docker/testutil/daemon"
+	"github.com/moby/moby/api/stdcopy"
 	"github.com/moby/moby/api/types"
 	"github.com/moby/moby/api/types/container"
 	"gotest.tools/v3/assert"

integration/secret/secret_test.go

@@ -10,8 +10,8 @@ import (
 
 	cerrdefs "github.com/containerd/errdefs"
 	"github.com/docker/docker/integration/internal/swarm"
-	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/docker/docker/testutil"
+	"github.com/moby/moby/api/stdcopy"
 	"github.com/moby/moby/api/types/container"
 	"github.com/moby/moby/api/types/filters"
 	swarmtypes "github.com/moby/moby/api/types/swarm"

pkg/stdcopy/stdcopy_deprecated.go

@@ -0,0 +1,35 @@
+package stdcopy // Deprecated: use [github.com/docker/docker/api/stdcopy] instead.
+
+import (
+	"io"
+
+	"github.com/moby/moby/api/stdcopy"
+)
+
+// TODO(thaJeztah): remove exception in hack/make.ps1 and hack/validate/pkg-imports when removing.
+
+// StdType is the type of standard stream
+// a writer can multiplex to.
+//
+// Deprecated: use [stdcopy.StdType]. This alias will be removed in the next release.
+type StdType = stdcopy.StdType
+
+const (
+	Stdin  = stdcopy.Stdin  // Deprecated: use [stdcopy.Stderr]. This alias will be removed in the next release.
+	Stdout = stdcopy.Stdout // Deprecated: use [stdcopy.Stdout]. This alias will be removed in the next release.
+	Stderr = stdcopy.Stderr // Deprecated: use [stdcopy.Stderr]. This alias will be removed in the next release.
+)
+
+// NewStdWriter instantiates a new Writer.
+//
+// Deprecated: use [stdcopy.NewStdWriter]. This alias will be removed in the next release.
+func NewStdWriter(w io.Writer, t stdcopy.StdType) io.Writer {
+	return stdcopy.NewStdWriter(w, t)
+}
+
+// StdCopy is a modified version of io.Copy.
+//
+// Deprecated: use [stdcopy.StdCopy]. This alias will be removed in the next release.
+func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, _ error) {
+	return stdcopy.StdCopy(dstout, dsterr, src)
+}

vendor/github.com/moby/moby/api/stdcopy/stdcopy.go

@@ -0,0 +1,190 @@
+package stdcopy
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"sync"
+)
+
+// StdType is the type of standard stream
+// a writer can multiplex to.
+type StdType byte
+
+const (
+	// Stdin represents standard input stream type.
+	Stdin StdType = iota
+	// Stdout represents standard output stream type.
+	Stdout
+	// Stderr represents standard error steam type.
+	Stderr
+	// Systemerr represents errors originating from the system that make it
+	// into the multiplexed stream.
+	Systemerr
+
+	stdWriterPrefixLen = 8
+	stdWriterFdIndex   = 0
+	stdWriterSizeIndex = 4
+
+	startingBufLen = 32*1024 + stdWriterPrefixLen + 1
+)
+
+var bufPool = &sync.Pool{New: func() interface{} { return bytes.NewBuffer(nil) }}
+
+// stdWriter is wrapper of io.Writer with extra customized info.
+type stdWriter struct {
+	io.Writer
+	prefix byte
+}
+
+// Write sends the buffer to the underneath writer.
+// It inserts the prefix header before the buffer,
+// so stdcopy.StdCopy knows where to multiplex the output.
+// It makes stdWriter to implement io.Writer.
+func (w *stdWriter) Write(p []byte) (int, error) {
+	if w == nil || w.Writer == nil {
+		return 0, errors.New("writer not instantiated")
+	}
+	if p == nil {
+		return 0, nil
+	}
+
+	header := [stdWriterPrefixLen]byte{stdWriterFdIndex: w.prefix}
+	binary.BigEndian.PutUint32(header[stdWriterSizeIndex:], uint32(len(p)))
+	buf := bufPool.Get().(*bytes.Buffer)
+	buf.Write(header[:])
+	buf.Write(p)
+
+	n, err := w.Writer.Write(buf.Bytes())
+	n -= stdWriterPrefixLen
+	if n < 0 {
+		n = 0
+	}
+
+	buf.Reset()
+	bufPool.Put(buf)
+	return n, err
+}
+
+// NewStdWriter instantiates a new Writer.
+// Everything written to it will be encapsulated using a custom format,
+// and written to the underlying `w` stream.
+// This allows multiple write streams (e.g. stdout and stderr) to be muxed into a single connection.
+// `t` indicates the id of the stream to encapsulate.
+// It can be stdcopy.Stdin, stdcopy.Stdout, stdcopy.Stderr.
+func NewStdWriter(w io.Writer, t StdType) io.Writer {
+	return &stdWriter{
+		Writer: w,
+		prefix: byte(t),
+	}
+}
+
+// StdCopy is a modified version of io.Copy.
+//
+// StdCopy will demultiplex `src`, assuming that it contains two streams,
+// previously multiplexed together using a StdWriter instance.
+// As it reads from `src`, StdCopy will write to `dstout` and `dsterr`.
+//
+// StdCopy will read until it hits EOF on `src`. It will then return a nil error.
+// In other words: if `err` is non nil, it indicates a real underlying error.
+//
+// `written` will hold the total number of bytes written to `dstout` and `dsterr`.
+func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, _ error) {
+	var (
+		buf       = make([]byte, startingBufLen)
+		bufLen    = len(buf)
+		nr, nw    int
+		err       error
+		out       io.Writer
+		frameSize int
+	)
+
+	for {
+		// Make sure we have at least a full header
+		for nr < stdWriterPrefixLen {
+			var nr2 int
+			nr2, err = src.Read(buf[nr:])
+			nr += nr2
+			if errors.Is(err, io.EOF) {
+				if nr < stdWriterPrefixLen {
+					return written, nil
+				}
+				break
+			}
+			if err != nil {
+				return 0, err
+			}
+		}
+
+		stream := StdType(buf[stdWriterFdIndex])
+		// Check the first byte to know where to write
+		switch stream {
+		case Stdin:
+			fallthrough
+		case Stdout:
+			// Write on stdout
+			out = dstout
+		case Stderr:
+			// Write on stderr
+			out = dsterr
+		case Systemerr:
+			// If we're on Systemerr, we won't write anywhere.
+			// NB: if this code changes later, make sure you don't try to write
+			// to outstream if Systemerr is the stream
+			out = nil
+		default:
+			return 0, fmt.Errorf("Unrecognized input header: %d", buf[stdWriterFdIndex])
+		}
+
+		// Retrieve the size of the frame
+		frameSize = int(binary.BigEndian.Uint32(buf[stdWriterSizeIndex : stdWriterSizeIndex+4]))
+
+		// Check if the buffer is big enough to read the frame.
+		// Extend it if necessary.
+		if frameSize+stdWriterPrefixLen > bufLen {
+			buf = append(buf, make([]byte, frameSize+stdWriterPrefixLen-bufLen+1)...)
+			bufLen = len(buf)
+		}
+
+		// While the amount of bytes read is less than the size of the frame + header, we keep reading
+		for nr < frameSize+stdWriterPrefixLen {
+			var nr2 int
+			nr2, err = src.Read(buf[nr:])
+			nr += nr2
+			if errors.Is(err, io.EOF) {
+				if nr < frameSize+stdWriterPrefixLen {
+					return written, nil
+				}
+				break
+			}
+			if err != nil {
+				return 0, err
+			}
+		}
+
+		// we might have an error from the source mixed up in our multiplexed
+		// stream. if we do, return it.
+		if stream == Systemerr {
+			return written, fmt.Errorf("error from daemon in stream: %s", string(buf[stdWriterPrefixLen:frameSize+stdWriterPrefixLen]))
+		}
+
+		// Write the retrieved frame (without header)
+		nw, err = out.Write(buf[stdWriterPrefixLen : frameSize+stdWriterPrefixLen])
+		if err != nil {
+			return 0, err
+		}
+
+		// If the frame has not been fully written: error
+		if nw != frameSize {
+			return 0, io.ErrShortWrite
+		}
+		written += int64(nw)
+
+		// Move the rest of the buffer to the beginning
+		copy(buf, buf[frameSize+stdWriterPrefixLen:])
+		// Move the index
+		nr -= frameSize + stdWriterPrefixLen
+	}
+}

vendor/github.com/moby/moby/client/container_attach.go

@@ -31,7 +31,7 @@ import (
 // SIZE1, SIZE2, SIZE3, and SIZE4 are four bytes of uint32 encoded as big endian.
 // This is the size of OUTPUT.
 //
-// You can use github.com/docker/docker/pkg/stdcopy.StdCopy to demultiplex this
+// You can use github.com/moby/moby/api/stdcopy.StdCopy to demultiplex this
 // stream.
 func (cli *Client) ContainerAttach(ctx context.Context, containerID string, options container.AttachOptions) (types.HijackedResponse, error) {
 	containerID, err := trimID("container", containerID)

vendor/github.com/moby/moby/client/container_exec.go

@@ -68,7 +68,7 @@ func (cli *Client) ContainerExecStart(ctx context.Context, execID string, config
 //   - If the container is *not* using a TTY, streams for stdout and stderr are
 //     multiplexed.
 //
-// You can use [github.com/docker/docker/pkg/stdcopy.StdCopy] to demultiplex this
+// You can use [github.com/moby/moby/api/stdcopy.StdCopy] to demultiplex this
 // stream. Refer to [Client.ContainerAttach] for details about the multiplexed
 // stream.
 func (cli *Client) ContainerExecAttach(ctx context.Context, execID string, config container.ExecAttachOptions) (types.HijackedResponse, error) {

vendor/github.com/moby/moby/client/container_logs.go

@@ -31,7 +31,7 @@ import (
 // SIZE1, SIZE2, SIZE3, and SIZE4 are four bytes of uint32 encoded as big endian.
 // This is the size of OUTPUT.
 //
-// You can use github.com/docker/docker/pkg/stdcopy.StdCopy to demultiplex this
+// You can use github.com/moby/moby/api/stdcopy.StdCopy to demultiplex this
 // stream.
 func (cli *Client) ContainerLogs(ctx context.Context, containerID string, options container.LogsOptions) (io.ReadCloser, error) {
 	containerID, err := trimID("container", containerID)

vendor/modules.txt

@@ -939,6 +939,7 @@ github.com/moby/locker
 # github.com/moby/moby/api v0.0.0 => ./api
 ## explicit; go 1.23.0
 github.com/moby/moby/api
+github.com/moby/moby/api/stdcopy
 github.com/moby/moby/api/types
 github.com/moby/moby/api/types/auxprogress
 github.com/moby/moby/api/types/backend