mirror of
https://github.com/moby/moby.git
synced 2026-01-11 10:41:43 +00:00
0df791cb72
The Container.State struct holds the container's state, and most of
its fields are expected to change dynamically. Some o these state-changes
are explicit, for example, setting the container to be "stopped". Other
state changes can be more explicit, for example due to the containers'
process exiting or being "OOM" killed by the kernel.
The distinction between explicit ("desired") state changes and "state"
("actual state") is sometimes vague; for some properties, we clearly
separated them, for example if a user requested the container to be
stopped or restarted, we store state in the Container object itself;
HasBeenManuallyStopped bool // used for unless-stopped restart policy
HasBeenManuallyRestarted bool `json:"-"` // used to distinguish restart caused by restart policy from the manual one
Other properties are more ambiguous. such as "HasBeenStartedBefore" and
"RestartCount", which are stored on the Container (and persisted to
disk), but may be more related to "actual" state, and likely should
not be persisted;
RestartCount int
HasBeenStartedBefore bool
Given that (per the above) concurrency must be taken into account, most
changes to the `container.State` struct should be protected; here's where
things get blurry. While the `State` type provides various accessor methods,
only some of them take concurrency into account; for example, [State.IsRunning]
and [State.GetPID] acquire a lock, whereas [State.ExitCodeValue] does not.
Even the (commonly used) [State.StateString] has no locking at all.
The way to handle this is error-prone; [container.State] contains a mutex,
and it's exported. Given that its embedded in the [container.Container]
struct, it's also exposed as an exported mutex for the container. The
assumption here is that by "merging" the two, the caller to acquire a lock
when either the container _or_ its state must be mutated. However, because
some methods on `container.State` handle their own locking, consumers must
be deeply familiar with the internals; if both changes to the `Container`
AND `Container.State` must be made. This gets amplified more as some
(exported!) methods, such as [container.SetRunning] mutate multiple fields,
but don't acquire a lock (so expect the caller to hold one), but their
(also exported) counterpart (e.g. [State.IsRunning]) do.
It should be clear from the above, that this needs some architectural
changes; a clearer separation between "desired" and "actual" state (opening
the potential to update the container's config without manually touching
its `State`), possibly a method to obtain a read-only copy of the current
state (for those querying state), and reviewing which fields belong where
(and should be persisted to disk, or only remain in memory).
This PR preserves the status quo; it makes no structural changes, other
than exposing where we access the container's state. Where previously the
State fields and methods were referred to as "part of the container"
(e.g. `ctr.IsRunning()` or `ctr.Running`), we now explicitly reference
the embedded `State` (`ctr.State.IsRunning`, `ctr.State.Running`).
The exception (for now) is the mutex, which is still referenced through
the embedded struct (`ctr.Lock()` instead of `ctr.State.Lock()`), as this
is (mostly) by design to protect the container, and what's in it (including
its `State`).
[State.IsRunning]: c4afa77157/daemon/container/state.go (L205-L209)
[State.GetPID]: c4afa77157/daemon/container/state.go (L211-L216)
[State.ExitCodeValue]: c4afa77157/daemon/container/state.go (L218-L228)
[State.StateString]: c4afa77157/daemon/container/state.go (L102-L131)
[container.State]: c4afa77157/daemon/container/state.go (L15-L23)
[container.Container]: c4afa77157/daemon/container/container.go (L67-L75)
[container.SetRunning]: c4afa77157/daemon/container/state.go (L230-L277)
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
297 lines
11 KiB
Go
297 lines
11 KiB
Go
package container
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
containertypes "github.com/moby/moby/api/types/container"
|
|
"github.com/moby/moby/client"
|
|
realcontainer "github.com/moby/moby/v2/daemon/container"
|
|
"github.com/moby/moby/v2/integration/internal/container"
|
|
"github.com/moby/moby/v2/internal/testutil"
|
|
"github.com/moby/moby/v2/internal/testutil/daemon"
|
|
"golang.org/x/sys/unix"
|
|
"gotest.tools/v3/assert"
|
|
is "gotest.tools/v3/assert/cmp"
|
|
"gotest.tools/v3/assert/opt"
|
|
"gotest.tools/v3/skip"
|
|
)
|
|
|
|
// This is a regression test for #36145
|
|
// It ensures that a container can be started when the daemon was improperly
|
|
// shutdown when the daemon is brought back up.
|
|
//
|
|
// The regression is due to improper error handling preventing a container from
|
|
// being restored and as such have the resources cleaned up.
|
|
//
|
|
// To test this, we need to kill dockerd, then kill both the containerd-shim and
|
|
// the container process, then start dockerd back up and attempt to start the
|
|
// container again.
|
|
func TestContainerStartOnDaemonRestart(t *testing.T) {
|
|
skip.If(t, testEnv.IsRemoteDaemon, "cannot start daemon on remote test run")
|
|
skip.If(t, testEnv.DaemonInfo.OSType == "windows")
|
|
skip.If(t, testEnv.IsRootless)
|
|
t.Parallel()
|
|
|
|
ctx := testutil.StartSpan(baseContext, t)
|
|
|
|
d := daemon.New(t)
|
|
d.StartWithBusybox(ctx, t, "--iptables=false", "--ip6tables=false")
|
|
defer d.Stop(t)
|
|
|
|
c := d.NewClientT(t)
|
|
|
|
cID := container.Create(ctx, t, c)
|
|
defer c.ContainerRemove(ctx, cID, client.ContainerRemoveOptions{Force: true})
|
|
|
|
err := c.ContainerStart(ctx, cID, client.ContainerStartOptions{})
|
|
assert.Check(t, err, "error starting test container")
|
|
|
|
inspect, err := c.ContainerInspect(ctx, cID)
|
|
assert.Check(t, err, "error getting inspect data")
|
|
|
|
ppid := getContainerdShimPid(t, inspect)
|
|
|
|
err = d.Kill()
|
|
assert.Check(t, err, "failed to kill test daemon")
|
|
|
|
err = unix.Kill(inspect.State.Pid, unix.SIGKILL)
|
|
assert.Check(t, err, "failed to kill container process")
|
|
|
|
err = unix.Kill(ppid, unix.SIGKILL)
|
|
assert.Check(t, err, "failed to kill containerd-shim")
|
|
|
|
d.Start(t, "--iptables=false", "--ip6tables=false")
|
|
|
|
err = c.ContainerStart(ctx, cID, client.ContainerStartOptions{})
|
|
assert.Check(t, err, "failed to start test container")
|
|
}
|
|
|
|
func getContainerdShimPid(t *testing.T, c containertypes.InspectResponse) int {
|
|
statB, err := os.ReadFile(fmt.Sprintf("/proc/%d/stat", c.State.Pid))
|
|
assert.Check(t, err, "error looking up containerd-shim pid")
|
|
|
|
// ppid is the 4th entry in `/proc/pid/stat`
|
|
ppid, err := strconv.Atoi(strings.Fields(string(statB))[3])
|
|
assert.Check(t, err, "error converting ppid field to int")
|
|
|
|
assert.Check(t, ppid != 1, "got unexpected ppid")
|
|
return ppid
|
|
}
|
|
|
|
// TestDaemonRestartIpcMode makes sure a container keeps its ipc mode
|
|
// (derived from daemon default) even after the daemon is restarted
|
|
// with a different default ipc mode.
|
|
func TestDaemonRestartIpcMode(t *testing.T) {
|
|
skip.If(t, testEnv.IsRemoteDaemon, "cannot start daemon on remote test run")
|
|
skip.If(t, testEnv.DaemonInfo.OSType == "windows")
|
|
t.Parallel()
|
|
|
|
ctx := testutil.StartSpan(baseContext, t)
|
|
|
|
d := daemon.New(t)
|
|
d.StartWithBusybox(ctx, t, "--iptables=false", "--ip6tables=false", "--default-ipc-mode=private")
|
|
defer d.Stop(t)
|
|
|
|
c := d.NewClientT(t)
|
|
|
|
// check the container is created with private ipc mode as per daemon default
|
|
cID := container.Run(ctx, t, c,
|
|
container.WithCmd("top"),
|
|
container.WithRestartPolicy(containertypes.RestartPolicyAlways),
|
|
)
|
|
defer c.ContainerRemove(ctx, cID, client.ContainerRemoveOptions{Force: true})
|
|
|
|
inspect, err := c.ContainerInspect(ctx, cID)
|
|
assert.NilError(t, err)
|
|
assert.Check(t, is.Equal(string(inspect.HostConfig.IpcMode), "private"))
|
|
|
|
// restart the daemon with shareable default ipc mode
|
|
d.Restart(t, "--iptables=false", "--ip6tables=false", "--default-ipc-mode=shareable")
|
|
|
|
// check the container is still having private ipc mode
|
|
inspect, err = c.ContainerInspect(ctx, cID)
|
|
assert.NilError(t, err)
|
|
assert.Check(t, is.Equal(string(inspect.HostConfig.IpcMode), "private"))
|
|
|
|
// check a new container is created with shareable ipc mode as per new daemon default
|
|
cID = container.Run(ctx, t, c)
|
|
defer c.ContainerRemove(ctx, cID, client.ContainerRemoveOptions{Force: true})
|
|
|
|
inspect, err = c.ContainerInspect(ctx, cID)
|
|
assert.NilError(t, err)
|
|
assert.Check(t, is.Equal(string(inspect.HostConfig.IpcMode), "shareable"))
|
|
}
|
|
|
|
// TestDaemonHostGatewayIP verifies that when a magic string "host-gateway" is passed
|
|
// to ExtraHosts (--add-host) instead of an IP address, its value is set to
|
|
// 1. Daemon config flag value specified by host-gateway-ip or
|
|
// 2. IP of the default bridge network
|
|
// and is added to the /etc/hosts file
|
|
func TestDaemonHostGatewayIP(t *testing.T) {
|
|
skip.If(t, testEnv.IsRemoteDaemon)
|
|
skip.If(t, testEnv.DaemonInfo.OSType == "windows")
|
|
skip.If(t, testEnv.IsRootless, "rootless mode has different view of network")
|
|
t.Parallel()
|
|
|
|
ctx := testutil.StartSpan(baseContext, t)
|
|
|
|
// Verify the IP in /etc/hosts is same as host-gateway-ip
|
|
d := daemon.New(t)
|
|
// Verify the IP in /etc/hosts is same as the default bridge's IP
|
|
d.StartWithBusybox(ctx, t, "--iptables=false", "--ip6tables=false")
|
|
c := d.NewClientT(t)
|
|
cID := container.Run(ctx, t, c,
|
|
container.WithExtraHost("host.docker.internal:host-gateway"),
|
|
)
|
|
res, err := container.Exec(ctx, c, cID, []string{"cat", "/etc/hosts"})
|
|
assert.NilError(t, err)
|
|
assert.Assert(t, is.Len(res.Stderr(), 0))
|
|
assert.Equal(t, 0, res.ExitCode)
|
|
inspect, err := c.NetworkInspect(ctx, "bridge", client.NetworkInspectOptions{})
|
|
assert.NilError(t, err)
|
|
assert.Check(t, is.Contains(res.Stdout(), inspect.IPAM.Config[0].Gateway))
|
|
c.ContainerRemove(ctx, cID, client.ContainerRemoveOptions{Force: true})
|
|
d.Stop(t)
|
|
|
|
// Verify the IP in /etc/hosts is same as host-gateway-ip
|
|
d.StartWithBusybox(ctx, t, "--iptables=false", "--ip6tables=false", "--host-gateway-ip=6.7.8.9")
|
|
cID = container.Run(ctx, t, c,
|
|
container.WithExtraHost("host.docker.internal:host-gateway"),
|
|
)
|
|
res, err = container.Exec(ctx, c, cID, []string{"cat", "/etc/hosts"})
|
|
assert.NilError(t, err)
|
|
assert.Assert(t, is.Len(res.Stderr(), 0))
|
|
assert.Equal(t, 0, res.ExitCode)
|
|
assert.Check(t, is.Contains(res.Stdout(), "6.7.8.9"))
|
|
c.ContainerRemove(ctx, cID, client.ContainerRemoveOptions{Force: true})
|
|
d.Stop(t)
|
|
}
|
|
|
|
// TestRestartDaemonWithRestartingContainer simulates a case where a container is in "restarting" state when
|
|
// dockerd is killed (due to machine reset or something else).
|
|
//
|
|
// Related to moby/moby#41817
|
|
//
|
|
// In this test we'll change the container state to "restarting".
|
|
// This means that the container will not be 'alive' when we attempt to restore in on daemon startup.
|
|
//
|
|
// We could do the same with `docker run -d --restart=always busybox:latest exit 1`, and then
|
|
// `kill -9` dockerd while the container is in "restarting" state. This is difficult to reproduce reliably
|
|
// in an automated test, so we manipulate on disk state instead.
|
|
func TestRestartDaemonWithRestartingContainer(t *testing.T) {
|
|
skip.If(t, testEnv.IsRemoteDaemon, "cannot start daemon on remote test run")
|
|
skip.If(t, testEnv.DaemonInfo.OSType == "windows")
|
|
|
|
t.Parallel()
|
|
|
|
ctx := testutil.StartSpan(baseContext, t)
|
|
|
|
d := daemon.New(t)
|
|
defer d.Cleanup(t)
|
|
|
|
d.StartWithBusybox(ctx, t, "--iptables=false", "--ip6tables=false")
|
|
defer d.Stop(t)
|
|
|
|
apiClient := d.NewClientT(t)
|
|
|
|
// Just create the container, no need to start it to be started.
|
|
// We really want to make sure there is no process running when docker starts back up.
|
|
// We will manipulate the on disk state later
|
|
id := container.Create(ctx, t, apiClient, container.WithRestartPolicy(containertypes.RestartPolicyAlways), container.WithCmd("/bin/sh", "-c", "exit 1"))
|
|
|
|
d.Stop(t)
|
|
|
|
d.TamperWithContainerConfig(t, id, func(c *realcontainer.Container) {
|
|
c.State.SetRestarting(&realcontainer.ExitStatus{ExitCode: 1})
|
|
c.HasBeenStartedBefore = true
|
|
})
|
|
|
|
d.Start(t, "--iptables=false", "--ip6tables=false")
|
|
|
|
ctxTimeout, cancel := context.WithTimeout(ctx, 30*time.Second)
|
|
defer cancel()
|
|
chOk, chErr := apiClient.ContainerWait(ctxTimeout, id, containertypes.WaitConditionNextExit)
|
|
select {
|
|
case <-chOk:
|
|
case err := <-chErr:
|
|
assert.NilError(t, err)
|
|
}
|
|
}
|
|
|
|
// TestHardRestartWhenContainerIsRunning simulates a case where dockerd is
|
|
// killed while a container is running, and the container's task no longer
|
|
// exists when dockerd starts back up. This can happen if the system is
|
|
// hard-rebooted, for example.
|
|
//
|
|
// Regression test for moby/moby#45788
|
|
func TestHardRestartWhenContainerIsRunning(t *testing.T) {
|
|
skip.If(t, testEnv.IsRemoteDaemon, "cannot start daemon on remote test run")
|
|
skip.If(t, testEnv.DaemonInfo.OSType == "windows")
|
|
|
|
t.Parallel()
|
|
|
|
ctx := testutil.StartSpan(baseContext, t)
|
|
|
|
d := daemon.New(t)
|
|
defer d.Cleanup(t)
|
|
|
|
d.StartWithBusybox(ctx, t, "--iptables=false", "--ip6tables=false")
|
|
defer d.Stop(t)
|
|
|
|
apiClient := d.NewClientT(t)
|
|
|
|
// Just create the containers, no need to start them.
|
|
// We really want to make sure there is no process running when docker starts back up.
|
|
// We will manipulate the on disk state later.
|
|
noPolicy := container.Create(ctx, t, apiClient, container.WithCmd("/bin/sh", "-c", "exit 1"))
|
|
onFailure := container.Create(ctx, t, apiClient, container.WithRestartPolicy("on-failure"), container.WithCmd("/bin/sh", "-c", "sleep 60"))
|
|
|
|
d.Stop(t)
|
|
|
|
for _, id := range []string{noPolicy, onFailure} {
|
|
d.TamperWithContainerConfig(t, id, func(c *realcontainer.Container) {
|
|
c.State.SetRunning(nil, nil, time.Now())
|
|
c.HasBeenStartedBefore = true
|
|
})
|
|
}
|
|
|
|
d.Start(t, "--iptables=false", "--ip6tables=false")
|
|
|
|
t.Run("RestartPolicy=none", func(t *testing.T) {
|
|
ctx := testutil.StartSpan(ctx, t)
|
|
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
defer cancel()
|
|
inspect, err := apiClient.ContainerInspect(ctx, noPolicy)
|
|
assert.NilError(t, err)
|
|
assert.Check(t, is.Equal(inspect.State.Status, containertypes.StateExited))
|
|
assert.Check(t, is.Equal(inspect.State.ExitCode, 255))
|
|
finishedAt, err := time.Parse(time.RFC3339Nano, inspect.State.FinishedAt)
|
|
if assert.Check(t, err) {
|
|
assert.Check(t, is.DeepEqual(finishedAt, time.Now(), opt.TimeWithThreshold(time.Minute)))
|
|
}
|
|
})
|
|
|
|
t.Run("RestartPolicy=on-failure", func(t *testing.T) {
|
|
ctx := testutil.StartSpan(ctx, t)
|
|
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
defer cancel()
|
|
inspect, err := apiClient.ContainerInspect(ctx, onFailure)
|
|
assert.NilError(t, err)
|
|
assert.Check(t, is.Equal(inspect.State.Status, containertypes.StateRunning))
|
|
assert.Check(t, is.Equal(inspect.State.ExitCode, 0))
|
|
finishedAt, err := time.Parse(time.RFC3339Nano, inspect.State.FinishedAt)
|
|
if assert.Check(t, err) {
|
|
assert.Check(t, is.DeepEqual(finishedAt, time.Now(), opt.TimeWithThreshold(time.Minute)))
|
|
}
|
|
|
|
stopTimeout := 0
|
|
assert.Assert(t, apiClient.ContainerStop(ctx, onFailure, client.ContainerStopOptions{Timeout: &stopTimeout}))
|
|
})
|
|
}
|