daemon: Emit Image Create event when image is built

Signed-off-by: Paweł Gronowski <pawel.gronowski@docker.com>

api/server/router/system/system_routes.go

@@ -273,7 +273,18 @@ func (s *systemRouter) getEvents(ctx context.Context, w http.ResponseWriter, r *
 	buffered, l := s.backend.SubscribeToEvents(since, until, ef)
 	defer s.backend.UnsubscribeFromEvents(l)
 
+	shouldSkip := func(ev events.Message) bool { return false }
+	if versions.LessThan(httputils.VersionFromContext(ctx), "1.46") {
+		// Image create events were added in API 1.46
+		shouldSkip = func(ev events.Message) bool {
+			return ev.Type == "image" && ev.Action == "create"
+		}
+	}
+
 	for _, ev := range buffered {
+		if shouldSkip(ev) {
+			continue
+		}
 		if err := enc.Encode(ev); err != nil {
 			return err
 		}
@@ -291,6 +302,9 @@ func (s *systemRouter) getEvents(ctx context.Context, w http.ResponseWriter, r *
 				log.G(ctx).Warnf("unexpected event message: %q", ev)
 				continue
 			}
+			if shouldSkip(jev) {
+				continue
+			}
 			if err := enc.Encode(jev); err != nil {
 				return err
 			}

api/swagger.yaml

@@ -9507,7 +9507,7 @@ paths:
 
         Containers report these events: `attach`, `commit`, `copy`, `create`, `destroy`, `detach`, `die`, `exec_create`, `exec_detach`, `exec_start`, `exec_die`, `export`, `health_status`, `kill`, `oom`, `pause`, `rename`, `resize`, `restart`, `start`, `stop`, `top`, `unpause`, `update`, and `prune`
 
-        Images report these events: `delete`, `import`, `load`, `pull`, `push`, `save`, `tag`, `untag`, and `prune`
+        Images report these events: `create, `delete`, `import`, `load`, `pull`, `push`, `save`, `tag`, `untag`, and `prune`
 
         Volumes report these events: `create`, `mount`, `unmount`, `destroy`, and `prune`
 

cmd/dockerd/daemon.go

@@ -430,23 +430,24 @@ func newRouterOptions(ctx context.Context, config *config.Config, d *daemon.Daem
 	cgroupParent := newCgroupParent(config)
 
 	bk, err := buildkit.New(ctx, buildkit.Opt{
-		SessionManager:      sm,
-		Root:                filepath.Join(config.Root, "buildkit"),
-		EngineID:            d.ID(),
-		Dist:                d.DistributionServices(),
-		ImageTagger:         d.ImageService(),
-		NetworkController:   d.NetworkController(),
-		DefaultCgroupParent: cgroupParent,
-		RegistryHosts:       d.RegistryHosts,
-		BuilderConfig:       config.Builder,
-		Rootless:            daemon.Rootless(config),
-		IdentityMapping:     d.IdentityMapping(),
-		DNSConfig:           config.DNSConfig,
-		ApparmorProfile:     daemon.DefaultApparmorProfile(),
-		UseSnapshotter:      d.UsesSnapshotter(),
-		Snapshotter:         d.ImageService().StorageDriver(),
-		ContainerdAddress:   config.ContainerdAddr,
-		ContainerdNamespace: config.ContainerdNamespace,
+		SessionManager:        sm,
+		Root:                  filepath.Join(config.Root, "buildkit"),
+		EngineID:              d.ID(),
+		Dist:                  d.DistributionServices(),
+		ImageTagger:           d.ImageService(),
+		NetworkController:     d.NetworkController(),
+		DefaultCgroupParent:   cgroupParent,
+		RegistryHosts:         d.RegistryHosts,
+		BuilderConfig:         config.Builder,
+		Rootless:              daemon.Rootless(config),
+		IdentityMapping:       d.IdentityMapping(),
+		DNSConfig:             config.DNSConfig,
+		ApparmorProfile:       daemon.DefaultApparmorProfile(),
+		UseSnapshotter:        d.UsesSnapshotter(),
+		Snapshotter:           d.ImageService().StorageDriver(),
+		ContainerdAddress:     config.ContainerdAddr,
+		ContainerdNamespace:   config.ContainerdNamespace,
+		ImageExportedCallback: d.ImageExportedByBuildkit,
 	})
 	if err != nil {
 		return routerOptions{}, err

daemon/build.go

@@ -0,0 +1,16 @@
+package daemon
+
+import (
+	"context"
+
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// ImageExportedByBuildkit is a callback that is called when an image is exported by buildkit.
+// This is used to log the image creation event for untagged images.
+// When no tag is given, buildkit doesn't call the image service so it has no
+// way of knowing the image was created.
+func (daemon *Daemon) ImageExportedByBuildkit(ctx context.Context, id string, desc ocispec.Descriptor) error {
+	daemon.imageService.LogImageEvent(id, id, "create")
+	return nil
+}

daemon/containerd/image_builder.go

@@ -23,9 +23,11 @@ import (
 	"github.com/distribution/reference"
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/events"
 	"github.com/docker/docker/api/types/registry"
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/errdefs"
+	"github.com/docker/docker/image"
 	dimage "github.com/docker/docker/image"
 	"github.com/docker/docker/layer"
 	"github.com/docker/docker/pkg/archive"
@@ -533,11 +535,14 @@ func (i *ImageService) createImageOCI(ctx context.Context, imgToCreate imagespec
 		}
 	}
 
+	id := image.ID(createdImage.Target.Digest)
+	i.LogImageEvent(id.String(), id.String(), events.ActionCreate)
+
 	if err := i.unpackImage(ctx, i.StorageDriver(), img, manifestDesc); err != nil {
 		return "", err
 	}
 
-	return dimage.ID(createdImage.Target.Digest), nil
+	return id, nil
 }
 
 // writeContentsForImage will commit oci image config and manifest into containerd's content store.

daemon/images/image_commit.go

@@ -6,6 +6,7 @@ import (
 	"io"
 
 	"github.com/docker/docker/api/types/backend"
+	"github.com/docker/docker/api/types/events"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/layer"
 	"github.com/docker/docker/pkg/ioutils"
@@ -62,6 +63,9 @@ func (i *ImageService) CommitImage(ctx context.Context, c backend.CommitConfig)
 	if err != nil {
 		return "", err
 	}
+
+	i.LogImageEvent(id.String(), id.String(), events.ActionCreate)
+
 	if err := i.imageStore.SetBuiltLocally(id); err != nil {
 		return "", err
 	}

docs/api/version-history.md

@@ -32,6 +32,8 @@ keywords: "API, Docker, rcli, REST, documentation"
   the multi-platform image.
 * `POST /containers/create` now takes `Options` as part of `HostConfig.Mounts.TmpfsOptions` to set options for tmpfs mounts.
 * `POST /services/create` now takes `Options` as part of `ContainerSpec.Mounts.TmpfsOptions`, to set options for tmpfs mounts.
+* `GET /events` now supports image `create` event that is emitted when a new
+  image is built regardless if it was tagged or not.
 
 ### Deprecated Config fields in `GET /images/{name}/json` response
 

integration-cli/cli/build/build.go

@@ -2,6 +2,7 @@ package build // import "github.com/docker/docker/integration-cli/cli/build"
 
 import (
 	"io"
+	"os"
 	"strings"
 	"testing"
 
@@ -30,6 +31,21 @@ func WithDockerfile(dockerfile string) func(*icmd.Cmd) func() {
 	}
 }
 
+// WithBuildkit sets an DOCKER_BUILDKIT environment variable to make the build use buildkit (or not)
+func WithBuildkit(useBuildkit bool) func(*icmd.Cmd) func() {
+	return func(cmd *icmd.Cmd) func() {
+		val := "0"
+		if useBuildkit {
+			val = "1"
+		}
+		if cmd.Env == nil {
+			cmd.Env = os.Environ()
+		}
+		cmd.Env = append(cmd.Env, "DOCKER_BUILDKIT="+val)
+		return nil
+	}
+}
+
 // WithoutCache makes the build ignore cache
 func WithoutCache(cmd *icmd.Cmd) func() {
 	cmd.Command = append(cmd.Command, "--no-cache")

integration-cli/cli/cli.go

@@ -3,6 +3,7 @@ package cli // import "github.com/docker/docker/integration-cli/cli"
 import (
 	"fmt"
 	"io"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -161,7 +162,10 @@ func WithTimeout(timeout time.Duration) func(cmd *icmd.Cmd) func() {
 // WithEnvironmentVariables sets the specified environment variables for the command to run
 func WithEnvironmentVariables(envs ...string) func(cmd *icmd.Cmd) func() {
 	return func(cmd *icmd.Cmd) func() {
-		cmd.Env = envs
+		if cmd.Env == nil {
+			cmd.Env = os.Environ()
+		}
+		cmd.Env = append(cmd.Env, envs...)
 		return nil
 	}
 }

integration-cli/docker_cli_build_test.go

@@ -6192,3 +6192,41 @@ func (s *DockerCLIBuildSuite) TestBuildIidFileCleanupOnFail(c *testing.T) {
 	assert.ErrorContains(c, err, "")
 	assert.Equal(c, os.IsNotExist(err), true)
 }
+
+func (s *DockerCLIBuildSuite) TestBuildEmitsImageCreateEvent(t *testing.T) {
+	for _, tc := range []struct {
+		buildkit bool
+	}{
+		{buildkit: false},
+		{buildkit: true},
+	} {
+		tc := tc
+		t.Run(fmt.Sprintf("buildkit=%v", tc.buildkit), func(t *testing.T) {
+			skip.If(t, DaemonIsWindows, "Buildkit is not supported on Windows")
+
+			before := time.Now()
+
+			b := cli.Docker(cli.Args("build"),
+				build.WithoutCache,
+				build.WithDockerfile("FROM busybox\nRUN echo hi >/hello"),
+				build.WithBuildkit(tc.buildkit),
+			)
+			b.Assert(t, icmd.Success)
+			t.Log(b.Stdout())
+			t.Log(b.Stderr())
+
+			cmd := cli.Docker(
+				cli.Args("events",
+					"--filter", "action=create,type=image",
+					"--since", before.Format(time.RFC3339),
+				),
+				cli.WithTimeout(time.Millisecond*300),
+				cli.WithEnvironmentVariables("DOCKER_API_VERSION=v1.46"), // FIXME(thaJeztah): integration-cli runs docker CLI 17.06; we're "upgrading" the API version to a version it doesn't support here ;)
+			)
+
+			t.Log(cmd.Stdout())
+
+			assert.Check(t, is.Contains(cmd.Stdout(), "image create"))
+		})
+	}
+}

integration-cli/docker_cli_events_test.go

@@ -338,9 +338,10 @@ func (s *DockerCLIEventSuite) TestEventsFilterImageLabels(c *testing.T) {
 	label := "io.docker.testing=image"
 
 	// Build a test image.
-	buildImageSuccessfully(c, name, build.WithDockerfile(fmt.Sprintf(`
-		FROM busybox:latest
-		LABEL %s`, label)))
+	buildImageSuccessfully(c, name,
+		build.WithDockerfile("FROM busybox:latest\nLABEL "+label),
+		build.WithoutCache, // Make sure image is actually built
+	)
 	cli.DockerCmd(c, "tag", name, "labelfiltertest:tag1")
 	cli.DockerCmd(c, "tag", name, "labelfiltertest:tag2")
 	cli.DockerCmd(c, "tag", "busybox:latest", "labelfiltertest:tag3")
@@ -560,9 +561,10 @@ func (s *DockerCLIEventSuite) TestEventsFilterType(c *testing.T) {
 	label := "io.docker.testing=image"
 
 	// Build a test image.
-	buildImageSuccessfully(c, name, build.WithDockerfile(fmt.Sprintf(`
-		FROM busybox:latest
-		LABEL %s`, label)))
+	buildImageSuccessfully(c, name,
+		build.WithDockerfile("FROM busybox:latest\nLABEL "+label),
+		build.WithoutCache, // Make sure image is actually built
+	)
 	cli.DockerCmd(c, "tag", name, "labelfiltertest:tag1")
 	cli.DockerCmd(c, "tag", name, "labelfiltertest:tag2")
 	cli.DockerCmd(c, "tag", "busybox:latest", "labelfiltertest:tag3")
@@ -571,7 +573,7 @@ func (s *DockerCLIEventSuite) TestEventsFilterType(c *testing.T) {
 		"events",
 		"--since", since,
 		"--until", daemonUnixTime(c),
-		"--filter", fmt.Sprintf("label=%s", label),
+		"--filter", "label="+label,
 		"--filter", "type=image",
 	).Stdout()
 

integration/build/build_test.go

@@ -3,14 +3,17 @@ package build // import "github.com/docker/docker/integration/build"
 import (
 	"archive/tar"
 	"bytes"
+	"context"
 	"encoding/json"
 	"io"
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/events"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/pkg/jsonmessage"
@@ -727,6 +730,68 @@ func TestBuildWorkdirNoCacheMiss(t *testing.T) {
 	}
 }
 
+func TestBuildEmitsImageCreateEvent(t *testing.T) {
+	ctx := setupTest(t)
+
+	dockerfile := "FROM busybox\nRUN echo hello > /hello"
+	source := fakecontext.New(t, "", fakecontext.WithDockerfile(dockerfile))
+	defer source.Close()
+
+	apiClient := testEnv.APIClient()
+
+	for _, builderVersion := range []types.BuilderVersion{types.BuilderV1, types.BuilderBuildKit} {
+		builderVersion := builderVersion
+		t.Run("v"+string(builderVersion), func(t *testing.T) {
+			if builderVersion == types.BuilderBuildKit {
+				skip.If(t, testEnv.UsingSnapshotter(),
+					"FIXME: Passing a context via a tarball is not supported with the containerd image store. See: https://github.com/moby/moby/issues/47717")
+				skip.If(t, testEnv.DaemonInfo.OSType == "windows", "Buildkit is not supported on Windows")
+			}
+
+			ctx, cancel := context.WithCancel(ctx)
+			defer cancel()
+
+			since := time.Now()
+
+			resp, err := apiClient.ImageBuild(ctx, source.AsTarReader(t), types.ImageBuildOptions{
+				Version: builderVersion,
+				NoCache: true,
+			})
+			assert.NilError(t, err)
+
+			defer resp.Body.Close()
+
+			out := bytes.NewBuffer(nil)
+			_, err = io.Copy(out, resp.Body)
+			assert.NilError(t, err)
+
+			t.Log(out.String())
+
+			eventsChan, errs := apiClient.Events(ctx, events.ListOptions{
+				Since: since.Format(time.RFC3339Nano),
+				Until: time.Now().Format(time.RFC3339Nano),
+			})
+			imageCreateEvts := 0
+			finished := false
+			for !finished {
+				select {
+				case evt := <-eventsChan:
+					t.Log("Got event type:", evt.Type, "action:", evt.Action)
+
+					if evt.Type == events.ImageEventType && evt.Action == events.ActionCreate {
+						imageCreateEvts++
+					}
+				case err := <-errs:
+					assert.Check(t, err == nil || err == io.EOF)
+					finished = true
+				}
+			}
+
+			assert.Check(t, is.Equal(1, imageCreateEvts))
+		})
+	}
+}
+
 func readBuildImageIDs(t *testing.T, rd io.Reader) string {
 	t.Helper()
 	decoder := json.NewDecoder(rd)