Merge pull request #22340 from Microsoft/jstarks/split-service

Windows: Support running dockerd as a service

cmd/docker/docker_windows.go

@@ -1,5 +1,5 @@
 package main
 
 import (
-	_ "github.com/docker/docker/autogen/winresources/dockerd"
+	_ "github.com/docker/docker/autogen/winresources/docker"
 )

cmd/dockerd/daemon.go

@@ -53,6 +53,9 @@ type DaemonCli struct {
 	*daemon.Config
 	commonFlags *cli.CommonFlags
 	configFile  *string
+
+	api *apiserver.Server
+	d   *daemon.Daemon
 }
 
 func presentInHelp(usage string) string { return usage }
@@ -121,7 +124,10 @@ func migrateKey() (err error) {
 	return nil
 }
 
-func (cli *DaemonCli) start() {
+func (cli *DaemonCli) start() (err error) {
+	stopc := make(chan bool)
+	defer close(stopc)
+
 	// warn from uuid package when running the daemon
 	uuid.Loggerf = logrus.Warnf
 
@@ -133,8 +139,7 @@ func (cli *DaemonCli) start() {
 	}
 	cliConfig, err := loadDaemonCliConfig(cli.Config, flags, cli.commonFlags, *cli.configFile)
 	if err != nil {
-		fmt.Fprint(os.Stderr, err)
-		os.Exit(1)
+		return err
 	}
 	cli.Config = cliConfig
 
@@ -152,24 +157,22 @@ func (cli *DaemonCli) start() {
 	})
 
 	if err := setDefaultUmask(); err != nil {
-		logrus.Fatalf("Failed to set umask: %v", err)
+		return fmt.Errorf("Failed to set umask: %v", err)
 	}
 
 	if len(cli.LogConfig.Config) > 0 {
 		if err := logger.ValidateLogOpts(cli.LogConfig.Type, cli.LogConfig.Config); err != nil {
-			logrus.Fatalf("Failed to set log opts: %v", err)
+			return fmt.Errorf("Failed to set log opts: %v", err)
 		}
 	}
 
-	var pfile *pidfile.PIDFile
 	if cli.Pidfile != "" {
 		pf, err := pidfile.New(cli.Pidfile)
 		if err != nil {
-			logrus.Fatalf("Error starting daemon: %v", err)
+			return fmt.Errorf("Error starting daemon: %v", err)
 		}
-		pfile = pf
 		defer func() {
-			if err := pfile.Remove(); err != nil {
+			if err := pf.Remove(); err != nil {
 				logrus.Error(err)
 			}
 		}()
@@ -196,7 +199,7 @@ func (cli *DaemonCli) start() {
 		}
 		tlsConfig, err := tlsconfig.Server(tlsOptions)
 		if err != nil {
-			logrus.Fatal(err)
+			return err
 		}
 		serverConfig.TLSConfig = tlsConfig
 	}
@@ -210,13 +213,13 @@ func (cli *DaemonCli) start() {
 	for i := 0; i < len(cli.Config.Hosts); i++ {
 		var err error
 		if cli.Config.Hosts[i], err = opts.ParseHost(cli.Config.TLS, cli.Config.Hosts[i]); err != nil {
-			logrus.Fatalf("error parsing -H %s : %v", cli.Config.Hosts[i], err)
+			return fmt.Errorf("error parsing -H %s : %v", cli.Config.Hosts[i], err)
 		}
 
 		protoAddr := cli.Config.Hosts[i]
 		protoAddrParts := strings.SplitN(protoAddr, "://", 2)
 		if len(protoAddrParts) != 2 {
-			logrus.Fatalf("bad format %s, expected PROTO://ADDR", protoAddr)
+			return fmt.Errorf("bad format %s, expected PROTO://ADDR", protoAddr)
 		}
 
 		proto := protoAddrParts[0]
@@ -228,12 +231,12 @@ func (cli *DaemonCli) start() {
 		}
 		l, err := listeners.Init(proto, addr, serverConfig.SocketGroup, serverConfig.TLSConfig)
 		if err != nil {
-			logrus.Fatal(err)
+			return err
 		}
 		// If we're binding to a TCP port, make sure that a container doesn't try to use it.
 		if proto == "tcp" {
 			if err := allocateDaemonPort(addr); err != nil {
-				logrus.Fatal(err)
+				return err
 			}
 		}
 		logrus.Debugf("Listener created for HTTP on %s (%s)", protoAddrParts[0], protoAddrParts[1])
@@ -241,24 +244,19 @@ func (cli *DaemonCli) start() {
 	}
 
 	if err := migrateKey(); err != nil {
-		logrus.Fatal(err)
+		return err
 	}
 	cli.TrustKeyPath = cli.commonFlags.TrustKey
 
 	registryService := registry.NewService(cli.Config.ServiceOptions)
 	containerdRemote, err := libcontainerd.New(cli.getLibcontainerdRoot(), cli.getPlatformRemoteOptions()...)
 	if err != nil {
-		logrus.Fatal(err)
+		return err
 	}
 
 	d, err := daemon.NewDaemon(cli.Config, registryService, containerdRemote)
 	if err != nil {
-		if pfile != nil {
-			if err := pfile.Remove(); err != nil {
-				logrus.Error(err)
-			}
-		}
-		logrus.Fatalf("Error starting daemon: %v", err)
+		return fmt.Errorf("Error starting daemon: %v", err)
 	}
 
 	logrus.Info("Daemon has completed initialization")
@@ -272,26 +270,9 @@ func (cli *DaemonCli) start() {
 	cli.initMiddlewares(api, serverConfig)
 	initRouter(api, d)
 
-	reload := func(config *daemon.Config) {
-		if err := d.Reload(config); err != nil {
-			logrus.Errorf("Error reconfiguring the daemon: %v", err)
-			return
-		}
-		if config.IsValueSet("debug") {
-			debugEnabled := utils.IsDebugEnabled()
-			switch {
-			case debugEnabled && !config.Debug: // disable debug
-				utils.DisableDebug()
-				api.DisableProfiler()
-			case config.Debug && !debugEnabled: // enable debug
-				utils.EnableDebug()
-				api.EnableProfiler()
-			}
-
-		}
-	}
-
-	setupConfigReloadTrap(*cli.configFile, flags, reload)
+	cli.d = d
+	cli.api = api
+	cli.setupConfigReloadTrap()
 
 	// The serve API routine never exits unless an error occurs
 	// We need to start it as a goroutine and wait on it so
@@ -300,14 +281,8 @@ func (cli *DaemonCli) start() {
 	go api.Wait(serveAPIWait)
 
 	signal.Trap(func() {
-		api.Close()
-		<-serveAPIWait
-		shutdownDaemon(d, 15)
-		if pfile != nil {
-			if err := pfile.Remove(); err != nil {
-				logrus.Error(err)
-			}
-		}
+		cli.stop()
+		<-stopc // wait for daemonCli.start() to return
 	})
 
 	// after the daemon is done setting up we can notify systemd api
@@ -319,13 +294,39 @@ func (cli *DaemonCli) start() {
 	shutdownDaemon(d, 15)
 	containerdRemote.Cleanup()
 	if errAPI != nil {
-		if pfile != nil {
-			if err := pfile.Remove(); err != nil {
-				logrus.Error(err)
-			}
-		}
-		logrus.Fatalf("Shutting down due to ServeAPI error: %v", errAPI)
+		return fmt.Errorf("Shutting down due to ServeAPI error: %v", errAPI)
 	}
+
+	return nil
+}
+
+func (cli *DaemonCli) reloadConfig() {
+	reload := func(config *daemon.Config) {
+		if err := cli.d.Reload(config); err != nil {
+			logrus.Errorf("Error reconfiguring the daemon: %v", err)
+			return
+		}
+		if config.IsValueSet("debug") {
+			debugEnabled := utils.IsDebugEnabled()
+			switch {
+			case debugEnabled && !config.Debug: // disable debug
+				utils.DisableDebug()
+				cli.api.DisableProfiler()
+			case config.Debug && !debugEnabled: // enable debug
+				utils.EnableDebug()
+				cli.api.EnableProfiler()
+			}
+
+		}
+	}
+
+	if err := daemon.ReloadConfiguration(*cli.configFile, flag.CommandLine, reload); err != nil {
+		logrus.Error(err)
+	}
+}
+
+func (cli *DaemonCli) stop() {
+	cli.api.Close()
 }
 
 // shutdownDaemon just wraps daemon.Shutdown() to handle a timeout in case

cmd/dockerd/daemon_unix.go

@@ -11,10 +11,8 @@ import (
 	"strconv"
 	"syscall"
 
-	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/daemon"
 	"github.com/docker/docker/libcontainerd"
-	"github.com/docker/docker/pkg/mflag"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/libnetwork/portallocator"
 )
@@ -49,14 +47,12 @@ func getDaemonConfDir() string {
 }
 
 // setupConfigReloadTrap configures the USR2 signal to reload the configuration.
-func setupConfigReloadTrap(configFile string, flags *mflag.FlagSet, reload func(*daemon.Config)) {
+func (cli *DaemonCli) setupConfigReloadTrap() {
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, syscall.SIGHUP)
 	go func() {
 		for range c {
-			if err := daemon.ReloadConfiguration(configFile, flags, reload); err != nil {
-				logrus.Error(err)
-			}
+			cli.reloadConfig()
 		}
 	}()
 }
@@ -111,3 +107,7 @@ func allocateDaemonPort(addr string) error {
 	}
 	return nil
 }
+
+// notifyShutdown is called after the daemon shuts down but before the process exits.
+func notifyShutdown(err error) {
+}

cmd/dockerd/daemon_windows.go

@@ -6,9 +6,7 @@ import (
 	"syscall"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/docker/docker/daemon"
 	"github.com/docker/docker/libcontainerd"
-	"github.com/docker/docker/pkg/mflag"
 	"github.com/docker/docker/pkg/system"
 )
 
@@ -31,10 +29,23 @@ func getDaemonConfDir() string {
 
 // notifySystem sends a message to the host when the server is ready to be used
 func notifySystem() {
+	if service != nil {
+		err := service.started()
+		if err != nil {
+			logrus.Fatal(err)
+		}
+	}
+}
+
+// notifyShutdown is called after the daemon shuts down but before the process exits.
+func notifyShutdown(err error) {
+	if service != nil {
+		service.stopped(err)
+	}
 }
 
 // setupConfigReloadTrap configures a Win32 event to reload the configuration.
-func setupConfigReloadTrap(configFile string, flags *mflag.FlagSet, reload func(*daemon.Config)) {
+func (cli *DaemonCli) setupConfigReloadTrap() {
 	go func() {
 		sa := syscall.SecurityAttributes{
 			Length: 0,
@@ -44,9 +55,7 @@ func setupConfigReloadTrap(configFile string, flags *mflag.FlagSet, reload func(
 			logrus.Debugf("Config reload - waiting signal at %s", ev)
 			for {
 				syscall.WaitForSingleObject(h, syscall.INFINITE)
-				if err := daemon.ReloadConfiguration(configFile, flags, reload); err != nil {
-					logrus.Error(err)
-				}
+				cli.reloadConfig()
 			}
 		}
 	}()

cmd/dockerd/docker.go

@@ -56,7 +56,21 @@ func main() {
 		flag.Usage()
 		return
 	}
-	daemonCli.start()
+
+	// On Windows, this may be launching as a service or with an option to
+	// register the service.
+	stop, err := initService()
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	if !stop {
+		err = daemonCli.start()
+		notifyShutdown(err)
+		if err != nil {
+			logrus.Fatal(err)
+		}
+	}
 }
 
 func showVersion() {

cmd/dockerd/docker_windows.go

@@ -1,5 +1,5 @@
 package main
 
 import (
-	_ "github.com/docker/docker/autogen/winresources/docker"
+	_ "github.com/docker/docker/autogen/winresources/dockerd"
 )

cmd/dockerd/service_unsupported.go

@@ -0,0 +1,7 @@
+// +build !windows
+
+package main
+
+func initService() (bool, error) {
+	return false, nil
+}

cmd/dockerd/service_windows.go

@@ -0,0 +1,369 @@
+package main
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"syscall"
+
+	"github.com/Sirupsen/logrus"
+	flag "github.com/docker/docker/pkg/mflag"
+	"golang.org/x/sys/windows"
+	"golang.org/x/sys/windows/svc"
+	"golang.org/x/sys/windows/svc/debug"
+	"golang.org/x/sys/windows/svc/eventlog"
+	"golang.org/x/sys/windows/svc/mgr"
+)
+
+var (
+	flServiceName       = flag.String([]string{"-service-name"}, "docker", "Set the Windows service name")
+	flRegisterService   = flag.Bool([]string{"-register-service"}, false, "Register the service and exit")
+	flUnregisterService = flag.Bool([]string{"-unregister-service"}, false, "Unregister the service and exit")
+	flRunService        = flag.Bool([]string{"-run-service"}, false, "")
+
+	setStdHandle = syscall.NewLazyDLL("kernel32.dll").NewProc("SetStdHandle")
+	oldStderr    syscall.Handle
+	panicFile    *os.File
+
+	service *handler
+)
+
+const (
+	// These should match the values in event_messages.mc.
+	eventInfo  = 1
+	eventWarn  = 1
+	eventError = 1
+	eventDebug = 2
+	eventPanic = 3
+	eventFatal = 4
+
+	eventExtraOffset = 10 // Add this to any event to get a string that supports extended data
+)
+
+type handler struct {
+	tosvc   chan bool
+	fromsvc chan error
+}
+
+type etwHook struct {
+	log *eventlog.Log
+}
+
+func (h *etwHook) Levels() []logrus.Level {
+	return []logrus.Level{
+		logrus.PanicLevel,
+		logrus.FatalLevel,
+		logrus.ErrorLevel,
+		logrus.WarnLevel,
+		logrus.InfoLevel,
+		logrus.DebugLevel,
+	}
+}
+
+func (h *etwHook) Fire(e *logrus.Entry) error {
+	var (
+		etype uint16
+		eid   uint32
+	)
+
+	switch e.Level {
+	case logrus.PanicLevel:
+		etype = windows.EVENTLOG_ERROR_TYPE
+		eid = eventPanic
+	case logrus.FatalLevel:
+		etype = windows.EVENTLOG_ERROR_TYPE
+		eid = eventFatal
+	case logrus.ErrorLevel:
+		etype = windows.EVENTLOG_ERROR_TYPE
+		eid = eventError
+	case logrus.WarnLevel:
+		etype = windows.EVENTLOG_WARNING_TYPE
+		eid = eventWarn
+	case logrus.InfoLevel:
+		etype = windows.EVENTLOG_INFORMATION_TYPE
+		eid = eventInfo
+	case logrus.DebugLevel:
+		etype = windows.EVENTLOG_INFORMATION_TYPE
+		eid = eventDebug
+	default:
+		return errors.New("unknown level")
+	}
+
+	// If there is additional data, include it as a second string.
+	exts := ""
+	if len(e.Data) > 0 {
+		fs := bytes.Buffer{}
+		for k, v := range e.Data {
+			fs.WriteString(k)
+			fs.WriteByte('=')
+			fmt.Fprint(&fs, v)
+			fs.WriteByte(' ')
+		}
+
+		exts = fs.String()[:fs.Len()-1]
+		eid += eventExtraOffset
+	}
+
+	if h.log == nil {
+		fmt.Fprintf(os.Stderr, "%s [%s]\n", e.Message, exts)
+		return nil
+	}
+
+	var (
+		ss  [2]*uint16
+		err error
+	)
+
+	ss[0], err = syscall.UTF16PtrFromString(e.Message)
+	if err != nil {
+		return err
+	}
+
+	count := uint16(1)
+	if exts != "" {
+		ss[1], err = syscall.UTF16PtrFromString(exts)
+		if err != nil {
+			return err
+		}
+
+		count++
+	}
+
+	return windows.ReportEvent(h.log.Handle, etype, 0, eid, 0, count, 0, &ss[0], nil)
+}
+
+func getServicePath() (string, error) {
+	p, err := exec.LookPath(os.Args[0])
+	if err != nil {
+		return "", err
+	}
+	return filepath.Abs(p)
+}
+
+func registerService() error {
+	p, err := getServicePath()
+	if err != nil {
+		return err
+	}
+	m, err := mgr.Connect()
+	if err != nil {
+		return err
+	}
+	defer m.Disconnect()
+	c := mgr.Config{
+		ServiceType:  windows.SERVICE_WIN32_OWN_PROCESS,
+		StartType:    mgr.StartAutomatic,
+		ErrorControl: mgr.ErrorNormal,
+		DisplayName:  "Docker Engine",
+	}
+
+	// Configure the service to launch with the arguments that were just passed.
+	args := []string{"--run-service"}
+	for _, a := range os.Args[1:] {
+		if a != "--register-service" && a != "--unregister-service" {
+			args = append(args, a)
+		}
+	}
+
+	s, err := m.CreateService(*flServiceName, p, c, args...)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+	err = eventlog.Install(*flServiceName, p, false, eventlog.Info|eventlog.Warning|eventlog.Error)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func unregisterService() error {
+	m, err := mgr.Connect()
+	if err != nil {
+		return err
+	}
+	defer m.Disconnect()
+
+	s, err := m.OpenService(*flServiceName)
+	if err != nil {
+		return err
+	}
+	defer s.Close()
+
+	eventlog.Remove(*flServiceName)
+	err = s.Delete()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func initService() (bool, error) {
+	if *flUnregisterService {
+		if *flRegisterService {
+			return true, errors.New("--register-service and --unregister-service cannot be used together")
+		}
+		return true, unregisterService()
+	}
+
+	if *flRegisterService {
+		return true, registerService()
+	}
+
+	if !*flRunService {
+		return false, nil
+	}
+
+	interactive, err := svc.IsAnInteractiveSession()
+	if err != nil {
+		return false, err
+	}
+
+	h := &handler{
+		tosvc:   make(chan bool),
+		fromsvc: make(chan error),
+	}
+
+	var log *eventlog.Log
+	if !interactive {
+		log, err = eventlog.Open(*flServiceName)
+		if err != nil {
+			return false, err
+		}
+	}
+
+	logrus.AddHook(&etwHook{log})
+	logrus.SetOutput(ioutil.Discard)
+
+	service = h
+	go func() {
+		if interactive {
+			err = debug.Run(*flServiceName, h)
+		} else {
+			err = svc.Run(*flServiceName, h)
+		}
+
+		h.fromsvc <- err
+	}()
+
+	// Wait for the first signal from the service handler.
+	err = <-h.fromsvc
+	if err != nil {
+		return false, err
+	}
+	return false, nil
+}
+
+func (h *handler) started() error {
+	// This must be delayed until daemonCli initializes Config.Root
+	err := initPanicFile(filepath.Join(daemonCli.Config.Root, "panic.log"))
+	if err != nil {
+		return err
+	}
+
+	h.tosvc <- false
+	return nil
+}
+
+func (h *handler) stopped(err error) {
+	logrus.Debugf("Stopping service: %v", err)
+	h.tosvc <- err != nil
+	<-h.fromsvc
+}
+
+func (h *handler) Execute(_ []string, r <-chan svc.ChangeRequest, s chan<- svc.Status) (bool, uint32) {
+	s <- svc.Status{State: svc.StartPending, Accepts: 0}
+	// Unblock initService()
+	h.fromsvc <- nil
+
+	// Wait for initialization to complete.
+	failed := <-h.tosvc
+	if failed {
+		logrus.Debugf("Aborting service start due to failure during initializtion")
+		return true, 1
+	}
+
+	s <- svc.Status{State: svc.Running, Accepts: svc.AcceptStop | svc.AcceptShutdown | svc.Accepted(windows.SERVICE_ACCEPT_PARAMCHANGE)}
+	logrus.Debugf("Service running")
+Loop:
+	for {
+		select {
+		case failed = <-h.tosvc:
+			break Loop
+		case c := <-r:
+			switch c.Cmd {
+			case svc.Cmd(windows.SERVICE_CONTROL_PARAMCHANGE):
+				daemonCli.reloadConfig()
+			case svc.Interrogate:
+				s <- c.CurrentStatus
+			case svc.Stop, svc.Shutdown:
+				s <- svc.Status{State: svc.StopPending, Accepts: 0}
+				daemonCli.stop()
+			}
+		}
+	}
+
+	removePanicFile()
+	if failed {
+		return true, 1
+	}
+	return false, 0
+}
+
+func initPanicFile(path string) error {
+	var err error
+	panicFile, err = os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0)
+	if err != nil {
+		return err
+	}
+
+	st, err := panicFile.Stat()
+	if err != nil {
+		return err
+	}
+
+	// If there are contents in the file already, move the file out of the way
+	// and replace it.
+	if st.Size() > 0 {
+		panicFile.Close()
+		os.Rename(path, path+".old")
+		panicFile, err = os.Create(path)
+		if err != nil {
+			return err
+		}
+	}
+
+	// Update STD_ERROR_HANDLE to point to the panic file so that Go writes to
+	// it when it panics. Remember the old stderr to restore it before removing
+	// the panic file.
+	sh := syscall.STD_ERROR_HANDLE
+	h, err := syscall.GetStdHandle(sh)
+	if err != nil {
+		return err
+	}
+
+	oldStderr = h
+
+	r, _, err := setStdHandle.Call(uintptr(sh), uintptr(panicFile.Fd()))
+	if r == 0 && err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func removePanicFile() {
+	if st, err := panicFile.Stat(); err == nil {
+		if st.Size() == 0 {
+			sh := syscall.STD_ERROR_HANDLE
+			setStdHandle.Call(uintptr(sh), uintptr(oldStderr))
+			panicFile.Close()
+			os.Remove(panicFile.Name())
+		}
+	}
+}