worker: Fix orphaned containers on node shutdown

CHANGELOG_PENDING.md

@@ -16,4 +16,8 @@
 
 ### Bug Fixes 🐞
 
+#### General
+
+* [#3779](https://github.com/livepeer/go-livepeer/pull/3779) worker: Fix orphaned containers on node shutdown (@victorges)
+
 #### CLI

ai/worker/docker.go

@@ -411,12 +411,9 @@ func (m *DockerManager) createContainer(ctx context.Context, pipeline string, mo
 	}
 
 	restartPolicy := container.RestartPolicy{
-		Name:              "on-failure",
+		Name:              container.RestartPolicyOnFailure,
 		MaximumRetryCount: 3,
 	}
-	if keepWarm {
-		restartPolicy = container.RestartPolicy{Name: "always"}
-	}
 
 	hostConfig := &container.HostConfig{
 		Resources: container.Resources{

cmd/livepeer/livepeer.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"os/signal"
 	"runtime"
+	"syscall"
 	"time"
 
 	"github.com/livepeer/go-livepeer/cmd/livepeer/starter"
@@ -20,6 +21,8 @@ import (
 	"github.com/livepeer/go-livepeer/core"
 )
 
+const shutdownTimeout = 10 * time.Second
+
 func main() {
 	// Override the default flag set since there are dependencies that
 	// incorrectly add their own flags (specifically, due to the 'testing'
@@ -76,17 +79,25 @@ func main() {
 	lc := make(chan struct{})
 
 	go func() {
+		defer close(lc)
 		starter.StartLivepeer(ctx, cfg)
-		lc <- struct{}{}
 	}()
 
-	c := make(chan os.Signal)
-	signal.Notify(c, os.Interrupt)
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+
 	select {
 	case sig := <-c:
 		glog.Infof("Exiting Livepeer: %v", sig)
 		cancel()
-		time.Sleep(time.Second * 2) //Give time for other processes to shut down completely
 	case <-lc:
+		// fallthrough to normal shutdown below
+	}
+	select {
+	case <-lc:
+		glog.Infof("Graceful shutdown complete")
+	case <-time.After(shutdownTimeout):
+		glog.Infof("Shutdown timed out, forcing exit")
+		os.Exit(1)
 	}
 }