diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index f59628caaf..dcb85e6bb4 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -16,4 +16,8 @@ ### Bug Fixes 🐞 +#### General + +* [#3779](https://github.com/livepeer/go-livepeer/pull/3779) worker: Fix orphaned containers on node shutdown (@victorges) + #### CLI diff --git a/ai/worker/docker.go b/ai/worker/docker.go index 4e16d5d0bc..75bb741db1 100644 --- a/ai/worker/docker.go +++ b/ai/worker/docker.go @@ -411,12 +411,9 @@ func (m *DockerManager) createContainer(ctx context.Context, pipeline string, mo } restartPolicy := container.RestartPolicy{ - Name: "on-failure", + Name: container.RestartPolicyOnFailure, MaximumRetryCount: 3, } - if keepWarm { - restartPolicy = container.RestartPolicy{Name: "always"} - } hostConfig := &container.HostConfig{ Resources: container.Resources{ diff --git a/cmd/livepeer/livepeer.go b/cmd/livepeer/livepeer.go index 5502a09cd2..af895ab503 100755 --- a/cmd/livepeer/livepeer.go +++ b/cmd/livepeer/livepeer.go @@ -10,6 +10,7 @@ import ( "os" "os/signal" "runtime" + "syscall" "time" "github.com/livepeer/go-livepeer/cmd/livepeer/starter" @@ -20,6 +21,8 @@ import ( "github.com/livepeer/go-livepeer/core" ) +const shutdownTimeout = 10 * time.Second + func main() { // Override the default flag set since there are dependencies that // incorrectly add their own flags (specifically, due to the 'testing' @@ -76,17 +79,25 @@ func main() { lc := make(chan struct{}) go func() { + defer close(lc) starter.StartLivepeer(ctx, cfg) - lc <- struct{}{} }() - c := make(chan os.Signal) - signal.Notify(c, os.Interrupt) + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + select { case sig := <-c: glog.Infof("Exiting Livepeer: %v", sig) cancel() - time.Sleep(time.Second * 2) //Give time for other processes to shut down completely case <-lc: + // fallthrough to normal shutdown below + } + select { + case <-lc: + glog.Infof("Graceful shutdown complete") + case <-time.After(shutdownTimeout): + glog.Infof("Shutdown timed out, forcing exit") + os.Exit(1) } }