failsafe.go

package kvm

import (
	"io"
	"os"
	"strings"

	"github.com/jetkvm/kvm/internal/supervisor"
	"github.com/jetkvm/kvm/internal/sync"
)

const (
	failsafeDefaultLastCrashPath = "/userdata/jetkvm/crashdump/last-crash.log"
	failsafeFile                 = "/userdata/jetkvm/.enablefailsafe"
	failsafeLastCrashEnv         = "JETKVM_LAST_ERROR_PATH"
	failsafeEnv                  = "JETKVM_FORCE_FAILSAFE"
)

var (
	failsafeOnce       sync.Once
	failsafeCrashLog   = ""
	failsafeModeActive = false
	failsafeModeReason = ""
)

type FailsafeModeNotification struct {
	Active bool   `json:"active"`
	Reason string `json:"reason"`
}

// this function has side effects and can be only executed once
func checkFailsafeReason() {
	failsafeOnce.Do(func() {
		// check if the failsafe environment variable is set
		if os.Getenv(failsafeEnv) == "1" {
			failsafeModeActive = true
			failsafeModeReason = "failsafe_env_set"
			return
		}

		// check if the failsafe file exists
		if _, err := os.Stat(failsafeFile); err == nil {
			failsafeModeActive = true
			failsafeModeReason = "failsafe_file_exists"
			_ = os.Remove(failsafeFile)
			return
		}

		// get the last crash log path from the environment variable
		lastCrashPath := os.Getenv(failsafeLastCrashEnv)
		if lastCrashPath == "" {
			lastCrashPath = failsafeDefaultLastCrashPath
		}

		// check if the last crash log file exists
		l := failsafeLogger.With().Str("path", lastCrashPath).Logger()
		fi, err := os.Lstat(lastCrashPath)
		if err != nil {
			if !os.IsNotExist(err) {
				l.Warn().Err(err).Msg("failed to stat last crash log")
			}
			return
		}

		if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
			l.Warn().Msg("last crash log is not a symlink, ignoring")
			return
		}

		// open the last crash log file and find if it contains the string "panic"
		// read only the last 50KB to avoid memory issues with large log files
		content, err := readFileTail(lastCrashPath, 50*1024)
		if err != nil {
			l.Warn().Err(err).Msg("failed to read last crash log")
			return
		}

		// unlink the last crash log file
		failsafeCrashLog = content
		_ = os.Remove(lastCrashPath)

		// TODO: read the goroutine stack trace and check which goroutine is panicking
		failsafeModeActive = true
		if strings.Contains(failsafeCrashLog, supervisor.FailsafeReasonVideoMaxRestartAttemptsReached) {
			failsafeModeReason = "video"
			return
		}
		if strings.Contains(failsafeCrashLog, "runtime.cgocall") {
			failsafeModeReason = "video"
			return
		} else {
			failsafeModeReason = "unknown"
		}
	})
}

// readFileTail reads at most maxBytes from the end of a file.
// This prevents memory issues when reading potentially large log files.
func readFileTail(path string, maxBytes int64) (string, error) {
	f, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		return "", err
	}

	size := fi.Size()
	if size > maxBytes {
		if _, err := f.Seek(size-maxBytes, io.SeekStart); err != nil {
			return "", err
		}
	}

	data, err := io.ReadAll(f)
	if err != nil {
		return "", err
	}
	return string(data), nil
}

func notifyFailsafeMode(session *Session) {
	if !failsafeModeActive || session == nil {
		return
	}

	jsonRpcLogger.Info().Str("reason", failsafeModeReason).Msg("sending failsafe mode notification")

	writeJSONRPCEvent("failsafeMode", FailsafeModeNotification{
		Active: true,
		Reason: failsafeModeReason,
	}, session)
}