Add controller error state handling

Resolves #579

We can check the controller error variable first to easily get any
error state. If the controller does not yet support this, we still
check other pieces of information.

Author: saxon-milton

cmd/oceantv/broadcast_hardware_machine.go

@@ -234,6 +234,35 @@ func (s *hardwareStarting) enter() {
 		return
 	}
 
+	// The first check for any known hardware error states.
+	hwErr, err := s.camera.error(s.broadcastContext)
+	if err != nil {
+		errWrapped := fmt.Errorf("could not get hardware error state: %w", err)
+		s.log(errWrapped.Error())
+		// NOTE here we could do this, however it's not certain that all ESPs
+		// will have the latest firmware that supports this, and so it's not
+		// necessarily a showstopper.
+		// s.bus.publish(invalidConfigurationEvent{errWrapped})
+		// return
+	}
+
+	switch {
+	case errors.Is(hwErr, LowVoltageAlarm):
+		s.log("controller voltage is low, waiting for recovery before starting")
+		s.bus.publish(lowVoltageEvent{})
+		return
+	case errors.Is(hwErr, None):
+		// Continue other checks, this is good.
+	case hwErr != nil:
+		errWrapped := fmt.Errorf("unhandled controller hardware error: %w", hwErr)
+		s.log(errWrapped.Error())
+		s.bus.publish(invalidConfigurationEvent{errWrapped})
+		return
+	default:
+		// This means heErr get failed to get, which at this stage just means
+		// we have a controller that doesn't have the latest firmware.
+	}
+
 	voltage, err := s.camera.voltage(s.broadcastContext)
 	if err != nil {
 		errWrapped := fmt.Errorf("could not get hardware voltage: %w", err)
@@ -1015,10 +1044,32 @@ type hardwareManager interface {
 	shutdown(ctx *broadcastContext)
 	stop(ctx *broadcastContext)
 	publishEventIfStatus(ctx *broadcastContext, event event, status bool, mac int64, store Store, log func(format string, args ...interface{}), publish func(event event))
+	error(ctx *broadcastContext) (error, error)
 }
 
 type revidCameraClient struct{}
 
+type ControllerError string
+
+const (
+	None            ControllerError = ""
+	LowVoltageAlarm ControllerError = "LowVoltageAlarm"
+)
+
+func (e ControllerError) Error() string {
+	return string(e)
+}
+
+func (e ControllerError) Is(target error) bool {
+	if target == nil {
+		return false
+	}
+	if t, ok := target.(ControllerError); ok {
+		return e == t
+	}
+	return false
+}
+
 func (c *revidCameraClient) voltage(ctx *broadcastContext) (float64, error) {
 	// Get battery voltage sensor, which we'll use to get scale factor and current voltage value.
 	sensor, err := model.GetSensorV2(context.Background(), ctx.store, ctx.cfg.ControllerMAC, ctx.cfg.BatteryVoltagePin)
@@ -1131,6 +1182,15 @@ func (c *revidCameraClient) publishEventIfStatus(ctx *broadcastContext, event ev
 	}
 }
 
+func (c *revidCameraClient) error(ctx *broadcastContext) (error, error) {
+	controllerMACHex := (&model.Device{Mac: ctx.cfg.ControllerMAC}).Hex()
+	devErr, err := model.GetVariable(context.Background(), ctx.store, ctx.cfg.SKey, controllerMACHex+".error")
+	if err != nil {
+		return nil, fmt.Errorf("could not get controller error variable: %w", err)
+	}
+	return ControllerError(devErr.Value), nil
+}
+
 func (sm *hardwareStateMachine) saveHardwareStateToConfig() error {
 	sm.log("saving hardware state to config: %v", hardwareStateToString(sm.currentState))
 	hardwareState := hardwareStateToString(sm.currentState)

cmd/oceantv/broadcast_machine_test.go

@@ -1763,6 +1763,60 @@ func TestHardwareVoltageAndFaultHandling(t *testing.T) {
 			expectedLogs:   []string{},
 			expectedNotify: map[int64]map[notify.Kind][]string{},
 		},
+
+		{
+			desc: "direct broadcast; start with low voltage alarm error, then enter recovery",
+			cfg: func(c *BroadcastConfig) {
+				c.Enabled = true
+				c.SKey = testSiteKey
+				c.Start = time.Now().Add(-1 * time.Hour)
+				c.End = time.Now().Add(1 * time.Hour)
+				c.HardwareState = "hardwareOff"
+				c.ControllerMAC = 1
+			},
+			initialBroadcastState: &directIdle{},
+			finalBroadcastState:   &directStarting{},
+			finalHardwareState:    &hardwareRecoveringVoltage{},
+			hardwareMan:           newDummyHardwareManager(withLowVoltage(), withHardwareError(LowVoltageAlarm)),
+			newBroadcastMan: func(t *testing.T, c *BroadcastConfig) BroadcastManager {
+				return newDummyManager(t, c)
+			},
+			expectedEvents: []event{timeEvent{}, startEvent{}, hardwareStartRequestEvent{}, lowVoltageEvent{}},
+			expectedLogs:   []string{},
+			expectedNotify: map[int64]map[notify.Kind][]string{},
+		},
+
+		{
+			desc: "direct broadcast; low voltage alarm error, successful voltage recovery",
+			cfg: func(c *BroadcastConfig) {
+				c.Enabled = true
+				c.SKey = testSiteKey
+				c.Start = time.Now().Add(-1 * time.Hour)
+				c.End = time.Now().Add(1 * time.Hour)
+				c.HardwareState = "hardwareOff"
+				c.ControllerMAC = 1
+			},
+			initialBroadcastState: &directIdle{},
+			finalBroadcastState:   &directStarting{},
+			finalHardwareState:    &hardwareStarting{},
+			hardwareMan:           newDummyHardwareManager(withLowVoltage(), withHardwareError(LowVoltageAlarm)),
+			newBroadcastMan: func(t *testing.T, c *BroadcastConfig) BroadcastManager {
+				return newDummyManager(t, c)
+			},
+			requiredTicks: 60,
+			expectedEvents: append(
+				append(
+					[]event{
+						timeEvent{},
+						startEvent{},
+						hardwareStartRequestEvent{},
+						lowVoltageEvent{},
+					}, timeEvents(49)...),
+				[]event{voltageRecoveredEvent{}}...,
+			),
+			expectedLogs:   []string{},
+			expectedNotify: map[int64]map[notify.Kind][]string{},
+		},
 	}
 
 	for _, tt := range tests {

cmd/oceantv/broadcast_test.go

@@ -312,6 +312,7 @@ type dummyHardwareManager struct {
 	controllerMAC     string
 	cameraMAC         string
 	latestRequest     request
+	hwErr             error
 }
 
 func withHardwareFault() func(*dummyHardwareManager) {
@@ -326,6 +327,12 @@ func withLowVoltage() func(*dummyHardwareManager) {
 	}
 }
 
+func withHardwareError(err error) func(*dummyHardwareManager) {
+	return func(h *dummyHardwareManager) {
+		h.hwErr = err
+	}
+}
+
 func withMACSanitisation() func(*dummyHardwareManager) {
 	return func(h *dummyHardwareManager) {
 		h.checkMAC = true
@@ -456,6 +463,12 @@ func (h *dummyHardwareManager) publishEventIfStatus(ctx *broadcastContext, event
 		publish(event)
 	}
 }
+func (h *dummyHardwareManager) error(ctx *broadcastContext) (error, error) {
+	if h.volts > h.alarmVolts {
+		return None, nil
+	}
+	return h.hwErr, nil
+}
 
 // mockNotifier to implement Notifier interface.
 type mockNotifier struct {

cmd/oceantv/main.go

@@ -47,7 +47,7 @@ import (
 
 const (
 	projectID          = "oceantv"
-	version            = "v0.10.5"
+	version            = "v0.11.0"
 	projectURL         = "https://oceantv.appspot.com"
 	cronServiceAccount = "oceancron@appspot.gserviceaccount.com"
 	locationID         = "Australia/Adelaide" // TODO: Use site location.