Add controller error state handling

cmd/oceantv/broadcast_hardware_machine.go

@@ -234,6 +234,35 @@ func (s *hardwareStarting) enter() {
 		return
 	}
 
+	// The first check for any known hardware error states.
+	hwErr, err := s.camera.error(s.broadcastContext)
+	if err != nil {
+		errWrapped := fmt.Errorf("could not get hardware error state: %w", err)
+		s.log(errWrapped.Error())
+		// NOTE here we could do this, however it's not certain that all ESPs
+		// will have the latest firmware that supports this, and so it's not
+		// necessarily a showstopper.
+		// s.bus.publish(invalidConfigurationEvent{errWrapped})
+		// return
+	}
+
+	switch {
+	case errors.Is(hwErr, LowVoltageAlarm):
+		s.log("controller voltage is low, waiting for recovery before starting")
+		s.bus.publish(lowVoltageEvent{})
+		return
+	case errors.Is(hwErr, None):
+		// Continue other checks, this is good.
+	case hwErr != nil:
+		errWrapped := fmt.Errorf("unhandled controller hardware error: %w", hwErr)
+		s.log(errWrapped.Error())
+		s.bus.publish(invalidConfigurationEvent{errWrapped})
+		return
+	default:
+		// This means we failed to get hwErr, which at this stage just means
+		// we have a controller that doesn't have the latest firmware.
+	}
+
 	voltage, err := s.camera.voltage(s.broadcastContext)
 	if err != nil {
 		errWrapped := fmt.Errorf("could not get hardware voltage: %w", err)
@@ -1012,10 +1041,32 @@ type hardwareManager interface {
 	shutdown(ctx *broadcastContext)
 	stop(ctx *broadcastContext)
 	publishEventIfStatus(ctx *broadcastContext, event event, status bool, mac int64, store Store, log func(format string, args ...interface{}), publish func(event event))
+	error(ctx *broadcastContext) (error, error)
 }
 
 type revidCameraClient struct{}
 
+type ControllerError string
+
+const (
+	None            ControllerError = ""
+	LowVoltageAlarm ControllerError = "LowVoltageAlarm"
+)
+
+func (e ControllerError) Error() string {
+	return string(e)
+}
+
+func (e ControllerError) Is(target error) bool {
+	if target == nil {
+		return false
+	}
+	if t, ok := target.(ControllerError); ok {
+		return e == t
+	}
+	return false
+}
+
 func (c *revidCameraClient) voltage(ctx *broadcastContext) (float64, error) {
 	// Get battery voltage sensor, which we'll use to get scale factor and current voltage value.
 	sensor, err := model.GetSensorV2(context.Background(), ctx.store, ctx.cfg.ControllerMAC, ctx.cfg.BatteryVoltagePin)
@@ -1128,6 +1179,15 @@ func (c *revidCameraClient) publishEventIfStatus(ctx *broadcastContext, event ev
 	}
 }
 
+func (c *revidCameraClient) error(ctx *broadcastContext) (error, error) {
+	controllerMACHex := (&model.Device{Mac: ctx.cfg.ControllerMAC}).Hex()
+	devErr, err := model.GetVariable(context.Background(), ctx.store, ctx.cfg.SKey, controllerMACHex+".error")
+	if err != nil {
+		return nil, fmt.Errorf("could not get controller error variable: %w", err)
+	}
+	return ControllerError(devErr.Value), nil
+}
+
 func (sm *hardwareStateMachine) saveHardwareStateToConfig() error {
 	sm.log("saving hardware state to config: %v", hardwareStateToString(sm.currentState))
 	hardwareState := hardwareStateToString(sm.currentState)

cmd/oceantv/broadcast_machine_test.go

@@ -1763,6 +1763,60 @@ func TestHardwareVoltageAndFaultHandling(t *testing.T) {
 			expectedLogs:   []string{},
 			expectedNotify: map[int64]map[notify.Kind][]string{},
 		},
+
+		{
+			desc: "direct broadcast; start with low voltage alarm error, then enter recovery",
+			cfg: func(c *BroadcastConfig) {
+				c.Enabled = true
+				c.SKey = testSiteKey
+				c.Start = time.Now().Add(-1 * time.Hour)
+				c.End = time.Now().Add(1 * time.Hour)
+				c.HardwareState = "hardwareOff"
+				c.ControllerMAC = 1
+			},
+			initialBroadcastState: &directIdle{},
+			finalBroadcastState:   &directStarting{},
+			finalHardwareState:    &hardwareRecoveringVoltage{},
+			hardwareMan:           newDummyHardwareManager(withLowVoltage(), withHardwareError(LowVoltageAlarm)),
+			newBroadcastMan: func(t *testing.T, c *BroadcastConfig) BroadcastManager {
+				return newDummyManager(t, c)
+			},
+			expectedEvents: []event{timeEvent{}, startEvent{}, hardwareStartRequestEvent{}, lowVoltageEvent{}},
+			expectedLogs:   []string{},
+			expectedNotify: map[int64]map[notify.Kind][]string{},
+		},
+
+		{
+			desc: "direct broadcast; low voltage alarm error, successful voltage recovery",
+			cfg: func(c *BroadcastConfig) {
+				c.Enabled = true
+				c.SKey = testSiteKey
+				c.Start = time.Now().Add(-1 * time.Hour)
+				c.End = time.Now().Add(1 * time.Hour)
+				c.HardwareState = "hardwareOff"
+				c.ControllerMAC = 1
+			},
+			initialBroadcastState: &directIdle{},
+			finalBroadcastState:   &directStarting{},
+			finalHardwareState:    &hardwareStarting{},
+			hardwareMan:           newDummyHardwareManager(withLowVoltage(), withHardwareError(LowVoltageAlarm)),
+			newBroadcastMan: func(t *testing.T, c *BroadcastConfig) BroadcastManager {
+				return newDummyManager(t, c)
+			},
+			requiredTicks: 60,
+			expectedEvents: append(
+				append(
+					[]event{
+						timeEvent{},
+						startEvent{},
+						hardwareStartRequestEvent{},
+						lowVoltageEvent{},
+					}, timeEvents(49)...),
+				[]event{voltageRecoveredEvent{}}...,
+			),
+			expectedLogs:   []string{},
+			expectedNotify: map[int64]map[notify.Kind][]string{},
+		},
 	}
 
 	for _, tt := range tests {

cmd/oceantv/broadcast_test.go

@@ -312,6 +312,7 @@ type dummyHardwareManager struct {
 	controllerMAC     string
 	cameraMAC         string
 	latestRequest     request
+	hwErr             error
 }
 
 func withHardwareFault() func(*dummyHardwareManager) {
@@ -326,6 +327,12 @@ func withLowVoltage() func(*dummyHardwareManager) {
 	}
 }
 
+func withHardwareError(err error) func(*dummyHardwareManager) {
+	return func(h *dummyHardwareManager) {
+		h.hwErr = err
+	}
+}
+
 func withMACSanitisation() func(*dummyHardwareManager) {
 	return func(h *dummyHardwareManager) {
 		h.checkMAC = true
@@ -456,6 +463,12 @@ func (h *dummyHardwareManager) publishEventIfStatus(ctx *broadcastContext, event
 		publish(event)
 	}
 }
+func (h *dummyHardwareManager) error(ctx *broadcastContext) (error, error) {
+	if h.volts > h.alarmVolts {
+		return None, nil
+	}
+	return h.hwErr, nil
+}
 
 // mockNotifier to implement Notifier interface.
 type mockNotifier struct {

cmd/oceantv/main.go

@@ -47,7 +47,7 @@ import (
 
 const (
 	projectID          = "oceantv"
-	version            = "v0.11.1"
+	version            = "v0.11.2"
 	projectURL         = "https://oceantv.appspot.com"
 	cronServiceAccount = "oceancron@appspot.gserviceaccount.com"
 	locationID         = "Australia/Adelaide" // TODO: Use site location.