Print stacktrace in questionable situations

It would be useful to see what the stack looks like when we get
consecutive watchdog patting failures. We can then try to work out
what's ultimately causing this issue.

Approved-by: Trek Hopton

cmd/vidforward/main.go

@@ -138,6 +138,7 @@ func terminationCallback(m *broadcastManager) func() {
 			return
 		}
 		m.log.Info("successfully saved broadcast manager state on termination signal")
+		logTrace(m.log.Debug,m.log.Warning)
 	}
 }
 

cmd/vidforward/notifier.go

@@ -92,12 +92,20 @@ func (n *watchdogNotifier) notify() {
 		n.termCallback()
 	}()
 
+	var consecutiveUnhealthyStates int
 	for {
+		const nUnhealthyStatesForTrace = 10
 		if n.handlersUnhealthy() {
+			consecutiveUnhealthyStates++
+			if consecutiveUnhealthyStates >= nUnhealthyStatesForTrace {
+				logTrace(n.log.Debug,n.log.Warning)
+				consecutiveUnhealthyStates = 0
+			}
 			const unhealthyHandlerWait = 1 * time.Second
 			time.Sleep(unhealthyHandlerWait)
 			continue
 		}
+		consecutiveUnhealthyStates = 0
 
 		<-notifyTicker.C
 

cmd/vidforward/utils.go

@@ -28,6 +28,7 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
+	"runtime"
 
 	"bitbucket.org/ausocean/av/codec/codecutil"
 	"bitbucket.org/ausocean/av/revid"
@@ -86,3 +87,18 @@ func isMac(m string) bool {
 	}
 	return true
 }
+
+type Log func(msg string, args ...interface{})
+
+func logTrace(debug, warning Log){
+	const (
+		maxStackTraceSize = 100000
+		allStacks         = true
+	)
+	buf := make([]byte, maxStackTraceSize)
+	n := runtime.Stack(buf, allStacks)
+	if n > maxStackTraceSize && warning != nil {
+		warning("stacktrace exceeded buffer size")
+	}
+	debug("got stacktrace at termination", "stacktrace", string(buf[:n]))
+}