av/cmd/rv/probe.go

//go:build !nocv
// +build !nocv

/*
DESCRIPTION
  Provides the methods for the turbidity probe using GoCV. Turbidity probe
  will collect the most recent frames in a buffer and write the latest sharpness
  and contrast scores to the probe.

AUTHORS
  Russell Stanley <russell@ausocean.org>

LICENSE
  Copyright (C) 2021-2022 the Australian Ocean Lab (AusOcean)

  It is free software: you can redistribute it and/or modify them
  under the terms of the GNU General Public License as published by the
  Free Software Foundation, either version 3 of the License, or (at your
  option) any later version.

  It is distributed in the hope that it will be useful, but WITHOUT
  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
  for more details.

  You should have received a copy of the GNU General Public License
  in gpl.txt.  If not, see http://www.gnu.org/licenses.
*/

package main

import (
	"bytes"
	"os"
	"time"

	"gocv.io/x/gocv"
	"gonum.org/v1/gonum/stat"

	"bitbucket.org/ausocean/av/codec/h264"
	"bitbucket.org/ausocean/av/turbidity"
	"bitbucket.org/ausocean/utils/logger"
)

// Misc constants.
const (
	maxImages   = 1     // Max number of images read when evaluating turbidity.
	bufferLimit = 20000 // 20KB
)

// Turbidity sensor constants.
const (
	k1, k2     = 4, 4 // Block size, must be divisible by the size template with no remainder.
	filterSize = 3    // Sobel filter size.
	scale      = 1.0  // Amount of scale applied to sobel filter values.
	alpha      = 1.0  // Paramater for contrast equation.
)

// turbidityProbe will hold the latest video data and calculate the sharpness and contrast scores.
// These scores will be sent to netreceiver based on the given delay.
type turbidityProbe struct {
	sharpness, contrast float64
	delay               time.Duration
	ticker              time.Ticker
	ts                  *turbidity.TurbiditySensor
	log                 logger.Logger
	buffer              *bytes.Buffer
}

// NewTurbidityProbe returns a new turbidity probe.
func NewTurbidityProbe(log logger.Logger, delay time.Duration) (*turbidityProbe, error) {
	tp := new(turbidityProbe)
	tp.log = log
	tp.delay = delay
	tp.ticker = *time.NewTicker(delay)
	tp.buffer = bytes.NewBuffer(*new([]byte))

	// Create the turbidity sensor.
	standard := gocv.IMRead("../../turbidity/images/default.jpg", gocv.IMReadGrayScale)
	template := gocv.IMRead("../../turbidity/images/template.jpg", gocv.IMReadGrayScale)
	ts, err := turbidity.NewTurbiditySensor(template, standard, k1, k2, filterSize, scale, alpha, log)
	if err != nil {
		log.Error("failed create turbidity sensor", "error", err.Error())
	}
	tp.ts = ts
	return tp, nil
}

// Write, reads input h264 frames in the form of a byte stream and writes the the sharpness and contrast
// scores of a video to the the turbidity probe.
func (tp *turbidityProbe) Write(p []byte) (int, error) {
	out := len(p)
	if tp.buffer.Len() == 0 {
		// The first entry in the buffer must be a keyframe to speed up decoding.
		video, err := h264.Trim(p)
		if err != nil {
			tp.log.Log(logger.Debug, "no key frame detected", "error", err.Error())
			return 0, nil
		}
		n, err := tp.buffer.Write(video)
		if err != nil {
			tp.log.Log(logger.Error, "could not write trimmed video to buffer", "error", err.Error())
			tp.buffer.Reset()
			return 0, nil
		}
		tp.log.Log(logger.Debug, "video trimmed, write keyframe complete", "size(bytes)", n)
	} else if tp.buffer.Len() < bufferLimit {
		// Buffer is limited to speed up decoding.
		n, err := tp.buffer.Write(p)
		if err != nil {
			tp.log.Log(logger.Error, "could not write to buffer, reseting", "error", err.Error())
			tp.buffer.Reset()
			return 0, nil
		}
		tp.log.Log(logger.Debug, "write to video buffer complete", "size(bytes)", n)
	} else {
		out = 0
	}

	select {
	case <-tp.ticker.C:
		tp.log.Log(logger.Debug, "beginning turbidity calculation")
		startTime := time.Now()
		tp.turbidityCalculation()
		tp.log.Log(logger.Debug, "finished turbidity calculation", "total duration (sec)", time.Since(startTime).Seconds())
	default:
	}
	return out, nil
}

func (tp *turbidityProbe) Close() error {
	return nil
}

func (tp *turbidityProbe) turbidityCalculation() {
	var imgs []gocv.Mat
	img := gocv.NewMat()
	// Write byte array to a temp file.
	file, err := os.CreateTemp("temp", "video*.h264")
	if err != nil {
		tp.log.Error("failed to create temp file", "error", err.Error())
		return
	}
	tp.log.Log(logger.Debug, "writing to file", "buffer size(bytes)", tp.buffer.Len())
	_, err = file.Write(tp.buffer.Bytes())
	if err != nil {
		tp.log.Error("failed to write to temporary file", "error", err.Error())
		return
	}
	tp.log.Log(logger.Debug, "write to file success", "buffer size(bytes)", tp.buffer.Len())
	tp.buffer.Reset()

	// Open the video file.
	startTime := time.Now()
	vc, err := gocv.VideoCaptureFile(file.Name())
	if err != nil {
		tp.log.Error("failed to open video file", "error", err.Error())
		return
	}
	tp.log.Log(logger.Debug, "video capture open", "total duration (sec)", time.Since(startTime).Seconds())

	// Store each frame untill max.
	startTime = time.Now()
	for vc.Read(&img) && len(imgs) < maxImages {
		imgs = append(imgs, img.Clone())
	}
	if len(imgs) <= 0 {
		tp.log.Log(logger.Warning, "no frames found")
		return
	}
	tp.log.Log(logger.Debug, "read time", "total duration (sec)", time.Since(startTime).Seconds())

	// Process video data to get saturation and contrast scores.
	res, err := tp.ts.Evaluate(imgs)
	if err != nil {
		tp.log.Error("evaluate failed", "error", err.Error())
	} else {
		tp.contrast = stat.Mean(res.Contrast, nil)
		tp.sharpness = stat.Mean(res.Sharpness, nil)
	}

	err = os.Remove(file.Name())
	if err != nil {
		tp.log.Error("could not remove file", "error", err.Error())
	}
	err = vc.Close()
	if err != nil {
		tp.log.Error("could not close video capture", "error", err.Error())
	}
}