//go:build !nocv // +build !nocv /* DESCRIPTION Provides the methods for the turbidity probe using GoCV. Turbidity probe will collect the most recent frames in a buffer and write the latest sharpness and contrast scores to the probe. AUTHORS Russell Stanley LICENSE Copyright (C) 2021-2022 the Australian Ocean Lab (AusOcean) It is free software: you can redistribute it and/or modify them under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. It is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License in gpl.txt. If not, see http://www.gnu.org/licenses. */ package main import ( "bytes" "errors" "fmt" "os" "time" "gocv.io/x/gocv" "gonum.org/v1/gonum/stat" "bitbucket.org/ausocean/av/codec/h264" "bitbucket.org/ausocean/av/revid/config" "bitbucket.org/ausocean/av/turbidity" "bitbucket.org/ausocean/utils/logger" ) // Misc constants. const ( maxImages = 1 // Max number of images read when evaluating turbidity. bufferLimit = 20000 // 20KB trimTolerance = 200 // Number of times trim can be called where no keyframe is found. ) // Turbidity sensor constants. const ( k1, k2 = 4, 4 // Block size, must be divisible by the size template with no remainder. filterSize = 3 // Sobel filter size. scale = 1.0 // Amount of scale applied to sobel filter values. alpha = 1.0 // Paramater for contrast equation. ) // turbidityProbe will hold the latest video data and calculate the sharpness and contrast scores. // These scores will be sent to netreceiver based on the given delay. type turbidityProbe struct { sharpness, contrast float64 delay time.Duration ticker time.Ticker ts *turbidity.TurbiditySensor log logger.Logger buffer *bytes.Buffer transform []float64 trimCounter int } // NewTurbidityProbe returns a new turbidity probe. func NewTurbidityProbe(log logger.Logger, delay time.Duration) (*turbidityProbe, error) { tp := new(turbidityProbe) tp.log = log tp.delay = delay tp.ticker = *time.NewTicker(delay) tp.buffer = bytes.NewBuffer(*new([]byte)) tp.transform = []float64{0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0} H, err := floatToMat(tp.transform) if err != nil { return nil, fmt.Errorf("failed to convert float slice to mat: %w", err) } // Create the turbidity sensor. template := gocv.IMRead("../../turbidity/images/template.jpg", gocv.IMReadGrayScale) ts, err := turbidity.NewTurbiditySensor(template, H, k1, k2, filterSize, scale, alpha, log) if err != nil { return nil, fmt.Errorf("failed to create turbidity sensor: %w", err) } tp.ts = ts return tp, nil } // Write, reads input h264 frames in the form of a byte stream and writes the the sharpness and contrast // scores of a video to the the turbidity probe. func (tp *turbidityProbe) Write(p []byte) (int, error) { if tp.buffer.Len() == 0 { // The first entry in the buffer must be a keyframe to speed up decoding. video, err := h264.Trim(p) if err != nil { tp.trimCounter++ if tp.trimCounter >= trimTolerance { return 0, fmt.Errorf("could not trim h264 within tolerance: %w", err) } return len(p), nil } else { tp.log.Log(logger.Debug, "trim successful", "keyframe error counter", tp.trimCounter) tp.trimCounter = 0 } n, err := tp.buffer.Write(video) if err != nil { tp.buffer.Reset() return 0, fmt.Errorf("could not write trimmed video to buffer: %w", err) } tp.log.Log(logger.Debug, "video trimmed, write keyframe complete", "size(bytes)", n) } else if tp.buffer.Len() < bufferLimit { // Buffer size is limited to speed up decoding. _, err := tp.buffer.Write(p) if err != nil { tp.buffer.Reset() return 0, fmt.Errorf("could not write to buffer: %w", err) } } else { // Buffer is large enough to begin turbidity calculation. select { case <-tp.ticker.C: tp.log.Log(logger.Debug, "beginning turbidity calculation") startTime := time.Now() err := tp.turbidityCalculation() if err != nil { return 0, fmt.Errorf("could not calculate turbidity: %w", err) } tp.log.Log(logger.Debug, "finished turbidity calculation", "total duration (sec)", time.Since(startTime).Seconds()) default: } } return len(p), nil } func (tp *turbidityProbe) Close() error { return nil } func (tp *turbidityProbe) Update(config config.Config) error { if len(config.TransformMatrix) != 9 { return errors.New("transformation matrix has incorrect size") } for i := range tp.transform { if tp.transform[i] != config.TransformMatrix[i] { // Update the turbidity sensor with new transformation tp.log.Log(logger.Debug, "updating the turbidity sensor with new transformation") tp.transform = config.TransformMatrix newTransform, err := floatToMat(tp.transform) if err != nil { return fmt.Errorf("failed to convert float slice to mat: %w", err) } tp.ts.H = newTransform return nil } } tp.log.Log(logger.Debug, "no update requried") return nil } func (tp *turbidityProbe) turbidityCalculation() error { var imgs []gocv.Mat img := gocv.NewMat() // Write byte array to a temp file. file, err := os.CreateTemp("temp", "video*.h264") if err != nil { return fmt.Errorf("failed to create temp file: %w", err) } tp.log.Log(logger.Debug, "writing to file", "buffer size(bytes)", tp.buffer.Len()) _, err = file.Write(tp.buffer.Bytes()) if err != nil { return fmt.Errorf("failed to write to temporary file: %w", err) } tp.log.Log(logger.Debug, "write to file success", "buffer size(bytes)", tp.buffer.Len()) tp.buffer.Reset() // Open the video file. startTime := time.Now() vc, err := gocv.VideoCaptureFile(file.Name()) if err != nil { return fmt.Errorf("failed to open video file: %w", err) } tp.log.Log(logger.Debug, "video capture open", "total duration (sec)", time.Since(startTime).Seconds()) // Store each frame until maximum amount is reached. startTime = time.Now() for vc.Read(&img) && len(imgs) < maxImages { imgs = append(imgs, img.Clone()) } if len(imgs) <= 0 { return errors.New("no frames found") } tp.log.Log(logger.Debug, "read time", "total duration (sec)", time.Since(startTime).Seconds()) // Process video data to get saturation and contrast scores. res, err := tp.ts.Evaluate(imgs) if err != nil { err_ := cleanUp(file.Name(), vc) if err_ != nil { return fmt.Errorf("could not clean up: %v, after evaluation error: %w", err_, err) } return fmt.Errorf("evaluation error: %w", err) } tp.contrast = stat.Mean(res.Contrast, nil) tp.sharpness = stat.Mean(res.Sharpness, nil) err = cleanUp(file.Name(), vc) if err != nil { return fmt.Errorf("could not clean up: %w", err) } return nil } func cleanUp(file string, vc *gocv.VideoCapture) error { err := os.Remove(file) if err != nil { return fmt.Errorf("could not remove temp file: %w", err) } err = vc.Close() if err != nil { return fmt.Errorf("could not close video capture device: %w", err) } return nil } func floatToMat(array []float64) (gocv.Mat, error) { H := gocv.NewMatWithSize(3, 3, gocv.MatTypeCV64F) for i := 0; i < H.Rows(); i++ { for j := 0; j < H.Cols(); j++ { H.SetDoubleAt(i, j, array[i*H.Cols()+j]) } } return H, nil }