Merged in target-location (pull request #478)

Add functionality to set the location of the chessboard

Approved-by: Saxon Milton

cmd/rv/main.go

@@ -175,12 +175,12 @@ func main() {
 	time.Sleep(runPreDelay)
 
 	log.Log(logger.Debug, "beginning main loop")
-	run(rv, ns, log, netLog)
+	run(rv, ns, log, netLog, p)
 }
 
 // run starts the main loop. This will run netsender on every pass of the loop
 // (sleeping inbetween), check vars, and if changed, update revid as appropriate.
-func run(rv *revid.Revid, ns *netsender.Sender, l *logger.Logger, nl *netlogger.Logger) {
+func run(rv *revid.Revid, ns *netsender.Sender, l *logger.Logger, nl *netlogger.Logger, p *turbidityProbe) {
 	var vs int
 	for {
 		l.Log(logger.Debug, "running netsender")
@@ -225,6 +225,12 @@ func run(rv *revid.Revid, ns *netsender.Sender, l *logger.Logger, nl *netlogger.
 		}
 		l.Log(logger.Info, "revid successfully reconfigured")
 
+		// Update transform matrix based on new revid variables.
+		err = p.Update(rv.Config().TransformMatrix)
+		if err != nil {
+			l.Log(logger.Error, "could not update turbidity probe", "error", err.Error())
+		}
+
 		l.Log(logger.Debug, "checking mode")
 		switch ns.Mode() {
 		case modePaused:

cmd/rv/probe.go

@@ -49,6 +49,7 @@ const (
 	maxImages     = 1     // Max number of images read when evaluating turbidity.
 	bufferLimit   = 20000 // 20KB
 	trimTolerance = 200   // Number of times trim can be called where no keyframe is found.
+	transformSize = 9     // Size of the square projective matrix.
 )
 
 // Turbidity sensor constants.
@@ -68,6 +69,7 @@ type turbidityProbe struct {
 	ts                  *turbidity.TurbiditySensor
 	log                 logger.Logger
 	buffer              *bytes.Buffer
+	transform           []float64
 	trimCounter         int
 }
 
@@ -79,10 +81,12 @@ func NewTurbidityProbe(log logger.Logger, delay time.Duration) (*turbidityProbe,
 	tp.ticker = *time.NewTicker(delay)
 	tp.buffer = bytes.NewBuffer(*new([]byte))
 
+	tp.transform = make([]float64, transformSize)
+	transformMatrix := floatToMat(tp.transform)
+
 	// Create the turbidity sensor.
-	standard := gocv.IMRead("../../turbidity/images/default.jpg", gocv.IMReadGrayScale)
 	template := gocv.IMRead("../../turbidity/images/template.jpg", gocv.IMReadGrayScale)
-	ts, err := turbidity.NewTurbiditySensor(template, standard, k1, k2, filterSize, scale, alpha, log)
+	ts, err := turbidity.NewTurbiditySensor(template, transformMatrix, k1, k2, filterSize, scale, alpha, log)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create turbidity sensor: %w", err)
 	}
@@ -141,6 +145,26 @@ func (tp *turbidityProbe) Close() error {
 	return nil
 }
 
+// Update will update the probe and turbidity sensor with the new transformation matrix if it has been changed.
+func (tp *turbidityProbe) Update(transformMatrix []float64) error {
+	if len(transformMatrix) != transformSize {
+		return errors.New("transformation matrix has incorrect size")
+	}
+	for i := range tp.transform {
+		if tp.transform[i] == transformMatrix[i] {
+			continue
+		}
+		// Update the turbidity sensor with new transformation.
+		tp.log.Log(logger.Debug, "updating the transformation matrix")
+		tp.transform = transformMatrix
+		newTransform := floatToMat(tp.transform)
+		tp.ts.TransformMatrix = newTransform
+		return nil
+	}
+	tp.log.Log(logger.Debug, "no change to the transformation matrix")
+	return nil
+}
+
 func (tp *turbidityProbe) turbidityCalculation() error {
 	var imgs []gocv.Mat
 	img := gocv.NewMat()
@@ -208,3 +232,14 @@ func cleanUp(file string, vc *gocv.VideoCapture) error {
 	}
 	return nil
 }
+
+// floatToMat will convert a slice of 9 floats to a gocv.Mat.
+func floatToMat(array []float64) gocv.Mat {
+	mat := gocv.NewMatWithSize(3, 3, gocv.MatTypeCV64F)
+	for i := 0; i < mat.Rows(); i++ {
+		for j := 0; j < mat.Cols(); j++ {
+			mat.SetDoubleAt(i, j, array[i*mat.Cols()+j])
+		}
+	}
+	return mat
+}

cmd/rv/probe_circleci.go

@@ -46,10 +46,8 @@ func NewTurbidityProbe(log logger.Logger, delay time.Duration) (*turbidityProbe,
 }
 
 // Write performs no operation for CircleCI testing only.
-func (tp *turbidityProbe) Write(p []byte) (int, error) {
-	return 0, nil
-}
+func (tp *turbidityProbe) Write(p []byte) (int, error) { return 0, nil }
 
-func (tp *turbidityProbe) Close() error {
-	return nil
-}
+func (tp *turbidityProbe) Update(mat []float64) error { return nil }
+
+func (tp *turbidityProbe) Close() error { return nil }

cmd/rv/probe_test.go

@@ -44,11 +44,18 @@ func TestProbe(t *testing.T) {
 		MaxAge:     logMaxAge,
 	}
 	log := logger.New(logVerbosity, io.MultiWriter(fileLog), logSuppress)
+	updatedMatrix := []float64{-0.2731048063, -0.0020501869, 661.0275911942, 0.0014327789, -0.2699443748, 339.3921028016, 0.0000838317, 0.0000476486, 1.0}
 
 	ts, err := NewTurbidityProbe(*log, time.Microsecond)
 	if err != nil {
 		t.Fatalf("failed to create turbidity probe")
 	}
+
+	err = ts.Update(updatedMatrix)
+	if err != nil {
+		t.Fatalf("could not update probe: %v", err)
+	}
+
 	video, err := ioutil.ReadFile("logo.h264")
 	if err != nil {
 		t.Fatalf("failed to read file: %v", err)

revid/config/config.go

@@ -273,6 +273,10 @@ type Config struct {
 
 	VerticalFlip bool // VerticalFlip flips video vertically for Raspivid input.
 	Width        uint // Width defines the input video width Raspivid input.
+
+	// TransformMatrix describes the projective transformation matrix to extract a target from the
+	// video data for turbidty calculations.
+	TransformMatrix []float64
 }
 
 // Validate checks for any errors in the config fields and defaults settings

revid/config/config_test.go

@@ -122,6 +122,7 @@ func TestUpdate(t *testing.T) {
 		"VBRQuality":        "excellent",
 		"VerticalFlip":      "true",
 		"Width":             "300",
+		"TransformMatrix":   "0.1,  0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8,0.9",
 	}
 
 	dl := &dumbLogger{}
@@ -172,6 +173,7 @@ func TestUpdate(t *testing.T) {
 		VBRQuality:        QualityExcellent,
 		VerticalFlip:      true,
 		Width:             300,
+		TransformMatrix:   []float64{0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9},
 	}
 
 	got := Config{Logger: dl}

revid/config/variables.go

@@ -99,6 +99,7 @@ const (
 	KeyVBRQuality           = "VBRQuality"
 	KeyVerticalFlip         = "VerticalFlip"
 	KeyWidth                = "Width"
+	KeyTransformMatrix      = "TransformMatrix"
 )
 
 // Config map parameter types.
@@ -144,6 +145,29 @@ var Variables = []struct {
 	Update   func(*Config, string)
 	Validate func(*Config)
 }{
+	{
+		Name: KeyTransformMatrix,
+		Type: typeString,
+		Update: func(c *Config, v string) {
+			c.Logger.Log(logger.Debug, "updating transform matrix", "string", v)
+			v = strings.Replace(v, " ", "", -1)
+			vals := make([]float64, 0)
+			if v == "" {
+				c.TransformMatrix = vals
+				return
+			}
+
+			elements := strings.Split(v, ",")
+			for _, e := range elements {
+				vFloat, err := strconv.ParseFloat(e, 64)
+				if err != nil {
+					c.Logger.Log(logger.Warning, "invalid TransformMatrix param", "value", e)
+				}
+				vals = append(vals, vFloat)
+			}
+			c.TransformMatrix = vals
+		},
+	},
 	{
 		Name:   KeyAutoWhiteBalance,
 		Type:   "enum:off,auto,sun,cloud,shade,tungsten,fluorescent,incandescent,flash,horizon",

turbidity/transform.go

@@ -0,0 +1,73 @@
+//go:build !nocv
+// +build !nocv
+
+/*
+DESCRIPTION
+  Provides a function which can extract the transformation matrix.
+
+AUTHORS
+  Russell Stanley <russell@ausocean.org>
+
+LICENSE
+  Copyright (C) 2021-2022 the Australian Ocean Lab (AusOcean)
+
+  It is free software: you can redistribute it and/or modify them
+  under the terms of the GNU General Public License as published by the
+  Free Software Foundation, either version 3 of the License, or (at your
+  option) any later version.
+
+  It is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+  for more details.
+
+  You should have received a copy of the GNU General Public License
+  in gpl.txt.  If not, see http://www.gnu.org/licenses.
+*/
+
+package turbidity
+
+import (
+	"errors"
+	"image"
+
+	"gocv.io/x/gocv"
+)
+
+// Perspective transformation constants.
+const (
+	ransacThreshold = 3.0   // Maximum allowed reprojection error to treat a point pair as an inlier.
+	maxIter         = 2000  // The maximum number of RANSAC iterations.
+	confidence      = 0.995 // Confidence level, between 0 and 1.
+)
+
+// FindTransform, given a template and standard image the perspetive transformation matrix will be determined.
+// the matrix will be returned and logged for use in vidgrind.
+func FindTransform(standardPath, templatePath string) (gocv.Mat, error) {
+	mask := gocv.NewMat()
+	std := gocv.IMRead(standardPath, gocv.IMReadColor)
+	stdCorners := gocv.NewMat()
+
+	template := gocv.IMRead(templatePath, gocv.IMReadGrayScale)
+	templateCorners := gocv.NewMat()
+	transformMatrix := gocv.NewMat()
+
+	// Validate template image is not empty and has valid corners.
+	if template.Empty() {
+		return transformMatrix, errors.New("template image is empty")
+	}
+	if !gocv.FindChessboardCorners(template, image.Pt(3, 3), &templateCorners, gocv.CalibCBNormalizeImage) {
+		return transformMatrix, errors.New("could not find corners in template image")
+	}
+
+	// Validate standard image is not empty and has valid corners.
+	if std.Empty() {
+		return transformMatrix, errors.New("standard image is empty")
+	}
+	if !gocv.FindChessboardCorners(std, image.Pt(3, 3), &stdCorners, gocv.CalibCBNormalizeImage) {
+		return transformMatrix, errors.New("could not find corners in standard image")
+	}
+
+	transformMatrix = gocv.FindHomography(stdCorners, &templateCorners, gocv.HomograpyMethodRANSAC, ransacThreshold, &mask, maxIter, confidence)
+	return transformMatrix, nil
+}

turbidity/turbidity.go

@@ -44,39 +44,24 @@ import (
 // TurbiditySensor is a software based turbidity sensor that uses CV to determine sharpness and constrast level
 // of a chessboard-like target submerged in water that can be correlated to turbidity/visibility values.
 type TurbiditySensor struct {
-	template, templateCorners gocv.Mat
-	standard, standardCorners gocv.Mat
+	template                gocv.Mat // Holds the image of the target.
+	TransformMatrix         gocv.Mat // The current perspective transformation matrix to extract the target from the frame.
 	k1, k2, sobelFilterSize int
 	scale, alpha            float64
 	log                     logger.Logger
 }
 
 // NewTurbiditySensor returns a new TurbiditySensor.
-func NewTurbiditySensor(template, standard gocv.Mat, k1, k2, sobelFilterSize int, scale, alpha float64, log logger.Logger) (*TurbiditySensor, error) {
+func NewTurbiditySensor(template, transformMatrix gocv.Mat, k1, k2, sobelFilterSize int, scale, alpha float64, log logger.Logger) (*TurbiditySensor, error) {
 	ts := new(TurbiditySensor)
-	templateCorners := gocv.NewMat()
-	standardCorners := gocv.NewMat()
 
 	// Validate template image is not empty and has valid corners.
 	if template.Empty() {
 		return nil, errors.New("template image is empty")
 	}
-	if !gocv.FindChessboardCorners(template, image.Pt(3, 3), &templateCorners, gocv.CalibCBNormalizeImage) {
-		return nil, errors.New("could not find corners in template image")
-	}
+
 	ts.template = template
-	ts.templateCorners = templateCorners
-
-	// Validate standard image is not empty and has valid corners.
-	if standard.Empty() {
-		return nil, errors.New("standard image is empty")
-	}
-	if !gocv.FindChessboardCorners(standard, image.Pt(3, 3), &standardCorners, gocv.CalibCBNormalizeImage) {
-		return nil, errors.New("could not find corners in standard image")
-	}
-	ts.standard = standard
-	ts.standardCorners = standardCorners
-
+	ts.TransformMatrix = transformMatrix
 	ts.k1, ts.k2, ts.sobelFilterSize = k1, k2, sobelFilterSize
 	ts.alpha, ts.scale = alpha, scale
 	ts.log = log
@@ -191,13 +176,6 @@ func (ts TurbiditySensor) evaluateBlockAMEE(img gocv.Mat, xStart, yStart, xEnd,
 // transform will search img for matching template. Returns the transformed image which best match the template.
 func (ts TurbiditySensor) transform(img gocv.Mat) (gocv.Mat, error) {
 	out := gocv.NewMat()
-	mask := gocv.NewMat()
-	imgCorners := ts.standardCorners
-	const (
-		ransacThreshold = 3.0   // Maximum allowed reprojection error to treat a point pair as an inlier.
-		maxIter         = 2000  // The maximum number of RANSAC iterations.
-		confidence      = 0.995 // Confidence level, between 0 and 1.
-	)
 
 	if img.Empty() {
 		return out, errors.New("image is empty, cannot transform")
@@ -206,8 +184,7 @@ func (ts TurbiditySensor) transform(img gocv.Mat) (gocv.Mat, error) {
 	// if !gocv.FindChessboardCorners(img, image.Pt(3, 3), &imgCorners, gocv.CalibCBFastCheck) {}
 
 	// Find and apply transformation.
-	H := gocv.FindHomography(imgCorners, &ts.templateCorners, gocv.HomograpyMethodRANSAC, ransacThreshold, &mask, maxIter, confidence)
-	gocv.WarpPerspective(img, &out, H, image.Pt(ts.template.Rows(), ts.template.Cols()))
+	gocv.WarpPerspective(img, &out, ts.TransformMatrix, image.Pt(ts.template.Rows(), ts.template.Cols()))
 	gocv.CvtColor(out, &out, gocv.ColorRGBToGray)
 	return out, nil
 }

turbidity/turbidity_test.go

@@ -42,8 +42,8 @@ import (
 )
 
 const (
-	nImages   = 10  // Number of images to test. (Max 13)
-	nSamples  = 1   // Number of samples for each image. (Max 10)
+	nImages   = 13  // Number of images to test. (Max 13)
+	nSamples  = 10  // Number of samples for each image. (Max 10)
 	increment = 2.5 // Increment of the turbidity level.
 )
 
@@ -62,7 +62,7 @@ const (
 func TestImages(t *testing.T) {
 
 	const (
-		k1, k2       = 8, 8
+		k1, k2       = 4, 4
 		filterSize   = 3
 		scale, alpha = 1.0, 1.0
 	)
@@ -77,7 +77,11 @@ func TestImages(t *testing.T) {
 	log := *logger.New(logVerbosity, io.MultiWriter(fileLog), logSuppress)
 
 	template := gocv.IMRead("images/template.jpg", gocv.IMReadGrayScale)
-	standard := gocv.IMRead("images/default.jpg", gocv.IMReadGrayScale)
+	transformMatrix, err := FindTransform("images/default.jpg", "images/template.jpg")
+	if err != nil {
+		t.Fatalf("could not find transformation: %v", err)
+	}
+	t.Log(formatMat(transformMatrix))
 
 	imgs := make([][]gocv.Mat, nImages)
 
@@ -85,11 +89,11 @@ func TestImages(t *testing.T) {
 	for i := range imgs {
 		imgs[i] = make([]gocv.Mat, nSamples)
 		for j := range imgs[i] {
-			imgs[i][j] = gocv.IMRead(fmt.Sprintf("images/t-%v/000%v.jpg", i, j), gocv.IMReadGrayScale)
+			imgs[i][j] = gocv.IMRead(fmt.Sprintf("images/t-%v/000%v.jpg", i, j), gocv.IMReadColor)
 		}
 	}
 
-	ts, err := NewTurbiditySensor(template, standard, k1, k2, filterSize, scale, alpha, log)
+	ts, err := NewTurbiditySensor(template, transformMatrix, k1, k2, filterSize, scale, alpha, log)
 	if err != nil {
 		t.Fatalf("could not create turbidity sensor: %v", err)
 	}
@@ -111,7 +115,7 @@ func TestImages(t *testing.T) {
 		results.Update(stat.Mean(sample_result.Sharpness, nil), stat.Mean(sample_result.Contrast, nil), float64(i)*increment, i)
 	}
 
-	err = plotResults(results.Turbidity, results.Sharpness, results.Contrast)
+	err = plotResults(results.Turbidity, normalize(results.Sharpness), normalize(results.Contrast))
 	if err != nil {
 		t.Fatalf("plotting Failed: %v", err)
 	}
@@ -138,3 +142,17 @@ func plotResults(x, sharpness, contrast []float64) error {
 	}
 	return nil
 }
+
+// formatMat creates a formatted transformation matrix string for use in vidgrind.
+func formatMat(transformMatrix gocv.Mat) string {
+	var out string
+	for i := 0; i < transformMatrix.Rows(); i++ {
+		for j := 0; j < transformMatrix.Cols(); j++ {
+			out += fmt.Sprintf(" %.10f", transformMatrix.GetDoubleAt(i, j))
+			if i < 2 || j < 2 {
+				out += ","
+			}
+		}
+	}
+	return out
+}