From 3047704ca08ad60ac8d486832838a2a166456f04 Mon Sep 17 00:00:00 2001 From: Russell Stanley Date: Mon, 11 Apr 2022 09:23:40 +0930 Subject: [PATCH 01/10] revid: add transfrom matrix variable --- cmd/rv/main.go | 2 +- cmd/rv/probe.go | 5 +- cmd/rv/probe_test.go | 3 +- revid/config/config.go | 2 + revid/config/variables.go | 140 ++++++++++++++++++++---------------- turbidity/turbidity.go | 21 +++--- turbidity/turbidity_test.go | 17 +++-- 7 files changed, 112 insertions(+), 78 deletions(-) diff --git a/cmd/rv/main.go b/cmd/rv/main.go index bd22e8b9..c430faed 100644 --- a/cmd/rv/main.go +++ b/cmd/rv/main.go @@ -146,7 +146,7 @@ func main() { p *turbidityProbe ) - p, err := NewTurbidityProbe(*log, 60*time.Second) + p, err := NewTurbidityProbe(*log, 60*time.Second, rv.Config().TransformMatrix) if err != nil { log.Log(logger.Fatal, "could not create new turbidity probe", "error", err.Error()) } diff --git a/cmd/rv/probe.go b/cmd/rv/probe.go index 9bda6931..5e1ca408 100644 --- a/cmd/rv/probe.go +++ b/cmd/rv/probe.go @@ -68,16 +68,18 @@ type turbidityProbe struct { ts *turbidity.TurbiditySensor log logger.Logger buffer *bytes.Buffer + transform []float64 trimCounter int } // NewTurbidityProbe returns a new turbidity probe. -func NewTurbidityProbe(log logger.Logger, delay time.Duration) (*turbidityProbe, error) { +func NewTurbidityProbe(log logger.Logger, delay time.Duration, transformMatrix []float64) (*turbidityProbe, error) { tp := new(turbidityProbe) tp.log = log tp.delay = delay tp.ticker = *time.NewTicker(delay) tp.buffer = bytes.NewBuffer(*new([]byte)) + tp.transform = transformMatrix // Create the turbidity sensor. standard := gocv.IMRead("../../turbidity/images/default.jpg", gocv.IMReadGrayScale) @@ -125,6 +127,7 @@ func (tp *turbidityProbe) Write(p []byte) (int, error) { select { case <-tp.ticker.C: tp.log.Log(logger.Debug, "beginning turbidity calculation") + tp.log.Log(logger.Debug, "transformation matrix", tp.transform[0]) startTime := time.Now() err := tp.turbidityCalculation() if err != nil { diff --git a/cmd/rv/probe_test.go b/cmd/rv/probe_test.go index 4f40c702..f04d6b10 100644 --- a/cmd/rv/probe_test.go +++ b/cmd/rv/probe_test.go @@ -44,8 +44,9 @@ func TestProbe(t *testing.T) { MaxAge: logMaxAge, } log := logger.New(logVerbosity, io.MultiWriter(fileLog), logSuppress) + transformMatrix := []float64{0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0} - ts, err := NewTurbidityProbe(*log, time.Microsecond) + ts, err := NewTurbidityProbe(*log, time.Microsecond, transformMatrix) if err != nil { t.Fatalf("failed to create turbidity probe") } diff --git a/revid/config/config.go b/revid/config/config.go index 7cf9864a..2e665f72 100644 --- a/revid/config/config.go +++ b/revid/config/config.go @@ -273,6 +273,8 @@ type Config struct { VerticalFlip bool // VerticalFlip flips video vertically for Raspivid input. Width uint // Width defines the input video width Raspivid input. + + TransformMatrix []float64 // Describes the transformation matrix to extract the target. } // Validate checks for any errors in the config fields and defaults settings diff --git a/revid/config/variables.go b/revid/config/variables.go index db16aa33..8098eb68 100644 --- a/revid/config/variables.go +++ b/revid/config/variables.go @@ -99,6 +99,7 @@ const ( KeyVBRQuality = "VBRQuality" KeyVerticalFlip = "VerticalFlip" KeyWidth = "Width" + KeyTransformMatrix = "TransformMatrix" ) // Config map parameter types. @@ -140,38 +141,55 @@ const ( // this variable in a Config, and a function for validating the value of the variable. var Variables = []struct { Name string - Type string + Type string Update func(*Config, string) Validate func(*Config) }{ + { + Name: KeyTransformMatrix, + Type: typeString, + Update: func(c *Config, v string) { + v = strings.Replace(v, " ", "", 0) + elements := strings.Split(v, ",") + vals := make([]float64, len(elements)) + for _, i := range elements { + vFloat, err := strconv.ParseFloat(i, 64) + if err != nil { + c.Logger.Log(logger.Warning, "invalid TransformMatrix param", "value", i) + } + vals = append(vals, vFloat) + } + c.TransformMatrix = vals + }, + }, { Name: KeyAutoWhiteBalance, - Type: "enum:off,auto,sun,cloud,shade,tungsten,fluorescent,incandescent,flash,horizon", + Type: "enum:off,auto,sun,cloud,shade,tungsten,fluorescent,incandescent,flash,horizon", Update: func(c *Config, v string) { c.AutoWhiteBalance = v }, }, { Name: KeyAWBGains, - Type: typeString, + Type: typeString, Update: func(c *Config, v string) { c.AWBGains = v }, }, { Name: KeyBitDepth, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.BitDepth = parseUint(KeyBitDepth, v, c) }, }, { Name: KeyBitrate, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.Bitrate = parseUint(KeyBitrate, v, c) }, }, { Name: KeyBrightness, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.Brightness = parseUint(KeyBrightness, v, c) }, }, { Name: KeyBurstPeriod, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.BurstPeriod = parseUint(KeyBurstPeriod, v, c) }, Validate: func(c *Config) { if c.BurstPeriod <= 0 { @@ -182,12 +200,12 @@ var Variables = []struct { }, { Name: KeyCameraChan, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.CameraChan = uint8(parseUint(KeyCameraChan, v, c)) }, }, { Name: KeyCameraIP, - Type: typeString, + Type: typeString, Update: func(c *Config, v string) { c.CameraIP = v }, Validate: func(c *Config) { if c.CameraIP == "" { @@ -198,11 +216,11 @@ var Variables = []struct { }, { Name: KeyCBR, - Type: typeBool, + Type: typeBool, Update: func(c *Config, v string) { c.CBR = parseBool(KeyCBR, v, c) }, }, { - Name: KeyClipDuration, + Name: KeyClipDuration, Type: typeUint, Update: func(c *Config, v string) { _v, err := strconv.Atoi(v) @@ -220,27 +238,27 @@ var Variables = []struct { }, { Name: KeyChannels, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.Channels = parseUint(KeyChannels, v, c) }, }, { Name: KeyContrast, - Type: typeInt, + Type: typeInt, Update: func(c *Config, v string) { c.Contrast = parseInt(KeyContrast, v, c) }, }, { Name: KeyEV, - Type: typeInt, + Type: typeInt, Update: func(c *Config, v string) { c.EV = parseInt(KeyEV, v, c) }, }, { Name: KeyExposure, - Type: "enum:auto,night,nightpreview,backlight,spotlight,sports,snow,beach,verylong,fixedfps,antishake,fireworks", + Type: "enum:auto,night,nightpreview,backlight,spotlight,sports,snow,beach,verylong,fixedfps,antishake,fireworks", Update: func(c *Config, v string) { c.Exposure = v }, }, { Name: KeyFileFPS, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.FileFPS = parseUint(KeyFileFPS, v, c) }, Validate: func(c *Config) { if c.FileFPS <= 0 || (c.FileFPS > 0 && c.Input != InputFile) { @@ -250,7 +268,7 @@ var Variables = []struct { }, }, { - Name: KeyFilters, + Name: KeyFilters, Type: "enums:NoOp,MOG,VariableFPS,KNN,Difference,Basic", Update: func(c *Config, v string) { filters := strings.Split(v, ",") @@ -267,7 +285,7 @@ var Variables = []struct { }, { Name: KeyFrameRate, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.FrameRate = parseUint(KeyFrameRate, v, c) }, Validate: func(c *Config) { if c.FrameRate <= 0 || c.FrameRate > 60 { @@ -278,21 +296,21 @@ var Variables = []struct { }, { Name: KeyHeight, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.Height = parseUint(KeyHeight, v, c) }, }, { Name: KeyHorizontalFlip, - Type: typeBool, + Type: typeBool, Update: func(c *Config, v string) { c.HorizontalFlip = parseBool(KeyHorizontalFlip, v, c) }, }, { Name: KeyHTTPAddress, - Type: typeString, + Type: typeString, Update: func(c *Config, v string) { c.HTTPAddress = v }, }, { - Name: KeyInput, + Name: KeyInput, Type: "enum:raspivid,raspistill,rtsp,v4l,file,audio", Update: func(c *Config, v string) { c.Input = parseEnum( @@ -319,7 +337,7 @@ var Variables = []struct { }, }, { - Name: KeyInputCodec, + Name: KeyInputCodec, Type: "enum:h264,h265,mjpeg,jpeg,pcm,adpcm", Update: func(c *Config, v string) { c.InputCodec = v @@ -333,16 +351,16 @@ var Variables = []struct { }, { Name: KeyInputPath, - Type: typeString, + Type: typeString, Update: func(c *Config, v string) { c.InputPath = v }, }, { Name: KeyISO, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.ISO = parseUint(KeyISO, v, c) }, }, { - Name: KeyLogging, + Name: KeyLogging, Type: "enum:Debug,Info,Warning,Error,Fatal", Update: func(c *Config, v string) { switch v { @@ -371,18 +389,18 @@ var Variables = []struct { }, { Name: KeyLoop, - Type: typeBool, + Type: typeBool, Update: func(c *Config, v string) { c.Loop = parseBool(KeyLoop, v, c) }, }, { Name: KeyMinFPS, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.MinFPS = parseUint(KeyMinFPS, v, c) }, Validate: func(c *Config) { c.MinFPS = lessThanOrEqual(KeyMinFPS, c.MinFPS, 0, c, defaultMinFPS) }, }, { Name: KeyMinFrames, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.MinFrames = parseUint(KeyMinFrames, v, c) }, Validate: func(c *Config) { const maxMinFrames = 1000 @@ -393,7 +411,7 @@ var Variables = []struct { }, }, { - Name: KeyMode, + Name: KeyMode, Type: "enum:Normal,Paused,Burst,Loop", Update: func(c *Config, v string) { c.Loop = false @@ -404,26 +422,26 @@ var Variables = []struct { }, { Name: KeyMotionDownscaling, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.MotionDownscaling = parseUint(KeyMotionDownscaling, v, c) }, }, { Name: KeyMotionHistory, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.MotionHistory = parseUint(KeyMotionHistory, v, c) }, }, { Name: KeyMotionInterval, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.MotionInterval = parseUint(KeyMotionInterval, v, c) }, }, { Name: KeyMotionKernel, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.MotionKernel = parseUint(KeyMotionKernel, v, c) }, }, { - Name: KeyMotionMinArea, + Name: KeyMotionMinArea, Type: typeFloat, Update: func(c *Config, v string) { f, err := strconv.ParseFloat(v, 64) @@ -435,16 +453,16 @@ var Variables = []struct { }, { Name: KeyMotionPadding, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.MotionPadding = parseUint(KeyMotionPadding, v, c) }, }, { Name: KeyMotionPixels, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.MotionPixels = parseUint(KeyMotionPixels, v, c) }, }, { - Name: KeyMotionThreshold, + Name: KeyMotionThreshold, Type: typeFloat, Update: func(c *Config, v string) { f, err := strconv.ParseFloat(v, 64) @@ -455,7 +473,7 @@ var Variables = []struct { }, }, { - Name: KeyOutput, + Name: KeyOutput, Type: "enum:File,HTTP,RTMP,RTP", Update: func(c *Config, v string) { c.Outputs = make([]uint8, 1) @@ -477,11 +495,11 @@ var Variables = []struct { }, { Name: KeyOutputPath, - Type: typeString, + Type: typeString, Update: func(c *Config, v string) { c.OutputPath = v }, }, { - Name: KeyOutputs, + Name: KeyOutputs, Type: "enums:File,HTTP,RTMP,RTP", Update: func(c *Config, v string) { outputs := strings.Split(v, ",") @@ -512,18 +530,18 @@ var Variables = []struct { }, { Name: KeyPSITime, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.PSITime = parseUint(KeyPSITime, v, c) }, Validate: func(c *Config) { c.PSITime = lessThanOrEqual(KeyPSITime, c.PSITime, 0, c, defaultPSITime) }, }, { Name: KeyQuantization, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.Quantization = parseUint(KeyQuantization, v, c) }, }, { Name: KeyPoolCapacity, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.PoolCapacity = parseUint(KeyPoolCapacity, v, c) }, Validate: func(c *Config) { c.PoolCapacity = lessThanOrEqual(KeyPoolCapacity, c.PoolCapacity, 0, c, defaultPoolCapacity) @@ -531,7 +549,7 @@ var Variables = []struct { }, { Name: KeyPoolStartElementSize, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.PoolStartElementSize = parseUint("PoolStartElementSize", v, c) }, Validate: func(c *Config) { c.PoolStartElementSize = lessThanOrEqual("PoolStartElementSize", c.PoolStartElementSize, 0, c, defaultPoolStartElementSize) @@ -539,14 +557,14 @@ var Variables = []struct { }, { Name: KeyPoolWriteTimeout, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.PoolWriteTimeout = parseUint(KeyPoolWriteTimeout, v, c) }, Validate: func(c *Config) { c.PoolWriteTimeout = lessThanOrEqual(KeyPoolWriteTimeout, c.PoolWriteTimeout, 0, c, defaultPoolWriteTimeout) }, }, { - Name: KeyRecPeriod, + Name: KeyRecPeriod, Type: typeFloat, Update: func(c *Config, v string) { _v, err := strconv.ParseFloat(v, 64) @@ -558,17 +576,17 @@ var Variables = []struct { }, { Name: KeyRotation, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.Rotation = parseUint(KeyRotation, v, c) }, }, { Name: KeyRTMPURL, - Type: typeString, + Type: typeString, Update: func(c *Config, v string) { c.RTMPURL = v }, }, { Name: KeyRTPAddress, - Type: typeString, + Type: typeString, Update: func(c *Config, v string) { c.RTPAddress = v }, Validate: func(c *Config) { if c.RTPAddress == "" { @@ -579,21 +597,21 @@ var Variables = []struct { }, { Name: KeySampleRate, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.SampleRate = parseUint(KeySampleRate, v, c) }, }, { Name: KeySaturation, - Type: typeInt, + Type: typeInt, Update: func(c *Config, v string) { c.Saturation = parseInt(KeySaturation, v, c) }, }, { Name: KeySharpness, - Type: typeInt, + Type: typeInt, Update: func(c *Config, v string) { c.Sharpness = parseInt(KeySharpness, v, c) }, }, { - Name: KeyJPEGQuality, + Name: KeyJPEGQuality, Type: typeUint, Update: func(c *Config, v string) { _v, err := strconv.Atoi(v) @@ -604,7 +622,7 @@ var Variables = []struct { }, }, { - Name: KeySuppress, + Name: KeySuppress, Type: typeBool, Update: func(c *Config, v string) { c.Suppress = parseBool(KeySuppress, v, c) @@ -612,7 +630,7 @@ var Variables = []struct { }, }, { - Name: KeyTimelapseInterval, + Name: KeyTimelapseInterval, Type: typeUint, Update: func(c *Config, v string) { _v, err := strconv.Atoi(v) @@ -623,7 +641,7 @@ var Variables = []struct { }, }, { - Name: KeyTimelapseDuration, + Name: KeyTimelapseDuration, Type: typeUint, Update: func(c *Config, v string) { _v, err := strconv.Atoi(v) @@ -635,11 +653,11 @@ var Variables = []struct { }, { Name: KeyVBRBitrate, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.VBRBitrate = parseUint(KeyVBRBitrate, v, c) }, }, { - Name: KeyVBRQuality, + Name: KeyVBRQuality, Type: "enum:standard,fair,good,great,excellent", Update: func(c *Config, v string) { c.VBRQuality = Quality(parseEnum( @@ -658,12 +676,12 @@ var Variables = []struct { }, { Name: KeyVerticalFlip, - Type: typeBool, + Type: typeBool, Update: func(c *Config, v string) { c.VerticalFlip = parseBool(KeyVerticalFlip, v, c) }, }, { Name: KeyWidth, - Type: typeUint, + Type: typeUint, Update: func(c *Config, v string) { c.Width = parseUint(KeyWidth, v, c) }, }, } diff --git a/turbidity/turbidity.go b/turbidity/turbidity.go index 8bbc4d6e..cc8578c2 100644 --- a/turbidity/turbidity.go +++ b/turbidity/turbidity.go @@ -41,11 +41,19 @@ import ( "gocv.io/x/gocv" ) +// Homography constants +const ( + ransacThreshold = 3.0 // Maximum allowed reprojection error to treat a point pair as an inlier. + maxIter = 2000 // The maximum number of RANSAC iterations. + confidence = 0.995 // Confidence level, between 0 and 1. +) + // TurbiditySensor is a software based turbidity sensor that uses CV to determine sharpness and constrast level // of a chessboard-like target submerged in water that can be correlated to turbidity/visibility values. type TurbiditySensor struct { template, templateCorners gocv.Mat standard, standardCorners gocv.Mat + H gocv.Mat k1, k2, sobelFilterSize int scale, alpha float64 log logger.Logger @@ -56,6 +64,7 @@ func NewTurbiditySensor(template, standard gocv.Mat, k1, k2, sobelFilterSize int ts := new(TurbiditySensor) templateCorners := gocv.NewMat() standardCorners := gocv.NewMat() + mask := gocv.NewMat() // Validate template image is not empty and has valid corners. if template.Empty() { @@ -74,8 +83,10 @@ func NewTurbiditySensor(template, standard gocv.Mat, k1, k2, sobelFilterSize int if !gocv.FindChessboardCorners(standard, image.Pt(3, 3), &standardCorners, gocv.CalibCBNormalizeImage) { return nil, errors.New("could not find corners in standard image") } + ts.standard = standard ts.standardCorners = standardCorners + ts.H = gocv.FindHomography(ts.standardCorners, &ts.templateCorners, gocv.HomograpyMethodRANSAC, ransacThreshold, &mask, maxIter, confidence) ts.k1, ts.k2, ts.sobelFilterSize = k1, k2, sobelFilterSize ts.alpha, ts.scale = alpha, scale @@ -191,13 +202,6 @@ func (ts TurbiditySensor) evaluateBlockAMEE(img gocv.Mat, xStart, yStart, xEnd, // transform will search img for matching template. Returns the transformed image which best match the template. func (ts TurbiditySensor) transform(img gocv.Mat) (gocv.Mat, error) { out := gocv.NewMat() - mask := gocv.NewMat() - imgCorners := ts.standardCorners - const ( - ransacThreshold = 3.0 // Maximum allowed reprojection error to treat a point pair as an inlier. - maxIter = 2000 // The maximum number of RANSAC iterations. - confidence = 0.995 // Confidence level, between 0 and 1. - ) if img.Empty() { return out, errors.New("image is empty, cannot transform") @@ -206,8 +210,7 @@ func (ts TurbiditySensor) transform(img gocv.Mat) (gocv.Mat, error) { // if !gocv.FindChessboardCorners(img, image.Pt(3, 3), &imgCorners, gocv.CalibCBFastCheck) {} // Find and apply transformation. - H := gocv.FindHomography(imgCorners, &ts.templateCorners, gocv.HomograpyMethodRANSAC, ransacThreshold, &mask, maxIter, confidence) - gocv.WarpPerspective(img, &out, H, image.Pt(ts.template.Rows(), ts.template.Cols())) + gocv.WarpPerspective(img, &out, ts.H, image.Pt(ts.template.Rows(), ts.template.Cols())) gocv.CvtColor(out, &out, gocv.ColorRGBToGray) return out, nil } diff --git a/turbidity/turbidity_test.go b/turbidity/turbidity_test.go index 4c97b65a..72f4e5bf 100644 --- a/turbidity/turbidity_test.go +++ b/turbidity/turbidity_test.go @@ -42,8 +42,8 @@ import ( ) const ( - nImages = 10 // Number of images to test. (Max 13) - nSamples = 1 // Number of samples for each image. (Max 10) + nImages = 12 // Number of images to test. (Max 13) + nSamples = 10 // Number of samples for each image. (Max 10) increment = 2.5 // Increment of the turbidity level. ) @@ -62,7 +62,7 @@ const ( func TestImages(t *testing.T) { const ( - k1, k2 = 8, 8 + k1, k2 = 4, 4 filterSize = 3 scale, alpha = 1.0, 1.0 ) @@ -85,7 +85,7 @@ func TestImages(t *testing.T) { for i := range imgs { imgs[i] = make([]gocv.Mat, nSamples) for j := range imgs[i] { - imgs[i][j] = gocv.IMRead(fmt.Sprintf("images/t-%v/000%v.jpg", i, j), gocv.IMReadGrayScale) + imgs[i][j] = gocv.IMRead(fmt.Sprintf("images/t-%v/000%v.jpg", i, j), gocv.IMReadColor) } } @@ -94,6 +94,13 @@ func TestImages(t *testing.T) { t.Fatalf("could not create turbidity sensor: %v", err) } + for i := 0; i < ts.H.Rows(); i++ { + for j := 0; j < ts.H.Cols(); j++ { + fmt.Printf(" %v,\t", ts.H.GetDoubleAt(i, j)) + } + fmt.Println() + } + results, err := NewResults(nImages) if err != nil { t.Fatalf("could not create results: %v", err) @@ -111,7 +118,7 @@ func TestImages(t *testing.T) { results.Update(stat.Mean(sample_result.Sharpness, nil), stat.Mean(sample_result.Contrast, nil), float64(i)*increment, i) } - err = plotResults(results.Turbidity, results.Sharpness, results.Contrast) + err = plotResults(results.Turbidity, normalize(results.Sharpness), normalize(results.Contrast)) if err != nil { t.Fatalf("plotting Failed: %v", err) } From 175cfc4925079fb6812cf174ea29689e2518c026 Mon Sep 17 00:00:00 2001 From: Russell Stanley Date: Mon, 11 Apr 2022 13:15:07 +0930 Subject: [PATCH 02/10] revid/config: add transform matrix test case --- cmd/rv/probe_circleci.go | 2 +- revid/config/config_test.go | 2 ++ revid/config/variables.go | 16 +++++++++++----- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/cmd/rv/probe_circleci.go b/cmd/rv/probe_circleci.go index 3c82038e..060424d1 100644 --- a/cmd/rv/probe_circleci.go +++ b/cmd/rv/probe_circleci.go @@ -40,7 +40,7 @@ type turbidityProbe struct { } // NewTurbidityProbe returns an empty turbidity probe for CircleCI testing only. -func NewTurbidityProbe(log logger.Logger, delay time.Duration) (*turbidityProbe, error) { +func NewTurbidityProbe(log logger.Logger, delay time.Duration, transformMatrix []float64) (*turbidityProbe, error) { tp := new(turbidityProbe) return tp, nil } diff --git a/revid/config/config_test.go b/revid/config/config_test.go index 4983d2bd..d2a25a2f 100644 --- a/revid/config/config_test.go +++ b/revid/config/config_test.go @@ -122,6 +122,7 @@ func TestUpdate(t *testing.T) { "VBRQuality": "excellent", "VerticalFlip": "true", "Width": "300", + "TransformMatrix": "0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8,0.9", } dl := &dumbLogger{} @@ -172,6 +173,7 @@ func TestUpdate(t *testing.T) { VBRQuality: QualityExcellent, VerticalFlip: true, Width: 300, + TransformMatrix: []float64{0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9}, } got := Config{Logger: dl} diff --git a/revid/config/variables.go b/revid/config/variables.go index 8098eb68..fc52dd5a 100644 --- a/revid/config/variables.go +++ b/revid/config/variables.go @@ -149,13 +149,19 @@ var Variables = []struct { Name: KeyTransformMatrix, Type: typeString, Update: func(c *Config, v string) { - v = strings.Replace(v, " ", "", 0) + c.Logger.Log(logger.Debug, "updating transform matrix", "string", v) + v = strings.Replace(v, " ", "", -1) + vals := make([]float64, 0) + if v == "" { + c.TransformMatrix = vals + return + } + elements := strings.Split(v, ",") - vals := make([]float64, len(elements)) - for _, i := range elements { - vFloat, err := strconv.ParseFloat(i, 64) + for _, e := range elements { + vFloat, err := strconv.ParseFloat(e, 64) if err != nil { - c.Logger.Log(logger.Warning, "invalid TransformMatrix param", "value", i) + c.Logger.Log(logger.Warning, "invalid TransformMatrix param", "value", e) } vals = append(vals, vFloat) } From 798e691c06aba2c9b4d049dde5c183184364be88 Mon Sep 17 00:00:00 2001 From: Russell Stanley Date: Mon, 11 Apr 2022 16:26:22 +0930 Subject: [PATCH 03/10] cmd/rv/main.go: add probe update after revid variable update --- cmd/rv/main.go | 17 ++++++++++------- cmd/rv/probe.go | 1 - 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cmd/rv/main.go b/cmd/rv/main.go index c430faed..50869bbe 100644 --- a/cmd/rv/main.go +++ b/cmd/rv/main.go @@ -146,11 +146,6 @@ func main() { p *turbidityProbe ) - p, err := NewTurbidityProbe(*log, 60*time.Second, rv.Config().TransformMatrix) - if err != nil { - log.Log(logger.Fatal, "could not create new turbidity probe", "error", err.Error()) - } - log.Log(logger.Debug, "initialising netsender client") ns, err := netsender.New(log, nil, readPin(p, rv, log), nil, createVarMap()) if err != nil { @@ -163,6 +158,11 @@ func main() { log.Log(logger.Fatal, pkg+"could not initialise revid", "error", err.Error()) } + p, err = NewTurbidityProbe(*log, 60*time.Second, rv.Config().TransformMatrix) + if err != nil { + log.Log(logger.Fatal, "could not create new turbidity probe", "error", err.Error()) + } + err = rv.SetProbe(p) if err != nil { log.Log(logger.Error, pkg+"could not set probe", "error", err.Error()) @@ -175,12 +175,12 @@ func main() { time.Sleep(runPreDelay) log.Log(logger.Debug, "beginning main loop") - run(rv, ns, log, netLog) + run(rv, ns, log, netLog, p) } // run starts the main loop. This will run netsender on every pass of the loop // (sleeping inbetween), check vars, and if changed, update revid as appropriate. -func run(rv *revid.Revid, ns *netsender.Sender, l *logger.Logger, nl *netlogger.Logger) { +func run(rv *revid.Revid, ns *netsender.Sender, l *logger.Logger, nl *netlogger.Logger, p *turbidityProbe) { var vs int for { l.Log(logger.Debug, "running netsender") @@ -225,6 +225,9 @@ func run(rv *revid.Revid, ns *netsender.Sender, l *logger.Logger, nl *netlogger. } l.Log(logger.Info, "revid successfully reconfigured") + // Update transform matrix based on new revid variables. + p.transform = rv.Config().TransformMatrix + l.Log(logger.Debug, "checking mode") switch ns.Mode() { case modePaused: diff --git a/cmd/rv/probe.go b/cmd/rv/probe.go index 5e1ca408..8f697c80 100644 --- a/cmd/rv/probe.go +++ b/cmd/rv/probe.go @@ -127,7 +127,6 @@ func (tp *turbidityProbe) Write(p []byte) (int, error) { select { case <-tp.ticker.C: tp.log.Log(logger.Debug, "beginning turbidity calculation") - tp.log.Log(logger.Debug, "transformation matrix", tp.transform[0]) startTime := time.Now() err := tp.turbidityCalculation() if err != nil { From f6505488bbb4cf39cdc755dce3902a720c43b42c Mon Sep 17 00:00:00 2001 From: Russell Stanley Date: Tue, 12 Apr 2022 13:32:34 +0930 Subject: [PATCH 04/10] turbidity: incorporate transform matrix variable into turbidity sensor --- cmd/rv/main.go | 2 +- cmd/rv/probe.go | 44 ++++++++++++++++++-- cmd/rv/probe_circleci.go | 2 +- cmd/rv/probe_test.go | 28 ++++++++++++- turbidity/transform.go | 80 +++++++++++++++++++++++++++++++++++++ turbidity/turbidity.go | 42 ++++--------------- turbidity/turbidity_test.go | 14 +++---- 7 files changed, 161 insertions(+), 51 deletions(-) create mode 100644 turbidity/transform.go diff --git a/cmd/rv/main.go b/cmd/rv/main.go index 50869bbe..242889f1 100644 --- a/cmd/rv/main.go +++ b/cmd/rv/main.go @@ -158,7 +158,7 @@ func main() { log.Log(logger.Fatal, pkg+"could not initialise revid", "error", err.Error()) } - p, err = NewTurbidityProbe(*log, 60*time.Second, rv.Config().TransformMatrix) + p, err = NewTurbidityProbe(*log, 60*time.Second) if err != nil { log.Log(logger.Fatal, "could not create new turbidity probe", "error", err.Error()) } diff --git a/cmd/rv/probe.go b/cmd/rv/probe.go index 8f697c80..d2cb42fb 100644 --- a/cmd/rv/probe.go +++ b/cmd/rv/probe.go @@ -40,6 +40,7 @@ import ( "gonum.org/v1/gonum/stat" "bitbucket.org/ausocean/av/codec/h264" + "bitbucket.org/ausocean/av/revid/config" "bitbucket.org/ausocean/av/turbidity" "bitbucket.org/ausocean/utils/logger" ) @@ -73,18 +74,22 @@ type turbidityProbe struct { } // NewTurbidityProbe returns a new turbidity probe. -func NewTurbidityProbe(log logger.Logger, delay time.Duration, transformMatrix []float64) (*turbidityProbe, error) { +func NewTurbidityProbe(log logger.Logger, delay time.Duration) (*turbidityProbe, error) { tp := new(turbidityProbe) tp.log = log tp.delay = delay tp.ticker = *time.NewTicker(delay) tp.buffer = bytes.NewBuffer(*new([]byte)) - tp.transform = transformMatrix + + tp.transform = []float64{0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0} + H, err := floatToMat(tp.transform) + if err != nil { + return nil, fmt.Errorf("failed to convert float slice to mat: %w", err) + } // Create the turbidity sensor. - standard := gocv.IMRead("../../turbidity/images/default.jpg", gocv.IMReadGrayScale) template := gocv.IMRead("../../turbidity/images/template.jpg", gocv.IMReadGrayScale) - ts, err := turbidity.NewTurbiditySensor(template, standard, k1, k2, filterSize, scale, alpha, log) + ts, err := turbidity.NewTurbiditySensor(template, H, k1, k2, filterSize, scale, alpha, log) if err != nil { return nil, fmt.Errorf("failed to create turbidity sensor: %w", err) } @@ -143,6 +148,27 @@ func (tp *turbidityProbe) Close() error { return nil } +func (tp *turbidityProbe) Update(config config.Config) error { + if len(config.TransformMatrix) != 9 { + return errors.New("transformation matrix has incorrect size") + } + for i := range tp.transform { + if tp.transform[i] != config.TransformMatrix[i] { + // Update the turbidity sensor with new transformation + tp.log.Log(logger.Debug, "updating the turbidity sensor with new transformation") + tp.transform = config.TransformMatrix + newTransform, err := floatToMat(tp.transform) + if err != nil { + return fmt.Errorf("failed to convert float slice to mat: %w", err) + } + tp.ts.H = newTransform + return nil + } + } + tp.log.Log(logger.Debug, "not update requried") + return nil +} + func (tp *turbidityProbe) turbidityCalculation() error { var imgs []gocv.Mat img := gocv.NewMat() @@ -210,3 +236,13 @@ func cleanUp(file string, vc *gocv.VideoCapture) error { } return nil } + +func floatToMat(array []float64) (gocv.Mat, error) { + H := gocv.NewMatWithSize(3, 3, gocv.MatTypeCV64F) + for i := 0; i < H.Rows(); i++ { + for j := 0; j < H.Cols(); j++ { + H.SetDoubleAt(i, j, array[i*H.Cols()+j]) + } + } + return H, nil +} diff --git a/cmd/rv/probe_circleci.go b/cmd/rv/probe_circleci.go index 060424d1..3c82038e 100644 --- a/cmd/rv/probe_circleci.go +++ b/cmd/rv/probe_circleci.go @@ -40,7 +40,7 @@ type turbidityProbe struct { } // NewTurbidityProbe returns an empty turbidity probe for CircleCI testing only. -func NewTurbidityProbe(log logger.Logger, delay time.Duration, transformMatrix []float64) (*turbidityProbe, error) { +func NewTurbidityProbe(log logger.Logger, delay time.Duration) (*turbidityProbe, error) { tp := new(turbidityProbe) return tp, nil } diff --git a/cmd/rv/probe_test.go b/cmd/rv/probe_test.go index f04d6b10..3b5327a1 100644 --- a/cmd/rv/probe_test.go +++ b/cmd/rv/probe_test.go @@ -30,6 +30,7 @@ import ( "testing" "time" + "bitbucket.org/ausocean/av/revid/config" "bitbucket.org/ausocean/utils/logger" "gopkg.in/natefinch/lumberjack.v2" ) @@ -44,12 +45,21 @@ func TestProbe(t *testing.T) { MaxAge: logMaxAge, } log := logger.New(logVerbosity, io.MultiWriter(fileLog), logSuppress) - transformMatrix := []float64{0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0} + updatedMatrix := []float64{-0.2731048063, -0.0020501869, 661.0275911942, 0.0014327789, -0.2699443748, 339.3921028016, 0.0000838317, 0.0000476486, 1.0} + config := config.Config{TransformMatrix: updatedMatrix} - ts, err := NewTurbidityProbe(*log, time.Microsecond, transformMatrix) + ts, err := NewTurbidityProbe(*log, time.Microsecond) if err != nil { t.Fatalf("failed to create turbidity probe") } + + t.Log(ts.transform) + err = ts.Update(config) + if err != nil { + t.Fatalf("could not update probe: %v", err) + } + t.Log(ts.transform) + video, err := ioutil.ReadFile("logo.h264") if err != nil { t.Fatalf("failed to read file: %v", err) @@ -61,3 +71,17 @@ func TestProbe(t *testing.T) { } t.Logf("contrast: %v, sharpness: %v\n", ts.contrast, ts.sharpness) } + +/* +func printMat(mat gocv.Mat) { + for i := 0; i < mat.Rows(); i++ { + for j := 0; j < mat.Cols(); j++ { + fmt.Printf(" %.10f", mat.GetDoubleAt(i, j)) + if i < 2 || j < 2 { + fmt.Print(",") + } + } + } + fmt.Println() +} +*/ diff --git a/turbidity/transform.go b/turbidity/transform.go new file mode 100644 index 00000000..d855ba19 --- /dev/null +++ b/turbidity/transform.go @@ -0,0 +1,80 @@ +//go:build !nocv +// +build !nocv + +/* +DESCRIPTION + +AUTHORS + Russell Stanley + +LICENSE + Copyright (C) 2021-2022 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ + +package turbidity + +import ( + "errors" + "fmt" + "image" + + "gocv.io/x/gocv" +) + +// Homography constants +const ( + ransacThreshold = 3.0 // Maximum allowed reprojection error to treat a point pair as an inlier. + maxIter = 2000 // The maximum number of RANSAC iterations. + confidence = 0.995 // Confidence level, between 0 and 1. +) + +func FindTransform(standardPath, templatePath string) (gocv.Mat, error) { + mask := gocv.NewMat() + standard := gocv.IMRead(standardPath, gocv.IMReadColor) + standardCorners := gocv.NewMat() + + template := gocv.IMRead(templatePath, gocv.IMReadGrayScale) + templateCorners := gocv.NewMat() + + // Validate template image is not empty and has valid corners. + if template.Empty() { + return gocv.NewMat(), errors.New("template image is empty") + } + if !gocv.FindChessboardCorners(template, image.Pt(3, 3), &templateCorners, gocv.CalibCBNormalizeImage) { + return gocv.NewMat(), errors.New("could not find corners in template image") + } + + // Validate standard image is not empty and has valid corners. + if standard.Empty() { + return gocv.NewMat(), errors.New("standard image is empty") + } + if !gocv.FindChessboardCorners(standard, image.Pt(3, 3), &standardCorners, gocv.CalibCBNormalizeImage) { + return gocv.NewMat(), errors.New("could not find corners in standard image") + } + + H := gocv.FindHomography(standardCorners, &templateCorners, gocv.HomograpyMethodRANSAC, ransacThreshold, &mask, maxIter, confidence) + fmt.Print(H.Type()) + for i := 0; i < H.Rows(); i++ { + for j := 0; j < H.Cols(); j++ { + fmt.Printf(" %.10f", H.GetDoubleAt(i, j)) + if i < 2 || j < 2 { + fmt.Print(",") + } + } + } + fmt.Println() + return H, nil +} diff --git a/turbidity/turbidity.go b/turbidity/turbidity.go index cc8578c2..dd5bec8a 100644 --- a/turbidity/turbidity.go +++ b/turbidity/turbidity.go @@ -41,53 +41,27 @@ import ( "gocv.io/x/gocv" ) -// Homography constants -const ( - ransacThreshold = 3.0 // Maximum allowed reprojection error to treat a point pair as an inlier. - maxIter = 2000 // The maximum number of RANSAC iterations. - confidence = 0.995 // Confidence level, between 0 and 1. -) - // TurbiditySensor is a software based turbidity sensor that uses CV to determine sharpness and constrast level // of a chessboard-like target submerged in water that can be correlated to turbidity/visibility values. type TurbiditySensor struct { - template, templateCorners gocv.Mat - standard, standardCorners gocv.Mat - H gocv.Mat - k1, k2, sobelFilterSize int - scale, alpha float64 - log logger.Logger + template gocv.Mat + H gocv.Mat + k1, k2, sobelFilterSize int + scale, alpha float64 + log logger.Logger } // NewTurbiditySensor returns a new TurbiditySensor. -func NewTurbiditySensor(template, standard gocv.Mat, k1, k2, sobelFilterSize int, scale, alpha float64, log logger.Logger) (*TurbiditySensor, error) { +func NewTurbiditySensor(template, H gocv.Mat, k1, k2, sobelFilterSize int, scale, alpha float64, log logger.Logger) (*TurbiditySensor, error) { ts := new(TurbiditySensor) - templateCorners := gocv.NewMat() - standardCorners := gocv.NewMat() - mask := gocv.NewMat() // Validate template image is not empty and has valid corners. if template.Empty() { return nil, errors.New("template image is empty") } - if !gocv.FindChessboardCorners(template, image.Pt(3, 3), &templateCorners, gocv.CalibCBNormalizeImage) { - return nil, errors.New("could not find corners in template image") - } + ts.template = template - ts.templateCorners = templateCorners - - // Validate standard image is not empty and has valid corners. - if standard.Empty() { - return nil, errors.New("standard image is empty") - } - if !gocv.FindChessboardCorners(standard, image.Pt(3, 3), &standardCorners, gocv.CalibCBNormalizeImage) { - return nil, errors.New("could not find corners in standard image") - } - - ts.standard = standard - ts.standardCorners = standardCorners - ts.H = gocv.FindHomography(ts.standardCorners, &ts.templateCorners, gocv.HomograpyMethodRANSAC, ransacThreshold, &mask, maxIter, confidence) - + ts.H = H ts.k1, ts.k2, ts.sobelFilterSize = k1, k2, sobelFilterSize ts.alpha, ts.scale = alpha, scale ts.log = log diff --git a/turbidity/turbidity_test.go b/turbidity/turbidity_test.go index 72f4e5bf..efe8f240 100644 --- a/turbidity/turbidity_test.go +++ b/turbidity/turbidity_test.go @@ -77,7 +77,10 @@ func TestImages(t *testing.T) { log := *logger.New(logVerbosity, io.MultiWriter(fileLog), logSuppress) template := gocv.IMRead("images/template.jpg", gocv.IMReadGrayScale) - standard := gocv.IMRead("images/default.jpg", gocv.IMReadGrayScale) + H, err := FindTransform("images/default.jpg", "images/template.jpg") + if err != nil { + t.Fatalf("could not find transformation: %v", err) + } imgs := make([][]gocv.Mat, nImages) @@ -89,18 +92,11 @@ func TestImages(t *testing.T) { } } - ts, err := NewTurbiditySensor(template, standard, k1, k2, filterSize, scale, alpha, log) + ts, err := NewTurbiditySensor(template, H, k1, k2, filterSize, scale, alpha, log) if err != nil { t.Fatalf("could not create turbidity sensor: %v", err) } - for i := 0; i < ts.H.Rows(); i++ { - for j := 0; j < ts.H.Cols(); j++ { - fmt.Printf(" %v,\t", ts.H.GetDoubleAt(i, j)) - } - fmt.Println() - } - results, err := NewResults(nImages) if err != nil { t.Fatalf("could not create results: %v", err) From 6dd32ea7861d50654e7de4df553d686035fc2bce Mon Sep 17 00:00:00 2001 From: Russell Stanley Date: Tue, 12 Apr 2022 13:51:17 +0930 Subject: [PATCH 05/10] cmd/rv: fix build issues --- cmd/rv/main.go | 2 +- cmd/rv/probe_circleci.go | 5 +++++ turbidity/transform.go | 1 - 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/rv/main.go b/cmd/rv/main.go index 242889f1..6becfae3 100644 --- a/cmd/rv/main.go +++ b/cmd/rv/main.go @@ -226,7 +226,7 @@ func run(rv *revid.Revid, ns *netsender.Sender, l *logger.Logger, nl *netlogger. l.Log(logger.Info, "revid successfully reconfigured") // Update transform matrix based on new revid variables. - p.transform = rv.Config().TransformMatrix + p.Update(rv.Config()) l.Log(logger.Debug, "checking mode") switch ns.Mode() { diff --git a/cmd/rv/probe_circleci.go b/cmd/rv/probe_circleci.go index 3c82038e..7a3d0cb4 100644 --- a/cmd/rv/probe_circleci.go +++ b/cmd/rv/probe_circleci.go @@ -32,6 +32,7 @@ package main import ( "time" + "bitbucket.org/ausocean/av/revid/config" "bitbucket.org/ausocean/utils/logger" ) @@ -50,6 +51,10 @@ func (tp *turbidityProbe) Write(p []byte) (int, error) { return 0, nil } +func (tp *turbidityProbe) Update(config config.Config) error { + return nil +} + func (tp *turbidityProbe) Close() error { return nil } diff --git a/turbidity/transform.go b/turbidity/transform.go index d855ba19..9fada5b7 100644 --- a/turbidity/transform.go +++ b/turbidity/transform.go @@ -66,7 +66,6 @@ func FindTransform(standardPath, templatePath string) (gocv.Mat, error) { } H := gocv.FindHomography(standardCorners, &templateCorners, gocv.HomograpyMethodRANSAC, ransacThreshold, &mask, maxIter, confidence) - fmt.Print(H.Type()) for i := 0; i < H.Rows(); i++ { for j := 0; j < H.Cols(); j++ { fmt.Printf(" %.10f", H.GetDoubleAt(i, j)) From c58643e20774887ec7401bfced25367818adeed8 Mon Sep 17 00:00:00 2001 From: Russell Stanley Date: Thu, 21 Apr 2022 09:12:29 +0930 Subject: [PATCH 06/10] cmd/rv/main.go fix probe initialization --- cmd/rv/main.go | 10 +++++----- cmd/rv/probe.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/rv/main.go b/cmd/rv/main.go index 6becfae3..60bab7c9 100644 --- a/cmd/rv/main.go +++ b/cmd/rv/main.go @@ -146,6 +146,11 @@ func main() { p *turbidityProbe ) + p, err := NewTurbidityProbe(*log, 60*time.Second) + if err != nil { + log.Log(logger.Fatal, "could not create new turbidity probe", "error", err.Error()) + } + log.Log(logger.Debug, "initialising netsender client") ns, err := netsender.New(log, nil, readPin(p, rv, log), nil, createVarMap()) if err != nil { @@ -158,11 +163,6 @@ func main() { log.Log(logger.Fatal, pkg+"could not initialise revid", "error", err.Error()) } - p, err = NewTurbidityProbe(*log, 60*time.Second) - if err != nil { - log.Log(logger.Fatal, "could not create new turbidity probe", "error", err.Error()) - } - err = rv.SetProbe(p) if err != nil { log.Log(logger.Error, pkg+"could not set probe", "error", err.Error()) diff --git a/cmd/rv/probe.go b/cmd/rv/probe.go index d2cb42fb..d42d8fb0 100644 --- a/cmd/rv/probe.go +++ b/cmd/rv/probe.go @@ -165,7 +165,7 @@ func (tp *turbidityProbe) Update(config config.Config) error { return nil } } - tp.log.Log(logger.Debug, "not update requried") + tp.log.Log(logger.Debug, "no update requried") return nil } From 83317e9bbce560d3bc0427bec5d25e2e60686873 Mon Sep 17 00:00:00 2001 From: Russell Stanley Date: Fri, 22 Apr 2022 15:09:23 +0930 Subject: [PATCH 07/10] turbidity: added comments and improved debug logs --- cmd/rv/probe.go | 20 ++++++++------------ turbidity/transform.go | 1 + 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/cmd/rv/probe.go b/cmd/rv/probe.go index d42d8fb0..2585ec5d 100644 --- a/cmd/rv/probe.go +++ b/cmd/rv/probe.go @@ -82,10 +82,7 @@ func NewTurbidityProbe(log logger.Logger, delay time.Duration) (*turbidityProbe, tp.buffer = bytes.NewBuffer(*new([]byte)) tp.transform = []float64{0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0} - H, err := floatToMat(tp.transform) - if err != nil { - return nil, fmt.Errorf("failed to convert float slice to mat: %w", err) - } + H := floatToMat(tp.transform) // Create the turbidity sensor. template := gocv.IMRead("../../turbidity/images/template.jpg", gocv.IMReadGrayScale) @@ -148,6 +145,7 @@ func (tp *turbidityProbe) Close() error { return nil } +// Update, will update the probe and turbidity sensor with the new transformation matrix if it has been changed. func (tp *turbidityProbe) Update(config config.Config) error { if len(config.TransformMatrix) != 9 { return errors.New("transformation matrix has incorrect size") @@ -155,17 +153,14 @@ func (tp *turbidityProbe) Update(config config.Config) error { for i := range tp.transform { if tp.transform[i] != config.TransformMatrix[i] { // Update the turbidity sensor with new transformation - tp.log.Log(logger.Debug, "updating the turbidity sensor with new transformation") + tp.log.Log(logger.Debug, "updating the transformation matrix") tp.transform = config.TransformMatrix - newTransform, err := floatToMat(tp.transform) - if err != nil { - return fmt.Errorf("failed to convert float slice to mat: %w", err) - } + newTransform := floatToMat(tp.transform) tp.ts.H = newTransform return nil } } - tp.log.Log(logger.Debug, "no update requried") + tp.log.Log(logger.Debug, "no change to the transformation matrix") return nil } @@ -237,12 +232,13 @@ func cleanUp(file string, vc *gocv.VideoCapture) error { return nil } -func floatToMat(array []float64) (gocv.Mat, error) { +// floatToMat will convert a slice of 9 floats to a gocv.Mat. +func floatToMat(array []float64) gocv.Mat { H := gocv.NewMatWithSize(3, 3, gocv.MatTypeCV64F) for i := 0; i < H.Rows(); i++ { for j := 0; j < H.Cols(); j++ { H.SetDoubleAt(i, j, array[i*H.Cols()+j]) } } - return H, nil + return H } diff --git a/turbidity/transform.go b/turbidity/transform.go index 9fada5b7..21e01101 100644 --- a/turbidity/transform.go +++ b/turbidity/transform.go @@ -3,6 +3,7 @@ /* DESCRIPTION + Provides a function which can extract the transformation matrix. AUTHORS Russell Stanley From fe0c9ffcdbbd05f8d74874f282d13f95905b3ab4 Mon Sep 17 00:00:00 2001 From: Russell Stanley Date: Fri, 22 Apr 2022 15:23:25 +0930 Subject: [PATCH 08/10] cmd/rv/probe_test.go fix test with nocv build --- cmd/rv/probe_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/rv/probe_test.go b/cmd/rv/probe_test.go index 3b5327a1..bb3faef7 100644 --- a/cmd/rv/probe_test.go +++ b/cmd/rv/probe_test.go @@ -53,12 +53,10 @@ func TestProbe(t *testing.T) { t.Fatalf("failed to create turbidity probe") } - t.Log(ts.transform) err = ts.Update(config) if err != nil { t.Fatalf("could not update probe: %v", err) } - t.Log(ts.transform) video, err := ioutil.ReadFile("logo.h264") if err != nil { From 445ab1d78537f88e9a44ed5d0d29d94086b87bfe Mon Sep 17 00:00:00 2001 From: Russell Stanley Date: Thu, 28 Apr 2022 13:43:49 +0930 Subject: [PATCH 09/10] turbidity: updated naming convention and comments per comments on PR --- cmd/rv/main.go | 5 ++++- cmd/rv/probe.go | 39 +++++++++++++++++++------------------ cmd/rv/probe_circleci.go | 3 +-- cmd/rv/probe_test.go | 18 +---------------- revid/config/config.go | 4 +++- turbidity/transform.go | 35 +++++++++++++-------------------- turbidity/turbidity.go | 10 +++++----- turbidity/turbidity_test.go | 21 +++++++++++++++++--- 8 files changed, 66 insertions(+), 69 deletions(-) diff --git a/cmd/rv/main.go b/cmd/rv/main.go index 60bab7c9..fa8b7c02 100644 --- a/cmd/rv/main.go +++ b/cmd/rv/main.go @@ -226,7 +226,10 @@ func run(rv *revid.Revid, ns *netsender.Sender, l *logger.Logger, nl *netlogger. l.Log(logger.Info, "revid successfully reconfigured") // Update transform matrix based on new revid variables. - p.Update(rv.Config()) + err = p.Update(rv.Config().TransformMatrix) + if err != nil { + l.Log(logger.Error, "could not update turbidity probe", "error", err.Error()) + } l.Log(logger.Debug, "checking mode") switch ns.Mode() { diff --git a/cmd/rv/probe.go b/cmd/rv/probe.go index 2585ec5d..7d5525a0 100644 --- a/cmd/rv/probe.go +++ b/cmd/rv/probe.go @@ -40,7 +40,6 @@ import ( "gonum.org/v1/gonum/stat" "bitbucket.org/ausocean/av/codec/h264" - "bitbucket.org/ausocean/av/revid/config" "bitbucket.org/ausocean/av/turbidity" "bitbucket.org/ausocean/utils/logger" ) @@ -50,6 +49,7 @@ const ( maxImages = 1 // Max number of images read when evaluating turbidity. bufferLimit = 20000 // 20KB trimTolerance = 200 // Number of times trim can be called where no keyframe is found. + transformSize = 9 // Size of the square projective matrix. ) // Turbidity sensor constants. @@ -81,12 +81,12 @@ func NewTurbidityProbe(log logger.Logger, delay time.Duration) (*turbidityProbe, tp.ticker = *time.NewTicker(delay) tp.buffer = bytes.NewBuffer(*new([]byte)) - tp.transform = []float64{0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0} - H := floatToMat(tp.transform) + tp.transform = make([]float64, transformSize) + transformMatrix := floatToMat(tp.transform) // Create the turbidity sensor. template := gocv.IMRead("../../turbidity/images/template.jpg", gocv.IMReadGrayScale) - ts, err := turbidity.NewTurbiditySensor(template, H, k1, k2, filterSize, scale, alpha, log) + ts, err := turbidity.NewTurbiditySensor(template, transformMatrix, k1, k2, filterSize, scale, alpha, log) if err != nil { return nil, fmt.Errorf("failed to create turbidity sensor: %w", err) } @@ -145,20 +145,21 @@ func (tp *turbidityProbe) Close() error { return nil } -// Update, will update the probe and turbidity sensor with the new transformation matrix if it has been changed. -func (tp *turbidityProbe) Update(config config.Config) error { - if len(config.TransformMatrix) != 9 { +// Update will update the probe and turbidity sensor with the new transformation matrix if it has been changed. +func (tp *turbidityProbe) Update(transformMatrix []float64) error { + if len(transformMatrix) != transformSize { return errors.New("transformation matrix has incorrect size") } for i := range tp.transform { - if tp.transform[i] != config.TransformMatrix[i] { - // Update the turbidity sensor with new transformation - tp.log.Log(logger.Debug, "updating the transformation matrix") - tp.transform = config.TransformMatrix - newTransform := floatToMat(tp.transform) - tp.ts.H = newTransform - return nil + if tp.transform[i] == transformMatrix[i] { + continue } + // Update the turbidity sensor with new transformation. + tp.log.Log(logger.Debug, "updating the transformation matrix") + tp.transform = transformMatrix + newTransform := floatToMat(tp.transform) + tp.ts.TransformMatrix = newTransform + return nil } tp.log.Log(logger.Debug, "no change to the transformation matrix") return nil @@ -234,11 +235,11 @@ func cleanUp(file string, vc *gocv.VideoCapture) error { // floatToMat will convert a slice of 9 floats to a gocv.Mat. func floatToMat(array []float64) gocv.Mat { - H := gocv.NewMatWithSize(3, 3, gocv.MatTypeCV64F) - for i := 0; i < H.Rows(); i++ { - for j := 0; j < H.Cols(); j++ { - H.SetDoubleAt(i, j, array[i*H.Cols()+j]) + mat := gocv.NewMatWithSize(3, 3, gocv.MatTypeCV64F) + for i := 0; i < mat.Rows(); i++ { + for j := 0; j < mat.Cols(); j++ { + mat.SetDoubleAt(i, j, array[i*mat.Cols()+j]) } } - return H + return mat } diff --git a/cmd/rv/probe_circleci.go b/cmd/rv/probe_circleci.go index 7a3d0cb4..1b69e15a 100644 --- a/cmd/rv/probe_circleci.go +++ b/cmd/rv/probe_circleci.go @@ -32,7 +32,6 @@ package main import ( "time" - "bitbucket.org/ausocean/av/revid/config" "bitbucket.org/ausocean/utils/logger" ) @@ -51,7 +50,7 @@ func (tp *turbidityProbe) Write(p []byte) (int, error) { return 0, nil } -func (tp *turbidityProbe) Update(config config.Config) error { +func (tp *turbidityProbe) Update(mat []float64) error { return nil } diff --git a/cmd/rv/probe_test.go b/cmd/rv/probe_test.go index bb3faef7..a87c70ef 100644 --- a/cmd/rv/probe_test.go +++ b/cmd/rv/probe_test.go @@ -30,7 +30,6 @@ import ( "testing" "time" - "bitbucket.org/ausocean/av/revid/config" "bitbucket.org/ausocean/utils/logger" "gopkg.in/natefinch/lumberjack.v2" ) @@ -46,14 +45,13 @@ func TestProbe(t *testing.T) { } log := logger.New(logVerbosity, io.MultiWriter(fileLog), logSuppress) updatedMatrix := []float64{-0.2731048063, -0.0020501869, 661.0275911942, 0.0014327789, -0.2699443748, 339.3921028016, 0.0000838317, 0.0000476486, 1.0} - config := config.Config{TransformMatrix: updatedMatrix} ts, err := NewTurbidityProbe(*log, time.Microsecond) if err != nil { t.Fatalf("failed to create turbidity probe") } - err = ts.Update(config) + err = ts.Update(updatedMatrix) if err != nil { t.Fatalf("could not update probe: %v", err) } @@ -69,17 +67,3 @@ func TestProbe(t *testing.T) { } t.Logf("contrast: %v, sharpness: %v\n", ts.contrast, ts.sharpness) } - -/* -func printMat(mat gocv.Mat) { - for i := 0; i < mat.Rows(); i++ { - for j := 0; j < mat.Cols(); j++ { - fmt.Printf(" %.10f", mat.GetDoubleAt(i, j)) - if i < 2 || j < 2 { - fmt.Print(",") - } - } - } - fmt.Println() -} -*/ diff --git a/revid/config/config.go b/revid/config/config.go index 2e665f72..23106570 100644 --- a/revid/config/config.go +++ b/revid/config/config.go @@ -274,7 +274,9 @@ type Config struct { VerticalFlip bool // VerticalFlip flips video vertically for Raspivid input. Width uint // Width defines the input video width Raspivid input. - TransformMatrix []float64 // Describes the transformation matrix to extract the target. + // TransformMatrix describes the projective transformation matrix to extract a target from the + // video data for turbidty calculations. + TransformMatrix []float64 } // Validate checks for any errors in the config fields and defaults settings diff --git a/turbidity/transform.go b/turbidity/transform.go index 21e01101..3eff1efd 100644 --- a/turbidity/transform.go +++ b/turbidity/transform.go @@ -29,52 +29,45 @@ package turbidity import ( "errors" - "fmt" "image" "gocv.io/x/gocv" ) -// Homography constants +// Perspective transformation constants. const ( ransacThreshold = 3.0 // Maximum allowed reprojection error to treat a point pair as an inlier. maxIter = 2000 // The maximum number of RANSAC iterations. confidence = 0.995 // Confidence level, between 0 and 1. ) +// FindTransform, given a template and standard image the perspetive transformation matrix will be determined. +// the matrix will be returned and logged for use in vidgrind. func FindTransform(standardPath, templatePath string) (gocv.Mat, error) { mask := gocv.NewMat() - standard := gocv.IMRead(standardPath, gocv.IMReadColor) - standardCorners := gocv.NewMat() + std := gocv.IMRead(standardPath, gocv.IMReadColor) + stdCorners := gocv.NewMat() template := gocv.IMRead(templatePath, gocv.IMReadGrayScale) templateCorners := gocv.NewMat() + transformMatrix := gocv.NewMat() // Validate template image is not empty and has valid corners. if template.Empty() { - return gocv.NewMat(), errors.New("template image is empty") + return transformMatrix, errors.New("template image is empty") } if !gocv.FindChessboardCorners(template, image.Pt(3, 3), &templateCorners, gocv.CalibCBNormalizeImage) { - return gocv.NewMat(), errors.New("could not find corners in template image") + return transformMatrix, errors.New("could not find corners in template image") } // Validate standard image is not empty and has valid corners. - if standard.Empty() { - return gocv.NewMat(), errors.New("standard image is empty") + if std.Empty() { + return transformMatrix, errors.New("standard image is empty") } - if !gocv.FindChessboardCorners(standard, image.Pt(3, 3), &standardCorners, gocv.CalibCBNormalizeImage) { - return gocv.NewMat(), errors.New("could not find corners in standard image") + if !gocv.FindChessboardCorners(std, image.Pt(3, 3), &stdCorners, gocv.CalibCBNormalizeImage) { + return transformMatrix, errors.New("could not find corners in standard image") } - H := gocv.FindHomography(standardCorners, &templateCorners, gocv.HomograpyMethodRANSAC, ransacThreshold, &mask, maxIter, confidence) - for i := 0; i < H.Rows(); i++ { - for j := 0; j < H.Cols(); j++ { - fmt.Printf(" %.10f", H.GetDoubleAt(i, j)) - if i < 2 || j < 2 { - fmt.Print(",") - } - } - } - fmt.Println() - return H, nil + transformMatrix = gocv.FindHomography(stdCorners, &templateCorners, gocv.HomograpyMethodRANSAC, ransacThreshold, &mask, maxIter, confidence) + return transformMatrix, nil } diff --git a/turbidity/turbidity.go b/turbidity/turbidity.go index dd5bec8a..b3b75a58 100644 --- a/turbidity/turbidity.go +++ b/turbidity/turbidity.go @@ -44,15 +44,15 @@ import ( // TurbiditySensor is a software based turbidity sensor that uses CV to determine sharpness and constrast level // of a chessboard-like target submerged in water that can be correlated to turbidity/visibility values. type TurbiditySensor struct { - template gocv.Mat - H gocv.Mat + template gocv.Mat // Holds the image of the target. + TransformMatrix gocv.Mat // The current perspective transformation matrix to extract the target from the frame. k1, k2, sobelFilterSize int scale, alpha float64 log logger.Logger } // NewTurbiditySensor returns a new TurbiditySensor. -func NewTurbiditySensor(template, H gocv.Mat, k1, k2, sobelFilterSize int, scale, alpha float64, log logger.Logger) (*TurbiditySensor, error) { +func NewTurbiditySensor(template, transformMatrix gocv.Mat, k1, k2, sobelFilterSize int, scale, alpha float64, log logger.Logger) (*TurbiditySensor, error) { ts := new(TurbiditySensor) // Validate template image is not empty and has valid corners. @@ -61,7 +61,7 @@ func NewTurbiditySensor(template, H gocv.Mat, k1, k2, sobelFilterSize int, scale } ts.template = template - ts.H = H + ts.TransformMatrix = transformMatrix ts.k1, ts.k2, ts.sobelFilterSize = k1, k2, sobelFilterSize ts.alpha, ts.scale = alpha, scale ts.log = log @@ -184,7 +184,7 @@ func (ts TurbiditySensor) transform(img gocv.Mat) (gocv.Mat, error) { // if !gocv.FindChessboardCorners(img, image.Pt(3, 3), &imgCorners, gocv.CalibCBFastCheck) {} // Find and apply transformation. - gocv.WarpPerspective(img, &out, ts.H, image.Pt(ts.template.Rows(), ts.template.Cols())) + gocv.WarpPerspective(img, &out, ts.TransformMatrix, image.Pt(ts.template.Rows(), ts.template.Cols())) gocv.CvtColor(out, &out, gocv.ColorRGBToGray) return out, nil } diff --git a/turbidity/turbidity_test.go b/turbidity/turbidity_test.go index efe8f240..32ec1654 100644 --- a/turbidity/turbidity_test.go +++ b/turbidity/turbidity_test.go @@ -42,7 +42,7 @@ import ( ) const ( - nImages = 12 // Number of images to test. (Max 13) + nImages = 13 // Number of images to test. (Max 13) nSamples = 10 // Number of samples for each image. (Max 10) increment = 2.5 // Increment of the turbidity level. ) @@ -77,10 +77,11 @@ func TestImages(t *testing.T) { log := *logger.New(logVerbosity, io.MultiWriter(fileLog), logSuppress) template := gocv.IMRead("images/template.jpg", gocv.IMReadGrayScale) - H, err := FindTransform("images/default.jpg", "images/template.jpg") + transformMatrix, err := FindTransform("images/default.jpg", "images/template.jpg") if err != nil { t.Fatalf("could not find transformation: %v", err) } + t.Log(formatMat(transformMatrix)) imgs := make([][]gocv.Mat, nImages) @@ -92,7 +93,7 @@ func TestImages(t *testing.T) { } } - ts, err := NewTurbiditySensor(template, H, k1, k2, filterSize, scale, alpha, log) + ts, err := NewTurbiditySensor(template, transformMatrix, k1, k2, filterSize, scale, alpha, log) if err != nil { t.Fatalf("could not create turbidity sensor: %v", err) } @@ -141,3 +142,17 @@ func plotResults(x, sharpness, contrast []float64) error { } return nil } + +// formatMat creates a formatted transformation matrix string for use in vidgrind. +func formatMat(transformMatrix gocv.Mat) string { + var out string + for i := 0; i < transformMatrix.Rows(); i++ { + for j := 0; j < transformMatrix.Cols(); j++ { + out += fmt.Sprintf(" %.10f", transformMatrix.GetDoubleAt(i, j)) + if i < 2 || j < 2 { + out += "," + } + } + } + return out +} From 5202f5914e4d7acd1193d36e27a770f0a6ba9e95 Mon Sep 17 00:00:00 2001 From: Russell Stanley Date: Fri, 29 Apr 2022 13:53:52 +0930 Subject: [PATCH 10/10] cmd/rv/probe_circleci.go code cleanup --- cmd/rv/probe_circleci.go | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/cmd/rv/probe_circleci.go b/cmd/rv/probe_circleci.go index 1b69e15a..5eb733fa 100644 --- a/cmd/rv/probe_circleci.go +++ b/cmd/rv/probe_circleci.go @@ -46,14 +46,8 @@ func NewTurbidityProbe(log logger.Logger, delay time.Duration) (*turbidityProbe, } // Write performs no operation for CircleCI testing only. -func (tp *turbidityProbe) Write(p []byte) (int, error) { - return 0, nil -} +func (tp *turbidityProbe) Write(p []byte) (int, error) { return 0, nil } -func (tp *turbidityProbe) Update(mat []float64) error { - return nil -} +func (tp *turbidityProbe) Update(mat []float64) error { return nil } -func (tp *turbidityProbe) Close() error { - return nil -} +func (tp *turbidityProbe) Close() error { return nil }