From 57b3f53bf3b057bac411b57f91554f102a807046 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 12 Feb 2020 13:10:24 +1030 Subject: [PATCH] filter: time based filter for overcoming discontinuities When the filter has short discontinuities in detection, those frames are sent so that the video is more continuous. --- filter/{difference.go => diff.go} | 70 ++++++---------- filter/knn.go | 105 +++++++----------------- filter/mog.go | 106 +++++++----------------- filter/motion.go | 132 ++++++++++++++++++++++++++++++ revid/config/config.go | 2 + revid/revid.go | 7 ++ 6 files changed, 225 insertions(+), 197 deletions(-) rename filter/{difference.go => diff.go} (56%) create mode 100644 filter/motion.go diff --git a/filter/difference.go b/filter/diff.go similarity index 56% rename from filter/difference.go rename to filter/diff.go index 137a0b0d..d56524e2 100644 --- a/filter/difference.go +++ b/filter/diff.go @@ -11,7 +11,7 @@ AUTHORS Scott Barnard LICENSE - difference.go is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) + diff.go is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) It is free software: you can redistribute it and/or modify them under the terms of the GNU General Public License as published by the @@ -39,65 +39,54 @@ import ( const defaultDiffThreshold = 3 -// Diff is a filter that provides basic motion detection. Difference calculates -// the absolute difference for each pixel between two frames, then finds the mean. If -// the mean is above a given threshold, then it is considered motion. -type Diff struct { - debugging debugWindows - dst io.WriteCloser - thresh float64 - prev gocv.Mat -} +// NewDiff returns a pointer to a new difference motion filter. +func NewDiff(dst io.WriteCloser, c config.Config) *Motion { -// NewDiff returns a pointer to a new Diff struct. -func NewDiff(dst io.WriteCloser, c config.Config) *Diff { // Validate parameters. if c.MotionThreshold <= 0 { c.LogInvalidField("MotionThreshold", defaultDiffThreshold) c.MotionThreshold = defaultDiffThreshold } - return &Diff{ - dst: dst, + alg := &Diff{ thresh: c.MotionThreshold, prev: gocv.NewMat(), debugging: newWindows("DIFF"), } + + return NewMotion(dst, alg, c) } -// Implements io.Closer. -// Close frees resources used by gocv, because it has to be done manually, due to -// it using c-go. +// Diff is a motion detection algorithm. It calculates the absolute +// difference for each pixel between two frames, then finds the mean. +// If the mean is above a given threshold, it is considered motion. +type Diff struct { + debugging debugWindows + thresh float64 + prev gocv.Mat +} + +// Close frees resources used by gocv. It has to be done manually, +// due to gocv using c-go. func (d *Diff) Close() error { - d.prev.Close() d.debugging.close() + d.prev.Close() return nil } -// Implements io.Writer. -// Write applies the motion filter to the video stream. Only frames with motion -// are written to the destination encoder, frames without are discarded. -func (d *Diff) Write(f []byte) (int, error) { +// Detect performs the motion detection on a frame. It returns true +// if motion is detected. +func (d *Diff) Detect(img *gocv.Mat) bool { if d.prev.Empty() { - var err error - d.prev, err = gocv.IMDecode(f, gocv.IMReadColor) - if err != nil { - return 0, err - } - return len(f), nil - } - - img, err := gocv.IMDecode(f, gocv.IMReadColor) - defer img.Close() - if err != nil { - return 0, err + d.prev = img.Clone() + return false } imgDelta := gocv.NewMat() defer imgDelta.Close() // Seperate foreground and background. - gocv.AbsDiff(img, d.prev, &imgDelta) + gocv.AbsDiff(*img, d.prev, &imgDelta) gocv.CvtColor(imgDelta, &imgDelta, gocv.ColorBGRToGray) mean := imgDelta.Mean().Val1 @@ -106,13 +95,8 @@ func (d *Diff) Write(f []byte) (int, error) { d.prev = img.Clone() // Draw debug information. - d.debugging.show(img, imgDelta, mean > d.thresh, nil, fmt.Sprintf("Mean: %f", mean), fmt.Sprintf("Threshold: %f", d.thresh)) + d.debugging.show(*img, imgDelta, mean > d.thresh, nil, fmt.Sprintf("Mean: %f", mean), fmt.Sprintf("Threshold: %f", d.thresh)) - // Don't write to destination if there is no motion. - if mean < d.thresh { - return len(f), nil - } - - // Write to destination. - return d.dst.Write(f) + // Return if there is motion. + return mean > d.thresh } diff --git a/filter/knn.go b/filter/knn.go index 31c97c27..005dc204 100644 --- a/filter/knn.go +++ b/filter/knn.go @@ -29,7 +29,6 @@ LICENSE package filter import ( - "fmt" "image" "io" @@ -38,30 +37,15 @@ import ( ) const ( - defaultKNNMinArea = 25.0 - defaultKNNThreshold = 300 - defaultKNNHistory = 300 - defaultKNNKernel = 4 - defaultKNNDownscaling = 2 - defaultKNNInterval = 1 + defaultKNNMinArea = 25.0 + defaultKNNThreshold = 300 + defaultKNNHistory = 300 + defaultKNNKernel = 4 ) -// KNN is a filter that provides basic motion detection. KNN is short for -// K-Nearest Neighbours method. -type KNN struct { - debugging debugWindows - dst io.WriteCloser // Destination to which motion containing frames go. - area float64 // The minimum area that a contour can be found in. - bs *gocv.BackgroundSubtractorKNN // Uses the KNN algorithm to find the difference between the current and background frame. - knl gocv.Mat // Matrix that is used for calculations. - hold [][]byte // Will hold all frames up to hf (so only every hf frame is motion detected). - hf int // The number of frames to be held. - hfCount int // Counter for the hold array. - scale float64 // The factor that frames will be downscaled by for motion detection. -} +// NewKNN returns a pointer to a new KNN motion filter. +func NewKNN(dst io.WriteCloser, c config.Config) *Motion { -// NewKNN returns a pointer to a new KNN filter struct. -func NewKNN(dst io.WriteCloser, c config.Config) *KNN { // Validate parameters. if c.MotionMinArea <= 0 { c.LogInvalidField("MotionMinArea", defaultKNNMinArea) @@ -75,37 +59,33 @@ func NewKNN(dst io.WriteCloser, c config.Config) *KNN { c.LogInvalidField("MotionHistory", defaultKNNHistory) c.MotionHistory = defaultKNNHistory } - if c.MotionDownscaling <= 0 { - c.LogInvalidField("MotionDownscaling", defaultKNNDownscaling) - c.MotionDownscaling = defaultKNNDownscaling - } - if c.MotionInterval <= 0 { - c.LogInvalidField("MotionInterval", defaultKNNInterval) - c.MotionInterval = defaultKNNInterval - } if c.MotionKernel <= 0 { c.LogInvalidField("MotionKernel", defaultKNNKernel) c.MotionKernel = defaultKNNKernel } bs := gocv.NewBackgroundSubtractorKNNWithParams(int(c.MotionHistory), c.MotionThreshold, false) - k := gocv.GetStructuringElement(gocv.MorphRect, image.Pt(int(c.MotionKernel), int(c.MotionKernel))) - - return &KNN{ - dst: dst, + alg := &KNN{ area: c.MotionMinArea, bs: &bs, - knl: k, - hold: make([][]byte, c.MotionInterval-1), - hf: c.MotionInterval, - scale: 1 / float64(c.MotionDownscaling), + knl: gocv.GetStructuringElement(gocv.MorphRect, image.Pt(int(c.MotionKernel), int(c.MotionKernel))), debugging: newWindows("KNN"), } + + return NewMotion(dst, alg, c) } -// Implements io.Closer. -// Close frees resources used by gocv, because it has to be done manually, due to -// it using c-go. +// KNN is motion detection algorithm. KNN is short for +// K-Nearest Neighbours method. +type KNN struct { + debugging debugWindows + area float64 // The minimum area that a contour can be found in. + bs *gocv.BackgroundSubtractorKNN // Uses the KNN algorithm to find the difference between the current and background frame. + knl gocv.Mat // Matrix that is used for calculations. +} + +// Close frees resources used by gocv. It has to be done manually, +// due to gocv using c-go. func (m *KNN) Close() error { m.bs.Close() m.knl.Close() @@ -113,31 +93,14 @@ func (m *KNN) Close() error { return nil } -// Implements io.Writer. -// Write applies the motion filter to the video stream. Only frames with motion -// are written to the destination encoder, frames without are discarded. -func (m *KNN) Write(f []byte) (int, error) { - if m.hfCount < (m.hf - 1) { - m.hold[m.hfCount] = f - m.hfCount++ - return len(f), nil - } - m.hfCount = 0 - - img, err := gocv.IMDecode(f, gocv.IMReadColor) - if err != nil { - return 0, fmt.Errorf("can't decode image: %w", err) - } - defer img.Close() - +// Detect performs the motion detection on a frame. It returns true +// if motion is detected. +func (m *KNN) Detect(img *gocv.Mat) bool { imgDelta := gocv.NewMat() defer imgDelta.Close() - // Downsize image to speed up calculations. - gocv.Resize(img, &img, image.Point{}, m.scale, m.scale, gocv.InterpolationNearestNeighbor) - // Seperate foreground and background. - m.bs.Apply(img, &imgDelta) + m.bs.Apply(*img, &imgDelta) // Threshold imgDelta. gocv.Threshold(imgDelta, &imgDelta, 25, 255, gocv.ThresholdBinary) @@ -160,20 +123,8 @@ func (m *KNN) Write(f []byte) (int, error) { } // Draw debug information. - m.debugging.show(img, imgDelta, len(contours) > 0, &contours) + m.debugging.show(*img, imgDelta, len(contours) > 0, &contours) - // Don't write to destination if there is no motion. - if len(contours) == 0 { - return len(f), nil - } - - // Write to destination, past 4 frames then current frame. - for i, h := range m.hold { - _, err := m.dst.Write(h) - m.hold[i] = nil - if err != nil { - return len(f), fmt.Errorf("could not write previous frames: %w", err) - } - } - return m.dst.Write(f) + // Return if there is motion. + return len(contours) > 0 } diff --git a/filter/mog.go b/filter/mog.go index 93bd4f92..ab0ac2bc 100644 --- a/filter/mog.go +++ b/filter/mog.go @@ -3,7 +3,7 @@ /* DESCRIPTION A filter that detects motion and discards frames without motion. The - filter uses a Mixture of Gaussians method (MoG) to determine what is + algorithm uses a Mixture of Gaussians method (MoG) to determine what is background and what is foreground. AUTHORS @@ -29,7 +29,6 @@ LICENSE package filter import ( - "fmt" "image" "io" @@ -38,30 +37,15 @@ import ( ) const ( - defaultMOGMinArea = 25.0 - defaultMOGThreshold = 20.0 - defaultMOGHistory = 500 - defaultMOGKernel = 3 - defaultMOGDownscaling = 2 - defaultMOGInterval = 1 + defaultMOGMinArea = 25.0 + defaultMOGThreshold = 20.0 + defaultMOGHistory = 500 + defaultMOGKernel = 3 ) -// MOG is a filter that provides basic motion detection. MoG is short for -// Mixture of Gaussians method. -type MOG struct { - debugging debugWindows - dst io.WriteCloser // Destination to which motion containing frames go. - area float64 // The minimum area that a contour can be found in. - bs *gocv.BackgroundSubtractorMOG2 // Uses the MOG algorithm to find the difference between the current and background frame. - knl gocv.Mat // Matrix that is used for calculations. - hold [][]byte // Will hold all frames up to hf (so only every hf frame is motion detected). - hf int // The number of frames to be held. - hfCount int // Counter for the hold array. - scale float64 // The factor that frames will be downscaled by for motion detection. -} +// NewMOG returns a pointer to a new MOG motion filter. +func NewMOG(dst io.WriteCloser, c config.Config) *Motion { -// NewMOG returns a pointer to a new MOG filter struct. -func NewMOG(dst io.WriteCloser, c config.Config) *MOG { // Validate parameters. if c.MotionMinArea <= 0 { c.LogInvalidField("MotionMinArea", defaultMOGMinArea) @@ -75,36 +59,33 @@ func NewMOG(dst io.WriteCloser, c config.Config) *MOG { c.LogInvalidField("MotionHistory", defaultMOGHistory) c.MotionHistory = defaultMOGHistory } - if c.MotionDownscaling <= 0 { - c.LogInvalidField("MotionDownscaling", defaultMOGDownscaling) - c.MotionDownscaling = defaultMOGDownscaling - } - if c.MotionInterval <= 0 { - c.LogInvalidField("MotionInterval", defaultMOGInterval) - c.MotionInterval = defaultMOGInterval - } if c.MotionKernel <= 0 { c.LogInvalidField("MotionKernel", defaultMOGKernel) c.MotionKernel = defaultMOGKernel } bs := gocv.NewBackgroundSubtractorMOG2WithParams(int(c.MotionHistory), c.MotionThreshold, false) - k := gocv.GetStructuringElement(gocv.MorphRect, image.Pt(3, 3)) - return &MOG{ - dst: dst, + alg := &MOG{ area: c.MotionMinArea, bs: &bs, - knl: k, - hold: make([][]byte, c.MotionInterval-1), - hf: c.MotionInterval, - scale: 1 / float64(c.MotionDownscaling), + knl: gocv.GetStructuringElement(gocv.MorphRect, image.Pt(3, 3)), debugging: newWindows("MOG"), } + + return NewMotion(dst, alg, c) } -// Implements io.Closer. -// Close frees resources used by gocv, because it has to be done manually, due to -// it using c-go. +// MOG is a motion detection algorithm. MoG is short for +// Mixture of Gaussians method. +type MOG struct { + debugging debugWindows + area float64 // The minimum area that a contour can be found in. + bs *gocv.BackgroundSubtractorMOG2 // Uses the MOG algorithm to find the difference between the current and background frame. + knl gocv.Mat // Matrix that is used for calculations. +} + +// Close frees resources used by gocv. It has to be done manually, +// due to gocv using c-go. func (m *MOG) Close() error { m.bs.Close() m.knl.Close() @@ -112,31 +93,14 @@ func (m *MOG) Close() error { return nil } -// Implements io.Writer. -// Write applies the motion filter to the video stream. Only frames with motion -// are written to the destination encoder, frames without are discarded. -func (m *MOG) Write(f []byte) (int, error) { - if m.hfCount < (m.hf - 1) { - m.hold[m.hfCount] = f - m.hfCount++ - return len(f), nil - } - - m.hfCount = 0 - img, err := gocv.IMDecode(f, gocv.IMReadColor) - if err != nil { - return 0, fmt.Errorf("image can't be decoded: %w", err) - } - defer img.Close() - +// Detect performs the motion detection on a frame. It returns true +// if motion is detected. +func (m *MOG) Detect(img *gocv.Mat) bool { imgDelta := gocv.NewMat() defer imgDelta.Close() - // Downsize image to speed up calculations. - gocv.Resize(img, &img, image.Point{}, m.scale, m.scale, gocv.InterpolationNearestNeighbor) - // Seperate foreground and background. - m.bs.Apply(img, &imgDelta) + m.bs.Apply(*img, &imgDelta) // Threshold imgDelta. gocv.Threshold(imgDelta, &imgDelta, 25, 255, gocv.ThresholdBinary) @@ -159,20 +123,8 @@ func (m *MOG) Write(f []byte) (int, error) { } // Draw debug information. - m.debugging.show(img, imgDelta, len(contours) > 0, &contours) + m.debugging.show(*img, imgDelta, len(contours) > 0, &contours) - // Don't write to destination if there is no motion. - if len(contours) == 0 { - return len(f), nil - } - - // Write to destination, past 4 frames then current frame. - for i, h := range m.hold { - _, err := m.dst.Write(h) - m.hold[i] = nil - if err != nil { - return len(f), fmt.Errorf("could not write previous frames: %w", err) - } - } - return m.dst.Write(f) + // Return if there is motion. + return len(contours) > 0 } diff --git a/filter/motion.go b/filter/motion.go new file mode 100644 index 00000000..d2af00d9 --- /dev/null +++ b/filter/motion.go @@ -0,0 +1,132 @@ +// +build !circleci + +/* +DESCRIPTION + A filter that detects motion and discards frames without motion. This + filter can use different algorithms for motion detection. + +AUTHORS + Scott Barnard + +LICENSE + motion.go is Copyright (C) 2019 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ + +package filter + +import ( + "fmt" + "image" + "io" + + "bitbucket.org/ausocean/av/revid/config" + "gocv.io/x/gocv" +) + +const ( + defaultMotionDownscaling = 1 + defaultMotionInterval = 5 + defaultMotionPadding = 10 +) + +// MotionAlgorithm is the interface the motion filter expects for +// motion detection algorithms. +type MotionAlgorithm interface { + Detect(img *gocv.Mat) bool + Close() error +} + +// Motion is a filter that performs motion detection using a supplied +// motion detection algorithm. +type Motion struct { + dst io.WriteCloser // Destination to which motion containing frames go. + algorithm MotionAlgorithm // Algorithm to use for motion detection. + scale float64 // The factor that frames will be downscaled by for motion detection. + sample uint // Interval that motion detection is performed at. + padding uint // The amount of frames before and after motion that will be kept. + + t uint // Frame counter. + send uint // Amount of frames to send. + + frames chan []byte // Used for storing frames. +} + +// NewMotion returns a pointer to a new Motion filter struct. +func NewMotion(dst io.WriteCloser, alg MotionAlgorithm, c config.Config) *Motion { + + // Validate parameters. + if c.MotionPadding == 0 { + c.LogInvalidField("MotionPadding", defaultMotionPadding) + c.MotionPadding = defaultMotionPadding + } + if c.MotionDownscaling <= 0 { + c.LogInvalidField("MotionDownscaling", defaultMotionDownscaling) + c.MotionDownscaling = defaultMotionDownscaling + } + if c.MotionInterval <= 0 { + c.LogInvalidField("MotionInterval", defaultMotionInterval) + c.MotionInterval = defaultMotionInterval + } + + return &Motion{ + dst: dst, + algorithm: alg, + scale: 1 / float64(c.MotionDownscaling), + sample: uint(c.MotionInterval), + padding: c.MotionPadding, + frames: make(chan []byte, c.MotionInterval+int(c.MotionPadding)), + } +} + +// Implements io.Closer. +// Close frees resources used by gocv, because it has to be done manually, due to +// it using c-go. +func (m *Motion) Close() error { + return m.algorithm.Close() +} + +// Write applies the motion filter to the video stream. Only frames with motion +// are written to the destination encoder, frames without are discarded. +func (m *Motion) Write(f []byte) (int, error) { + // Decode image into Mat. + img, err := gocv.IMDecode(f, gocv.IMReadColor) + if err != nil { + return 0, fmt.Errorf("image can't be decoded: %w", err) + } + defer img.Close() + + // Downsize image to speed up calculations. + gocv.Resize(img, &img, image.Point{}, m.scale, m.scale, gocv.InterpolationNearestNeighbor) + + // Filter on an interval. + if m.t == m.sample/2 { + if m.algorithm.Detect(&img) { + m.send = m.sample + 2*m.padding - 1 + } + } + m.t = (m.t + 1) % m.sample // Increment counter. + + // Send frames. + m.frames <- f // Put current frame into buffer. + toSend := <-m.frames // Get oldest frame out of circular buffer. + + if m.send > 0 { + m.send-- + return m.dst.Write(toSend) + } + + return len(f), nil +} diff --git a/revid/config/config.go b/revid/config/config.go index 9414b5ae..48a9f288 100644 --- a/revid/config/config.go +++ b/revid/config/config.go @@ -276,6 +276,7 @@ type Config struct { MotionHistory uint // Length of filter's history (KNN & MOG only). MotionKernel uint // Size of kernel used for filling holes and removing noise (KNN only). MotionPixels int // Number of pixels with motion that is needed for a whole frame to be considered as moving (Basic only). + MotionPadding uint // Number of frames to keep before and after motion detected. // If true will restart reading of input after an io.EOF. Loop bool @@ -315,6 +316,7 @@ var TypeData = map[string]string{ "MotionInterval": "int", "MotionKernel": "uint", "MotionMinArea": "float", + "MotionPadding": "uint", "MotionPixels": "int", "MotionThreshold": "float", "Output": "enum:File,Http,Rtmp,Rtp", diff --git a/revid/revid.go b/revid/revid.go index 1ef886d8..69646547 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -821,6 +821,13 @@ func (r *Revid) Update(vars map[string]string) error { break } r.cfg.MotionHistory = uint(v) + case "MotionPadding": + v, err := strconv.Atoi(value) + if err != nil || v <= 0 { + r.cfg.Logger.Log(logger.Warning, pkg+"invalid MotionPadding var", "value", value) + break + } + r.cfg.MotionPadding = uint(v) case "MotionPixels": v, err := strconv.Atoi(value) if err != nil {