diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 4e340b4c..bced0a4f 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -282,7 +282,7 @@ func run(cfg config.Config) { return nil // Return error only if we want NetSender to generate an error } - ns, err := netsender.New(log, nil, readPin, nil) + ns, err := netsender.New(log, nil, readPin, nil, config.TypeData) if err != nil { log.Log(logger.Fatal, pkg+"could not initialise netsender client: "+err.Error()) } diff --git a/container/mts/encoder.go b/container/mts/encoder.go index 251607e6..3447eebf 100644 --- a/container/mts/encoder.go +++ b/container/mts/encoder.go @@ -155,9 +155,12 @@ type Encoder struct { continuity map[uint16]byte - nalBasedPSI bool + psiMethod int pktCount int psiSendCount int + psiTime time.Duration + psiSetTime time.Duration + startTime time.Time mediaPid uint16 streamID byte } @@ -167,21 +170,24 @@ type Encoder struct { func NewEncoder(dst io.WriteCloser, rate float64, mediaType int, options ...func(*Encoder) error) (*Encoder, error) { var mPID uint16 var sID byte - nbp := true + psiM := timeBased switch mediaType { case EncodeAudio: mPID = AudioPid sID = audioStreamID - nbp = false + psiM = pktBased case EncodeH265: mPID = VideoPid sID = H265ID + psiM = nalBased case EncodeH264: mPID = VideoPid sID = H264ID + psiM = nalBased case EncodeMJPEG: mPID = VideoPid sID = MJPEGID + psiM = timeBased } pmt := BasePMT @@ -202,7 +208,7 @@ func NewEncoder(dst io.WriteCloser, rate float64, mediaType int, options ...func writePeriod: time.Duration(float64(time.Second) / rate), ptsOffset: ptsOffset, - nalBasedPSI: nbp, + psiMethod: psiM, pktCount: 8, @@ -225,22 +231,52 @@ func NewEncoder(dst io.WriteCloser, rate float64, mediaType int, options ...func return e, nil } +// These three constants are used to select between the three different +// methods of when the PSI is sent. +const ( + pktBased = iota + timeBased + nalBased +) + // PacketBasedPSI is an option that can be passed to NewEncoder to select // packet based PSI writing, i.e. PSI are written to the destination every // sendCount packets. func PacketBasedPSI(sendCount int) func(*Encoder) error { return func(e *Encoder) error { - e.nalBasedPSI = false + e.psiMethod = pktBased e.psiSendCount = sendCount e.pktCount = e.psiSendCount return nil } } +// TimeBasedPSI is another option that can be passed to NewEncoder to select +// time based PSI writing, i.e. PSI are written to the destination every dur (duration) +// (defualt is 2 seconds). +func TimeBasedPSI(dur time.Duration) func(*Encoder) error { + return func(e *Encoder) error { + e.psiMethod = timeBased + e.psiTime = 0 + e.psiSetTime = dur + e.startTime = time.Now() + return nil + } +} + // Write implements io.Writer. Write takes raw video or audio data and encodes into MPEG-TS, // then sending it to the encoder's io.Writer destination. func (e *Encoder) Write(data []byte) (int, error) { - if e.nalBasedPSI { + switch e.psiMethod { + case pktBased: + if e.pktCount >= e.psiSendCount { + e.pktCount = 0 + err := e.writePSI() + if err != nil { + return 0, err + } + } + case nalBased: nalType, err := h264.NALType(data) if err != nil { return 0, fmt.Errorf("could not get type from NAL unit, failed with error: %w", err) @@ -252,12 +288,18 @@ func (e *Encoder) Write(data []byte) (int, error) { return 0, err } } - } else if e.pktCount >= e.psiSendCount { - e.pktCount = 0 - err := e.writePSI() - if err != nil { - return 0, err + case timeBased: + if time.Now().Sub(e.startTime) >= e.psiTime { + e.psiTime = e.psiSetTime + e.startTime = time.Now() + err := e.writePSI() + if err != nil { + return 0, err + } + } + default: + panic("Undefined PSI method") } // Prepare PES data. @@ -319,6 +361,7 @@ func (e *Encoder) writePSI() error { } e.pktCount++ pmtTable, err = updateMeta(pmtTable) + if err != nil { return err } diff --git a/exp/gocv-exp/gocv-exp b/exp/gocv-exp/gocv-exp new file mode 100644 index 00000000..7a51493d Binary files /dev/null and b/exp/gocv-exp/gocv-exp differ diff --git a/exp/gocv-exp/leatherjacket.mp4 b/exp/gocv-exp/leatherjacket.mp4 new file mode 100644 index 00000000..55815d30 Binary files /dev/null and b/exp/gocv-exp/leatherjacket.mp4 differ diff --git a/exp/gocv-exp/littleFish.mp4 b/exp/gocv-exp/littleFish.mp4 new file mode 100644 index 00000000..4b8d9d9d Binary files /dev/null and b/exp/gocv-exp/littleFish.mp4 differ diff --git a/exp/gocv-exp/littleFish2.mp4 b/exp/gocv-exp/littleFish2.mp4 new file mode 100644 index 00000000..41817558 Binary files /dev/null and b/exp/gocv-exp/littleFish2.mp4 differ diff --git a/exp/gocv-exp/main.go b/exp/gocv-exp/main.go new file mode 100644 index 00000000..9de10184 --- /dev/null +++ b/exp/gocv-exp/main.go @@ -0,0 +1,117 @@ +// What it does: +// +// This example detects motion using a delta threshold from the first frame, +// and then finds contours to determine where the object is located. +// +// Very loosely based on Adrian Rosebrock code located at: +// http://www.pyimagesearch.com/2015/06/01/home-surveillance-and-motion-detection-with-the-raspberry-pi-python-and-opencv/ + +package main + +import ( + "fmt" + "image" + "image/color" + "os" + "strconv" + "time" + + "gocv.io/x/gocv" +) + +const MinimumArea = 1000 + +func main() { + if len(os.Args) < 2 { + fmt.Println("How to run:\n\tmotion-detect [camera ID]") + return + } + + // parse args + deviceID := os.Args[1] + minArea, _ := strconv.Atoi(os.Args[2]) + + webcam, err := gocv.OpenVideoCapture(deviceID) + if err != nil { + fmt.Printf("Error opening video capture device: %v\n", deviceID) + return + } + defer webcam.Close() + + window1 := gocv.NewWindow("Motion Window") + defer window1.Close() + + window2 := gocv.NewWindow("Threshold window") + defer window2.Close() + + time.Sleep(2 * time.Second) + + img := gocv.NewMat() + defer img.Close() + + imgDelta := gocv.NewMat() + defer imgDelta.Close() + + imgThresh := gocv.NewMat() + defer imgThresh.Close() + + mog2 := gocv.NewBackgroundSubtractorMOG2() + defer mog2.Close() + + status := "Ready" + + fmt.Printf("Start reading device: %v\n", deviceID) + for { + if ok := webcam.Read(&img); !ok { + fmt.Printf("Device closed: %v\n", deviceID) + return + } + if img.Empty() { + continue + } + + status = "READY" + statusColor := color.RGBA{0, 255, 0, 0} + + // first phase of cleaning up image, obtain foreground only + mog2.Apply(img, &imgDelta) + + // remaining cleanup of the image to use for finding contours. + // first use threshold + gocv.Threshold(imgDelta, &imgThresh, 25, 255, gocv.ThresholdBinary) + + // then dilate + kernel := gocv.GetStructuringElement(gocv.MorphRect, image.Pt(3, 3)) + defer kernel.Close() + + // Erode + gocv.Erode(imgThresh, &imgThresh, kernel) + // Dilate + gocv.Dilate(imgThresh, &imgThresh, kernel) + + // now find contours + contours := gocv.FindContours(imgThresh, gocv.RetrievalExternal, gocv.ChainApproxSimple) + for _, c := range contours { + area := gocv.ContourArea(c) + if area < float64(minArea) { + continue + } + + status = "MOTION DETECTED" + statusColor = color.RGBA{255, 0, 0, 0} + + //gocv.DrawContours(&img, contours, i, statusColor, 2) + + rect := gocv.BoundingRect(c) + gocv.Rectangle(&img, rect, color.RGBA{0, 0, 255, 0}, 1) + } + + gocv.PutText(&img, status, image.Pt(10, 20), gocv.FontHersheyPlain, 1.2, statusColor, 2) + + window1.IMShow(img) + window2.IMShow(imgThresh) + if window1.WaitKey(1) == 27 { + break + } + } +} diff --git a/exp/gocv-exp/portJackson.mp4 b/exp/gocv-exp/portJackson.mp4 new file mode 100644 index 00000000..0563f292 Binary files /dev/null and b/exp/gocv-exp/portJackson.mp4 differ diff --git a/exp/gocv-exp/school.mp4 b/exp/gocv-exp/school.mp4 new file mode 100644 index 00000000..581e06b0 Binary files /dev/null and b/exp/gocv-exp/school.mp4 differ diff --git a/exp/gocv-exp/squid.mp4 b/exp/gocv-exp/squid.mp4 new file mode 100644 index 00000000..6a2ffedf Binary files /dev/null and b/exp/gocv-exp/squid.mp4 differ diff --git a/exp/gocv-exp/squid2.mp4 b/exp/gocv-exp/squid2.mp4 new file mode 100644 index 00000000..4d29057d Binary files /dev/null and b/exp/gocv-exp/squid2.mp4 differ diff --git a/exp/gocv-exp/squid3.mp4 b/exp/gocv-exp/squid3.mp4 new file mode 100644 index 00000000..169181b0 Binary files /dev/null and b/exp/gocv-exp/squid3.mp4 differ diff --git a/filter/mog.go b/filter/mog.go new file mode 100644 index 00000000..99fe332b --- /dev/null +++ b/filter/mog.go @@ -0,0 +1,127 @@ +/* +DESCRIPTION + A filter that detects motion and discards frames without motion. The + filter uses a Mixture of Gaussians method (MoG) to determine what is + background and what is foreground. + +AUTHORS + Scott Barnard + +LICENSE + mog.go is Copyright (C) 2019 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ + +package filter + +import ( + "image" + "image/color" + "io" + + "gocv.io/x/gocv" +) + +// MOGFilter is a filter that provides basic motion detection. MoG is short for +// Mixture of Gaussians method. +type MOGFilter struct { + dst io.WriteCloser + area float64 + bs *gocv.BackgroundSubtractorMOG2 + knl gocv.Mat + debug bool + windows []*gocv.Window +} + +// NewMOGFilter returns a pointer to a new MOGFilter. +func NewMOGFilter(dst io.WriteCloser, area, threshold float64, history, kernelSize int, debug bool) *MogFilter { + bs := gocv.NewBackgroundSubtractorMOG2WithParams(history, threshold, false) + k := gocv.GetStructuringElement(gocv.MorphRect, image.Pt(kernelSize, kernelSize)) + var windows []*gocv.Window + if debug { + windows = []*gocv.Window{gocv.NewWindow("Debug: Bounding boxes"), gocv.NewWindow("Debug: Motion")} + } + return &MOGFilter{dst, area, &bs, k, debug, windows} +} + +// Implements io.Closer. +// Close frees resources used by gocv, because it has to be done manually, due to +// it using c-go. +func (m *MOGFilter) Close() error { + m.bs.Close() + m.knl.Close() + for _, window := range m.windows { + window.Close() + } + return m.dst.Close() +} + +// Implements io.Writer. +// Write applies the motion filter to the video stream. Only frames with motion +// are written to the destination encoder, frames without are discarded. +func (m *MOGFilter) Write(f []byte) (int, error) { + img, _ := gocv.IMDecode(f, gocv.IMReadColor) + defer img.Close() + + imgDelta := gocv.NewMat() + defer imgDelta.Close() + + // Seperate foreground and background. + m.bs.Apply(img, &imgDelta) + + // Threshold imgDelta. + gocv.Threshold(imgDelta, &imgDelta, 25, 255, gocv.ThresholdBinary) + + // Remove noise. + gocv.Erode(imgDelta, &imgDelta, m.knl) + gocv.Dilate(imgDelta, &imgDelta, m.knl) + + // Fill small holes. + gocv.Dilate(imgDelta, &imgDelta, m.knl) + gocv.Erode(imgDelta, &imgDelta, m.knl) + + // Find contours and reject ones with a small area. + var contours [][]image.Point + allContours := gocv.FindContours(imgDelta, gocv.RetrievalExternal, gocv.ChainApproxSimple) + for _, c := range allContours { + if gocv.ContourArea(c) > m.area { + contours = append(contours, c) + } + } + + // Draw debug information. + if m.debug { + for _, c := range contours { + rect := gocv.BoundingRect(c) + gocv.Rectangle(&img, rect, color.RGBA{0, 0, 255, 0}, 1) + } + + if len(contours) > 0 { + gocv.PutText(&img, "Motion", image.Pt(32, 32), gocv.FontHersheyPlain, 2.0, color.RGBA{255, 0, 0, 0}, 2) + } + + m.windows[0].IMShow(img) + m.windows[1].IMShow(imgDelta) + m.windows[0].WaitKey(1) + } + + // Don't write to destination if there is no motion. + if len(contours) == 0 { + return -1, nil + } + + // Write to destination. + return m.dst.Write(f) +} diff --git a/go.mod b/go.mod index a1860670..1ab98014 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,14 @@ module bitbucket.org/ausocean/av go 1.13 require ( - bitbucket.org/ausocean/iot v1.2.8 + bitbucket.org/ausocean/iot v1.2.9 bitbucket.org/ausocean/utils v1.2.11 github.com/Comcast/gots v0.0.0-20190305015453-8d56e473f0f7 github.com/go-audio/audio v0.0.0-20181013203223-7b2a6ca21480 github.com/go-audio/wav v0.0.0-20181013172942-de841e69b884 - github.com/kr/pretty v0.1.0 // indirect github.com/mewkiz/flac v1.0.5 github.com/pkg/errors v0.8.1 github.com/yobert/alsa v0.0.0-20180630182551-d38d89fa843e + gocv.io/x/gocv v0.21.0 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect ) diff --git a/go.sum b/go.sum index 7870ba89..e6eee428 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ bitbucket.org/ausocean/iot v1.2.8 h1:m2UhfAbG/6RKPBzY4OJ5S7zLxTmuzyMKPNbu0qaxFIw= bitbucket.org/ausocean/iot v1.2.8/go.mod h1:wCLOYeEDCxDquneSZ/zTEcKGXZ6uan+6sgZyTMlNVDo= +bitbucket.org/ausocean/iot v1.2.9 h1:3tzgiekH+Z0yXhkwnqBzxxe8qQJ2O7YTkz4s0T6stgw= +bitbucket.org/ausocean/iot v1.2.9/go.mod h1:Q5FwaOKnCty3dVeVtki6DLwYa5vhNpOaeu1lwLyPCg8= bitbucket.org/ausocean/utils v1.2.11 h1:zA0FOaPjN960ryp8PKCkV5y50uWBYrIxCVnXjwbvPqg= bitbucket.org/ausocean/utils v1.2.11/go.mod h1:uXzX9z3PLemyURTMWRhVI8uLhPX4uuvaaO85v2hcob8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= @@ -52,6 +54,8 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +gocv.io/x/gocv v0.21.0 h1:dVjagrupZrfCRY0qPEaYWgoNMRpBel6GYDH4mvQOK8Y= +gocv.io/x/gocv v0.21.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= golang.org/x/sys v0.0.0-20190913121621-c3b328c6e5a7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/revid/config/config.go b/revid/config/config.go index 1e21b7c0..9457199b 100644 --- a/revid/config/config.go +++ b/revid/config/config.go @@ -82,6 +82,7 @@ const ( defaultWriteRate = 25 defaultClipDuration = 0 defaultAudioInputCodec = codecutil.ADPCM + defaultPSITime = 2 // MTS ring buffer defaults. defaultMTSRBSize = 100 @@ -238,14 +239,16 @@ type Config struct { Channels int // Number of audio channels, 1 for mono, 2 for stereo. BitDepth int // Sample bit depth. - RTPAddress string // RTPAddress defines the RTP output destination. - BurstPeriod uint // BurstPeriod defines the revid burst period in seconds. - Rotation uint // Rotation defines the video rotation angle in degrees Raspivid input. - Height uint // Height defines the input video height Raspivid input. - Width uint // Width defines the input video width Raspivid input. - Bitrate uint // Bitrate specifies the bitrate for constant bitrate in kbps. - HorizontalFlip bool // HorizontalFlip flips video horizontally for Raspivid input. - VerticalFlip bool // VerticalFlip flips video vertically for Raspivid input. + RTPAddress string // RTPAddress defines the RTP output destination. + BurstPeriod uint // BurstPeriod defines the revid burst period in seconds. + Rotation uint // Rotation defines the video rotation angle in degrees Raspivid input. + Height uint // Height defines the input video height Raspivid input. + Width uint // Width defines the input video width Raspivid input. + Bitrate uint // Bitrate specifies the bitrate for constant bitrate in kbps. + + HorizontalFlip bool // HorizontalFlip flips video horizontally for Raspivid input. + VerticalFlip bool // VerticalFlip flips video vertically for Raspivid input. + PSITime int // Sets the time between a packet being sent // RTMP ring buffer parameters. RTMPRBSize int // The number of elements in the RTMP sender ringbuffer. @@ -258,6 +261,45 @@ type Config struct { MTSRBWriteTimeout int // The ringbuffer write timeout in seconds. } +// TypeData contains information about all of the variables that +// can be set over the web. It is a psuedo const. +var TypeData = map[string]string{ + "AutoWhiteBalance": "enum:off,auto,sun,cloud,shade,tungsten,fluorescent,incandescent,flash,horizon", + "BitDepth": "int", + "Brightness": "uint", + "BurstPeriod": "uint", + "CameraChan": "int", + "CBR": "bool", + "ClipDuration": "uint", + "Exposure": "enum:auto,night,nightpreview,backlight,spotlight,sports,snow,beach,verylong,fixedfps,antishake,fireworks", + "FrameRate": "uint", + "Height": "uint", + "HorizontalFlip": "bool", + "HTTPAddress": "string", + "Input": "enum:raspivid,rtsp,v4l,file", + "InputCodec": "enum:H264,MJPEG", + "InputPath": "string", + "Logging": "enum:Debug,Info,Warning,Error,Fatal", + "MinFrames": "uint", + "MTSRBElementSize": "int", + "MTSRBSize": "int", + "MTSRBWriteTimeout": "int", + "OutputPath": "string", + "Outputs": "string", + "Quantization": "uint", + "Rotation": "uint", + "RTMPRBElementSize": "int", + "RTMPRBSize": "int", + "RTMPRBWriteTimeout": "int", + "RTMPURL": "string", + "RTPAddress": "string", + "Saturation": "int", + "VBRBitrate": "int", + "VBRQuality": "enum:standard,fair,good,great,excellent", + "VerticalFlip": "bool", + "Width": "uint", +} + // Validation errors. var ( errInvalidQuantization = errors.New("invalid quantization") @@ -385,6 +427,11 @@ func (c *Config) Validate() error { c.MTSRBWriteTimeout = defaultMTSRBWriteTimeout } + if c.PSITime <= 0 { + c.Logger.Log(logger.Info, pkg+"PSITime bad or unset, defaulting", "PSITime", defaultPSITime) + c.PSITime = defaultPSITime + } + return nil } diff --git a/revid/revid.go b/revid/revid.go index bd657ef1..b2b399dc 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -186,7 +186,7 @@ func (r *Revid) reset(c config.Config) error { st = mts.EncodeH264 case codecutil.MJPEG: st = mts.EncodeMJPEG - encOptions = append(encOptions, mts.PacketBasedPSI(int(r.cfg.MinFrames))) + encOptions = append(encOptions, mts.TimeBasedPSI(time.Duration(r.cfg.PSITime)*time.Second)) default: panic("unknown input codec for raspivid input") } @@ -196,7 +196,7 @@ func (r *Revid) reset(c config.Config) error { st = mts.EncodeH264 case codecutil.MJPEG: st = mts.EncodeMJPEG - encOptions = append(encOptions, mts.PacketBasedPSI(int(r.cfg.MinFrames))) + encOptions = append(encOptions, mts.TimeBasedPSI(time.Duration(r.cfg.PSITime)*time.Second)) default: panic("unknown input codec for v4l or input file input") } @@ -208,7 +208,7 @@ func (r *Revid) reset(c config.Config) error { st = mts.EncodeH264 case codecutil.MJPEG: st = mts.EncodeMJPEG - encOptions = append(encOptions, mts.PacketBasedPSI(int(r.cfg.MinFrames))) + encOptions = append(encOptions, mts.TimeBasedPSI(time.Duration(r.cfg.PSITime)*time.Second)) default: panic("unknown input codec for RTSP input") } @@ -611,6 +611,13 @@ func (r *Revid) Update(vars map[string]string) error { default: r.cfg.Logger.Log(logger.Warning, pkg+"invalid VerticalFlip param", "value", value) } + case "PSITime": + v, err := strconv.Atoi(value) + if err != nil || v < 0 { + r.cfg.Logger.Log(logger.Warning, pkg+"invalid PSITime var", "value", value) + break + } + r.cfg.PSITime = v case "BurstPeriod": v, err := strconv.Atoi(value) if err != nil {