From 9454ef88c3385e72651a0483bed1a9f244bdc1e7 Mon Sep 17 00:00:00 2001 From: Alan Noble Date: Wed, 15 Jan 2020 13:46:56 +1030 Subject: [PATCH 01/46] revid/config/config.go: fixed typo in outputs data type (enums->enum) --- revid/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/revid/config/config.go b/revid/config/config.go index 04d51a63..2ae14feb 100644 --- a/revid/config/config.go +++ b/revid/config/config.go @@ -332,7 +332,7 @@ var TypeData = map[string]string{ "RBWriteTimeout": "uint", "Output": "enum:File,Http,Rtmp,Rtp", "OutputPath": "string", - "Outputs": "enums:File,Http,Rtmp,Rtp", + "Outputs": "enum:File,Http,Rtmp,Rtp", "Quantization": "uint", "Rotation": "uint", "RTMPURL": "string", From 82e2df5d88affb2e08dd42caa95e40542943768d Mon Sep 17 00:00:00 2001 From: Alan Noble Date: Wed, 15 Jan 2020 13:58:51 +1030 Subject: [PATCH 02/46] revid/config/config.go: reverted incorrect change --- revid/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/revid/config/config.go b/revid/config/config.go index 2ae14feb..04d51a63 100644 --- a/revid/config/config.go +++ b/revid/config/config.go @@ -332,7 +332,7 @@ var TypeData = map[string]string{ "RBWriteTimeout": "uint", "Output": "enum:File,Http,Rtmp,Rtp", "OutputPath": "string", - "Outputs": "enum:File,Http,Rtmp,Rtp", + "Outputs": "enums:File,Http,Rtmp,Rtp", "Quantization": "uint", "Rotation": "uint", "RTMPURL": "string", From 39d0fa12d04df2ec947a4cfa5e2538fae465f2d0 Mon Sep 17 00:00:00 2001 From: Saxon Date: Sat, 18 Jan 2020 09:54:12 +1030 Subject: [PATCH 03/46] cmd/revid-cli: send logs using smartlogger --- cmd/revid-cli/main.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index bced0a4f..87185323 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -72,7 +72,10 @@ const ( var canProfile = true // The logger that will be used throughout. -var log *logger.Logger +var ( + smartLog *smartlogger.Smartlogger + log *logger.Logger +) const ( metaPreambleKey = "copyright" @@ -160,7 +163,8 @@ func handleFlags() config.Config { cfg.LogLevel = defaultLogVerbosity } - log = logger.New(cfg.LogLevel, &smartlogger.New(*logPathPtr).LogRoller, true) + smartLog = smartlogger.New(*logPathPtr) + log = logger.New(cfg.LogLevel, &smartLog.LogRoller, true) cfg.Logger = log @@ -351,6 +355,8 @@ func run(cfg config.Config) { ns.SetMode(paused, &vs) } + smartLog.SendLogs(ns) + sleep: sleepTime, err := strconv.Atoi(ns.Param("mp")) if err != nil { From b1a7adc487f2b1b75c0f11b2c8b53c582afa6c25 Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 20 Jan 2020 10:32:22 +1030 Subject: [PATCH 04/46] cmd/revid-cli: send logs using netlogger --- cmd/revid-cli/main.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 87185323..d5cc51db 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -32,7 +32,9 @@ package main import ( "flag" + "io" "os" + "path/filepath" "runtime/pprof" "strconv" "strings" @@ -44,10 +46,11 @@ import ( "bitbucket.org/ausocean/av/device/raspivid" "bitbucket.org/ausocean/av/revid" "bitbucket.org/ausocean/av/revid/config" + "bitbucket.org/ausocean/iot/pi/netlogger" "bitbucket.org/ausocean/iot/pi/netsender" "bitbucket.org/ausocean/iot/pi/sds" - "bitbucket.org/ausocean/iot/pi/smartlogger" "bitbucket.org/ausocean/utils/logger" + "gopkg.in/natefinch/lumberjack.v2" ) // Revid modes @@ -73,8 +76,8 @@ var canProfile = true // The logger that will be used throughout. var ( - smartLog *smartlogger.Smartlogger - log *logger.Logger + netLogger *netlogger.NetLogger + log *logger.Logger ) const ( @@ -163,8 +166,16 @@ func handleFlags() config.Config { cfg.LogLevel = defaultLogVerbosity } - smartLog = smartlogger.New(*logPathPtr) - log = logger.New(cfg.LogLevel, &smartLog.LogRoller, true) + netLogger = netlogger.New() + log = logger.New(cfg.LogLevel, io.MultiWriter( + &lumberjack.Logger{ + Filename: filepath.Join(*logPathPtr, "netsender.log"), + MaxSize: 500, // megabytes + MaxBackups: 10, + MaxAge: 28, // days + }, + netLogger, + ), true) cfg.Logger = log @@ -355,7 +366,7 @@ func run(cfg config.Config) { ns.SetMode(paused, &vs) } - smartLog.SendLogs(ns) + netLogger.SendLogs(ns) sleep: sleepTime, err := strconv.Atoi(ns.Param("mp")) From e4315d8203fdf909cef7517280cd58eca8845da7 Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 20 Jan 2020 10:50:02 +1030 Subject: [PATCH 05/46] cmd/revid-cli: added error checking for sending of logs --- cmd/revid-cli/main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index d5cc51db..d319d4fe 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -311,6 +311,11 @@ func run(cfg config.Config) { continue } + err = netLogger.SendLogs(ns) + if err != nil { + log.Log(logger.Warning, pkg+"Logs could not be sent", "error", err.Error()) + } + // If var sum hasn't changed we continue. var vars map[string]string newVs := ns.VarSum() @@ -366,8 +371,6 @@ func run(cfg config.Config) { ns.SetMode(paused, &vs) } - netLogger.SendLogs(ns) - sleep: sleepTime, err := strconv.Atoi(ns.Param("mp")) if err != nil { From 8c82a03c2551a14f71e9fe7a9d39bbeb692ad27a Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 20 Jan 2020 16:20:35 +1030 Subject: [PATCH 06/46] cmd/revid-cli/main.go: Renamed SendLogs to Send. --- cmd/revid-cli/main.go | 2 +- go.mod | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index d319d4fe..ee892df9 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -311,7 +311,7 @@ func run(cfg config.Config) { continue } - err = netLogger.SendLogs(ns) + err = netLogger.Send(ns) if err != nil { log.Log(logger.Warning, pkg+"Logs could not be sent", "error", err.Error()) } diff --git a/go.mod b/go.mod index 632c0ebb..f9a618a0 100644 --- a/go.mod +++ b/go.mod @@ -12,4 +12,5 @@ require ( github.com/pkg/errors v0.8.1 github.com/yobert/alsa v0.0.0-20180630182551-d38d89fa843e gocv.io/x/gocv v0.21.0 + gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) From 9d2fd533358b02e7d2ad3ac1139fa1be930c0cd3 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 22 Jan 2020 15:33:23 +1030 Subject: [PATCH 07/46] megabytes -> MB --- cmd/revid-cli/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index ee892df9..d7bdd267 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -170,7 +170,7 @@ func handleFlags() config.Config { log = logger.New(cfg.LogLevel, io.MultiWriter( &lumberjack.Logger{ Filename: filepath.Join(*logPathPtr, "netsender.log"), - MaxSize: 500, // megabytes + MaxSize: 500, // MB MaxBackups: 10, MaxAge: 28, // days }, From 93f7d5849e8dfc47e3b255ac5a59c1060d2bd216 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 22 Jan 2020 15:36:20 +1030 Subject: [PATCH 08/46] Added newlines to logger.New --- cmd/revid-cli/main.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index d7bdd267..67cddc0a 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -167,15 +167,19 @@ func handleFlags() config.Config { } netLogger = netlogger.New() - log = logger.New(cfg.LogLevel, io.MultiWriter( - &lumberjack.Logger{ - Filename: filepath.Join(*logPathPtr, "netsender.log"), - MaxSize: 500, // MB - MaxBackups: 10, - MaxAge: 28, // days - }, - netLogger, - ), true) + log = logger.New( + cfg.LogLevel, + io.MultiWriter( + &lumberjack.Logger{ + Filename: filepath.Join(*logPathPtr, "netsender.log"), + MaxSize: 500, // MB + MaxBackups: 10, + MaxAge: 28, // days + }, + netLogger, + ), + true, + ) cfg.Logger = log From 0cfa93a314aa8782c99491c338bf887eb5679695 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 22 Jan 2020 16:34:20 +1030 Subject: [PATCH 09/46] Uses iot v1.2.12 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f9a618a0..c7100214 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module bitbucket.org/ausocean/av go 1.13 require ( - bitbucket.org/ausocean/iot v1.2.11 + bitbucket.org/ausocean/iot v1.2.12 bitbucket.org/ausocean/utils v1.2.12 github.com/Comcast/gots v0.0.0-20190305015453-8d56e473f0f7 github.com/go-audio/audio v0.0.0-20181013203223-7b2a6ca21480 diff --git a/go.sum b/go.sum index b0fee827..87d44284 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ bitbucket.org/ausocean/iot v1.2.10 h1:TTu+ykH5gQA8wU/pN0aS55ySQ/XcGxV4s4LKx3Wye5 bitbucket.org/ausocean/iot v1.2.10/go.mod h1:Q5FwaOKnCty3dVeVtki6DLwYa5vhNpOaeu1lwLyPCg8= bitbucket.org/ausocean/iot v1.2.11 h1:MwYQK1F2ESA5jPVSCB0lBUN8HBiNDHGkh/OMGJKw8Oc= bitbucket.org/ausocean/iot v1.2.11/go.mod h1:Q5FwaOKnCty3dVeVtki6DLwYa5vhNpOaeu1lwLyPCg8= +bitbucket.org/ausocean/iot v1.2.12 h1:Ixf0CTmWOMJVrJ6IYMEluTrCLlu9LM1eNSBZ+ZUnDmU= +bitbucket.org/ausocean/iot v1.2.12/go.mod h1:Q5FwaOKnCty3dVeVtki6DLwYa5vhNpOaeu1lwLyPCg8= bitbucket.org/ausocean/utils v1.2.11 h1:zA0FOaPjN960ryp8PKCkV5y50uWBYrIxCVnXjwbvPqg= bitbucket.org/ausocean/utils v1.2.11/go.mod h1:uXzX9z3PLemyURTMWRhVI8uLhPX4uuvaaO85v2hcob8= bitbucket.org/ausocean/utils v1.2.12 h1:VnskjWTDM475TnQRhBQE0cNp9D6Y6OELrd4UkD2VVIQ= From 0e94a0172bb454948a1823ab4d8f930f0c8e7e24 Mon Sep 17 00:00:00 2001 From: Scott Date: Thu, 23 Jan 2020 09:41:54 +1030 Subject: [PATCH 10/46] cmd/revid-cli/main.go: netlogger.NetLogger -> netlogger.Logger --- cmd/revid-cli/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 67cddc0a..38cb71b4 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -76,7 +76,7 @@ var canProfile = true // The logger that will be used throughout. var ( - netLogger *netlogger.NetLogger + netLogger *netlogger.Logger log *logger.Logger ) From 62141926ff8f00d9afa15490f21d75d562328f7e Mon Sep 17 00:00:00 2001 From: Scott Date: Thu, 23 Jan 2020 10:01:16 +1030 Subject: [PATCH 11/46] =?UTF-8?q?main:=20netLogger=20=E2=86=92=20netLog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/revid-cli/main.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 38cb71b4..db6e17ec 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -76,8 +76,8 @@ var canProfile = true // The logger that will be used throughout. var ( - netLogger *netlogger.Logger - log *logger.Logger + netLog *netlogger.Logger + log *logger.Logger ) const ( @@ -166,7 +166,7 @@ func handleFlags() config.Config { cfg.LogLevel = defaultLogVerbosity } - netLogger = netlogger.New() + netLog = netlogger.New() log = logger.New( cfg.LogLevel, io.MultiWriter( @@ -176,7 +176,7 @@ func handleFlags() config.Config { MaxBackups: 10, MaxAge: 28, // days }, - netLogger, + netLog, ), true, ) @@ -315,7 +315,7 @@ func run(cfg config.Config) { continue } - err = netLogger.Send(ns) + err = netLog.Send(ns) if err != nil { log.Log(logger.Warning, pkg+"Logs could not be sent", "error", err.Error()) } From 44508fd2691ed2403bf3d5b7dc441508fd9caa8a Mon Sep 17 00:00:00 2001 From: Scott Date: Fri, 24 Jan 2020 15:36:27 +1030 Subject: [PATCH 12/46] use iot v1.2.13 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c7100214..2ed4c734 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module bitbucket.org/ausocean/av go 1.13 require ( - bitbucket.org/ausocean/iot v1.2.12 + bitbucket.org/ausocean/iot v1.2.13 bitbucket.org/ausocean/utils v1.2.12 github.com/Comcast/gots v0.0.0-20190305015453-8d56e473f0f7 github.com/go-audio/audio v0.0.0-20181013203223-7b2a6ca21480 diff --git a/go.sum b/go.sum index 87d44284..ef1fb9ff 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ bitbucket.org/ausocean/iot v1.2.11 h1:MwYQK1F2ESA5jPVSCB0lBUN8HBiNDHGkh/OMGJKw8O bitbucket.org/ausocean/iot v1.2.11/go.mod h1:Q5FwaOKnCty3dVeVtki6DLwYa5vhNpOaeu1lwLyPCg8= bitbucket.org/ausocean/iot v1.2.12 h1:Ixf0CTmWOMJVrJ6IYMEluTrCLlu9LM1eNSBZ+ZUnDmU= bitbucket.org/ausocean/iot v1.2.12/go.mod h1:Q5FwaOKnCty3dVeVtki6DLwYa5vhNpOaeu1lwLyPCg8= +bitbucket.org/ausocean/iot v1.2.13 h1:E9LcW3HYqRgJqxNhPJUCfVRvoV2IAU4B7JSDNxB/x2k= +bitbucket.org/ausocean/iot v1.2.13/go.mod h1:Q5FwaOKnCty3dVeVtki6DLwYa5vhNpOaeu1lwLyPCg8= bitbucket.org/ausocean/utils v1.2.11 h1:zA0FOaPjN960ryp8PKCkV5y50uWBYrIxCVnXjwbvPqg= bitbucket.org/ausocean/utils v1.2.11/go.mod h1:uXzX9z3PLemyURTMWRhVI8uLhPX4uuvaaO85v2hcob8= bitbucket.org/ausocean/utils v1.2.12 h1:VnskjWTDM475TnQRhBQE0cNp9D6Y6OELrd4UkD2VVIQ= From 9a0fa098798f5ce6dbf925eaf4c681726f176c13 Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 20 Jan 2020 17:40:45 +1030 Subject: [PATCH 13/46] revid/senders.go,revid.go: Added bitrate calculations to revid. --- revid/revid.go | 12 +++++++---- revid/senders.go | 53 ++++++++++++++++++++++++++++-------------------- 2 files changed, 39 insertions(+), 26 deletions(-) diff --git a/revid/revid.go b/revid/revid.go index 8b1b42da..5cd0fbfc 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -7,6 +7,7 @@ AUTHORS Alan Noble Dan Kortschak Trek Hopton + Scott Barnard LICENSE revid is Copyright (C) 2017-2020 the Australian Ocean Lab (AusOcean) @@ -52,6 +53,7 @@ import ( "bitbucket.org/ausocean/av/filter" "bitbucket.org/ausocean/av/revid/config" "bitbucket.org/ausocean/iot/pi/netsender" + "bitbucket.org/ausocean/utils/bitrate" "bitbucket.org/ausocean/utils/ioext" "bitbucket.org/ausocean/utils/logger" "bitbucket.org/ausocean/utils/vring" @@ -115,6 +117,9 @@ type Revid struct { // err will channel errors from revid routines to the handle errors routine. err chan error + + // bitrate is used for bitrate calculations. + bitrate bitrate.Calculator } // New returns a pointer to a new Revid with the desired configuration, and/or @@ -148,10 +153,8 @@ func (r *Revid) handleErrors() { } // Bitrate returns the result of the most recent bitrate check. -// -// TODO: get this working again. func (r *Revid) Bitrate() int { - return -1 + return r.bitrate.Bitrate() } // reset swaps the current config of a Revid with the passed @@ -266,7 +269,7 @@ func (r *Revid) setupPipeline(mtsEnc func(dst io.WriteCloser, rate float64) (io. return fmt.Errorf("could not initialise MTS ring buffer: %w", err) } w = newMtsSender( - newHttpSender(r.ns, r.cfg.Logger.Log), + newHttpSender(r.ns, r.cfg.Logger.Log, r.bitrate.Report), r.cfg.Logger.Log, rb, r.cfg.ClipDuration, @@ -295,6 +298,7 @@ func (r *Revid) setupPipeline(mtsEnc func(dst io.WriteCloser, rate float64) (io. rtmpConnectionMaxTries, rb, r.cfg.Logger.Log, + r.bitrate.Report, ) if err != nil { r.cfg.Logger.Log(logger.Warning, pkg+"rtmp connect error", "error", err.Error()) diff --git a/revid/senders.go b/revid/senders.go index 4871ddb5..216b191b 100644 --- a/revid/senders.go +++ b/revid/senders.go @@ -59,21 +59,27 @@ const ( // httpSender provides an implemntation of io.Writer to perform sends to a http // destination. type httpSender struct { - client *netsender.Sender - log func(lvl int8, msg string, args ...interface{}) + client *netsender.Sender + log func(lvl int8, msg string, args ...interface{}) + reportSent func(sent int) } // newHttpSender returns a pointer to a new httpSender. -func newHttpSender(ns *netsender.Sender, log func(lvl int8, msg string, args ...interface{})) *httpSender { +func newHttpSender(ns *netsender.Sender, log func(lvl int8, msg string, args ...interface{}), reportSent func(sent int)) *httpSender { return &httpSender{ - client: ns, - log: log, + client: ns, + log: log, + reportSent: reportSent, } } // Write implements io.Writer. func (s *httpSender) Write(d []byte) (int, error) { - return len(d), httpSend(d, s.client, s.log) + err := httpSend(d, s.client, s.log) + if err == nil { + s.reportSent(len(d)) + } + return len(d), err } func (s *httpSender) Close() error { return nil } @@ -268,17 +274,18 @@ func (s *mtsSender) Close() error { // rtmpSender implements loadSender for a native RTMP destination. type rtmpSender struct { - conn *rtmp.Conn - url string - timeout uint - retries int - log func(lvl int8, msg string, args ...interface{}) - ring *vring.Buffer - done chan struct{} - wg sync.WaitGroup + conn *rtmp.Conn + url string + timeout uint + retries int + log func(lvl int8, msg string, args ...interface{}) + ring *vring.Buffer + done chan struct{} + wg sync.WaitGroup + reportSent func(sent int) } -func newRtmpSender(url string, timeout uint, retries int, rb *vring.Buffer, log func(lvl int8, msg string, args ...interface{})) (*rtmpSender, error) { +func newRtmpSender(url string, timeout uint, retries int, rb *vring.Buffer, log func(lvl int8, msg string, args ...interface{}), reportSent func(sent int)) (*rtmpSender, error) { var conn *rtmp.Conn var err error for n := 0; n < retries; n++ { @@ -292,13 +299,14 @@ func newRtmpSender(url string, timeout uint, retries int, rb *vring.Buffer, log } } s := &rtmpSender{ - conn: conn, - url: url, - timeout: timeout, - retries: retries, - log: log, - ring: rb, - done: make(chan struct{}), + conn: conn, + url: url, + timeout: timeout, + retries: retries, + log: log, + ring: rb, + done: make(chan struct{}), + reportSent: reportSent, } s.wg.Add(1) go s.output() @@ -364,6 +372,7 @@ func (s *rtmpSender) Write(d []byte) (int, error) { if err != nil { s.log(logger.Warning, pkg+"rtmpSender: ring buffer write error", "error", err.Error()) } + s.reportSent(len(d)) return len(d), nil } From cdd74c7a22625e59caf47b18510aa5d7f2c2e484 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 22 Jan 2020 15:26:14 +1030 Subject: [PATCH 14/46] revid/senders.go: reportSent -> report --- revid/senders.go | 54 ++++++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/revid/senders.go b/revid/senders.go index 216b191b..6e74d519 100644 --- a/revid/senders.go +++ b/revid/senders.go @@ -59,17 +59,17 @@ const ( // httpSender provides an implemntation of io.Writer to perform sends to a http // destination. type httpSender struct { - client *netsender.Sender - log func(lvl int8, msg string, args ...interface{}) - reportSent func(sent int) + client *netsender.Sender + log func(lvl int8, msg string, args ...interface{}) + report func(sent int) } // newHttpSender returns a pointer to a new httpSender. -func newHttpSender(ns *netsender.Sender, log func(lvl int8, msg string, args ...interface{}), reportSent func(sent int)) *httpSender { +func newHttpSender(ns *netsender.Sender, log func(lvl int8, msg string, args ...interface{}), report func(sent int)) *httpSender { return &httpSender{ - client: ns, - log: log, - reportSent: reportSent, + client: ns, + log: log, + report: report, } } @@ -77,7 +77,7 @@ func newHttpSender(ns *netsender.Sender, log func(lvl int8, msg string, args ... func (s *httpSender) Write(d []byte) (int, error) { err := httpSend(d, s.client, s.log) if err == nil { - s.reportSent(len(d)) + s.report(len(d)) } return len(d), err } @@ -274,18 +274,18 @@ func (s *mtsSender) Close() error { // rtmpSender implements loadSender for a native RTMP destination. type rtmpSender struct { - conn *rtmp.Conn - url string - timeout uint - retries int - log func(lvl int8, msg string, args ...interface{}) - ring *vring.Buffer - done chan struct{} - wg sync.WaitGroup - reportSent func(sent int) + conn *rtmp.Conn + url string + timeout uint + retries int + log func(lvl int8, msg string, args ...interface{}) + ring *vring.Buffer + done chan struct{} + wg sync.WaitGroup + report func(sent int) } -func newRtmpSender(url string, timeout uint, retries int, rb *vring.Buffer, log func(lvl int8, msg string, args ...interface{}), reportSent func(sent int)) (*rtmpSender, error) { +func newRtmpSender(url string, timeout uint, retries int, rb *vring.Buffer, log func(lvl int8, msg string, args ...interface{}), report func(sent int)) (*rtmpSender, error) { var conn *rtmp.Conn var err error for n := 0; n < retries; n++ { @@ -299,14 +299,14 @@ func newRtmpSender(url string, timeout uint, retries int, rb *vring.Buffer, log } } s := &rtmpSender{ - conn: conn, - url: url, - timeout: timeout, - retries: retries, - log: log, - ring: rb, - done: make(chan struct{}), - reportSent: reportSent, + conn: conn, + url: url, + timeout: timeout, + retries: retries, + log: log, + ring: rb, + done: make(chan struct{}), + report: report, } s.wg.Add(1) go s.output() @@ -372,7 +372,7 @@ func (s *rtmpSender) Write(d []byte) (int, error) { if err != nil { s.log(logger.Warning, pkg+"rtmpSender: ring buffer write error", "error", err.Error()) } - s.reportSent(len(d)) + s.report(len(d)) return len(d), nil } From 1876fe9dea14bbd950f00608565517b9608df5ab Mon Sep 17 00:00:00 2001 From: Scott Date: Thu, 23 Jan 2020 10:07:28 +1030 Subject: [PATCH 15/46] Added bitrate reporting to RTP sender. --- revid/revid.go | 2 +- revid/senders.go | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/revid/revid.go b/revid/revid.go index 5cd0fbfc..1c95f05a 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -276,7 +276,7 @@ func (r *Revid) setupPipeline(mtsEnc func(dst io.WriteCloser, rate float64) (io. ) mtsSenders = append(mtsSenders, w) case config.OutputRTP: - w, err := newRtpSender(r.cfg.RTPAddress, r.cfg.Logger.Log, r.cfg.FrameRate) + w, err := newRtpSender(r.cfg.RTPAddress, r.cfg.Logger.Log, r.cfg.FrameRate, r.bitrate.Report) if err != nil { r.cfg.Logger.Log(logger.Warning, pkg+"rtp connect error", "error", err.Error()) } diff --git a/revid/senders.go b/revid/senders.go index 6e74d519..4929bd40 100644 --- a/revid/senders.go +++ b/revid/senders.go @@ -413,9 +413,10 @@ type rtpSender struct { log func(lvl int8, msg string, args ...interface{}) encoder *rtp.Encoder data []byte + report func(sent int) } -func newRtpSender(addr string, log func(lvl int8, msg string, args ...interface{}), fps uint) (*rtpSender, error) { +func newRtpSender(addr string, log func(lvl int8, msg string, args ...interface{}), fps uint, report func(sent int)) (*rtpSender, error) { conn, err := net.Dial("udp", addr) if err != nil { return nil, err @@ -423,6 +424,7 @@ func newRtpSender(addr string, log func(lvl int8, msg string, args ...interface{ s := &rtpSender{ log: log, encoder: rtp.NewEncoder(conn, int(fps)), + report: report, } return s, nil } @@ -435,6 +437,7 @@ func (s *rtpSender) Write(d []byte) (int, error) { if err != nil { s.log(logger.Warning, pkg+"rtpSender: write error", err.Error()) } + s.report(len(d)) return len(d), nil } From 7419bdff7ad579a373b3983a9c7128ddd47d7ae1 Mon Sep 17 00:00:00 2001 From: Scott Date: Thu, 23 Jan 2020 14:28:21 +1030 Subject: [PATCH 16/46] Using utils v1.2.13 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2ed4c734..dfa6f093 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( bitbucket.org/ausocean/iot v1.2.13 - bitbucket.org/ausocean/utils v1.2.12 + bitbucket.org/ausocean/utils v1.2.13 github.com/Comcast/gots v0.0.0-20190305015453-8d56e473f0f7 github.com/go-audio/audio v0.0.0-20181013203223-7b2a6ca21480 github.com/go-audio/wav v0.0.0-20181013172942-de841e69b884 diff --git a/go.sum b/go.sum index ef1fb9ff..d74fe617 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ bitbucket.org/ausocean/utils v1.2.11 h1:zA0FOaPjN960ryp8PKCkV5y50uWBYrIxCVnXjwbv bitbucket.org/ausocean/utils v1.2.11/go.mod h1:uXzX9z3PLemyURTMWRhVI8uLhPX4uuvaaO85v2hcob8= bitbucket.org/ausocean/utils v1.2.12 h1:VnskjWTDM475TnQRhBQE0cNp9D6Y6OELrd4UkD2VVIQ= bitbucket.org/ausocean/utils v1.2.12/go.mod h1:uXzX9z3PLemyURTMWRhVI8uLhPX4uuvaaO85v2hcob8= +bitbucket.org/ausocean/utils v1.2.13 h1:tUaIywtoMc1+zl1GCVQokX4mL5X7LNHX5O51AgAPrWA= +bitbucket.org/ausocean/utils v1.2.13/go.mod h1:uXzX9z3PLemyURTMWRhVI8uLhPX4uuvaaO85v2hcob8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Comcast/gots v0.0.0-20190305015453-8d56e473f0f7 h1:LdOc9B9Bj6LEsKiXShkLA3/kpxXb6LJpH+ekU2krbzw= From bd52a5e658c6b9555d656830418422bbc3a4ce70 Mon Sep 17 00:00:00 2001 From: Trek H Date: Fri, 24 Jan 2020 19:12:07 +1030 Subject: [PATCH 17/46] mjpeg-player: added back changes that were overwritten --- cmd/mjpeg-player/eventemitter3/index.js | 13 +- cmd/mjpeg-player/url-toolkit/url-toolkit.js | 286 ++++++++++---------- 2 files changed, 140 insertions(+), 159 deletions(-) diff --git a/cmd/mjpeg-player/eventemitter3/index.js b/cmd/mjpeg-player/eventemitter3/index.js index 6ea485c4..7d65945d 100644 --- a/cmd/mjpeg-player/eventemitter3/index.js +++ b/cmd/mjpeg-player/eventemitter3/index.js @@ -10,7 +10,7 @@ var has = Object.prototype.hasOwnProperty * @constructor * @private */ -function Events() {} +function Events() { } // // We try to not inherit from `Object.prototype`. In some engines creating an @@ -185,7 +185,7 @@ EventEmitter.prototype.emit = function emit(event, a1, a2, a3, a4, a5) { case 6: return listeners.fn.call(listeners.context, a1, a2, a3, a4, a5), true; } - for (i = 1, args = new Array(len -1); i < len; i++) { + for (i = 1, args = new Array(len - 1); i < len; i++) { args[i - 1] = arguments[i]; } @@ -203,7 +203,7 @@ EventEmitter.prototype.emit = function emit(event, a1, a2, a3, a4, a5) { case 3: listeners[i].fn.call(listeners[i].context, a1, a2); break; case 4: listeners[i].fn.call(listeners[i].context, a1, a2, a3); break; default: - if (!args) for (j = 1, args = new Array(len -1); j < len; j++) { + if (!args) for (j = 1, args = new Array(len - 1); j < len; j++) { args[j - 1] = arguments[j]; } @@ -328,9 +328,4 @@ EventEmitter.prefixed = prefix; // EventEmitter.EventEmitter = EventEmitter; -// -// Expose the module. -// -if ('undefined' !== typeof module) { - module.exports = EventEmitter; -} +export default EventEmitter; diff --git a/cmd/mjpeg-player/url-toolkit/url-toolkit.js b/cmd/mjpeg-player/url-toolkit/url-toolkit.js index 44337aaa..75c7f78a 100644 --- a/cmd/mjpeg-player/url-toolkit/url-toolkit.js +++ b/cmd/mjpeg-player/url-toolkit/url-toolkit.js @@ -1,163 +1,149 @@ // see https://tools.ietf.org/html/rfc1808 -/* jshint ignore:start */ -(function(root) { -/* jshint ignore:end */ +var URL_REGEX = /^((?:[a-zA-Z0-9+\-.]+:)?)(\/\/[^\/?#]*)?((?:[^\/\?#]*\/)*.*?)??(;.*?)?(\?.*?)?(#.*?)?$/; +var FIRST_SEGMENT_REGEX = /^([^\/?#]*)(.*)$/; +var SLASH_DOT_REGEX = /(?:\/|^)\.(?=\/)/g; +var SLASH_DOT_DOT_REGEX = /(?:\/|^)\.\.\/(?!\.\.\/).*?(?=\/)/g; - var URL_REGEX = /^((?:[a-zA-Z0-9+\-.]+:)?)(\/\/[^\/?#]*)?((?:[^\/\?#]*\/)*.*?)??(;.*?)?(\?.*?)?(#.*?)?$/; - var FIRST_SEGMENT_REGEX = /^([^\/?#]*)(.*)$/; - var SLASH_DOT_REGEX = /(?:\/|^)\.(?=\/)/g; - var SLASH_DOT_DOT_REGEX = /(?:\/|^)\.\.\/(?!\.\.\/).*?(?=\/)/g; - - var URLToolkit = { // jshint ignore:line - // If opts.alwaysNormalize is true then the path will always be normalized even when it starts with / or // - // E.g - // With opts.alwaysNormalize = false (default, spec compliant) - // http://a.com/b/cd + /e/f/../g => http://a.com/e/f/../g - // With opts.alwaysNormalize = true (not spec compliant) - // http://a.com/b/cd + /e/f/../g => http://a.com/e/g - buildAbsoluteURL: function(baseURL, relativeURL, opts) { - opts = opts || {}; - // remove any remaining space and CRLF - baseURL = baseURL.trim(); - relativeURL = relativeURL.trim(); - if (!relativeURL) { - // 2a) If the embedded URL is entirely empty, it inherits the - // entire base URL (i.e., is set equal to the base URL) - // and we are done. - if (!opts.alwaysNormalize) { - return baseURL; - } - var basePartsForNormalise = URLToolkit.parseURL(baseURL); - if (!basePartsForNormalise) { - throw new Error('Error trying to parse base URL.'); - } - basePartsForNormalise.path = URLToolkit.normalizePath(basePartsForNormalise.path); - return URLToolkit.buildURLFromParts(basePartsForNormalise); +var URLToolkit = { // jshint ignore:line + // If opts.alwaysNormalize is true then the path will always be normalized even when it starts with / or // + // E.g + // With opts.alwaysNormalize = false (default, spec compliant) + // http://a.com/b/cd + /e/f/../g => http://a.com/e/f/../g + // With opts.alwaysNormalize = true (not spec compliant) + // http://a.com/b/cd + /e/f/../g => http://a.com/e/g + buildAbsoluteURL: function (baseURL, relativeURL, opts) { + opts = opts || {}; + // remove any remaining space and CRLF + baseURL = baseURL.trim(); + relativeURL = relativeURL.trim(); + if (!relativeURL) { + // 2a) If the embedded URL is entirely empty, it inherits the + // entire base URL (i.e., is set equal to the base URL) + // and we are done. + if (!opts.alwaysNormalize) { + return baseURL; } - var relativeParts = URLToolkit.parseURL(relativeURL); - if (!relativeParts) { - throw new Error('Error trying to parse relative URL.'); - } - if (relativeParts.scheme) { - // 2b) If the embedded URL starts with a scheme name, it is - // interpreted as an absolute URL and we are done. - if (!opts.alwaysNormalize) { - return relativeURL; - } - relativeParts.path = URLToolkit.normalizePath(relativeParts.path); - return URLToolkit.buildURLFromParts(relativeParts); - } - var baseParts = URLToolkit.parseURL(baseURL); - if (!baseParts) { + var basePartsForNormalise = URLToolkit.parseURL(baseURL); + if (!basePartsForNormalise) { throw new Error('Error trying to parse base URL.'); } - if (!baseParts.netLoc && baseParts.path && baseParts.path[0] !== '/') { - // If netLoc missing and path doesn't start with '/', assume everthing before the first '/' is the netLoc - // This causes 'example.com/a' to be handled as '//example.com/a' instead of '/example.com/a' - var pathParts = FIRST_SEGMENT_REGEX.exec(baseParts.path); - baseParts.netLoc = pathParts[1]; - baseParts.path = pathParts[2]; + basePartsForNormalise.path = URLToolkit.normalizePath(basePartsForNormalise.path); + return URLToolkit.buildURLFromParts(basePartsForNormalise); + } + var relativeParts = URLToolkit.parseURL(relativeURL); + if (!relativeParts) { + throw new Error('Error trying to parse relative URL.'); + } + if (relativeParts.scheme) { + // 2b) If the embedded URL starts with a scheme name, it is + // interpreted as an absolute URL and we are done. + if (!opts.alwaysNormalize) { + return relativeURL; } - if (baseParts.netLoc && !baseParts.path) { - baseParts.path = '/'; - } - var builtParts = { - // 2c) Otherwise, the embedded URL inherits the scheme of - // the base URL. - scheme: baseParts.scheme, - netLoc: relativeParts.netLoc, - path: null, - params: relativeParts.params, - query: relativeParts.query, - fragment: relativeParts.fragment - }; - if (!relativeParts.netLoc) { - // 3) If the embedded URL's is non-empty, we skip to - // Step 7. Otherwise, the embedded URL inherits the - // (if any) of the base URL. - builtParts.netLoc = baseParts.netLoc; - // 4) If the embedded URL path is preceded by a slash "/", the - // path is not relative and we skip to Step 7. - if (relativeParts.path[0] !== '/') { - if (!relativeParts.path) { - // 5) If the embedded URL path is empty (and not preceded by a - // slash), then the embedded URL inherits the base URL path - builtParts.path = baseParts.path; - // 5a) if the embedded URL's is non-empty, we skip to - // step 7; otherwise, it inherits the of the base - // URL (if any) and - if (!relativeParts.params) { - builtParts.params = baseParts.params; - // 5b) if the embedded URL's is non-empty, we skip to - // step 7; otherwise, it inherits the of the base - // URL (if any) and we skip to step 7. - if (!relativeParts.query) { - builtParts.query = baseParts.query; - } + relativeParts.path = URLToolkit.normalizePath(relativeParts.path); + return URLToolkit.buildURLFromParts(relativeParts); + } + var baseParts = URLToolkit.parseURL(baseURL); + if (!baseParts) { + throw new Error('Error trying to parse base URL.'); + } + if (!baseParts.netLoc && baseParts.path && baseParts.path[0] !== '/') { + // If netLoc missing and path doesn't start with '/', assume everthing before the first '/' is the netLoc + // This causes 'example.com/a' to be handled as '//example.com/a' instead of '/example.com/a' + var pathParts = FIRST_SEGMENT_REGEX.exec(baseParts.path); + baseParts.netLoc = pathParts[1]; + baseParts.path = pathParts[2]; + } + if (baseParts.netLoc && !baseParts.path) { + baseParts.path = '/'; + } + var builtParts = { + // 2c) Otherwise, the embedded URL inherits the scheme of + // the base URL. + scheme: baseParts.scheme, + netLoc: relativeParts.netLoc, + path: null, + params: relativeParts.params, + query: relativeParts.query, + fragment: relativeParts.fragment + }; + if (!relativeParts.netLoc) { + // 3) If the embedded URL's is non-empty, we skip to + // Step 7. Otherwise, the embedded URL inherits the + // (if any) of the base URL. + builtParts.netLoc = baseParts.netLoc; + // 4) If the embedded URL path is preceded by a slash "/", the + // path is not relative and we skip to Step 7. + if (relativeParts.path[0] !== '/') { + if (!relativeParts.path) { + // 5) If the embedded URL path is empty (and not preceded by a + // slash), then the embedded URL inherits the base URL path + builtParts.path = baseParts.path; + // 5a) if the embedded URL's is non-empty, we skip to + // step 7; otherwise, it inherits the of the base + // URL (if any) and + if (!relativeParts.params) { + builtParts.params = baseParts.params; + // 5b) if the embedded URL's is non-empty, we skip to + // step 7; otherwise, it inherits the of the base + // URL (if any) and we skip to step 7. + if (!relativeParts.query) { + builtParts.query = baseParts.query; } - } else { - // 6) The last segment of the base URL's path (anything - // following the rightmost slash "/", or the entire path if no - // slash is present) is removed and the embedded URL's path is - // appended in its place. - var baseURLPath = baseParts.path; - var newPath = baseURLPath.substring(0, baseURLPath.lastIndexOf('/') + 1) + relativeParts.path; - builtParts.path = URLToolkit.normalizePath(newPath); } + } else { + // 6) The last segment of the base URL's path (anything + // following the rightmost slash "/", or the entire path if no + // slash is present) is removed and the embedded URL's path is + // appended in its place. + var baseURLPath = baseParts.path; + var newPath = baseURLPath.substring(0, baseURLPath.lastIndexOf('/') + 1) + relativeParts.path; + builtParts.path = URLToolkit.normalizePath(newPath); } } - if (builtParts.path === null) { - builtParts.path = opts.alwaysNormalize ? URLToolkit.normalizePath(relativeParts.path) : relativeParts.path; - } - return URLToolkit.buildURLFromParts(builtParts); - }, - parseURL: function(url) { - var parts = URL_REGEX.exec(url); - if (!parts) { - return null; - } - return { - scheme: parts[1] || '', - netLoc: parts[2] || '', - path: parts[3] || '', - params: parts[4] || '', - query: parts[5] || '', - fragment: parts[6] || '' - }; - }, - normalizePath: function(path) { - // The following operations are - // then applied, in order, to the new path: - // 6a) All occurrences of "./", where "." is a complete path - // segment, are removed. - // 6b) If the path ends with "." as a complete path segment, - // that "." is removed. - path = path.split('').reverse().join('').replace(SLASH_DOT_REGEX, ''); - // 6c) All occurrences of "/../", where is a - // complete path segment not equal to "..", are removed. - // Removal of these path segments is performed iteratively, - // removing the leftmost matching pattern on each iteration, - // until no matching pattern remains. - // 6d) If the path ends with "/..", where is a - // complete path segment not equal to "..", that - // "/.." is removed. - while (path.length !== (path = path.replace(SLASH_DOT_DOT_REGEX, '')).length) {} // jshint ignore:line - return path.split('').reverse().join(''); - }, - buildURLFromParts: function(parts) { - return parts.scheme + parts.netLoc + parts.path + parts.params + parts.query + parts.fragment; } - }; + if (builtParts.path === null) { + builtParts.path = opts.alwaysNormalize ? URLToolkit.normalizePath(relativeParts.path) : relativeParts.path; + } + return URLToolkit.buildURLFromParts(builtParts); + }, + parseURL: function (url) { + var parts = URL_REGEX.exec(url); + if (!parts) { + return null; + } + return { + scheme: parts[1] || '', + netLoc: parts[2] || '', + path: parts[3] || '', + params: parts[4] || '', + query: parts[5] || '', + fragment: parts[6] || '' + }; + }, + normalizePath: function (path) { + // The following operations are + // then applied, in order, to the new path: + // 6a) All occurrences of "./", where "." is a complete path + // segment, are removed. + // 6b) If the path ends with "." as a complete path segment, + // that "." is removed. + path = path.split('').reverse().join('').replace(SLASH_DOT_REGEX, ''); + // 6c) All occurrences of "/../", where is a + // complete path segment not equal to "..", are removed. + // Removal of these path segments is performed iteratively, + // removing the leftmost matching pattern on each iteration, + // until no matching pattern remains. + // 6d) If the path ends with "/..", where is a + // complete path segment not equal to "..", that + // "/.." is removed. + while (path.length !== (path = path.replace(SLASH_DOT_DOT_REGEX, '')).length) { } // jshint ignore:line + return path.split('').reverse().join(''); + }, + buildURLFromParts: function (parts) { + return parts.scheme + parts.netLoc + parts.path + parts.params + parts.query + parts.fragment; + } +}; -/* jshint ignore:start */ - if(typeof exports === 'object' && typeof module === 'object') - module.exports = URLToolkit; - else if(typeof define === 'function' && define.amd) - define([], function() { return URLToolkit; }); - else if(typeof exports === 'object') - exports["URLToolkit"] = URLToolkit; - else - root["URLToolkit"] = URLToolkit; -})(this); -/* jshint ignore:end */ +export default URLToolkit; From 6be40c8849d4d389be14a4950690ab543e9ceb37 Mon Sep 17 00:00:00 2001 From: Trek H Date: Fri, 24 Jan 2020 19:17:35 +1030 Subject: [PATCH 18/46] mjpeg-player: added back changes that were overwritten --- .../hlsjs/controller/stream-controller.js | 1326 +---------------- cmd/mjpeg-player/hlsjs/hls.js | 694 +-------- 2 files changed, 84 insertions(+), 1936 deletions(-) diff --git a/cmd/mjpeg-player/hlsjs/controller/stream-controller.js b/cmd/mjpeg-player/hlsjs/controller/stream-controller.js index 092a55bc..31fd0828 100644 --- a/cmd/mjpeg-player/hlsjs/controller/stream-controller.js +++ b/cmd/mjpeg-player/hlsjs/controller/stream-controller.js @@ -1,801 +1,93 @@ +/* +AUTHOR + Trek Hopton + +LICENSE + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see http://www.gnu.org/licenses. + + For hls.js Copyright notice and license, see LICENSE file. +*/ + /* * Stream Controller */ -import BinarySearch from '../utils/binary-search'; -import { BufferHelper } from '../utils/buffer-helper'; -import Demuxer from '../demux/demuxer'; -import Event from '../events'; -import { FragmentState } from './fragment-tracker'; -import { ElementaryStreamTypes } from '../loader/fragment'; -import { PlaylistLevelType } from '../types/loader'; -import * as LevelHelper from './level-helper'; -import TimeRanges from '../utils/time-ranges'; -import { ErrorDetails } from '../errors'; -import { logger } from '../utils/logger'; -import { alignStream } from '../utils/discontinuities'; -import { findFragmentByPDT, findFragmentByPTS } from './fragment-finders'; -import GapController from './gap-controller'; -import BaseStreamController, { State } from './base-stream-controller'; +import Event from '../events.js'; +import EventHandler from '../event-handler.js'; -const TICK_INTERVAL = 100; // how often to tick in ms - -class StreamController extends BaseStreamController { - constructor (hls, fragmentTracker) { +class StreamController extends EventHandler { + constructor(hls) { super(hls, - Event.MEDIA_ATTACHED, - Event.MEDIA_DETACHING, - Event.MANIFEST_LOADING, - Event.MANIFEST_PARSED, Event.LEVEL_LOADED, - Event.KEY_LOADED, - Event.FRAG_LOADED, - Event.FRAG_LOAD_EMERGENCY_ABORTED, - Event.FRAG_PARSING_INIT_SEGMENT, - Event.FRAG_PARSING_DATA, - Event.FRAG_PARSED, - Event.ERROR, - Event.AUDIO_TRACK_SWITCHING, - Event.AUDIO_TRACK_SWITCHED, - Event.BUFFER_CREATED, - Event.BUFFER_APPENDED, - Event.BUFFER_FLUSHED); - - this.fragmentTracker = fragmentTracker; + Event.FRAG_LOADED); + this.hls = hls; this.config = hls.config; this.audioCodecSwap = false; - this._state = State.STOPPED; this.stallReported = false; this.gapController = null; + this.currentFrag = 0; } - startLoad (startPosition) { - if (this.levels) { - let lastCurrentTime = this.lastCurrentTime, hls = this.hls; - this.stopLoad(); - this.setInterval(TICK_INTERVAL); - this.level = -1; - this.fragLoadError = 0; - if (!this.startFragRequested) { - // determine load level - let startLevel = hls.startLevel; - if (startLevel === -1) { - // -1 : guess start Level by doing a bitrate test by loading first fragment of lowest quality level - startLevel = 0; - this.bitrateTest = true; - } - // set new level to playlist loader : this will trigger start level load - // hls.nextLoadLevel remains until it is set to a new value or until a new frag is successfully loaded - this.level = hls.nextLoadLevel = startLevel; - this.loadedmetadata = false; - } - // if startPosition undefined but lastCurrentTime set, set startPosition to last currentTime - if (lastCurrentTime > 0 && startPosition === -1) { - logger.log(`override startPosition with lastCurrentTime @${lastCurrentTime.toFixed(3)}`); - startPosition = lastCurrentTime; - } - this.state = State.IDLE; - this.nextLoadPosition = this.startPosition = this.lastCurrentTime = startPosition; - this.tick(); - } else { - this.forceStartLoad = true; - this.state = State.STOPPED; - } + _fetchPayloadOrEos(levelDetails) { + this.fragments = levelDetails.fragments; + this._loadFragment(); } - stopLoad () { - this.forceStartLoad = false; - super.stopLoad(); - } - - doTick () { - switch (this.state) { - case State.BUFFER_FLUSHING: - // in buffer flushing state, reset fragLoadError counter - this.fragLoadError = 0; - break; - case State.IDLE: - this._doTickIdle(); - break; - case State.WAITING_LEVEL: - var level = this.levels[this.level]; - // check if playlist is already loaded - if (level && level.details) { - this.state = State.IDLE; - } - - break; - case State.FRAG_LOADING_WAITING_RETRY: - var now = window.performance.now(); - var retryDate = this.retryDate; - // if current time is gt than retryDate, or if media seeking let's switch to IDLE state to retry loading - if (!retryDate || (now >= retryDate) || (this.media && this.media.seeking)) { - logger.log('mediaController: retryDate reached, switch back to IDLE state'); - this.state = State.IDLE; - } - break; - case State.ERROR: - case State.STOPPED: - case State.FRAG_LOADING: - case State.PARSING: - case State.PARSED: - case State.ENDED: - break; - default: - break; - } - // check buffer - this._checkBuffer(); - // check/update current fragment - this._checkFragmentChanged(); - } - - // Ironically the "idle" state is the on we do the most logic in it seems .... - // NOTE: Maybe we could rather schedule a check for buffer length after half of the currently - // played segment, or on pause/play/seek instead of naively checking every 100ms? - _doTickIdle () { - const hls = this.hls, - config = hls.config, - media = this.media; - - // if start level not parsed yet OR - // if video not attached AND start fragment already requested OR start frag prefetch disable - // exit loop, as we either need more info (level not parsed) or we need media to be attached to load new fragment - if (this.levelLastLoaded === undefined || ( - !media && (this.startFragRequested || !config.startFragPrefetch))) { + _loadFragment() { + let fragLen = this.fragments.length; + if (this.currentFrag >= fragLen) { return; } - - // if we have not yet loaded any fragment, start loading from start position - let pos; - if (this.loadedmetadata) { - pos = media.currentTime; - } else { - pos = this.nextLoadPosition; - } - - // determine next load level - let level = hls.nextLoadLevel, - levelInfo = this.levels[level]; - - if (!levelInfo) { - return; - } - - let levelBitrate = levelInfo.bitrate, - maxBufLen; - - // compute max Buffer Length that we could get from this load level, based on level bitrate. - if (levelBitrate) { - maxBufLen = Math.max(8 * config.maxBufferSize / levelBitrate, config.maxBufferLength); - } else { - maxBufLen = config.maxBufferLength; - } - - maxBufLen = Math.min(maxBufLen, config.maxMaxBufferLength); - - // determine next candidate fragment to be loaded, based on current position and end of buffer position - // ensure up to `config.maxMaxBufferLength` of buffer upfront - - const bufferInfo = BufferHelper.bufferInfo(this.mediaBuffer ? this.mediaBuffer : media, pos, config.maxBufferHole), - bufferLen = bufferInfo.len; - // Stay idle if we are still with buffer margins - if (bufferLen >= maxBufLen) { - return; - } - - // if buffer length is less than maxBufLen try to load a new fragment ... - logger.trace(`buffer length of ${bufferLen.toFixed(3)} is below max of ${maxBufLen.toFixed(3)}. checking for more payload ...`); - - // set next load level : this will trigger a playlist load if needed - this.level = hls.nextLoadLevel = level; - - const levelDetails = levelInfo.details; - // if level info not retrieved yet, switch state and wait for level retrieval - // if live playlist, ensure that new playlist has been refreshed to avoid loading/try to load - // a useless and outdated fragment (that might even introduce load error if it is already out of the live playlist) - if (!levelDetails || (levelDetails.live && this.levelLastLoaded !== level)) { - this.state = State.WAITING_LEVEL; - return; - } - - if (this._streamEnded(bufferInfo, levelDetails)) { - const data = {}; - if (this.altAudio) { - data.type = 'video'; - } - - this.hls.trigger(Event.BUFFER_EOS, data); - this.state = State.ENDED; - return; - } - // if we have the levelDetails for the selected variant, lets continue enrichen our stream (load keys/fragments or trigger EOS, etc..) - this._fetchPayloadOrEos(pos, bufferInfo, levelDetails); + this.hls.trigger(Event.FRAG_LOADING, { frag: this.fragments[this.currentFrag++] }); } - _fetchPayloadOrEos (pos, bufferInfo, levelDetails) { - const fragPrevious = this.fragPrevious, - level = this.level, - fragments = levelDetails.fragments, - fragLen = fragments.length; - - // empty playlist - if (fragLen === 0) { - return; - } - - // find fragment index, contiguous with end of buffer position - let start = fragments[0].start, - end = fragments[fragLen - 1].start + fragments[fragLen - 1].duration, - bufferEnd = bufferInfo.end, - frag; - - if (levelDetails.initSegment && !levelDetails.initSegment.data) { - frag = levelDetails.initSegment; - } else { - // in case of live playlist we need to ensure that requested position is not located before playlist start - if (levelDetails.live) { - let initialLiveManifestSize = this.config.initialLiveManifestSize; - if (fragLen < initialLiveManifestSize) { - logger.warn(`Can not start playback of a level, reason: not enough fragments ${fragLen} < ${initialLiveManifestSize}`); - return; - } - - frag = this._ensureFragmentAtLivePoint(levelDetails, bufferEnd, start, end, fragPrevious, fragments, fragLen); - // if it explicitely returns null don't load any fragment and exit function now - if (frag === null) { - return; - } - } else { - // VoD playlist: if bufferEnd before start of playlist, load first fragment - if (bufferEnd < start) { - frag = fragments[0]; - } - } - } - if (!frag) { - frag = this._findFragment(start, fragPrevious, fragLen, fragments, bufferEnd, end, levelDetails); - } - - if (frag) { - if (frag.encrypted) { - logger.log(`Loading key for ${frag.sn} of [${levelDetails.startSN} ,${levelDetails.endSN}],level ${level}`); - this._loadKey(frag); - } else { - logger.log(`Loading ${frag.sn} of [${levelDetails.startSN} ,${levelDetails.endSN}],level ${level}, currentTime:${pos.toFixed(3)},bufferEnd:${bufferEnd.toFixed(3)}`); - this._loadFragment(frag); - } - } - } - - _ensureFragmentAtLivePoint (levelDetails, bufferEnd, start, end, fragPrevious, fragments, fragLen) { - const config = this.hls.config, media = this.media; - - let frag; - - // check if requested position is within seekable boundaries : - // logger.log(`start/pos/bufEnd/seeking:${start.toFixed(3)}/${pos.toFixed(3)}/${bufferEnd.toFixed(3)}/${this.media.seeking}`); - let maxLatency = config.liveMaxLatencyDuration !== undefined ? config.liveMaxLatencyDuration : config.liveMaxLatencyDurationCount * levelDetails.targetduration; - - if (bufferEnd < Math.max(start - config.maxFragLookUpTolerance, end - maxLatency)) { - let liveSyncPosition = this.liveSyncPosition = this.computeLivePosition(start, levelDetails); - logger.log(`buffer end: ${bufferEnd.toFixed(3)} is located too far from the end of live sliding playlist, reset currentTime to : ${liveSyncPosition.toFixed(3)}`); - bufferEnd = liveSyncPosition; - if (media && media.readyState && media.duration > liveSyncPosition) { - media.currentTime = liveSyncPosition; - } - - this.nextLoadPosition = liveSyncPosition; - } - - // if end of buffer greater than live edge, don't load any fragment - // this could happen if live playlist intermittently slides in the past. - // level 1 loaded [182580161,182580167] - // level 1 loaded [182580162,182580169] - // Loading 182580168 of [182580162 ,182580169],level 1 .. - // Loading 182580169 of [182580162 ,182580169],level 1 .. - // level 1 loaded [182580162,182580168] <============= here we should have bufferEnd > end. in that case break to avoid reloading 182580168 - // level 1 loaded [182580164,182580171] - // - // don't return null in case media not loaded yet (readystate === 0) - if (levelDetails.PTSKnown && bufferEnd > end && media && media.readyState) { - return null; - } - - if (this.startFragRequested && !levelDetails.PTSKnown) { - /* we are switching level on live playlist, but we don't have any PTS info for that quality level ... - try to load frag matching with next SN. - even if SN are not synchronized between playlists, loading this frag will help us - compute playlist sliding and find the right one after in case it was not the right consecutive one */ - if (fragPrevious) { - if (levelDetails.hasProgramDateTime) { - // Relies on PDT in order to switch bitrates (Support EXT-X-DISCONTINUITY without EXT-X-DISCONTINUITY-SEQUENCE) - logger.log(`live playlist, switching playlist, load frag with same PDT: ${fragPrevious.programDateTime}`); - frag = findFragmentByPDT(fragments, fragPrevious.endProgramDateTime, config.maxFragLookUpTolerance); - } else { - // Uses buffer and sequence number to calculate switch segment (required if using EXT-X-DISCONTINUITY-SEQUENCE) - const targetSN = fragPrevious.sn + 1; - if (targetSN >= levelDetails.startSN && targetSN <= levelDetails.endSN) { - const fragNext = fragments[targetSN - levelDetails.startSN]; - if (fragPrevious.cc === fragNext.cc) { - frag = fragNext; - logger.log(`live playlist, switching playlist, load frag with next SN: ${frag.sn}`); - } - } - // next frag SN not available (or not with same continuity counter) - // look for a frag sharing the same CC - if (!frag) { - frag = BinarySearch.search(fragments, function (frag) { - return fragPrevious.cc - frag.cc; - }); - if (frag) { - logger.log(`live playlist, switching playlist, load frag with same CC: ${frag.sn}`); - } - } - } - } - if (!frag) { - /* we have no idea about which fragment should be loaded. - so let's load mid fragment. it will help computing playlist sliding and find the right one - */ - frag = fragments[Math.min(fragLen - 1, Math.round(fragLen / 2))]; - logger.log(`live playlist, switching playlist, unknown, load middle frag : ${frag.sn}`); - } - } - - return frag; - } - - _findFragment (start, fragPrevious, fragLen, fragments, bufferEnd, end, levelDetails) { - const config = this.hls.config; - let frag; - - if (bufferEnd < end) { - const lookupTolerance = (bufferEnd > end - config.maxFragLookUpTolerance) ? 0 : config.maxFragLookUpTolerance; - // Remove the tolerance if it would put the bufferEnd past the actual end of stream - // Uses buffer and sequence number to calculate switch segment (required if using EXT-X-DISCONTINUITY-SEQUENCE) - frag = findFragmentByPTS(fragPrevious, fragments, bufferEnd, lookupTolerance); - } else { - // reach end of playlist - frag = fragments[fragLen - 1]; - } - if (frag) { - const curSNIdx = frag.sn - levelDetails.startSN; - const sameLevel = fragPrevious && frag.level === fragPrevious.level; - const prevFrag = fragments[curSNIdx - 1]; - const nextFrag = fragments[curSNIdx + 1]; - // logger.log('find SN matching with pos:' + bufferEnd + ':' + frag.sn); - if (fragPrevious && frag.sn === fragPrevious.sn) { - if (sameLevel && !frag.backtracked) { - if (frag.sn < levelDetails.endSN) { - let deltaPTS = fragPrevious.deltaPTS; - // if there is a significant delta between audio and video, larger than max allowed hole, - // and if previous remuxed fragment did not start with a keyframe. (fragPrevious.dropped) - // let's try to load previous fragment again to get last keyframe - // then we will reload again current fragment (that way we should be able to fill the buffer hole ...) - if (deltaPTS && deltaPTS > config.maxBufferHole && fragPrevious.dropped && curSNIdx) { - frag = prevFrag; - logger.warn('SN just loaded, with large PTS gap between audio and video, maybe frag is not starting with a keyframe ? load previous one to try to overcome this'); - } else { - frag = nextFrag; - logger.log(`SN just loaded, load next one: ${frag.sn}`, frag); - } - } else { - frag = null; - } - } else if (frag.backtracked) { - // Only backtrack a max of 1 consecutive fragment to prevent sliding back too far when little or no frags start with keyframes - if (nextFrag && nextFrag.backtracked) { - logger.warn(`Already backtracked from fragment ${nextFrag.sn}, will not backtrack to fragment ${frag.sn}. Loading fragment ${nextFrag.sn}`); - frag = nextFrag; - } else { - // If a fragment has dropped frames and it's in a same level/sequence, load the previous fragment to try and find the keyframe - // Reset the dropped count now since it won't be reset until we parse the fragment again, which prevents infinite backtracking on the same segment - logger.warn('Loaded fragment with dropped frames, backtracking 1 segment to find a keyframe'); - frag.dropped = 0; - if (prevFrag) { - frag = prevFrag; - frag.backtracked = true; - } else if (curSNIdx) { - // can't backtrack on very first fragment - frag = null; - } - } - } - } - } - return frag; - } - - _loadKey (frag) { - this.state = State.KEY_LOADING; - this.hls.trigger(Event.KEY_LOADING, { frag }); - } - - _loadFragment (frag) { - // Check if fragment is not loaded - let fragState = this.fragmentTracker.getState(frag); - - this.fragCurrent = frag; - if (frag.sn !== 'initSegment') { - this.startFragRequested = true; - } - // Don't update nextLoadPosition for fragments which are not buffered - if (Number.isFinite(frag.sn) && !frag.bitrateTest) { - this.nextLoadPosition = frag.start + frag.duration; - } - - // Allow backtracked fragments to load - if (frag.backtracked || fragState === FragmentState.NOT_LOADED || fragState === FragmentState.PARTIAL) { - frag.autoLevel = this.hls.autoLevelEnabled; - frag.bitrateTest = this.bitrateTest; - - this.hls.trigger(Event.FRAG_LOADING, { frag }); - // lazy demuxer init, as this could take some time ... do it during frag loading - if (!this.demuxer) { - this.demuxer = new Demuxer(this.hls, 'main'); - } - - this.state = State.FRAG_LOADING; - } else if (fragState === FragmentState.APPENDING) { - // Lower the buffer size and try again - if (this._reduceMaxBufferLength(frag.duration)) { - this.fragmentTracker.removeFragment(frag); - } - } - } - - set state (nextState) { - if (this.state !== nextState) { - const previousState = this.state; - this._state = nextState; - logger.log(`main stream:${previousState}->${nextState}`); - this.hls.trigger(Event.STREAM_STATE_TRANSITION, { previousState, nextState }); - } - } - - get state () { - return this._state; - } - - getBufferedFrag (position) { - return this.fragmentTracker.getBufferedFrag(position, PlaylistLevelType.MAIN); - } - - get currentLevel () { - let media = this.media; - if (media) { - const frag = this.getBufferedFrag(media.currentTime); - if (frag) { - return frag.level; - } - } - return -1; - } - - get nextBufferedFrag () { - let media = this.media; - if (media) { - // first get end range of current fragment - return this.followingBufferedFrag(this.getBufferedFrag(media.currentTime)); - } else { - return null; - } - } - - followingBufferedFrag (frag) { - if (frag) { - // try to get range of next fragment (500ms after this range) - return this.getBufferedFrag(frag.endPTS + 0.5); - } - return null; - } - - get nextLevel () { - const frag = this.nextBufferedFrag; - if (frag) { - return frag.level; - } else { - return -1; - } - } - - _checkFragmentChanged () { - let fragPlayingCurrent, currentTime, video = this.media; - if (video && video.readyState && video.seeking === false) { - currentTime = video.currentTime; - /* if video element is in seeked state, currentTime can only increase. - (assuming that playback rate is positive ...) - As sometimes currentTime jumps back to zero after a - media decode error, check this, to avoid seeking back to - wrong position after a media decode error - */ - if (currentTime > this.lastCurrentTime) { - this.lastCurrentTime = currentTime; - } - - if (BufferHelper.isBuffered(video, currentTime)) { - fragPlayingCurrent = this.getBufferedFrag(currentTime); - } else if (BufferHelper.isBuffered(video, currentTime + 0.1)) { - /* ensure that FRAG_CHANGED event is triggered at startup, - when first video frame is displayed and playback is paused. - add a tolerance of 100ms, in case current position is not buffered, - check if current pos+100ms is buffered and use that buffer range - for FRAG_CHANGED event reporting */ - fragPlayingCurrent = this.getBufferedFrag(currentTime + 0.1); - } - if (fragPlayingCurrent) { - let fragPlaying = fragPlayingCurrent; - if (fragPlaying !== this.fragPlaying) { - this.hls.trigger(Event.FRAG_CHANGED, { frag: fragPlaying }); - const fragPlayingLevel = fragPlaying.level; - if (!this.fragPlaying || this.fragPlaying.level !== fragPlayingLevel) { - this.hls.trigger(Event.LEVEL_SWITCHED, { level: fragPlayingLevel }); - } - - this.fragPlaying = fragPlaying; - } - } - } - } - - /* - on immediate level switch : - - pause playback if playing - - cancel any pending load request - - and trigger a buffer flush - */ - immediateLevelSwitch () { - logger.log('immediateLevelSwitch'); - if (!this.immediateSwitch) { - this.immediateSwitch = true; - let media = this.media, previouslyPaused; - if (media) { - previouslyPaused = media.paused; - media.pause(); - } else { - // don't restart playback after instant level switch in case media not attached - previouslyPaused = true; - } - this.previouslyPaused = previouslyPaused; - } - let fragCurrent = this.fragCurrent; - if (fragCurrent && fragCurrent.loader) { - fragCurrent.loader.abort(); - } - - this.fragCurrent = null; - // flush everything - this.flushMainBuffer(0, Number.POSITIVE_INFINITY); - } - - /** - * on immediate level switch end, after new fragment has been buffered: - * - nudge video decoder by slightly adjusting video currentTime (if currentTime buffered) - * - resume the playback if needed - */ - immediateLevelSwitchEnd () { - const media = this.media; - if (media && media.buffered.length) { - this.immediateSwitch = false; - if (BufferHelper.isBuffered(media, media.currentTime)) { - // only nudge if currentTime is buffered - media.currentTime -= 0.0001; - } - if (!this.previouslyPaused) { - media.play(); - } - } - } - - /** - * try to switch ASAP without breaking video playback: - * in order to ensure smooth but quick level switching, - * we need to find the next flushable buffer range - * we should take into account new segment fetch time - */ - nextLevelSwitch () { - const media = this.media; - // ensure that media is defined and that metadata are available (to retrieve currentTime) - if (media && media.readyState) { - let fetchdelay, fragPlayingCurrent, nextBufferedFrag; - fragPlayingCurrent = this.getBufferedFrag(media.currentTime); - if (fragPlayingCurrent && fragPlayingCurrent.startPTS > 1) { - // flush buffer preceding current fragment (flush until current fragment start offset) - // minus 1s to avoid video freezing, that could happen if we flush keyframe of current video ... - this.flushMainBuffer(0, fragPlayingCurrent.startPTS - 1); - } - if (!media.paused) { - // add a safety delay of 1s - let nextLevelId = this.hls.nextLoadLevel, nextLevel = this.levels[nextLevelId], fragLastKbps = this.fragLastKbps; - if (fragLastKbps && this.fragCurrent) { - fetchdelay = this.fragCurrent.duration * nextLevel.bitrate / (1000 * fragLastKbps) + 1; - } else { - fetchdelay = 0; - } - } else { - fetchdelay = 0; - } - // logger.log('fetchdelay:'+fetchdelay); - // find buffer range that will be reached once new fragment will be fetched - nextBufferedFrag = this.getBufferedFrag(media.currentTime + fetchdelay); - if (nextBufferedFrag) { - // we can flush buffer range following this one without stalling playback - nextBufferedFrag = this.followingBufferedFrag(nextBufferedFrag); - if (nextBufferedFrag) { - // if we are here, we can also cancel any loading/demuxing in progress, as they are useless - let fragCurrent = this.fragCurrent; - if (fragCurrent && fragCurrent.loader) { - fragCurrent.loader.abort(); - } - - this.fragCurrent = null; - // start flush position is the start PTS of next buffered frag. - // we use frag.naxStartPTS which is max(audio startPTS, video startPTS). - // in case there is a small PTS Delta between audio and video, using maxStartPTS avoids flushing last samples from current fragment - this.flushMainBuffer(nextBufferedFrag.maxStartPTS, Number.POSITIVE_INFINITY); - } - } - } - } - - flushMainBuffer (startOffset, endOffset) { - this.state = State.BUFFER_FLUSHING; - let flushScope = { startOffset: startOffset, endOffset: endOffset }; - // if alternate audio tracks are used, only flush video, otherwise flush everything - if (this.altAudio) { - flushScope.type = 'video'; - } - - this.hls.trigger(Event.BUFFER_FLUSHING, flushScope); - } - - onMediaAttached (data) { - let media = this.media = this.mediaBuffer = data.media; - this.onvseeking = this.onMediaSeeking.bind(this); - this.onvseeked = this.onMediaSeeked.bind(this); - this.onvended = this.onMediaEnded.bind(this); - media.addEventListener('seeking', this.onvseeking); - media.addEventListener('seeked', this.onvseeked); - media.addEventListener('ended', this.onvended); - let config = this.config; - if (this.levels && config.autoStartLoad) { - this.hls.startLoad(config.startPosition); - } - - this.gapController = new GapController(config, media, this.fragmentTracker, this.hls); - } - - onMediaDetaching () { - let media = this.media; - if (media && media.ended) { - logger.log('MSE detaching and video ended, reset startPosition'); - this.startPosition = this.lastCurrentTime = 0; - } - - // reset fragment backtracked flag - let levels = this.levels; - if (levels) { - levels.forEach(level => { - if (level.details) { - level.details.fragments.forEach(fragment => { - fragment.backtracked = undefined; - }); - } - }); - } - // remove video listeners - if (media) { - media.removeEventListener('seeking', this.onvseeking); - media.removeEventListener('seeked', this.onvseeked); - media.removeEventListener('ended', this.onvended); - this.onvseeking = this.onvseeked = this.onvended = null; - } - this.media = this.mediaBuffer = null; - this.loadedmetadata = false; - this.stopLoad(); - } - - onMediaSeeked () { - const media = this.media, currentTime = media ? media.currentTime : undefined; - if (Number.isFinite(currentTime)) { - logger.log(`media seeked to ${currentTime.toFixed(3)}`); - } - - // tick to speed up FRAGMENT_PLAYING triggering - this.tick(); - } - - onManifestLoading () { - // reset buffer on manifest loading - logger.log('trigger BUFFER_RESET'); - this.hls.trigger(Event.BUFFER_RESET); - this.fragmentTracker.removeAllFragments(); - this.stalled = false; - this.startPosition = this.lastCurrentTime = 0; - } - - onManifestParsed (data) { - let aac = false, heaac = false, codec; - data.levels.forEach(level => { - // detect if we have different kind of audio codecs used amongst playlists - codec = level.audioCodec; - if (codec) { - if (codec.indexOf('mp4a.40.2') !== -1) { - aac = true; - } - - if (codec.indexOf('mp4a.40.5') !== -1) { - heaac = true; - } - } - }); - this.audioCodecSwitch = (aac && heaac); - if (this.audioCodecSwitch) { - logger.log('both AAC/HE-AAC audio found in levels; declaring level codec as HE-AAC'); - } - - this.levels = data.levels; - this.startFragRequested = false; - let config = this.config; - if (config.autoStartLoad || this.forceStartLoad) { - this.hls.startLoad(config.startPosition); - } - } - - onLevelLoaded (data) { + onLevelLoaded(data) { const newDetails = data.details; const newLevelId = data.level; - const lastLevel = this.levels[this.levelLastLoaded]; - const curLevel = this.levels[newLevelId]; + const levelDetails = data.details; const duration = newDetails.totalduration; let sliding = 0; - logger.log(`level ${newLevelId} loaded [${newDetails.startSN},${newDetails.endSN}],duration:${duration}`); + console.log(`level ${newLevelId} loaded [${newDetails.startSN},${newDetails.endSN}],duration:${duration}`); if (newDetails.live) { - let curDetails = curLevel.details; - if (curDetails && newDetails.fragments.length > 0) { - // we already have details for that level, merge them - LevelHelper.mergeDetails(curDetails, newDetails); - sliding = newDetails.fragments[0].start; - this.liveSyncPosition = this.computeLivePosition(sliding, curDetails); - if (newDetails.PTSKnown && Number.isFinite(sliding)) { - logger.log(`live playlist sliding:${sliding.toFixed(3)}`); - } else { - logger.log('live playlist - outdated PTS, unknown sliding'); - alignStream(this.fragPrevious, lastLevel, newDetails); - } - } else { - logger.log('live playlist - first load, unknown sliding'); - newDetails.PTSKnown = false; - alignStream(this.fragPrevious, lastLevel, newDetails); - } + console.log("handling of this case is not implemented"); } else { newDetails.PTSKnown = false; } // override level info - curLevel.details = newDetails; this.levelLastLoaded = newLevelId; this.hls.trigger(Event.LEVEL_UPDATED, { details: newDetails, level: newLevelId }); if (this.startFragRequested === false) { - // compute start position if set to -1. use it straight away if value is defined + // compute start position if set to -1. use it straight away if value is defined if (this.startPosition === -1 || this.lastCurrentTime === -1) { // first, check if start time offset has been set in playlist, if yes, use this value let startTimeOffset = newDetails.startTimeOffset; if (Number.isFinite(startTimeOffset)) { if (startTimeOffset < 0) { - logger.log(`negative start time offset ${startTimeOffset}, count from end of last fragment`); + console.log(`negative start time offset ${startTimeOffset}, count from end of last fragment`); startTimeOffset = sliding + duration + startTimeOffset; } - logger.log(`start time offset found in playlist, adjust startPosition to ${startTimeOffset}`); + console.log(`start time offset found in playlist, adjust startPosition to ${startTimeOffset}`); this.startPosition = startTimeOffset; } else { // if live playlist, set start position to be fragment N-this.config.liveSyncDurationCount (usually 3) if (newDetails.live) { - this.startPosition = this.computeLivePosition(sliding, newDetails); - logger.log(`configure startPosition to ${this.startPosition}`); + console.log("handling of this case is not implemented"); } else { this.startPosition = 0; } @@ -804,534 +96,14 @@ class StreamController extends BaseStreamController { } this.nextLoadPosition = this.startPosition; } - // only switch batck to IDLE state if we were waiting for level to start downloading a new fragment - if (this.state === State.WAITING_LEVEL) { - this.state = State.IDLE; - } - // trigger handler right now - this.tick(); + this._fetchPayloadOrEos(levelDetails); } - onKeyLoaded () { - if (this.state === State.KEY_LOADING) { - this.state = State.IDLE; - this.tick(); - } - } + onFragLoaded(data) { + this.hls.loadSuccess(data.payload); + this._loadFragment(); - onFragLoaded (data) { - const { fragCurrent, hls, levels, media } = this; - const fragLoaded = data.frag; - if (this.state === State.FRAG_LOADING && - fragCurrent && - fragLoaded.type === 'main' && - fragLoaded.level === fragCurrent.level && - fragLoaded.sn === fragCurrent.sn) { - const stats = data.stats; - const currentLevel = levels[fragCurrent.level]; - const details = currentLevel.details; - // reset frag bitrate test in any case after frag loaded event - // if this frag was loaded to perform a bitrate test AND if hls.nextLoadLevel is greater than 0 - // then this means that we should be able to load a fragment at a higher quality level - this.bitrateTest = false; - this.stats = stats; - - logger.log(`Loaded ${fragCurrent.sn} of [${details.startSN} ,${details.endSN}],level ${fragCurrent.level}`); - if (fragLoaded.bitrateTest && hls.nextLoadLevel) { - // switch back to IDLE state ... we just loaded a fragment to determine adequate start bitrate and initialize autoswitch algo - this.state = State.IDLE; - this.startFragRequested = false; - stats.tparsed = stats.tbuffered = window.performance.now(); - hls.trigger(Event.FRAG_BUFFERED, { stats: stats, frag: fragCurrent, id: 'main' }); - this.tick(); - } else if (fragLoaded.sn === 'initSegment') { - this.state = State.IDLE; - stats.tparsed = stats.tbuffered = window.performance.now(); - details.initSegment.data = data.payload; - hls.trigger(Event.FRAG_BUFFERED, { stats: stats, frag: fragCurrent, id: 'main' }); - this.tick(); - } else { - logger.log(`Parsing ${fragCurrent.sn} of [${details.startSN} ,${details.endSN}],level ${fragCurrent.level}, cc ${fragCurrent.cc}`); - this.state = State.PARSING; - this.pendingBuffering = true; - this.appended = false; - - // Bitrate test frags are not usually buffered so the fragment tracker ignores them. If Hls.js decides to buffer - // it (and therefore ends up at this line), then the fragment tracker needs to be manually informed. - if (fragLoaded.bitrateTest) { - fragLoaded.bitrateTest = false; - this.fragmentTracker.onFragLoaded({ - frag: fragLoaded - }); - } - - // time Offset is accurate if level PTS is known, or if playlist is not sliding (not live) and if media is not seeking (this is to overcome potential timestamp drifts between playlists and fragments) - const accurateTimeOffset = !(media && media.seeking) && (details.PTSKnown || !details.live); - const initSegmentData = details.initSegment ? details.initSegment.data : []; - const audioCodec = this._getAudioCodec(currentLevel); - - // transmux the MPEG-TS data to ISO-BMFF segments - const demuxer = this.demuxer = this.demuxer || new Demuxer(this.hls, 'main'); - demuxer.push( - data.payload, - initSegmentData, - audioCodec, - currentLevel.videoCodec, - fragCurrent, - details.totalduration, - accurateTimeOffset - ); - } - } - this.fragLoadError = 0; - } - - onFragParsingInitSegment (data) { - const fragCurrent = this.fragCurrent; - const fragNew = data.frag; - - if (fragCurrent && - data.id === 'main' && - fragNew.sn === fragCurrent.sn && - fragNew.level === fragCurrent.level && - this.state === State.PARSING) { - let tracks = data.tracks, trackName, track; - - // if audio track is expected to come from audio stream controller, discard any coming from main - if (tracks.audio && this.altAudio) { - delete tracks.audio; - } - - // include levelCodec in audio and video tracks - track = tracks.audio; - if (track) { - let audioCodec = this.levels[this.level].audioCodec, - ua = navigator.userAgent.toLowerCase(); - if (audioCodec && this.audioCodecSwap) { - logger.log('swapping playlist audio codec'); - if (audioCodec.indexOf('mp4a.40.5') !== -1) { - audioCodec = 'mp4a.40.2'; - } else { - audioCodec = 'mp4a.40.5'; - } - } - // in case AAC and HE-AAC audio codecs are signalled in manifest - // force HE-AAC , as it seems that most browsers prefers that way, - // except for mono streams OR on FF - // these conditions might need to be reviewed ... - if (this.audioCodecSwitch) { - // don't force HE-AAC if mono stream - if (track.metadata.channelCount !== 1 && - // don't force HE-AAC if firefox - ua.indexOf('firefox') === -1) { - audioCodec = 'mp4a.40.5'; - } - } - // HE-AAC is broken on Android, always signal audio codec as AAC even if variant manifest states otherwise - if (ua.indexOf('android') !== -1 && track.container !== 'audio/mpeg') { // Exclude mpeg audio - audioCodec = 'mp4a.40.2'; - logger.log(`Android: force audio codec to ${audioCodec}`); - } - track.levelCodec = audioCodec; - track.id = data.id; - } - track = tracks.video; - if (track) { - track.levelCodec = this.levels[this.level].videoCodec; - track.id = data.id; - } - this.hls.trigger(Event.BUFFER_CODECS, tracks); - // loop through tracks that are going to be provided to bufferController - for (trackName in tracks) { - track = tracks[trackName]; - logger.log(`main track:${trackName},container:${track.container},codecs[level/parsed]=[${track.levelCodec}/${track.codec}]`); - let initSegment = track.initSegment; - if (initSegment) { - this.appended = true; - // arm pending Buffering flag before appending a segment - this.pendingBuffering = true; - this.hls.trigger(Event.BUFFER_APPENDING, { type: trackName, data: initSegment, parent: 'main', content: 'initSegment' }); - } - } - // trigger handler right now - this.tick(); - } - } - - onFragParsingData (data) { - const fragCurrent = this.fragCurrent; - const fragNew = data.frag; - if (fragCurrent && - data.id === 'main' && - fragNew.sn === fragCurrent.sn && - fragNew.level === fragCurrent.level && - !(data.type === 'audio' && this.altAudio) && // filter out main audio if audio track is loaded through audio stream controller - this.state === State.PARSING) { - let level = this.levels[this.level], - frag = fragCurrent; - if (!Number.isFinite(data.endPTS)) { - data.endPTS = data.startPTS + fragCurrent.duration; - data.endDTS = data.startDTS + fragCurrent.duration; - } - - if (data.hasAudio === true) { - frag.addElementaryStream(ElementaryStreamTypes.AUDIO); - } - - if (data.hasVideo === true) { - frag.addElementaryStream(ElementaryStreamTypes.VIDEO); - } - - logger.log(`Parsed ${data.type},PTS:[${data.startPTS.toFixed(3)},${data.endPTS.toFixed(3)}],DTS:[${data.startDTS.toFixed(3)}/${data.endDTS.toFixed(3)}],nb:${data.nb},dropped:${data.dropped || 0}`); - - // Detect gaps in a fragment and try to fix it by finding a keyframe in the previous fragment (see _findFragments) - if (data.type === 'video') { - frag.dropped = data.dropped; - if (frag.dropped) { - if (!frag.backtracked) { - const levelDetails = level.details; - if (levelDetails && frag.sn === levelDetails.startSN) { - logger.warn('missing video frame(s) on first frag, appending with gap', frag.sn); - } else { - logger.warn('missing video frame(s), backtracking fragment', frag.sn); - // Return back to the IDLE state without appending to buffer - // Causes findFragments to backtrack a segment and find the keyframe - // Audio fragments arriving before video sets the nextLoadPosition, causing _findFragments to skip the backtracked fragment - this.fragmentTracker.removeFragment(frag); - frag.backtracked = true; - this.nextLoadPosition = data.startPTS; - this.state = State.IDLE; - this.fragPrevious = frag; - this.tick(); - return; - } - } else { - logger.warn('Already backtracked on this fragment, appending with the gap', frag.sn); - } - } else { - // Only reset the backtracked flag if we've loaded the frag without any dropped frames - frag.backtracked = false; - } - } - - let drift = LevelHelper.updateFragPTSDTS(level.details, frag, data.startPTS, data.endPTS, data.startDTS, data.endDTS), - hls = this.hls; - hls.trigger(Event.LEVEL_PTS_UPDATED, { details: level.details, level: this.level, drift: drift, type: data.type, start: data.startPTS, end: data.endPTS }); - // has remuxer dropped video frames located before first keyframe ? - [data.data1, data.data2].forEach(buffer => { - // only append in PARSING state (rationale is that an appending error could happen synchronously on first segment appending) - // in that case it is useless to append following segments - if (buffer && buffer.length && this.state === State.PARSING) { - this.appended = true; - // arm pending Buffering flag before appending a segment - this.pendingBuffering = true; - hls.trigger(Event.BUFFER_APPENDING, { type: data.type, data: buffer, parent: 'main', content: 'data' }); - } - }); - // trigger handler right now - this.tick(); - } - } - - onFragParsed (data) { - const fragCurrent = this.fragCurrent; - const fragNew = data.frag; - if (fragCurrent && - data.id === 'main' && - fragNew.sn === fragCurrent.sn && - fragNew.level === fragCurrent.level && - this.state === State.PARSING) { - this.stats.tparsed = window.performance.now(); - this.state = State.PARSED; - this._checkAppendedParsed(); - } - } - - onAudioTrackSwitching (data) { - // if any URL found on new audio track, it is an alternate audio track - let altAudio = !!data.url, - trackId = data.id; - // if we switch on main audio, ensure that main fragment scheduling is synced with media.buffered - // don't do anything if we switch to alt audio: audio stream controller is handling it. - // we will just have to change buffer scheduling on audioTrackSwitched - if (!altAudio) { - if (this.mediaBuffer !== this.media) { - logger.log('switching on main audio, use media.buffered to schedule main fragment loading'); - this.mediaBuffer = this.media; - let fragCurrent = this.fragCurrent; - // we need to refill audio buffer from main: cancel any frag loading to speed up audio switch - if (fragCurrent.loader) { - logger.log('switching to main audio track, cancel main fragment load'); - fragCurrent.loader.abort(); - } - this.fragCurrent = null; - this.fragPrevious = null; - // destroy demuxer to force init segment generation (following audio switch) - if (this.demuxer) { - this.demuxer.destroy(); - this.demuxer = null; - } - // switch to IDLE state to load new fragment - this.state = State.IDLE; - } - let hls = this.hls; - // switching to main audio, flush all audio and trigger track switched - hls.trigger(Event.BUFFER_FLUSHING, { startOffset: 0, endOffset: Number.POSITIVE_INFINITY, type: 'audio' }); - hls.trigger(Event.AUDIO_TRACK_SWITCHED, { id: trackId }); - this.altAudio = false; - } - } - - onAudioTrackSwitched (data) { - let trackId = data.id, - altAudio = !!this.hls.audioTracks[trackId].url; - if (altAudio) { - let videoBuffer = this.videoBuffer; - // if we switched on alternate audio, ensure that main fragment scheduling is synced with video sourcebuffer buffered - if (videoBuffer && this.mediaBuffer !== videoBuffer) { - logger.log('switching on alternate audio, use video.buffered to schedule main fragment loading'); - this.mediaBuffer = videoBuffer; - } - } - this.altAudio = altAudio; - this.tick(); - } - - onBufferCreated (data) { - let tracks = data.tracks, mediaTrack, name, alternate = false; - for (let type in tracks) { - let track = tracks[type]; - if (track.id === 'main') { - name = type; - mediaTrack = track; - // keep video source buffer reference - if (type === 'video') { - this.videoBuffer = tracks[type].buffer; - } - } else { - alternate = true; - } - } - if (alternate && mediaTrack) { - logger.log(`alternate track found, use ${name}.buffered to schedule main fragment loading`); - this.mediaBuffer = mediaTrack.buffer; - } else { - this.mediaBuffer = this.media; - } - } - - onBufferAppended (data) { - if (data.parent === 'main') { - const state = this.state; - if (state === State.PARSING || state === State.PARSED) { - // check if all buffers have been appended - this.pendingBuffering = (data.pending > 0); - this._checkAppendedParsed(); - } - } - } - - _checkAppendedParsed () { - // trigger handler right now - if (this.state === State.PARSED && (!this.appended || !this.pendingBuffering)) { - const frag = this.fragCurrent; - if (frag) { - const media = this.mediaBuffer ? this.mediaBuffer : this.media; - logger.log(`main buffered : ${TimeRanges.toString(media.buffered)}`); - this.fragPrevious = frag; - const stats = this.stats; - stats.tbuffered = window.performance.now(); - // we should get rid of this.fragLastKbps - this.fragLastKbps = Math.round(8 * stats.total / (stats.tbuffered - stats.tfirst)); - this.hls.trigger(Event.FRAG_BUFFERED, { stats: stats, frag: frag, id: 'main' }); - this.state = State.IDLE; - } - this.tick(); - } - } - - onError (data) { - let frag = data.frag || this.fragCurrent; - // don't handle frag error not related to main fragment - if (frag && frag.type !== 'main') { - return; - } - - // 0.5 : tolerance needed as some browsers stalls playback before reaching buffered end - let mediaBuffered = !!this.media && BufferHelper.isBuffered(this.media, this.media.currentTime) && BufferHelper.isBuffered(this.media, this.media.currentTime + 0.5); - - switch (data.details) { - case ErrorDetails.FRAG_LOAD_ERROR: - case ErrorDetails.FRAG_LOAD_TIMEOUT: - case ErrorDetails.KEY_LOAD_ERROR: - case ErrorDetails.KEY_LOAD_TIMEOUT: - if (!data.fatal) { - // keep retrying until the limit will be reached - if ((this.fragLoadError + 1) <= this.config.fragLoadingMaxRetry) { - // exponential backoff capped to config.fragLoadingMaxRetryTimeout - let delay = Math.min(Math.pow(2, this.fragLoadError) * this.config.fragLoadingRetryDelay, this.config.fragLoadingMaxRetryTimeout); - logger.warn(`mediaController: frag loading failed, retry in ${delay} ms`); - this.retryDate = window.performance.now() + delay; - // retry loading state - // if loadedmetadata is not set, it means that we are emergency switch down on first frag - // in that case, reset startFragRequested flag - if (!this.loadedmetadata) { - this.startFragRequested = false; - this.nextLoadPosition = this.startPosition; - } - this.fragLoadError++; - this.state = State.FRAG_LOADING_WAITING_RETRY; - } else { - logger.error(`mediaController: ${data.details} reaches max retry, redispatch as fatal ...`); - // switch error to fatal - data.fatal = true; - this.state = State.ERROR; - } - } - break; - case ErrorDetails.LEVEL_LOAD_ERROR: - case ErrorDetails.LEVEL_LOAD_TIMEOUT: - if (this.state !== State.ERROR) { - if (data.fatal) { - // if fatal error, stop processing - this.state = State.ERROR; - logger.warn(`streamController: ${data.details},switch to ${this.state} state ...`); - } else { - // in case of non fatal error while loading level, if level controller is not retrying to load level , switch back to IDLE - if (!data.levelRetry && this.state === State.WAITING_LEVEL) { - this.state = State.IDLE; - } - } - } - break; - case ErrorDetails.BUFFER_FULL_ERROR: - // if in appending state - if (data.parent === 'main' && (this.state === State.PARSING || this.state === State.PARSED)) { - // reduce max buf len if current position is buffered - if (mediaBuffered) { - this._reduceMaxBufferLength(this.config.maxBufferLength); - this.state = State.IDLE; - } else { - // current position is not buffered, but browser is still complaining about buffer full error - // this happens on IE/Edge, refer to https://github.com/video-dev/hls.js/pull/708 - // in that case flush the whole buffer to recover - logger.warn('buffer full error also media.currentTime is not buffered, flush everything'); - this.fragCurrent = null; - // flush everything - this.flushMainBuffer(0, Number.POSITIVE_INFINITY); - } - } - break; - default: - break; - } - } - - _reduceMaxBufferLength (minLength) { - let config = this.config; - if (config.maxMaxBufferLength >= minLength) { - // reduce max buffer length as it might be too high. we do this to avoid loop flushing ... - config.maxMaxBufferLength /= 2; - logger.warn(`main:reduce max buffer length to ${config.maxMaxBufferLength}s`); - return true; - } - return false; - } - - /** - * Checks the health of the buffer and attempts to resolve playback stalls. - * @private - */ - _checkBuffer () { - const { media } = this; - if (!media || media.readyState === 0) { - // Exit early if we don't have media or if the media hasn't bufferd anything yet (readyState 0) - return; - } - - const mediaBuffer = this.mediaBuffer ? this.mediaBuffer : media; - const buffered = mediaBuffer.buffered; - - if (!this.loadedmetadata && buffered.length) { - this.loadedmetadata = true; - this._seekToStartPos(); - } else if (this.immediateSwitch) { - this.immediateLevelSwitchEnd(); - } else { - this.gapController.poll(this.lastCurrentTime, buffered); - } - } - - onFragLoadEmergencyAborted () { - this.state = State.IDLE; - // if loadedmetadata is not set, it means that we are emergency switch down on first frag - // in that case, reset startFragRequested flag - if (!this.loadedmetadata) { - this.startFragRequested = false; - this.nextLoadPosition = this.startPosition; - } - this.tick(); - } - - onBufferFlushed () { - /* after successful buffer flushing, filter flushed fragments from bufferedFrags - use mediaBuffered instead of media (so that we will check against video.buffered ranges in case of alt audio track) - */ - const media = this.mediaBuffer ? this.mediaBuffer : this.media; - if (media) { - // filter fragments potentially evicted from buffer. this is to avoid memleak on live streams - this.fragmentTracker.detectEvictedFragments(ElementaryStreamTypes.VIDEO, media.buffered); - } - // move to IDLE once flush complete. this should trigger new fragment loading - this.state = State.IDLE; - // reset reference to frag - this.fragPrevious = null; - } - - swapAudioCodec () { - this.audioCodecSwap = !this.audioCodecSwap; - } - /** - * Seeks to the set startPosition if not equal to the mediaElement's current time. - * @private - */ - _seekToStartPos () { - const { media } = this; - const currentTime = media.currentTime; - // only adjust currentTime if different from startPosition or if startPosition not buffered - // at that stage, there should be only one buffered range, as we reach that code after first fragment has been buffered - const startPosition = media.seeking ? currentTime : this.startPosition; - // if currentTime not matching with expected startPosition or startPosition not buffered but close to first buffered - if (currentTime !== startPosition) { - // if startPosition not buffered, let's seek to buffered.start(0) - logger.log(`target start position not buffered, seek to buffered.start(0) ${startPosition} from current time ${currentTime} `); - media.currentTime = startPosition; - } - } - - _getAudioCodec (currentLevel) { - let audioCodec = this.config.defaultAudioCodec || currentLevel.audioCodec; - if (this.audioCodecSwap) { - logger.log('swapping playlist audio codec'); - if (audioCodec) { - if (audioCodec.indexOf('mp4a.40.5') !== -1) { - audioCodec = 'mp4a.40.2'; - } else { - audioCodec = 'mp4a.40.5'; - } - } - } - - return audioCodec; - } - - get liveSyncPosition () { - return this._liveSyncPosition; - } - - set liveSyncPosition (value) { - this._liveSyncPosition = value; } } export default StreamController; diff --git a/cmd/mjpeg-player/hlsjs/hls.js b/cmd/mjpeg-player/hlsjs/hls.js index 88107af2..a344d330 100644 --- a/cmd/mjpeg-player/hlsjs/hls.js +++ b/cmd/mjpeg-player/hlsjs/hls.js @@ -1,674 +1,50 @@ -import * as URLToolkit from 'url-toolkit'; +/* +AUTHOR + Trek Hopton -import { - ErrorTypes, - ErrorDetails -} from './errors'; +LICENSE + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) -import PlaylistLoader from './loader/playlist-loader'; -import FragmentLoader from './loader/fragment-loader'; -import KeyLoader from './loader/key-loader'; + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. -import { FragmentTracker } from './controller/fragment-tracker'; -import StreamController from './controller/stream-controller'; -import LevelController from './controller/level-controller'; -import ID3TrackController from './controller/id3-track-controller'; + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. -import { isSupported } from './is-supported'; -import { logger, enableLogs } from './utils/logger'; -import { hlsDefaultConfig, HlsConfig } from './config'; + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see http://www.gnu.org/licenses. -import HlsEvents from './events'; + For hls.js Copyright notice and license, see LICENSE file. +*/ -import { Observer } from './observer'; +import URLToolkit from '../url-toolkit/url-toolkit.js'; +import HlsEvents from './events.js'; +import PlaylistLoader from './loader/playlist-loader.js'; +import FragmentLoader from './loader/fragment-loader.js'; +import StreamController from './controller/stream-controller.js'; +import { hlsDefaultConfig } from './config.js'; +import { Observer } from './observer.js'; -/** - * @module Hls - * @class - * @constructor - */ -export default class Hls extends Observer { - public static defaultConfig?: HlsConfig; - public config: HlsConfig; - - private _autoLevelCapping: number; - private abrController: any; - private capLevelController: any; - private levelController: any; - private streamController: any; - private networkControllers: any[]; - private audioTrackController: any; - private subtitleTrackController: any; - private emeController: any; - private coreComponents: any[]; - private media: HTMLMediaElement | null = null; - private url: string | null = null; - - /** - * @type {string} - */ - static get version (): string { - return __VERSION__; - } - - /** - * @type {boolean} - */ - static isSupported (): boolean { - return isSupported(); - } - - /** - * @type {HlsEvents} - */ - static get Events () { - return HlsEvents; - } - - /** - * @type {HlsErrorTypes} - */ - static get ErrorTypes () { - return ErrorTypes; - } - - /** - * @type {HlsErrorDetails} - */ - static get ErrorDetails () { - return ErrorDetails; - } - - /** - * @type {HlsConfig} - */ - static get DefaultConfig (): HlsConfig { - if (!Hls.defaultConfig) { - return hlsDefaultConfig; - } - - return Hls.defaultConfig; - } - - /** - * @type {HlsConfig} - */ - static set DefaultConfig (defaultConfig: HlsConfig) { - Hls.defaultConfig = defaultConfig; - } - - /** - * Creates an instance of an HLS client that can attach to exactly one `HTMLMediaElement`. - * - * @constructs Hls - * @param {HlsConfig} config - */ - constructor (userConfig: Partial = {}) { +class Hls extends Observer { + constructor() { super(); + this.pLoader = new PlaylistLoader(this); + this.streamController = new StreamController(this); + this.fragmentLoader = new FragmentLoader(this); - const defaultConfig = Hls.DefaultConfig; - - if ((userConfig.liveSyncDurationCount || userConfig.liveMaxLatencyDurationCount) && (userConfig.liveSyncDuration || userConfig.liveMaxLatencyDuration)) { - throw new Error('Illegal hls.js config: don\'t mix up liveSyncDurationCount/liveMaxLatencyDurationCount and liveSyncDuration/liveMaxLatencyDuration'); - } - - // Shallow clone - this.config = { - ...defaultConfig, - ...userConfig - }; - - const { config } = this; - - if (config.liveMaxLatencyDurationCount !== void 0 && config.liveMaxLatencyDurationCount <= config.liveSyncDurationCount) { - throw new Error('Illegal hls.js config: "liveMaxLatencyDurationCount" must be gt "liveSyncDurationCount"'); - } - - if (config.liveMaxLatencyDuration !== void 0 && (config.liveSyncDuration === void 0 || config.liveMaxLatencyDuration <= config.liveSyncDuration)) { - throw new Error('Illegal hls.js config: "liveMaxLatencyDuration" must be gt "liveSyncDuration"'); - } - - enableLogs(config.debug); - - this._autoLevelCapping = -1; - - // core controllers and network loaders - - /** - * @member {AbrController} abrController - */ - const abrController = this.abrController = new config.abrController(this); // eslint-disable-line new-cap - const bufferController = new config.bufferController(this); // eslint-disable-line new-cap - const capLevelController = this.capLevelController = new config.capLevelController(this); // eslint-disable-line new-cap - const fpsController = new config.fpsController(this); // eslint-disable-line new-cap - const playListLoader = new PlaylistLoader(this); - const fragmentLoader = new FragmentLoader(this); - const keyLoader = new KeyLoader(this); - const id3TrackController = new ID3TrackController(this); - - // network controllers - - /** - * @member {LevelController} levelController - */ - const levelController = this.levelController = new LevelController(this); - - // FIXME: FragmentTracker must be defined before StreamController because the order of event handling is important - const fragmentTracker = new FragmentTracker(this); - - /** - * @member {StreamController} streamController - */ - const streamController = this.streamController = new StreamController(this, fragmentTracker); - - let networkControllers = [levelController, streamController]; - - // optional audio stream controller - /** - * @var {ICoreComponent | Controller} - */ - let Controller = config.audioStreamController; - if (Controller) { - networkControllers.push(new Controller(this, fragmentTracker)); - } - - /** - * @member {INetworkController[]} networkControllers - */ - this.networkControllers = networkControllers; - - /** - * @var {ICoreComponent[]} - */ - const coreComponents = [ - playListLoader, - fragmentLoader, - keyLoader, - abrController, - bufferController, - capLevelController, - fpsController, - id3TrackController, - fragmentTracker - ]; - - // optional audio track and subtitle controller - Controller = config.audioTrackController; - if (Controller) { - const audioTrackController = new Controller(this); - - /** - * @member {AudioTrackController} audioTrackController - */ - this.audioTrackController = audioTrackController; - coreComponents.push(audioTrackController); - } - - Controller = config.subtitleTrackController; - if (Controller) { - const subtitleTrackController = new Controller(this); - - /** - * @member {SubtitleTrackController} subtitleTrackController - */ - this.subtitleTrackController = subtitleTrackController; - networkControllers.push(subtitleTrackController); - } - - Controller = config.emeController; - if (Controller) { - const emeController = new Controller(this); - - /** - * @member {EMEController} emeController - */ - this.emeController = emeController; - coreComponents.push(emeController); - } - - // optional subtitle controllers - Controller = config.subtitleStreamController; - if (Controller) { - networkControllers.push(new Controller(this, fragmentTracker)); - } - Controller = config.timelineController; - if (Controller) { - coreComponents.push(new Controller(this)); - } - - /** - * @member {ICoreComponent[]} - */ - this.coreComponents = coreComponents; + this.config = hlsDefaultConfig; } - /** - * Dispose of the instance - */ - destroy () { - logger.log('destroy'); - this.trigger(HlsEvents.DESTROYING); - this.detachMedia(); - this.coreComponents.concat(this.networkControllers).forEach(component => { - component.destroy(); - }); - this.url = null; - this.removeAllListeners(); - this._autoLevelCapping = -1; - } - - /** - * Attach a media element - * @param {HTMLMediaElement} media - */ - attachMedia (media: HTMLMediaElement) { - logger.log('attachMedia'); - this.media = media; - this.trigger(HlsEvents.MEDIA_ATTACHING, { media: media }); - } - - /** - * Detach from the media - */ - detachMedia () { - logger.log('detachMedia'); - this.trigger(HlsEvents.MEDIA_DETACHING); - this.media = null; - } - - /** - * Set the source URL. Can be relative or absolute. - * @param {string} url - */ - loadSource (url: string) { + // url is the source URL. Can be relative or absolute. + loadSource(url, callback) { + this.loadSuccess = callback; url = URLToolkit.buildAbsoluteURL(window.location.href, url, { alwaysNormalize: true }); - logger.log(`loadSource:${url}`); - this.url = url; - // when attaching to a source URL, trigger a playlist load this.trigger(HlsEvents.MANIFEST_LOADING, { url: url }); } - - /** - * Start loading data from the stream source. - * Depending on default config, client starts loading automatically when a source is set. - * - * @param {number} startPosition Set the start position to stream from - * @default -1 None (from earliest point) - */ - startLoad (startPosition: number = -1) { - logger.log(`startLoad(${startPosition})`); - this.networkControllers.forEach(controller => { - controller.startLoad(startPosition); - }); - } - - /** - * Stop loading of any stream data. - */ - stopLoad () { - logger.log('stopLoad'); - this.networkControllers.forEach(controller => { - controller.stopLoad(); - }); - } - - /** - * Swap through possible audio codecs in the stream (for example to switch from stereo to 5.1) - */ - swapAudioCodec () { - logger.log('swapAudioCodec'); - this.streamController.swapAudioCodec(); - } - - /** - * When the media-element fails, this allows to detach and then re-attach it - * as one call (convenience method). - * - * Automatic recovery of media-errors by this process is configurable. - */ - recoverMediaError () { - logger.log('recoverMediaError'); - let media = this.media; - this.detachMedia(); - if (media) { - this.attachMedia(media); - } - } - - /** - * @type {QualityLevel[]} - */ - // todo(typescript-levelController) - get levels (): any[] { - return this.levelController.levels; - } - - /** - * Index of quality level currently played - * @type {number} - */ - get currentLevel (): number { - return this.streamController.currentLevel; - } - - /** - * Set quality level index immediately . - * This will flush the current buffer to replace the quality asap. - * That means playback will interrupt at least shortly to re-buffer and re-sync eventually. - * @type {number} -1 for automatic level selection - */ - set currentLevel (newLevel: number) { - logger.log(`set currentLevel:${newLevel}`); - this.loadLevel = newLevel; - this.streamController.immediateLevelSwitch(); - } - - /** - * Index of next quality level loaded as scheduled by stream controller. - * @type {number} - */ - get nextLevel (): number { - return this.streamController.nextLevel; - } - - /** - * Set quality level index for next loaded data. - * This will switch the video quality asap, without interrupting playback. - * May abort current loading of data, and flush parts of buffer (outside currently played fragment region). - * @type {number} -1 for automatic level selection - */ - set nextLevel (newLevel: number) { - logger.log(`set nextLevel:${newLevel}`); - this.levelController.manualLevel = newLevel; - this.streamController.nextLevelSwitch(); - } - - /** - * Return the quality level of the currently or last (of none is loaded currently) segment - * @type {number} - */ - get loadLevel (): number { - return this.levelController.level; - } - - /** - * Set quality level index for next loaded data in a conservative way. - * This will switch the quality without flushing, but interrupt current loading. - * Thus the moment when the quality switch will appear in effect will only be after the already existing buffer. - * @type {number} newLevel -1 for automatic level selection - */ - set loadLevel (newLevel: number) { - logger.log(`set loadLevel:${newLevel}`); - this.levelController.manualLevel = newLevel; - } - - /** - * get next quality level loaded - * @type {number} - */ - get nextLoadLevel (): number { - return this.levelController.nextLoadLevel; - } - - /** - * Set quality level of next loaded segment in a fully "non-destructive" way. - * Same as `loadLevel` but will wait for next switch (until current loading is done). - * @type {number} level - */ - set nextLoadLevel (level: number) { - this.levelController.nextLoadLevel = level; - } - - /** - * Return "first level": like a default level, if not set, - * falls back to index of first level referenced in manifest - * @type {number} - */ - get firstLevel (): number { - return Math.max(this.levelController.firstLevel, this.minAutoLevel); - } - - /** - * Sets "first-level", see getter. - * @type {number} - */ - set firstLevel (newLevel: number) { - logger.log(`set firstLevel:${newLevel}`); - this.levelController.firstLevel = newLevel; - } - - /** - * Return start level (level of first fragment that will be played back) - * if not overrided by user, first level appearing in manifest will be used as start level - * if -1 : automatic start level selection, playback will start from level matching download bandwidth - * (determined from download of first segment) - * @type {number} - */ - get startLevel (): number { - return this.levelController.startLevel; - } - - /** - * set start level (level of first fragment that will be played back) - * if not overrided by user, first level appearing in manifest will be used as start level - * if -1 : automatic start level selection, playback will start from level matching download bandwidth - * (determined from download of first segment) - * @type {number} newLevel - */ - set startLevel (newLevel: number) { - logger.log(`set startLevel:${newLevel}`); - // if not in automatic start level detection, ensure startLevel is greater than minAutoLevel - if (newLevel !== -1) { - newLevel = Math.max(newLevel, this.minAutoLevel); - } - - this.levelController.startLevel = newLevel; - } - - /** - * set dynamically set capLevelToPlayerSize against (`CapLevelController`) - * - * @type {boolean} - */ - set capLevelToPlayerSize (shouldStartCapping: boolean) { - const newCapLevelToPlayerSize = !!shouldStartCapping; - - if (newCapLevelToPlayerSize !== this.config.capLevelToPlayerSize) { - if (newCapLevelToPlayerSize) { - this.capLevelController.startCapping(); // If capping occurs, nextLevelSwitch will happen based on size. - } else { - this.capLevelController.stopCapping(); - this.autoLevelCapping = -1; - this.streamController.nextLevelSwitch(); // Now we're uncapped, get the next level asap. - } - - this.config.capLevelToPlayerSize = newCapLevelToPlayerSize; - } - } - - /** - * Capping/max level value that should be used by automatic level selection algorithm (`ABRController`) - * @type {number} - */ - get autoLevelCapping (): number { - return this._autoLevelCapping; - } - - /** - * get bandwidth estimate - * @type {number} - */ - get bandwidthEstimate (): number { - const bwEstimator = this.abrController._bwEstimator; - return bwEstimator ? bwEstimator.getEstimate() : NaN; - } - - /** - * Capping/max level value that should be used by automatic level selection algorithm (`ABRController`) - * @type {number} - */ - set autoLevelCapping (newLevel: number) { - logger.log(`set autoLevelCapping:${newLevel}`); - this._autoLevelCapping = newLevel; - } - - /** - * True when automatic level selection enabled - * @type {boolean} - */ - get autoLevelEnabled (): boolean { - return (this.levelController.manualLevel === -1); - } - - /** - * Level set manually (if any) - * @type {number} - */ - get manualLevel (): number { - return this.levelController.manualLevel; - } - - /** - * min level selectable in auto mode according to config.minAutoBitrate - * @type {number} - */ - get minAutoLevel (): number { - const { levels, config: { minAutoBitrate } } = this; - const len = levels ? levels.length : 0; - - for (let i = 0; i < len; i++) { - const levelNextBitrate = levels[i].realBitrate - ? Math.max(levels[i].realBitrate, levels[i].bitrate) - : levels[i].bitrate; - - if (levelNextBitrate > minAutoBitrate) { - return i; - } - } - - return 0; - } - - /** - * max level selectable in auto mode according to autoLevelCapping - * @type {number} - */ - get maxAutoLevel (): number { - const { levels, autoLevelCapping } = this; - - let maxAutoLevel; - if (autoLevelCapping === -1 && levels && levels.length) { - maxAutoLevel = levels.length - 1; - } else { - maxAutoLevel = autoLevelCapping; - } - - return maxAutoLevel; - } - - /** - * next automatically selected quality level - * @type {number} - */ - get nextAutoLevel (): number { - // ensure next auto level is between min and max auto level - return Math.min(Math.max(this.abrController.nextAutoLevel, this.minAutoLevel), this.maxAutoLevel); - } - - /** - * this setter is used to force next auto level. - * this is useful to force a switch down in auto mode: - * in case of load error on level N, hls.js can set nextAutoLevel to N-1 for example) - * forced value is valid for one fragment. upon succesful frag loading at forced level, - * this value will be resetted to -1 by ABR controller. - * @type {number} - */ - set nextAutoLevel (nextLevel: number) { - this.abrController.nextAutoLevel = Math.max(this.minAutoLevel, nextLevel); - } - - /** - * @type {AudioTrack[]} - */ - // todo(typescript-audioTrackController) - get audioTracks (): any[] { - const audioTrackController = this.audioTrackController; - return audioTrackController ? audioTrackController.audioTracks : []; - } - - /** - * index of the selected audio track (index in audio track lists) - * @type {number} - */ - get audioTrack (): number { - const audioTrackController = this.audioTrackController; - return audioTrackController ? audioTrackController.audioTrack : -1; - } - - /** - * selects an audio track, based on its index in audio track lists - * @type {number} - */ - set audioTrack (audioTrackId: number) { - const audioTrackController = this.audioTrackController; - if (audioTrackController) { - audioTrackController.audioTrack = audioTrackId; - } - } - - /** - * @type {Seconds} - */ - get liveSyncPosition (): number { - return this.streamController.liveSyncPosition; - } - - /** - * get alternate subtitle tracks list from playlist - * @type {SubtitleTrack[]} - */ - // todo(typescript-subtitleTrackController) - get subtitleTracks (): any[] { - const subtitleTrackController = this.subtitleTrackController; - return subtitleTrackController ? subtitleTrackController.subtitleTracks : []; - } - - /** - * index of the selected subtitle track (index in subtitle track lists) - * @type {number} - */ - get subtitleTrack (): number { - const subtitleTrackController = this.subtitleTrackController; - return subtitleTrackController ? subtitleTrackController.subtitleTrack : -1; - } - - /** - * select an subtitle track, based on its index in subtitle track lists - * @type {number} - */ - set subtitleTrack (subtitleTrackId: number) { - const subtitleTrackController = this.subtitleTrackController; - if (subtitleTrackController) { - subtitleTrackController.subtitleTrack = subtitleTrackId; - } - } - - /** - * @type {boolean} - */ - get subtitleDisplay (): boolean { - const subtitleTrackController = this.subtitleTrackController; - return subtitleTrackController ? subtitleTrackController.subtitleDisplay : false; - } - - /** - * Enable/disable subtitle display rendering - * @type {boolean} - */ - set subtitleDisplay (value: boolean) { - const subtitleTrackController = this.subtitleTrackController; - if (subtitleTrackController) { - subtitleTrackController.subtitleDisplay = value; - } - } } + +export default Hls \ No newline at end of file From 99b931f948f875cb1b61d95aab8dc4670c9fd2e8 Mon Sep 17 00:00:00 2001 From: Saxon Date: Fri, 24 Jan 2020 19:21:18 +1030 Subject: [PATCH 19/46] revid & cmd/revid-cli: added loop mode so that input may be restarted after completion Loop flag has been added to command line flags and in turn sets the Loop field that has been added to the config.Config struct. mode variable now also checked to see if value set to Loop, in which case revid config.Config.Loop = true. Revid.processFrom modified so that when input source has completed Revid.cfg.Loop is checked and input restarted if true. --- cmd/revid-cli/main.go | 3 +++ revid/config/config.go | 3 +++ revid/revid.go | 43 +++++++++++++++++++++++++++++++----------- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index bced0a4f..fc4d863f 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -122,6 +122,7 @@ func handleFlags() config.Config { httpAddressPtr = flag.String("HttpAddress", "", "Destination address of http posts") verticalFlipPtr = flag.Bool("VerticalFlip", false, "Flip video vertically: Yes, No") horizontalFlipPtr = flag.Bool("HorizontalFlip", false, "Flip video horizontally: Yes, No") + loopPtr = flag.Bool("Loop", false, "Loop video if EOF encountered: true, false") bitratePtr = flag.Uint("Bitrate", 0, "Bitrate of recorded video") heightPtr = flag.Uint("Height", 0, "Height in pixels") widthPtr = flag.Uint("Width", 0, "Width in pixels") @@ -179,6 +180,8 @@ func handleFlags() config.Config { } } + cfg.Loop = *loopPtr + switch *inputPtr { case "Raspivid": cfg.Input = config.InputRaspivid diff --git a/revid/config/config.go b/revid/config/config.go index 194faf33..47a8eaf9 100644 --- a/revid/config/config.go +++ b/revid/config/config.go @@ -297,6 +297,9 @@ type Config struct { MOGMinArea float64 // Used to ignore small areas of motion detection. MOGThreshold float64 // Intensity value from the KNN motion detection algorithm that is considered motion. MOGHistory uint // Length of MOG filter's history + + // If true will restart reading of input after an io.EOF. + Loop bool } // TypeData contains information about all of the variables that diff --git a/revid/revid.go b/revid/revid.go index 8b1b42da..9c414480 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -428,13 +428,8 @@ func (r *Revid) Start() error { return err } - err = r.input.Start() - if err != nil { - return fmt.Errorf("could not start input device: %w", err) - } - r.wg.Add(1) - go r.processFrom(r.input, 0) + go r.processFrom(r.input, (1000/25)*time.Millisecond) r.running = true return nil @@ -845,6 +840,11 @@ func (r *Revid) Update(vars map[string]string) error { break } r.cfg.MOGHistory = uint(v) + case "mode": + r.cfg.Loop = false + if value == "Loop" { + r.cfg.Loop = true + } } } r.cfg.Logger.Log(logger.Info, pkg+"revid config changed", "config", fmt.Sprintf("%+v", r.cfg)) @@ -853,14 +853,35 @@ func (r *Revid) Update(vars map[string]string) error { // processFrom is run as a routine to read from a input data source, lex and // then send individual access units to revid's encoders. -func (r *Revid) processFrom(read io.Reader, delay time.Duration) { - err := r.lexTo(r.filters[0], read, delay) - r.cfg.Logger.Log(logger.Debug, pkg+"finished lexing") +func (r *Revid) processFrom(in device.AVDevice, delay time.Duration) { +restart: + err := in.Start() + if err != nil { + r.err <- fmt.Errorf("could not start input device: %w", err) + r.wg.Done() + return + } + + // Lex data from input device, in, until finished or an error is encountered. + // For a continuous source e.g. a camera or microphone, we should remain + // in this call indefinitely unless in.Stop() is called and an io.EOF is forced. + err = r.lexTo(r.filters[0], in, delay) switch err { - case nil: // Do nothing. - case io.EOF: // TODO: handle this depending on loop mode. + case nil, io.EOF, io.ErrUnexpectedEOF: default: r.err <- err } + + err = in.Stop() + if err != nil { + r.err <- fmt.Errorf("could not stop input source: %w", err) + } + + if r.cfg.Loop { + r.cfg.Logger.Log(logger.Info, pkg+"looping input") + goto restart + } + + r.cfg.Logger.Log(logger.Info, pkg+"finished lexing") r.wg.Done() } From 640f50543d41eba30633f37e642394fc9d788961 Mon Sep 17 00:00:00 2001 From: Trek H Date: Fri, 24 Jan 2020 19:22:51 +1030 Subject: [PATCH 20/46] mjpeg-player: added back changes that were overwritten --- cmd/mjpeg-player/index.html | 69 ++++++++++----- cmd/mjpeg-player/main.js | 168 +++++++++++++++++++++++++----------- cmd/mjpeg-player/player.js | 153 ++++++++++++++++++++------------ 3 files changed, 264 insertions(+), 126 deletions(-) diff --git a/cmd/mjpeg-player/index.html b/cmd/mjpeg-player/index.html index 064f436c..60a40429 100644 --- a/cmd/mjpeg-player/index.html +++ b/cmd/mjpeg-player/index.html @@ -1,34 +1,59 @@ + + - - Mjpeg Player - - + + Mjpeg Player + -
-
-
-
- -
-
-
- Frame Rate: fps -
-
- -
+
+
+
+
+
+
+
+ + + +
+
+ Frame Rate: fps +
+
+ +
-
-
- ©2019 Australian Ocean Laboratory Limited (AusOcean) (License) -
-
+
+
+
+ ©2019 Australian Ocean Laboratory Limited (AusOcean) (License) +
+
\ No newline at end of file diff --git a/cmd/mjpeg-player/main.js b/cmd/mjpeg-player/main.js index 037b31ed..e4a0eba0 100644 --- a/cmd/mjpeg-player/main.js +++ b/cmd/mjpeg-player/main.js @@ -1,12 +1,9 @@ /* -NAME - main.js - AUTHOR Trek Hopton LICENSE - This file is Copyright (C) 2019 the Australian Ocean Lab (AusOcean) + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) It is free software: you can redistribute it and/or modify them under the terms of the GNU General Public License as published by the @@ -19,57 +16,128 @@ LICENSE for more details. You should have received a copy of the GNU General Public License in gpl.txt. - If not, see http://www.gnu.org/licenses. + If not, see http://www.gnu.org/licenses.. */ -// play will process and play the chosen target file. -function play() { - const viewer = document.getElementById('viewer'); - const input = event.target.files[0]; - const reader = new FileReader(); +import Hls from "./hlsjs/hls.js"; - reader.onload = event => { - const player = new Worker("player.js"); +let started = false; +let player, viewer; - let rate = document.getElementById('rate'); - if (rate.value && rate.value > 0) { - player.postMessage({ msg: "setFrameRate", data: rate.value }); - } +// init gets DOM elements once the document has been loaded and adds listeners where necessary. +function init() { + document.addEventListener('DOMContentLoaded', load); + document.addEventListener('DOMContentLoaded', function () { + document.getElementById('urlBtn').addEventListener('click', load); + document.getElementById('fileInput').addEventListener('change', play); + viewer = document.getElementById('viewer'); + } + ); +} - player.onmessage = e => { - switch (e.data.msg) { - case "frame": - const blob = new Blob([new Uint8Array(e.data.data)], { - type: 'video/x-motion-jpeg' - }); - const url = URL.createObjectURL(blob); - viewer.src = url; - break; - case "log": - console.log(e.data.data); - break; - case "stop": - break; - default: - console.error("unknown message from player"); - break; - } - }; +init(); - switch (input.name.split('.')[1]) { - case "mjpeg": - case "mjpg": - player.postMessage({ msg: "loadMjpeg", data: event.target.result }, [event.target.result]); - break; - case "ts": - player.postMessage({ msg: "loadMtsMjpeg", data: event.target.result }, [event.target.result]); - break; - default: - console.error("unknown file format"); - break; - } - }; - reader.onerror = error => reject(error); - reader.readAsArrayBuffer(input); +// load gets the URL from the URL input element or the browser's URL bar +// and creates an Hls instance to load the content from the URL. +function load() { + let url = document.getElementById('url').value; + if (url == "") { + url = getQuery() + document.getElementById('url').value = url; + } + if (url[0] == '/') { + url = window.location.protocol + '//' + window.location.host + url; + } + if (url == "") { + return; + } + + let hls = new Hls(); + hls.loadSource(url, append); +} + +// getQuery gets everything after the question mark from the URL in the browser's URL bar. +function getQuery() { + let regex = new RegExp("\\?(.*)"); + let match = regex.exec(window.location.href); + if (match == null) { + return ''; + } else { + return decodeURIComponent(match[1].replace(/\+/g, " ")); + } +} + +// append, on the first call, starts a player worker and passes it a frame rate and the video data, +// on subsequent calls it passes the video data to the player worker. +function append(data) { + if (!started) { + player = new Worker("player.js"); + + let rate = document.getElementById('rate'); + if (rate.value && rate.value > 0) { + player.postMessage({ msg: "setFrameRate", data: rate.value }); + } + + player.onmessage = handleMessage; + + player.postMessage({ msg: "loadMtsMjpeg", data: data }, [data]); + started = true; + } else { + player.postMessage({ msg: "appendMtsMjpeg", data: data }, [data]); + } } + +// play will process and play the target file chosen with the file input element. +function play() { + const input = event.target.files[0]; + const reader = new FileReader(); + + reader.onload = event => { + const player = new Worker("player.js"); + + let rate = document.getElementById('rate'); + if (rate.value && rate.value > 0) { + player.postMessage({ msg: "setFrameRate", data: rate.value }); + } + + player.onmessage = handleMessage; + + switch (input.name.split('.')[1]) { + case "mjpeg": + case "mjpg": + player.postMessage({ msg: "loadMjpeg", data: event.target.result }, [event.target.result]); + break; + case "ts": + player.postMessage({ msg: "loadMtsMjpeg", data: event.target.result }, [event.target.result]); + break; + default: + console.error("unknown file format"); + break; + } + }; + reader.onerror = error => reject(error); + reader.readAsArrayBuffer(input); +} + +// handleMessage handles messgaes from the player workers, its main job is to update the display when a frame is received. +function handleMessage(e) { + switch (e.data.msg) { + case "frame": + const blob = new Blob([new Uint8Array(e.data.data)], { + type: 'video/x-motion-jpeg' + }); + const url = URL.createObjectURL(blob); + viewer.src = url; + break; + case "log": + console.log(e.data.data); + break; + case "stop": + console.log("stopped"); + break; + default: + console.error("unknown message from player"); + break; + } +} \ No newline at end of file diff --git a/cmd/mjpeg-player/player.js b/cmd/mjpeg-player/player.js index 34ff75ab..ea96e031 100644 --- a/cmd/mjpeg-player/player.js +++ b/cmd/mjpeg-player/player.js @@ -1,12 +1,9 @@ /* -NAME - player.js - AUTHOR Trek Hopton LICENSE - This file is Copyright (C) 2019 the Australian Ocean Lab (AusOcean) + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) It is free software: you can redistribute it and/or modify them under the terms of the GNU General Public License as published by the @@ -22,68 +19,116 @@ LICENSE If not, see http://www.gnu.org/licenses. */ -let frameRate = 30; +let frameRate = 25; //Keeps track of the frame rate, default is 25fps. +self.importScripts('./lex-mjpeg.js'); +self.importScripts('./hlsjs/mts-demuxer.js'); +const codecs = { + MJPEG: 1, + MTS_MJPEG: 2, +} + +// onmessage is called whenever the main thread sends this worker a message. onmessage = e => { - switch (e.data.msg) { - case "setFrameRate": - frameRate = e.data.data; - break; - case "loadMjpeg": - self.importScripts('./lex-mjpeg.js'); - let mjpeg = new Uint8Array(e.data.data); - let lex = new MJPEGLexer(mjpeg); - player = new Player(lex); - player.setFrameRate(frameRate); - player.start(); - break; - case "loadMtsMjpeg": - self.importScripts('./hlsjs/mts-demuxer.js'); - let mtsMjpeg = new Uint8Array(e.data.data); - let demux = new MTSDemuxer(); - let tracks = demux._getTracks(); - demux.append(mtsMjpeg); - buf = new FrameBuffer(tracks.video.data); - player = new Player(buf); - player.setFrameRate(frameRate); //TODO: read frame rate from metadata. - player.start(); - break; - default: - console.error("unknown message from main thread"); - break; - } + switch (e.data.msg) { + case "setFrameRate": + frameRate = e.data.data; + break; + case "loadMjpeg": + player = new PlayerWorker(); + player.init(codecs.MJPEG); + player.append(e.data.data); + player.setFrameRate(frameRate); + player.start(); + break; + case "loadMtsMjpeg": + player = new PlayerWorker(); + player.init(codecs.MTS_MJPEG); + player.append(e.data.data); + player.start(); + break; + case "appendMtsMjpeg": + player.append(e.data.data); + break; + default: + console.error("unknown message from main thread"); + break; + } }; -class Player { - constructor(buffer) { - this.buffer = buffer; - this.frameRate = frameRate; +// PlayerWorker has a FrameBuffer to hold frames and once started, passes them one at a time to the main thread. +class PlayerWorker { + init(codec) { + this.frameRate = frameRate; + this.codec = codec; + switch (codec) { + case codecs.MJPEG: + this.frameSrc = new MJPEGLexer(); + break; + case codecs.MTS_MJPEG: + this.frameSrc = new FrameBuffer(); + break; } + } - setFrameRate(rate) { - this.frameRate = rate; - } + setFrameRate(rate) { + this.frameRate = rate; + } - start() { - let frame = this.buffer.read(); - if (frame != null) { - postMessage({ msg: "frame", data: frame.buffer }, [frame.buffer]); - setTimeout(() => { this.start(); }, 1000 / this.frameRate); - } else { - postMessage({ msg: "stop" }); - } + start() { + let frame = this.frameSrc.read(); + if (frame != null) { + postMessage({ msg: "frame", data: frame.buffer }, [frame.buffer]); + setTimeout(() => { this.start(); }, 1000 / this.frameRate); + } else { + postMessage({ msg: "stop" }); } + } + + append(data) { + this.frameSrc.append(data); + } } // FrameBuffer allows an array of subarrays (MJPEG frames) to be read one at a time. class FrameBuffer { - constructor(src) { - this.src = src; - this.off = 0; - } + constructor() { + this.segments = []; + this.off = { segment: 0, frame: 0 }; + this.demuxer = new MTSDemuxer(); + } - // read returns the next single frame. - read() { - return this.src[this.off++]; + // read returns the next single frame. + read() { + let off = this.off; + let prevOff = off; + if (this.incrementOff()) { + return this.segments[prevOff.segment][prevOff.frame]; + } else { + return null; } + } + + append(data) { + let demuxed = this.demuxer.demux(new Uint8Array(data)); + this.segments.push(demuxed.data); + } + + incrementOff() { + if (!this.segments || !this.segments[this.off.segment]) { + return false; + } + if (this.off.frame + 1 >= this.segments[this.off.segment].length) { + if (this.off.segment + 1 >= this.segments.length) { + return false; + } else { + this.off.segment++; + this.off.frame = 0; + return true; + } + } else { + this.off.frame++; + return true; + } + } } \ No newline at end of file From 179f9cfa034aca07c48fa1bfd2ecdc185276b890 Mon Sep 17 00:00:00 2001 From: Trek H Date: Fri, 24 Jan 2020 19:41:43 +1030 Subject: [PATCH 21/46] mjpeg-player: reduced hlsjs code Reduced the lengthy lists of config vars and events used by hlsjs player that we no longer need. Also removed typescript typing from config.js --- cmd/mjpeg-player/hlsjs/config.js | 215 ++++--------------------------- cmd/mjpeg-player/hlsjs/events.js | 103 ++++----------- 2 files changed, 50 insertions(+), 268 deletions(-) diff --git a/cmd/mjpeg-player/hlsjs/config.js b/cmd/mjpeg-player/hlsjs/config.js index efc8e27c..ed548ff9 100644 --- a/cmd/mjpeg-player/hlsjs/config.js +++ b/cmd/mjpeg-player/hlsjs/config.js @@ -1,170 +1,32 @@ -/** - * HLS config - */ +/* +AUTHOR + Trek Hopton -import AbrController from './controller/abr-controller'; -import BufferController from './controller/buffer-controller'; -import CapLevelController from './controller/cap-level-controller'; -import FPSController from './controller/fps-controller'; -import XhrLoader from './utils/xhr-loader'; -// import FetchLoader from './utils/fetch-loader'; +LICENSE + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) -import AudioTrackController from './controller/audio-track-controller'; -import AudioStreamController from './controller/audio-stream-controller'; + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. -import * as Cues from './utils/cues'; -import TimelineController from './controller/timeline-controller'; -import SubtitleTrackController from './controller/subtitle-track-controller'; -import { SubtitleStreamController } from './controller/subtitle-stream-controller'; -import EMEController from './controller/eme-controller'; -import { requestMediaKeySystemAccess, MediaKeyFunc } from './utils/mediakeys-helper'; + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. -type ABRControllerConfig = { - abrEwmaFastLive: number, - abrEwmaSlowLive: number, - abrEwmaFastVoD: number, - abrEwmaSlowVoD: number, - abrEwmaDefaultEstimate: number, - abrBandWidthFactor: number, - abrBandWidthUpFactor: number, - abrMaxWithRealBitrate: boolean, - maxStarvationDelay: number, - maxLoadingDelay: number, -}; + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see http://www.gnu.org/licenses. -export type BufferControllerConfig = { - appendErrorMaxRetry: number, - liveDurationInfinity: boolean, - liveBackBufferLength: number, -}; + For hls.js Copyright notice and license, see LICENSE file. +*/ -type CapLevelControllerConfig = { - capLevelToPlayerSize: boolean -}; - -export type EMEControllerConfig = { - licenseXhrSetup?: (xhr: XMLHttpRequest, url: string) => void, - emeEnabled: boolean, - widevineLicenseUrl?: string, - requestMediaKeySystemAccessFunc: MediaKeyFunc | null, -}; - -type FragmentLoaderConfig = { - fLoader: any, // TODO(typescript-loader): Once Loader is typed fill this in - - fragLoadingTimeOut: number, - fragLoadingMaxRetry: number, - fragLoadingRetryDelay: number, - fragLoadingMaxRetryTimeout: number, -}; - -type FPSControllerConfig = { - capLevelOnFPSDrop: boolean, - fpsDroppedMonitoringPeriod: number, - fpsDroppedMonitoringThreshold: number, -}; - -type LevelControllerConfig = { - startLevel?: number -}; - -type MP4RemuxerConfig = { - stretchShortVideoTrack: boolean, - maxAudioFramesDrift: number, -}; - -type PlaylistLoaderConfig = { - pLoader: any, // TODO(typescript-loader): Once Loader is typed fill this in - - manifestLoadingTimeOut: number, - manifestLoadingMaxRetry: number, - manifestLoadingRetryDelay: number, - manifestLoadingMaxRetryTimeout: number, - - levelLoadingTimeOut: number, - levelLoadingMaxRetry: number, - levelLoadingRetryDelay: number, - levelLoadingMaxRetryTimeout: number -}; - -type StreamControllerConfig = { - autoStartLoad: boolean, - startPosition: number, - defaultAudioCodec?: string, - initialLiveManifestSize: number, - maxBufferLength: number, - maxBufferSize: number, - maxBufferHole: number, - - lowBufferWatchdogPeriod: number, - highBufferWatchdogPeriod: number, - nudgeOffset: number, - nudgeMaxRetry: number, - maxFragLookUpTolerance: number, - liveSyncDurationCount: number, - liveMaxLatencyDurationCount: number, - liveSyncDuration?: number, - liveMaxLatencyDuration?: number, - maxMaxBufferLength: number, - - startFragPrefetch: boolean, -}; - -type TimelineControllerConfig = { - cueHandler: any, // TODO(typescript-cues): Type once file is done - enableCEA708Captions: boolean, - enableWebVTT: boolean, - captionsTextTrack1Label: string, - captionsTextTrack1LanguageCode: string, - captionsTextTrack2Label: string, - captionsTextTrack2LanguageCode: string, -}; - -type TSDemuxerConfig = { - forceKeyFrameOnDiscontinuity: boolean, -}; - -export type HlsConfig = - { - debug: boolean, - enableWorker: boolean, - enableSoftwareAES: boolean, - minAutoBitrate: number, - loader: any, // TODO(typescript-xhrloader): Type once XHR is done - xhrSetup?: (xhr: XMLHttpRequest, url: string) => void, - - // Alt Audio - audioStreamController?: any, // TODO(typescript-audiostreamcontroller): Type once file is done - audioTrackController?: any, // TODO(typescript-audiotrackcontroller): Type once file is done - // Subtitle - subtitleStreamController?: any, // TODO(typescript-subtitlestreamcontroller): Type once file is done - subtitleTrackController?: any, // TODO(typescript-subtitletrackcontroller): Type once file is done - timelineController?: any, // TODO(typescript-timelinecontroller): Type once file is done - // EME - emeController?: typeof EMEController, - - abrController: any, // TODO(typescript-abrcontroller): Type once file is done - bufferController: typeof BufferController, - capLevelController: any, // TODO(typescript-caplevelcontroller): Type once file is done - fpsController: any, // TODO(typescript-fpscontroller): Type once file is done - } & - ABRControllerConfig & - BufferControllerConfig & - CapLevelControllerConfig & - EMEControllerConfig & - FPSControllerConfig & - FragmentLoaderConfig & - LevelControllerConfig & - MP4RemuxerConfig & - PlaylistLoaderConfig & - StreamControllerConfig & - Partial & - TSDemuxerConfig; +import XhrLoader from './utils/xhr-loader.js'; // If possible, keep hlsDefaultConfig shallow // It is cloned whenever a new Hls instance is created, by keeping the config // shallow the properties are cloned, and we don't end up manipulating the default -export const hlsDefaultConfig: HlsConfig = { +export const hlsDefaultConfig = { autoStartLoad: true, // used by stream-controller startPosition: -1, // used by stream-controller defaultAudioCodec: void 0, // used by stream-controller @@ -213,11 +75,11 @@ export const hlsDefaultConfig: HlsConfig = { pLoader: void 0, // used by playlist-loader xhrSetup: void 0, // used by xhr-loader licenseXhrSetup: void 0, // used by eme-controller - // fetchSetup: void 0, - abrController: AbrController, - bufferController: BufferController, - capLevelController: CapLevelController, - fpsController: FPSController, + fetchSetup: void 0, + // abrController: AbrController, + // bufferController: BufferController, + // capLevelController: CapLevelController, + // fpsController: FPSController, stretchShortVideoTrack: false, // used by mp4-remuxer maxAudioFramesDrift: 1, // used by mp4-remuxer forceKeyFrameOnDiscontinuity: true, // used by ts-demuxer @@ -234,31 +96,6 @@ export const hlsDefaultConfig: HlsConfig = { minAutoBitrate: 0, // used by hls emeEnabled: false, // used by eme-controller widevineLicenseUrl: void 0, // used by eme-controller - requestMediaKeySystemAccessFunc: requestMediaKeySystemAccess, // used by eme-controller + // requestMediaKeySystemAccessFunc: requestMediaKeySystemAccess, // used by eme-controller - // Dynamic Modules - ...timelineConfig(), - subtitleStreamController: (__USE_SUBTITLES__) ? SubtitleStreamController : void 0, - subtitleTrackController: (__USE_SUBTITLES__) ? SubtitleTrackController : void 0, - timelineController: (__USE_SUBTITLES__) ? TimelineController : void 0, - audioStreamController: (__USE_ALT_AUDIO__) ? AudioStreamController : void 0, - audioTrackController: (__USE_ALT_AUDIO__) ? AudioTrackController : void 0, - emeController: (__USE_EME_DRM__) ? EMEController : void 0 -}; - -function timelineConfig (): TimelineControllerConfig { - if (!__USE_SUBTITLES__) { - // intentionally doing this over returning Partial above - // this has the added nice property of still requiring the object below to completely define all props. - return {} as any; - } - return { - cueHandler: Cues, // used by timeline-controller - enableCEA708Captions: true, // used by timeline-controller - enableWebVTT: true, // used by timeline-controller - captionsTextTrack1Label: 'English', // used by timeline-controller - captionsTextTrack1LanguageCode: 'en', // used by timeline-controller - captionsTextTrack2Label: 'Spanish', // used by timeline-controller - captionsTextTrack2LanguageCode: 'es' // used by timeline-controller - }; -} +}; \ No newline at end of file diff --git a/cmd/mjpeg-player/hlsjs/events.js b/cmd/mjpeg-player/hlsjs/events.js index 82314117..e71d71c5 100644 --- a/cmd/mjpeg-player/hlsjs/events.js +++ b/cmd/mjpeg-player/hlsjs/events.js @@ -1,110 +1,55 @@ +/* +AUTHOR + Trek Hopton + +LICENSE + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see http://www.gnu.org/licenses. + + For hls.js Copyright notice and license, see LICENSE file. +*/ + /** * @readonly * @enum {string} */ const HlsEvents = { - // fired before MediaSource is attaching to media element - data: { media } - MEDIA_ATTACHING: 'hlsMediaAttaching', - // fired when MediaSource has been succesfully attached to media element - data: { } - MEDIA_ATTACHED: 'hlsMediaAttached', - // fired before detaching MediaSource from media element - data: { } - MEDIA_DETACHING: 'hlsMediaDetaching', - // fired when MediaSource has been detached from media element - data: { } - MEDIA_DETACHED: 'hlsMediaDetached', - // fired when we buffer is going to be reset - data: { } - BUFFER_RESET: 'hlsBufferReset', - // fired when we know about the codecs that we need buffers for to push into - data: {tracks : { container, codec, levelCodec, initSegment, metadata }} - BUFFER_CODECS: 'hlsBufferCodecs', - // fired when sourcebuffers have been created - data: { tracks : tracks } - BUFFER_CREATED: 'hlsBufferCreated', - // fired when we append a segment to the buffer - data: { segment: segment object } - BUFFER_APPENDING: 'hlsBufferAppending', - // fired when we are done with appending a media segment to the buffer - data : { parent : segment parent that triggered BUFFER_APPENDING, pending : nb of segments waiting for appending for this segment parent} - BUFFER_APPENDED: 'hlsBufferAppended', - // fired when the stream is finished and we want to notify the media buffer that there will be no more data - data: { } - BUFFER_EOS: 'hlsBufferEos', - // fired when the media buffer should be flushed - data { startOffset, endOffset } - BUFFER_FLUSHING: 'hlsBufferFlushing', - // fired when the media buffer has been flushed - data: { } - BUFFER_FLUSHED: 'hlsBufferFlushed', // fired to signal that a manifest loading starts - data: { url : manifestURL} MANIFEST_LOADING: 'hlsManifestLoading', // fired after manifest has been loaded - data: { levels : [available quality levels], audioTracks : [ available audio tracks], url : manifestURL, stats : { trequest, tfirst, tload, mtime}} MANIFEST_LOADED: 'hlsManifestLoaded', - // fired after manifest has been parsed - data: { levels : [available quality levels], firstLevel : index of first quality level appearing in Manifest} - MANIFEST_PARSED: 'hlsManifestParsed', - // fired when a level switch is requested - data: { level : id of new level } - LEVEL_SWITCHING: 'hlsLevelSwitching', - // fired when a level switch is effective - data: { level : id of new level } - LEVEL_SWITCHED: 'hlsLevelSwitched', // fired when a level playlist loading starts - data: { url : level URL, level : id of level being loaded} LEVEL_LOADING: 'hlsLevelLoading', // fired when a level playlist loading finishes - data: { details : levelDetails object, level : id of loaded level, stats : { trequest, tfirst, tload, mtime} } LEVEL_LOADED: 'hlsLevelLoaded', // fired when a level's details have been updated based on previous details, after it has been loaded - data: { details : levelDetails object, level : id of updated level } LEVEL_UPDATED: 'hlsLevelUpdated', - // fired when a level's PTS information has been updated after parsing a fragment - data: { details : levelDetails object, level : id of updated level, drift: PTS drift observed when parsing last fragment } - LEVEL_PTS_UPDATED: 'hlsLevelPtsUpdated', - // fired to notify that audio track lists has been updated - data: { audioTracks : audioTracks } - AUDIO_TRACKS_UPDATED: 'hlsAudioTracksUpdated', - // fired when an audio track switching is requested - data: { id : audio track id } - AUDIO_TRACK_SWITCHING: 'hlsAudioTrackSwitching', - // fired when an audio track switch actually occurs - data: { id : audio track id } - AUDIO_TRACK_SWITCHED: 'hlsAudioTrackSwitched', // fired when an audio track loading starts - data: { url : audio track URL, id : audio track id } AUDIO_TRACK_LOADING: 'hlsAudioTrackLoading', // fired when an audio track loading finishes - data: { details : levelDetails object, id : audio track id, stats : { trequest, tfirst, tload, mtime } } AUDIO_TRACK_LOADED: 'hlsAudioTrackLoaded', - // fired to notify that subtitle track lists has been updated - data: { subtitleTracks : subtitleTracks } - SUBTITLE_TRACKS_UPDATED: 'hlsSubtitleTracksUpdated', - // fired when an subtitle track switch occurs - data: { id : subtitle track id } - SUBTITLE_TRACK_SWITCH: 'hlsSubtitleTrackSwitch', // fired when a subtitle track loading starts - data: { url : subtitle track URL, id : subtitle track id } SUBTITLE_TRACK_LOADING: 'hlsSubtitleTrackLoading', // fired when a subtitle track loading finishes - data: { details : levelDetails object, id : subtitle track id, stats : { trequest, tfirst, tload, mtime } } SUBTITLE_TRACK_LOADED: 'hlsSubtitleTrackLoaded', - // fired when a subtitle fragment has been processed - data: { success : boolean, frag : the processed frag } - SUBTITLE_FRAG_PROCESSED: 'hlsSubtitleFragProcessed', - // fired when the first timestamp is found - data: { id : demuxer id, initPTS: initPTS, frag : fragment object } - INIT_PTS_FOUND: 'hlsInitPtsFound', // fired when a fragment loading starts - data: { frag : fragment object } FRAG_LOADING: 'hlsFragLoading', // fired when a fragment loading is progressing - data: { frag : fragment object, { trequest, tfirst, loaded } } FRAG_LOAD_PROGRESS: 'hlsFragLoadProgress', - // Identifier for fragment load aborting for emergency switch down - data: { frag : fragment object } - FRAG_LOAD_EMERGENCY_ABORTED: 'hlsFragLoadEmergencyAborted', // fired when a fragment loading is completed - data: { frag : fragment object, payload : fragment payload, stats : { trequest, tfirst, tload, length } } - FRAG_LOADED: 'hlsFragLoaded', - // fired when a fragment has finished decrypting - data: { id : demuxer id, frag: fragment object, payload : fragment payload, stats : { tstart, tdecrypt } } - FRAG_DECRYPTED: 'hlsFragDecrypted', - // fired when Init Segment has been extracted from fragment - data: { id : demuxer id, frag: fragment object, moov : moov MP4 box, codecs : codecs found while parsing fragment } - FRAG_PARSING_INIT_SEGMENT: 'hlsFragParsingInitSegment', - // fired when parsing sei text is completed - data: { id : demuxer id, frag: fragment object, samples : [ sei samples pes ] } - FRAG_PARSING_USERDATA: 'hlsFragParsingUserdata', - // fired when parsing id3 is completed - data: { id : demuxer id, frag: fragment object, samples : [ id3 samples pes ] } - FRAG_PARSING_METADATA: 'hlsFragParsingMetadata', - // fired when data have been extracted from fragment - data: { id : demuxer id, frag: fragment object, data1 : moof MP4 box or TS fragments, data2 : mdat MP4 box or null} - FRAG_PARSING_DATA: 'hlsFragParsingData', - // fired when fragment parsing is completed - data: { id : demuxer id, frag: fragment object } - FRAG_PARSED: 'hlsFragParsed', - // fired when fragment remuxed MP4 boxes have all been appended into SourceBuffer - data: { id : demuxer id, frag : fragment object, stats : { trequest, tfirst, tload, tparsed, tbuffered, length, bwEstimate } } - FRAG_BUFFERED: 'hlsFragBuffered', - // fired when fragment matching with current media position is changing - data : { id : demuxer id, frag : fragment object } - FRAG_CHANGED: 'hlsFragChanged', - // Identifier for a FPS drop event - data: { curentDropped, currentDecoded, totalDroppedFrames } - FPS_DROP: 'hlsFpsDrop', - // triggered when FPS drop triggers auto level capping - data: { level, droppedlevel } - FPS_DROP_LEVEL_CAPPING: 'hlsFpsDropLevelCapping', - // Identifier for an error event - data: { type : error type, details : error details, fatal : if true, hls.js cannot/will not try to recover, if false, hls.js will try to recover,other error specific data } - ERROR: 'hlsError', - // fired when hls.js instance starts destroying. Different from MEDIA_DETACHED as one could want to detach and reattach a media to the instance of hls.js to handle mid-rolls for example - data: { } - DESTROYING: 'hlsDestroying', - // fired when a decrypt key loading starts - data: { frag : fragment object } - KEY_LOADING: 'hlsKeyLoading', - // fired when a decrypt key loading is completed - data: { frag : fragment object, payload : key payload, stats : { trequest, tfirst, tload, length } } - KEY_LOADED: 'hlsKeyLoaded', - // fired upon stream controller state transitions - data: { previousState, nextState } - STREAM_STATE_TRANSITION: 'hlsStreamStateTransition' + FRAG_LOADED: 'hlsFragLoaded' }; export default HlsEvents; From 2e297101a7fb6912c237965acbc29be2b7f3ad53 Mon Sep 17 00:00:00 2001 From: Trek H Date: Fri, 24 Jan 2020 19:50:48 +1030 Subject: [PATCH 22/46] mjpeg-player: ts code to js code This PR is for the hls.js files where the only modifications made are for typescript to javascript conversion, and updated import statements. --- cmd/mjpeg-player/hlsjs/event-handler.js | 55 +++-- cmd/mjpeg-player/hlsjs/loader/fragment.js | 151 +++++++------ cmd/mjpeg-player/hlsjs/loader/level-key.js | 43 +++- cmd/mjpeg-player/hlsjs/loader/m3u8-parser.js | 222 ++++++++++--------- cmd/mjpeg-player/hlsjs/observer.js | 27 ++- cmd/mjpeg-player/hlsjs/types/loader.js | 146 +++--------- cmd/mjpeg-player/hlsjs/utils/codecs.js | 29 ++- 7 files changed, 352 insertions(+), 321 deletions(-) diff --git a/cmd/mjpeg-player/hlsjs/event-handler.js b/cmd/mjpeg-player/hlsjs/event-handler.js index 06c089da..24735225 100644 --- a/cmd/mjpeg-player/hlsjs/event-handler.js +++ b/cmd/mjpeg-player/hlsjs/event-handler.js @@ -1,13 +1,32 @@ +/* +AUTHOR + Trek Hopton + +LICENSE + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see http://www.gnu.org/licenses. + + For hls.js Copyright notice and license, see LICENSE file. +*/ + /* * * All objects in the event handling chain should inherit from this class * */ - -import { logger } from './utils/logger'; -import { ErrorTypes, ErrorDetails } from './errors'; -import Event from './events'; -import Hls from './hls'; +import Event from './events.js'; const FORBIDDEN_EVENT_NAMES = { 'hlsEventGeneric': true, @@ -16,11 +35,7 @@ const FORBIDDEN_EVENT_NAMES = { }; class EventHandler { - hls: Hls; - handledEvents: any[]; - useGenericHandler: boolean; - - constructor (hls: Hls, ...events: any[]) { + constructor(hls, ...events) { this.hls = hls; this.onEvent = this.onEvent.bind(this); this.handledEvents = events; @@ -29,20 +44,20 @@ class EventHandler { this.registerListeners(); } - destroy () { + destroy() { this.onHandlerDestroying(); this.unregisterListeners(); this.onHandlerDestroyed(); } - protected onHandlerDestroying () {} - protected onHandlerDestroyed () {} + onHandlerDestroying() { } + onHandlerDestroyed() { } - isEventHandler () { + isEventHandler() { return typeof this.handledEvents === 'object' && this.handledEvents.length && typeof this.onEvent === 'function'; } - registerListeners () { + registerListeners() { if (this.isEventHandler()) { this.handledEvents.forEach(function (event) { if (FORBIDDEN_EVENT_NAMES[event]) { @@ -54,7 +69,7 @@ class EventHandler { } } - unregisterListeners () { + unregisterListeners() { if (this.isEventHandler()) { this.handledEvents.forEach(function (event) { this.hls.off(event, this.onEvent); @@ -65,12 +80,12 @@ class EventHandler { /** * arguments: event (string), data (any) */ - onEvent (event: string, data: any) { + onEvent(event, data) { this.onEventGeneric(event, data); } - onEventGeneric (event: string, data: any) { - let eventToFunction = function (event: string, data: any) { + onEventGeneric(event, data) { + let eventToFunction = function (event, data) { let funcName = 'on' + event.replace('hls', ''); if (typeof this[funcName] !== 'function') { throw new Error(`Event ${event} has no generic handler in this ${this.constructor.name} class (tried ${funcName})`); @@ -81,7 +96,7 @@ class EventHandler { try { eventToFunction.call(this, event, data).call(); } catch (err) { - logger.error(`An internal error happened while handling event ${event}. Error message: "${err.message}". Here is a stacktrace:`, err); + console.error(`An internal error happened while handling event ${event}. Error message: "${err.message}". Here is a stacktrace:`, err); this.hls.trigger(Event.ERROR, { type: ErrorTypes.OTHER_ERROR, details: ErrorDetails.INTERNAL_EXCEPTION, fatal: false, event: event, err: err }); } } diff --git a/cmd/mjpeg-player/hlsjs/loader/fragment.js b/cmd/mjpeg-player/hlsjs/loader/fragment.js index fb7c0532..01fb0509 100644 --- a/cmd/mjpeg-player/hlsjs/loader/fragment.js +++ b/cmd/mjpeg-player/hlsjs/loader/fragment.js @@ -1,69 +1,90 @@ +/* +AUTHOR + Trek Hopton -import { buildAbsoluteURL } from 'url-toolkit'; -import { logger } from '../utils/logger'; -import LevelKey from './level-key'; -import { PlaylistLevelType } from '../types/loader'; +LICENSE + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) -export enum ElementaryStreamTypes { - AUDIO = 'audio', - VIDEO = 'video', + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see http://www.gnu.org/licenses. + + For hls.js Copyright notice and license, see LICENSE file. +*/ + +import URLToolkit from '../../url-toolkit/url-toolkit.js'; +import LevelKey from './level-key.js'; + +export const ElementaryStreamTypes = { + AUDIO: 'audio', + VIDEO: 'video' } export default class Fragment { - private _url: string | null = null; - private _byteRange: number[] | null = null; - private _decryptdata: LevelKey | null = null; + constructor() { + this._url = null; + this._byteRange = null; + this._decryptdata = null; - // Holds the types of data this fragment supports - private _elementaryStreams: Record = { - [ElementaryStreamTypes.AUDIO]: false, - [ElementaryStreamTypes.VIDEO]: false - }; + // Holds the types of data this fragment supports + this._elementaryStreams = { + [ElementaryStreamTypes.AUDIO]: false, + [ElementaryStreamTypes.VIDEO]: false + }; - // deltaPTS tracks the change in presentation timestamp between fragments - public deltaPTS: number = 0; + // deltaPTS tracks the change in presentation timestamp between fragments + this.deltaPTS = 0; - public rawProgramDateTime: string | null = null; - public programDateTime: number | null = null; - public title: string | null = null; - public tagList: Array = []; + this.rawProgramDateTime = null; + this.programDateTime = null; + this.title = null; + this.tagList = []; - // TODO: Move at least baseurl to constructor. - // Currently we do a two-pass construction as use the Fragment class almost like a object for holding parsing state. - // It may make more sense to just use a POJO to keep state during the parsing phase. - // Have Fragment be the representation once we have a known state? - // Something to think on. + // TODO: Move at least baseurl to constructor. + // Currently we do a two-pass construction as use the Fragment class almost like a object for holding parsing state. + // It may make more sense to just use a POJO to keep state during the parsing phase. + // Have Fragment be the representation once we have a known state? + // Something to think on. - // Discontinuity Counter - public cc!: number; + // Discontinuity Counter + this.cc; + this.type; + // relurl is the portion of the URL that comes from inside the playlist. + this.relurl; + // baseurl is the URL to the playlist + this.baseurl; + // EXTINF has to be present for a m3u8 to be considered valid + this.duration; + // When this segment starts in the timeline + this.start; + // sn notates the sequence number for a segment, and if set to a string can be 'initSegment' + this.sn = 0; - public type!: PlaylistLevelType; - // relurl is the portion of the URL that comes from inside the playlist. - public relurl!: string; - // baseurl is the URL to the playlist - public baseurl!: string; - // EXTINF has to be present for a m3u8 to be considered valid - public duration!: number; - // When this segment starts in the timeline - public start!: number; - // sn notates the sequence number for a segment, and if set to a string can be 'initSegment' - public sn: number | 'initSegment' = 0; + this.urlId = 0; + // level matches this fragment to a index playlist + this.level = 0; + // levelkey is the EXT-X-KEY that applies to this segment for decryption + // core difference from the private field _decryptdata is the lack of the initialized IV + // _decryptdata will set the IV for this segment based on the segment number in the fragment + this.levelkey; - public urlId: number = 0; - // level matches this fragment to a index playlist - public level: number = 0; - // levelkey is the EXT-X-KEY that applies to this segment for decryption - // core difference from the private field _decryptdata is the lack of the initialized IV - // _decryptdata will set the IV for this segment based on the segment number in the fragment - public levelkey?: LevelKey; - - // TODO(typescript-xhrloader) - public loader: any; + // TODO(typescript-xhrloader) + this.loader; + } // setByteRange converts a EXT-X-BYTERANGE attribute into a two element array - setByteRange (value: string, previousFrag?: Fragment) { + setByteRange(value, previousFrag) { const params = value.split('@', 2); - const byteRange: number[] = []; + const byteRange = []; if (params.length === 1) { byteRange[0] = previousFrag ? previousFrag.byteRangeEndOffset : 0; } else { @@ -73,19 +94,19 @@ export default class Fragment { this._byteRange = byteRange; } - get url () { + get url() { if (!this._url && this.relurl) { - this._url = buildAbsoluteURL(this.baseurl, this.relurl, { alwaysNormalize: true }); + this._url = URLToolkit.buildAbsoluteURL(this.baseurl, this.relurl, { alwaysNormalize: true }); } return this._url; } - set url (value) { + set url(value) { this._url = value; } - get byteRange (): number[] { + get byteRange() { if (!this._byteRange) { return []; } @@ -96,15 +117,15 @@ export default class Fragment { /** * @type {number} */ - get byteRangeStartOffset () { + get byteRangeStartOffset() { return this.byteRange[0]; } - get byteRangeEndOffset () { + get byteRangeEndOffset() { return this.byteRange[1]; } - get decryptdata (): LevelKey | null { + get decryptdata() { if (!this.levelkey && !this._decryptdata) { return null; } @@ -116,7 +137,7 @@ export default class Fragment { // If the segment was encrypted with AES-128 // It must have an IV defined. We cannot substitute the Segment Number in. if (this.levelkey && this.levelkey.method === 'AES-128' && !this.levelkey.iv) { - logger.warn(`missing IV for initialization segment with method="${this.levelkey.method}" - compliance issue`); + console.warn(`missing IV for initialization segment with method="${this.levelkey.method}" - compliance issue`); } /* @@ -134,7 +155,7 @@ export default class Fragment { return this._decryptdata; } - get endProgramDateTime () { + get endProgramDateTime() { if (this.programDateTime === null) { return null; } @@ -148,21 +169,21 @@ export default class Fragment { return this.programDateTime + (duration * 1000); } - get encrypted () { + get encrypted() { return !!((this.decryptdata && this.decryptdata.uri !== null) && (this.decryptdata.key === null)); } /** * @param {ElementaryStreamTypes} type */ - addElementaryStream (type: ElementaryStreamTypes) { + addElementaryStream(type) { this._elementaryStreams[type] = true; } /** * @param {ElementaryStreamTypes} type */ - hasElementaryStream (type: ElementaryStreamTypes) { + hasElementaryStream(type) { return this._elementaryStreams[type] === true; } @@ -171,7 +192,7 @@ export default class Fragment { * @param {number} segmentNumber - segment number to generate IV with * @returns {Uint8Array} */ - createInitializationVector (segmentNumber: number): Uint8Array { + createInitializationVector(segmentNumber) { let uint8View = new Uint8Array(16); for (let i = 12; i < 16; i++) { @@ -187,7 +208,7 @@ export default class Fragment { * @param segmentNumber - the fragment's segment number * @returns {LevelKey} - an object to be applied as a fragment's decryptdata */ - setDecryptDataFromLevelKey (levelkey: LevelKey, segmentNumber: number): LevelKey { + setDecryptDataFromLevelKey(levelkey, segmentNumber) { let decryptdata = levelkey; if (levelkey && levelkey.method && levelkey.uri && !levelkey.iv) { diff --git a/cmd/mjpeg-player/hlsjs/loader/level-key.js b/cmd/mjpeg-player/hlsjs/loader/level-key.js index f5abc95e..8bfe4331 100644 --- a/cmd/mjpeg-player/hlsjs/loader/level-key.js +++ b/cmd/mjpeg-player/hlsjs/loader/level-key.js @@ -1,22 +1,45 @@ -import { buildAbsoluteURL } from 'url-toolkit'; +/* +AUTHOR + Trek Hopton + +LICENSE + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see http://www.gnu.org/licenses. + + For hls.js Copyright notice and license, see LICENSE file. +*/ + +import URLToolkit from '../../url-toolkit/url-toolkit.js'; export default class LevelKey { - private _uri: string | null = null; + constructor(baseURI, relativeURI) { + this._uri = null; - public baseuri: string; - public reluri: string; - public method: string | null = null; - public key: Uint8Array | null = null; - public iv: Uint8Array | null = null; + this.baseuri; + this.reluri; + this.method = null; + this.key = null; + this.iv = null; - constructor (baseURI: string, relativeURI: string) { this.baseuri = baseURI; this.reluri = relativeURI; } - get uri () { + get uri() { if (!this._uri && this.reluri) { - this._uri = buildAbsoluteURL(this.baseuri, this.reluri, { alwaysNormalize: true }); + this._uri = URLToolkit.buildAbsoluteURL(this.baseuri, this.reluri, { alwaysNormalize: true }); } return this._uri; diff --git a/cmd/mjpeg-player/hlsjs/loader/m3u8-parser.js b/cmd/mjpeg-player/hlsjs/loader/m3u8-parser.js index ea4c2538..8778dc75 100644 --- a/cmd/mjpeg-player/hlsjs/loader/m3u8-parser.js +++ b/cmd/mjpeg-player/hlsjs/loader/m3u8-parser.js @@ -1,14 +1,32 @@ -import * as URLToolkit from 'url-toolkit'; +/* +AUTHOR + Trek Hopton -import Fragment from './fragment'; -import Level from './level'; -import LevelKey from './level-key'; +LICENSE + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) -import AttrList from '../utils/attr-list'; -import { logger } from '../utils/logger'; -import { isCodecType, CodecType } from '../utils/codecs'; -import { MediaPlaylist, AudioGroup, MediaPlaylistType } from '../types/media-playlist'; -import { PlaylistLevelType } from '../types/loader'; + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see http://www.gnu.org/licenses. + + For hls.js Copyright notice and license, see LICENSE file. +*/ + +import URLToolkit from '../../url-toolkit/url-toolkit.js'; +import Fragment from './fragment.js'; +import Level from './level.js'; +import LevelKey from './level-key.js'; +import AttrList from '../utils/attr-list.js'; +import { isCodecType } from '../utils/codecs.js'; /** * M3U8 parser @@ -32,7 +50,7 @@ const LEVEL_PLAYLIST_REGEX_SLOW = /(?:(?:#(EXTM3U))|(?:#EXT-X-(PLAYLIST-TYPE):(. const MP4_REGEX_SUFFIX = /\.(mp4|m4s|m4v|m4a)$/i; export default class M3U8Parser { - static findGroup (groups: Array, mediaGroupId: string): AudioGroup | undefined { + static findGroup(groups, mediaGroupId) { for (let i = 0; i < groups.length; i++) { const group = groups[i]; if (group.id === mediaGroupId) { @@ -41,7 +59,7 @@ export default class M3U8Parser { } } - static convertAVC1ToAVCOTI (codec) { + static convertAVC1ToAVCOTI(codec) { let avcdata = codec.split('.'); let result; if (avcdata.length > 2) { @@ -54,18 +72,18 @@ export default class M3U8Parser { return result; } - static resolve (url, baseUrl) { + static resolve(url, baseUrl) { return URLToolkit.buildAbsoluteURL(baseUrl, url, { alwaysNormalize: true }); } - static parseMasterPlaylist (string: string, baseurl: string) { + static parseMasterPlaylist(string, baseurl) { // TODO(typescript-level) - let levels: Array = []; + let levels = []; MASTER_PLAYLIST_REGEX.lastIndex = 0; // TODO(typescript-level) - function setCodecs (codecs: Array, level: any) { - ['video', 'audio'].forEach((type: CodecType) => { + function setCodecs(codecs, level) { + ['video', 'audio'].forEach((type) => { const filtered = codecs.filter((codec) => isCodecType(codec, type)); if (filtered.length) { const preferred = filtered.filter((codec) => { @@ -81,10 +99,10 @@ export default class M3U8Parser { level.unknownCodecs = codecs; } - let result: RegExpExecArray | null; + let result; while ((result = MASTER_PLAYLIST_REGEX.exec(string)) != null) { // TODO(typescript-level) - const level: any = {}; + const level = {}; const attrs = level.attrs = new AttrList(result[1]); level.url = M3U8Parser.resolve(result[2], baseurl); @@ -108,15 +126,15 @@ export default class M3U8Parser { return levels; } - static parseMasterPlaylistMedia (string: string, baseurl: string, type: MediaPlaylistType, audioGroups: Array = []): Array { - let result: RegExpExecArray | null; - let medias: Array = []; + static parseMasterPlaylistMedia(string, baseurl, type, audioGroups = []) { + let result; + let medias = []; let id = 0; MASTER_PLAYLIST_MEDIA_REGEX.lastIndex = 0; while ((result = MASTER_PLAYLIST_MEDIA_REGEX.exec(string)) !== null) { const attrs = new AttrList(result[1]); if (attrs.TYPE === type) { - const media: MediaPlaylist = { + const media = { id: id++, groupId: attrs['GROUP-ID'], name: attrs.NAME || attrs.LANGUAGE, @@ -146,16 +164,16 @@ export default class M3U8Parser { return medias; } - static parseLevelPlaylist (string: string, baseurl: string, id: number, type: PlaylistLevelType, levelUrlId: number) { + static parseLevelPlaylist(string, baseurl, id, type, levelUrlId) { let currentSN = 0; let totalduration = 0; let level = new Level(baseurl); let discontinuityCounter = 0; - let prevFrag: Fragment | null = null; - let frag: Fragment | null = new Fragment(); - let result: RegExpExecArray | RegExpMatchArray | null; - let i: number; - let levelkey: LevelKey | undefined; + let prevFrag = null; + let frag = new Fragment(); + let result; + let i; + let levelkey; let firstPdtIndex = null; @@ -168,7 +186,7 @@ export default class M3U8Parser { // avoid sliced strings https://github.com/video-dev/hls.js/issues/939 const title = (' ' + result[2]).slice(1); frag.title = title || null; - frag.tagList.push(title ? [ 'INF', duration, title ] : [ 'INF', duration ]); + frag.tagList.push(title ? ['INF', duration, title] : ['INF', duration]); } else if (result[3]) { // url if (Number.isFinite(frag.duration)) { const sn = currentSN++; @@ -209,7 +227,7 @@ export default class M3U8Parser { } else { result = result[0].match(LEVEL_PLAYLIST_REGEX_SLOW); if (!result) { - logger.warn('No matches on slow regex match for level playlist!'); + console.warn('No matches on slow regex match for level playlist!'); continue; } for (i = 1; i < result.length; i++) { @@ -223,84 +241,84 @@ export default class M3U8Parser { const value2 = (' ' + result[i + 2]).slice(1); switch (result[i]) { - case '#': - frag.tagList.push(value2 ? [ value1, value2 ] : [ value1 ]); - break; - case 'PLAYLIST-TYPE': - level.type = value1.toUpperCase(); - break; - case 'MEDIA-SEQUENCE': - currentSN = level.startSN = parseInt(value1); - break; - case 'TARGETDURATION': - level.targetduration = parseFloat(value1); - break; - case 'VERSION': - level.version = parseInt(value1); - break; - case 'EXTM3U': - break; - case 'ENDLIST': - level.live = false; - break; - case 'DIS': - discontinuityCounter++; - frag.tagList.push(['DIS']); - break; - case 'DISCONTINUITY-SEQ': - discontinuityCounter = parseInt(value1); - break; - case 'KEY': { - // https://tools.ietf.org/html/draft-pantos-http-live-streaming-08#section-3.4.4 - const decryptparams = value1; - const keyAttrs = new AttrList(decryptparams); - const decryptmethod = keyAttrs.enumeratedString('METHOD'); - const decrypturi = keyAttrs.URI; - const decryptiv = keyAttrs.hexadecimalInteger('IV'); + case '#': + frag.tagList.push(value2 ? [value1, value2] : [value1]); + break; + case 'PLAYLIST-TYPE': + level.type = value1.toUpperCase(); + break; + case 'MEDIA-SEQUENCE': + currentSN = level.startSN = parseInt(value1); + break; + case 'TARGETDURATION': + level.targetduration = parseFloat(value1); + break; + case 'VERSION': + level.version = parseInt(value1); + break; + case 'EXTM3U': + break; + case 'ENDLIST': + level.live = false; + break; + case 'DIS': + discontinuityCounter++; + frag.tagList.push(['DIS']); + break; + case 'DISCONTINUITY-SEQ': + discontinuityCounter = parseInt(value1); + break; + case 'KEY': { + // https://tools.ietf.org/html/draft-pantos-http-live-streaming-08#section-3.4.4 + const decryptparams = value1; + const keyAttrs = new AttrList(decryptparams); + const decryptmethod = keyAttrs.enumeratedString('METHOD'); + const decrypturi = keyAttrs.URI; + const decryptiv = keyAttrs.hexadecimalInteger('IV'); - if (decryptmethod) { - levelkey = new LevelKey(baseurl, decrypturi); - if ((decrypturi) && (['AES-128', 'SAMPLE-AES', 'SAMPLE-AES-CENC'].indexOf(decryptmethod) >= 0)) { - levelkey.method = decryptmethod; - levelkey.key = null; - // Initialization Vector (IV) - levelkey.iv = decryptiv; + if (decryptmethod) { + levelkey = new LevelKey(baseurl, decrypturi); + if ((decrypturi) && (['AES-128', 'SAMPLE-AES', 'SAMPLE-AES-CENC'].indexOf(decryptmethod) >= 0)) { + levelkey.method = decryptmethod; + levelkey.key = null; + // Initialization Vector (IV) + levelkey.iv = decryptiv; + } } + break; } - break; - } - case 'START': { - const startAttrs = new AttrList(value1); - const startTimeOffset = startAttrs.decimalFloatingPoint('TIME-OFFSET'); - // TIME-OFFSET can be 0 - if (Number.isFinite(startTimeOffset)) { - level.startTimeOffset = startTimeOffset; + case 'START': { + const startAttrs = new AttrList(value1); + const startTimeOffset = startAttrs.decimalFloatingPoint('TIME-OFFSET'); + // TIME-OFFSET can be 0 + if (Number.isFinite(startTimeOffset)) { + level.startTimeOffset = startTimeOffset; + } + break; } - break; - } - case 'MAP': { - const mapAttrs = new AttrList(value1); - frag.relurl = mapAttrs.URI; - if (mapAttrs.BYTERANGE) { - frag.setByteRange(mapAttrs.BYTERANGE); + case 'MAP': { + const mapAttrs = new AttrList(value1); + frag.relurl = mapAttrs.URI; + if (mapAttrs.BYTERANGE) { + frag.setByteRange(mapAttrs.BYTERANGE); + } + frag.baseurl = baseurl; + frag.level = id; + frag.type = type; + frag.sn = 'initSegment'; + level.initSegment = frag; + frag = new Fragment(); + frag.rawProgramDateTime = level.initSegment.rawProgramDateTime; + break; } - frag.baseurl = baseurl; - frag.level = id; - frag.type = type; - frag.sn = 'initSegment'; - level.initSegment = frag; - frag = new Fragment(); - frag.rawProgramDateTime = level.initSegment.rawProgramDateTime; - break; - } - default: - logger.warn(`line parsed but not handled: ${result}`); - break; + default: + console.warn(`line parsed but not handled: ${result}`); + break; } } } frag = prevFrag; - // logger.log('found ' + level.fragments.length + ' fragments'); + // console.log('found ' + level.fragments.length + ' fragments'); if (frag && !frag.relurl) { level.fragments.pop(); totalduration -= frag.duration; @@ -316,7 +334,7 @@ export default class M3U8Parser { // if the fragments are TS or MP4, except if we download them :/ // but this is to be able to handle SIDX. if (level.fragments.every((frag) => MP4_REGEX_SUFFIX.test(frag.relurl))) { - logger.warn('MP4 fragments found but no init segment (probably no MAP, incomplete M3U8), trying to fetch SIDX'); + console.warn('MP4 fragments found but no init segment (probably no MAP, incomplete M3U8), trying to fetch SIDX'); frag = new Fragment(); frag.relurl = level.fragments[0].relurl; @@ -347,7 +365,7 @@ export default class M3U8Parser { } } -function backfillProgramDateTimes (fragments, startIndex) { +function backfillProgramDateTimes(fragments, startIndex) { let fragPrev = fragments[startIndex]; for (let i = startIndex - 1; i >= 0; i--) { const frag = fragments[i]; @@ -356,7 +374,7 @@ function backfillProgramDateTimes (fragments, startIndex) { } } -function assignProgramDateTime (frag, prevFrag) { +function assignProgramDateTime(frag, prevFrag) { if (frag.rawProgramDateTime) { frag.programDateTime = Date.parse(frag.rawProgramDateTime); } else if (prevFrag && prevFrag.programDateTime) { diff --git a/cmd/mjpeg-player/hlsjs/observer.js b/cmd/mjpeg-player/hlsjs/observer.js index 33d265eb..9f300d23 100644 --- a/cmd/mjpeg-player/hlsjs/observer.js +++ b/cmd/mjpeg-player/hlsjs/observer.js @@ -1,4 +1,27 @@ -import { EventEmitter } from 'eventemitter3'; +/* +AUTHOR + Trek Hopton + +LICENSE + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see http://www.gnu.org/licenses. + + For hls.js Copyright notice and license, see LICENSE file. +*/ + +import EventEmitter from '../eventemitter3/index.js'; /** * Simple adapter sub-class of Nodejs-like EventEmitter. @@ -9,7 +32,7 @@ export class Observer extends EventEmitter { * in every call to a handler, which is the purpose of our `trigger` method * extending the standard API. */ - trigger (event: string, ...data: Array): void { + trigger(event, ...data) { this.emit(event, event, ...data); } } diff --git a/cmd/mjpeg-player/hlsjs/types/loader.js b/cmd/mjpeg-player/hlsjs/types/loader.js index f99aa1da..2a23cb1b 100644 --- a/cmd/mjpeg-player/hlsjs/types/loader.js +++ b/cmd/mjpeg-player/hlsjs/types/loader.js @@ -1,132 +1,42 @@ -import Level from '../loader/level'; +/* +AUTHOR + Trek Hopton -export interface LoaderContext { - // target URL - url: string - // loader response type (arraybuffer or default response type for playlist) - responseType: string - // start byte range offset - rangeStart?: number - // end byte range offset - rangeEnd?: number - // true if onProgress should report partial chunk of loaded content - progressData?: boolean -} +LICENSE + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) -export interface LoaderConfiguration { - // Max number of load retries - maxRetry: number - // Timeout after which `onTimeOut` callback will be triggered - // (if loading is still not finished after that delay) - timeout: number - // Delay between an I/O error and following connection retry (ms). - // This to avoid spamming the server - retryDelay: number - // max connection retry delay (ms) - maxRetryDelay: number -} + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. -export interface LoaderResponse { - url: string, - // TODO(jstackhouse): SharedArrayBuffer, es2017 extension to TS - data: string | ArrayBuffer -} + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. -export interface LoaderStats { - // performance.now() just after load() has been called - trequest: number - // performance.now() of first received byte - tfirst: number - // performance.now() on load complete - tload: number - // performance.now() on parse completion - tparsed: number - // number of loaded bytes - loaded: number - // total number of bytes - total: number -} + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see http://www.gnu.org/licenses. -type LoaderOnSuccess < T extends LoaderContext > = ( - response: LoaderResponse, - stats: LoaderStats, - context: T, - networkDetails: any -) => void; - -type LoaderOnProgress < T extends LoaderContext > = ( - stats: LoaderStats, - context: T, - data: string | ArrayBuffer, - networkDetails: any, -) => void; - -type LoaderOnError < T extends LoaderContext > = ( - error: { - // error status code - code: number, - // error description - text: string, - }, - context: T, - networkDetails: any, -) => void; - -type LoaderOnTimeout < T extends LoaderContext > = ( - stats: LoaderStats, - context: T, -) => void; - -export interface LoaderCallbacks{ - onSuccess: LoaderOnSuccess, - onError: LoaderOnError, - onTimeout: LoaderOnTimeout, - onProgress?: LoaderOnProgress, -} - -export interface Loader { - destroy(): void - abort(): void - load( - context: LoaderContext, - config: LoaderConfiguration, - callbacks: LoaderCallbacks, - ): void - - context: T -} + For hls.js Copyright notice and license, see LICENSE file. +*/ /** - * `type` property values for this loaders' context object - * @enum - * + * @readonly + * @enum {string} */ -export enum PlaylistContextType { - MANIFEST = 'manifest', - LEVEL = 'level', - AUDIO_TRACK = 'audioTrack', - SUBTITLE_TRACK= 'subtitleTrack' +export const PlaylistContextType = { + MANIFEST: 'manifest', + LEVEL: 'level', + AUDIO_TRACK: 'audioTrack', + SUBTITLE_TRACK: 'subtitleTrack' } /** * @enum {string} */ -export enum PlaylistLevelType { - MAIN = 'main', - AUDIO = 'audio', - SUBTITLE = 'subtitle' -} - -export interface PlaylistLoaderContext extends LoaderContext { - loader?: Loader - - type: PlaylistContextType - // the level index to load - level: number | null - // TODO: what is id? - id: number | null - // defines if the loader is handling a sidx request for the playlist - isSidxRequest?: boolean - // internal reprsentation of a parsed m3u8 level playlist - levelDetails?: Level +export const PlaylistLevelType = { + MAIN: 'main', + AUDIO: 'audio', + SUBTITLE: 'subtitle' } diff --git a/cmd/mjpeg-player/hlsjs/utils/codecs.js b/cmd/mjpeg-player/hlsjs/utils/codecs.js index 60ae8a94..a9b345aa 100644 --- a/cmd/mjpeg-player/hlsjs/utils/codecs.js +++ b/cmd/mjpeg-player/hlsjs/utils/codecs.js @@ -1,3 +1,26 @@ +/* +AUTHOR + Trek Hopton + +LICENSE + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see http://www.gnu.org/licenses. + + For hls.js Copyright notice and license, see LICENSE file. +*/ + // from http://mp4ra.org/codecs.html const sampleEntryCodesISO = { audio: { @@ -63,14 +86,12 @@ const sampleEntryCodesISO = { } }; -export type CodecType = 'audio' | 'video'; - -function isCodecType (codec: string, type: CodecType): boolean { +function isCodecType(codec, type) { const typeCodes = sampleEntryCodesISO[type]; return !!typeCodes && typeCodes[codec.slice(0, 4)] === true; } -function isCodecSupportedInMp4 (codec: string, type: CodecType): boolean { +function isCodecSupportedInMp4(codec, type) { return MediaSource.isTypeSupported(`${type || 'video'}/mp4;codecs="${codec}"`); } From 7126f7aa12898c38f6565306477bbca678ee561e Mon Sep 17 00:00:00 2001 From: Trek H Date: Fri, 24 Jan 2020 20:02:17 +1030 Subject: [PATCH 23/46] mjpeg-player: mtsdemuxer functionality change Changed MTSDemuxer's 'append' function to 'demux' because it is not appending. Updated MTSDemuxer so that it doesn't store the stream after demuxing it, it simply demuxes the data that it's given and returns the demuxed data. Any PES packets that were truncated are kept to be tried at the next call to demux. --- cmd/mjpeg-player/hlsjs/mts-demuxer.js | 52 ++++++++++----------------- 1 file changed, 19 insertions(+), 33 deletions(-) diff --git a/cmd/mjpeg-player/hlsjs/mts-demuxer.js b/cmd/mjpeg-player/hlsjs/mts-demuxer.js index 22ac59ca..5fe7c351 100644 --- a/cmd/mjpeg-player/hlsjs/mts-demuxer.js +++ b/cmd/mjpeg-player/hlsjs/mts-demuxer.js @@ -1,12 +1,9 @@ /* -NAME - mts-demuxer.js - AUTHOR Trek Hopton LICENSE - This file is Copyright (C) 2019 the Australian Ocean Lab (AusOcean) + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) It is free software: you can redistribute it and/or modify them under the terms of the GNU General Public License as published by the @@ -36,10 +33,6 @@ class MTSDemuxer { init() { this.pmtParsed = false; this._pmtId = -1; - - this._videoTrack = MTSDemuxer.createTrack('video'); - this._audioTrack = MTSDemuxer.createTrack('audio'); - this._id3Track = MTSDemuxer.createTrack('id3'); } // createTrack creates and returns a track model. @@ -55,15 +48,6 @@ class MTSDemuxer { }; } - // _getTracks returns this MTSDemuxer's tracks. - _getTracks() { - return { - video: this._videoTrack, - audio: this._audioTrack, - id3: this._id3Track - }; - } - // _syncOffset scans the first 'maxScanWindow' bytes and returns an offset to the beginning of the first three MTS packets, // or -1 if three are not found. // A TS fragment should contain at least 3 TS packets, a PAT, a PMT, and one PID, each starting with 0x47. @@ -81,20 +65,20 @@ class MTSDemuxer { return -1; } - append(data) { + demux(data) { let start, len = data.length, pusi, pid, afc, offset, pes, unknownPIDs = false; let pmtParsed = this.pmtParsed, - videoTrack = this._videoTrack, - audioTrack = this._audioTrack, - id3Track = this._id3Track, - videoId = videoTrack.pid, - audioId = audioTrack.pid, - id3Id = id3Track.pid, + videoTrack = MTSDemuxer.createTrack('video'), + audioTrack = MTSDemuxer.createTrack('audio'), + id3Track = MTSDemuxer.createTrack('id3'), + videoId, + audioId, + id3Id, pmtId = this._pmtId, - videoData = videoTrack.pesData, - audioData = audioTrack.pesData, - id3Data = id3Track.pesData, + videoData = this.videoPesData, + audioData = this.audioPesData, + id3Data = this.id3PesData, parsePAT = this._parsePAT, parsePMT = this._parsePMT, parsePES = this._parsePES; @@ -209,27 +193,29 @@ class MTSDemuxer { // Try to parse last PES packets. if (videoData && (pes = parsePES(videoData)) && pes.pts !== undefined) { videoTrack.data.push(pes.data); - videoTrack.pesData = null; + this.videoPesData = null; } else { // Either pesPkts null or PES truncated, keep it for next frag parsing. - videoTrack.pesData = videoData; + this.videoPesData = videoData; } if (audioData && (pes = parsePES(audioData)) && pes.pts !== undefined) { audioTrack.data.push(pes.data); - audioTrack.pesData = null; + this.audioPesData = null; } else { // Either pesPkts null or PES truncated, keep it for next frag parsing. - audioTrack.pesData = audioData; + this.audioPesData = audioData; } if (id3Data && (pes = parsePES(id3Data)) && pes.pts !== undefined) { id3Track.data.push(pes.data); - id3Track.pesData = null; + this.id3PesData = null; } else { // Either pesPkts null or PES truncated, keep it for next frag parsing. - id3Track.pesData = id3Data; + this.id3PesData = id3Data; } + + return videoTrack; } _parsePAT(data, offset) { From 0f5aaf6cb5afac81fbb1d1b871d4d08313db3681 Mon Sep 17 00:00:00 2001 From: Saxon Date: Fri, 24 Jan 2020 20:05:43 +1030 Subject: [PATCH 24/46] revid & cmd/revid-cli: added InputFPS config.Config field The InputFPS field can control rate at which we lex frames from the input source. This has not been a useful feature until now; we now want to simulate realtime input device using file input. This requires firstly the Loop mode, and now also realistic input rate. --- cmd/revid-cli/main.go | 5 +++-- revid/config/config.go | 14 +++++++++++++- revid/revid.go | 5 ++++- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index fc4d863f..d0b32e9f 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -133,6 +133,7 @@ func handleFlags() config.Config { saturationPtr = flag.Int("Saturation", 0, "Set Saturation. (100-100)") exposurePtr = flag.String("Exposure", "auto", "Set exposure mode. ("+strings.Join(raspivid.ExposureModes[:], ",")+")") autoWhiteBalancePtr = flag.String("Awb", "auto", "Set automatic white balance mode. ("+strings.Join(raspivid.AutoWhiteBalanceModes[:], ",")+")") + inputFPSPtr = flag.Int("InputFPS", 0, "Input source processing FPS") // Audio specific flags. sampleRatePtr = flag.Int("SampleRate", 48000, "Sample rate of recorded audio") @@ -180,8 +181,6 @@ func handleFlags() config.Config { } } - cfg.Loop = *loopPtr - switch *inputPtr { case "Raspivid": cfg.Input = config.InputRaspivid @@ -238,6 +237,8 @@ func handleFlags() config.Config { netsender.ConfigFile = *configFilePtr } + cfg.InputFPS = *inputFPSPtr + cfg.Loop = *loopPtr cfg.CameraIP = *cameraIPPtr cfg.Rotation = *rotationPtr cfg.HorizontalFlip = *horizontalFlipPtr diff --git a/revid/config/config.go b/revid/config/config.go index 47a8eaf9..8ee31ee5 100644 --- a/revid/config/config.go +++ b/revid/config/config.go @@ -86,6 +86,7 @@ const ( defaultAudioInputCodec = codecutil.ADPCM defaultPSITime = 2 defaultMotionInterval = 5 + defaultInputFPS = 0 // Ring buffer defaults. defaultRBMaxElements = 10000 @@ -300,6 +301,10 @@ type Config struct { // If true will restart reading of input after an io.EOF. Loop bool + + // Defines the rate at which an input source is processed. If reading + // from a realtime source, InputFPS is not necessary and should be 0 (default). + InputFPS int } // TypeData contains information about all of the variables that @@ -321,15 +326,17 @@ var TypeData = map[string]string{ "HTTPAddress": "string", "Input": "enum:raspivid,rtsp,v4l,file", "InputCodec": "enum:H264,MJPEG", + "InputFPS": "int", "InputPath": "string", "KNNHistory": "uint", "KNNKernel": "float", "KNNMinArea": "float", "KNNThreshold": "float", "logging": "enum:Debug,Info,Warning,Error,Fatal", + "Loop": "bool", "MinFPS": "float", "MinFrames": "uint", - "mode": "enum:Normal,Paused,Burst", + "mode": "enum:Normal,Paused,Burst,Loop", "MOGHistory": "uint", "MOGMinArea": "float", "MOGThreshold": "float", @@ -524,6 +531,11 @@ func (c *Config) Validate() error { } } + if c.InputFPS <= 0 { + c.Logger.Log(logger.Info, pkg+"InputFPS bad or unset, defaulting", "InputFPS", defaultInputFPS) + c.InputFPS = defaultInputFPS + } + return nil } diff --git a/revid/revid.go b/revid/revid.go index 9c414480..b442e34a 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -428,8 +428,11 @@ func (r *Revid) Start() error { return err } + // Calculate delay between frames based on InputFPS. + d := time.Duration(1000/r.cfg.InputFPS) * time.Millisecond + r.wg.Add(1) - go r.processFrom(r.input, (1000/25)*time.Millisecond) + go r.processFrom(r.input, d) r.running = true return nil From 58747c0b059fef090fe455b50be10d8f0db1d26e Mon Sep 17 00:00:00 2001 From: Trek H Date: Fri, 24 Jan 2020 20:07:54 +1030 Subject: [PATCH 25/46] mjpeg-player: playlist loader modification Typescript code has been removed. Functionality that is not being used has been removed. --- .../hlsjs/loader/playlist-loader.js | 505 ++++++------------ 1 file changed, 159 insertions(+), 346 deletions(-) diff --git a/cmd/mjpeg-player/hlsjs/loader/playlist-loader.js b/cmd/mjpeg-player/hlsjs/loader/playlist-loader.js index cd8c6c2b..e34f236b 100644 --- a/cmd/mjpeg-player/hlsjs/loader/playlist-loader.js +++ b/cmd/mjpeg-player/hlsjs/loader/playlist-loader.js @@ -1,71 +1,95 @@ -/** - * PlaylistLoader - delegate for media manifest/playlist loading tasks. Takes care of parsing media to internal data-models. - * - * Once loaded, dispatches events with parsed data-models of manifest/levels/audio/subtitle tracks. - * - * Uses loader(s) set in config to do actual internal loading of resource tasks. - * - * @module - * - */ +/* +AUTHOR + Trek Hopton -import Event from '../events'; -import EventHandler from '../event-handler'; -import { ErrorTypes, ErrorDetails } from '../errors'; -import { logger } from '../utils/logger'; -import { Loader, PlaylistContextType, PlaylistLoaderContext, PlaylistLevelType, LoaderCallbacks, LoaderResponse, LoaderStats, LoaderConfiguration } from '../types/loader'; -import MP4Demuxer from '../demux/mp4demuxer'; -import M3U8Parser from './m3u8-parser'; -import { AudioGroup } from '../types/media-playlist'; +LICENSE + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see http://www.gnu.org/licenses. + + For hls.js Copyright notice and license, see LICENSE file. +*/ + +import { PlaylistContextType, PlaylistLevelType } from '../types/loader.js'; +import Event from '../events.js'; +import EventHandler from '../event-handler.js'; +import M3U8Parser from './m3u8-parser.js'; const { performance } = window; -/** - * @constructor - */ class PlaylistLoader extends EventHandler { - private loaders: Partial>> = {}; - - /** - * @constructs - * @param {Hls} hls - */ - constructor (hls) { + constructor(hls) { super(hls, Event.MANIFEST_LOADING, Event.LEVEL_LOADING, Event.AUDIO_TRACK_LOADING, Event.SUBTITLE_TRACK_LOADING); + this.hls = hls; + this.loaders = {}; } /** - * @param {PlaylistContextType} type - * @returns {boolean} - */ - static canHaveQualityLevels (type: PlaylistContextType): boolean { + * @param {PlaylistContextType} type + * @returns {boolean} + */ + static canHaveQualityLevels(type) { return (type !== PlaylistContextType.AUDIO_TRACK && type !== PlaylistContextType.SUBTITLE_TRACK); } - /** - * Map context.type to LevelType - * @param {PlaylistLoaderContext} context - * @returns {LevelType} - */ - static mapContextToLevelType (context: PlaylistLoaderContext): PlaylistLevelType { - const { type } = context; - - switch (type) { - case PlaylistContextType.AUDIO_TRACK: - return PlaylistLevelType.AUDIO; - case PlaylistContextType.SUBTITLE_TRACK: - return PlaylistLevelType.SUBTITLE; - default: - return PlaylistLevelType.MAIN; - } + onManifestLoading(data) { + this.load({ + url: data.url, + type: PlaylistContextType.MANIFEST, + level: 0, + id: null, + responseType: 'text' + }); } - static getResponseUrl (response: LoaderResponse, context: PlaylistLoaderContext): string { + // param 1 -> data: { url: string; level: number | null; id: number | null; } + onLevelLoading(data) { + this.load({ + url: data.url, + type: PlaylistContextType.LEVEL, + level: data.level, + id: data.id, + responseType: 'text' + }); + } + + onAudioTrackLoading(data) { + this.load({ + url: data.url, + type: PlaylistContextType.AUDIO_TRACK, + level: null, + id: data.id, + responseType: 'text' + }); + } + + onSubtitleTrackLoading(data) { + this.load({ + url: data.url, + type: PlaylistContextType.SUBTITLE_TRACK, + level: null, + id: data.id, + responseType: 'text' + }); + } + + static getResponseUrl(response, context) { let url = response.url; // responseURL not supported on some browsers (it is used to detect URL redirection) // data-uri mode also not supported (but no need to detect redirection) @@ -76,13 +100,35 @@ class PlaylistLoader extends EventHandler { return url; } + /** + * Map context.type to LevelType + * @param {PlaylistLoaderContext} context + * @returns {LevelType} + */ + static mapContextToLevelType(context) { + const { type } = context; + + switch (type) { + case PlaylistContextType.AUDIO_TRACK: + return PlaylistLevelType.AUDIO; + case PlaylistContextType.SUBTITLE_TRACK: + return PlaylistLevelType.SUBTITLE; + default: + return PlaylistLevelType.MAIN; + } + } + + getInternalLoader(context) { + return this.loaders[context.type]; + } + /** * Returns defaults or configured loader-type overloads (pLoader and loader config params) * Default loader is XHRLoader (see utils) * @param {PlaylistLoaderContext} context * @returns {Loader} or other compatible configured overload */ - createInternalLoader (context: PlaylistLoaderContext): Loader { + createInternalLoader(context) { const config = this.hls.config; const PLoader = config.pLoader; const Loader = config.loader; @@ -98,146 +144,77 @@ class PlaylistLoader extends EventHandler { return loader; } - getInternalLoader (context: PlaylistLoaderContext): Loader | undefined { - return this.loaders[context.type]; - } - - resetInternalLoader (contextType: PlaylistContextType) { + resetInternalLoader(contextType) { if (this.loaders[contextType]) { delete this.loaders[contextType]; } } - /** - * Call `destroy` on all internal loader instances mapped (one per context type) - */ - destroyInternalLoaders () { - for (let contextType in this.loaders) { - let loader = this.loaders[contextType]; - if (loader) { - loader.destroy(); - } - - this.resetInternalLoader(contextType as PlaylistContextType); - } - } - - destroy () { - this.destroyInternalLoaders(); - - super.destroy(); - } - - onManifestLoading (data: { url: string; }) { - this.load({ - url: data.url, - type: PlaylistContextType.MANIFEST, - level: 0, - id: null, - responseType: 'text' - }); - } - - onLevelLoading (data: { url: string; level: number | null; id: number | null; }) { - this.load({ - url: data.url, - type: PlaylistContextType.LEVEL, - level: data.level, - id: data.id, - responseType: 'text' - }); - } - - onAudioTrackLoading (data: { url: string; id: number | null; }) { - this.load({ - url: data.url, - type: PlaylistContextType.AUDIO_TRACK, - level: null, - id: data.id, - responseType: 'text' - }); - } - - onSubtitleTrackLoading (data: { url: string; id: number | null; }) { - this.load({ - url: data.url, - type: PlaylistContextType.SUBTITLE_TRACK, - level: null, - id: data.id, - responseType: 'text' - }); - } - - load (context: PlaylistLoaderContext): boolean { + load(context) { const config = this.hls.config; - logger.debug(`Loading playlist of type ${context.type}, level: ${context.level}, id: ${context.id}`); - // Check if a loader for this context already exists let loader = this.getInternalLoader(context); if (loader) { const loaderContext = loader.context; if (loaderContext && loaderContext.url === context.url) { // same URL can't overlap - logger.trace('playlist request ongoing'); return false; } else { - logger.warn(`aborting previous loader for type: ${context.type}`); loader.abort(); } } - let maxRetry: number; - let timeout: number; - let retryDelay: number; - let maxRetryDelay: number; + let maxRetry; + let timeout; + let retryDelay; + let maxRetryDelay; // apply different configs for retries depending on // context (manifest, level, audio/subs playlist) switch (context.type) { - case PlaylistContextType.MANIFEST: - maxRetry = config.manifestLoadingMaxRetry; - timeout = config.manifestLoadingTimeOut; - retryDelay = config.manifestLoadingRetryDelay; - maxRetryDelay = config.manifestLoadingMaxRetryTimeout; - break; - case PlaylistContextType.LEVEL: - // Disable internal loader retry logic, since we are managing retries in Level Controller - maxRetry = 0; - maxRetryDelay = 0; - retryDelay = 0; - timeout = config.levelLoadingTimeOut; - // TODO Introduce retry settings for audio-track and subtitle-track, it should not use level retry config - break; - default: - maxRetry = config.levelLoadingMaxRetry; - timeout = config.levelLoadingTimeOut; - retryDelay = config.levelLoadingRetryDelay; - maxRetryDelay = config.levelLoadingMaxRetryTimeout; - break; + case PlaylistContextType.MANIFEST: + maxRetry = config.manifestLoadingMaxRetry; + timeout = config.manifestLoadingTimeOut; + retryDelay = config.manifestLoadingRetryDelay; + maxRetryDelay = config.manifestLoadingMaxRetryTimeout; + break; + case PlaylistContextType.LEVEL: + // Disable internal loader retry logic, since we are managing retries in Level Controller + maxRetry = 0; + maxRetryDelay = 0; + retryDelay = 0; + timeout = config.levelLoadingTimeOut; + // TODO Introduce retry settings for audio-track and subtitle-track, it should not use level retry config + break; + default: + maxRetry = config.levelLoadingMaxRetry; + timeout = config.levelLoadingTimeOut; + retryDelay = config.levelLoadingRetryDelay; + maxRetryDelay = config.levelLoadingMaxRetryTimeout; + break; } loader = this.createInternalLoader(context); - const loaderConfig: LoaderConfiguration = { + const loaderConfig = { timeout, maxRetry, retryDelay, maxRetryDelay }; - const loaderCallbacks: LoaderCallbacks = { + const loaderCallbacks = { onSuccess: this.loadsuccess.bind(this), onError: this.loaderror.bind(this), onTimeout: this.loadtimeout.bind(this) }; - logger.debug(`Calling internal loader delegate for URL: ${context.url}`); loader.load(context, loaderConfig, loaderCallbacks); return true; } - loadsuccess (response: LoaderResponse, stats: LoaderStats, context: PlaylistLoaderContext, networkDetails: unknown = null) { + loadsuccess(response, stats, context, networkDetails = null) { if (context.isSidxRequest) { this._handleSidxRequest(response, context); this._handlePlaylistLoaded(response, stats, context, networkDetails); @@ -252,11 +229,10 @@ class PlaylistLoader extends EventHandler { const string = response.data; stats.tload = performance.now(); - // stats.mtime = new Date(target.getResponseHeader('Last-Modified')); // Validate if it is an M3U8 at all if (string.indexOf('#EXTM3U') !== 0) { - this._handleManifestParsingError(response, context, 'no EXTM3U delimiter', networkDetails); + console.error("no EXTM3U delimiter"); return; } @@ -264,77 +240,21 @@ class PlaylistLoader extends EventHandler { if (string.indexOf('#EXTINF:') > 0 || string.indexOf('#EXT-X-TARGETDURATION:') > 0) { this._handleTrackOrLevelPlaylist(response, stats, context, networkDetails); } else { - this._handleMasterPlaylist(response, stats, context, networkDetails); - } - } - - loaderror (response: LoaderResponse, context: PlaylistLoaderContext, networkDetails = null) { - this._handleNetworkError(context, networkDetails, false, response); - } - - loadtimeout (stats: LoaderStats, context: PlaylistLoaderContext, networkDetails = null) { - this._handleNetworkError(context, networkDetails, true); - } - - // TODO(typescript-config): networkDetails can currently be a XHR or Fetch impl, - // but with custom loaders it could be generic investigate this further when config is typed - _handleMasterPlaylist (response: LoaderResponse, stats: LoaderStats, context: PlaylistLoaderContext, networkDetails: unknown) { - const hls = this.hls; - const string = response.data as string; - - const url = PlaylistLoader.getResponseUrl(response, context); - const levels = M3U8Parser.parseMasterPlaylist(string, url); - if (!levels.length) { - this._handleManifestParsingError(response, context, 'no level found in manifest', networkDetails); - return; + console.log("handling of master playlists is not implemented"); + // this._handleMasterPlaylist(response, stats, context, networkDetails); } - // multi level playlist, parse level info - const audioGroups: Array = levels.map(level => ({ - id: level.attrs.AUDIO, - codec: level.audioCodec - })); - - const audioTracks = M3U8Parser.parseMasterPlaylistMedia(string, url, 'AUDIO', audioGroups); - const subtitles = M3U8Parser.parseMasterPlaylistMedia(string, url, 'SUBTITLES'); - - if (audioTracks.length) { - // check if we have found an audio track embedded in main playlist (audio track without URI attribute) - let embeddedAudioFound = false; - audioTracks.forEach(audioTrack => { - if (!audioTrack.url) { - embeddedAudioFound = true; - } - }); - - // if no embedded audio track defined, but audio codec signaled in quality level, - // we need to signal this main audio track this could happen with playlists with - // alt audio rendition in which quality levels (main) - // contains both audio+video. but with mixed audio track not signaled - if (embeddedAudioFound === false && levels[0].audioCodec && !levels[0].attrs.AUDIO) { - logger.log('audio codec signaled in quality level, but no embedded audio track signaled, create one'); - audioTracks.unshift({ - type: 'main', - name: 'main', - default: false, - autoselect: false, - forced: false, - id: -1 - }); - } - } - - hls.trigger(Event.MANIFEST_LOADED, { - levels, - audioTracks, - subtitles, - url, - stats, - networkDetails - }); } - _handleTrackOrLevelPlaylist (response: LoaderResponse, stats: LoaderStats, context: PlaylistLoaderContext, networkDetails: unknown) { + loaderror(response, context, networkDetails = null) { + console.error("network error while loading", response); + } + + loadtimeout(stats, context, networkDetails = null) { + console.error("network timeout while loading", stats); + } + + _handleTrackOrLevelPlaylist(response, stats, context, networkDetails) { const hls = this.hls; const { id, level, type } = context; @@ -342,15 +262,15 @@ class PlaylistLoader extends EventHandler { const url = PlaylistLoader.getResponseUrl(response, context); // if the values are null, they will result in the else conditional - const levelUrlId = Number.isFinite(id as number) ? id as number : 0; - const levelId = Number.isFinite(level as number) ? level as number : levelUrlId; + const levelUrlId = Number.isFinite(id) ? id : 0; + const levelId = Number.isFinite(level) ? level : levelUrlId; const levelType = PlaylistLoader.mapContextToLevelType(context); - const levelDetails = M3U8Parser.parseLevelPlaylist(response.data as string, url, levelId, levelType, levelUrlId); + const levelDetails = M3U8Parser.parseLevelPlaylist(response.data, url, levelId, levelType, levelUrlId); // set stats on level structure // TODO(jstackhouse): why? mixing concerns, is it just treated as value bag? - (levelDetails as any).tload = stats.tload; + (levelDetails).tload = stats.tload; // We have done our first request (Manifest-type) and receive // not a master playlist but a chunk-list (track/level) @@ -374,124 +294,17 @@ class PlaylistLoader extends EventHandler { // save parsing time stats.tparsed = performance.now(); - // in case we need SIDX ranges - // return early after calling load for - // the SIDX box. - if (levelDetails.needSidxRanges) { - const sidxUrl = levelDetails.initSegment.url; - this.load({ - url: sidxUrl, - isSidxRequest: true, - type, - level, - levelDetails, - id, - rangeStart: 0, - rangeEnd: 2048, - responseType: 'arraybuffer' - }); - return; - } - // extend the context with the new levelDetails property context.levelDetails = levelDetails; this._handlePlaylistLoaded(response, stats, context, networkDetails); } - _handleSidxRequest (response: LoaderResponse, context: PlaylistLoaderContext) { - if (typeof response.data === 'string') { - throw new Error('sidx request must be made with responseType of array buffer'); - } - - const sidxInfo = MP4Demuxer.parseSegmentIndex(new Uint8Array(response.data)); - // if provided fragment does not contain sidx, early return - if (!sidxInfo) { - return; - } - const sidxReferences = sidxInfo.references; - const levelDetails = context.levelDetails; - sidxReferences.forEach((segmentRef, index) => { - const segRefInfo = segmentRef.info; - if (!levelDetails) { - return; - } - const frag = levelDetails.fragments[index]; - if (frag.byteRange.length === 0) { - frag.setByteRange(String(1 + segRefInfo.end - segRefInfo.start) + '@' + String(segRefInfo.start)); - } - }); - - if (levelDetails) { - levelDetails.initSegment.setByteRange(String(sidxInfo.moovEndOffset) + '@0'); - } - } - - _handleManifestParsingError (response: LoaderResponse, context: PlaylistLoaderContext, reason: string, networkDetails: unknown) { - this.hls.trigger(Event.ERROR, { - type: ErrorTypes.NETWORK_ERROR, - details: ErrorDetails.MANIFEST_PARSING_ERROR, - fatal: true, - url: response.url, - reason, - networkDetails - }); - } - - _handleNetworkError (context: PlaylistLoaderContext, networkDetails: unknown, timeout: boolean = false, response: LoaderResponse | null = null) { - logger.info(`A network error occured while loading a ${context.type}-type playlist`); - - let details; - let fatal; - - const loader = this.getInternalLoader(context); - - switch (context.type) { - case PlaylistContextType.MANIFEST: - details = (timeout ? ErrorDetails.MANIFEST_LOAD_TIMEOUT : ErrorDetails.MANIFEST_LOAD_ERROR); - fatal = true; - break; - case PlaylistContextType.LEVEL: - details = (timeout ? ErrorDetails.LEVEL_LOAD_TIMEOUT : ErrorDetails.LEVEL_LOAD_ERROR); - fatal = false; - break; - case PlaylistContextType.AUDIO_TRACK: - details = (timeout ? ErrorDetails.AUDIO_TRACK_LOAD_TIMEOUT : ErrorDetails.AUDIO_TRACK_LOAD_ERROR); - fatal = false; - break; - default: - // details = ...? - fatal = false; - } - - if (loader) { - loader.abort(); - this.resetInternalLoader(context.type); - } - - // TODO(typescript-events): when error events are handled, type this - let errorData: any = { - type: ErrorTypes.NETWORK_ERROR, - details, - fatal, - url: context.url, - loader, - context, - networkDetails - }; - - if (response) { - errorData.response = response; - } - - this.hls.trigger(Event.ERROR, errorData); - } - - _handlePlaylistLoaded (response: LoaderResponse, stats: LoaderStats, context: PlaylistLoaderContext, networkDetails: unknown) { + _handlePlaylistLoaded(response, stats, context, networkDetails) { const { type, level, id, levelDetails } = context; if (!levelDetails || !levelDetails.targetduration) { - this._handleManifestParsingError(response, context, 'invalid target duration', networkDetails); + console.error("manifest parsing error"); return; } @@ -506,22 +319,22 @@ class PlaylistLoader extends EventHandler { }); } else { switch (type) { - case PlaylistContextType.AUDIO_TRACK: - this.hls.trigger(Event.AUDIO_TRACK_LOADED, { - details: levelDetails, - id, - stats, - networkDetails - }); - break; - case PlaylistContextType.SUBTITLE_TRACK: - this.hls.trigger(Event.SUBTITLE_TRACK_LOADED, { - details: levelDetails, - id, - stats, - networkDetails - }); - break; + case PlaylistContextType.AUDIO_TRACK: + this.hls.trigger(Event.AUDIO_TRACK_LOADED, { + details: levelDetails, + id, + stats, + networkDetails + }); + break; + case PlaylistContextType.SUBTITLE_TRACK: + this.hls.trigger(Event.SUBTITLE_TRACK_LOADED, { + details: levelDetails, + id, + stats, + networkDetails + }); + break; } } } From 5647946bacee4ec10d7409f709a948aadc03dd65 Mon Sep 17 00:00:00 2001 From: Trek H Date: Fri, 24 Jan 2020 20:15:04 +1030 Subject: [PATCH 26/46] mjpeg-player: minor changes to xhr-loader and frag-loader Changes include: using console for logger, import statements, formatting. --- .../hlsjs/loader/fragment-loader.js | 45 +++++++++++----- cmd/mjpeg-player/hlsjs/utils/xhr-loader.js | 52 +++++++++++++------ 2 files changed, 70 insertions(+), 27 deletions(-) diff --git a/cmd/mjpeg-player/hlsjs/loader/fragment-loader.js b/cmd/mjpeg-player/hlsjs/loader/fragment-loader.js index ac5f3d51..b7e1f9e1 100644 --- a/cmd/mjpeg-player/hlsjs/loader/fragment-loader.js +++ b/cmd/mjpeg-player/hlsjs/loader/fragment-loader.js @@ -1,19 +1,40 @@ +/* +AUTHOR + Trek Hopton + +LICENSE + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see http://www.gnu.org/licenses. + + For hls.js Copyright notice and license, see LICENSE file. +*/ + /* * Fragment Loader */ -import Event from '../events'; -import EventHandler from '../event-handler'; -import { ErrorTypes, ErrorDetails } from '../errors'; -import { logger } from '../utils/logger'; +import Event from '../events.js'; +import EventHandler from '../event-handler.js'; class FragmentLoader extends EventHandler { - constructor (hls) { + constructor(hls) { super(hls, Event.FRAG_LOADING); this.loaders = {}; } - destroy () { + destroy() { let loaders = this.loaders; for (let loaderName in loaders) { let loader = loaders[loaderName]; @@ -26,7 +47,7 @@ class FragmentLoader extends EventHandler { super.destroy(); } - onFragLoading (data) { + onFragLoading(data) { const frag = data.frag, type = frag.type, loaders = this.loaders, @@ -39,7 +60,7 @@ class FragmentLoader extends EventHandler { let loader = loaders[type]; if (loader) { - logger.warn(`abort previous fragment loader for type: ${type}`); + console.warn(`abort previous fragment loader for type: ${type}`); loader.abort(); } @@ -75,7 +96,7 @@ class FragmentLoader extends EventHandler { loader.load(loaderContext, loaderConfig, loaderCallbacks); } - loadsuccess (response, stats, context, networkDetails = null) { + loadsuccess(response, stats, context, networkDetails = null) { let payload = response.data, frag = context.frag; // detach fragment loader on load success frag.loader = undefined; @@ -83,7 +104,7 @@ class FragmentLoader extends EventHandler { this.hls.trigger(Event.FRAG_LOADED, { payload: payload, frag: frag, stats: stats, networkDetails: networkDetails }); } - loaderror (response, context, networkDetails = null) { + loaderror(response, context, networkDetails = null) { const frag = context.frag; let loader = frag.loader; if (loader) { @@ -94,7 +115,7 @@ class FragmentLoader extends EventHandler { this.hls.trigger(Event.ERROR, { type: ErrorTypes.NETWORK_ERROR, details: ErrorDetails.FRAG_LOAD_ERROR, fatal: false, frag: context.frag, response: response, networkDetails: networkDetails }); } - loadtimeout (stats, context, networkDetails = null) { + loadtimeout(stats, context, networkDetails = null) { const frag = context.frag; let loader = frag.loader; if (loader) { @@ -106,7 +127,7 @@ class FragmentLoader extends EventHandler { } // data will be used for progressive parsing - loadprogress (stats, context, data, networkDetails = null) { // jshint ignore:line + loadprogress(stats, context, data, networkDetails = null) { // jshint ignore:line let frag = context.frag; frag.loaded = stats.loaded; this.hls.trigger(Event.FRAG_LOAD_PROGRESS, { frag: frag, stats: stats, networkDetails: networkDetails }); diff --git a/cmd/mjpeg-player/hlsjs/utils/xhr-loader.js b/cmd/mjpeg-player/hlsjs/utils/xhr-loader.js index 10ff29c7..67280a87 100644 --- a/cmd/mjpeg-player/hlsjs/utils/xhr-loader.js +++ b/cmd/mjpeg-player/hlsjs/utils/xhr-loader.js @@ -1,24 +1,45 @@ -/** - * XHR based logger +/* +AUTHOR + Trek Hopton + +LICENSE + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see http://www.gnu.org/licenses. + + For hls.js Copyright notice and license, see LICENSE file. */ -import { logger } from '../utils/logger'; +/** + * XHR based loader +*/ const { performance, XMLHttpRequest } = window; class XhrLoader { - constructor (config) { + constructor(config) { if (config && config.xhrSetup) { this.xhrSetup = config.xhrSetup; } } - destroy () { + destroy() { this.abort(); this.loader = null; } - abort () { + abort() { let loader = this.loader; if (loader && loader.readyState !== 4) { this.stats.aborted = true; @@ -31,7 +52,7 @@ class XhrLoader { this.retryTimeout = null; } - load (context, config, callbacks) { + load(context, config, callbacks) { this.context = context; this.config = config; this.callbacks = callbacks; @@ -40,10 +61,11 @@ class XhrLoader { this.loadInternal(); } - loadInternal () { + loadInternal() { let xhr, context = this.context; xhr = this.loader = new XMLHttpRequest(); - + window.console.log("load internal xhr: " + context.url); + let stats = this.stats; stats.tfirst = 0; stats.loaded = 0; @@ -82,7 +104,7 @@ class XhrLoader { xhr.send(); } - readystatechange (event) { + readystatechange(event) { let xhr = event.currentTarget, readyState = xhr.readyState, stats = this.stats, @@ -121,11 +143,11 @@ class XhrLoader { } else { // if max nb of retries reached or if http status between 400 and 499 (such error cannot be recovered, retrying is useless), return error if (stats.retry >= config.maxRetry || (status >= 400 && status < 499)) { - logger.error(`${status} while loading ${context.url}`); + console.error(`${status} while loading ${context.url}`); this.callbacks.onError({ code: status, text: xhr.statusText }, context, xhr); } else { // retry - logger.warn(`${status} while loading ${context.url}, retrying in ${this.retryDelay}...`); + console.warn(`${status} while loading ${context.url}, retrying in ${this.retryDelay}...`); // aborts and resets internal state this.destroy(); // schedule retry @@ -142,12 +164,12 @@ class XhrLoader { } } - loadtimeout () { - logger.warn(`timeout while loading ${this.context.url}`); + loadtimeout() { + console.warn(`timeout while loading ${this.context.url}`); this.callbacks.onTimeout(this.stats, this.context, null); } - loadprogress (event) { + loadprogress(event) { let xhr = event.currentTarget, stats = this.stats; From 4e7e5ebca3cd5fa95f448dbf3f3baf7cb7788639 Mon Sep 17 00:00:00 2001 From: Saxon Date: Fri, 24 Jan 2020 20:15:40 +1030 Subject: [PATCH 27/46] revid/revid.go: cleaned up prcoessFrom (added deger for waitgroup.Done and added info log) --- revid/revid.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/revid/revid.go b/revid/revid.go index b442e34a..7aca384e 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -857,17 +857,19 @@ func (r *Revid) Update(vars map[string]string) error { // processFrom is run as a routine to read from a input data source, lex and // then send individual access units to revid's encoders. func (r *Revid) processFrom(in device.AVDevice, delay time.Duration) { + defer r.wg.Done() + restart: err := in.Start() if err != nil { r.err <- fmt.Errorf("could not start input device: %w", err) - r.wg.Done() return } // Lex data from input device, in, until finished or an error is encountered. // For a continuous source e.g. a camera or microphone, we should remain // in this call indefinitely unless in.Stop() is called and an io.EOF is forced. + r.cfg.Logger.Log(logger.Info, pkg+"lexing") err = r.lexTo(r.filters[0], in, delay) switch err { case nil, io.EOF, io.ErrUnexpectedEOF: @@ -886,5 +888,4 @@ restart: } r.cfg.Logger.Log(logger.Info, pkg+"finished lexing") - r.wg.Done() } From 0e14c0a056966b82a269d0124b3914cf7e666570 Mon Sep 17 00:00:00 2001 From: Saxon Date: Fri, 24 Jan 2020 20:24:23 +1030 Subject: [PATCH 28/46] cmd/revid-cli: improved commentfor Loop flag description --- cmd/revid-cli/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index d0b32e9f..3d03916b 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -122,7 +122,7 @@ func handleFlags() config.Config { httpAddressPtr = flag.String("HttpAddress", "", "Destination address of http posts") verticalFlipPtr = flag.Bool("VerticalFlip", false, "Flip video vertically: Yes, No") horizontalFlipPtr = flag.Bool("HorizontalFlip", false, "Flip video horizontally: Yes, No") - loopPtr = flag.Bool("Loop", false, "Loop video if EOF encountered: true, false") + loopPtr = flag.Bool("Loop", false, "Loop input source on completion (true/false)") bitratePtr = flag.Uint("Bitrate", 0, "Bitrate of recorded video") heightPtr = flag.Uint("Height", 0, "Height in pixels") widthPtr = flag.Uint("Width", 0, "Width in pixels") From e8766ce5cb91b762ffc4118ae07103bdf2939d2d Mon Sep 17 00:00:00 2001 From: Trek H Date: Fri, 24 Jan 2020 21:45:03 +1030 Subject: [PATCH 29/46] mjpeg-player: matched order of functions to simplify PR diff --- .../hlsjs/loader/playlist-loader.js | 122 +++++++++--------- 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/cmd/mjpeg-player/hlsjs/loader/playlist-loader.js b/cmd/mjpeg-player/hlsjs/loader/playlist-loader.js index e34f236b..bf2422f9 100644 --- a/cmd/mjpeg-player/hlsjs/loader/playlist-loader.js +++ b/cmd/mjpeg-player/hlsjs/loader/playlist-loader.js @@ -48,6 +48,67 @@ class PlaylistLoader extends EventHandler { type !== PlaylistContextType.SUBTITLE_TRACK); } + /** + * Map context.type to LevelType + * @param {PlaylistLoaderContext} context + * @returns {LevelType} + */ + static mapContextToLevelType(context) { + const { type } = context; + + switch (type) { + case PlaylistContextType.AUDIO_TRACK: + return PlaylistLevelType.AUDIO; + case PlaylistContextType.SUBTITLE_TRACK: + return PlaylistLevelType.SUBTITLE; + default: + return PlaylistLevelType.MAIN; + } + } + + static getResponseUrl(response, context) { + let url = response.url; + // responseURL not supported on some browsers (it is used to detect URL redirection) + // data-uri mode also not supported (but no need to detect redirection) + if (url === undefined || url.indexOf('data:') === 0) { + // fallback to initial URL + url = context.url; + } + return url; + } + + /** + * Returns defaults or configured loader-type overloads (pLoader and loader config params) + * Default loader is XHRLoader (see utils) + * @param {PlaylistLoaderContext} context + * @returns {Loader} or other compatible configured overload + */ + createInternalLoader(context) { + const config = this.hls.config; + const PLoader = config.pLoader; + const Loader = config.loader; + // TODO(typescript-config): Verify once config is typed that InternalLoader always returns a Loader + const InternalLoader = PLoader || Loader; + + const loader = new InternalLoader(config); + + // TODO - Do we really need to assign the instance or if the dep has been lost + context.loader = loader; + this.loaders[context.type] = loader; + + return loader; + } + + getInternalLoader(context) { + return this.loaders[context.type]; + } + + resetInternalLoader(contextType) { + if (this.loaders[contextType]) { + delete this.loaders[contextType]; + } + } + onManifestLoading(data) { this.load({ url: data.url, @@ -89,67 +150,6 @@ class PlaylistLoader extends EventHandler { }); } - static getResponseUrl(response, context) { - let url = response.url; - // responseURL not supported on some browsers (it is used to detect URL redirection) - // data-uri mode also not supported (but no need to detect redirection) - if (url === undefined || url.indexOf('data:') === 0) { - // fallback to initial URL - url = context.url; - } - return url; - } - - /** - * Map context.type to LevelType - * @param {PlaylistLoaderContext} context - * @returns {LevelType} - */ - static mapContextToLevelType(context) { - const { type } = context; - - switch (type) { - case PlaylistContextType.AUDIO_TRACK: - return PlaylistLevelType.AUDIO; - case PlaylistContextType.SUBTITLE_TRACK: - return PlaylistLevelType.SUBTITLE; - default: - return PlaylistLevelType.MAIN; - } - } - - getInternalLoader(context) { - return this.loaders[context.type]; - } - - /** - * Returns defaults or configured loader-type overloads (pLoader and loader config params) - * Default loader is XHRLoader (see utils) - * @param {PlaylistLoaderContext} context - * @returns {Loader} or other compatible configured overload - */ - createInternalLoader(context) { - const config = this.hls.config; - const PLoader = config.pLoader; - const Loader = config.loader; - // TODO(typescript-config): Verify once config is typed that InternalLoader always returns a Loader - const InternalLoader = PLoader || Loader; - - const loader = new InternalLoader(config); - - // TODO - Do we really need to assign the instance or if the dep has been lost - context.loader = loader; - this.loaders[context.type] = loader; - - return loader; - } - - resetInternalLoader(contextType) { - if (this.loaders[contextType]) { - delete this.loaders[contextType]; - } - } - load(context) { const config = this.hls.config; From 64754f7e0f33a6ae37000ed732a4f70d9a2f2771 Mon Sep 17 00:00:00 2001 From: Saxon Date: Fri, 24 Jan 2020 21:25:27 +1030 Subject: [PATCH 30/46] revid: use loop in processFrom instead of goto --- revid/revid.go | 44 ++++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/revid/revid.go b/revid/revid.go index 7aca384e..3d9d26c5 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -859,32 +859,28 @@ func (r *Revid) Update(vars map[string]string) error { func (r *Revid) processFrom(in device.AVDevice, delay time.Duration) { defer r.wg.Done() -restart: - err := in.Start() - if err != nil { - r.err <- fmt.Errorf("could not start input device: %w", err) - return - } + for l := true; l; l = r.cfg.Loop { + err := in.Start() + if err != nil { + r.err <- fmt.Errorf("could not start input device: %w", err) + return + } - // Lex data from input device, in, until finished or an error is encountered. - // For a continuous source e.g. a camera or microphone, we should remain - // in this call indefinitely unless in.Stop() is called and an io.EOF is forced. - r.cfg.Logger.Log(logger.Info, pkg+"lexing") - err = r.lexTo(r.filters[0], in, delay) - switch err { - case nil, io.EOF, io.ErrUnexpectedEOF: - default: - r.err <- err - } + // Lex data from input device, in, until finished or an error is encountered. + // For a continuous source e.g. a camera or microphone, we should remain + // in this call indefinitely unless in.Stop() is called and an io.EOF is forced. + r.cfg.Logger.Log(logger.Info, pkg+"lexing") + err = r.lexTo(r.filters[0], in, delay) + switch err { + case nil, io.EOF, io.ErrUnexpectedEOF: + default: + r.err <- err + } - err = in.Stop() - if err != nil { - r.err <- fmt.Errorf("could not stop input source: %w", err) - } - - if r.cfg.Loop { - r.cfg.Logger.Log(logger.Info, pkg+"looping input") - goto restart + err = in.Stop() + if err != nil { + r.err <- fmt.Errorf("could not stop input source: %w", err) + } } r.cfg.Logger.Log(logger.Info, pkg+"finished lexing") From f82ab422463dcc7db3fccc335011bc0b0754212c Mon Sep 17 00:00:00 2001 From: Saxon Date: Sat, 25 Jan 2020 09:39:42 +1030 Subject: [PATCH 31/46] revid & cmd/revid-cli: InputFPS field now FileFPS and added check so that only used with File input --- cmd/revid-cli/main.go | 2 +- revid/config/config.go | 15 +++++++-------- revid/revid.go | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 3d03916b..5db8e9a1 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -133,7 +133,7 @@ func handleFlags() config.Config { saturationPtr = flag.Int("Saturation", 0, "Set Saturation. (100-100)") exposurePtr = flag.String("Exposure", "auto", "Set exposure mode. ("+strings.Join(raspivid.ExposureModes[:], ",")+")") autoWhiteBalancePtr = flag.String("Awb", "auto", "Set automatic white balance mode. ("+strings.Join(raspivid.AutoWhiteBalanceModes[:], ",")+")") - inputFPSPtr = flag.Int("InputFPS", 0, "Input source processing FPS") + fileFPSPtr = flag.Int("FileFPS", 0, "File source frame processing FPS") // Audio specific flags. sampleRatePtr = flag.Int("SampleRate", 48000, "Sample rate of recorded audio") diff --git a/revid/config/config.go b/revid/config/config.go index 8ee31ee5..b70a01b3 100644 --- a/revid/config/config.go +++ b/revid/config/config.go @@ -86,7 +86,7 @@ const ( defaultAudioInputCodec = codecutil.ADPCM defaultPSITime = 2 defaultMotionInterval = 5 - defaultInputFPS = 0 + defaultFileFPS = 0 // Ring buffer defaults. defaultRBMaxElements = 10000 @@ -302,9 +302,8 @@ type Config struct { // If true will restart reading of input after an io.EOF. Loop bool - // Defines the rate at which an input source is processed. If reading - // from a realtime source, InputFPS is not necessary and should be 0 (default). - InputFPS int + // Defines the rate at which frames from a file source are processed. + FileFPS int } // TypeData contains information about all of the variables that @@ -319,6 +318,7 @@ var TypeData = map[string]string{ "CBR": "bool", "ClipDuration": "uint", "Exposure": "enum:auto,night,nightpreview,backlight,spotlight,sports,snow,beach,verylong,fixedfps,antishake,fireworks", + "FileFPS": "int", "Filters": "enums:NoOp,MOG,VariableFPS,KNN", "FrameRate": "uint", "Height": "uint", @@ -326,7 +326,6 @@ var TypeData = map[string]string{ "HTTPAddress": "string", "Input": "enum:raspivid,rtsp,v4l,file", "InputCodec": "enum:H264,MJPEG", - "InputFPS": "int", "InputPath": "string", "KNNHistory": "uint", "KNNKernel": "float", @@ -531,9 +530,9 @@ func (c *Config) Validate() error { } } - if c.InputFPS <= 0 { - c.Logger.Log(logger.Info, pkg+"InputFPS bad or unset, defaulting", "InputFPS", defaultInputFPS) - c.InputFPS = defaultInputFPS + if c.FileFPS <= 0 || (c.FileFPS > 0 && c.Input != InputFile) { + c.Logger.Log(logger.Info, pkg+"FileFPS bad or unset, defaulting", "FileFPS", defaultFileFPS) + c.FileFPS = defaultFileFPS } return nil diff --git a/revid/revid.go b/revid/revid.go index 3d9d26c5..5d8c81d0 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -429,7 +429,7 @@ func (r *Revid) Start() error { } // Calculate delay between frames based on InputFPS. - d := time.Duration(1000/r.cfg.InputFPS) * time.Millisecond + d := time.Duration(1000/r.cfg.FileFPS) * time.Millisecond r.wg.Add(1) go r.processFrom(r.input, d) From a0a99e17dbf62ca7534b869475fdddf14f0af1c5 Mon Sep 17 00:00:00 2001 From: Saxon Date: Sat, 25 Jan 2020 09:42:32 +1030 Subject: [PATCH 32/46] revid/revid.go: logging unexpected EOF from input source --- revid/revid.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/revid/revid.go b/revid/revid.go index 5d8c81d0..aa94daf0 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -872,7 +872,9 @@ func (r *Revid) processFrom(in device.AVDevice, delay time.Duration) { r.cfg.Logger.Log(logger.Info, pkg+"lexing") err = r.lexTo(r.filters[0], in, delay) switch err { - case nil, io.EOF, io.ErrUnexpectedEOF: + case nil, io.EOF: + case io.ErrUnexpectedEOF: + r.cfg.Logger.Log(logger.Info, pkg+"unexpected EOF from input") default: r.err <- err } From dcafcbf69ebc83b500eb4a6edcb649b849188744 Mon Sep 17 00:00:00 2001 From: Saxon Date: Sat, 25 Jan 2020 09:53:21 +1030 Subject: [PATCH 33/46] cmd/revid-cli: fixed build error --- cmd/revid-cli/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 5db8e9a1..abc049ed 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -237,7 +237,7 @@ func handleFlags() config.Config { netsender.ConfigFile = *configFilePtr } - cfg.InputFPS = *inputFPSPtr + cfg.FileFPS = *fileFPSPtr cfg.Loop = *loopPtr cfg.CameraIP = *cameraIPPtr cfg.Rotation = *rotationPtr From cfd3f0fc3c62d961ac906fe81f789717dd1e8a68 Mon Sep 17 00:00:00 2001 From: Saxon Date: Sat, 25 Jan 2020 10:15:27 +1030 Subject: [PATCH 34/46] revid: consider when r.cfg.FileFPS ==0 --- revid/revid.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/revid/revid.go b/revid/revid.go index aa94daf0..94bd4a6f 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -428,8 +428,11 @@ func (r *Revid) Start() error { return err } - // Calculate delay between frames based on InputFPS. - d := time.Duration(1000/r.cfg.FileFPS) * time.Millisecond + // Calculate delay between frames based on FileFPS. + d := time.Duration(0) + if r.cfg.FileFPS != 0 { + d = time.Duration(1000/r.cfg.FileFPS) * time.Millisecond + } r.wg.Add(1) go r.processFrom(r.input, d) From ced8727c070b2077366ab4329e2e639a6923098a Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 22 Jan 2020 12:06:14 +1030 Subject: [PATCH 35/46] filter: create a simple difference motion filter using gocv --- filter/difference.go | 114 +++++++++++++++++++++++++++++++++++++++++ revid/config/config.go | 12 +++-- revid/revid.go | 11 +++- 3 files changed, 132 insertions(+), 5 deletions(-) create mode 100644 filter/difference.go diff --git a/filter/difference.go b/filter/difference.go new file mode 100644 index 00000000..97df94b9 --- /dev/null +++ b/filter/difference.go @@ -0,0 +1,114 @@ +// +build !circleci + +/* +DESCRIPTION + A filter that detects motion and discards frames without motion. The + algorithm calculates the absolute difference for each pixel between + two frames, then finds the mean. If the mean is above a given threshold, + then it is considered motion. + +AUTHORS + Scott Barnard + +LICENSE + difference.go is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ + +package filter + +import ( + "fmt" + "image" + "image/color" + "io" + + "gocv.io/x/gocv" +) + +// DiffFilter is a filter that provides basic motion detection. DiffFilter calculates +// the absolute difference for each pixel between two frames, then finds the mean. If +// the mean is above a given threshold, then it is considered motion. +type DiffFilter struct { + dst io.WriteCloser + thresh float64 + prev gocv.Mat + debug bool + windows []*gocv.Window +} + +// NewDiffFilter returns a pointer to a new DiffFilter. +func NewDiffFilter(dst io.WriteCloser, debug bool, threshold float64) *DiffFilter { + var windows []*gocv.Window + if debug { + windows = []*gocv.Window{gocv.NewWindow("Diff: Bounding boxes"), gocv.NewWindow("Diff: Motion")} + } + return &DiffFilter{dst, threshold, gocv.NewMat(), debug, windows} +} + +// Implements io.Closer. +// Close frees resources used by gocv, because it has to be done manually, due to +// it using c-go. +func (d *DiffFilter) Close() error { + d.prev.Close() + for _, window := range d.windows { + window.Close() + } + return nil +} + +// Implements io.Writer. +// Write applies the motion filter to the video stream. Only frames with motion +// are written to the destination encoder, frames without are discarded. +func (d *DiffFilter) Write(f []byte) (int, error) { + if d.prev.Empty() { + d.prev, _ = gocv.IMDecode(f, gocv.IMReadColor) + return 0, nil + } + + img, _ := gocv.IMDecode(f, gocv.IMReadColor) + defer img.Close() + + imgDelta := gocv.NewMat() + defer imgDelta.Close() + + // Seperate foreground and background. + gocv.AbsDiff(img, d.prev, &imgDelta) + gocv.CvtColor(imgDelta, &imgDelta, gocv.ColorBGRToGray) + + mean := imgDelta.Mean().Val1 + + // Update History + d.prev = img.Clone() + + // Draw debug information. + if d.debug { + if mean >= d.thresh { + gocv.PutText(&img, fmt.Sprintf("motion - mean:%f", mean), image.Pt(32, 32), gocv.FontHersheyPlain, 2.0, color.RGBA{255, 0, 0, 0}, 2) + } + + d.windows[0].IMShow(img) + d.windows[1].IMShow(imgDelta) + d.windows[0].WaitKey(1) + } + + // Don't write to destination if there is no motion. + if mean < d.thresh { + return 0, nil + } + + // Write to destination. + return d.dst.Write(f) +} diff --git a/revid/config/config.go b/revid/config/config.go index b70a01b3..0192b1bd 100644 --- a/revid/config/config.go +++ b/revid/config/config.go @@ -127,6 +127,7 @@ const ( FilterMOG FilterVariableFPS FilterKNN + FilterDifference ) // OS names @@ -304,6 +305,8 @@ type Config struct { // Defines the rate at which frames from a file source are processed. FileFPS int + // Difference filter parameters. + DiffThreshold float64 // Intensity value from the Difference motion detection algorithm that is considered motion. } // TypeData contains information about all of the variables that @@ -317,9 +320,10 @@ var TypeData = map[string]string{ "CameraIP": "string", "CBR": "bool", "ClipDuration": "uint", + "DiffThreshold": "float", "Exposure": "enum:auto,night,nightpreview,backlight,spotlight,sports,snow,beach,verylong,fixedfps,antishake,fireworks", "FileFPS": "int", - "Filters": "enums:NoOp,MOG,VariableFPS,KNN", + "Filters": "enums:NoOp,MOG,VariableFPS,KNN,Difference", "FrameRate": "uint", "Height": "uint", "HorizontalFlip": "bool", @@ -340,13 +344,13 @@ var TypeData = map[string]string{ "MOGMinArea": "float", "MOGThreshold": "float", "MotionInterval": "int", - "RBCapacity": "uint", - "RBMaxElements": "uint", - "RBWriteTimeout": "uint", "Output": "enum:File,Http,Rtmp,Rtp", "OutputPath": "string", "Outputs": "enums:File,Http,Rtmp,Rtp", "Quantization": "uint", + "RBCapacity": "uint", + "RBMaxElements": "uint", + "RBWriteTimeout": "uint", "Rotation": "uint", "RTMPURL": "string", "RTPAddress": "string", diff --git a/revid/revid.go b/revid/revid.go index 94bd4a6f..fd7614f1 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -342,6 +342,8 @@ func (r *Revid) setupPipeline(mtsEnc func(dst io.WriteCloser, rate float64) (io. r.filters[i] = filter.NewVariableFPSFilter(dst, r.cfg.MinFPS, filter.NewMOGFilter(dst, r.cfg.MOGMinArea, r.cfg.MOGThreshold, int(r.cfg.MOGHistory), r.cfg.ShowWindows, r.cfg.MotionInterval)) case config.FilterKNN: r.filters[i] = filter.NewKNNFilter(dst, r.cfg.KNNMinArea, r.cfg.KNNThreshold, int(r.cfg.KNNHistory), int(r.cfg.KNNKernel), r.cfg.ShowWindows, r.cfg.MotionInterval) + case config.FilterDifference: + r.filters[i] = filter.NewDiffFilter(dst, r.cfg.ShowWindows, r.cfg.DiffThreshold) default: panic("Undefined Filter") } @@ -663,7 +665,7 @@ func (r *Revid) Update(vars map[string]string) error { } case "Filters": filters := strings.Split(value, ",") - m := map[string]int{"NoOp": config.FilterNoOp, "MOG": config.FilterMOG, "VariableFPS": config.FilterVariableFPS, "KNN": config.FilterKNN} + m := map[string]int{"NoOp": config.FilterNoOp, "MOG": config.FilterMOG, "VariableFPS": config.FilterVariableFPS, "KNN": config.FilterKNN, "Difference": config.FilterDifference} r.cfg.Filters = make([]int, len(filters)) for i, filter := range filters { v, ok := m[filter] @@ -811,6 +813,13 @@ func (r *Revid) Update(vars map[string]string) error { break } r.cfg.KNNThreshold = v + case "DiffThreshold": + v, err := strconv.ParseFloat(value, 64) + if err != nil { + r.cfg.Logger.Log(logger.Warning, pkg+"invalid DiffThreshold var", "value", value) + break + } + r.cfg.DiffThreshold = v case "KNNKernel": v, err := strconv.Atoi(value) if err != nil { From 458933babb95bb105a7f6ffc7a39b509787ea27f Mon Sep 17 00:00:00 2001 From: Scott Date: Thu, 23 Jan 2020 14:03:08 +1030 Subject: [PATCH 36/46] filter: create function for satisfying circleci tests --- filter/filters_circleci.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/filter/filters_circleci.go b/filter/filters_circleci.go index 374e8b0a..d4c427d7 100644 --- a/filter/filters_circleci.go +++ b/filter/filters_circleci.go @@ -36,6 +36,12 @@ func NewMOGFilter(dst io.WriteCloser, area, threshold float64, history int, debu return &NoOp{dst: dst} } +// NewKNNFilter returns a pointer to a new NoOp struct for testing purposes only. func NewKNNFilter(dst io.WriteCloser, area, threshold float64, history, kernelSize int, debug bool, hf int) *NoOp { return &NoOp{dst: dst} } + +// NewDiffFilter returns a pointer to a new NoOp struct for testing purposes only. +func NewDiffFilter(dst io.WriteCloser, debug bool, threshold float64) *NoOp { + return &NoOp{dst: dst} +} From e0fa47490671fdbca93d70023e6c25cb97ca568d Mon Sep 17 00:00:00 2001 From: Scott Date: Fri, 24 Jan 2020 16:05:35 +1030 Subject: [PATCH 37/46] =?UTF-8?q?filter:=20DiffFilter=20=E2=86=92=20Differ?= =?UTF-8?q?ence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- filter/difference.go | 14 +++++++------- filter/filters_circleci.go | 4 ++-- revid/revid.go | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/filter/difference.go b/filter/difference.go index 97df94b9..00de000a 100644 --- a/filter/difference.go +++ b/filter/difference.go @@ -38,10 +38,10 @@ import ( "gocv.io/x/gocv" ) -// DiffFilter is a filter that provides basic motion detection. DiffFilter calculates +// Difference is a filter that provides basic motion detection. Difference calculates // the absolute difference for each pixel between two frames, then finds the mean. If // the mean is above a given threshold, then it is considered motion. -type DiffFilter struct { +type Difference struct { dst io.WriteCloser thresh float64 prev gocv.Mat @@ -49,19 +49,19 @@ type DiffFilter struct { windows []*gocv.Window } -// NewDiffFilter returns a pointer to a new DiffFilter. -func NewDiffFilter(dst io.WriteCloser, debug bool, threshold float64) *DiffFilter { +// NewDifference returns a pointer to a new Difference struct. +func NewDifference(dst io.WriteCloser, debug bool, threshold float64) *Difference { var windows []*gocv.Window if debug { windows = []*gocv.Window{gocv.NewWindow("Diff: Bounding boxes"), gocv.NewWindow("Diff: Motion")} } - return &DiffFilter{dst, threshold, gocv.NewMat(), debug, windows} + return &Difference{dst, threshold, gocv.NewMat(), debug, windows} } // Implements io.Closer. // Close frees resources used by gocv, because it has to be done manually, due to // it using c-go. -func (d *DiffFilter) Close() error { +func (d *Difference) Close() error { d.prev.Close() for _, window := range d.windows { window.Close() @@ -72,7 +72,7 @@ func (d *DiffFilter) Close() error { // Implements io.Writer. // Write applies the motion filter to the video stream. Only frames with motion // are written to the destination encoder, frames without are discarded. -func (d *DiffFilter) Write(f []byte) (int, error) { +func (d *Difference) Write(f []byte) (int, error) { if d.prev.Empty() { d.prev, _ = gocv.IMDecode(f, gocv.IMReadColor) return 0, nil diff --git a/filter/filters_circleci.go b/filter/filters_circleci.go index d4c427d7..aaca83f6 100644 --- a/filter/filters_circleci.go +++ b/filter/filters_circleci.go @@ -41,7 +41,7 @@ func NewKNNFilter(dst io.WriteCloser, area, threshold float64, history, kernelSi return &NoOp{dst: dst} } -// NewDiffFilter returns a pointer to a new NoOp struct for testing purposes only. -func NewDiffFilter(dst io.WriteCloser, debug bool, threshold float64) *NoOp { +// NewDiffference returns a pointer to a new NoOp struct for testing purposes only. +func NewDifference(dst io.WriteCloser, debug bool, threshold float64) *NoOp { return &NoOp{dst: dst} } diff --git a/revid/revid.go b/revid/revid.go index fd7614f1..b5f31a6e 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -343,7 +343,7 @@ func (r *Revid) setupPipeline(mtsEnc func(dst io.WriteCloser, rate float64) (io. case config.FilterKNN: r.filters[i] = filter.NewKNNFilter(dst, r.cfg.KNNMinArea, r.cfg.KNNThreshold, int(r.cfg.KNNHistory), int(r.cfg.KNNKernel), r.cfg.ShowWindows, r.cfg.MotionInterval) case config.FilterDifference: - r.filters[i] = filter.NewDiffFilter(dst, r.cfg.ShowWindows, r.cfg.DiffThreshold) + r.filters[i] = filter.NewDifference(dst, r.cfg.ShowWindows, r.cfg.DiffThreshold) default: panic("Undefined Filter") } From b15b649151d394d25d4313d8f66182595ada3d8b Mon Sep 17 00:00:00 2001 From: Scott Date: Fri, 24 Jan 2020 16:10:08 +1030 Subject: [PATCH 38/46] filter: check for errors after decoding --- filter/difference.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/filter/difference.go b/filter/difference.go index 00de000a..581cc8bf 100644 --- a/filter/difference.go +++ b/filter/difference.go @@ -74,12 +74,16 @@ func (d *Difference) Close() error { // are written to the destination encoder, frames without are discarded. func (d *Difference) Write(f []byte) (int, error) { if d.prev.Empty() { - d.prev, _ = gocv.IMDecode(f, gocv.IMReadColor) - return 0, nil + var err error + d.prev, err = gocv.IMDecode(f, gocv.IMReadColor) + return 0, err } - img, _ := gocv.IMDecode(f, gocv.IMReadColor) + img, err := gocv.IMDecode(f, gocv.IMReadColor) defer img.Close() + if err != nil { + return 0, err + } imgDelta := gocv.NewMat() defer imgDelta.Close() From fb43e9214ae5fb11d67e92ea4e2c1cec0ee9d40f Mon Sep 17 00:00:00 2001 From: Scott Date: Fri, 24 Jan 2020 16:29:06 +1030 Subject: [PATCH 39/46] filter: change formatting of code --- filter/difference.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/filter/difference.go b/filter/difference.go index 581cc8bf..4504c6ef 100644 --- a/filter/difference.go +++ b/filter/difference.go @@ -100,7 +100,15 @@ func (d *Difference) Write(f []byte) (int, error) { // Draw debug information. if d.debug { if mean >= d.thresh { - gocv.PutText(&img, fmt.Sprintf("motion - mean:%f", mean), image.Pt(32, 32), gocv.FontHersheyPlain, 2.0, color.RGBA{255, 0, 0, 0}, 2) + gocv.PutText( + &img, + fmt.Sprintf("motion - mean:%f", mean), + image.Pt(32, 32), + gocv.FontHersheyPlain, + 2.0, + color.RGBA{255, 0, 0, 0}, + 2, + ) } d.windows[0].IMShow(img) From a111f214279966885d1af40766dabd0ee9867453 Mon Sep 17 00:00:00 2001 From: Scott Date: Mon, 27 Jan 2020 11:23:56 +1030 Subject: [PATCH 40/46] filter: return length from the filter's Write method --- filter/difference.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/filter/difference.go b/filter/difference.go index 4504c6ef..d01cf0a7 100644 --- a/filter/difference.go +++ b/filter/difference.go @@ -76,7 +76,10 @@ func (d *Difference) Write(f []byte) (int, error) { if d.prev.Empty() { var err error d.prev, err = gocv.IMDecode(f, gocv.IMReadColor) - return 0, err + if err != nil { + return 0, err + } + return len(f), nil } img, err := gocv.IMDecode(f, gocv.IMReadColor) @@ -94,7 +97,7 @@ func (d *Difference) Write(f []byte) (int, error) { mean := imgDelta.Mean().Val1 - // Update History + // Update History. d.prev = img.Clone() // Draw debug information. @@ -118,7 +121,7 @@ func (d *Difference) Write(f []byte) (int, error) { // Don't write to destination if there is no motion. if mean < d.thresh { - return 0, nil + return len(f), nil } // Write to destination. From cae7e55723f7ce5ed032218354165f56e4354b4a Mon Sep 17 00:00:00 2001 From: Saxon Date: Mon, 27 Jan 2020 13:54:43 +1030 Subject: [PATCH 41/46] revid/revid.go: added handling for FileFPS var from vidgrind --- revid/revid.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/revid/revid.go b/revid/revid.go index 5e3a035d..5eb5b7bf 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -859,6 +859,13 @@ func (r *Revid) Update(vars map[string]string) error { break } r.cfg.MOGHistory = uint(v) + case "FileFPS": + v, err := strconv.Atoi(value) + if err != nil { + r.cfg.Logger.Log(logger.Warning, pkg+"invalid FileFPS var", "value", value) + break + } + r.cfg.FileFPS = v case "mode": r.cfg.Loop = false if value == "Loop" { From 09c254e1f0aee89cee34ba2d88d624625da5e865 Mon Sep 17 00:00:00 2001 From: Trek H Date: Tue, 28 Jan 2020 13:29:17 +1030 Subject: [PATCH 42/46] mjpeg-player: removed bootstrap classes, updated copyright year --- cmd/mjpeg-player/index.html | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/cmd/mjpeg-player/index.html b/cmd/mjpeg-player/index.html index 60a40429..d42069fb 100644 --- a/cmd/mjpeg-player/index.html +++ b/cmd/mjpeg-player/index.html @@ -29,29 +29,29 @@ LICENSE -
-
-
-
- +
+
+
+
+
-
+
-
- Frame Rate: fps +
+ Frame Rate: fps
-
+
-