Merged in rvcl (pull request #356)

cmd/rvcl: added command line interface for controlling revid API

Approved-by: Saxon Milton <saxon.milton@gmail.com>

exp/rvcl/main.go

@@ -0,0 +1,198 @@
+/*
+DESCRIPTION
+  rvcl is command line interface for revid. The user can provide configuration
+  by passing a JSON string directly, or by specifying a file containing the JSON.
+
+AUTHORS
+  Saxon A. Nelson-Milton <saxon@ausocean.org>
+  Dan Kortschak <dan@ausocean.org>
+  Scott Barnard <scott@ausocean.org>
+
+LICENSE
+  Copyright (C) 2020 the Australian Ocean Lab (AusOcean)
+
+  It is free software: you can redistribute it and/or modify them
+  under the terms of the GNU General Public License as published by the
+  Free Software Foundation, either version 3 of the License, or (at your
+  option) any later version.
+
+  It is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+  for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with revid in gpl.txt. If not, see http://www.gnu.org/licenses.
+
+USAGE
+  User may providied path to a config JSON file using the 'config-file' flag.
+  For example, to stream a H.264 file to youtube we will need to create a file
+  containing configuration JSON:
+  {
+    "Input":"file",
+    "InputPath":"<path to h264 file>",
+    "Output":"rtmp",
+    "RtmpUrl":"<rtmp url>",
+    "FileFPS":"25"
+  }
+
+  Then, using rvcl we can stream:
+
+  ./rvcl -config-file="config.json"
+
+  We can also use the 'config' flag to directly pass JSON as a string.
+  NB: Quotations must have backslashes, '\', proceeding them for rvcl to consider
+  the string valid JSON. For example, to play a file to youtube using the JSON
+  config described above, we would do the following:
+
+  ./rvcl -config="{\"Input\":\"file\",\"InputPath\":\"<file path>\",
+    \"Output\":\"rtmp\",\"RtmpUrl\":\"<rtmp url>\",\"FileFPS\":\"25\"}"
+
+  rvcl passes given variables to revid i.e. all variables defined by revid are
+  valid. See revid config package i.e. bitbucket.org/ausocean/av/revid/config for
+  revid specific variables, and see AVDevice implementations i.e.
+  bitbucket.org/ausocean/av/device for input device specific variables.
+*/
+
+// rvcl is a command line interface for revid.
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"runtime/pprof"
+
+	"gopkg.in/natefinch/lumberjack.v2"
+
+	"bitbucket.org/ausocean/av/container/mts"
+	"bitbucket.org/ausocean/av/container/mts/meta"
+	"bitbucket.org/ausocean/av/revid"
+	"bitbucket.org/ausocean/av/revid/config"
+	"bitbucket.org/ausocean/iot/pi/netsender"
+	"bitbucket.org/ausocean/utils/logger"
+)
+
+// Copyright information prefixed to all metadata.
+const (
+	metaPreambleKey  = "copyright"
+	metaPreambleData = "ausocean.org/license/content2019"
+)
+
+// Logging configuration.
+const (
+	logMaxSize    = 500 // MB
+	logMaxBackups = 10
+	logMaxAge     = 28 // days
+	logLevel      = logger.Debug
+	logPath       = "/var/log/netsender/netsender.log"
+	logSuppress   = false
+)
+
+// Misc consts.
+const (
+	pkg         = "rvcl: "
+	profilePath = "rvcl.prof"
+)
+
+// canProfile is set to false with revid-cli is built with "-tags profile".
+var canProfile = false
+
+// The logger that will be used throughout.
+var log *logger.Logger
+
+func main() {
+	mts.Meta = meta.NewWith([][2]string{{metaPreambleKey, metaPreambleData}})
+
+	// If built with profile tag, we will start CPU profiling.
+	if canProfile {
+		profile()
+		defer pprof.StopCPUProfile()
+	}
+
+	// User can provide config through single JSON string flag, or through JSON file.
+	var (
+		configPtr     = flag.String("config", "", "Provide configuration JSON to revid (see readme for further information).")
+		configFilePtr = flag.String("config-file", "", "Location of revid configuration file (see readme for further information).")
+	)
+	flag.Parse()
+
+	// Create config map according to flags or file, and panic if both are defined.
+	var (
+		cfg map[string]string
+		err error
+	)
+	switch {
+	case *configPtr != "" && *configFilePtr != "": // This doesn't make sense so panic.
+		panic("cannot define both command-line config and file config")
+	case *configPtr != "": // Decode JSON file to map.
+		err = json.Unmarshal([]byte(*configPtr), &cfg)
+		if err != nil {
+			panic(fmt.Sprintf("could not decode JSON config: %w", err))
+		}
+	case *configFilePtr != "": // Decode JSON string to map from command line flag.
+		f, err := os.Open(*configFilePtr)
+		if err != nil {
+			panic(fmt.Sprintf("could not open config file: %w", err))
+		}
+		err = json.NewDecoder(f).Decode(&cfg)
+		if err != nil {
+			panic(fmt.Sprintf("could not decode JSON config: %w", err))
+		}
+	default: // No config information has been provided; give empty map to force defaults.
+		cfg = map[string]string{}
+	}
+
+	// Create logger that methods will be called on by the netsender client and
+	// revid to log messages. Logs will go the lumberjack logger to handle file
+	// writing of messages.
+	log = logger.New(
+		logLevel,
+		&lumberjack.Logger{
+			Filename:   logPath,
+			MaxSize:    logMaxSize, // MB
+			MaxBackups: logMaxBackups,
+			MaxAge:     logMaxAge, // days
+		},
+		logSuppress,
+	)
+
+	// Create a netsender client. This is used only for HTTP sending of media
+	// in this binary.
+	ns, err := netsender.New(log, nil, nil, nil, nil)
+	if err != nil {
+		log.Log(logger.Fatal, pkg+"could not initialise netsender client: "+err.Error())
+	}
+
+	rv, err := revid.New(config.Config{Logger: log}, ns)
+	if err != nil {
+		panic(fmt.Sprintf("could not create revid: %w", err))
+	}
+
+	// Configure revid with configuration map obtained through flags or file.
+	// If config is empty, defaults will be adopted by revid.
+	err = rv.Update(cfg)
+	if err != nil {
+		panic(fmt.Sprintf("could not update revid config: %w", err))
+	}
+
+	err = rv.Start()
+	if err != nil {
+		panic(fmt.Sprintf("could not start revid: %w", err))
+	}
+
+	// Run indefinitely.
+	select {}
+}
+
+// profile creates a file to hold CPU profile metrics and begins CPU profiling.
+func profile() {
+	f, err := os.Create(profilePath)
+	if err != nil {
+		log.Log(logger.Fatal, pkg+"could not create CPU profile", "error", err.Error())
+	}
+	if err := pprof.StartCPUProfile(f); err != nil {
+		log.Log(logger.Fatal, pkg+"could not start CPU profile", "error", err.Error())
+	}
+}

exp/rvcl/profile.go

@@ -0,0 +1,34 @@
+// +build profile
+
+/*
+DESCRIPTION
+  profile.go provides an init function for setting canProfile flag to true to
+  allow for CPU profiling.
+
+AUTHORS
+  Dan Kortschak <dan@ausocean.org>
+
+LICENSE
+  Copyright (C) 2020 the Australian Ocean Lab (AusOcean)
+
+  It is free software: you can redistribute it and/or modify them
+  under the terms of the GNU General Public License as published by the
+  Free Software Foundation, either version 3 of the License, or (at your
+  option) any later version.
+
+  It is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+  for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with revid in gpl.txt. If not, see http://www.gnu.org/licenses.
+*/
+
+package main
+
+import _ "net/http/pprof"
+
+func init() {
+	canProfile = true
+}

exp/rvcl/run.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+REVIDPATH=$HOME/go/src/bitbucket.org/ausocean/av/cmd/rvcl
+cd $REVIDPATH
+sudo "PATH=$PATH:$REVIDPATH" ./rvcl -NetSender &

exp/rvcl/upgrade.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+# All-purpose upgrade script.
+# Upgrades source(s) to given Git tag, runs make in each directory,
+# and write tags to tags.conf upon success, exiting 0.
+# NB: Customize SrcDirs as needed to reflect dependencies.
+Usage="Usage: upgrade.sh [-d] tag"
+BaseDir=$GOPATH/src/bitbucket.org/ausocean
+VarDir=/var/netsender
+LogFile=/var/log/netsender/stream.log
+SrcDirs=($BaseDir/utils $BaseDir/iot $BaseDir/av)
+if [ "$1" == "-d" ]; then
+    set  -x
+    GitFlags=""
+    NewTag="$2"
+else
+    # capture stdout and stderr
+    exec 2> $LogFile
+    exec 1>&2
+    GitFlags="--quiet"
+    NewTag="$1"
+fi
+if [ -z "$GOPATH" ]; then
+    echo "Error: GOPATH not defined"
+    exit 1
+fi
+if [ -z "$NewTag" ]; then
+    echo "$Usage"
+    exit 1
+fi
+for dir in ${SrcDirs[@]}; do
+  pushd $dir
+  if [ ! "$?" == 0 ]; then
+    exit 1
+  fi
+  git fetch $GitFlags --depth=1 origin refs/tags/$NewTag:refs/tags/$NewTag
+  if [ ! "$?" == 0 ]; then
+    exit 1
+  fi
+  git checkout $GitFlags --force tags/$NewTag
+  if [ ! "$?" == 0 ]; then
+    exit 1
+  fi
+  if [ -e Makefile ]; then
+    make
+    if [ ! "$?" == 0 ]; then
+      exit 1
+    fi
+  fi
+  popd
+done
+if [ ! -d "$VarDir" ]; then
+  echo "Error: $VarDir does not exit."	
+  exit 1
+fi
+git tag > "$VarDir/tags.conf"
+exit $?

go.sum

@@ -1,17 +1,7 @@
-bitbucket.org/ausocean/iot v1.2.9 h1:3tzgiekH+Z0yXhkwnqBzxxe8qQJ2O7YTkz4s0T6stgw=
-bitbucket.org/ausocean/iot v1.2.9/go.mod h1:Q5FwaOKnCty3dVeVtki6DLwYa5vhNpOaeu1lwLyPCg8=
-bitbucket.org/ausocean/iot v1.2.10 h1:TTu+ykH5gQA8wU/pN0aS55ySQ/XcGxV4s4LKx3Wye5k=
-bitbucket.org/ausocean/iot v1.2.10/go.mod h1:Q5FwaOKnCty3dVeVtki6DLwYa5vhNpOaeu1lwLyPCg8=
-bitbucket.org/ausocean/iot v1.2.11 h1:MwYQK1F2ESA5jPVSCB0lBUN8HBiNDHGkh/OMGJKw8Oc=
-bitbucket.org/ausocean/iot v1.2.11/go.mod h1:Q5FwaOKnCty3dVeVtki6DLwYa5vhNpOaeu1lwLyPCg8=
-bitbucket.org/ausocean/iot v1.2.12 h1:Ixf0CTmWOMJVrJ6IYMEluTrCLlu9LM1eNSBZ+ZUnDmU=
-bitbucket.org/ausocean/iot v1.2.12/go.mod h1:Q5FwaOKnCty3dVeVtki6DLwYa5vhNpOaeu1lwLyPCg8=
 bitbucket.org/ausocean/iot v1.2.13 h1:E9LcW3HYqRgJqxNhPJUCfVRvoV2IAU4B7JSDNxB/x2k=
 bitbucket.org/ausocean/iot v1.2.13/go.mod h1:Q5FwaOKnCty3dVeVtki6DLwYa5vhNpOaeu1lwLyPCg8=
 bitbucket.org/ausocean/utils v1.2.11 h1:zA0FOaPjN960ryp8PKCkV5y50uWBYrIxCVnXjwbvPqg=
 bitbucket.org/ausocean/utils v1.2.11/go.mod h1:uXzX9z3PLemyURTMWRhVI8uLhPX4uuvaaO85v2hcob8=
-bitbucket.org/ausocean/utils v1.2.12 h1:VnskjWTDM475TnQRhBQE0cNp9D6Y6OELrd4UkD2VVIQ=
-bitbucket.org/ausocean/utils v1.2.12/go.mod h1:uXzX9z3PLemyURTMWRhVI8uLhPX4uuvaaO85v2hcob8=
 bitbucket.org/ausocean/utils v1.2.13 h1:tUaIywtoMc1+zl1GCVQokX4mL5X7LNHX5O51AgAPrWA=
 bitbucket.org/ausocean/utils v1.2.13/go.mod h1:uXzX9z3PLemyURTMWRhVI8uLhPX4uuvaaO85v2hcob8=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=

revid/revid.go

@@ -567,14 +567,14 @@ func (r *Revid) Update(vars map[string]string) error {
 			}
 		case "Output":
 			r.cfg.Outputs = make([]uint8, 1)
-			switch value {
+			switch strings.ToLower(value) {
-			case "File":
+			case "file":
 				r.cfg.Outputs[0] = config.OutputFile
-			case "Http":
+			case "http":
 				r.cfg.Outputs[0] = config.OutputHTTP
-			case "Rtmp":
+			case "rtmp":
 				r.cfg.Outputs[0] = config.OutputRTMP
-			case "Rtp":
+			case "rtp":
 				r.cfg.Outputs[0] = config.OutputRTP
 			default:
 				r.cfg.Logger.Log(logger.Warning, pkg+"invalid output param", "value", value)