From 3101d21d99bf9ec2c422b53608845fbc92e80d6b Mon Sep 17 00:00:00 2001 From: Saxon Nelson-Milton Date: Mon, 6 Jun 2022 20:46:14 +0930 Subject: [PATCH] cmd/rv: adding rv upgrade function. init: refactor Makefile and scripts for more flexibility --- cmd/rv/main.go | 102 +++++++++++++++++++++++++++-------------- cmd/rv/upgrade.sh | 56 ---------------------- go.mod | 2 +- go.sum | 4 +- init/Makefile | 30 +++++------- init/create_service.sh | 55 ++++++++++++++++++++++ init/permissions.sh | 6 +++ init/run.sh | 30 ++++++++---- init/rv.service | 10 ---- 9 files changed, 165 insertions(+), 130 deletions(-) delete mode 100644 cmd/rv/upgrade.sh create mode 100644 init/create_service.sh create mode 100644 init/permissions.sh delete mode 100644 init/rv.service diff --git a/cmd/rv/main.go b/cmd/rv/main.go index d6afcc0f..28c15a14 100644 --- a/cmd/rv/main.go +++ b/cmd/rv/main.go @@ -55,8 +55,10 @@ USAGE package main import ( + "fmt" "io" "os" + "os/exec" "runtime/pprof" "strconv" "time" @@ -138,7 +140,7 @@ func main() { if canProfile { profile(log) defer pprof.StopCPUProfile() - log.Info( "profiling started") + log.Info("profiling started") } var ( @@ -148,24 +150,31 @@ func main() { p, err := NewTurbidityProbe(log, 60*time.Second) if err != nil { - log.Fatal( "could not create new turbidity probe", "error", err.Error()) + log.Fatal("could not create new turbidity probe", "error", err.Error()) } - log.Debug( "initialising netsender client") - ns, err := netsender.New(log, nil, readPin(p, rv, log), nil, netsender.WithVarTypes(createVarMap())) + log.Debug("initialising netsender client") + ns, err := netsender.New( + log, + nil, + readPin(p, rv, log), + nil, + netsender.WithVarTypes(createVarMap()), + netsender.WithUpgrader(upgrade), + ) if err != nil { - log.Fatal( pkg+"could not initialise netsender client: "+err.Error()) + log.Fatal(pkg + "could not initialise netsender client: " + err.Error()) } - log.Debug( "initialising revid") + log.Debug("initialising revid") rv, err = revid.New(config.Config{Logger: log}, ns) if err != nil { - log.Fatal( pkg+"could not initialise revid", "error", err.Error()) + log.Fatal(pkg+"could not initialise revid", "error", err.Error()) } err = rv.SetProbe(p) if err != nil { - log.Error( pkg+"could not set probe", "error", err.Error()) + log.Error(pkg+"could not set probe", "error", err.Error()) } // NB: Problems were encountered with communicating with RTSP inputs. When trying to @@ -174,7 +183,7 @@ func main() { // a better way to solve this problem. time.Sleep(runPreDelay) - log.Debug( "beginning main loop") + log.Debug("beginning main loop") run(rv, ns, log, netLog, p) } @@ -183,80 +192,80 @@ func main() { func run(rv *revid.Revid, ns *netsender.Sender, l logging.Logger, nl *netlogger.Logger, p *turbidityProbe) { var vs int for { - l.Debug( "running netsender") + l.Debug("running netsender") err := ns.Run() if err != nil { - l.Warning( pkg+"Run Failed. Retrying...", "error", err.Error()) + l.Warning(pkg+"Run Failed. Retrying...", "error", err.Error()) time.Sleep(netSendRetryTime) continue } - l.Debug( "sending logs") + l.Debug("sending logs") err = nl.Send(ns) if err != nil { - l.Warning( pkg+"Logs could not be sent", "error", err.Error()) + l.Warning(pkg+"Logs could not be sent", "error", err.Error()) } - l.Debug( "checking varsum") + l.Debug("checking varsum") newVs := ns.VarSum() if vs == newVs { sleep(ns, l) continue } vs = newVs - l.Info( "varsum changed", "vs", vs) + l.Info("varsum changed", "vs", vs) - l.Debug( "getting new vars") + l.Debug("getting new vars") vars, err := ns.Vars() if err != nil { - l.Error( pkg+"netSender failed to get vars", "error", err.Error()) + l.Error(pkg+"netSender failed to get vars", "error", err.Error()) time.Sleep(netSendRetryTime) continue } - l.Debug( "got new vars", "vars", vars) + l.Debug("got new vars", "vars", vars) // Configure revid based on the vars. - l.Debug( "updating revid's configuration") + l.Debug("updating revid's configuration") err = rv.Update(vars) if err != nil { - l.Warning( pkg+"couldn't update revid", "error", err.Error()) + l.Warning(pkg+"couldn't update revid", "error", err.Error()) sleep(ns, l) continue } - l.Info( "revid successfully reconfigured") + l.Info("revid successfully reconfigured") // Update transform matrix based on new revid variables. err = p.Update(rv.Config().TransformMatrix) if err != nil { - l.Error( "could not update turbidity probe", "error", err.Error()) + l.Error("could not update turbidity probe", "error", err.Error()) } - l.Debug( "checking mode") + l.Debug("checking mode") switch ns.Mode() { case modePaused: - l.Debug( "mode is Paused, stopping revid") + l.Debug("mode is Paused, stopping revid") rv.Stop() case modeNormal, modeLoop: - l.Debug( "mode is Normal or Loop, starting revid") + l.Debug("mode is Normal or Loop, starting revid") err = rv.Start() if err != nil { - l.Error( pkg+"could not start revid", "error", err.Error()) + l.Error(pkg+"could not start revid", "error", err.Error()) ns.SetMode(modePaused, &vs) sleep(ns, l) continue } case modeBurst: - l.Debug( "mode is Burst, bursting revid") + l.Debug("mode is Burst, bursting revid") err = rv.Burst() if err != nil { - l.Warning( pkg+"could not start burst", "error", err.Error()) + l.Warning(pkg+"could not start burst", "error", err.Error()) ns.SetMode(modePaused, &vs) sleep(ns, l) continue } ns.SetMode(modePaused, &vs) } - l.Info( "revid updated with new mode") + l.Info("revid updated with new mode") sleep(ns, l) } @@ -275,24 +284,24 @@ func createVarMap() map[string]string { func profile(l logging.Logger) { f, err := os.Create(profilePath) if err != nil { - l.Fatal( pkg+"could not create CPU profile", "error", err.Error()) + l.Fatal(pkg+"could not create CPU profile", "error", err.Error()) } if err := pprof.StartCPUProfile(f); err != nil { - l.Fatal( pkg+"could not start CPU profile", "error", err.Error()) + l.Fatal(pkg+"could not start CPU profile", "error", err.Error()) } } // sleep uses a delay to halt the program based on the monitoring period // netsender parameter (mp) defined in the netsender.conf config. func sleep(ns *netsender.Sender, l logging.Logger) { - l.Debug( "sleeping") + l.Debug("sleeping") t, err := strconv.Atoi(ns.Param("mp")) if err != nil { - l.Error( pkg+"could not get sleep time, using default", "error", err) + l.Error(pkg+"could not get sleep time, using default", "error", err) t = defaultSleepTime } time.Sleep(time.Duration(t) * time.Second) - l.Debug( "finished sleeping") + l.Debug("finished sleeping") } // readPin provides a callback function of consistent signature for use by @@ -321,3 +330,28 @@ func readPin(p *turbidityProbe, rv *revid.Revid, l logging.Logger) func(pin *net return nil } } + +// upgrade is a callback to be provided to the netsender client initialiser through the +// netsender.WithUpgrader() option function. This function is called if the +// netsender client gets a remote upgrade request. The tag is used in this case +// to fetch, checkout and then build. +func upgrade(tag, user, gopath string) error { + srcDir := gopath + "/src/bitbucket.org/ausocean/av" + instrs := []struct { + Cmd, Dir string + Args []string + }{ + {Cmd: "git", Dir: srcDir, Args: []string{"fetch", "--depth=1", "origin", "refs/tags/" + tag + ":refs/tags/" + tag}}, + {Cmd: "git", Dir: srcDir, Args: []string{"checkout", "--force", "tags/" + tag}}, + {Cmd: "make", Dir: srcDir + "/init", Args: []string{"rebuild"}}, + } + for _, instr := range instrs { + cmd := exec.Command(instr.Cmd, instr.Args...) + cmd.Dir = instr.Dir + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("could not perform upgrade instruction: %w (%+v), out: %s", err, instr, string(out)) + } + } + return nil +} diff --git a/cmd/rv/upgrade.sh b/cmd/rv/upgrade.sh deleted file mode 100644 index 46cfac2a..00000000 --- a/cmd/rv/upgrade.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -# All-purpose upgrade script. -# Upgrades source(s) to given Git tag, runs make in each directory, -# and write tags to tags.conf upon success, exiting 0. -# NB: Customize SrcDirs as needed to reflect dependencies. -Usage="Usage: upgrade.sh [-d] tag" -BaseDir=$GOPATH/src/bitbucket.org/ausocean -VarDir=/var/netsender -LogFile=/var/log/netsender/stream.log -SrcDirs=($BaseDir/utils $BaseDir/iot $BaseDir/av) -if [ "$1" == "-d" ]; then - set -x - GitFlags="" - NewTag="$2" -else - # capture stdout and stderr - exec 2> $LogFile - exec 1>&2 - GitFlags="--quiet" - NewTag="$1" -fi -if [ -z "$GOPATH" ]; then - echo "Error: GOPATH not defined" - exit 1 -fi -if [ -z "$NewTag" ]; then - echo "$Usage" - exit 1 -fi -for dir in ${SrcDirs[@]}; do - pushd $dir - if [ ! "$?" == 0 ]; then - exit 1 - fi - git fetch $GitFlags --depth=1 origin refs/tags/$NewTag:refs/tags/$NewTag - if [ ! "$?" == 0 ]; then - exit 1 - fi - git checkout $GitFlags --force tags/$NewTag - if [ ! "$?" == 0 ]; then - exit 1 - fi - if [ -e Makefile ]; then - make - if [ ! "$?" == 0 ]; then - exit 1 - fi - fi - popd -done -if [ ! -d "$VarDir" ]; then - echo "Error: $VarDir does not exit." - exit 1 -fi -git tag > "$VarDir/tags.conf" -exit $? diff --git a/go.mod b/go.mod index 8ac0611c..13bbc5a1 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module bitbucket.org/ausocean/av go 1.16 require ( - bitbucket.org/ausocean/iot v1.3.2 + bitbucket.org/ausocean/iot v1.3.3 bitbucket.org/ausocean/utils v1.3.1 github.com/Comcast/gots v0.0.0-20190305015453-8d56e473f0f7 github.com/go-audio/audio v0.0.0-20181013203223-7b2a6ca21480 diff --git a/go.sum b/go.sum index 4af81063..4ab950e7 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,6 @@ bitbucket.org/ausocean/iot v1.3.0/go.mod h1:rRcWt6SoM/jgIZpP1zrpnKb5BhxIMulAJ+q1xTvLh94= -bitbucket.org/ausocean/iot v1.3.2 h1:aMLIipH4GT8ItrHTlFh/cwEbGxsp0fkGhvFZy3zP3KI= -bitbucket.org/ausocean/iot v1.3.2/go.mod h1:NbEg2PvYSHDdUsy5eMmihBySpWfqaHiMdspQDZdDe8o= +bitbucket.org/ausocean/iot v1.3.3 h1:xEu6jrvWEofzFo13154FS66g4i2Ojl94kheoUshFB/A= +bitbucket.org/ausocean/iot v1.3.3/go.mod h1:NbEg2PvYSHDdUsy5eMmihBySpWfqaHiMdspQDZdDe8o= bitbucket.org/ausocean/utils v1.2.11/go.mod h1:uXzX9z3PLemyURTMWRhVI8uLhPX4uuvaaO85v2hcob8= bitbucket.org/ausocean/utils v1.3.0/go.mod h1:yWsulKjbBgwL17/w55MQ6cIT9jmNeOkwpd2gUIxAcIY= bitbucket.org/ausocean/utils v1.3.1 h1:xESAyWsq2tExr8uNPqpysBAgihTJVoLYLF77ABZjX4g= diff --git a/init/Makefile b/init/Makefile index e01388bc..a0e2b86e 100644 --- a/init/Makefile +++ b/init/Makefile @@ -4,47 +4,41 @@ # sudo MA=mac DK=dk install_hard # NB: The default (soft) install does not override conf files. # USR can also be passsed to customise user under which the code is housed. -# For example, if you're trying to make rv on your laptop instead of a pi, -# set USR=. USER := $(shell whoami) PATH := /usr/local/go/bin:$(PATH) +BIN_DIR := /src/bitbucket.org/ausocean/av/cmd/rv +RUN_SCRIPT_DIR := /src/bitbucket.org/ausocean/av/init/run.sh + ifeq ($(MA),) MA := "00:E0:4C:00:00:01" endif ifeq ($(DK),) DK := 0 endif -ifeq ($(USR),) - USR := "pi" -endif .SILENT:make_dirs .SILENT:soft_copy_files .SILENT:hard_copy_files .SILENT:set_mac -.SILENT:syncreboot .SILENT:clean rebuild: - sed -i 's//$(USR)/' run.sh chmod +x run.sh cd ../cmd/rv; go build -tags nocv -install: as_root make_dirs soft_copy_files +install: as_root make_dirs soft_copy_files rebuild @echo "Install complete" -install_hard: as_root make_dirs hard_copy_files set_mac syncreboot +install_hard: as_root make_dirs hard_copy_files set_mac rebuild @echo "Hard install complete" -install_hard_treat: as_root make_dirs hard_copy_files_treat set_mac syncreboot +install_hard_treat: as_root make_dirs hard_copy_files_treat set_mac @echo "Hard install complete" as_root: ifneq ($(USER),root) $(error Must run as superuser!) endif - sed -i 's//$(USR)/' rv.service - sed -i 's//$(USR)/' run.sh make_dirs: if [ ! -d /var/netsender ] ; then \ @@ -60,30 +54,28 @@ soft_copy_files: if [ -f /etc/systemd/system/rv.service ] ; then \ echo "/etc/systemd/system/rv.service left unmodified" ; \ else \ - cp rv.service /etc/systemd/system; \ + bash create_service.sh $(RUN_SCRIPT_DIR) $(BIN_DIR); \ fi systemctl enable rv.service - chmod +x run.sh if [ -f /etc/netsender.conf ] ; then \ echo "/etc/netsender.conf left unmodified" ; \ else \ printf "ma $(MA)\ndk $(DK)\n" > /etc/netsender.conf; \ - chown $(USR) /etc/netsender.conf; \ + bash permissions.sh; \ fi hard_copy_files: if [ -f /etc/systemd/system/rv.service ] ; then \ echo "/etc/systemd/system/rv.service overwritten" ; \ fi - cp -f rv.service /etc/systemd/system + bash create_service.sh $(RUN_SCRIPT_DIR) $(BIN_DIR) systemctl enable rv.service - chmod +x run.sh if [ -f /etc/netsender.conf ] ; then \ echo "Backed up netsender.conf to /etc/netsender.conf.bak"; \ cp /etc/netsender.conf /etc/netsender.conf.bak ; \ fi printf "ma $(MA)\ndk $(DK)\n" > /etc/netsender.conf - chown $(USR) /etc/netsender.conf + bash permissions.sh hard_copy_files_treat: if [ -f /etc/systemd/system/treatment.service ] ; then \ @@ -97,7 +89,7 @@ hard_copy_files_treat: cp /etc/netsender.conf /etc/netsender.conf.bak ; \ fi printf "ma $(MA)\ndk $(DK)\n" > /etc/netsender.conf - chown $(USR) /etc/netsender.conf + bash permissions.sh set_mac: printf "ip link set eth0 address $(MA)\n" > /etc/dhcpcd.enter-hook diff --git a/init/create_service.sh b/init/create_service.sh new file mode 100644 index 00000000..c9104fb5 --- /dev/null +++ b/init/create_service.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# This script is utilised by Makefile for creation of a service responsible for the +# running of a netsender client. The service lines are stored in a string allowing us +# to substitute the GOPATH into the ExecStart path. + +if [ $# -ne 2 ]; then + echo "incorrect number of arguments, expected run script and binary directories" + exit 1 +fi + +# This corresponds to the path beyond the GOPATH corresponding to the run script +# of the client. e.g. "/src/bitbucket.org/ausocean/av/init/run.sh" +run_script_dir=$1 + +# This corresponds to the binary dir. e.g. /src/bitbucket.org/ausocean/av/cmd/rv +bin_dir=$2 + +# Get the bin name (assuming this is at the end of the bin_dir). +bin_name=$(basename $bin_dir) + +# First find the user that corresponds to this path (which is assumed to be at the +# base of the current working directory). +in=$(pwd) +arr_in=(${in//// }) +gopath_user=${arr_in[1]} + +# We can now form the gopath from the obtained user. +gopath="/home/$gopath_user/go" + +# Here are the lines that will go into the rv.service file. We'll set the +# ExecStart field as the GOPATH we've obtained + the passed run script dir. +service=" +[Unit] +Description=Netsender Client for Media Collection and Forwarding + +[Service] +Type=simple +ExecStart=$gopath$run_script_dir $gopath_user $bin_dir +Restart=on-failure + +[Install] +WantedBy=multi-user.target +" + +# The service name will just use the bin name. +service_name="$bin_name.service" + +# Now overwrite the service if it exists, or create the service then write. +service_dir=/etc/systemd/system/$service_name +if [ -f $service_dir ]; then + echo "$service" > $service_dir +else + touch $service_dir + echo "$service" > $service_dir +fi diff --git a/init/permissions.sh b/init/permissions.sh new file mode 100644 index 00000000..904417fa --- /dev/null +++ b/init/permissions.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# This script is used by Makefile to modify permissions required by some targets. +in=$(pwd) +arr_in=(${in//// }) +gopath_user=${arr_in[1]} +chown $gopath_user /etc/netsender.conf diff --git a/init/run.sh b/init/run.sh index fba789a9..79af2fa4 100755 --- a/init/run.sh +++ b/init/run.sh @@ -1,13 +1,27 @@ -#!/bin/sh -e -# This script launches rv on a , intended to run at boot time. +#!/bin/bash -e +# This script launches rv. This is used by the rv.service file that will become +# a systemd service after using the Makefile install* targets. -RVPATH=/home//go/src/bitbucket.org/ausocean/av/cmd/rv +# Check that we have the correct number of arguments passed. +if [ $# -ne 2 ]; then + echo "incorrect number of arguments, expected gopath user and binary directory" + exit 1 +fi + +# This is the user in the GOPATH e.g. for /home/pi/go the user is pi. +gopath_user=$1 + +# This is the dir of the binary from the GOPATH e.g. /src/bitbucket.org/ausocean/av/cmd/rv. +bin_dir=$2 + +# We'll get the bin name from the bin dir (assuming this is same as the bin dir name). +bin_name=$(basename $bin_dir) echo Set kernel parameters: # kernel settings to improve performance on Raspberry Pi # tell Linux to fork optimistically sudo sysctl -w vm.overcommit_memory=1 -# minimize swapng, without disabling it completely +# minimize swapping, without disabling it completely sudo sysctl -w vm.swappiness=1 # the following required directories _should_ already exist @@ -28,13 +42,13 @@ sudo ip addr show | grep inet exec 2> /var/log/netsender/stream.log exec 1>&2 -# set env, working dir and run rv as user -HOME=/home/ +# Now set all required variables. +HOME=/home/$gopath_user GOPATH=$HOME/go -RVPATH=$GOPATH/src/bitbucket.org/ausocean/av/cmd/rv +RVPATH=$GOPATH$bin_dir PATH=$PATH:/usr/local/go/bin:$RVPATH cd $RVPATH -sudo -u HOME=$HOME GOPATH=$GOPATH PATH=$PATH ./rv +sudo -u $gopath_user HOME=$HOME GOPATH=$GOPATH PATH=$PATH ./$bin_name if [ $? -eq 0 ] then echo "Successfully exited rv" diff --git a/init/rv.service b/init/rv.service deleted file mode 100644 index 13fb3127..00000000 --- a/init/rv.service +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Netsender Client for Media Collection and Forwarding - -[Service] -Type=simple -ExecStart=/home//go/src/bitbucket.org/ausocean/av/init/run.sh -Restart=on-failure - -[Install] -WantedBy=multi-user.target