Merged in fix-upgrade (pull request #480)

cmd/rv: adding rv upgrade function. init: refactor Makefile and scripts for more flexibility

Resolves issue #369

Approved-by: Trek Hopton

cmd/rv/main.go

@@ -55,8 +55,10 @@ USAGE
 package main
 
 import (
+	"fmt"
 	"io"
 	"os"
+	"os/exec"
 	"runtime/pprof"
 	"strconv"
 	"time"
@@ -138,7 +140,7 @@ func main() {
 	if canProfile {
 		profile(log)
 		defer pprof.StopCPUProfile()
-		log.Info( "profiling started")
+		log.Info("profiling started")
 	}
 
 	var (
@@ -148,24 +150,31 @@ func main() {
 
 	p, err := NewTurbidityProbe(log, 60*time.Second)
 	if err != nil {
-		log.Fatal( "could not create new turbidity probe", "error", err.Error())
+		log.Fatal("could not create new turbidity probe", "error", err.Error())
 	}
 
-	log.Debug( "initialising netsender client")
+	log.Debug("initialising netsender client")
-	ns, err := netsender.New(log, nil, readPin(p, rv, log), nil, netsender.WithVarTypes(createVarMap()))
+	ns, err := netsender.New(
+		log,
+		nil,
+		readPin(p, rv, log),
+		nil,
+		netsender.WithVarTypes(createVarMap()),
+		netsender.WithUpgrader(upgrade),
+	)
 	if err != nil {
-		log.Fatal( pkg+"could not initialise netsender client: "+err.Error())
+		log.Fatal(pkg + "could not initialise netsender client: " + err.Error())
 	}
 
-	log.Debug( "initialising revid")
+	log.Debug("initialising revid")
 	rv, err = revid.New(config.Config{Logger: log}, ns)
 	if err != nil {
-		log.Fatal( pkg+"could not initialise revid", "error", err.Error())
+		log.Fatal(pkg+"could not initialise revid", "error", err.Error())
 	}
 
 	err = rv.SetProbe(p)
 	if err != nil {
-		log.Error( pkg+"could not set probe", "error", err.Error())
+		log.Error(pkg+"could not set probe", "error", err.Error())
 	}
 
 	// NB: Problems were encountered with communicating with RTSP inputs. When trying to
@@ -174,7 +183,7 @@ func main() {
 	// a better way to solve this problem.
 	time.Sleep(runPreDelay)
 
-	log.Debug( "beginning main loop")
+	log.Debug("beginning main loop")
 	run(rv, ns, log, netLog, p)
 }
 
@@ -183,80 +192,80 @@ func main() {
 func run(rv *revid.Revid, ns *netsender.Sender, l logging.Logger, nl *netlogger.Logger, p *turbidityProbe) {
 	var vs int
 	for {
-		l.Debug( "running netsender")
+		l.Debug("running netsender")
 		err := ns.Run()
 		if err != nil {
-			l.Warning( pkg+"Run Failed. Retrying...", "error", err.Error())
+			l.Warning(pkg+"Run Failed. Retrying...", "error", err.Error())
 			time.Sleep(netSendRetryTime)
 			continue
 		}
 
-		l.Debug( "sending logs")
+		l.Debug("sending logs")
 		err = nl.Send(ns)
 		if err != nil {
-			l.Warning( pkg+"Logs could not be sent", "error", err.Error())
+			l.Warning(pkg+"Logs could not be sent", "error", err.Error())
 		}
 
-		l.Debug( "checking varsum")
+		l.Debug("checking varsum")
 		newVs := ns.VarSum()
 		if vs == newVs {
 			sleep(ns, l)
 			continue
 		}
 		vs = newVs
-		l.Info( "varsum changed", "vs", vs)
+		l.Info("varsum changed", "vs", vs)
 
-		l.Debug( "getting new vars")
+		l.Debug("getting new vars")
 		vars, err := ns.Vars()
 		if err != nil {
-			l.Error( pkg+"netSender failed to get vars", "error", err.Error())
+			l.Error(pkg+"netSender failed to get vars", "error", err.Error())
 			time.Sleep(netSendRetryTime)
 			continue
 		}
-		l.Debug( "got new vars", "vars", vars)
+		l.Debug("got new vars", "vars", vars)
 
 		// Configure revid based on the vars.
-		l.Debug( "updating revid's configuration")
+		l.Debug("updating revid's configuration")
 		err = rv.Update(vars)
 		if err != nil {
-			l.Warning( pkg+"couldn't update revid", "error", err.Error())
+			l.Warning(pkg+"couldn't update revid", "error", err.Error())
 			sleep(ns, l)
 			continue
 		}
-		l.Info( "revid successfully reconfigured")
+		l.Info("revid successfully reconfigured")
 
 		// Update transform matrix based on new revid variables.
 		err = p.Update(rv.Config().TransformMatrix)
 		if err != nil {
-			l.Error( "could not update turbidity probe", "error", err.Error())
+			l.Error("could not update turbidity probe", "error", err.Error())
 		}
 
-		l.Debug( "checking mode")
+		l.Debug("checking mode")
 		switch ns.Mode() {
 		case modePaused:
-			l.Debug( "mode is Paused, stopping revid")
+			l.Debug("mode is Paused, stopping revid")
 			rv.Stop()
 		case modeNormal, modeLoop:
-			l.Debug( "mode is Normal or Loop, starting revid")
+			l.Debug("mode is Normal or Loop, starting revid")
 			err = rv.Start()
 			if err != nil {
-				l.Error( pkg+"could not start revid", "error", err.Error())
+				l.Error(pkg+"could not start revid", "error", err.Error())
 				ns.SetMode(modePaused, &vs)
 				sleep(ns, l)
 				continue
 			}
 		case modeBurst:
-			l.Debug( "mode is Burst, bursting revid")
+			l.Debug("mode is Burst, bursting revid")
 			err = rv.Burst()
 			if err != nil {
-				l.Warning( pkg+"could not start burst", "error", err.Error())
+				l.Warning(pkg+"could not start burst", "error", err.Error())
 				ns.SetMode(modePaused, &vs)
 				sleep(ns, l)
 				continue
 			}
 			ns.SetMode(modePaused, &vs)
 		}
-		l.Info( "revid updated with new mode")
+		l.Info("revid updated with new mode")
 
 		sleep(ns, l)
 	}
@@ -275,24 +284,24 @@ func createVarMap() map[string]string {
 func profile(l logging.Logger) {
 	f, err := os.Create(profilePath)
 	if err != nil {
-		l.Fatal( pkg+"could not create CPU profile", "error", err.Error())
+		l.Fatal(pkg+"could not create CPU profile", "error", err.Error())
 	}
 	if err := pprof.StartCPUProfile(f); err != nil {
-		l.Fatal( pkg+"could not start CPU profile", "error", err.Error())
+		l.Fatal(pkg+"could not start CPU profile", "error", err.Error())
 	}
 }
 
 // sleep uses a delay to halt the program based on the monitoring period
 // netsender parameter (mp) defined in the netsender.conf config.
 func sleep(ns *netsender.Sender, l logging.Logger) {
-	l.Debug( "sleeping")
+	l.Debug("sleeping")
 	t, err := strconv.Atoi(ns.Param("mp"))
 	if err != nil {
-		l.Error( pkg+"could not get sleep time, using default", "error", err)
+		l.Error(pkg+"could not get sleep time, using default", "error", err)
 		t = defaultSleepTime
 	}
 	time.Sleep(time.Duration(t) * time.Second)
-	l.Debug( "finished sleeping")
+	l.Debug("finished sleeping")
 }
 
 // readPin provides a callback function of consistent signature for use by
@@ -321,3 +330,28 @@ func readPin(p *turbidityProbe, rv *revid.Revid, l logging.Logger) func(pin *net
 		return nil
 	}
 }
+
+// upgrade is a callback to be provided to the netsender client initialiser through the
+// netsender.WithUpgrader() option function. This function is called if the
+// netsender client gets a remote upgrade request. The tag is used in this case
+// to fetch, checkout and then build.
+func upgrade(tag, user, gopath string) error {
+	srcDir := gopath + "/src/bitbucket.org/ausocean/av"
+	instrs := []struct {
+		Cmd, Dir string
+		Args     []string
+	}{
+		{Cmd: "git", Dir: srcDir, Args: []string{"fetch", "--depth=1", "origin", "refs/tags/" + tag + ":refs/tags/" + tag}},
+		{Cmd: "git", Dir: srcDir, Args: []string{"checkout", "--force", "tags/" + tag}},
+		{Cmd: "make", Dir: srcDir + "/init", Args: []string{"rebuild"}},
+	}
+	for _, instr := range instrs {
+		cmd := exec.Command(instr.Cmd, instr.Args...)
+		cmd.Dir = instr.Dir
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			return fmt.Errorf("could not perform upgrade instruction: %w (%+v), out: %s", err, instr, string(out))
+		}
+	}
+	return nil
+}

cmd/rv/upgrade.sh

@@ -1,56 +0,0 @@
-#!/bin/bash
-# All-purpose upgrade script.
-# Upgrades source(s) to given Git tag, runs make in each directory,
-# and write tags to tags.conf upon success, exiting 0.
-# NB: Customize SrcDirs as needed to reflect dependencies.
-Usage="Usage: upgrade.sh [-d] tag"
-BaseDir=$GOPATH/src/bitbucket.org/ausocean
-VarDir=/var/netsender
-LogFile=/var/log/netsender/stream.log
-SrcDirs=($BaseDir/utils $BaseDir/iot $BaseDir/av)
-if [ "$1" == "-d" ]; then
-    set  -x
-    GitFlags=""
-    NewTag="$2"
-else
-    # capture stdout and stderr
-    exec 2> $LogFile
-    exec 1>&2
-    GitFlags="--quiet"
-    NewTag="$1"
-fi
-if [ -z "$GOPATH" ]; then
-    echo "Error: GOPATH not defined"
-    exit 1
-fi
-if [ -z "$NewTag" ]; then
-    echo "$Usage"
-    exit 1
-fi
-for dir in ${SrcDirs[@]}; do
-  pushd $dir
-  if [ ! "$?" == 0 ]; then
-    exit 1
-  fi
-  git fetch $GitFlags --depth=1 origin refs/tags/$NewTag:refs/tags/$NewTag
-  if [ ! "$?" == 0 ]; then
-    exit 1
-  fi
-  git checkout $GitFlags --force tags/$NewTag
-  if [ ! "$?" == 0 ]; then
-    exit 1
-  fi
-  if [ -e Makefile ]; then
-    make
-    if [ ! "$?" == 0 ]; then
-      exit 1
-    fi
-  fi
-  popd
-done
-if [ ! -d "$VarDir" ]; then
-  echo "Error: $VarDir does not exit."	
-  exit 1
-fi
-git tag > "$VarDir/tags.conf"
-exit $?

go.mod

@@ -3,7 +3,7 @@ module bitbucket.org/ausocean/av
 go 1.16
 
 require (
-	bitbucket.org/ausocean/iot v1.3.2
+	bitbucket.org/ausocean/iot v1.3.3
 	bitbucket.org/ausocean/utils v1.3.1
 	github.com/Comcast/gots v0.0.0-20190305015453-8d56e473f0f7
 	github.com/go-audio/audio v0.0.0-20181013203223-7b2a6ca21480

go.sum

@@ -1,6 +1,6 @@
 bitbucket.org/ausocean/iot v1.3.0/go.mod h1:rRcWt6SoM/jgIZpP1zrpnKb5BhxIMulAJ+q1xTvLh94=
-bitbucket.org/ausocean/iot v1.3.2 h1:aMLIipH4GT8ItrHTlFh/cwEbGxsp0fkGhvFZy3zP3KI=
+bitbucket.org/ausocean/iot v1.3.3 h1:xEu6jrvWEofzFo13154FS66g4i2Ojl94kheoUshFB/A=
-bitbucket.org/ausocean/iot v1.3.2/go.mod h1:NbEg2PvYSHDdUsy5eMmihBySpWfqaHiMdspQDZdDe8o=
+bitbucket.org/ausocean/iot v1.3.3/go.mod h1:NbEg2PvYSHDdUsy5eMmihBySpWfqaHiMdspQDZdDe8o=
 bitbucket.org/ausocean/utils v1.2.11/go.mod h1:uXzX9z3PLemyURTMWRhVI8uLhPX4uuvaaO85v2hcob8=
 bitbucket.org/ausocean/utils v1.3.0/go.mod h1:yWsulKjbBgwL17/w55MQ6cIT9jmNeOkwpd2gUIxAcIY=
 bitbucket.org/ausocean/utils v1.3.1 h1:xESAyWsq2tExr8uNPqpysBAgihTJVoLYLF77ABZjX4g=

init/Makefile

@@ -4,47 +4,41 @@
 # 	sudo MA=mac DK=dk install_hard
 # NB: The default (soft) install does not override conf files.
 # USR can also be passsed to customise user under which the code is housed.
-# For example, if you're trying to make rv on your laptop instead of a pi,
-# set USR=<your user name>.
 USER := $(shell whoami)
 PATH := /usr/local/go/bin:$(PATH)
+BIN_DIR := /src/bitbucket.org/ausocean/av/cmd/rv
+RUN_SCRIPT_DIR := /src/bitbucket.org/ausocean/av/init/run.sh
+
 ifeq ($(MA),)
 	MA := "00:E0:4C:00:00:01"
 endif
 ifeq ($(DK),)
 	DK := 0
 endif
-ifeq ($(USR),)
-	USR := "pi"
-endif
 
 .SILENT:make_dirs
 .SILENT:soft_copy_files
 .SILENT:hard_copy_files
 .SILENT:set_mac
-.SILENT:syncreboot
 .SILENT:clean
 
 rebuild:
-	sed -i 's/<user>/$(USR)/' run.sh
 	chmod +x run.sh
 	cd ../cmd/rv; go build -tags nocv
 
-install: as_root make_dirs soft_copy_files
+install: as_root make_dirs soft_copy_files rebuild
 	@echo "Install complete"
 
-install_hard: as_root make_dirs hard_copy_files set_mac syncreboot
+install_hard: as_root make_dirs hard_copy_files set_mac rebuild
 	@echo "Hard install complete"
 
-install_hard_treat: as_root make_dirs hard_copy_files_treat set_mac syncreboot
+install_hard_treat: as_root make_dirs hard_copy_files_treat set_mac
 	@echo "Hard install complete"
 
 as_root:
 ifneq ($(USER),root)
 	$(error Must run as superuser!)
 endif
-	sed -i 's/<user>/$(USR)/' rv.service
-	sed -i 's/<user>/$(USR)/' run.sh
 
 make_dirs:
 	if [ ! -d /var/netsender ] ; then \
@@ -60,30 +54,28 @@ soft_copy_files:
 	if [ -f /etc/systemd/system/rv.service ] ; then \
 		echo "/etc/systemd/system/rv.service left unmodified" ; \
 	else \
-		cp rv.service /etc/systemd/system; \
+		bash create_service.sh $(RUN_SCRIPT_DIR) $(BIN_DIR); \
 	fi
 	systemctl enable rv.service
-	chmod +x run.sh
 	if [ -f /etc/netsender.conf ] ; then \
 		echo "/etc/netsender.conf left unmodified" ; \
 	else \
 		printf "ma $(MA)\ndk $(DK)\n" > /etc/netsender.conf; \
-		chown $(USR) /etc/netsender.conf; \
+		bash permissions.sh; \
 	fi
 
 hard_copy_files:
 	if [ -f /etc/systemd/system/rv.service ] ; then \
 		echo "/etc/systemd/system/rv.service overwritten" ; \
 	fi
-	cp -f rv.service /etc/systemd/system
+	bash create_service.sh $(RUN_SCRIPT_DIR) $(BIN_DIR)
 	systemctl enable rv.service
-	chmod +x run.sh
 	if [ -f /etc/netsender.conf ] ; then \
 		echo "Backed up netsender.conf to /etc/netsender.conf.bak"; \
 		cp /etc/netsender.conf /etc/netsender.conf.bak ; \
 	fi
 	printf "ma $(MA)\ndk $(DK)\n" > /etc/netsender.conf
-	chown $(USR) /etc/netsender.conf
+	bash permissions.sh
 
 hard_copy_files_treat:
 	if [ -f /etc/systemd/system/treatment.service ] ; then \
@@ -97,7 +89,7 @@ hard_copy_files_treat:
 		cp /etc/netsender.conf /etc/netsender.conf.bak ; \
 	fi
 	printf "ma $(MA)\ndk $(DK)\n" > /etc/netsender.conf
-	chown $(USR) /etc/netsender.conf
+	bash permissions.sh
 
 set_mac:
 	printf "ip link set eth0 address $(MA)\n" > /etc/dhcpcd.enter-hook

init/create_service.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# This script is utilised by Makefile for creation of a service responsible for the
+# running of a netsender client. The service lines are stored in a string allowing us
+# to substitute the GOPATH into the ExecStart path.
+
+if [ $# -ne 2 ]; then
+  echo "incorrect number of arguments, expected run script and binary directories"
+  exit 1
+fi
+
+# This corresponds to the path beyond the GOPATH corresponding to the run script
+# of the client. e.g. "/src/bitbucket.org/ausocean/av/init/run.sh"
+run_script_dir=$1
+
+# This corresponds to the binary dir. e.g. /src/bitbucket.org/ausocean/av/cmd/rv
+bin_dir=$2
+
+# Get the bin name (assuming this is at the end of the bin_dir).
+bin_name=$(basename $bin_dir)
+
+# First find the user that corresponds to this path (which is assumed to be at the
+# base of the current working directory).
+in=$(pwd)
+arr_in=(${in//// })
+gopath_user=${arr_in[1]}
+
+# We can now form the gopath from the obtained user.
+gopath="/home/$gopath_user/go"
+
+# Here are the lines that will go into the rv.service file. We'll set the
+# ExecStart field as the GOPATH we've obtained + the passed run script dir.
+service="
+[Unit]
+Description=Netsender Client for Media Collection and Forwarding
+
+[Service]
+Type=simple
+ExecStart=$gopath$run_script_dir $gopath_user $bin_dir
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+"
+
+# The service name will just use the bin name.
+service_name="$bin_name.service"
+
+# Now overwrite the service if it exists, or create the service then write.
+service_dir=/etc/systemd/system/$service_name
+if [ -f $service_dir ]; then
+  echo "$service" > $service_dir
+else
+  touch $service_dir
+  echo "$service" > $service_dir
+fi

init/permissions.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# This script is used by Makefile to modify permissions required by some targets.
+in=$(pwd)
+arr_in=(${in//// })
+gopath_user=${arr_in[1]}
+chown $gopath_user /etc/netsender.conf

init/run.sh

@@ -1,13 +1,27 @@
-#!/bin/sh -e
+#!/bin/bash -e
-# This script launches rv on a <user>, intended to run at boot time.
+# This script launches rv. This is used by the rv.service file that will become
+# a systemd service after using the Makefile install* targets.
 
-RVPATH=/home/<user>/go/src/bitbucket.org/ausocean/av/cmd/rv
+# Check that we have the correct number of arguments passed.
+if [ $# -ne 2 ]; then
+  echo "incorrect number of arguments, expected gopath user and binary directory"
+  exit 1
+fi
+
+# This is the user in the GOPATH e.g. for /home/pi/go the user is pi.
+gopath_user=$1
+
+# This is the dir of the binary from the GOPATH e.g. /src/bitbucket.org/ausocean/av/cmd/rv.
+bin_dir=$2
+
+# We'll get the bin name from the bin dir (assuming this is same as the bin dir name).
+bin_name=$(basename $bin_dir)
 
 echo Set kernel parameters:
 # kernel settings to improve performance on Raspberry Pi
 # tell Linux to fork optimistically
 sudo sysctl -w vm.overcommit_memory=1
-# minimize swap<user>ng, without disabling it completely
+# minimize swapping, without disabling it completely
 sudo sysctl -w vm.swappiness=1
 
 # the following required directories _should_ already exist
@@ -28,13 +42,13 @@ sudo ip addr show | grep inet
 exec 2> /var/log/netsender/stream.log
 exec 1>&2
 
-# set env, working dir and run rv as <user> user
+# Now set all required variables.
-HOME=/home/<user>
+HOME=/home/$gopath_user
 GOPATH=$HOME/go
-RVPATH=$GOPATH/src/bitbucket.org/ausocean/av/cmd/rv
+RVPATH=$GOPATH$bin_dir
 PATH=$PATH:/usr/local/go/bin:$RVPATH
 cd $RVPATH
-sudo -u <user> HOME=$HOME GOPATH=$GOPATH PATH=$PATH ./rv
+sudo -u $gopath_user HOME=$HOME GOPATH=$GOPATH PATH=$PATH ./$bin_name
 if [ $? -eq 0 ]
 then
   echo "Successfully exited rv"

init/rv.service

@@ -1,10 +0,0 @@
-[Unit]
-Description=Netsender Client for Media Collection and Forwarding
-
-[Service]
-Type=simple
-ExecStart=/home/<user>/go/src/bitbucket.org/ausocean/av/init/run.sh
-Restart=on-failure
-
-[Install]
-WantedBy=multi-user.target