Merged in fix-upgrade (pull request #480)

cmd/rv: adding rv upgrade function. init: refactor Makefile and scripts for more flexibility

Resolves issue #369

Approved-by: Trek Hopton
This commit is contained in:
Saxon Milton 2022-06-07 05:52:12 +00:00
commit 84bf14aa0a
9 changed files with 165 additions and 130 deletions

View File

@ -55,8 +55,10 @@ USAGE
package main
import (
"fmt"
"io"
"os"
"os/exec"
"runtime/pprof"
"strconv"
"time"
@ -138,7 +140,7 @@ func main() {
if canProfile {
profile(log)
defer pprof.StopCPUProfile()
log.Info( "profiling started")
log.Info("profiling started")
}
var (
@ -148,24 +150,31 @@ func main() {
p, err := NewTurbidityProbe(log, 60*time.Second)
if err != nil {
log.Fatal( "could not create new turbidity probe", "error", err.Error())
log.Fatal("could not create new turbidity probe", "error", err.Error())
}
log.Debug( "initialising netsender client")
ns, err := netsender.New(log, nil, readPin(p, rv, log), nil, netsender.WithVarTypes(createVarMap()))
log.Debug("initialising netsender client")
ns, err := netsender.New(
log,
nil,
readPin(p, rv, log),
nil,
netsender.WithVarTypes(createVarMap()),
netsender.WithUpgrader(upgrade),
)
if err != nil {
log.Fatal( pkg+"could not initialise netsender client: "+err.Error())
log.Fatal(pkg + "could not initialise netsender client: " + err.Error())
}
log.Debug( "initialising revid")
log.Debug("initialising revid")
rv, err = revid.New(config.Config{Logger: log}, ns)
if err != nil {
log.Fatal( pkg+"could not initialise revid", "error", err.Error())
log.Fatal(pkg+"could not initialise revid", "error", err.Error())
}
err = rv.SetProbe(p)
if err != nil {
log.Error( pkg+"could not set probe", "error", err.Error())
log.Error(pkg+"could not set probe", "error", err.Error())
}
// NB: Problems were encountered with communicating with RTSP inputs. When trying to
@ -174,7 +183,7 @@ func main() {
// a better way to solve this problem.
time.Sleep(runPreDelay)
log.Debug( "beginning main loop")
log.Debug("beginning main loop")
run(rv, ns, log, netLog, p)
}
@ -183,80 +192,80 @@ func main() {
func run(rv *revid.Revid, ns *netsender.Sender, l logging.Logger, nl *netlogger.Logger, p *turbidityProbe) {
var vs int
for {
l.Debug( "running netsender")
l.Debug("running netsender")
err := ns.Run()
if err != nil {
l.Warning( pkg+"Run Failed. Retrying...", "error", err.Error())
l.Warning(pkg+"Run Failed. Retrying...", "error", err.Error())
time.Sleep(netSendRetryTime)
continue
}
l.Debug( "sending logs")
l.Debug("sending logs")
err = nl.Send(ns)
if err != nil {
l.Warning( pkg+"Logs could not be sent", "error", err.Error())
l.Warning(pkg+"Logs could not be sent", "error", err.Error())
}
l.Debug( "checking varsum")
l.Debug("checking varsum")
newVs := ns.VarSum()
if vs == newVs {
sleep(ns, l)
continue
}
vs = newVs
l.Info( "varsum changed", "vs", vs)
l.Info("varsum changed", "vs", vs)
l.Debug( "getting new vars")
l.Debug("getting new vars")
vars, err := ns.Vars()
if err != nil {
l.Error( pkg+"netSender failed to get vars", "error", err.Error())
l.Error(pkg+"netSender failed to get vars", "error", err.Error())
time.Sleep(netSendRetryTime)
continue
}
l.Debug( "got new vars", "vars", vars)
l.Debug("got new vars", "vars", vars)
// Configure revid based on the vars.
l.Debug( "updating revid's configuration")
l.Debug("updating revid's configuration")
err = rv.Update(vars)
if err != nil {
l.Warning( pkg+"couldn't update revid", "error", err.Error())
l.Warning(pkg+"couldn't update revid", "error", err.Error())
sleep(ns, l)
continue
}
l.Info( "revid successfully reconfigured")
l.Info("revid successfully reconfigured")
// Update transform matrix based on new revid variables.
err = p.Update(rv.Config().TransformMatrix)
if err != nil {
l.Error( "could not update turbidity probe", "error", err.Error())
l.Error("could not update turbidity probe", "error", err.Error())
}
l.Debug( "checking mode")
l.Debug("checking mode")
switch ns.Mode() {
case modePaused:
l.Debug( "mode is Paused, stopping revid")
l.Debug("mode is Paused, stopping revid")
rv.Stop()
case modeNormal, modeLoop:
l.Debug( "mode is Normal or Loop, starting revid")
l.Debug("mode is Normal or Loop, starting revid")
err = rv.Start()
if err != nil {
l.Error( pkg+"could not start revid", "error", err.Error())
l.Error(pkg+"could not start revid", "error", err.Error())
ns.SetMode(modePaused, &vs)
sleep(ns, l)
continue
}
case modeBurst:
l.Debug( "mode is Burst, bursting revid")
l.Debug("mode is Burst, bursting revid")
err = rv.Burst()
if err != nil {
l.Warning( pkg+"could not start burst", "error", err.Error())
l.Warning(pkg+"could not start burst", "error", err.Error())
ns.SetMode(modePaused, &vs)
sleep(ns, l)
continue
}
ns.SetMode(modePaused, &vs)
}
l.Info( "revid updated with new mode")
l.Info("revid updated with new mode")
sleep(ns, l)
}
@ -275,24 +284,24 @@ func createVarMap() map[string]string {
func profile(l logging.Logger) {
f, err := os.Create(profilePath)
if err != nil {
l.Fatal( pkg+"could not create CPU profile", "error", err.Error())
l.Fatal(pkg+"could not create CPU profile", "error", err.Error())
}
if err := pprof.StartCPUProfile(f); err != nil {
l.Fatal( pkg+"could not start CPU profile", "error", err.Error())
l.Fatal(pkg+"could not start CPU profile", "error", err.Error())
}
}
// sleep uses a delay to halt the program based on the monitoring period
// netsender parameter (mp) defined in the netsender.conf config.
func sleep(ns *netsender.Sender, l logging.Logger) {
l.Debug( "sleeping")
l.Debug("sleeping")
t, err := strconv.Atoi(ns.Param("mp"))
if err != nil {
l.Error( pkg+"could not get sleep time, using default", "error", err)
l.Error(pkg+"could not get sleep time, using default", "error", err)
t = defaultSleepTime
}
time.Sleep(time.Duration(t) * time.Second)
l.Debug( "finished sleeping")
l.Debug("finished sleeping")
}
// readPin provides a callback function of consistent signature for use by
@ -321,3 +330,28 @@ func readPin(p *turbidityProbe, rv *revid.Revid, l logging.Logger) func(pin *net
return nil
}
}
// upgrade is a callback to be provided to the netsender client initialiser through the
// netsender.WithUpgrader() option function. This function is called if the
// netsender client gets a remote upgrade request. The tag is used in this case
// to fetch, checkout and then build.
func upgrade(tag, user, gopath string) error {
srcDir := gopath + "/src/bitbucket.org/ausocean/av"
instrs := []struct {
Cmd, Dir string
Args []string
}{
{Cmd: "git", Dir: srcDir, Args: []string{"fetch", "--depth=1", "origin", "refs/tags/" + tag + ":refs/tags/" + tag}},
{Cmd: "git", Dir: srcDir, Args: []string{"checkout", "--force", "tags/" + tag}},
{Cmd: "make", Dir: srcDir + "/init", Args: []string{"rebuild"}},
}
for _, instr := range instrs {
cmd := exec.Command(instr.Cmd, instr.Args...)
cmd.Dir = instr.Dir
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("could not perform upgrade instruction: %w (%+v), out: %s", err, instr, string(out))
}
}
return nil
}

View File

@ -1,56 +0,0 @@
#!/bin/bash
# All-purpose upgrade script.
# Upgrades source(s) to given Git tag, runs make in each directory,
# and write tags to tags.conf upon success, exiting 0.
# NB: Customize SrcDirs as needed to reflect dependencies.
Usage="Usage: upgrade.sh [-d] tag"
BaseDir=$GOPATH/src/bitbucket.org/ausocean
VarDir=/var/netsender
LogFile=/var/log/netsender/stream.log
SrcDirs=($BaseDir/utils $BaseDir/iot $BaseDir/av)
if [ "$1" == "-d" ]; then
set -x
GitFlags=""
NewTag="$2"
else
# capture stdout and stderr
exec 2> $LogFile
exec 1>&2
GitFlags="--quiet"
NewTag="$1"
fi
if [ -z "$GOPATH" ]; then
echo "Error: GOPATH not defined"
exit 1
fi
if [ -z "$NewTag" ]; then
echo "$Usage"
exit 1
fi
for dir in ${SrcDirs[@]}; do
pushd $dir
if [ ! "$?" == 0 ]; then
exit 1
fi
git fetch $GitFlags --depth=1 origin refs/tags/$NewTag:refs/tags/$NewTag
if [ ! "$?" == 0 ]; then
exit 1
fi
git checkout $GitFlags --force tags/$NewTag
if [ ! "$?" == 0 ]; then
exit 1
fi
if [ -e Makefile ]; then
make
if [ ! "$?" == 0 ]; then
exit 1
fi
fi
popd
done
if [ ! -d "$VarDir" ]; then
echo "Error: $VarDir does not exit."
exit 1
fi
git tag > "$VarDir/tags.conf"
exit $?

2
go.mod
View File

@ -3,7 +3,7 @@ module bitbucket.org/ausocean/av
go 1.16
require (
bitbucket.org/ausocean/iot v1.3.2
bitbucket.org/ausocean/iot v1.3.3
bitbucket.org/ausocean/utils v1.3.1
github.com/Comcast/gots v0.0.0-20190305015453-8d56e473f0f7
github.com/go-audio/audio v0.0.0-20181013203223-7b2a6ca21480

4
go.sum
View File

@ -1,6 +1,6 @@
bitbucket.org/ausocean/iot v1.3.0/go.mod h1:rRcWt6SoM/jgIZpP1zrpnKb5BhxIMulAJ+q1xTvLh94=
bitbucket.org/ausocean/iot v1.3.2 h1:aMLIipH4GT8ItrHTlFh/cwEbGxsp0fkGhvFZy3zP3KI=
bitbucket.org/ausocean/iot v1.3.2/go.mod h1:NbEg2PvYSHDdUsy5eMmihBySpWfqaHiMdspQDZdDe8o=
bitbucket.org/ausocean/iot v1.3.3 h1:xEu6jrvWEofzFo13154FS66g4i2Ojl94kheoUshFB/A=
bitbucket.org/ausocean/iot v1.3.3/go.mod h1:NbEg2PvYSHDdUsy5eMmihBySpWfqaHiMdspQDZdDe8o=
bitbucket.org/ausocean/utils v1.2.11/go.mod h1:uXzX9z3PLemyURTMWRhVI8uLhPX4uuvaaO85v2hcob8=
bitbucket.org/ausocean/utils v1.3.0/go.mod h1:yWsulKjbBgwL17/w55MQ6cIT9jmNeOkwpd2gUIxAcIY=
bitbucket.org/ausocean/utils v1.3.1 h1:xESAyWsq2tExr8uNPqpysBAgihTJVoLYLF77ABZjX4g=

View File

@ -4,47 +4,41 @@
# sudo MA=mac DK=dk install_hard
# NB: The default (soft) install does not override conf files.
# USR can also be passsed to customise user under which the code is housed.
# For example, if you're trying to make rv on your laptop instead of a pi,
# set USR=<your user name>.
USER := $(shell whoami)
PATH := /usr/local/go/bin:$(PATH)
BIN_DIR := /src/bitbucket.org/ausocean/av/cmd/rv
RUN_SCRIPT_DIR := /src/bitbucket.org/ausocean/av/init/run.sh
ifeq ($(MA),)
MA := "00:E0:4C:00:00:01"
endif
ifeq ($(DK),)
DK := 0
endif
ifeq ($(USR),)
USR := "pi"
endif
.SILENT:make_dirs
.SILENT:soft_copy_files
.SILENT:hard_copy_files
.SILENT:set_mac
.SILENT:syncreboot
.SILENT:clean
rebuild:
sed -i 's/<user>/$(USR)/' run.sh
chmod +x run.sh
cd ../cmd/rv; go build -tags nocv
install: as_root make_dirs soft_copy_files
install: as_root make_dirs soft_copy_files rebuild
@echo "Install complete"
install_hard: as_root make_dirs hard_copy_files set_mac syncreboot
install_hard: as_root make_dirs hard_copy_files set_mac rebuild
@echo "Hard install complete"
install_hard_treat: as_root make_dirs hard_copy_files_treat set_mac syncreboot
install_hard_treat: as_root make_dirs hard_copy_files_treat set_mac
@echo "Hard install complete"
as_root:
ifneq ($(USER),root)
$(error Must run as superuser!)
endif
sed -i 's/<user>/$(USR)/' rv.service
sed -i 's/<user>/$(USR)/' run.sh
make_dirs:
if [ ! -d /var/netsender ] ; then \
@ -60,30 +54,28 @@ soft_copy_files:
if [ -f /etc/systemd/system/rv.service ] ; then \
echo "/etc/systemd/system/rv.service left unmodified" ; \
else \
cp rv.service /etc/systemd/system; \
bash create_service.sh $(RUN_SCRIPT_DIR) $(BIN_DIR); \
fi
systemctl enable rv.service
chmod +x run.sh
if [ -f /etc/netsender.conf ] ; then \
echo "/etc/netsender.conf left unmodified" ; \
else \
printf "ma $(MA)\ndk $(DK)\n" > /etc/netsender.conf; \
chown $(USR) /etc/netsender.conf; \
bash permissions.sh; \
fi
hard_copy_files:
if [ -f /etc/systemd/system/rv.service ] ; then \
echo "/etc/systemd/system/rv.service overwritten" ; \
fi
cp -f rv.service /etc/systemd/system
bash create_service.sh $(RUN_SCRIPT_DIR) $(BIN_DIR)
systemctl enable rv.service
chmod +x run.sh
if [ -f /etc/netsender.conf ] ; then \
echo "Backed up netsender.conf to /etc/netsender.conf.bak"; \
cp /etc/netsender.conf /etc/netsender.conf.bak ; \
fi
printf "ma $(MA)\ndk $(DK)\n" > /etc/netsender.conf
chown $(USR) /etc/netsender.conf
bash permissions.sh
hard_copy_files_treat:
if [ -f /etc/systemd/system/treatment.service ] ; then \
@ -97,7 +89,7 @@ hard_copy_files_treat:
cp /etc/netsender.conf /etc/netsender.conf.bak ; \
fi
printf "ma $(MA)\ndk $(DK)\n" > /etc/netsender.conf
chown $(USR) /etc/netsender.conf
bash permissions.sh
set_mac:
printf "ip link set eth0 address $(MA)\n" > /etc/dhcpcd.enter-hook

55
init/create_service.sh Normal file
View File

@ -0,0 +1,55 @@
#!/bin/bash
# This script is utilised by Makefile for creation of a service responsible for the
# running of a netsender client. The service lines are stored in a string allowing us
# to substitute the GOPATH into the ExecStart path.
if [ $# -ne 2 ]; then
echo "incorrect number of arguments, expected run script and binary directories"
exit 1
fi
# This corresponds to the path beyond the GOPATH corresponding to the run script
# of the client. e.g. "/src/bitbucket.org/ausocean/av/init/run.sh"
run_script_dir=$1
# This corresponds to the binary dir. e.g. /src/bitbucket.org/ausocean/av/cmd/rv
bin_dir=$2
# Get the bin name (assuming this is at the end of the bin_dir).
bin_name=$(basename $bin_dir)
# First find the user that corresponds to this path (which is assumed to be at the
# base of the current working directory).
in=$(pwd)
arr_in=(${in//// })
gopath_user=${arr_in[1]}
# We can now form the gopath from the obtained user.
gopath="/home/$gopath_user/go"
# Here are the lines that will go into the rv.service file. We'll set the
# ExecStart field as the GOPATH we've obtained + the passed run script dir.
service="
[Unit]
Description=Netsender Client for Media Collection and Forwarding
[Service]
Type=simple
ExecStart=$gopath$run_script_dir $gopath_user $bin_dir
Restart=on-failure
[Install]
WantedBy=multi-user.target
"
# The service name will just use the bin name.
service_name="$bin_name.service"
# Now overwrite the service if it exists, or create the service then write.
service_dir=/etc/systemd/system/$service_name
if [ -f $service_dir ]; then
echo "$service" > $service_dir
else
touch $service_dir
echo "$service" > $service_dir
fi

6
init/permissions.sh Normal file
View File

@ -0,0 +1,6 @@
#!/bin/bash
# This script is used by Makefile to modify permissions required by some targets.
in=$(pwd)
arr_in=(${in//// })
gopath_user=${arr_in[1]}
chown $gopath_user /etc/netsender.conf

View File

@ -1,13 +1,27 @@
#!/bin/sh -e
# This script launches rv on a <user>, intended to run at boot time.
#!/bin/bash -e
# This script launches rv. This is used by the rv.service file that will become
# a systemd service after using the Makefile install* targets.
RVPATH=/home/<user>/go/src/bitbucket.org/ausocean/av/cmd/rv
# Check that we have the correct number of arguments passed.
if [ $# -ne 2 ]; then
echo "incorrect number of arguments, expected gopath user and binary directory"
exit 1
fi
# This is the user in the GOPATH e.g. for /home/pi/go the user is pi.
gopath_user=$1
# This is the dir of the binary from the GOPATH e.g. /src/bitbucket.org/ausocean/av/cmd/rv.
bin_dir=$2
# We'll get the bin name from the bin dir (assuming this is same as the bin dir name).
bin_name=$(basename $bin_dir)
echo Set kernel parameters:
# kernel settings to improve performance on Raspberry Pi
# tell Linux to fork optimistically
sudo sysctl -w vm.overcommit_memory=1
# minimize swap<user>ng, without disabling it completely
# minimize swapping, without disabling it completely
sudo sysctl -w vm.swappiness=1
# the following required directories _should_ already exist
@ -28,13 +42,13 @@ sudo ip addr show | grep inet
exec 2> /var/log/netsender/stream.log
exec 1>&2
# set env, working dir and run rv as <user> user
HOME=/home/<user>
# Now set all required variables.
HOME=/home/$gopath_user
GOPATH=$HOME/go
RVPATH=$GOPATH/src/bitbucket.org/ausocean/av/cmd/rv
RVPATH=$GOPATH$bin_dir
PATH=$PATH:/usr/local/go/bin:$RVPATH
cd $RVPATH
sudo -u <user> HOME=$HOME GOPATH=$GOPATH PATH=$PATH ./rv
sudo -u $gopath_user HOME=$HOME GOPATH=$GOPATH PATH=$PATH ./$bin_name
if [ $? -eq 0 ]
then
echo "Successfully exited rv"

View File

@ -1,10 +0,0 @@
[Unit]
Description=Netsender Client for Media Collection and Forwarding
[Service]
Type=simple
ExecStart=/home/<user>/go/src/bitbucket.org/ausocean/av/init/run.sh
Restart=on-failure
[Install]
WantedBy=multi-user.target