From d10723122444e74e5201837c65477d99402fc74d Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Sat, 26 Jan 2019 18:04:21 +1030
Subject: [PATCH 01/83] stream/mts: starting to work out meta logistics in
 encoder.go and added incomplete AddDescriptor function in psi package

---
 stream/mts/encoder.go | 60 ++++++++++---------------------------------
 stream/mts/psi/psi.go |  8 ++++++
 2 files changed, 21 insertions(+), 47 deletions(-)

diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index 53342624..d4c2ecd6 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -82,43 +82,6 @@ var (
 			},
 		},
 	}
-
-	// standardPmtTimeLocation is a standard PMT with time and location
-	// descriptors, but time and location fields zeroed out.
-	standardPmtTimeLocation = psi.PSI{
-		Pf:  0x00,
-		Tid: 0x02,
-		Ssi: true,
-		Sl:  0x3e,
-		Tss: &psi.TSS{
-			Tide: 0x01,
-			V:    0,
-			Cni:  true,
-			Sn:   0,
-			Lsn:  0,
-			Sd: &psi.PMT{
-				Pcrpid: 0x0100,
-				Pil:    psi.PmtTimeLocationPil,
-				Pd: []psi.Desc{
-					{
-						Dt: psi.TimeDescTag,
-						Dl: psi.TimeDataSize,
-						Dd: make([]byte, psi.TimeDataSize),
-					},
-					{
-						Dt: psi.LocationDescTag,
-						Dl: psi.LocationDataSize,
-						Dd: make([]byte, psi.LocationDataSize),
-					},
-				},
-				Essd: &psi.ESSD{
-					St:   0x1b,
-					Epid: 0x0100,
-					Esil: 0x00,
-				},
-			},
-		},
-	}
 )
 
 const (
@@ -162,13 +125,24 @@ func (tl *timeLocation) Location() string {
 	return l
 }
 
+func updateMeta(b []byte) error {
+	var p psi.PSIBytes
+	p = b
+	// TODO: get length of meta data and format appropriately
+	metaLen := 0
+	metaData := ""
+
+	p.AddDescriptor(psi.MetadataTag, metaLen, metaData)
+	return nil
+}
+
 // MetData will hold time and location data which may be set externally if
 // this data is available. It is then inserted into mpegts packets outputted.
 var MetaData timeLocation
 
 var (
 	patTable = standardPat.Bytes()
-	pmtTable = standardPmtTimeLocation.Bytes()
+	pmtTable = standardPmt.Bytes()
 )
 
 // Time related constants.
@@ -283,15 +257,7 @@ func (e *Encoder) writePSI() error {
 		return err
 	}
 
-	// Update pmt table time and location.
-	err = psi.UpdateTime(pmtTable, MetaData.TimeStamp())
-	if err != nil {
-		return err
-	}
-	err = psi.UpdateLocation(pmtTable, MetaData.Location())
-	if err != nil {
-		return nil
-	}
+	updateMeta(pmtTable)
 
 	// Create mts packet from pmt table.
 	pmtPkt := Packet{
diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index 09028ba6..34063d0c 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -68,6 +68,14 @@ const (
 	crcSize          = 4
 )
 
+const MetadataTag = 0x26
+
+type PSIBytes []byte
+
+func (p *PSIBytes) AddDescriptor(tag, len int, data string) {
+
+}
+
 // Program specific information
 type PSI struct {
 	Pf  byte   // Point field

From 87ded6bf2ef84a3db52d6ba70625a561fcf1fb10 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Sat, 26 Jan 2019 21:53:19 +1030
Subject: [PATCH 02/83] stream/mts/encoder.go: implemented metadata receiver
 functions: Add, Get, All and Delete

---
 stream/mts/encoder.go | 70 ++++++++++++++++++++++++-------------------
 1 file changed, 39 insertions(+), 31 deletions(-)

diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index d4c2ecd6..9909a0fe 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -29,6 +29,8 @@ LICENSE
 package mts
 
 import (
+	"errors"
+	"fmt"
 	"io"
 	"sync"
 	"time"
@@ -88,43 +90,53 @@ const (
 	psiSndCnt = 7
 )
 
-// timeLocation holds time and location data
-type timeLocation struct {
-	mu       sync.RWMutex
-	time     uint64
-	location string
+// global Meta
+var meta Meta
+
+type Meta struct {
+	mu   sync.RWMutex
+	data map[string]string
 }
 
-// SetTimeStamp sets the time field of a TimeLocation.
-func (tl *timeLocation) SetTimeStamp(t uint64) {
-	tl.mu.Lock()
-	tl.time = t
-	tl.mu.Unlock()
+// Add adds metadata with key and val, if already exists return error
+func (m *Meta) Add(key, val string) error {
+	m.mu.Lock()
+	if _, exists := m.data[key]; !exists {
+		return errors.New(fmt.Sprintf("Metadata for: %v already exists", key))
+	}
+	m.data[key] = val
+	m.mu.Unlock()
+	return nil
 }
 
-// GetTimeStamp returns the location of a TimeLocation.
-func (tl *timeLocation) TimeStamp() uint64 {
-	tl.mu.RLock()
-	t := tl.time
-	tl.mu.RUnlock()
-	return t
+// All returns the a copy of the map containing the meta data
+func (m *Meta) All() map[string]string {
+	var cpy map[string]string
+	for k, v := range m.data {
+		cpy[k] = v
+	}
+	return cpy
 }
 
-// SetLocation sets the location of a TimeLocation.
-func (tl *timeLocation) SetLocation(l string) {
-	tl.mu.Lock()
-	tl.location = l
-	tl.mu.Unlock()
+// Get returns the meta data for the passed key
+func (m *Meta) Get(key string) (string, error) {
+	val, ok := m.data[key]
+	if !ok {
+		return "", errors.New("Key does not exist in metadata map")
+	}
+	return val, nil
 }
 
-// GetLocation returns the location of a TimeLocation.
-func (tl *timeLocation) Location() string {
-	tl.mu.RLock()
-	l := tl.location
-	tl.mu.RUnlock()
-	return l
+// Remove deletes a meta entry in the map and returns error if it doesn’t exist
+func (m *Meta) Delete(key string) error {
+	if _, ok := m.data[key]; ok {
+		delete(m.data, key)
+		return nil
+	}
+	return errors.New("Trying to delete map entry that doesn't exist")
 }
 
+// updateMeta ...
 func updateMeta(b []byte) error {
 	var p psi.PSIBytes
 	p = b
@@ -136,10 +148,6 @@ func updateMeta(b []byte) error {
 	return nil
 }
 
-// MetData will hold time and location data which may be set externally if
-// this data is available. It is then inserted into mpegts packets outputted.
-var MetaData timeLocation
-
 var (
 	patTable = standardPat.Bytes()
 	pmtTable = standardPmt.Bytes()

From 8f5a2352b294624bf00390880f82f0f44f70e211 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Sat, 26 Jan 2019 21:57:14 +1030
Subject: [PATCH 03/83] stream/mts: added meta.go file to contain struct and
 methods relating to Metadata and operations

---
 stream/mts/encoder.go | 46 -------------------------------------------
 1 file changed, 46 deletions(-)

diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index 9909a0fe..fb2afbed 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -29,10 +29,7 @@ LICENSE
 package mts
 
 import (
-	"errors"
-	"fmt"
 	"io"
-	"sync"
 	"time"
 
 	"bitbucket.org/ausocean/av/stream/mts/pes"
@@ -93,49 +90,6 @@ const (
 // global Meta
 var meta Meta
 
-type Meta struct {
-	mu   sync.RWMutex
-	data map[string]string
-}
-
-// Add adds metadata with key and val, if already exists return error
-func (m *Meta) Add(key, val string) error {
-	m.mu.Lock()
-	if _, exists := m.data[key]; !exists {
-		return errors.New(fmt.Sprintf("Metadata for: %v already exists", key))
-	}
-	m.data[key] = val
-	m.mu.Unlock()
-	return nil
-}
-
-// All returns the a copy of the map containing the meta data
-func (m *Meta) All() map[string]string {
-	var cpy map[string]string
-	for k, v := range m.data {
-		cpy[k] = v
-	}
-	return cpy
-}
-
-// Get returns the meta data for the passed key
-func (m *Meta) Get(key string) (string, error) {
-	val, ok := m.data[key]
-	if !ok {
-		return "", errors.New("Key does not exist in metadata map")
-	}
-	return val, nil
-}
-
-// Remove deletes a meta entry in the map and returns error if it doesn’t exist
-func (m *Meta) Delete(key string) error {
-	if _, ok := m.data[key]; ok {
-		delete(m.data, key)
-		return nil
-	}
-	return errors.New("Trying to delete map entry that doesn't exist")
-}
-
 // updateMeta ...
 func updateMeta(b []byte) error {
 	var p psi.PSIBytes

From df07f3ff48a0be27598c5f81ef12aec731b8563c Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Sat, 26 Jan 2019 22:36:34 +1030
Subject: [PATCH 04/83] stream/mts/meta.go: added Format func, but need to
 complete later once specs are clarified

---
 stream/mts/meta.go | 82 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 82 insertions(+)
 create mode 100644 stream/mts/meta.go

diff --git a/stream/mts/meta.go b/stream/mts/meta.go
new file mode 100644
index 00000000..8abdeb4f
--- /dev/null
+++ b/stream/mts/meta.go
@@ -0,0 +1,82 @@
+/*
+NAME
+  meta.go
+
+DESCRIPTION
+  See Readme.md
+
+AUTHOR
+  Saxon Nelson-Milton <saxon@ausocean.org>
+
+LICENSE
+  meta.go is Copyright (C) 2017-2019 the Australian Ocean Lab (AusOcean)
+
+  It is free software: you can redistribute it and/or modify them
+  under the terms of the GNU General Public License as published by the
+  Free Software Foundation, either version 3 of the License, or (at your
+  option) any later version.
+
+  It is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with revid in gpl.txt. If not, see http://www.gnu.org/licenses.
+*/
+
+package mts
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+)
+
+type Meta struct {
+	mu   sync.RWMutex
+	data map[string]string
+}
+
+// Add adds metadata with key and val, if already exists return error
+func (m *Meta) Add(key, val string) error {
+	m.mu.Lock()
+	if _, exists := m.data[key]; !exists {
+		return errors.New(fmt.Sprintf("Metadata for: %v already exists", key))
+	}
+	m.data[key] = val
+	m.mu.Unlock()
+	return nil
+}
+
+// All returns the a copy of the map containing the meta data
+func (m *Meta) All() map[string]string {
+	var cpy map[string]string
+	for k, v := range m.data {
+		cpy[k] = v
+	}
+	return cpy
+}
+
+// Get returns the meta data for the passed key
+func (m *Meta) Get(key string) (string, error) {
+	val, ok := m.data[key]
+	if !ok {
+		return "", errors.New("Key does not exist in metadata map")
+	}
+	return val, nil
+}
+
+// Remove deletes a meta entry in the map and returns error if it doesn’t exist
+func (m *Meta) Delete(key string) error {
+	if _, ok := m.data[key]; ok {
+		delete(m.data, key)
+		return nil
+	}
+	return errors.New("Trying to delete map entry that doesn't exist")
+}
+
+func (m *Meta) Format() []byte {
+	// TODO: complete this
+	return []byte("someData")
+}

From ecf7263bc190ed06b9246dc4216bf6ea3544a8a8 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Sat, 26 Jan 2019 22:42:31 +1030
Subject: [PATCH 05/83] stream/mts/meta.go: started using mutex where I was
 supposed to

---
 stream/mts/encoder.go | 22 ++++++++++------------
 stream/mts/meta.go    |  8 ++++++++
 stream/mts/psi/psi.go |  3 +--
 3 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index fb2afbed..9ce254e7 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -90,18 +90,6 @@ const (
 // global Meta
 var meta Meta
 
-// updateMeta ...
-func updateMeta(b []byte) error {
-	var p psi.PSIBytes
-	p = b
-	// TODO: get length of meta data and format appropriately
-	metaLen := 0
-	metaData := ""
-
-	p.AddDescriptor(psi.MetadataTag, metaLen, metaData)
-	return nil
-}
-
 var (
 	patTable = standardPat.Bytes()
 	pmtTable = standardPmt.Bytes()
@@ -259,3 +247,13 @@ func (e *Encoder) ccFor(pid int) byte {
 	e.continuity[pid] = (cc + 1) & continuityCounterMask
 	return cc
 }
+
+// updateMeta ...
+func updateMeta(b []byte) error {
+	var p psi.PSIBytes
+	p = b
+	// TODO: get length of meta data and format appropriately
+	m := meta.Format()
+	p.AddDescriptor(psi.MetadataTag, len(m), m)
+	return nil
+}
diff --git a/stream/mts/meta.go b/stream/mts/meta.go
index 8abdeb4f..c8d70ca2 100644
--- a/stream/mts/meta.go
+++ b/stream/mts/meta.go
@@ -51,24 +51,32 @@ func (m *Meta) Add(key, val string) error {
 
 // All returns the a copy of the map containing the meta data
 func (m *Meta) All() map[string]string {
+	m.mu.Lock()
 	var cpy map[string]string
 	for k, v := range m.data {
 		cpy[k] = v
 	}
+	m.mu.Unlock()
 	return cpy
 }
 
 // Get returns the meta data for the passed key
 func (m *Meta) Get(key string) (string, error) {
+	m.mu.Lock()
 	val, ok := m.data[key]
+	m.mu.Unlock()
 	if !ok {
 		return "", errors.New("Key does not exist in metadata map")
 	}
+
 	return val, nil
+
 }
 
 // Remove deletes a meta entry in the map and returns error if it doesn’t exist
 func (m *Meta) Delete(key string) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
 	if _, ok := m.data[key]; ok {
 		delete(m.data, key)
 		return nil
diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index 34063d0c..2942fbfb 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -72,8 +72,7 @@ const MetadataTag = 0x26
 
 type PSIBytes []byte
 
-func (p *PSIBytes) AddDescriptor(tag, len int, data string) {
-
+func (p *PSIBytes) AddDescriptor(tag, len int, data []byte) {
 }
 
 // Program specific information

From 6f421ab706962a0e29f1b994a63af867640242ec Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Sat, 26 Jan 2019 23:35:31 +1030
Subject: [PATCH 06/83] stream/mts/psi.go: added more to AddDescriptor and
 added signature for edistDesc

---
 stream/mts/encoder.go |  1 -
 stream/mts/psi/psi.go | 36 +++++++++++++++++++++++++++++++++++-
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index 9ce254e7..60fb6c07 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -252,7 +252,6 @@ func (e *Encoder) ccFor(pid int) byte {
 func updateMeta(b []byte) error {
 	var p psi.PSIBytes
 	p = b
-	// TODO: get length of meta data and format appropriately
 	m := meta.Format()
 	p.AddDescriptor(psi.MetadataTag, len(m), m)
 	return nil
diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index 2942fbfb..7571fb03 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -26,6 +26,12 @@ LICENSE
 
 package psi
 
+import (
+	"errors"
+
+	"github.com/Comcast/gots/psi"
+)
+
 const (
 	PacketSize = 184 // packet size of a psi.
 )
@@ -68,11 +74,39 @@ const (
 	crcSize          = 4
 )
 
+const (
+	SectionLenIdx1 = 2
+	SectionLenIdx2 = 3
+)
+
+const (
+	SectionLenMask1 = 0x03
+)
+
 const MetadataTag = 0x26
 
 type PSIBytes []byte
 
-func (p *PSIBytes) AddDescriptor(tag, len int, data []byte) {
+func (p *PSIBytes) AddDescriptor(tag int, data []byte) error {
+	// first check that we actually have table syntax section
+	// NB: if table syntax section doesn't exist (unlikely I think) then we could
+	// just add it
+
+	// Check that this is actually a PMT
+	if psi.TableID(*p) != pmtID {
+		return errors.New("trying to add descriptor, but not pmt")
+	}
+
+	if !p.editDesc(tag, data) {
+
+	}
+	// if exists update
+	// otherwise add
+	return nil
+}
+
+func (p *PSIBytes) editDesc(tag int, data []byte) bool {
+	return true
 }
 
 // Program specific information

From 9171b56d313d59ace1640a474c481516e739b29f Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Sun, 27 Jan 2019 16:55:00 +1030
Subject: [PATCH 07/83] stream/mts: created new type to represent Descriptor
 (typedef of []bytes) so that we can create receiver functions. Wrote
 AddDescriptor func to add or update a descriptor in a pmt. Wrote
 ProgramInfoLen func to return the program info length i.e. len of
 descriptors. Wrote HasDescriptor to check if descriptor exists, if so return
 the descriptor so that we can update. Wrote descriptors which returns []byte
 of all descriptors. Wrote create descriptor, which adds a descriptor to the
 existing if any i.e. shifts data downwards to accomodate new data. Wrote
 update func to update a descriptor.

---
 stream/mts/encoder.go |  2 +-
 stream/mts/psi/psi.go | 94 ++++++++++++++++++++++++++++++++-----------
 2 files changed, 72 insertions(+), 24 deletions(-)

diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index 60fb6c07..9fbf72a9 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -253,6 +253,6 @@ func updateMeta(b []byte) error {
 	var p psi.PSIBytes
 	p = b
 	m := meta.Format()
-	p.AddDescriptor(psi.MetadataTag, len(m), m)
+	p.Descriptor(psi.MetadataTag, len(m), m)
 	return nil
 }
diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index 7571fb03..167b2034 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -85,29 +85,10 @@ const (
 
 const MetadataTag = 0x26
 
-type PSIBytes []byte
-
-func (p *PSIBytes) AddDescriptor(tag int, data []byte) error {
-	// first check that we actually have table syntax section
-	// NB: if table syntax section doesn't exist (unlikely I think) then we could
-	// just add it
-
-	// Check that this is actually a PMT
-	if psi.TableID(*p) != pmtID {
-		return errors.New("trying to add descriptor, but not pmt")
-	}
-
-	if !p.editDesc(tag, data) {
-
-	}
-	// if exists update
-	// otherwise add
-	return nil
-}
-
-func (p *PSIBytes) editDesc(tag int, data []byte) bool {
-	return true
-}
+type (
+	PSIBytes   []byte
+	Descriptor []byte
+)
 
 // Program specific information
 type PSI struct {
@@ -246,3 +227,70 @@ func asByte(b bool) byte {
 	}
 	return 0x00
 }
+
+func (p *PSIBytes) AddDescriptor(tag int, data []byte) error {
+	if psi.TableID(*p) != pmtID {
+		return errors.New("trying to add descriptor, but not pmt")
+	}
+
+	desc := p.HasDescriptor(tag)
+	if desc == nil {
+		p.createDescriptor(tag, data)
+		return nil
+	}
+	desc.update(data)
+	return nil
+}
+
+func (p *PSIBytes) ProgramInfoLen() int {
+	return int((((*p)[ProgramInfoLenIdx1] & ProgramInfoLenMask1) << 8) | (*p)[ProgramInfoLenIdx2])
+}
+
+func (p *PSIBytes) HasDescriptor(int tag) Descriptor {
+	descs := p.descriptors()
+	if descs == nil {
+		return nil
+	}
+	for i := 0; i < len(descs); {
+		t := descs[i]
+		if t == tag {
+			return esc[i : i+int(descs[i+1])]
+		}
+		i += 1 + desc[i+1]
+	}
+	return nil
+}
+
+func (p *PSIBytes) descriptors() []byte {
+	return p[DescriptorsIdx : DescriptorsIdx+p.ProgramInfoLen()]
+}
+
+func (p *PSIBytes) createDescriptor(tag, data) {
+	curProgLen := p.ProgramInfoLen()
+	dataLen := len(data)
+
+	// Calculate the new descriptors index and length.
+	newDescIdx := DescriptorsIdx + curProgLen
+	newDescLen := dLen + 2
+
+	// Copy data down from newDescIdx to create room for the new descriptor.
+	copy(p[newDescIdx+newDescLen:], p[newDescIdx:dataLen-newDescLen])
+
+	// Set the tag, data len and data of the new desriptor.
+	p[newDescIdx] = tag
+	p[newDescIdx+1] = dataLen
+	copy(p[newDescIdx+2:newDescIdx+2+dataLen], data)
+
+	// Update the program info length to account for the new descriptor.
+	newProgInfoLen := curProgLen + dataLen
+	p[ProgramInfoLenIdx1] = newProgInfoLen >> 8
+	p[ProgramInfoLenIdx2] = byte(newProgInfoLen)
+}
+
+func (d *Descriptor) update(data) {
+	if len(data) > d[1] {
+		// TODO: implement resizing of descriptor
+		panic("Can't resize descriptor data")
+	}
+
+}

From 1be7e08b9e5afa1b664bebd2a4e2c879ad7a38f4 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Sun, 27 Jan 2019 16:56:14 +1030
Subject: [PATCH 08/83] stream/mts: renamed Format function for metadata to
 Encode

---
 stream/mts/meta.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stream/mts/meta.go b/stream/mts/meta.go
index c8d70ca2..7ab9cd3a 100644
--- a/stream/mts/meta.go
+++ b/stream/mts/meta.go
@@ -84,7 +84,7 @@ func (m *Meta) Delete(key string) error {
 	return errors.New("Trying to delete map entry that doesn't exist")
 }
 
-func (m *Meta) Format() []byte {
+func (m *Meta) Encode() []byte {
 	// TODO: complete this
 	return []byte("someData")
 }

From 46b2bc45204ec078da79305805b451b25ded8a76 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Sun, 27 Jan 2019 17:04:46 +1030
Subject: [PATCH 09/83] stream/mts: added some consts to describe indexes and
 masks, and fixed some syntax errors, so now it all builds

---
 stream/mts/psi/psi.go | 40 +++++++++++++++++++++++++---------------
 1 file changed, 25 insertions(+), 15 deletions(-)

diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index 167b2034..441a7221 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -83,6 +83,16 @@ const (
 	SectionLenMask1 = 0x03
 )
 
+const (
+	ProgramInfoLenIdx1  = 11
+	ProgramInfoLenIdx2  = 12
+	ProgramInfoLenMask1 = 0x03
+)
+
+const (
+	DescriptorsIdx = ProgramInfoLenIdx2 + 1
+)
+
 const MetadataTag = 0x26
 
 type (
@@ -246,49 +256,49 @@ func (p *PSIBytes) ProgramInfoLen() int {
 	return int((((*p)[ProgramInfoLenIdx1] & ProgramInfoLenMask1) << 8) | (*p)[ProgramInfoLenIdx2])
 }
 
-func (p *PSIBytes) HasDescriptor(int tag) Descriptor {
+func (p *PSIBytes) HasDescriptor(tag int) Descriptor {
 	descs := p.descriptors()
 	if descs == nil {
 		return nil
 	}
 	for i := 0; i < len(descs); {
-		t := descs[i]
+		t := int(descs[i])
 		if t == tag {
-			return esc[i : i+int(descs[i+1])]
+			return descs[i : i+int(descs[i+1])]
 		}
-		i += 1 + desc[i+1]
+		i += 1 + int(descs[i+1])
 	}
 	return nil
 }
 
 func (p *PSIBytes) descriptors() []byte {
-	return p[DescriptorsIdx : DescriptorsIdx+p.ProgramInfoLen()]
+	return (*p)[DescriptorsIdx : DescriptorsIdx+p.ProgramInfoLen()]
 }
 
-func (p *PSIBytes) createDescriptor(tag, data) {
+func (p *PSIBytes) createDescriptor(tag int, data []byte) {
 	curProgLen := p.ProgramInfoLen()
 	dataLen := len(data)
 
 	// Calculate the new descriptors index and length.
 	newDescIdx := DescriptorsIdx + curProgLen
-	newDescLen := dLen + 2
+	newDescLen := dataLen + 2
 
 	// Copy data down from newDescIdx to create room for the new descriptor.
-	copy(p[newDescIdx+newDescLen:], p[newDescIdx:dataLen-newDescLen])
+	copy((*p)[newDescIdx+newDescLen:], (*p)[newDescIdx:dataLen-newDescLen])
 
 	// Set the tag, data len and data of the new desriptor.
-	p[newDescIdx] = tag
-	p[newDescIdx+1] = dataLen
-	copy(p[newDescIdx+2:newDescIdx+2+dataLen], data)
+	(*p)[newDescIdx] = byte(tag)
+	(*p)[newDescIdx+1] = byte(dataLen)
+	copy((*p)[newDescIdx+2:newDescIdx+2+dataLen], data)
 
 	// Update the program info length to account for the new descriptor.
 	newProgInfoLen := curProgLen + dataLen
-	p[ProgramInfoLenIdx1] = newProgInfoLen >> 8
-	p[ProgramInfoLenIdx2] = byte(newProgInfoLen)
+	(*p)[ProgramInfoLenIdx1] = byte(newProgInfoLen >> 8)
+	(*p)[ProgramInfoLenIdx2] = byte(newProgInfoLen)
 }
 
-func (d *Descriptor) update(data) {
-	if len(data) > d[1] {
+func (d *Descriptor) update(data []byte) {
+	if len(data) > int((*d)[1]) {
 		// TODO: implement resizing of descriptor
 		panic("Can't resize descriptor data")
 	}

From c547c8bd19e28384999fe13a86631f9414e669b1 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Sun, 27 Jan 2019 17:16:44 +1030
Subject: [PATCH 10/83] stream/mts: added meta_test.go file and wrote todos for
 testing

---
 stream/mts/meta_test.go | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 stream/mts/meta_test.go

diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go
new file mode 100644
index 00000000..6796611f
--- /dev/null
+++ b/stream/mts/meta_test.go
@@ -0,0 +1,36 @@
+/*
+NAME
+  meta_test.go
+
+DESCRIPTION
+  See Readme.md
+
+AUTHOR
+  Saxon Nelson-Milton <saxon@ausocean.org>
+
+LICENSE
+  meta_test.go is Copyright (C) 2017-2019 the Australian Ocean Lab (AusOcean)
+
+  It is free software: you can redistribute it and/or modify them
+  under the terms of the GNU General Public License as published by the
+  Free Software Foundation, either version 3 of the License, or (at your
+  option) any later version.
+
+  It is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with revid in gpl.txt. If not, see http://www.gnu.org/licenses.
+*/
+
+package mts
+
+// TODO: test add when key doesn't exist
+// TODO: test add when key does exist
+// TODO: test all
+// TODO: test get when key exists
+// TODO: test get when key doesn't exist
+// TODO: test delete when key exists
+// TODO: test delete when key doesn't exist

From 3bb1ca9379634250c570d645b11d27051c5978ff Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Sun, 27 Jan 2019 17:57:42 +1030
Subject: [PATCH 11/83] stream/mts: wrote NewMeta func to initialize map in the
 Meta struct. Also wrote two tests, one TestAddAndGet to see if we can add and
 get metadata, and also second TestUpdate to see if we can correctly update
 metadata with the same key using Meta.Add

---
 stream/mts/encoder.go   |  4 ++--
 stream/mts/meta.go      | 12 +++++-------
 stream/mts/meta_test.go | 37 +++++++++++++++++++++++++++++++++++++
 3 files changed, 44 insertions(+), 9 deletions(-)

diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index 9fbf72a9..8f46a15a 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -252,7 +252,7 @@ func (e *Encoder) ccFor(pid int) byte {
 func updateMeta(b []byte) error {
 	var p psi.PSIBytes
 	p = b
-	m := meta.Format()
-	p.Descriptor(psi.MetadataTag, len(m), m)
+	m := meta.Encode()
+	p.AddDescriptor(psi.MetadataTag, m)
 	return nil
 }
diff --git a/stream/mts/meta.go b/stream/mts/meta.go
index 7ab9cd3a..57787452 100644
--- a/stream/mts/meta.go
+++ b/stream/mts/meta.go
@@ -29,7 +29,6 @@ package mts
 
 import (
 	"errors"
-	"fmt"
 	"sync"
 )
 
@@ -38,15 +37,15 @@ type Meta struct {
 	data map[string]string
 }
 
+func NewMeta() *Meta {
+	return &Meta{data: make(map[string]string)}
+}
+
 // Add adds metadata with key and val, if already exists return error
-func (m *Meta) Add(key, val string) error {
+func (m *Meta) Add(key, val string) {
 	m.mu.Lock()
-	if _, exists := m.data[key]; !exists {
-		return errors.New(fmt.Sprintf("Metadata for: %v already exists", key))
-	}
 	m.data[key] = val
 	m.mu.Unlock()
-	return nil
 }
 
 // All returns the a copy of the map containing the meta data
@@ -70,7 +69,6 @@ func (m *Meta) Get(key string) (string, error) {
 	}
 
 	return val, nil
-
 }
 
 // Remove deletes a meta entry in the map and returns error if it doesn’t exist
diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go
index 6796611f..3b0662c9 100644
--- a/stream/mts/meta_test.go
+++ b/stream/mts/meta_test.go
@@ -27,8 +27,45 @@ LICENSE
 
 package mts
 
+import (
+	"testing"
+)
+
 // TODO: test add when key doesn't exist
+func TestAddAndGet(t *testing.T) {
+	meta := NewMeta()
+	meta.Add("loc", "a,b,c")
+	meta.Add("ts", "12345678")
+
+	if data, err := meta.Get("loc"); err != nil {
+		t.Errorf("Could not get data for key: loc: %v", err.Error())
+		if data != "a,b,c" {
+			t.Errorf("Did not get expected data")
+		}
+	}
+
+	if data, err := meta.Get("ts"); err != nil {
+		t.Errorf("Could not get data for key: ts: %v", err.Error())
+		if data != "12345678" {
+			t.Errorf("Did not get expected data")
+		}
+	}
+}
+
 // TODO: test add when key does exist
+func TestUpdate(t *testing.T) {
+	meta := NewMeta()
+	meta.Add("loc", "a,b,c")
+	meta.Add("loc", "d,e,f")
+
+	if data, err := meta.Get("loc"); err != nil {
+		t.Errorf("Did not expect err: %v", err.Error())
+		if data != "d,e,f" {
+			t.Errorf("Data did not correctly update for key \"loc\"")
+		}
+	}
+}
+
 // TODO: test all
 // TODO: test get when key exists
 // TODO: test get when key doesn't exist

From 17d06f49f4063a9deea8262eaccc323f5519a0a1 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Sun, 27 Jan 2019 18:02:51 +1030
Subject: [PATCH 12/83] stream/mts/meta_test.go: added TestAll func to make
 sure Meta.All is working correctly

---
 stream/mts/meta_test.go | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go
index 3b0662c9..e5ae04e6 100644
--- a/stream/mts/meta_test.go
+++ b/stream/mts/meta_test.go
@@ -28,6 +28,7 @@ LICENSE
 package mts
 
 import (
+	"reflect"
 	"testing"
 )
 
@@ -66,6 +67,22 @@ func TestUpdate(t *testing.T) {
 	}
 }
 
+func TestAll(t *testing.T) {
+	meta := NewMeta()
+	tstMap := map[string]string{
+		"loc": "a,b,c",
+		"ts":  "12345678",
+	}
+
+	meta.Add("loc", "a,b,c")
+	meta.Add("ts", "12345678")
+	metaMap := meta.All()
+
+	if !reflect.DeepEqual(metaMap, tstMap) {
+		t.Errorf("Map not correct. Got: %v, want: %v", metaMap, tstMap)
+	}
+}
+
 // TODO: test all
 // TODO: test get when key exists
 // TODO: test get when key doesn't exist

From bd54dd128ba21775264c3b81f89f41c66f94a27f Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Sun, 27 Jan 2019 18:15:22 +1030
Subject: [PATCH 13/83] stream/mts/meta_test.go: added TestGetAbsent key to
 check that we get an error when we try and get data with key that doesn't
 exist in metadata map

---
 stream/mts/meta.go      |  8 ++++++--
 stream/mts/meta_test.go | 10 ++++++++--
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/stream/mts/meta.go b/stream/mts/meta.go
index 57787452..c1e29ab0 100644
--- a/stream/mts/meta.go
+++ b/stream/mts/meta.go
@@ -32,6 +32,10 @@ import (
 	"sync"
 )
 
+var (
+	errKeyAbsent = errors.New("Key does not exist in map")
+)
+
 type Meta struct {
 	mu   sync.RWMutex
 	data map[string]string
@@ -51,7 +55,7 @@ func (m *Meta) Add(key, val string) {
 // All returns the a copy of the map containing the meta data
 func (m *Meta) All() map[string]string {
 	m.mu.Lock()
-	var cpy map[string]string
+	cpy := make(map[string]string)
 	for k, v := range m.data {
 		cpy[k] = v
 	}
@@ -65,7 +69,7 @@ func (m *Meta) Get(key string) (string, error) {
 	val, ok := m.data[key]
 	m.mu.Unlock()
 	if !ok {
-		return "", errors.New("Key does not exist in metadata map")
+		return "", errKeyAbsent
 	}
 
 	return val, nil
diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go
index e5ae04e6..73727b5e 100644
--- a/stream/mts/meta_test.go
+++ b/stream/mts/meta_test.go
@@ -83,8 +83,14 @@ func TestAll(t *testing.T) {
 	}
 }
 
-// TODO: test all
-// TODO: test get when key exists
 // TODO: test get when key doesn't exist
+func TestGetAbsentKey(t *testing.T) {
+	meta := NewMeta()
+
+	if _, err := meta.Get("loc"); err != errKeyAbsent {
+		t.Errorf("Expected error: %v", errKeyAbsent.Error())
+	}
+}
+
 // TODO: test delete when key exists
 // TODO: test delete when key doesn't exist

From c16f1443729b399a3034ee382355356fd2159c7a Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Sun, 27 Jan 2019 18:21:49 +1030
Subject: [PATCH 14/83] stream/mts/meta_test.go: added TestDelete to check that
 deleting a particular metadata entry by key works

---
 stream/mts/meta_test.go | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go
index 73727b5e..70795a28 100644
--- a/stream/mts/meta_test.go
+++ b/stream/mts/meta_test.go
@@ -88,7 +88,18 @@ func TestGetAbsentKey(t *testing.T) {
 	meta := NewMeta()
 
 	if _, err := meta.Get("loc"); err != errKeyAbsent {
-		t.Errorf("Expected error: %v", errKeyAbsent.Error())
+		t.Errorf("Did not get expected err: %v", errKeyAbsent.Error())
+	}
+}
+
+func TestDelete(t *testing.T) {
+	meta := NewMeta()
+	meta.Add("loc", "a,b,c")
+	if err := meta.Delete("loc"); err != nil {
+		t.Errorf("Did not expect error: %v", err.Error())
+	}
+	if _, err := meta.Get("loc"); err != errKeyAbsent {
+		t.Errorf("Did not get expected err: %v", errKeyAbsent)
 	}
 }
 

From 7fc2b76e6b3230c8fa82d94d842d9d9006284a10 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Sun, 27 Jan 2019 18:24:26 +1030
Subject: [PATCH 15/83] stream/mts/meta_test.go: added TestDeleteKeyAbsent to
 check that we get correct err when we try to delete data of key that doesn't
 exist

---
 stream/mts/meta.go      | 2 +-
 stream/mts/meta_test.go | 9 ++++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/stream/mts/meta.go b/stream/mts/meta.go
index c1e29ab0..1f482790 100644
--- a/stream/mts/meta.go
+++ b/stream/mts/meta.go
@@ -83,7 +83,7 @@ func (m *Meta) Delete(key string) error {
 		delete(m.data, key)
 		return nil
 	}
-	return errors.New("Trying to delete map entry that doesn't exist")
+	return errKeyAbsent
 }
 
 func (m *Meta) Encode() []byte {
diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go
index 70795a28..6c81e4c5 100644
--- a/stream/mts/meta_test.go
+++ b/stream/mts/meta_test.go
@@ -99,7 +99,14 @@ func TestDelete(t *testing.T) {
 		t.Errorf("Did not expect error: %v", err.Error())
 	}
 	if _, err := meta.Get("loc"); err != errKeyAbsent {
-		t.Errorf("Did not get expected err: %v", errKeyAbsent)
+		t.Errorf("Did not get expected err: %v", errKeyAbsent.Error())
+	}
+}
+
+func TestDeleteAbsentKey(t *testing.T) {
+	meta := NewMeta()
+	if err := meta.Delete("loc"); err != errKeyAbsent {
+		t.Errorf("Did not get expected err: %v", errKeyAbsent.Error())
 	}
 }
 

From e1ac1ac5d418734e1b89629bb92d2cd111ee5787 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Sun, 27 Jan 2019 18:29:45 +1030
Subject: [PATCH 16/83] stream/mts/meta_test.go: using consts for commonly
 occuring test strings

---
 stream/mts/meta_test.go | 55 ++++++++++++++++++++++-------------------
 1 file changed, 29 insertions(+), 26 deletions(-)

diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go
index 6c81e4c5..fa8e545c 100644
--- a/stream/mts/meta_test.go
+++ b/stream/mts/meta_test.go
@@ -28,40 +28,47 @@ LICENSE
 package mts
 
 import (
+	"errors"
 	"reflect"
 	"testing"
 )
 
-// TODO: test add when key doesn't exist
+const (
+	tstKey1  = "loc"
+	tstData1 = "a,b,c"
+	tstKey2  = "ts"
+	tstData2 = "12345678"
+	tstData3 = "d,e,f"
+)
+
 func TestAddAndGet(t *testing.T) {
 	meta := NewMeta()
-	meta.Add("loc", "a,b,c")
-	meta.Add("ts", "12345678")
-
-	if data, err := meta.Get("loc"); err != nil {
+	meta.Add(tstKey1, tstData1)
+	meta.Add(tstKey2, tstData2)
+	errors.New("Trying to delete map entry that doesn't exist")
+	if data, err := meta.Get(tstKey1); err != nil {
 		t.Errorf("Could not get data for key: loc: %v", err.Error())
-		if data != "a,b,c" {
+		if data != tstData1 {
 			t.Errorf("Did not get expected data")
 		}
 	}
 
-	if data, err := meta.Get("ts"); err != nil {
+	if data, err := meta.Get(tstKey2); err != nil {
 		t.Errorf("Could not get data for key: ts: %v", err.Error())
-		if data != "12345678" {
+		if data != tstData2 {
 			t.Errorf("Did not get expected data")
 		}
 	}
 }
 
-// TODO: test add when key does exist
 func TestUpdate(t *testing.T) {
 	meta := NewMeta()
-	meta.Add("loc", "a,b,c")
-	meta.Add("loc", "d,e,f")
+	meta.Add(tstKey1, tstData1)
+	meta.Add(tstKey1, tstData3)
 
-	if data, err := meta.Get("loc"); err != nil {
+	if data, err := meta.Get(tstKey1); err != nil {
 		t.Errorf("Did not expect err: %v", err.Error())
-		if data != "d,e,f" {
+		if data != tstData2 {
 			t.Errorf("Data did not correctly update for key \"loc\"")
 		}
 	}
@@ -70,12 +77,12 @@ func TestUpdate(t *testing.T) {
 func TestAll(t *testing.T) {
 	meta := NewMeta()
 	tstMap := map[string]string{
-		"loc": "a,b,c",
-		"ts":  "12345678",
+		tstKey1: tstData1,
+		tstKey2: tstData2,
 	}
 
-	meta.Add("loc", "a,b,c")
-	meta.Add("ts", "12345678")
+	meta.Add(tstKey1, tstData1)
+	meta.Add(tstKey2, tstData2)
 	metaMap := meta.All()
 
 	if !reflect.DeepEqual(metaMap, tstMap) {
@@ -83,32 +90,28 @@ func TestAll(t *testing.T) {
 	}
 }
 
-// TODO: test get when key doesn't exist
 func TestGetAbsentKey(t *testing.T) {
 	meta := NewMeta()
 
-	if _, err := meta.Get("loc"); err != errKeyAbsent {
+	if _, err := meta.Get(tstKey1); err != errKeyAbsent {
 		t.Errorf("Did not get expected err: %v", errKeyAbsent.Error())
 	}
 }
 
 func TestDelete(t *testing.T) {
 	meta := NewMeta()
-	meta.Add("loc", "a,b,c")
-	if err := meta.Delete("loc"); err != nil {
+	meta.Add(tstKey1, tstData1)
+	if err := meta.Delete(tstKey1); err != nil {
 		t.Errorf("Did not expect error: %v", err.Error())
 	}
-	if _, err := meta.Get("loc"); err != errKeyAbsent {
+	if _, err := meta.Get(tstKey1); err != errKeyAbsent {
 		t.Errorf("Did not get expected err: %v", errKeyAbsent.Error())
 	}
 }
 
 func TestDeleteAbsentKey(t *testing.T) {
 	meta := NewMeta()
-	if err := meta.Delete("loc"); err != errKeyAbsent {
+	if err := meta.Delete(tstKey1); err != errKeyAbsent {
 		t.Errorf("Did not get expected err: %v", errKeyAbsent.Error())
 	}
 }
-
-// TODO: test delete when key exists
-// TODO: test delete when key doesn't exist

From 601351b021bf128e7fe85ad1b9b1f0bbcb8d4cd5 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Sun, 27 Jan 2019 18:33:15 +1030
Subject: [PATCH 17/83] stream/mts/meta_test.go: adding comments to
 meta_test.go testing functions

---
 stream/mts/meta_test.go | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go
index fa8e545c..14adf278 100644
--- a/stream/mts/meta_test.go
+++ b/stream/mts/meta_test.go
@@ -41,6 +41,7 @@ const (
 	tstData3 = "d,e,f"
 )
 
+// TestAddAndGet ensures that we can add metadata and then successfully get it.
 func TestAddAndGet(t *testing.T) {
 	meta := NewMeta()
 	meta.Add(tstKey1, tstData1)
@@ -61,6 +62,8 @@ func TestAddAndGet(t *testing.T) {
 	}
 }
 
+// TestUpdate checks that we can use Meta.Add to actually update metadata
+// if it already exists in the Meta map.
 func TestUpdate(t *testing.T) {
 	meta := NewMeta()
 	meta.Add(tstKey1, tstData1)
@@ -74,6 +77,7 @@ func TestUpdate(t *testing.T) {
 	}
 }
 
+// TestAll ensures we can get a correct map using Meta.All() after adding some data
 func TestAll(t *testing.T) {
 	meta := NewMeta()
 	tstMap := map[string]string{
@@ -90,6 +94,8 @@ func TestAll(t *testing.T) {
 	}
 }
 
+// TestGetAbsentKey ensures that we get the expected error when we try to get with
+// key that does not yet exist in the Meta map.
 func TestGetAbsentKey(t *testing.T) {
 	meta := NewMeta()
 
@@ -98,6 +104,7 @@ func TestGetAbsentKey(t *testing.T) {
 	}
 }
 
+// TestDelete ensures we can remove a data entry in the Meta map.
 func TestDelete(t *testing.T) {
 	meta := NewMeta()
 	meta.Add(tstKey1, tstData1)
@@ -109,6 +116,8 @@ func TestDelete(t *testing.T) {
 	}
 }
 
+// TestDeleteAbsentKey checks that we get an expected error when we try to delete
+// an entry in the Meta map that doesn't exist.
 func TestDeleteAbsentKey(t *testing.T) {
 	meta := NewMeta()
 	if err := meta.Delete(tstKey1); err != errKeyAbsent {

From 00816ecf84544c97178525ff9f72b1ae49e585b4 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Sun, 27 Jan 2019 18:34:35 +1030
Subject: [PATCH 18/83] stream/mts/meta_test.go: added test to meta_test.go to
 remind me to add a test function once the Meta.Encode() function is complete

---
 stream/mts/meta_test.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go
index 14adf278..8b871dcb 100644
--- a/stream/mts/meta_test.go
+++ b/stream/mts/meta_test.go
@@ -124,3 +124,5 @@ func TestDeleteAbsentKey(t *testing.T) {
 		t.Errorf("Did not get expected err: %v", errKeyAbsent.Error())
 	}
 }
+
+// TODO: add test function for Encoding the Meta map data

From 7d34fa1969007b4a531dd6fdeb8129eba082ba30 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Mon, 28 Jan 2019 17:29:15 +1030
Subject: [PATCH 19/83] stream/mts/meta.go: completed Meta.Encode() function

---
 stream/mts/meta.go | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/stream/mts/meta.go b/stream/mts/meta.go
index 1f482790..c25d3e85 100644
--- a/stream/mts/meta.go
+++ b/stream/mts/meta.go
@@ -32,6 +32,13 @@ import (
 	"sync"
 )
 
+const headSize = 4
+
+const (
+	majVer = 1
+	minVer = 0
+)
+
 var (
 	errKeyAbsent = errors.New("Key does not exist in map")
 )
@@ -39,10 +46,19 @@ var (
 type Meta struct {
 	mu   sync.RWMutex
 	data map[string]string
+	enc  []byte
 }
 
 func NewMeta() *Meta {
-	return &Meta{data: make(map[string]string)}
+	return &Meta{
+		data: make(map[string]string),
+		enc: []byte{
+			0x00,                   // Reserved byte
+			(majVer << 4) | minVer, // MS and LS versions
+			0x00,                   // Data len byte1
+			0x00,                   // Data len byte2
+		},
+	}
 }
 
 // Add adds metadata with key and val, if already exists return error
@@ -87,6 +103,12 @@ func (m *Meta) Delete(key string) error {
 }
 
 func (m *Meta) Encode() []byte {
-	// TODO: complete this
-	return []byte("someData")
+	m.enc = m.enc[:headSize]
+	for k, v := range m.data {
+		entry := k + "=" + v + "\t"
+		m.enc = append(m.enc, []byte(entry)...)
+	}
+	// Remove final tab
+	m.enc = m.enc[:len(m.enc)-1]
+	return m.enc
 }

From 960c110acb561ef8ff496885a63e9e6f39b063d6 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Mon, 28 Jan 2019 17:45:38 +1030
Subject: [PATCH 20/83] stream/mts/meta.go: fixed Meta.Encode() func so that it
 calculates data length correctly

---
 stream/mts/meta.go      | 25 +++++++++++++++++++++----
 stream/mts/meta_test.go |  4 +++-
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/stream/mts/meta.go b/stream/mts/meta.go
index c25d3e85..c12998bd 100644
--- a/stream/mts/meta.go
+++ b/stream/mts/meta.go
@@ -39,6 +39,11 @@ const (
 	minVer = 0
 )
 
+const (
+	dataLenIdx1 = 2
+	dataLenIdx2 = 3
+)
+
 var (
 	errKeyAbsent = errors.New("Key does not exist in map")
 )
@@ -104,11 +109,23 @@ func (m *Meta) Delete(key string) error {
 
 func (m *Meta) Encode() []byte {
 	m.enc = m.enc[:headSize]
+
+	// Iterate over map and append entries, only adding tab if we're not on the last entry
+	var i int
+	var entry string
 	for k, v := range m.data {
-		entry := k + "=" + v + "\t"
-		m.enc = append(m.enc, []byte(entry)...)
+		i++
+		entry += k + "=" + v
+		if i < len(m.data) {
+			entry += "\t"
+		}
 	}
-	// Remove final tab
-	m.enc = m.enc[:len(m.enc)-1]
+	m.enc = append(m.enc, []byte(entry)...)
+
+	// Calculate and set data length in encoded meta header.
+	dataLen := len(m.enc[headSize:])
+	m.enc[dataLenIdx1] = byte(dataLen >> 8)
+	m.enc[dataLenIdx2] = byte(dataLen)
+
 	return m.enc
 }
diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go
index 8b871dcb..07f0e5e2 100644
--- a/stream/mts/meta_test.go
+++ b/stream/mts/meta_test.go
@@ -125,4 +125,6 @@ func TestDeleteAbsentKey(t *testing.T) {
 	}
 }
 
-// TODO: add test function for Encoding the Meta map data
+func TestEncode(t *testing.T) {
+	meta := NewMeta()
+}

From 3aa0efc16a128fcd4fe8b7c12eb257f89f02b7f7 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Mon, 28 Jan 2019 18:07:34 +1030
Subject: [PATCH 21/83] stream/mts/meta_test.go: added TestEncode to test
 Meta.Encoding function

---
 stream/mts/meta_test.go | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go
index 07f0e5e2..010c9b82 100644
--- a/stream/mts/meta_test.go
+++ b/stream/mts/meta_test.go
@@ -28,6 +28,7 @@ LICENSE
 package mts
 
 import (
+	"bytes"
 	"errors"
 	"reflect"
 	"testing"
@@ -127,4 +128,23 @@ func TestDeleteAbsentKey(t *testing.T) {
 
 func TestEncode(t *testing.T) {
 	meta := NewMeta()
+	meta.Add(tstKey1, tstData1)
+	meta.Add(tstKey2, tstData2)
+
+	dataLen := len(tstKey1+tstData1+tstKey2+tstData2) + 3
+	expectedOut := []byte{
+		0x00,
+		0x10,
+		byte(dataLen >> 8),
+		byte(dataLen),
+	}
+	expectedOut = append(expectedOut, []byte(
+		tstKey1+"="+tstData1+"\t"+
+			tstKey2+"="+tstData2)...)
+
+	got := meta.Encode()
+
+	if !bytes.Equal(expectedOut, got) {
+		t.Errorf("Did not get expected out. \nGot : %v \nwant: %v", got, expectedOut)
+	}
 }

From 5aff27ac4dbebfc809d7b0b30cb3ba1433bc9e2b Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 29 Jan 2019 10:40:22 +1030
Subject: [PATCH 22/83] stream/mts/psi/descriptor_test.go: wrote test to check
 ProgramInfoLen func and descriptors func

---
 stream/mts/meta.go      | 2 ++
 stream/mts/meta_test.go | 1 +
 2 files changed, 3 insertions(+)

diff --git a/stream/mts/meta.go b/stream/mts/meta.go
index c12998bd..465a537e 100644
--- a/stream/mts/meta.go
+++ b/stream/mts/meta.go
@@ -107,6 +107,8 @@ func (m *Meta) Delete(key string) error {
 	return errKeyAbsent
 }
 
+// Encode takes the meta data map and encods into a byte slice with header
+// describing the version, length of data and data in TSV format.
 func (m *Meta) Encode() []byte {
 	m.enc = m.enc[:headSize]
 
diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go
index 010c9b82..daff9038 100644
--- a/stream/mts/meta_test.go
+++ b/stream/mts/meta_test.go
@@ -126,6 +126,7 @@ func TestDeleteAbsentKey(t *testing.T) {
 	}
 }
 
+// TestEncode checks that we're getting the correct byte slice from Meta.Encode().
 func TestEncode(t *testing.T) {
 	meta := NewMeta()
 	meta.Add(tstKey1, tstData1)

From dce113d1e458556ad222f00e312cf74a0c366461 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 29 Jan 2019 11:28:40 +1030
Subject: [PATCH 23/83] stream/mts/psi/descriptor_test.go: wrote HasDescriptor
 test and fixed HasDescriptor function after bug was found

---
 stream/mts/psi/psi.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index 441a7221..2f4ab5ad 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -264,9 +264,9 @@ func (p *PSIBytes) HasDescriptor(tag int) Descriptor {
 	for i := 0; i < len(descs); {
 		t := int(descs[i])
 		if t == tag {
-			return descs[i : i+int(descs[i+1])]
+			return descs[i : i+2+int(descs[i+1])]
 		}
-		i += 1 + int(descs[i+1])
+		i += 2 + int(descs[i+1])
 	}
 	return nil
 }

From 3a14b644802b5537edf14662e98306f7f5945031 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 29 Jan 2019 11:30:37 +1030
Subject: [PATCH 24/83] stream/mts/psi/psi.go: simplified HasDescriptor func

---
 stream/mts/psi/psi.go | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index 2f4ab5ad..c2e30f9e 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -261,12 +261,10 @@ func (p *PSIBytes) HasDescriptor(tag int) Descriptor {
 	if descs == nil {
 		return nil
 	}
-	for i := 0; i < len(descs); {
-		t := int(descs[i])
-		if t == tag {
+	for i := 0; i < len(descs); i += 2 + int(descs[i+1]) {
+		if int(descs[i]) == tag {
 			return descs[i : i+2+int(descs[i+1])]
 		}
-		i += 2 + int(descs[i+1])
 	}
 	return nil
 }

From af239838d0038d556985453a42e27d5171731529 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 29 Jan 2019 14:16:33 +1030
Subject: [PATCH 25/83] stream/mts/psi: modified how update crc works so that
 we're only giving it data that's included in crc calc as well as the crc
 field itself. Added some consts to allow working with SyntaxSectionLen. Added
 trimPadding func so that we can easily get rid of any padding. Fixed
 createDescriptor func which was found to be broken from TestCreateDescriptor

---
 stream/mts/psi/crc.go     |  4 ++--
 stream/mts/psi/helpers.go |  9 ++++-----
 stream/mts/psi/psi.go     | 32 +++++++++++++++++++++++++-------
 3 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/stream/mts/psi/crc.go b/stream/mts/psi/crc.go
index a361307d..cd9a1199 100644
--- a/stream/mts/psi/crc.go
+++ b/stream/mts/psi/crc.go
@@ -37,13 +37,13 @@ import (
 func addCrc(out []byte) []byte {
 	t := make([]byte, len(out)+4)
 	copy(t, out)
-	updateCrc(t)
+	updateCrc(t[1:])
 	return t
 }
 
 // updateCrc updates the crc of bytes slice, writing the checksum into the last four bytes.
 func updateCrc(b []byte) {
-	crc32 := crc32_Update(0xffffffff, crc32_MakeTable(bits.Reverse32(crc32.IEEE)), b[1:len(b)-4])
+	crc32 := crc32_Update(0xffffffff, crc32_MakeTable(bits.Reverse32(crc32.IEEE)), b[:len(b)-4])
 	binary.BigEndian.PutUint32(b[len(b)-4:], crc32)
 }
 
diff --git a/stream/mts/psi/helpers.go b/stream/mts/psi/helpers.go
index 82d7ce72..174b289b 100644
--- a/stream/mts/psi/helpers.go
+++ b/stream/mts/psi/helpers.go
@@ -64,15 +64,14 @@ func UpdateTime(dst []byte, t uint64) error {
 	for i := range dst[TimeDataIndx : TimeDataIndx+TimeDataSize] {
 		dst[i+TimeDataIndx] = ts[i]
 	}
-	updateCrc(dst)
+	updateCrc(dst[1:])
 	return nil
 }
 
 // SyntaxSecLenFrom takes a byte slice representation of a psi and extracts
 // it's syntax section length
-func SyntaxSecLenFrom(p []byte) (l uint8) {
-	l = uint8(p[syntaxSecLenIndx]) - crcSize
-	return
+func SyntaxSecLenFrom(p []byte) int {
+	return int(((p[SyntaxSecLenIdx1] & SyntaxSecLenMask1) << 8) | p[SyntaxSecLenIdx2])
 }
 
 // TimeFrom takes a byte slice representation of a psi-pmt and extracts it's
@@ -112,7 +111,7 @@ func UpdateLocation(d []byte, s string) error {
 	for i := range loc {
 		loc[i] = 0
 	}
-	updateCrc(d)
+	updateCrc(d[1:])
 	return nil
 }
 
diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index c2e30f9e..6a4a9103 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -70,8 +70,10 @@ const (
 
 // Other misc consts
 const (
-	syntaxSecLenIndx = 3
-	crcSize          = 4
+	SyntaxSecLenIdx1  = 2
+	SyntaxSecLenIdx2  = 3
+	SyntaxSecLenMask1 = 0x03
+	crcSize           = 4
 )
 
 const (
@@ -275,24 +277,40 @@ func (p *PSIBytes) descriptors() []byte {
 
 func (p *PSIBytes) createDescriptor(tag int, data []byte) {
 	curProgLen := p.ProgramInfoLen()
+	oldSyntaxSectionLen := SyntaxSecLenFrom(*p)
 	dataLen := len(data)
 
 	// Calculate the new descriptors index and length.
 	newDescIdx := DescriptorsIdx + curProgLen
 	newDescLen := dataLen + 2
-
 	// Copy data down from newDescIdx to create room for the new descriptor.
-	copy((*p)[newDescIdx+newDescLen:], (*p)[newDescIdx:dataLen-newDescLen])
-
+	copy((*p)[newDescIdx+newDescLen:], (*p)[newDescIdx:newDescIdx+newDescLen])
 	// Set the tag, data len and data of the new desriptor.
 	(*p)[newDescIdx] = byte(tag)
 	(*p)[newDescIdx+1] = byte(dataLen)
 	copy((*p)[newDescIdx+2:newDescIdx+2+dataLen], data)
 
 	// Update the program info length to account for the new descriptor.
-	newProgInfoLen := curProgLen + dataLen
-	(*p)[ProgramInfoLenIdx1] = byte(newProgInfoLen >> 8)
+	// TODO: put this in function set program info length
+	addedLen := dataLen + 2
+	newProgInfoLen := curProgLen + addedLen
+	(*p)[ProgramInfoLenIdx1] &= 0xff ^ ProgramInfoLenMask1
+	(*p)[ProgramInfoLenIdx1] |= byte(newProgInfoLen>>8) & ProgramInfoLenMask1
 	(*p)[ProgramInfoLenIdx2] = byte(newProgInfoLen)
+
+	// set section length
+	// TODO: put this in func set program info length
+	newSyntaxSectionLen := int(oldSyntaxSectionLen) + addedLen
+	(*p)[SyntaxSecLenIdx1] &= 0xff ^ SyntaxSecLenMask1
+	(*p)[SyntaxSecLenIdx1] |= byte(newSyntaxSectionLen>>8) & SyntaxSecLenMask1
+	(*p)[SyntaxSecLenIdx2] = byte(newSyntaxSectionLen)
+}
+
+func (p *PSIBytes) trimPadding() []byte {
+	sectionLength := SyntaxSecLenFrom(*p)
+	paddingIdx := (4 + sectionLength)
+	o := (*p)[:paddingIdx]
+	return o
 }
 
 func (d *Descriptor) update(data []byte) {

From a61bdac2de12e51858264303b7151a4f01c22625 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 29 Jan 2019 14:21:25 +1030
Subject: [PATCH 26/83] stream/mts/psi: added testing file with tests

---
 stream/mts/psi/descriptor_test.go | 224 ++++++++++++++++++++++++++++++
 1 file changed, 224 insertions(+)
 create mode 100644 stream/mts/psi/descriptor_test.go

diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go
new file mode 100644
index 00000000..cb25761c
--- /dev/null
+++ b/stream/mts/psi/descriptor_test.go
@@ -0,0 +1,224 @@
+/*
+NAME
+  descriptor_test.go
+
+DESCRIPTION
+  See Readme.md
+
+AUTHOR
+  Saxon Nelson-Milton <saxon@ausocean.org>
+
+LICENSE
+  descriptor_test.go is Copyright (C) 2017-2019 the Australian Ocean Lab (AusOcean)
+
+  It is free software: you can redistribute it and/or modify them
+  under the terms of the GNU General Public License as published by the
+  Free Software Foundation, either version 3 of the License, or (at your
+  option) any later version.
+
+  It is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with revid in gpl.txt.  If not, see http://www.gnu.org/licenses.
+*/
+
+package psi
+
+import (
+	"bytes"
+	"testing"
+)
+
+const (
+	errNotExpectedOut = "Did not get expected output: \ngot : %v, \nwant: %v"
+)
+
+var tstPsi1 = PSI{
+	Pf:  0x00,
+	Tid: 0x02,
+	Ssi: true,
+	Sl:  0x1c,
+	Tss: &TSS{
+		Tide: 0x01,
+		V:    0,
+		Cni:  true,
+		Sn:   0,
+		Lsn:  0,
+		Sd: &PMT{
+			Pcrpid: 0x0100, // wrong
+			Pil:    10,
+			Pd: []Desc{
+				{
+					Dt: TimeDescTag,
+					Dl: TimeDataSize,
+					Dd: make([]byte, TimeDataSize),
+				},
+			},
+			Essd: &ESSD{
+				St:   0x1b,
+				Epid: 0x0100,
+				Esil: 0x00,
+			},
+		},
+	},
+}
+
+var tstPsi2 = PSI{
+	Pf:  0x00,
+	Tid: 0x02,
+	Ssi: true,
+	Sl:  0x12,
+	Tss: &TSS{
+		Tide: 0x01,
+		V:    0,
+		Cni:  true,
+		Sn:   0,
+		Lsn:  0,
+		Sd: &PMT{
+			Pcrpid: 0x0100,
+			Pil:    0,
+			Essd: &ESSD{
+				St:   0x1b,
+				Epid: 0x0100,
+				Esil: 0x00,
+			},
+		},
+	},
+}
+
+var tstPsi3 = PSI{
+	Pf:  0x00,
+	Tid: 0x02,
+	Ssi: true,
+	Sl:  0x3e,
+	Tss: &TSS{
+		Tide: 0x01,
+		V:    0,
+		Cni:  true,
+		Sn:   0,
+		Lsn:  0,
+		Sd: &PMT{
+			Pcrpid: 0x0100,
+			Pil:    PmtTimeLocationPil,
+			Pd: []Desc{
+				{
+					Dt: TimeDescTag,
+					Dl: TimeDataSize,
+					Dd: make([]byte, TimeDataSize),
+				},
+				{
+					Dt: LocationDescTag,
+					Dl: LocationDataSize,
+					Dd: make([]byte, LocationDataSize),
+				},
+			},
+			Essd: &ESSD{
+				St:   0x1b,
+				Epid: 0x0100,
+				Esil: 0x00,
+			},
+		},
+	},
+}
+
+func TestHasDescriptor(t *testing.T) {
+	p := PSIBytes(tstPsi3.Bytes())
+
+	// Try getting descriptor that exists
+	got := p.HasDescriptor(LocationDescTag)
+	want := []byte{
+		LocationDescTag,
+		LocationDataSize,
+	}
+	want = append(want, make([]byte, LocationDataSize)...)
+
+	if !bytes.Equal(got, want) {
+		t.Errorf(errNotExpectedOut, got, want)
+	}
+
+	// Try getting descriptor that doesnt exist
+	const fakeTag = 236
+	got = p.HasDescriptor(fakeTag)
+	want = nil
+
+	if !bytes.Equal(got, want) {
+		t.Errorf(errNotExpectedOut, got, want)
+	}
+
+	// Try getting descriptor from psi that has no descriptors
+	p = PSIBytes(tstPsi2.Bytes())
+
+	got = p.HasDescriptor(LocationDescTag)
+	want = nil
+
+	if !bytes.Equal(got, want) {
+		t.Errorf(errNotExpectedOut, got, want)
+	}
+}
+
+func TestProgramInfoLen(t *testing.T) {
+	p := PSIBytes(tstPsi1.Bytes())
+	got := p.ProgramInfoLen()
+	want := 10
+	if got != want {
+		t.Errorf(errNotExpectedOut, got, want)
+	}
+}
+
+func TestDescriptors(t *testing.T) {
+	// Try psi with descriptors
+	p := PSIBytes(tstPsi1.Bytes())
+	got := p.descriptors()
+	want := []byte{
+		TimeDescTag,
+		TimeDataSize,
+	}
+	want = append(want, make([]byte, TimeDataSize)...)
+
+	if !bytes.Equal(got, want) {
+		t.Errorf(errNotExpectedOut, got, want)
+	}
+
+	// Now try psi with empty descriptors
+	p = PSIBytes(tstPsi2.Bytes())
+	got = p.descriptors()
+	want = nil
+	if !bytes.Equal(got, want) {
+		t.Errorf(errNotExpectedOut, got, want)
+	}
+}
+
+func TestCreateDescriptor(t *testing.T) {
+	// Test with PSI containing no descriptors
+	p := PSIBytes(tstPsi2.Bytes())
+	p.createDescriptor(TimeDescTag, make([]byte, TimeDataSize))
+	// update crc
+	noPadding := p.trimPadding()
+	updateCrc(noPadding[1:])
+	want := PSIBytes(tstPsi1.Bytes())
+	got := p
+
+	if !bytes.Equal(want, got) {
+		t.Errorf(errNotExpectedOut, got, want)
+	}
+
+	// Test with psi containing descriptor already
+	p = PSIBytes(tstPsi1.Bytes())
+	p.createDescriptor(LocationDescTag, make([]byte, LocationDataSize))
+	// update crc
+	noPadding = p.trimPadding()
+	updateCrc(noPadding[1:])
+	want = PSIBytes(tstPsi3.Bytes())
+	got = p
+
+	if !bytes.Equal(want, got) {
+		t.Errorf(errNotExpectedOut, got, want)
+	}
+}
+
+func TestUpdateDescriptor(t *testing.T) {
+
+}

From 4a3464252b78ecf3ee062ff03574ecda669ec746 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 29 Jan 2019 14:38:23 +1030
Subject: [PATCH 27/83] stream/mts/psi: changed HasDescriptor to also return
 index of descriptor in psi. Wrote func called SetProgInfoLen to set the
 program info length in a pmt. Started writing deleteDescriptor func to get
 rid of a descriptor.

---
 stream/mts/psi/descriptor_test.go | 10 +++++++---
 stream/mts/psi/psi.go             | 33 +++++++++++++++++++++++--------
 2 files changed, 32 insertions(+), 11 deletions(-)

diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go
index cb25761c..919653fa 100644
--- a/stream/mts/psi/descriptor_test.go
+++ b/stream/mts/psi/descriptor_test.go
@@ -128,7 +128,7 @@ func TestHasDescriptor(t *testing.T) {
 	p := PSIBytes(tstPsi3.Bytes())
 
 	// Try getting descriptor that exists
-	got := p.HasDescriptor(LocationDescTag)
+	_, got := p.HasDescriptor(LocationDescTag)
 	want := []byte{
 		LocationDescTag,
 		LocationDataSize,
@@ -141,7 +141,7 @@ func TestHasDescriptor(t *testing.T) {
 
 	// Try getting descriptor that doesnt exist
 	const fakeTag = 236
-	got = p.HasDescriptor(fakeTag)
+	_, got = p.HasDescriptor(fakeTag)
 	want = nil
 
 	if !bytes.Equal(got, want) {
@@ -151,7 +151,7 @@ func TestHasDescriptor(t *testing.T) {
 	// Try getting descriptor from psi that has no descriptors
 	p = PSIBytes(tstPsi2.Bytes())
 
-	got = p.HasDescriptor(LocationDescTag)
+	_, got = p.HasDescriptor(LocationDescTag)
 	want = nil
 
 	if !bytes.Equal(got, want) {
@@ -219,6 +219,10 @@ func TestCreateDescriptor(t *testing.T) {
 	}
 }
 
+func TestDeleteDescriptor(t *testing.T) {
+
+}
+
 func TestUpdateDescriptor(t *testing.T) {
 
 }
diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index 6a4a9103..8832dfb4 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -245,7 +245,7 @@ func (p *PSIBytes) AddDescriptor(tag int, data []byte) error {
 		return errors.New("trying to add descriptor, but not pmt")
 	}
 
-	desc := p.HasDescriptor(tag)
+	_, desc := p.HasDescriptor(tag)
 	if desc == nil {
 		p.createDescriptor(tag, data)
 		return nil
@@ -258,17 +258,17 @@ func (p *PSIBytes) ProgramInfoLen() int {
 	return int((((*p)[ProgramInfoLenIdx1] & ProgramInfoLenMask1) << 8) | (*p)[ProgramInfoLenIdx2])
 }
 
-func (p *PSIBytes) HasDescriptor(tag int) Descriptor {
+func (p *PSIBytes) HasDescriptor(tag int) (int, Descriptor) {
 	descs := p.descriptors()
 	if descs == nil {
-		return nil
+		return -1, nil
 	}
 	for i := 0; i < len(descs); i += 2 + int(descs[i+1]) {
 		if int(descs[i]) == tag {
-			return descs[i : i+2+int(descs[i+1])]
+			return i, descs[i : i+2+int(descs[i+1])]
 		}
 	}
-	return nil
+	return -1, nil
 }
 
 func (p *PSIBytes) descriptors() []byte {
@@ -283,8 +283,10 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) {
 	// Calculate the new descriptors index and length.
 	newDescIdx := DescriptorsIdx + curProgLen
 	newDescLen := dataLen + 2
+
 	// Copy data down from newDescIdx to create room for the new descriptor.
 	copy((*p)[newDescIdx+newDescLen:], (*p)[newDescIdx:newDescIdx+newDescLen])
+
 	// Set the tag, data len and data of the new desriptor.
 	(*p)[newDescIdx] = byte(tag)
 	(*p)[newDescIdx+1] = byte(dataLen)
@@ -294,9 +296,7 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) {
 	// TODO: put this in function set program info length
 	addedLen := dataLen + 2
 	newProgInfoLen := curProgLen + addedLen
-	(*p)[ProgramInfoLenIdx1] &= 0xff ^ ProgramInfoLenMask1
-	(*p)[ProgramInfoLenIdx1] |= byte(newProgInfoLen>>8) & ProgramInfoLenMask1
-	(*p)[ProgramInfoLenIdx2] = byte(newProgInfoLen)
+	p.SetProgInfoLen(newProgInfoLen)
 
 	// set section length
 	// TODO: put this in func set program info length
@@ -306,6 +306,7 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) {
 	(*p)[SyntaxSecLenIdx2] = byte(newSyntaxSectionLen)
 }
 
+// TODO: make this safer. If no padding, does this index out of range ?
 func (p *PSIBytes) trimPadding() []byte {
 	sectionLength := SyntaxSecLenFrom(*p)
 	paddingIdx := (4 + sectionLength)
@@ -313,6 +314,22 @@ func (p *PSIBytes) trimPadding() []byte {
 	return o
 }
 
+func (p *PSIBytes) SetProgInfoLen(l int) {
+	// TODO: check if pmt first
+	(*p)[ProgramInfoLenIdx1] &= 0xff ^ ProgramInfoLenMask1
+	(*p)[ProgramInfoLenIdx1] |= byte(l>>8) & ProgramInfoLenMask1
+	(*p)[ProgramInfoLenIdx2] = byte(l)
+}
+
+func (p *PSIBytes) deleteDescriptor(tag int) error {
+	if i, desc := p.HasDescriptor(tag); desc != nil {
+		descLen := len(desc)
+		newProgInfoLen := p.ProgramInfoLen() - descLen
+		p.SetProgInfoLen(newProgInfoLen)
+	}
+	return errors.New("Descriptor doesn't exist")
+}
+
 func (d *Descriptor) update(data []byte) {
 	if len(data) > int((*d)[1]) {
 		// TODO: implement resizing of descriptor

From 287238ddd170471cf6d476bc0c1c32642833d29e Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 29 Jan 2019 14:39:01 +1030
Subject: [PATCH 28/83] stream/mts/psi: removed deleteDescriptor func as we
 don't need it yet

---
 stream/mts/psi/psi.go | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index 8832dfb4..9d0c1906 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -321,15 +321,6 @@ func (p *PSIBytes) SetProgInfoLen(l int) {
 	(*p)[ProgramInfoLenIdx2] = byte(l)
 }
 
-func (p *PSIBytes) deleteDescriptor(tag int) error {
-	if i, desc := p.HasDescriptor(tag); desc != nil {
-		descLen := len(desc)
-		newProgInfoLen := p.ProgramInfoLen() - descLen
-		p.SetProgInfoLen(newProgInfoLen)
-	}
-	return errors.New("Descriptor doesn't exist")
-}
-
 func (d *Descriptor) update(data []byte) {
 	if len(data) > int((*d)[1]) {
 		// TODO: implement resizing of descriptor

From a5f7c5ad8750be4008cccc762556638a8e19aeac Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 29 Jan 2019 14:39:39 +1030
Subject: [PATCH 29/83] stream/mts/psi: removed TestDeleteDescriptor test as we
 don't need anymore

---
 stream/mts/psi/descriptor_test.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go
index 919653fa..4381f896 100644
--- a/stream/mts/psi/descriptor_test.go
+++ b/stream/mts/psi/descriptor_test.go
@@ -219,10 +219,6 @@ func TestCreateDescriptor(t *testing.T) {
 	}
 }
 
-func TestDeleteDescriptor(t *testing.T) {
-
-}
-
 func TestUpdateDescriptor(t *testing.T) {
 
 }

From 2145db71d45f8f31285fed1e04c3cc6bae45d57f Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 29 Jan 2019 15:16:08 +1030
Subject: [PATCH 30/83] stream/mts/psi: wrote SetSectionLen func

---
 stream/mts/psi/descriptor_test.go |  2 +-
 stream/mts/psi/psi.go             | 35 +++++++++++++++++++++----------
 2 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go
index 4381f896..d11053d1 100644
--- a/stream/mts/psi/descriptor_test.go
+++ b/stream/mts/psi/descriptor_test.go
@@ -219,6 +219,6 @@ func TestCreateDescriptor(t *testing.T) {
 	}
 }
 
-func TestUpdateDescriptor(t *testing.T) {
+func TestAddDescriptor(t *testing.T) {
 
 }
diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index 9d0c1906..7c7e150b 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -245,15 +245,32 @@ func (p *PSIBytes) AddDescriptor(tag int, data []byte) error {
 		return errors.New("trying to add descriptor, but not pmt")
 	}
 
-	_, desc := p.HasDescriptor(tag)
+	i, desc := p.HasDescriptor(tag)
 	if desc == nil {
 		p.createDescriptor(tag, data)
 		return nil
 	}
-	desc.update(data)
+	oldDescLen := desc.len()
+	// update the descriptor
+	oldDataLen := int(desc[1])
+	newDataLen := len(data)
+	newDescLen := 2 + newDataLen
+
+	// Shift data
+	copy(p[i+newDescLen:], data)
+
+	deltaLen := desc.len() - oldDescLen
+	newProgInfoLen := p.ProgramInfoLen() + deltaLen
+	p.SetProgInfoLen(newProgInfoLen)
+	newSectionLen := int(psi.SectionLength(*p)) + deltaLen
+	p.SetSectionLen(newSectionLen)
 	return nil
 }
 
+func (d *Descriptor) len() int {
+	return int(2 + (*d)[1])
+}
+
 func (p *PSIBytes) ProgramInfoLen() int {
 	return int((((*p)[ProgramInfoLenIdx1] & ProgramInfoLenMask1) << 8) | (*p)[ProgramInfoLenIdx2])
 }
@@ -301,9 +318,7 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) {
 	// set section length
 	// TODO: put this in func set program info length
 	newSyntaxSectionLen := int(oldSyntaxSectionLen) + addedLen
-	(*p)[SyntaxSecLenIdx1] &= 0xff ^ SyntaxSecLenMask1
-	(*p)[SyntaxSecLenIdx1] |= byte(newSyntaxSectionLen>>8) & SyntaxSecLenMask1
-	(*p)[SyntaxSecLenIdx2] = byte(newSyntaxSectionLen)
+	p.SetSectionLen(newSyntaxSectionLen)
 }
 
 // TODO: make this safer. If no padding, does this index out of range ?
@@ -321,10 +336,8 @@ func (p *PSIBytes) SetProgInfoLen(l int) {
 	(*p)[ProgramInfoLenIdx2] = byte(l)
 }
 
-func (d *Descriptor) update(data []byte) {
-	if len(data) > int((*d)[1]) {
-		// TODO: implement resizing of descriptor
-		panic("Can't resize descriptor data")
-	}
-
+func (p *PSIBytes) SetSectionLen(l int) {
+	(*p)[SyntaxSecLenIdx1] &= 0xff ^ SyntaxSecLenMask1
+	(*p)[SyntaxSecLenIdx1] |= byte(l>>8) & SyntaxSecLenMask1
+	(*p)[SyntaxSecLenIdx2] = byte(l)
 }

From 252e6680ed16e1885a22fdc3b797f34b844d78fa Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 29 Jan 2019 15:42:51 +1030
Subject: [PATCH 31/83] stream/mts/psi: modified way in which we add padding to
 psi - now we leave it up to the mts package to do this on creation of an ts
 packet. Also in the middle of writing AddDescriptor func, and finding issues,
 hence the mentioned change.

---
 stream/mts/psi/descriptor_test.go |  4 +++
 stream/mts/psi/psi.go             | 52 +++++++++++++++----------------
 2 files changed, 29 insertions(+), 27 deletions(-)

diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go
index d11053d1..b26fd007 100644
--- a/stream/mts/psi/descriptor_test.go
+++ b/stream/mts/psi/descriptor_test.go
@@ -168,6 +168,10 @@ func TestProgramInfoLen(t *testing.T) {
 	}
 }
 
+func TestSectionLen(t *testing.T) {
+
+}
+
 func TestDescriptors(t *testing.T) {
 	// Try psi with descriptors
 	p := PSIBytes(tstPsi1.Bytes())
diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index 7c7e150b..c41e1d98 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -26,12 +26,6 @@ LICENSE
 
 package psi
 
-import (
-	"errors"
-
-	"github.com/Comcast/gots/psi"
-)
-
 const (
 	PacketSize = 184 // packet size of a psi.
 )
@@ -170,7 +164,6 @@ func (p *PSI) Bytes() []byte {
 	out[3] = byte(p.Sl)
 	out = append(out, p.Tss.Bytes()...)
 	out = addCrc(out)
-	out = addPadding(out)
 	return out
 }
 
@@ -241,29 +234,31 @@ func asByte(b bool) byte {
 }
 
 func (p *PSIBytes) AddDescriptor(tag int, data []byte) error {
-	if psi.TableID(*p) != pmtID {
-		return errors.New("trying to add descriptor, but not pmt")
-	}
+	/*
+		if psi.TableID(*p) != pmtID {
+			return errors.New("trying to add descriptor, but not pmt")
+		}
 
-	i, desc := p.HasDescriptor(tag)
-	if desc == nil {
-		p.createDescriptor(tag, data)
-		return nil
-	}
-	oldDescLen := desc.len()
-	// update the descriptor
-	oldDataLen := int(desc[1])
-	newDataLen := len(data)
-	newDescLen := 2 + newDataLen
+		i, desc := p.HasDescriptor(tag)
+		if desc == nil {
+			p.createDescriptor(tag, data)
+			return nil
+		}
+		oldDescLen := desc.len()
+		// update the descriptor
+		oldDataLen := int(desc[1])
+		newDataLen := len(data)
+		newDescLen := 2 + newDataLen
 
-	// Shift data
-	copy(p[i+newDescLen:], data)
+		// Shift data
+		//	copy(p[i+newDescLen:], data)
 
-	deltaLen := desc.len() - oldDescLen
-	newProgInfoLen := p.ProgramInfoLen() + deltaLen
-	p.SetProgInfoLen(newProgInfoLen)
-	newSectionLen := int(psi.SectionLength(*p)) + deltaLen
-	p.SetSectionLen(newSectionLen)
+		deltaLen := desc.len() - oldDescLen
+		newProgInfoLen := p.ProgramInfoLen() + deltaLen
+		p.SetProgInfoLen(newProgInfoLen)
+		newSectionLen := int(psi.SectionLength(*p)) + deltaLen
+		p.SetSectionLen(newSectionLen)
+	*/
 	return nil
 }
 
@@ -302,6 +297,9 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) {
 	newDescLen := dataLen + 2
 
 	// Copy data down from newDescIdx to create room for the new descriptor.
+	tmp := make([]byte, len(*p)+newDescLen)
+	copy(tmp, *p)
+	*p = tmp
 	copy((*p)[newDescIdx+newDescLen:], (*p)[newDescIdx:newDescIdx+newDescLen])
 
 	// Set the tag, data len and data of the new desriptor.

From 1786ed26614df36521eb4e08671add6790273f9a Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 29 Jan 2019 16:14:00 +1030
Subject: [PATCH 32/83] stream/mts/psi: finished writing AddDescriptor

---
 stream/mts/encoder.go      | 20 ++++++------
 stream/mts/psi/psi.go      | 64 +++++++++++++++++++++++++-------------
 stream/mts/psi/psi_test.go |  2 +-
 3 files changed, 53 insertions(+), 33 deletions(-)

diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index 8f46a15a..33194ab8 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -196,12 +196,12 @@ func (e *Encoder) Encode(nalu []byte) error {
 func (e *Encoder) writePSI() error {
 	// Write PAT.
 	patPkt := Packet{
-		PUSI:    true,
-		PID:     PatPid,
-		CC:      e.ccFor(PatPid),
-		AFC:     HasPayload,
-		Payload: patTable,
+		PUSI: true,
+		PID:  PatPid,
+		CC:   e.ccFor(PatPid),
+		AFC:  HasPayload,
 	}
+	patPkt.FillPayload(patTable)
 	_, err := e.dst.Write(patPkt.Bytes(e.tsSpace[:PacketSize]))
 	if err != nil {
 		return err
@@ -211,12 +211,12 @@ func (e *Encoder) writePSI() error {
 
 	// Create mts packet from pmt table.
 	pmtPkt := Packet{
-		PUSI:    true,
-		PID:     PmtPid,
-		CC:      e.ccFor(PmtPid),
-		AFC:     HasPayload,
-		Payload: pmtTable,
+		PUSI: true,
+		PID:  PmtPid,
+		CC:   e.ccFor(PmtPid),
+		AFC:  HasPayload,
 	}
+	pmtPkt.FillPayload(pmtTable)
 	_, err = e.dst.Write(pmtPkt.Bytes(e.tsSpace[:PacketSize]))
 	if err != nil {
 		return err
diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index c41e1d98..9637594d 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -26,6 +26,12 @@ LICENSE
 
 package psi
 
+import (
+	"errors"
+
+	"github.com/Comcast/gots/psi"
+)
+
 const (
 	PacketSize = 184 // packet size of a psi.
 )
@@ -234,31 +240,45 @@ func asByte(b bool) byte {
 }
 
 func (p *PSIBytes) AddDescriptor(tag int, data []byte) error {
-	/*
-		if psi.TableID(*p) != pmtID {
-			return errors.New("trying to add descriptor, but not pmt")
-		}
+	if psi.TableID(*p) != pmtID {
+		return errors.New("trying to add descriptor, but not pmt")
+	}
 
-		i, desc := p.HasDescriptor(tag)
-		if desc == nil {
-			p.createDescriptor(tag, data)
-			return nil
-		}
-		oldDescLen := desc.len()
-		// update the descriptor
-		oldDataLen := int(desc[1])
-		newDataLen := len(data)
-		newDescLen := 2 + newDataLen
+	i, desc := p.HasDescriptor(tag)
+	if desc == nil {
+		p.createDescriptor(tag, data)
+		return nil
+	}
 
-		// Shift data
-		//	copy(p[i+newDescLen:], data)
+	// Old lengths
+	oldDescLen := desc.len()
+	oldDataLen := int(desc[1])
+
+	// New lengths
+	newDataLen := len(data)
+	newDescLen := 2 + newDataLen
+
+	delta := newDescLen - oldDescLen
+
+	if oldDataLen > newDataLen {
+		copy((*p)[i+newDescLen:], (*p)[i+oldDescLen:])
+		*p = (*p)[:len(*p)+delta]
+
+	} else if oldDataLen < newDataLen {
+		tmp := make([]byte, len(*p)+delta)
+		copy(tmp, *p)
+		*p = tmp
+		copy((*p)[i+newDescLen:], (*p)[i+oldDescLen:])
+
+	} else {
+		copy((*p)[i+2:], data)
+	}
+
+	newProgInfoLen := p.ProgramInfoLen() + delta
+	p.SetProgInfoLen(newProgInfoLen)
+	newSectionLen := int(psi.SectionLength(*p)) + delta
+	p.SetSectionLen(newSectionLen)
 
-		deltaLen := desc.len() - oldDescLen
-		newProgInfoLen := p.ProgramInfoLen() + deltaLen
-		p.SetProgInfoLen(newProgInfoLen)
-		newSectionLen := int(psi.SectionLength(*p)) + deltaLen
-		p.SetSectionLen(newSectionLen)
-	*/
 	return nil
 }
 
diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go
index e437066e..61a740a4 100644
--- a/stream/mts/psi/psi_test.go
+++ b/stream/mts/psi/psi_test.go
@@ -282,7 +282,7 @@ var bytesTests = []struct {
 func TestBytes(t *testing.T) {
 	for _, test := range bytesTests {
 		got := test.input.Bytes()
-		if !bytes.Equal(got, addPadding(addCrc(test.want))) {
+		if !bytes.Equal(got, addCrc(test.want)) {
 			t.Errorf("unexpected error for test %v: got:%v want:%v", test.name, got,
 				test.want)
 		}

From c7d418ce81c4f54d70ed798ab449c59c90e8091e Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 29 Jan 2019 16:15:18 +1030
Subject: [PATCH 33/83] stream/mts/psi: deleted some todos as they have been
 addressed now

---
 stream/mts/psi/psi.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index 9637594d..7d61f47b 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -328,13 +328,11 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) {
 	copy((*p)[newDescIdx+2:newDescIdx+2+dataLen], data)
 
 	// Update the program info length to account for the new descriptor.
-	// TODO: put this in function set program info length
 	addedLen := dataLen + 2
 	newProgInfoLen := curProgLen + addedLen
 	p.SetProgInfoLen(newProgInfoLen)
 
 	// set section length
-	// TODO: put this in func set program info length
 	newSyntaxSectionLen := int(oldSyntaxSectionLen) + addedLen
 	p.SetSectionLen(newSyntaxSectionLen)
 }

From 5cba86106350c4f9e7ff04232d4ba609c36f8b39 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 29 Jan 2019 16:16:37 +1030
Subject: [PATCH 34/83] stream/mts/psi: removed test that we don't need to do

---
 stream/mts/psi/descriptor_test.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go
index b26fd007..d11053d1 100644
--- a/stream/mts/psi/descriptor_test.go
+++ b/stream/mts/psi/descriptor_test.go
@@ -168,10 +168,6 @@ func TestProgramInfoLen(t *testing.T) {
 	}
 }
 
-func TestSectionLen(t *testing.T) {
-
-}
-
 func TestDescriptors(t *testing.T) {
 	// Try psi with descriptors
 	p := PSIBytes(tstPsi1.Bytes())

From a5e1763c9713999b32ffa72ba4f804f8f16e2bb1 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 29 Jan 2019 16:42:02 +1030
Subject: [PATCH 35/83] stream/mts/psi: finished writing AddDescriptor test
 -everything working fine now

---
 stream/mts/psi/descriptor_test.go | 25 +++++++++++++++++++++++++
 stream/mts/psi/psi.go             | 16 ++++++++++------
 2 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go
index d11053d1..99d005c2 100644
--- a/stream/mts/psi/descriptor_test.go
+++ b/stream/mts/psi/descriptor_test.go
@@ -34,6 +34,7 @@ import (
 
 const (
 	errNotExpectedOut = "Did not get expected output: \ngot : %v, \nwant: %v"
+	errUnexpectedErr  = "Unexpected error: %v\n"
 )
 
 var tstPsi1 = PSI{
@@ -220,5 +221,29 @@ func TestCreateDescriptor(t *testing.T) {
 }
 
 func TestAddDescriptor(t *testing.T) {
+	// Add descriptor to psi without descriptors
+	got := PSIBytes(tstPsi2.Bytes())
 
+	if err := got.AddDescriptor(TimeDescTag, make([]byte, TimeDataSize)); err != nil {
+		t.Errorf(errUnexpectedErr, err.Error())
+	}
+
+	want := PSIBytes(tstPsi1.Bytes())
+
+	if !bytes.Equal(got, want) {
+		t.Errorf(errNotExpectedOut, got, want)
+	}
+
+	// Add to psi already with descriptor
+	got = PSIBytes(tstPsi1.Bytes())
+
+	if err := got.AddDescriptor(LocationDescTag, make([]byte, LocationDataSize)); err != nil {
+		t.Errorf(errUnexpectedErr, err.Error())
+	}
+
+	want = PSIBytes(tstPsi3.Bytes())
+
+	if !bytes.Equal(got, want) {
+		t.Errorf(errNotExpectedOut, got, want)
+	}
 }
diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index 7d61f47b..2bace251 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -275,9 +275,11 @@ func (p *PSIBytes) AddDescriptor(tag int, data []byte) error {
 	}
 
 	newProgInfoLen := p.ProgramInfoLen() + delta
-	p.SetProgInfoLen(newProgInfoLen)
+	p.setProgInfoLen(newProgInfoLen)
 	newSectionLen := int(psi.SectionLength(*p)) + delta
-	p.SetSectionLen(newSectionLen)
+	p.setSectionLen(newSectionLen)
+
+	updateCrc((*p)[1:])
 
 	return nil
 }
@@ -330,11 +332,13 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) {
 	// Update the program info length to account for the new descriptor.
 	addedLen := dataLen + 2
 	newProgInfoLen := curProgLen + addedLen
-	p.SetProgInfoLen(newProgInfoLen)
+	p.setProgInfoLen(newProgInfoLen)
 
 	// set section length
 	newSyntaxSectionLen := int(oldSyntaxSectionLen) + addedLen
-	p.SetSectionLen(newSyntaxSectionLen)
+	p.setSectionLen(newSyntaxSectionLen)
+
+	updateCrc((*p)[1:])
 }
 
 // TODO: make this safer. If no padding, does this index out of range ?
@@ -345,14 +349,14 @@ func (p *PSIBytes) trimPadding() []byte {
 	return o
 }
 
-func (p *PSIBytes) SetProgInfoLen(l int) {
+func (p *PSIBytes) setProgInfoLen(l int) {
 	// TODO: check if pmt first
 	(*p)[ProgramInfoLenIdx1] &= 0xff ^ ProgramInfoLenMask1
 	(*p)[ProgramInfoLenIdx1] |= byte(l>>8) & ProgramInfoLenMask1
 	(*p)[ProgramInfoLenIdx2] = byte(l)
 }
 
-func (p *PSIBytes) SetSectionLen(l int) {
+func (p *PSIBytes) setSectionLen(l int) {
 	(*p)[SyntaxSecLenIdx1] &= 0xff ^ SyntaxSecLenMask1
 	(*p)[SyntaxSecLenIdx1] |= byte(l>>8) & SyntaxSecLenMask1
 	(*p)[SyntaxSecLenIdx2] = byte(l)

From 23e03eeddcae399ecbf349b3d0211f6e27258fa7 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 29 Jan 2019 16:47:55 +1030
Subject: [PATCH 36/83] cmd: removed ts-repair - which shouldn't have been on
 this branch

---
 cmd/ts-repair/main.go | 217 ------------------------------------------
 1 file changed, 217 deletions(-)
 delete mode 100644 cmd/ts-repair/main.go

diff --git a/cmd/ts-repair/main.go b/cmd/ts-repair/main.go
deleted file mode 100644
index f3f73bb2..00000000
--- a/cmd/ts-repair/main.go
+++ /dev/null
@@ -1,217 +0,0 @@
-package main
-
-import (
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"os"
-
-	"bitbucket.org/ausocean/av/stream/mts"
-	"github.com/Comcast/gots/packet"
-)
-
-// Various errors that we can encounter.
-const (
-	errBadInPath         = "No file path provided, or file does not exist"
-	errCantCreateOut     = "Can't create output file"
-	errCantGetPid        = "Can't get pid from packet"
-	errReadFail          = "Read failed"
-	errWriteFail         = "Write to file failed"
-	errBadMode           = "Bad fix mode"
-	errAdaptationPresent = "Adaptation field is already present in packet"
-	errNoAdaptationField = "No adaptation field in this packet"
-)
-
-// Consts describing flag usage.
-const (
-	inUsage   = "The path to the file to be repaired"
-	outUsage  = "Output file path"
-	modeUsage = "Fix mode: 0 = cc-shift, 1 = di-update"
-)
-
-// Repair modes.
-const (
-	ccShift = iota
-	diUpdate
-)
-
-var ccMap = map[int]byte{
-	mts.PatPid:   16,
-	mts.PmtPid:   16,
-	mts.VideoPid: 16,
-}
-
-// packetNo will keep track of the ts packet number for reference.
-var packetNo int
-
-// Option defines a func that performs an action on p in order to change a ts option.
-type Option func(p *Packet)
-
-// Packet is a byte array of size mts.PacketSize i.e. 188 bytes. We define this
-// to allow us to write receiver funcs for the [mts.PacketSize]byte type.
-type Packet [mts.PacketSize]byte
-
-// CC returns the CC of p.
-func (p *Packet) CC() byte {
-	return (*p)[3] & 0x0f
-}
-
-// setCC sets the CC of p.
-func (p *Packet) setCC(cc byte) {
-	(*p)[3] |= cc & 0xf
-}
-
-// setDI sets the discontinuity counter of p.
-func (p *Packet) setDI(di bool) {
-	if di {
-		p[5] |= 0x80
-	} else {
-		p[5] &= 0x7f
-	}
-}
-
-// addAdaptationField adds an adaptation field to p, and applys the passed options to this field.
-// TODO: this will probably break if we already have adaptation field.
-func (p *Packet) addAdaptationField(options ...Option) error {
-	if p.hasAdaptation() {
-		return errors.New(errAdaptationPresent)
-	}
-	// Create space for adaptation field.
-	copy(p[mts.HeadSize+mts.DefaultAdaptationSize:], p[mts.HeadSize:len(p)-mts.DefaultAdaptationSize])
-
-	// TODO: seperate into own function
-	// Update adaptation field control.
-	p[mts.AdaptationControlIdx] &= 0xff ^ mts.AdaptationControlMask
-	p[mts.AdaptationControlIdx] |= mts.AdaptationControlMask
-	// Default the adaptationfield.
-	p.resetAdaptation()
-
-	// Apply and options that have bee passed.
-	for _, option := range options {
-		option(p)
-	}
-	return nil
-}
-
-// resetAdaptation sets fields in ps adaptation field to 0 if the adaptation field
-// exists, otherwise an error is returned.
-func (p *Packet) resetAdaptation() error {
-	if !p.hasAdaptation() {
-		return errors.New(errNoAdaptationField)
-	}
-	p[mts.AdaptationIdx] = mts.DefaultAdaptationBodySize
-	p[mts.AdaptationBodyIdx] = 0x00
-	return nil
-}
-
-// hasAdaptation returns true if p has an adaptation field and false otherwise.
-func (p *Packet) hasAdaptation() bool {
-	afc := p[mts.AdaptationControlIdx] & mts.AdaptationControlMask
-	if afc == 0x20 || afc == 0x30 {
-		return true
-	} else {
-		return false
-	}
-}
-
-// DiscontinuityIndicator returns and Option that will set p's discontinuity
-// indicator according to f.
-func DiscontinuityIndicator(f bool) Option {
-	return func(p *Packet) {
-		set := byte(mts.DiscontinuityIndicatorMask)
-		if !f {
-			set = 0x00
-		}
-		p[mts.DiscontinuityIndicatorIdx] &= 0xff ^ mts.DiscontinuityIndicatorMask
-		p[mts.DiscontinuityIndicatorIdx] |= mts.DiscontinuityIndicatorMask & set
-	}
-}
-
-func main() {
-	// Deal with input flags
-	inPtr := flag.String("in", "", inUsage)
-	outPtr := flag.String("out", "out.ts", outUsage)
-	modePtr := flag.Int("mode", diUpdate, modeUsage)
-	flag.Parse()
-
-	// Try and open the given input file, otherwise panic - we can't do anything
-	inFile, err := os.Open(*inPtr)
-	defer inFile.Close()
-	if err != nil {
-		panic(errBadInPath)
-	}
-
-	// Try and create output file, otherwise panic - we can't do anything
-	outFile, err := os.Create(*outPtr)
-	defer outFile.Close()
-	if err != nil {
-		panic(errCantCreateOut)
-	}
-
-	// Read each packet from the input file reader
-	var p Packet
-	for {
-		// If we get an end of file then return, otherwise we panic - can't do anything else
-		if _, err := inFile.Read(p[:mts.PacketSize]); err == io.EOF {
-			return
-		} else if err != nil {
-			panic(errReadFail + ": " + err.Error())
-		}
-		packetNo++
-
-		// Get the pid from the packet
-		pid, err := packet.Pid((*packet.Packet)(&p))
-		if err != nil {
-			panic(errCantGetPid)
-		}
-
-		// Get the cc from the packet and also the expected cc (if exists)
-		cc := p.CC()
-		expect, exists := expectedCC(int(pid))
-		if !exists {
-			updateCCMap(int(pid), cc)
-		} else {
-			switch *modePtr {
-			// ccShift mode shifts all CC regardless of presence of Discontinuities or not
-			case ccShift:
-				p.setCC(expect)
-				// diUpdate mode finds discontinuities and sets the discontinuity indicator to true.
-				// If we have a pat or pmt then we need to add an adaptation field and then set the DI.
-			case diUpdate:
-				if cc != expect {
-					fmt.Printf("***** Discontinuity found (packetNo: %v pid: %v, cc: %v, expect: %v)\n", packetNo, pid, cc, expect)
-					if p.hasAdaptation() {
-						p.setDI(true)
-					} else {
-						p.addAdaptationField(DiscontinuityIndicator(true))
-					}
-					updateCCMap(int(pid), p.CC())
-				}
-			default:
-				panic(errBadMode)
-			}
-		}
-
-		// Write this packet to the output file.
-		if _, err := outFile.Write(p[:]); err != nil {
-			panic(errWriteFail + ": " + err.Error())
-		}
-	}
-}
-
-// expectedCC returns the expected cc for the given pid. If the cc hasn't been
-// used yet, then 16 and false is returned.
-func expectedCC(pid int) (byte, bool) {
-	cc := ccMap[pid]
-	if cc == 16 {
-		return 16, false
-	}
-	ccMap[pid] = (cc + 1) & 0xf
-	return cc, true
-}
-
-// updateCCMap updates the cc for the passed pid.
-func updateCCMap(pid int, cc byte) {
-	ccMap[pid] = (cc + 1) & 0xf
-}

From 568840f69180180380ddf9729108fc7085c07a1e Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 29 Jan 2019 18:22:57 +1030
Subject: [PATCH 37/83] stream/mts/psi: cleaned up descriptor_test.go

---
 stream/mts/psi/descriptor_test.go | 264 ++++++++++++++----------------
 1 file changed, 124 insertions(+), 140 deletions(-)

diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go
index 99d005c2..dd763b82 100644
--- a/stream/mts/psi/descriptor_test.go
+++ b/stream/mts/psi/descriptor_test.go
@@ -37,124 +37,123 @@ const (
 	errUnexpectedErr  = "Unexpected error: %v\n"
 )
 
-var tstPsi1 = PSI{
-	Pf:  0x00,
-	Tid: 0x02,
-	Ssi: true,
-	Sl:  0x1c,
-	Tss: &TSS{
-		Tide: 0x01,
-		V:    0,
-		Cni:  true,
-		Sn:   0,
-		Lsn:  0,
-		Sd: &PMT{
-			Pcrpid: 0x0100, // wrong
-			Pil:    10,
-			Pd: []Desc{
-				{
-					Dt: TimeDescTag,
-					Dl: TimeDataSize,
-					Dd: make([]byte, TimeDataSize),
+var (
+	tstPsi1 = PSI{
+		Pf:  0x00,
+		Tid: 0x02,
+		Ssi: true,
+		Sl:  0x1c,
+		Tss: &TSS{
+			Tide: 0x01,
+			V:    0,
+			Cni:  true,
+			Sn:   0,
+			Lsn:  0,
+			Sd: &PMT{
+				Pcrpid: 0x0100, // wrong
+				Pil:    10,
+				Pd: []Desc{
+					{
+						Dt: TimeDescTag,
+						Dl: TimeDataSize,
+						Dd: make([]byte, TimeDataSize),
+					},
+				},
+				Essd: &ESSD{
+					St:   0x1b,
+					Epid: 0x0100,
+					Esil: 0x00,
 				},
 			},
-			Essd: &ESSD{
-				St:   0x1b,
-				Epid: 0x0100,
-				Esil: 0x00,
-			},
 		},
-	},
-}
+	}
 
-var tstPsi2 = PSI{
-	Pf:  0x00,
-	Tid: 0x02,
-	Ssi: true,
-	Sl:  0x12,
-	Tss: &TSS{
-		Tide: 0x01,
-		V:    0,
-		Cni:  true,
-		Sn:   0,
-		Lsn:  0,
-		Sd: &PMT{
-			Pcrpid: 0x0100,
-			Pil:    0,
-			Essd: &ESSD{
-				St:   0x1b,
-				Epid: 0x0100,
-				Esil: 0x00,
-			},
-		},
-	},
-}
-
-var tstPsi3 = PSI{
-	Pf:  0x00,
-	Tid: 0x02,
-	Ssi: true,
-	Sl:  0x3e,
-	Tss: &TSS{
-		Tide: 0x01,
-		V:    0,
-		Cni:  true,
-		Sn:   0,
-		Lsn:  0,
-		Sd: &PMT{
-			Pcrpid: 0x0100,
-			Pil:    PmtTimeLocationPil,
-			Pd: []Desc{
-				{
-					Dt: TimeDescTag,
-					Dl: TimeDataSize,
-					Dd: make([]byte, TimeDataSize),
-				},
-				{
-					Dt: LocationDescTag,
-					Dl: LocationDataSize,
-					Dd: make([]byte, LocationDataSize),
+	tstPsi2 = PSI{
+		Pf:  0x00,
+		Tid: 0x02,
+		Ssi: true,
+		Sl:  0x12,
+		Tss: &TSS{
+			Tide: 0x01,
+			V:    0,
+			Cni:  true,
+			Sn:   0,
+			Lsn:  0,
+			Sd: &PMT{
+				Pcrpid: 0x0100,
+				Pil:    0,
+				Essd: &ESSD{
+					St:   0x1b,
+					Epid: 0x0100,
+					Esil: 0x00,
 				},
 			},
-			Essd: &ESSD{
-				St:   0x1b,
-				Epid: 0x0100,
-				Esil: 0x00,
+		},
+	}
+
+	tstPsi3 = PSI{
+		Pf:  0x00,
+		Tid: 0x02,
+		Ssi: true,
+		Sl:  0x3e,
+		Tss: &TSS{
+			Tide: 0x01,
+			V:    0,
+			Cni:  true,
+			Sn:   0,
+			Lsn:  0,
+			Sd: &PMT{
+				Pcrpid: 0x0100,
+				Pil:    PmtTimeLocationPil,
+				Pd: []Desc{
+					{
+						Dt: TimeDescTag,
+						Dl: TimeDataSize,
+						Dd: make([]byte, TimeDataSize),
+					},
+					{
+						Dt: LocationDescTag,
+						Dl: LocationDataSize,
+						Dd: make([]byte, LocationDataSize),
+					},
+				},
+				Essd: &ESSD{
+					St:   0x1b,
+					Epid: 0x0100,
+					Esil: 0x00,
+				},
 			},
 		},
-	},
-}
+	}
+)
 
-func TestHasDescriptor(t *testing.T) {
+func TestHasDescriptorExists(t *testing.T) {
 	p := PSIBytes(tstPsi3.Bytes())
-
-	// Try getting descriptor that exists
 	_, got := p.HasDescriptor(LocationDescTag)
 	want := []byte{
 		LocationDescTag,
 		LocationDataSize,
 	}
 	want = append(want, make([]byte, LocationDataSize)...)
-
 	if !bytes.Equal(got, want) {
 		t.Errorf(errNotExpectedOut, got, want)
 	}
+}
 
-	// Try getting descriptor that doesnt exist
+func TestHasDescriptorAbsent(t *testing.T) {
+	p := PSIBytes(tstPsi3.Bytes())
 	const fakeTag = 236
-	_, got = p.HasDescriptor(fakeTag)
-	want = nil
-
+	_, got := p.HasDescriptor(fakeTag)
+	var want []byte
 	if !bytes.Equal(got, want) {
 		t.Errorf(errNotExpectedOut, got, want)
 	}
+}
 
-	// Try getting descriptor from psi that has no descriptors
-	p = PSIBytes(tstPsi2.Bytes())
-
-	_, got = p.HasDescriptor(LocationDescTag)
-	want = nil
-
+func TestHasDescriptorNone(t *testing.T) {
+	p := PSIBytes(tstPsi2.Bytes())
+	_, got := p.HasDescriptor(LocationDescTag)
+	var want []byte
 	if !bytes.Equal(got, want) {
 		t.Errorf(errNotExpectedOut, got, want)
 	}
@@ -170,7 +169,6 @@ func TestProgramInfoLen(t *testing.T) {
 }
 
 func TestDescriptors(t *testing.T) {
-	// Try psi with descriptors
 	p := PSIBytes(tstPsi1.Bytes())
 	got := p.descriptors()
 	want := []byte{
@@ -178,71 +176,57 @@ func TestDescriptors(t *testing.T) {
 		TimeDataSize,
 	}
 	want = append(want, make([]byte, TimeDataSize)...)
-
-	if !bytes.Equal(got, want) {
-		t.Errorf(errNotExpectedOut, got, want)
-	}
-
-	// Now try psi with empty descriptors
-	p = PSIBytes(tstPsi2.Bytes())
-	got = p.descriptors()
-	want = nil
 	if !bytes.Equal(got, want) {
 		t.Errorf(errNotExpectedOut, got, want)
 	}
 }
 
-func TestCreateDescriptor(t *testing.T) {
-	// Test with PSI containing no descriptors
+func TestDescriptorsNone(t *testing.T) {
 	p := PSIBytes(tstPsi2.Bytes())
-	p.createDescriptor(TimeDescTag, make([]byte, TimeDataSize))
-	// update crc
-	noPadding := p.trimPadding()
-	updateCrc(noPadding[1:])
-	want := PSIBytes(tstPsi1.Bytes())
-	got := p
-
-	if !bytes.Equal(want, got) {
+	got := p.descriptors()
+	var want []byte
+	if !bytes.Equal(got, want) {
 		t.Errorf(errNotExpectedOut, got, want)
 	}
+}
 
-	// Test with psi containing descriptor already
-	p = PSIBytes(tstPsi1.Bytes())
-	p.createDescriptor(LocationDescTag, make([]byte, LocationDataSize))
-	// update crc
-	noPadding = p.trimPadding()
-	updateCrc(noPadding[1:])
-	want = PSIBytes(tstPsi3.Bytes())
-	got = p
-
+func TestCreateDescriptorEmpty(t *testing.T) {
+	got := PSIBytes(tstPsi2.Bytes())
+	got.createDescriptor(TimeDescTag, make([]byte, TimeDataSize))
+	updateCrc(got[1:])
+	want := PSIBytes(tstPsi1.Bytes())
 	if !bytes.Equal(want, got) {
 		t.Errorf(errNotExpectedOut, got, want)
 	}
 }
 
-func TestAddDescriptor(t *testing.T) {
-	// Add descriptor to psi without descriptors
-	got := PSIBytes(tstPsi2.Bytes())
+func TestCreateDescriptorNotEmpty(t *testing.T) {
+	got := PSIBytes(tstPsi1.Bytes())
+	got.createDescriptor(LocationDescTag, make([]byte, LocationDataSize))
+	updateCrc(got[1:])
+	want := PSIBytes(tstPsi3.Bytes())
+	if !bytes.Equal(want, got) {
+		t.Errorf(errNotExpectedOut, got, want)
+	}
+}
 
+func TestAddDescriptorEmpty(t *testing.T) {
+	got := PSIBytes(tstPsi2.Bytes())
 	if err := got.AddDescriptor(TimeDescTag, make([]byte, TimeDataSize)); err != nil {
 		t.Errorf(errUnexpectedErr, err.Error())
 	}
-
 	want := PSIBytes(tstPsi1.Bytes())
-
-	if !bytes.Equal(got, want) {
-		t.Errorf(errNotExpectedOut, got, want)
-	}
-
-	// Add to psi already with descriptor
-	got = PSIBytes(tstPsi1.Bytes())
-
-	if err := got.AddDescriptor(LocationDescTag, make([]byte, LocationDataSize)); err != nil {
-		t.Errorf(errUnexpectedErr, err.Error())
-	}
-
-	want = PSIBytes(tstPsi3.Bytes())
-
+	if !bytes.Equal(got, want) {
+		t.Errorf(errNotExpectedOut, got, want)
+	}
+}
+
+func TestAddDescriptorNonEmpty(t *testing.T) {
+	got := PSIBytes(tstPsi1.Bytes())
+	if err := got.AddDescriptor(LocationDescTag, make([]byte, LocationDataSize)); err != nil {
+		t.Errorf(errUnexpectedErr, err.Error())
+	}
+	want := PSIBytes(tstPsi3.Bytes())
 	if !bytes.Equal(got, want) {
 		t.Errorf(errNotExpectedOut, got, want)
 	}

From 7f8465f1adc241295a71d05e9b0d52b8f25fd668 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 29 Jan 2019 22:07:06 +1030
Subject: [PATCH 38/83] stream/mts/psi/descriptor_test.go: added comments to
 test funcs - as well as some todos for further testing.

---
 stream/mts/psi/descriptor_test.go | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go
index dd763b82..cbb743a2 100644
--- a/stream/mts/psi/descriptor_test.go
+++ b/stream/mts/psi/descriptor_test.go
@@ -127,6 +127,10 @@ var (
 	}
 )
 
+// TestHasDescriptorExists checks that PSIBytes.HasDescriptor performs as expected
+// when the PSI we're interested in has the descriptor of interest. HasDescriptor
+// should return the descriptor bytes.
+// TODO: HasDescriptor also returns index of descriptor - we should check this.
 func TestHasDescriptorExists(t *testing.T) {
 	p := PSIBytes(tstPsi3.Bytes())
 	_, got := p.HasDescriptor(LocationDescTag)
@@ -140,6 +144,10 @@ func TestHasDescriptorExists(t *testing.T) {
 	}
 }
 
+// TestHasDescriptorAbsent checks that PSIBytes.HasDescriptor performs as expected
+// when the PSI does not have the descriptor of interest. HasDescriptor should
+// return a nil slice and a negative index.
+// TODO: check index here as well.
 func TestHasDescriptorAbsent(t *testing.T) {
 	p := PSIBytes(tstPsi3.Bytes())
 	const fakeTag = 236
@@ -150,6 +158,10 @@ func TestHasDescriptorAbsent(t *testing.T) {
 	}
 }
 
+// TestHasDescriptorNone checks that PSIBytes.HasDescriptor behaves as expected
+// when the PSI does not have any descriptors. HasDescriptor should return a nil
+// slice.
+// TODO: again check index here.
 func TestHasDescriptorNone(t *testing.T) {
 	p := PSIBytes(tstPsi2.Bytes())
 	_, got := p.HasDescriptor(LocationDescTag)
@@ -159,6 +171,8 @@ func TestHasDescriptorNone(t *testing.T) {
 	}
 }
 
+// TestProgramInfoLen checks that PSIBytes.ProgramInfoLen correctly extracts
+// the program info length from a PSI.
 func TestProgramInfoLen(t *testing.T) {
 	p := PSIBytes(tstPsi1.Bytes())
 	got := p.ProgramInfoLen()
@@ -168,6 +182,8 @@ func TestProgramInfoLen(t *testing.T) {
 	}
 }
 
+// TestDescriptors checks that PSIBytes.descriptors correctly returns the descriptors
+// from a PSI when descriptors exist.
 func TestDescriptors(t *testing.T) {
 	p := PSIBytes(tstPsi1.Bytes())
 	got := p.descriptors()
@@ -181,6 +197,8 @@ func TestDescriptors(t *testing.T) {
 	}
 }
 
+// TestDescriptors checks that PSIBYtes.desriptors correctly returns nil when
+// we try to get descriptors from a psi without any descriptors.
 func TestDescriptorsNone(t *testing.T) {
 	p := PSIBytes(tstPsi2.Bytes())
 	got := p.descriptors()
@@ -190,6 +208,8 @@ func TestDescriptorsNone(t *testing.T) {
 	}
 }
 
+// TestCreateDescriptorEmpty checks that PSIBytes.createDescriptor correctly adds
+// a descriptor to the descriptors list in a PSI when it has no descriptors already.
 func TestCreateDescriptorEmpty(t *testing.T) {
 	got := PSIBytes(tstPsi2.Bytes())
 	got.createDescriptor(TimeDescTag, make([]byte, TimeDataSize))
@@ -200,6 +220,9 @@ func TestCreateDescriptorEmpty(t *testing.T) {
 	}
 }
 
+// TestCreateDescriptorNotEmpty checks that PSIBytes.createDescriptor correctly adds
+// a descriptor to the descriptors list in a PSI when it already has one with
+// a different tag.
 func TestCreateDescriptorNotEmpty(t *testing.T) {
 	got := PSIBytes(tstPsi1.Bytes())
 	got.createDescriptor(LocationDescTag, make([]byte, LocationDataSize))
@@ -210,6 +233,8 @@ func TestCreateDescriptorNotEmpty(t *testing.T) {
 	}
 }
 
+// TestAddDescriptorEmpty checks that PSIBytes.AddDescriptor correctly adds a descriptor
+// when there are no other descriptors present in the PSI.
 func TestAddDescriptorEmpty(t *testing.T) {
 	got := PSIBytes(tstPsi2.Bytes())
 	if err := got.AddDescriptor(TimeDescTag, make([]byte, TimeDataSize)); err != nil {
@@ -221,6 +246,8 @@ func TestAddDescriptorEmpty(t *testing.T) {
 	}
 }
 
+// TestAddDescriptorNonEmpty checks that PSIBytes.AddDescriptor correctly adds a
+// descriptor when there is already a descriptor of a differing type in a PSI.
 func TestAddDescriptorNonEmpty(t *testing.T) {
 	got := PSIBytes(tstPsi1.Bytes())
 	if err := got.AddDescriptor(LocationDescTag, make([]byte, LocationDataSize)); err != nil {
@@ -231,3 +258,5 @@ func TestAddDescriptorNonEmpty(t *testing.T) {
 		t.Errorf(errNotExpectedOut, got, want)
 	}
 }
+
+// TODO: check that we can update descriptor with AddDescriptor

From d49a8b8c6b38d31a6f3325d30db2e2c1a7cfcb26 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 29 Jan 2019 22:09:17 +1030
Subject: [PATCH 39/83] stream/mts/psi.go: removed trimPadding function as we
 don't need this anymore

---
 stream/mts/psi/psi.go | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index 2bace251..11e36d38 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -341,14 +341,6 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) {
 	updateCrc((*p)[1:])
 }
 
-// TODO: make this safer. If no padding, does this index out of range ?
-func (p *PSIBytes) trimPadding() []byte {
-	sectionLength := SyntaxSecLenFrom(*p)
-	paddingIdx := (4 + sectionLength)
-	o := (*p)[:paddingIdx]
-	return o
-}
-
 func (p *PSIBytes) setProgInfoLen(l int) {
 	// TODO: check if pmt first
 	(*p)[ProgramInfoLenIdx1] &= 0xff ^ ProgramInfoLenMask1

From 55bee1532e15a60b75d67363171809bb49bebbea Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Wed, 30 Jan 2019 11:01:04 +1030
Subject: [PATCH 40/83] stream/mts/psi/psi.go: added function comments and
 improved commenting and layout inside functions

---
 stream/mts/psi/psi.go | 48 ++++++++++++++++++++++++++-----------------
 1 file changed, 29 insertions(+), 19 deletions(-)

diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index 11e36d38..f1033be7 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -239,6 +239,11 @@ func asByte(b bool) byte {
 	return 0x00
 }
 
+// AddDescriptor adds or updates a descriptor in a PSI given a descriptor tag
+// and data. If the psi is not a pmt, then an error is returned. If a descriptor
+// with the given tag is not found in the psi, room is made and a descriptor with
+// given tag and data is created. If a descriptor with the tag is found, the
+// descriptor is resized as required and the new data is copied in.
 func (p *PSIBytes) AddDescriptor(tag int, data []byte) error {
 	if psi.TableID(*p) != pmtID {
 		return errors.New("trying to add descriptor, but not pmt")
@@ -250,27 +255,26 @@ func (p *PSIBytes) AddDescriptor(tag int, data []byte) error {
 		return nil
 	}
 
-	// Old lengths
 	oldDescLen := desc.len()
 	oldDataLen := int(desc[1])
-
-	// New lengths
 	newDataLen := len(data)
 	newDescLen := 2 + newDataLen
-
 	delta := newDescLen - oldDescLen
 
-	if oldDataLen > newDataLen {
+	// If the old data length is more than the new data length, we need shift data
+	// after descriptor up, and then trim the psi. If the oldDataLen is less than
+	// new data then we need reseize psi and shift data down. If data is same size
+	// just copy new data in.
+	switch {
+	case oldDataLen > newDataLen:
 		copy((*p)[i+newDescLen:], (*p)[i+oldDescLen:])
 		*p = (*p)[:len(*p)+delta]
-
-	} else if oldDataLen < newDataLen {
+	case oldDataLen < newDataLen:
 		tmp := make([]byte, len(*p)+delta)
 		copy(tmp, *p)
 		*p = tmp
 		copy((*p)[i+newDescLen:], (*p)[i+oldDescLen:])
-
-	} else {
+	default:
 		copy((*p)[i+2:], data)
 	}
 
@@ -278,20 +282,27 @@ func (p *PSIBytes) AddDescriptor(tag int, data []byte) error {
 	p.setProgInfoLen(newProgInfoLen)
 	newSectionLen := int(psi.SectionLength(*p)) + delta
 	p.setSectionLen(newSectionLen)
-
 	updateCrc((*p)[1:])
-
 	return nil
 }
 
+// len returns the length of a descriptor in bytes.
 func (d *Descriptor) len() int {
 	return int(2 + (*d)[1])
 }
 
+// ProgramInfoLen returns the program info length of a PSI.
+//
+// TODO: check if pmt - if not return 0 ? or -1 ?
 func (p *PSIBytes) ProgramInfoLen() int {
 	return int((((*p)[ProgramInfoLenIdx1] & ProgramInfoLenMask1) << 8) | (*p)[ProgramInfoLenIdx2])
 }
 
+// HasDescriptor checks if a descriptor of the given tag exists in a PSI. If the descriptor
+// of the given tag exists, an index of this descriptor, as well as the Descriptor is returned.
+// If the descriptor of the given tag cannot be found, -1 and a nil slice is returned.
+//
+// TODO: check if pmt, return error if not ?
 func (p *PSIBytes) HasDescriptor(tag int) (int, Descriptor) {
 	descs := p.descriptors()
 	if descs == nil {
@@ -305,20 +316,21 @@ func (p *PSIBytes) HasDescriptor(tag int) (int, Descriptor) {
 	return -1, nil
 }
 
+// descriptors returns the descriptors in a psi if they exist, otherwise
+// a nil slice is returned.
 func (p *PSIBytes) descriptors() []byte {
 	return (*p)[DescriptorsIdx : DescriptorsIdx+p.ProgramInfoLen()]
 }
 
+// createDescriptor creates a descriptor in a psi given a tag and data.
 func (p *PSIBytes) createDescriptor(tag int, data []byte) {
 	curProgLen := p.ProgramInfoLen()
 	oldSyntaxSectionLen := SyntaxSecLenFrom(*p)
 	dataLen := len(data)
-
-	// Calculate the new descriptors index and length.
 	newDescIdx := DescriptorsIdx + curProgLen
 	newDescLen := dataLen + 2
 
-	// Copy data down from newDescIdx to create room for the new descriptor.
+	// Increase size of psi and copy data down to make room for new descriptor.
 	tmp := make([]byte, len(*p)+newDescLen)
 	copy(tmp, *p)
 	*p = tmp
@@ -329,25 +341,23 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) {
 	(*p)[newDescIdx+1] = byte(dataLen)
 	copy((*p)[newDescIdx+2:newDescIdx+2+dataLen], data)
 
-	// Update the program info length to account for the new descriptor.
+	// Set length fields and update the psi crc.
 	addedLen := dataLen + 2
 	newProgInfoLen := curProgLen + addedLen
 	p.setProgInfoLen(newProgInfoLen)
-
-	// set section length
 	newSyntaxSectionLen := int(oldSyntaxSectionLen) + addedLen
 	p.setSectionLen(newSyntaxSectionLen)
-
 	updateCrc((*p)[1:])
 }
 
+// setProgInfoLen sets the program information length in a psi with a pmt.
 func (p *PSIBytes) setProgInfoLen(l int) {
-	// TODO: check if pmt first
 	(*p)[ProgramInfoLenIdx1] &= 0xff ^ ProgramInfoLenMask1
 	(*p)[ProgramInfoLenIdx1] |= byte(l>>8) & ProgramInfoLenMask1
 	(*p)[ProgramInfoLenIdx2] = byte(l)
 }
 
+// setSectionLen sets section length in a psi.
 func (p *PSIBytes) setSectionLen(l int) {
 	(*p)[SyntaxSecLenIdx1] &= 0xff ^ SyntaxSecLenMask1
 	(*p)[SyntaxSecLenIdx1] |= byte(l>>8) & SyntaxSecLenMask1

From c2112e58ac3203a40377e5de8a68bf263a026f7f Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Wed, 30 Jan 2019 12:08:55 +1030
Subject: [PATCH 41/83] stream/mts/psi: added some more tests for
 AddDescriptor, and in the process fixed some bugs with AddDescriptor

---
 stream/mts/psi/descriptor_test.go | 55 ++++++++++++++++++++++++++++++-
 stream/mts/psi/psi.go             | 51 ++++++++++++++--------------
 2 files changed, 81 insertions(+), 25 deletions(-)

diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go
index cbb743a2..1fed7b2b 100644
--- a/stream/mts/psi/descriptor_test.go
+++ b/stream/mts/psi/descriptor_test.go
@@ -127,6 +127,24 @@ var (
 	}
 )
 
+var (
+	pmtTimeBytesResizedBigger = []byte{
+		0x00, 0x02, 0xb0, 0x1e, 0x00, 0x01, 0xc1, 0x00, 0x00, 0xe1, 0x00, 0xf0, 0x0c,
+		TimeDescTag,                                                // Descriptor tag
+		0x0a,                                                       // Length of bytes to follow
+		0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, // timestamp
+		0x1b, 0xe1, 0x00, 0xf0, 0x00,
+	}
+
+	pmtTimeBytesResizedSmaller = []byte{
+		0x00, 0x02, 0xb0, 0x1a, 0x00, 0x01, 0xc1, 0x00, 0x00, 0xe1, 0x00, 0xf0, 0x08,
+		TimeDescTag,                        // Descriptor tag
+		0x06,                               // Length of bytes to follow
+		0x01, 0x02, 0x03, 0x04, 0x05, 0x06, // timestamp
+		0x1b, 0xe1, 0x00, 0xf0, 0x00,
+	}
+)
+
 // TestHasDescriptorExists checks that PSIBytes.HasDescriptor performs as expected
 // when the PSI we're interested in has the descriptor of interest. HasDescriptor
 // should return the descriptor bytes.
@@ -259,4 +277,39 @@ func TestAddDescriptorNonEmpty(t *testing.T) {
 	}
 }
 
-// TODO: check that we can update descriptor with AddDescriptor
+func TestAddDescriptorUpdateSame(t *testing.T) {
+	newData := [8]byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}
+	want := PSIBytes(tstPsi2.Bytes())
+	want.createDescriptor(TimeDescTag, newData[:])
+
+	got := PSIBytes(tstPsi1.Bytes())
+	if err := got.AddDescriptor(TimeDescTag, newData[:]); err != nil {
+		t.Errorf(errUnexpectedErr, err.Error())
+	}
+
+	if !bytes.Equal(got, want) {
+		t.Errorf(errNotExpectedOut, got, want)
+	}
+}
+
+func TestAddDescriptorUpdateBigger(t *testing.T) {
+	got := PSIBytes(tstPsi1.Bytes())
+	if err := got.AddDescriptor(TimeDescTag, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a}); err != nil {
+		t.Errorf(errUnexpectedErr, err.Error())
+	}
+	want := addCrc(pmtTimeBytesResizedBigger)
+	if !bytes.Equal(got, want) {
+		t.Errorf(errNotExpectedOut, got, want)
+	}
+}
+
+func TestAddDescriptorUpdateSmaller(t *testing.T) {
+	got := PSIBytes(tstPsi1.Bytes())
+	if err := got.AddDescriptor(TimeDescTag, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}); err != nil {
+		t.Errorf(errUnexpectedErr, err.Error())
+	}
+	want := addCrc(pmtTimeBytesResizedSmaller)
+	if !bytes.Equal(got, want) {
+		t.Errorf(errNotExpectedOut, got, want)
+	}
+}
diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index f1033be7..dca9e57a 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -263,8 +263,7 @@ func (p *PSIBytes) AddDescriptor(tag int, data []byte) error {
 
 	// If the old data length is more than the new data length, we need shift data
 	// after descriptor up, and then trim the psi. If the oldDataLen is less than
-	// new data then we need reseize psi and shift data down. If data is same size
-	// just copy new data in.
+	// new data then we need reseize psi and shift data down. If same do nothing.
 	switch {
 	case oldDataLen > newDataLen:
 		copy((*p)[i+newDescLen:], (*p)[i+oldDescLen:])
@@ -274,10 +273,12 @@ func (p *PSIBytes) AddDescriptor(tag int, data []byte) error {
 		copy(tmp, *p)
 		*p = tmp
 		copy((*p)[i+newDescLen:], (*p)[i+oldDescLen:])
-	default:
-		copy((*p)[i+2:], data)
 	}
 
+	// Copy in new data
+	(*p)[i+1] = byte(newDataLen)
+	copy((*p)[i+2:], data)
+
 	newProgInfoLen := p.ProgramInfoLen() + delta
 	p.setProgInfoLen(newProgInfoLen)
 	newSectionLen := int(psi.SectionLength(*p)) + delta
@@ -286,18 +287,6 @@ func (p *PSIBytes) AddDescriptor(tag int, data []byte) error {
 	return nil
 }
 
-// len returns the length of a descriptor in bytes.
-func (d *Descriptor) len() int {
-	return int(2 + (*d)[1])
-}
-
-// ProgramInfoLen returns the program info length of a PSI.
-//
-// TODO: check if pmt - if not return 0 ? or -1 ?
-func (p *PSIBytes) ProgramInfoLen() int {
-	return int((((*p)[ProgramInfoLenIdx1] & ProgramInfoLenMask1) << 8) | (*p)[ProgramInfoLenIdx2])
-}
-
 // HasDescriptor checks if a descriptor of the given tag exists in a PSI. If the descriptor
 // of the given tag exists, an index of this descriptor, as well as the Descriptor is returned.
 // If the descriptor of the given tag cannot be found, -1 and a nil slice is returned.
@@ -310,19 +299,15 @@ func (p *PSIBytes) HasDescriptor(tag int) (int, Descriptor) {
 	}
 	for i := 0; i < len(descs); i += 2 + int(descs[i+1]) {
 		if int(descs[i]) == tag {
-			return i, descs[i : i+2+int(descs[i+1])]
+			return i + DescriptorsIdx, descs[i : i+2+int(descs[i+1])]
 		}
 	}
 	return -1, nil
 }
 
-// descriptors returns the descriptors in a psi if they exist, otherwise
-// a nil slice is returned.
-func (p *PSIBytes) descriptors() []byte {
-	return (*p)[DescriptorsIdx : DescriptorsIdx+p.ProgramInfoLen()]
-}
-
-// createDescriptor creates a descriptor in a psi given a tag and data.
+// createDescriptor creates a descriptor in a psi given a tag and data. It does so
+// by resizing the psi, shifting existing data down and copying in new descriptor
+// in new space.
 func (p *PSIBytes) createDescriptor(tag int, data []byte) {
 	curProgLen := p.ProgramInfoLen()
 	oldSyntaxSectionLen := SyntaxSecLenFrom(*p)
@@ -363,3 +348,21 @@ func (p *PSIBytes) setSectionLen(l int) {
 	(*p)[SyntaxSecLenIdx1] |= byte(l>>8) & SyntaxSecLenMask1
 	(*p)[SyntaxSecLenIdx2] = byte(l)
 }
+
+// descriptors returns the descriptors in a psi if they exist, otherwise
+// a nil slice is returned.
+func (p *PSIBytes) descriptors() []byte {
+	return (*p)[DescriptorsIdx : DescriptorsIdx+p.ProgramInfoLen()]
+}
+
+// len returns the length of a descriptor in bytes.
+func (d *Descriptor) len() int {
+	return int(2 + (*d)[1])
+}
+
+// ProgramInfoLen returns the program info length of a PSI.
+//
+// TODO: check if pmt - if not return 0 ? or -1 ?
+func (p *PSIBytes) ProgramInfoLen() int {
+	return int((((*p)[ProgramInfoLenIdx1] & ProgramInfoLenMask1) << 8) | (*p)[ProgramInfoLenIdx2])
+}

From 60c2eafd8cd7b18e4a29a0102cdd7f9df33a1798 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Wed, 30 Jan 2019 12:13:29 +1030
Subject: [PATCH 42/83] stream/mts/psi/descriptor_test.go: added some function
 comments above new test funcs

---
 stream/mts/psi/descriptor_test.go | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go
index 1fed7b2b..0f7d300e 100644
--- a/stream/mts/psi/descriptor_test.go
+++ b/stream/mts/psi/descriptor_test.go
@@ -277,21 +277,25 @@ func TestAddDescriptorNonEmpty(t *testing.T) {
 	}
 }
 
+// TestAddDescriptorUpdateSame checks that PSIBytes.AddDescriptor correctly updates data in a descriptor
+// with the same given tag, with data being the same size. AddDescriptor should just copy new data into
+// the descriptors data field.
 func TestAddDescriptorUpdateSame(t *testing.T) {
 	newData := [8]byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}
 	want := PSIBytes(tstPsi2.Bytes())
 	want.createDescriptor(TimeDescTag, newData[:])
-
 	got := PSIBytes(tstPsi1.Bytes())
 	if err := got.AddDescriptor(TimeDescTag, newData[:]); err != nil {
 		t.Errorf(errUnexpectedErr, err.Error())
 	}
-
 	if !bytes.Equal(got, want) {
 		t.Errorf(errNotExpectedOut, got, want)
 	}
 }
 
+// TestAddDescriptorUpdateBigger checks that PSIBytes.AddDescriptor correctly resizes descriptor with same given tag
+// to a bigger size and copies in new data. AddDescriptor should find descriptor with same tag, increase size of psi,
+// shift data to make room for update descriptor, and then copy in the new data.
 func TestAddDescriptorUpdateBigger(t *testing.T) {
 	got := PSIBytes(tstPsi1.Bytes())
 	if err := got.AddDescriptor(TimeDescTag, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a}); err != nil {
@@ -303,6 +307,9 @@ func TestAddDescriptorUpdateBigger(t *testing.T) {
 	}
 }
 
+// TestAddDescriptorUpdateSmaller checks that PSIBytes.AddDescriptor correctly resizes descriptor with same given tag
+// in a psi to a smaller size and copies in new data. AddDescriptor should find tag with same descrtiptor, shift data
+// after descriptor upwards, trim the psi to new size, and then copy in new data.
 func TestAddDescriptorUpdateSmaller(t *testing.T) {
 	got := PSIBytes(tstPsi1.Bytes())
 	if err := got.AddDescriptor(TimeDescTag, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}); err != nil {

From 83ac98fe84a336df3780c81baee2a1800f742268 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Wed, 30 Jan 2019 12:26:51 +1030
Subject: [PATCH 43/83] stream/mts: added metaEncode_test.go a file that will
 contain tests that will use the Meta struct to actually encode metadata into
 psi

---
 stream/mts/metaEncode_test.go | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 stream/mts/metaEncode_test.go

diff --git a/stream/mts/metaEncode_test.go b/stream/mts/metaEncode_test.go
new file mode 100644
index 00000000..d422c700
--- /dev/null
+++ b/stream/mts/metaEncode_test.go
@@ -0,0 +1,34 @@
+/*
+NAME
+  metaEncode_test.go
+
+DESCRIPTION
+  See Readme.md
+
+AUTHOR
+  Saxon Nelson-Milton <saxon@ausocean.org>
+
+LICENSE
+  metaEncode_test.go is Copyright (C) 2017-2019 the Australian Ocean Lab (AusOcean)
+
+  It is free software: you can redistribute it and/or modify them
+  under the terms of the GNU General Public License as published by the
+  Free Software Foundation, either version 3 of the License, or (at your
+  option) any later version.
+
+  It is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with revid in gpl.txt. If not, see http://www.gnu.org/licenses.
+*/
+
+package mts
+
+import "testing"
+
+func TestMetaEncode(t *testing.T) {
+
+}

From 7dd1ce99e1454a7208cf12ab264f4738d97f6784 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Wed, 30 Jan 2019 13:23:07 +1030
Subject: [PATCH 44/83] stream/mts: started writing metaEncode_test.go file

---
 stream/mts/encoder.go         |  8 ++++++--
 stream/mts/meta.go            | 16 ++++++++--------
 stream/mts/metaEncode_test.go | 24 ++++++++++++++++++++++--
 3 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index 33194ab8..4567aea2 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -88,7 +88,11 @@ const (
 )
 
 // global Meta
-var meta Meta
+var Meta *Metadata
+
+func init() {
+	Meta = NewMeta()
+}
 
 var (
 	patTable = standardPat.Bytes()
@@ -252,7 +256,7 @@ func (e *Encoder) ccFor(pid int) byte {
 func updateMeta(b []byte) error {
 	var p psi.PSIBytes
 	p = b
-	m := meta.Encode()
+	m := Meta.Encode()
 	p.AddDescriptor(psi.MetadataTag, m)
 	return nil
 }
diff --git a/stream/mts/meta.go b/stream/mts/meta.go
index 465a537e..62097d30 100644
--- a/stream/mts/meta.go
+++ b/stream/mts/meta.go
@@ -48,14 +48,14 @@ var (
 	errKeyAbsent = errors.New("Key does not exist in map")
 )
 
-type Meta struct {
+type Metadata struct {
 	mu   sync.RWMutex
 	data map[string]string
 	enc  []byte
 }
 
-func NewMeta() *Meta {
-	return &Meta{
+func NewMeta() *Metadata {
+	return &Metadata{
 		data: make(map[string]string),
 		enc: []byte{
 			0x00,                   // Reserved byte
@@ -67,14 +67,14 @@ func NewMeta() *Meta {
 }
 
 // Add adds metadata with key and val, if already exists return error
-func (m *Meta) Add(key, val string) {
+func (m *Metadata) Add(key, val string) {
 	m.mu.Lock()
 	m.data[key] = val
 	m.mu.Unlock()
 }
 
 // All returns the a copy of the map containing the meta data
-func (m *Meta) All() map[string]string {
+func (m *Metadata) All() map[string]string {
 	m.mu.Lock()
 	cpy := make(map[string]string)
 	for k, v := range m.data {
@@ -85,7 +85,7 @@ func (m *Meta) All() map[string]string {
 }
 
 // Get returns the meta data for the passed key
-func (m *Meta) Get(key string) (string, error) {
+func (m *Metadata) Get(key string) (string, error) {
 	m.mu.Lock()
 	val, ok := m.data[key]
 	m.mu.Unlock()
@@ -97,7 +97,7 @@ func (m *Meta) Get(key string) (string, error) {
 }
 
 // Remove deletes a meta entry in the map and returns error if it doesn’t exist
-func (m *Meta) Delete(key string) error {
+func (m *Metadata) Delete(key string) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	if _, ok := m.data[key]; ok {
@@ -109,7 +109,7 @@ func (m *Meta) Delete(key string) error {
 
 // Encode takes the meta data map and encods into a byte slice with header
 // describing the version, length of data and data in TSV format.
-func (m *Meta) Encode() []byte {
+func (m *Metadata) Encode() []byte {
 	m.enc = m.enc[:headSize]
 
 	// Iterate over map and append entries, only adding tab if we're not on the last entry
diff --git a/stream/mts/metaEncode_test.go b/stream/mts/metaEncode_test.go
index d422c700..849f5c0d 100644
--- a/stream/mts/metaEncode_test.go
+++ b/stream/mts/metaEncode_test.go
@@ -27,8 +27,28 @@ LICENSE
 
 package mts
 
-import "testing"
+import (
+	"bytes"
+	"testing"
+
+	"bitbucket.org/ausocean/av/stream/mts/psi"
+)
+
+const fps = 25
 
 func TestMetaEncode(t *testing.T) {
-
+	var b []byte
+	buf := bytes.NewBuffer(b)
+	e := NewEncoder(buf, fps)
+	Meta.Add("ts", "12345678")
+	e.writePSI()
+	out := buf.Bytes()
+	got := out[PacketSize:]
+	want := []byte{
+		0x00, 0x02, 0xb0, 0x12, 0x00, 0x01, 0xc1, 0x00, 0x00, 0xe1, 0x00, 0xf0, 0x0a,
+		psi.MetadataTag,                                // Descriptor tag
+		0x08,                                           // Length of bytes to follow
+		0x00, 0x00, 0x00, 0x00, 0x49, 0xa2, 0x36, 0x0b, // timestamp
+		0x1b, 0xe1, 0x00, 0xf0, 0x00,
+	}
 }

From d373f85b850e32c864a9d20719c6f2ec6a853ea2 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Wed, 30 Jan 2019 15:37:15 +1030
Subject: [PATCH 45/83] stream/mts: wrote test in metaEncode_test.go and found
 bug which was fixed

---
 stream/mts/encoder.go             | 34 ++++++++++++++++---------------
 stream/mts/metaEncode_test.go     | 25 +++++++++++++++++------
 stream/mts/meta_test.go           |  1 -
 stream/mts/psi/crc.go             |  6 +++---
 stream/mts/psi/descriptor_test.go |  8 ++++----
 stream/mts/psi/helpers.go         |  6 +++---
 stream/mts/psi/psi.go             |  7 +++----
 stream/mts/psi/psi_test.go        |  6 +++---
 8 files changed, 53 insertions(+), 40 deletions(-)

diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index 4567aea2..a1527e8d 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -200,27 +200,29 @@ func (e *Encoder) Encode(nalu []byte) error {
 func (e *Encoder) writePSI() error {
 	// Write PAT.
 	patPkt := Packet{
-		PUSI: true,
-		PID:  PatPid,
-		CC:   e.ccFor(PatPid),
-		AFC:  HasPayload,
+		PUSI:    true,
+		PID:     PatPid,
+		CC:      e.ccFor(PatPid),
+		AFC:     HasPayload,
+		Payload: psi.AddPadding(patTable),
 	}
-	patPkt.FillPayload(patTable)
 	_, err := e.dst.Write(patPkt.Bytes(e.tsSpace[:PacketSize]))
 	if err != nil {
 		return err
 	}
 
-	updateMeta(pmtTable)
+	if err = updateMeta(&pmtTable); err != nil {
+		return err
+	}
 
 	// Create mts packet from pmt table.
 	pmtPkt := Packet{
-		PUSI: true,
-		PID:  PmtPid,
-		CC:   e.ccFor(PmtPid),
-		AFC:  HasPayload,
+		PUSI:    true,
+		PID:     PmtPid,
+		CC:      e.ccFor(PmtPid),
+		AFC:     HasPayload,
+		Payload: psi.AddPadding(pmtTable),
 	}
-	pmtPkt.FillPayload(pmtTable)
 	_, err = e.dst.Write(pmtPkt.Bytes(e.tsSpace[:PacketSize]))
 	if err != nil {
 		return err
@@ -253,10 +255,10 @@ func (e *Encoder) ccFor(pid int) byte {
 }
 
 // updateMeta ...
-func updateMeta(b []byte) error {
-	var p psi.PSIBytes
-	p = b
+func updateMeta(b *[]byte) error {
 	m := Meta.Encode()
-	p.AddDescriptor(psi.MetadataTag, m)
-	return nil
+	p := psi.PSIBytes(*b)
+	err := p.AddDescriptor(psi.MetadataTag, m)
+	*b = []byte(p)
+	return err
 }
diff --git a/stream/mts/metaEncode_test.go b/stream/mts/metaEncode_test.go
index 849f5c0d..a914da2b 100644
--- a/stream/mts/metaEncode_test.go
+++ b/stream/mts/metaEncode_test.go
@@ -34,6 +34,11 @@ import (
 	"bitbucket.org/ausocean/av/stream/mts/psi"
 )
 
+const (
+	errNotExpectedOut = "Unexpected output. \n Got : %v\n, Want: %v\n"
+	errUnexpectedErr  = "Unexpected error: %v\n"
+)
+
 const fps = 25
 
 func TestMetaEncode(t *testing.T) {
@@ -41,14 +46,22 @@ func TestMetaEncode(t *testing.T) {
 	buf := bytes.NewBuffer(b)
 	e := NewEncoder(buf, fps)
 	Meta.Add("ts", "12345678")
-	e.writePSI()
+	if err := e.writePSI(); err != nil {
+		t.Errorf(errUnexpectedErr, err.Error())
+	}
 	out := buf.Bytes()
-	got := out[PacketSize:]
+	got := out[PacketSize+4:]
+
 	want := []byte{
-		0x00, 0x02, 0xb0, 0x12, 0x00, 0x01, 0xc1, 0x00, 0x00, 0xe1, 0x00, 0xf0, 0x0a,
-		psi.MetadataTag,                                // Descriptor tag
-		0x08,                                           // Length of bytes to follow
-		0x00, 0x00, 0x00, 0x00, 0x49, 0xa2, 0x36, 0x0b, // timestamp
+		0x00, 0x02, 0xb0, 0x23, 0x00, 0x01, 0xc1, 0x00, 0x00, 0xe1, 0x00, 0xf0, 0x11,
+		psi.MetadataTag,                                                               // Descriptor tag
+		0x0f,                                                                          // Length of bytes to follow
+		0x00, 0x10, 0x00, 0x0b, 't', 's', '=', '1', '2', '3', '4', '5', '6', '7', '8', // timestamp
 		0x1b, 0xe1, 0x00, 0xf0, 0x00,
 	}
+	want = psi.AddCrc(want)
+	want = psi.AddPadding(want)
+	if !bytes.Equal(got, want) {
+		t.Errorf(errNotExpectedOut, got, want)
+	}
 }
diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go
index daff9038..11003b9b 100644
--- a/stream/mts/meta_test.go
+++ b/stream/mts/meta_test.go
@@ -144,7 +144,6 @@ func TestEncode(t *testing.T) {
 			tstKey2+"="+tstData2)...)
 
 	got := meta.Encode()
-
 	if !bytes.Equal(expectedOut, got) {
 		t.Errorf("Did not get expected out. \nGot : %v \nwant: %v", got, expectedOut)
 	}
diff --git a/stream/mts/psi/crc.go b/stream/mts/psi/crc.go
index cd9a1199..e0ac7deb 100644
--- a/stream/mts/psi/crc.go
+++ b/stream/mts/psi/crc.go
@@ -34,15 +34,15 @@ import (
 )
 
 // addCrc appends a crc table to a given psi table in bytes
-func addCrc(out []byte) []byte {
+func AddCrc(out []byte) []byte {
 	t := make([]byte, len(out)+4)
 	copy(t, out)
-	updateCrc(t[1:])
+	UpdateCrc(t[1:])
 	return t
 }
 
 // updateCrc updates the crc of bytes slice, writing the checksum into the last four bytes.
-func updateCrc(b []byte) {
+func UpdateCrc(b []byte) {
 	crc32 := crc32_Update(0xffffffff, crc32_MakeTable(bits.Reverse32(crc32.IEEE)), b[:len(b)-4])
 	binary.BigEndian.PutUint32(b[len(b)-4:], crc32)
 }
diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go
index 0f7d300e..94441277 100644
--- a/stream/mts/psi/descriptor_test.go
+++ b/stream/mts/psi/descriptor_test.go
@@ -231,7 +231,7 @@ func TestDescriptorsNone(t *testing.T) {
 func TestCreateDescriptorEmpty(t *testing.T) {
 	got := PSIBytes(tstPsi2.Bytes())
 	got.createDescriptor(TimeDescTag, make([]byte, TimeDataSize))
-	updateCrc(got[1:])
+	UpdateCrc(got[1:])
 	want := PSIBytes(tstPsi1.Bytes())
 	if !bytes.Equal(want, got) {
 		t.Errorf(errNotExpectedOut, got, want)
@@ -244,7 +244,7 @@ func TestCreateDescriptorEmpty(t *testing.T) {
 func TestCreateDescriptorNotEmpty(t *testing.T) {
 	got := PSIBytes(tstPsi1.Bytes())
 	got.createDescriptor(LocationDescTag, make([]byte, LocationDataSize))
-	updateCrc(got[1:])
+	UpdateCrc(got[1:])
 	want := PSIBytes(tstPsi3.Bytes())
 	if !bytes.Equal(want, got) {
 		t.Errorf(errNotExpectedOut, got, want)
@@ -301,7 +301,7 @@ func TestAddDescriptorUpdateBigger(t *testing.T) {
 	if err := got.AddDescriptor(TimeDescTag, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a}); err != nil {
 		t.Errorf(errUnexpectedErr, err.Error())
 	}
-	want := addCrc(pmtTimeBytesResizedBigger)
+	want := AddCrc(pmtTimeBytesResizedBigger)
 	if !bytes.Equal(got, want) {
 		t.Errorf(errNotExpectedOut, got, want)
 	}
@@ -315,7 +315,7 @@ func TestAddDescriptorUpdateSmaller(t *testing.T) {
 	if err := got.AddDescriptor(TimeDescTag, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}); err != nil {
 		t.Errorf(errUnexpectedErr, err.Error())
 	}
-	want := addCrc(pmtTimeBytesResizedSmaller)
+	want := AddCrc(pmtTimeBytesResizedSmaller)
 	if !bytes.Equal(got, want) {
 		t.Errorf(errNotExpectedOut, got, want)
 	}
diff --git a/stream/mts/psi/helpers.go b/stream/mts/psi/helpers.go
index 174b289b..b8bab6b5 100644
--- a/stream/mts/psi/helpers.go
+++ b/stream/mts/psi/helpers.go
@@ -64,7 +64,7 @@ func UpdateTime(dst []byte, t uint64) error {
 	for i := range dst[TimeDataIndx : TimeDataIndx+TimeDataSize] {
 		dst[i+TimeDataIndx] = ts[i]
 	}
-	updateCrc(dst[1:])
+	UpdateCrc(dst[1:])
 	return nil
 }
 
@@ -111,7 +111,7 @@ func UpdateLocation(d []byte, s string) error {
 	for i := range loc {
 		loc[i] = 0
 	}
-	updateCrc(d[1:])
+	UpdateCrc(d[1:])
 	return nil
 }
 
@@ -126,7 +126,7 @@ func trimTo(d []byte, t byte) []byte {
 
 // addPadding adds an appropriate amount of padding to a pat or pmt table for
 // addition to an mpegts packet
-func addPadding(d []byte) []byte {
+func AddPadding(d []byte) []byte {
 	t := make([]byte, PacketSize)
 	copy(t, d)
 	padding := t[len(d):]
diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index dca9e57a..91d8fade 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -169,7 +169,7 @@ func (p *PSI) Bytes() []byte {
 	out[2] = 0x80 | 0x30 | (0x03 & byte(p.Sl>>8))
 	out[3] = byte(p.Sl)
 	out = append(out, p.Tss.Bytes()...)
-	out = addCrc(out)
+	out = AddCrc(out)
 	return out
 }
 
@@ -283,7 +283,7 @@ func (p *PSIBytes) AddDescriptor(tag int, data []byte) error {
 	p.setProgInfoLen(newProgInfoLen)
 	newSectionLen := int(psi.SectionLength(*p)) + delta
 	p.setSectionLen(newSectionLen)
-	updateCrc((*p)[1:])
+	UpdateCrc((*p)[1:])
 	return nil
 }
 
@@ -320,7 +320,6 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) {
 	copy(tmp, *p)
 	*p = tmp
 	copy((*p)[newDescIdx+newDescLen:], (*p)[newDescIdx:newDescIdx+newDescLen])
-
 	// Set the tag, data len and data of the new desriptor.
 	(*p)[newDescIdx] = byte(tag)
 	(*p)[newDescIdx+1] = byte(dataLen)
@@ -332,7 +331,7 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) {
 	p.setProgInfoLen(newProgInfoLen)
 	newSyntaxSectionLen := int(oldSyntaxSectionLen) + addedLen
 	p.setSectionLen(newSyntaxSectionLen)
-	updateCrc((*p)[1:])
+	UpdateCrc((*p)[1:])
 }
 
 // setProgInfoLen sets the program information length in a psi with a pmt.
diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go
index 61a740a4..7e3a3104 100644
--- a/stream/mts/psi/psi_test.go
+++ b/stream/mts/psi/psi_test.go
@@ -282,7 +282,7 @@ var bytesTests = []struct {
 func TestBytes(t *testing.T) {
 	for _, test := range bytesTests {
 		got := test.input.Bytes()
-		if !bytes.Equal(got, addCrc(test.want)) {
+		if !bytes.Equal(got, AddCrc(test.want)) {
 			t.Errorf("unexpected error for test %v: got:%v want:%v", test.name, got,
 				test.want)
 		}
@@ -301,7 +301,7 @@ func TestTimestampToBytes(t *testing.T) {
 func TestTimeUpdate(t *testing.T) {
 	cpy := make([]byte, len(pmtTimeBytes1))
 	copy(cpy, pmtTimeBytes1)
-	cpy = addCrc(cpy)
+	cpy = AddCrc(cpy)
 	err := UpdateTime(cpy, tstTime2)
 	cpy = cpy[:len(cpy)-4]
 	if err != nil {
@@ -343,7 +343,7 @@ func TestLocationGet(t *testing.T) {
 func TestLocationUpdate(t *testing.T) {
 	cpy := make([]byte, len(pmtWithMetaTst1))
 	copy(cpy, pmtWithMetaTst1)
-	cpy = addCrc(cpy)
+	cpy = AddCrc(cpy)
 	err := UpdateLocation(cpy, locationTstStr2)
 	cpy = cpy[:len(cpy)-4]
 	if err != nil {

From 1d9cb575053190a57fc65bfd9fc4fe228a0c6c78 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Wed, 30 Jan 2019 16:22:57 +1030
Subject: [PATCH 46/83] stream/mts/metaEncode_test.go: added another test to
 check behaviour when more meta data is added.

---
 stream/mts/metaEncode_test.go | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/stream/mts/metaEncode_test.go b/stream/mts/metaEncode_test.go
index a914da2b..d372f20a 100644
--- a/stream/mts/metaEncode_test.go
+++ b/stream/mts/metaEncode_test.go
@@ -41,7 +41,7 @@ const (
 
 const fps = 25
 
-func TestMetaEncode(t *testing.T) {
+func TestMetaEncode1(t *testing.T) {
 	var b []byte
 	buf := bytes.NewBuffer(b)
 	e := NewEncoder(buf, fps)
@@ -65,3 +65,29 @@ func TestMetaEncode(t *testing.T) {
 		t.Errorf(errNotExpectedOut, got, want)
 	}
 }
+
+func TestMetaEncode2(t *testing.T) {
+	var b []byte
+	buf := bytes.NewBuffer(b)
+	e := NewEncoder(buf, fps)
+	Meta.Add("ts", "12345678")
+	Meta.Add("loc", "1234,4321,1234")
+	if err := e.writePSI(); err != nil {
+		t.Errorf(errUnexpectedErr, err.Error())
+	}
+	out := buf.Bytes()
+	got := out[PacketSize+4:]
+	want := []byte{
+		0x00, 0x02, 0xb0, 0x36, 0x00, 0x01, 0xc1, 0x00, 0x00, 0xe1, 0x00, 0xf0, 0x24,
+		psi.MetadataTag,                                                                     // Descriptor tag
+		0x22,                                                                                // Length of bytes to follow
+		0x00, 0x10, 0x00, 0x1e, 't', 's', '=', '1', '2', '3', '4', '5', '6', '7', '8', '\t', // timestamp
+		'l', 'o', 'c', '=', '1', '2', '3', '4', ',', '4', '3', '2', '1', ',', '1', '2', '3', '4', // location
+		0x1b, 0xe1, 0x00, 0xf0, 0x00,
+	}
+	want = psi.AddCrc(want)
+	want = psi.AddPadding(want)
+	if !bytes.Equal(got, want) {
+		t.Errorf(errNotExpectedOut, got, want)
+	}
+}

From 9011c090d9fa946b1ab73780841c1ebe27ce1252 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Thu, 31 Jan 2019 13:26:16 +1030
Subject: [PATCH 47/83] stream/mts/meta.go: wrote ReadFrom func - still need to
 write tests for it

---
 stream/mts/meta.go | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/stream/mts/meta.go b/stream/mts/meta.go
index 62097d30..8bd5365b 100644
--- a/stream/mts/meta.go
+++ b/stream/mts/meta.go
@@ -29,6 +29,7 @@ package mts
 
 import (
 	"errors"
+	"strings"
 	"sync"
 )
 
@@ -131,3 +132,14 @@ func (m *Metadata) Encode() []byte {
 
 	return m.enc
 }
+
+func ReadFrom(d []byte, key string) (string, error) {
+	entries := strings.Split(string(d), "\t")
+	for _, entry := range entries {
+		kv := strings.Split(entry, "=")
+		if kv[0] == key {
+			return kv[1], nil
+		}
+	}
+	return "", errors.New("could not find key in metadata")
+}

From 10343375a30c5dcac68bc81ff905bd0561fc1612 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Thu, 31 Jan 2019 21:23:06 +1030
Subject: [PATCH 48/83] stream/mts: moved meta.go and meta_test.go into meta
 package

---
 stream/mts/meta/meta.go      | 145 +++++++++++++++++++++++++++++++++
 stream/mts/meta/meta_test.go | 150 +++++++++++++++++++++++++++++++++++
 2 files changed, 295 insertions(+)
 create mode 100644 stream/mts/meta/meta.go
 create mode 100644 stream/mts/meta/meta_test.go

diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go
new file mode 100644
index 00000000..8bd5365b
--- /dev/null
+++ b/stream/mts/meta/meta.go
@@ -0,0 +1,145 @@
+/*
+NAME
+  meta.go
+
+DESCRIPTION
+  See Readme.md
+
+AUTHOR
+  Saxon Nelson-Milton <saxon@ausocean.org>
+
+LICENSE
+  meta.go is Copyright (C) 2017-2019 the Australian Ocean Lab (AusOcean)
+
+  It is free software: you can redistribute it and/or modify them
+  under the terms of the GNU General Public License as published by the
+  Free Software Foundation, either version 3 of the License, or (at your
+  option) any later version.
+
+  It is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with revid in gpl.txt. If not, see http://www.gnu.org/licenses.
+*/
+
+package mts
+
+import (
+	"errors"
+	"strings"
+	"sync"
+)
+
+const headSize = 4
+
+const (
+	majVer = 1
+	minVer = 0
+)
+
+const (
+	dataLenIdx1 = 2
+	dataLenIdx2 = 3
+)
+
+var (
+	errKeyAbsent = errors.New("Key does not exist in map")
+)
+
+type Metadata struct {
+	mu   sync.RWMutex
+	data map[string]string
+	enc  []byte
+}
+
+func NewMeta() *Metadata {
+	return &Metadata{
+		data: make(map[string]string),
+		enc: []byte{
+			0x00,                   // Reserved byte
+			(majVer << 4) | minVer, // MS and LS versions
+			0x00,                   // Data len byte1
+			0x00,                   // Data len byte2
+		},
+	}
+}
+
+// Add adds metadata with key and val, if already exists return error
+func (m *Metadata) Add(key, val string) {
+	m.mu.Lock()
+	m.data[key] = val
+	m.mu.Unlock()
+}
+
+// All returns the a copy of the map containing the meta data
+func (m *Metadata) All() map[string]string {
+	m.mu.Lock()
+	cpy := make(map[string]string)
+	for k, v := range m.data {
+		cpy[k] = v
+	}
+	m.mu.Unlock()
+	return cpy
+}
+
+// Get returns the meta data for the passed key
+func (m *Metadata) Get(key string) (string, error) {
+	m.mu.Lock()
+	val, ok := m.data[key]
+	m.mu.Unlock()
+	if !ok {
+		return "", errKeyAbsent
+	}
+
+	return val, nil
+}
+
+// Remove deletes a meta entry in the map and returns error if it doesn’t exist
+func (m *Metadata) Delete(key string) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if _, ok := m.data[key]; ok {
+		delete(m.data, key)
+		return nil
+	}
+	return errKeyAbsent
+}
+
+// Encode takes the meta data map and encods into a byte slice with header
+// describing the version, length of data and data in TSV format.
+func (m *Metadata) Encode() []byte {
+	m.enc = m.enc[:headSize]
+
+	// Iterate over map and append entries, only adding tab if we're not on the last entry
+	var i int
+	var entry string
+	for k, v := range m.data {
+		i++
+		entry += k + "=" + v
+		if i < len(m.data) {
+			entry += "\t"
+		}
+	}
+	m.enc = append(m.enc, []byte(entry)...)
+
+	// Calculate and set data length in encoded meta header.
+	dataLen := len(m.enc[headSize:])
+	m.enc[dataLenIdx1] = byte(dataLen >> 8)
+	m.enc[dataLenIdx2] = byte(dataLen)
+
+	return m.enc
+}
+
+func ReadFrom(d []byte, key string) (string, error) {
+	entries := strings.Split(string(d), "\t")
+	for _, entry := range entries {
+		kv := strings.Split(entry, "=")
+		if kv[0] == key {
+			return kv[1], nil
+		}
+	}
+	return "", errors.New("could not find key in metadata")
+}
diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go
new file mode 100644
index 00000000..11003b9b
--- /dev/null
+++ b/stream/mts/meta/meta_test.go
@@ -0,0 +1,150 @@
+/*
+NAME
+  meta_test.go
+
+DESCRIPTION
+  See Readme.md
+
+AUTHOR
+  Saxon Nelson-Milton <saxon@ausocean.org>
+
+LICENSE
+  meta_test.go is Copyright (C) 2017-2019 the Australian Ocean Lab (AusOcean)
+
+  It is free software: you can redistribute it and/or modify them
+  under the terms of the GNU General Public License as published by the
+  Free Software Foundation, either version 3 of the License, or (at your
+  option) any later version.
+
+  It is distributed in the hope that it will be useful, but WITHOUT
+  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with revid in gpl.txt. If not, see http://www.gnu.org/licenses.
+*/
+
+package mts
+
+import (
+	"bytes"
+	"errors"
+	"reflect"
+	"testing"
+)
+
+const (
+	tstKey1  = "loc"
+	tstData1 = "a,b,c"
+	tstKey2  = "ts"
+	tstData2 = "12345678"
+	tstData3 = "d,e,f"
+)
+
+// TestAddAndGet ensures that we can add metadata and then successfully get it.
+func TestAddAndGet(t *testing.T) {
+	meta := NewMeta()
+	meta.Add(tstKey1, tstData1)
+	meta.Add(tstKey2, tstData2)
+	errors.New("Trying to delete map entry that doesn't exist")
+	if data, err := meta.Get(tstKey1); err != nil {
+		t.Errorf("Could not get data for key: loc: %v", err.Error())
+		if data != tstData1 {
+			t.Errorf("Did not get expected data")
+		}
+	}
+
+	if data, err := meta.Get(tstKey2); err != nil {
+		t.Errorf("Could not get data for key: ts: %v", err.Error())
+		if data != tstData2 {
+			t.Errorf("Did not get expected data")
+		}
+	}
+}
+
+// TestUpdate checks that we can use Meta.Add to actually update metadata
+// if it already exists in the Meta map.
+func TestUpdate(t *testing.T) {
+	meta := NewMeta()
+	meta.Add(tstKey1, tstData1)
+	meta.Add(tstKey1, tstData3)
+
+	if data, err := meta.Get(tstKey1); err != nil {
+		t.Errorf("Did not expect err: %v", err.Error())
+		if data != tstData2 {
+			t.Errorf("Data did not correctly update for key \"loc\"")
+		}
+	}
+}
+
+// TestAll ensures we can get a correct map using Meta.All() after adding some data
+func TestAll(t *testing.T) {
+	meta := NewMeta()
+	tstMap := map[string]string{
+		tstKey1: tstData1,
+		tstKey2: tstData2,
+	}
+
+	meta.Add(tstKey1, tstData1)
+	meta.Add(tstKey2, tstData2)
+	metaMap := meta.All()
+
+	if !reflect.DeepEqual(metaMap, tstMap) {
+		t.Errorf("Map not correct. Got: %v, want: %v", metaMap, tstMap)
+	}
+}
+
+// TestGetAbsentKey ensures that we get the expected error when we try to get with
+// key that does not yet exist in the Meta map.
+func TestGetAbsentKey(t *testing.T) {
+	meta := NewMeta()
+
+	if _, err := meta.Get(tstKey1); err != errKeyAbsent {
+		t.Errorf("Did not get expected err: %v", errKeyAbsent.Error())
+	}
+}
+
+// TestDelete ensures we can remove a data entry in the Meta map.
+func TestDelete(t *testing.T) {
+	meta := NewMeta()
+	meta.Add(tstKey1, tstData1)
+	if err := meta.Delete(tstKey1); err != nil {
+		t.Errorf("Did not expect error: %v", err.Error())
+	}
+	if _, err := meta.Get(tstKey1); err != errKeyAbsent {
+		t.Errorf("Did not get expected err: %v", errKeyAbsent.Error())
+	}
+}
+
+// TestDeleteAbsentKey checks that we get an expected error when we try to delete
+// an entry in the Meta map that doesn't exist.
+func TestDeleteAbsentKey(t *testing.T) {
+	meta := NewMeta()
+	if err := meta.Delete(tstKey1); err != errKeyAbsent {
+		t.Errorf("Did not get expected err: %v", errKeyAbsent.Error())
+	}
+}
+
+// TestEncode checks that we're getting the correct byte slice from Meta.Encode().
+func TestEncode(t *testing.T) {
+	meta := NewMeta()
+	meta.Add(tstKey1, tstData1)
+	meta.Add(tstKey2, tstData2)
+
+	dataLen := len(tstKey1+tstData1+tstKey2+tstData2) + 3
+	expectedOut := []byte{
+		0x00,
+		0x10,
+		byte(dataLen >> 8),
+		byte(dataLen),
+	}
+	expectedOut = append(expectedOut, []byte(
+		tstKey1+"="+tstData1+"\t"+
+			tstKey2+"="+tstData2)...)
+
+	got := meta.Encode()
+	if !bytes.Equal(expectedOut, got) {
+		t.Errorf("Did not get expected out. \nGot : %v \nwant: %v", got, expectedOut)
+	}
+}

From 1f1546a28498d720da0959748fa848509a45b4f4 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Thu, 31 Jan 2019 21:30:08 +1030
Subject: [PATCH 49/83] revid: updated code in senders.go to work with meta
 changes

---
 revid/senders.go             |   5 +-
 stream/mts/encoder.go        |   5 +-
 stream/mts/meta.go           | 145 ---------------------------------
 stream/mts/meta/meta.go      |   6 +-
 stream/mts/meta/meta_test.go |  16 ++--
 stream/mts/meta_test.go      | 150 -----------------------------------
 6 files changed, 17 insertions(+), 310 deletions(-)
 delete mode 100644 stream/mts/meta.go
 delete mode 100644 stream/mts/meta_test.go

diff --git a/revid/senders.go b/revid/senders.go
index a73523d5..e86cb24b 100644
--- a/revid/senders.go
+++ b/revid/senders.go
@@ -35,6 +35,7 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"strconv"
 
 	"bitbucket.org/ausocean/av/rtmp"
 	"bitbucket.org/ausocean/av/stream/mts"
@@ -172,7 +173,7 @@ func (s *httpSender) extractMeta(r string) error {
 		s.log(logger.Warning, pkg+"No timestamp in reply")
 	} else {
 		s.log(logger.Debug, fmt.Sprintf("%v got timestamp: %v", pkg, t))
-		mts.MetaData.SetTimeStamp(uint64(t))
+		mts.Meta.Add("ts", strconv.Itoa(t))
 	}
 
 	// Extract location from reply
@@ -181,7 +182,7 @@ func (s *httpSender) extractMeta(r string) error {
 		s.log(logger.Warning, pkg+"No location in reply")
 	} else {
 		s.log(logger.Debug, fmt.Sprintf("%v got location: %v", pkg, g))
-		mts.MetaData.SetLocation(g)
+		mts.Meta.Add("loc", g)
 	}
 
 	return nil
diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index a1527e8d..833c18d9 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -32,6 +32,7 @@ import (
 	"io"
 	"time"
 
+	"bitbucket.org/ausocean/av/stream/mts/meta"
 	"bitbucket.org/ausocean/av/stream/mts/pes"
 	"bitbucket.org/ausocean/av/stream/mts/psi"
 )
@@ -88,10 +89,10 @@ const (
 )
 
 // global Meta
-var Meta *Metadata
+var Meta *meta.Metadata
 
 func init() {
-	Meta = NewMeta()
+	Meta = meta.New()
 }
 
 var (
diff --git a/stream/mts/meta.go b/stream/mts/meta.go
deleted file mode 100644
index 8bd5365b..00000000
--- a/stream/mts/meta.go
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
-NAME
-  meta.go
-
-DESCRIPTION
-  See Readme.md
-
-AUTHOR
-  Saxon Nelson-Milton <saxon@ausocean.org>
-
-LICENSE
-  meta.go is Copyright (C) 2017-2019 the Australian Ocean Lab (AusOcean)
-
-  It is free software: you can redistribute it and/or modify them
-  under the terms of the GNU General Public License as published by the
-  Free Software Foundation, either version 3 of the License, or (at your
-  option) any later version.
-
-  It is distributed in the hope that it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with revid in gpl.txt. If not, see http://www.gnu.org/licenses.
-*/
-
-package mts
-
-import (
-	"errors"
-	"strings"
-	"sync"
-)
-
-const headSize = 4
-
-const (
-	majVer = 1
-	minVer = 0
-)
-
-const (
-	dataLenIdx1 = 2
-	dataLenIdx2 = 3
-)
-
-var (
-	errKeyAbsent = errors.New("Key does not exist in map")
-)
-
-type Metadata struct {
-	mu   sync.RWMutex
-	data map[string]string
-	enc  []byte
-}
-
-func NewMeta() *Metadata {
-	return &Metadata{
-		data: make(map[string]string),
-		enc: []byte{
-			0x00,                   // Reserved byte
-			(majVer << 4) | minVer, // MS and LS versions
-			0x00,                   // Data len byte1
-			0x00,                   // Data len byte2
-		},
-	}
-}
-
-// Add adds metadata with key and val, if already exists return error
-func (m *Metadata) Add(key, val string) {
-	m.mu.Lock()
-	m.data[key] = val
-	m.mu.Unlock()
-}
-
-// All returns the a copy of the map containing the meta data
-func (m *Metadata) All() map[string]string {
-	m.mu.Lock()
-	cpy := make(map[string]string)
-	for k, v := range m.data {
-		cpy[k] = v
-	}
-	m.mu.Unlock()
-	return cpy
-}
-
-// Get returns the meta data for the passed key
-func (m *Metadata) Get(key string) (string, error) {
-	m.mu.Lock()
-	val, ok := m.data[key]
-	m.mu.Unlock()
-	if !ok {
-		return "", errKeyAbsent
-	}
-
-	return val, nil
-}
-
-// Remove deletes a meta entry in the map and returns error if it doesn’t exist
-func (m *Metadata) Delete(key string) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	if _, ok := m.data[key]; ok {
-		delete(m.data, key)
-		return nil
-	}
-	return errKeyAbsent
-}
-
-// Encode takes the meta data map and encods into a byte slice with header
-// describing the version, length of data and data in TSV format.
-func (m *Metadata) Encode() []byte {
-	m.enc = m.enc[:headSize]
-
-	// Iterate over map and append entries, only adding tab if we're not on the last entry
-	var i int
-	var entry string
-	for k, v := range m.data {
-		i++
-		entry += k + "=" + v
-		if i < len(m.data) {
-			entry += "\t"
-		}
-	}
-	m.enc = append(m.enc, []byte(entry)...)
-
-	// Calculate and set data length in encoded meta header.
-	dataLen := len(m.enc[headSize:])
-	m.enc[dataLenIdx1] = byte(dataLen >> 8)
-	m.enc[dataLenIdx2] = byte(dataLen)
-
-	return m.enc
-}
-
-func ReadFrom(d []byte, key string) (string, error) {
-	entries := strings.Split(string(d), "\t")
-	for _, entry := range entries {
-		kv := strings.Split(entry, "=")
-		if kv[0] == key {
-			return kv[1], nil
-		}
-	}
-	return "", errors.New("could not find key in metadata")
-}
diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go
index 8bd5365b..2f26a797 100644
--- a/stream/mts/meta/meta.go
+++ b/stream/mts/meta/meta.go
@@ -25,7 +25,7 @@ LICENSE
   along with revid in gpl.txt. If not, see http://www.gnu.org/licenses.
 */
 
-package mts
+package meta
 
 import (
 	"errors"
@@ -55,7 +55,7 @@ type Metadata struct {
 	enc  []byte
 }
 
-func NewMeta() *Metadata {
+func New() *Metadata {
 	return &Metadata{
 		data: make(map[string]string),
 		enc: []byte{
@@ -67,7 +67,7 @@ func NewMeta() *Metadata {
 	}
 }
 
-// Add adds metadata with key and val, if already exists return error
+// Add adds metadata with key and val
 func (m *Metadata) Add(key, val string) {
 	m.mu.Lock()
 	m.data[key] = val
diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go
index 11003b9b..6df47e5c 100644
--- a/stream/mts/meta/meta_test.go
+++ b/stream/mts/meta/meta_test.go
@@ -25,7 +25,7 @@ LICENSE
   along with revid in gpl.txt. If not, see http://www.gnu.org/licenses.
 */
 
-package mts
+package meta
 
 import (
 	"bytes"
@@ -44,7 +44,7 @@ const (
 
 // TestAddAndGet ensures that we can add metadata and then successfully get it.
 func TestAddAndGet(t *testing.T) {
-	meta := NewMeta()
+	meta := New()
 	meta.Add(tstKey1, tstData1)
 	meta.Add(tstKey2, tstData2)
 	errors.New("Trying to delete map entry that doesn't exist")
@@ -66,7 +66,7 @@ func TestAddAndGet(t *testing.T) {
 // TestUpdate checks that we can use Meta.Add to actually update metadata
 // if it already exists in the Meta map.
 func TestUpdate(t *testing.T) {
-	meta := NewMeta()
+	meta := New()
 	meta.Add(tstKey1, tstData1)
 	meta.Add(tstKey1, tstData3)
 
@@ -80,7 +80,7 @@ func TestUpdate(t *testing.T) {
 
 // TestAll ensures we can get a correct map using Meta.All() after adding some data
 func TestAll(t *testing.T) {
-	meta := NewMeta()
+	meta := New()
 	tstMap := map[string]string{
 		tstKey1: tstData1,
 		tstKey2: tstData2,
@@ -98,7 +98,7 @@ func TestAll(t *testing.T) {
 // TestGetAbsentKey ensures that we get the expected error when we try to get with
 // key that does not yet exist in the Meta map.
 func TestGetAbsentKey(t *testing.T) {
-	meta := NewMeta()
+	meta := New()
 
 	if _, err := meta.Get(tstKey1); err != errKeyAbsent {
 		t.Errorf("Did not get expected err: %v", errKeyAbsent.Error())
@@ -107,7 +107,7 @@ func TestGetAbsentKey(t *testing.T) {
 
 // TestDelete ensures we can remove a data entry in the Meta map.
 func TestDelete(t *testing.T) {
-	meta := NewMeta()
+	meta := New()
 	meta.Add(tstKey1, tstData1)
 	if err := meta.Delete(tstKey1); err != nil {
 		t.Errorf("Did not expect error: %v", err.Error())
@@ -120,7 +120,7 @@ func TestDelete(t *testing.T) {
 // TestDeleteAbsentKey checks that we get an expected error when we try to delete
 // an entry in the Meta map that doesn't exist.
 func TestDeleteAbsentKey(t *testing.T) {
-	meta := NewMeta()
+	meta := New()
 	if err := meta.Delete(tstKey1); err != errKeyAbsent {
 		t.Errorf("Did not get expected err: %v", errKeyAbsent.Error())
 	}
@@ -128,7 +128,7 @@ func TestDeleteAbsentKey(t *testing.T) {
 
 // TestEncode checks that we're getting the correct byte slice from Meta.Encode().
 func TestEncode(t *testing.T) {
-	meta := NewMeta()
+	meta := New()
 	meta.Add(tstKey1, tstData1)
 	meta.Add(tstKey2, tstData2)
 
diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go
deleted file mode 100644
index 11003b9b..00000000
--- a/stream/mts/meta_test.go
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
-NAME
-  meta_test.go
-
-DESCRIPTION
-  See Readme.md
-
-AUTHOR
-  Saxon Nelson-Milton <saxon@ausocean.org>
-
-LICENSE
-  meta_test.go is Copyright (C) 2017-2019 the Australian Ocean Lab (AusOcean)
-
-  It is free software: you can redistribute it and/or modify them
-  under the terms of the GNU General Public License as published by the
-  Free Software Foundation, either version 3 of the License, or (at your
-  option) any later version.
-
-  It is distributed in the hope that it will be useful, but WITHOUT
-  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- for more details.
-
-  You should have received a copy of the GNU General Public License
-  along with revid in gpl.txt. If not, see http://www.gnu.org/licenses.
-*/
-
-package mts
-
-import (
-	"bytes"
-	"errors"
-	"reflect"
-	"testing"
-)
-
-const (
-	tstKey1  = "loc"
-	tstData1 = "a,b,c"
-	tstKey2  = "ts"
-	tstData2 = "12345678"
-	tstData3 = "d,e,f"
-)
-
-// TestAddAndGet ensures that we can add metadata and then successfully get it.
-func TestAddAndGet(t *testing.T) {
-	meta := NewMeta()
-	meta.Add(tstKey1, tstData1)
-	meta.Add(tstKey2, tstData2)
-	errors.New("Trying to delete map entry that doesn't exist")
-	if data, err := meta.Get(tstKey1); err != nil {
-		t.Errorf("Could not get data for key: loc: %v", err.Error())
-		if data != tstData1 {
-			t.Errorf("Did not get expected data")
-		}
-	}
-
-	if data, err := meta.Get(tstKey2); err != nil {
-		t.Errorf("Could not get data for key: ts: %v", err.Error())
-		if data != tstData2 {
-			t.Errorf("Did not get expected data")
-		}
-	}
-}
-
-// TestUpdate checks that we can use Meta.Add to actually update metadata
-// if it already exists in the Meta map.
-func TestUpdate(t *testing.T) {
-	meta := NewMeta()
-	meta.Add(tstKey1, tstData1)
-	meta.Add(tstKey1, tstData3)
-
-	if data, err := meta.Get(tstKey1); err != nil {
-		t.Errorf("Did not expect err: %v", err.Error())
-		if data != tstData2 {
-			t.Errorf("Data did not correctly update for key \"loc\"")
-		}
-	}
-}
-
-// TestAll ensures we can get a correct map using Meta.All() after adding some data
-func TestAll(t *testing.T) {
-	meta := NewMeta()
-	tstMap := map[string]string{
-		tstKey1: tstData1,
-		tstKey2: tstData2,
-	}
-
-	meta.Add(tstKey1, tstData1)
-	meta.Add(tstKey2, tstData2)
-	metaMap := meta.All()
-
-	if !reflect.DeepEqual(metaMap, tstMap) {
-		t.Errorf("Map not correct. Got: %v, want: %v", metaMap, tstMap)
-	}
-}
-
-// TestGetAbsentKey ensures that we get the expected error when we try to get with
-// key that does not yet exist in the Meta map.
-func TestGetAbsentKey(t *testing.T) {
-	meta := NewMeta()
-
-	if _, err := meta.Get(tstKey1); err != errKeyAbsent {
-		t.Errorf("Did not get expected err: %v", errKeyAbsent.Error())
-	}
-}
-
-// TestDelete ensures we can remove a data entry in the Meta map.
-func TestDelete(t *testing.T) {
-	meta := NewMeta()
-	meta.Add(tstKey1, tstData1)
-	if err := meta.Delete(tstKey1); err != nil {
-		t.Errorf("Did not expect error: %v", err.Error())
-	}
-	if _, err := meta.Get(tstKey1); err != errKeyAbsent {
-		t.Errorf("Did not get expected err: %v", errKeyAbsent.Error())
-	}
-}
-
-// TestDeleteAbsentKey checks that we get an expected error when we try to delete
-// an entry in the Meta map that doesn't exist.
-func TestDeleteAbsentKey(t *testing.T) {
-	meta := NewMeta()
-	if err := meta.Delete(tstKey1); err != errKeyAbsent {
-		t.Errorf("Did not get expected err: %v", errKeyAbsent.Error())
-	}
-}
-
-// TestEncode checks that we're getting the correct byte slice from Meta.Encode().
-func TestEncode(t *testing.T) {
-	meta := NewMeta()
-	meta.Add(tstKey1, tstData1)
-	meta.Add(tstKey2, tstData2)
-
-	dataLen := len(tstKey1+tstData1+tstKey2+tstData2) + 3
-	expectedOut := []byte{
-		0x00,
-		0x10,
-		byte(dataLen >> 8),
-		byte(dataLen),
-	}
-	expectedOut = append(expectedOut, []byte(
-		tstKey1+"="+tstData1+"\t"+
-			tstKey2+"="+tstData2)...)
-
-	got := meta.Encode()
-	if !bytes.Equal(expectedOut, got) {
-		t.Errorf("Did not get expected out. \nGot : %v \nwant: %v", got, expectedOut)
-	}
-}

From 38d5d6f0fdea7985dc8726bf9148af3da13ea657 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Fri, 1 Feb 2019 09:27:43 +1030
Subject: [PATCH 50/83] stream/mts/meta/meta_test.go: adding test for ReadFrom

---
 stream/mts/meta/meta_test.go | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go
index 6df47e5c..d40e26d5 100644
--- a/stream/mts/meta/meta_test.go
+++ b/stream/mts/meta/meta_test.go
@@ -42,6 +42,10 @@ const (
 	tstData3 = "d,e,f"
 )
 
+var (
+	errNotExpectedOut = "Did not get expected out. \nGot : %v, \nwant: %v"
+)
+
 // TestAddAndGet ensures that we can add metadata and then successfully get it.
 func TestAddAndGet(t *testing.T) {
 	meta := New()
@@ -145,6 +149,26 @@ func TestEncode(t *testing.T) {
 
 	got := meta.Encode()
 	if !bytes.Equal(expectedOut, got) {
-		t.Errorf("Did not get expected out. \nGot : %v \nwant: %v", got, expectedOut)
+		t.Errorf(errNotExpectedOut, got, expectedOut)
+	}
+}
+
+func TestReadFrom(t *testing.T) {
+	tstStr := "loc=a,b,c\tts=12345"
+	got, err := ReadFrom([]byte(tstStr), "loc")
+	if err != nil {
+		t.Errorf("Got unexpected error: %v", err.Error())
+	}
+	want := "a,b,c"
+	if got != want {
+		t.Errorf(errNotExpectedOut, got, want)
+	}
+
+	if got, err = ReadFrom([]byte(tstStr), "ts"); err != nil {
+		t.Errorf("Got unexpected error: %v", err.Error())
+	}
+	want = "12345"
+	if got != want {
+		t.Errorf(errNotExpectedOut, got, want)
 	}
 }

From 4d4a8e04ec8b4a3019c4e451ae42cf4831c2c0e3 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Fri, 1 Feb 2019 11:17:53 +1030
Subject: [PATCH 51/83] stream/mts/meta/meta_test.go: created global err vars

---
 stream/mts/meta/meta_test.go | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go
index d40e26d5..0fcda13b 100644
--- a/stream/mts/meta/meta_test.go
+++ b/stream/mts/meta/meta_test.go
@@ -43,7 +43,9 @@ const (
 )
 
 var (
-	errNotExpectedOut = "Did not get expected out. \nGot : %v, \nwant: %v"
+	errNotExpectedOut = "Did not get expected out. \nGot : %v, \nwant: %v\n"
+	errUnexpectedErr  = "Unexpected err: %v\n"
+	errNotExpectedErr = "Not expected err: %v\n"
 )
 
 // TestAddAndGet ensures that we can add metadata and then successfully get it.
@@ -75,7 +77,7 @@ func TestUpdate(t *testing.T) {
 	meta.Add(tstKey1, tstData3)
 
 	if data, err := meta.Get(tstKey1); err != nil {
-		t.Errorf("Did not expect err: %v", err.Error())
+		t.Errorf(errUnexpectedErr, err.Error())
 		if data != tstData2 {
 			t.Errorf("Data did not correctly update for key \"loc\"")
 		}
@@ -105,7 +107,7 @@ func TestGetAbsentKey(t *testing.T) {
 	meta := New()
 
 	if _, err := meta.Get(tstKey1); err != errKeyAbsent {
-		t.Errorf("Did not get expected err: %v", errKeyAbsent.Error())
+		t.Errorf(errNotExpectedErr, errKeyAbsent.Error())
 	}
 }
 
@@ -114,10 +116,10 @@ func TestDelete(t *testing.T) {
 	meta := New()
 	meta.Add(tstKey1, tstData1)
 	if err := meta.Delete(tstKey1); err != nil {
-		t.Errorf("Did not expect error: %v", err.Error())
+		t.Errorf(errUnexpectedErr, err.Error())
 	}
 	if _, err := meta.Get(tstKey1); err != errKeyAbsent {
-		t.Errorf("Did not get expected err: %v", errKeyAbsent.Error())
+		t.Errorf(errNotExpectedErr, errKeyAbsent.Error())
 	}
 }
 
@@ -126,7 +128,7 @@ func TestDelete(t *testing.T) {
 func TestDeleteAbsentKey(t *testing.T) {
 	meta := New()
 	if err := meta.Delete(tstKey1); err != errKeyAbsent {
-		t.Errorf("Did not get expected err: %v", errKeyAbsent.Error())
+		t.Errorf(errNotExpectedErr, errKeyAbsent.Error())
 	}
 }
 
@@ -157,7 +159,7 @@ func TestReadFrom(t *testing.T) {
 	tstStr := "loc=a,b,c\tts=12345"
 	got, err := ReadFrom([]byte(tstStr), "loc")
 	if err != nil {
-		t.Errorf("Got unexpected error: %v", err.Error())
+		t.Errorf(errUnexpectedErr, err.Error())
 	}
 	want := "a,b,c"
 	if got != want {
@@ -165,7 +167,7 @@ func TestReadFrom(t *testing.T) {
 	}
 
 	if got, err = ReadFrom([]byte(tstStr), "ts"); err != nil {
-		t.Errorf("Got unexpected error: %v", err.Error())
+		t.Errorf(errUnexpectedErr, err.Error())
 	}
 	want = "12345"
 	if got != want {

From eb5487140272059bc4d6d0102e9a68df6e3cddf2 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Fri, 1 Feb 2019 11:39:47 +1030
Subject: [PATCH 52/83] stream/mts/encoder.go: simplified updateMeta

---
 stream/mts/encoder.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index 833c18d9..cc4a699a 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -257,9 +257,8 @@ func (e *Encoder) ccFor(pid int) byte {
 
 // updateMeta ...
 func updateMeta(b *[]byte) error {
-	m := Meta.Encode()
 	p := psi.PSIBytes(*b)
-	err := p.AddDescriptor(psi.MetadataTag, m)
+	err := p.AddDescriptor(psi.MetadataTag, Meta.Encode())
 	*b = []byte(p)
 	return err
 }

From 1570974ab7cfa84716be3879d75726cf3dec62ba Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Mon, 4 Feb 2019 22:14:17 +1030
Subject: [PATCH 53/83] stream/mts/encoder.go: removed init func where Meta was
 being initialised - just doing initialisation on same line

---
 stream/mts/encoder.go | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index cc4a699a..6b406536 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -89,11 +89,7 @@ const (
 )
 
 // global Meta
-var Meta *meta.Metadata
-
-func init() {
-	Meta = meta.New()
-}
+var Meta = meta.New()
 
 var (
 	patTable = standardPat.Bytes()

From 0a96d18a10a7c65b0b171f28c9e1fd07809a8b30 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Mon, 4 Feb 2019 22:17:39 +1030
Subject: [PATCH 54/83] stream/mts/encoder.go: finished comment for updateMeta
 function

---
 stream/mts/encoder.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index 6b406536..093dbee5 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -251,7 +251,8 @@ func (e *Encoder) ccFor(pid int) byte {
 	return cc
 }
 
-// updateMeta ...
+// updateMeta adds/updates a metaData descriptor in the given psi bytes using data
+// contained in the global Meta struct.
 func updateMeta(b *[]byte) error {
 	p := psi.PSIBytes(*b)
 	err := p.AddDescriptor(psi.MetadataTag, Meta.Encode())

From 953d363b3a50a1ee9d5c98da0d50e338b66a11c5 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Mon, 4 Feb 2019 22:38:11 +1030
Subject: [PATCH 55/83] stream/mts/meta.go: improved const and function
 commenting

---
 stream/mts/meta/meta.go | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go
index 2f26a797..1e4c72b8 100644
--- a/stream/mts/meta/meta.go
+++ b/stream/mts/meta/meta.go
@@ -33,6 +33,8 @@ import (
 	"sync"
 )
 
+// This is the headsize of our metadata string,
+// which is encoded int the data body of a pmt descriptor.
 const headSize = 4
 
 const (
@@ -40,6 +42,7 @@ const (
 	minVer = 0
 )
 
+// Indices of bytes for uint16 metadata length.
 const (
 	dataLenIdx1 = 2
 	dataLenIdx2 = 3
@@ -49,12 +52,15 @@ var (
 	errKeyAbsent = errors.New("Key does not exist in map")
 )
 
+// Metadata provides functionality for the storage and encoding of metadata
+// using a map.
 type Metadata struct {
 	mu   sync.RWMutex
 	data map[string]string
 	enc  []byte
 }
 
+// New returns a pointer to a new Metadata.
 func New() *Metadata {
 	return &Metadata{
 		data: make(map[string]string),
@@ -67,14 +73,14 @@ func New() *Metadata {
 	}
 }
 
-// Add adds metadata with key and val
+// Add adds metadata with key and val.
 func (m *Metadata) Add(key, val string) {
 	m.mu.Lock()
 	m.data[key] = val
 	m.mu.Unlock()
 }
 
-// All returns the a copy of the map containing the meta data
+// All returns the a copy of the map containing the meta data.
 func (m *Metadata) All() map[string]string {
 	m.mu.Lock()
 	cpy := make(map[string]string)
@@ -85,7 +91,7 @@ func (m *Metadata) All() map[string]string {
 	return cpy
 }
 
-// Get returns the meta data for the passed key
+// Get returns the meta data for the passed key.
 func (m *Metadata) Get(key string) (string, error) {
 	m.mu.Lock()
 	val, ok := m.data[key]
@@ -93,11 +99,10 @@ func (m *Metadata) Get(key string) (string, error) {
 	if !ok {
 		return "", errKeyAbsent
 	}
-
 	return val, nil
 }
 
-// Remove deletes a meta entry in the map and returns error if it doesn’t exist
+// Delete deletes a meta entry in the map and returns error if it doesn’t exist.
 func (m *Metadata) Delete(key string) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
@@ -108,12 +113,13 @@ func (m *Metadata) Delete(key string) error {
 	return errKeyAbsent
 }
 
-// Encode takes the meta data map and encods into a byte slice with header
+// Encode takes the meta data map and encodes into a byte slice with header
 // describing the version, length of data and data in TSV format.
 func (m *Metadata) Encode() []byte {
 	m.enc = m.enc[:headSize]
 
-	// Iterate over map and append entries, only adding tab if we're not on the last entry
+	// Iterate over map and append entries, only adding tab if we're not on the
+	// last entry.
 	var i int
 	var entry string
 	for k, v := range m.data {
@@ -133,6 +139,9 @@ func (m *Metadata) Encode() []byte {
 	return m.enc
 }
 
+// ReadFrom extracts a value from a metadata string d, for the given key. If the
+// key is not present in the metadata string, an error is returned. If the
+// metadata header is not present in the string, an error is returned.
 func ReadFrom(d []byte, key string) (string, error) {
 	entries := strings.Split(string(d), "\t")
 	for _, entry := range entries {
@@ -141,5 +150,5 @@ func ReadFrom(d []byte, key string) (string, error) {
 			return kv[1], nil
 		}
 	}
-	return "", errors.New("could not find key in metadata")
+	return "", errKeyAbsent
 }

From ad25785b77f6b06a83bbe34ad8d8fa60b3c9aaf5 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Mon, 4 Feb 2019 22:48:51 +1030
Subject: [PATCH 56/83] stream/mts/meta/meta_test.go: improved ReadFrom by
 checking for valid header

---
 stream/mts/meta/meta.go      | 11 ++++++++++-
 stream/mts/meta/meta_test.go |  6 +++---
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go
index 1e4c72b8..7a863d0c 100644
--- a/stream/mts/meta/meta.go
+++ b/stream/mts/meta/meta.go
@@ -28,6 +28,7 @@ LICENSE
 package meta
 
 import (
+	"encoding/binary"
 	"errors"
 	"strings"
 	"sync"
@@ -49,7 +50,9 @@ const (
 )
 
 var (
-	errKeyAbsent = errors.New("Key does not exist in map")
+	errKeyAbsent     = errors.New("Key does not exist in map")
+	errNoHeader      = errors.New("Metadata string does not contain header")
+	errInvalidHeader = errors.New("Metadata string does not contain valid header")
 )
 
 // Metadata provides functionality for the storage and encoding of metadata
@@ -143,6 +146,12 @@ func (m *Metadata) Encode() []byte {
 // key is not present in the metadata string, an error is returned. If the
 // metadata header is not present in the string, an error is returned.
 func ReadFrom(d []byte, key string) (string, error) {
+	if d[0] != 0 {
+		return "", errNoHeader
+	} else if d[0] == 0 && binary.BigEndian.Uint16(d[2:headSize]) != uint16(len(d[headSize:])) {
+		return "", errInvalidHeader
+	}
+	d = d[headSize:]
 	entries := strings.Split(string(d), "\t")
 	for _, entry := range entries {
 		kv := strings.Split(entry, "=")
diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go
index 0fcda13b..15789ebf 100644
--- a/stream/mts/meta/meta_test.go
+++ b/stream/mts/meta/meta_test.go
@@ -156,8 +156,8 @@ func TestEncode(t *testing.T) {
 }
 
 func TestReadFrom(t *testing.T) {
-	tstStr := "loc=a,b,c\tts=12345"
-	got, err := ReadFrom([]byte(tstStr), "loc")
+	tstMeta := append([]byte{0x00, 0x10, 0x00, 0x12}, "loc=a,b,c\tts=12345"...)
+	got, err := ReadFrom([]byte(tstMeta), "loc")
 	if err != nil {
 		t.Errorf(errUnexpectedErr, err.Error())
 	}
@@ -166,7 +166,7 @@ func TestReadFrom(t *testing.T) {
 		t.Errorf(errNotExpectedOut, got, want)
 	}
 
-	if got, err = ReadFrom([]byte(tstStr), "ts"); err != nil {
+	if got, err = ReadFrom([]byte(tstMeta), "ts"); err != nil {
 		t.Errorf(errUnexpectedErr, err.Error())
 	}
 	want = "12345"

From 834625a9f75705b19878a9d6b280952712589b89 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 5 Feb 2019 12:55:32 +1030
Subject: [PATCH 57/83] stream/mts/meta: using order slice so that encoded
 order reamins consistent with order that we added metadata. And also now
 using binary.BigEndian.PutUint16() to encode len into metdata rather than
 doing ugly byte manipulation

---
 stream/mts/meta/meta.go | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go
index 7a863d0c..eaa0713e 100644
--- a/stream/mts/meta/meta.go
+++ b/stream/mts/meta/meta.go
@@ -45,8 +45,7 @@ const (
 
 // Indices of bytes for uint16 metadata length.
 const (
-	dataLenIdx1 = 2
-	dataLenIdx2 = 3
+	dataLenIdx = 2
 )
 
 var (
@@ -58,9 +57,10 @@ var (
 // Metadata provides functionality for the storage and encoding of metadata
 // using a map.
 type Metadata struct {
-	mu   sync.RWMutex
-	data map[string]string
-	enc  []byte
+	mu    sync.RWMutex
+	data  map[string]string
+	order []string
+	enc   []byte
 }
 
 // New returns a pointer to a new Metadata.
@@ -80,6 +80,7 @@ func New() *Metadata {
 func (m *Metadata) Add(key, val string) {
 	m.mu.Lock()
 	m.data[key] = val
+	m.order = append(m.order, key)
 	m.mu.Unlock()
 }
 
@@ -111,6 +112,12 @@ func (m *Metadata) Delete(key string) error {
 	defer m.mu.Unlock()
 	if _, ok := m.data[key]; ok {
 		delete(m.data, key)
+		for i, k := range m.order {
+			if k == key {
+				m.order = append(m.order[:i], m.order[i+1:]...)
+				break
+			}
+		}
 		return nil
 	}
 	return errKeyAbsent
@@ -123,12 +130,11 @@ func (m *Metadata) Encode() []byte {
 
 	// Iterate over map and append entries, only adding tab if we're not on the
 	// last entry.
-	var i int
 	var entry string
-	for k, v := range m.data {
-		i++
+	for i, k := range m.order {
+		v := m.data[k]
 		entry += k + "=" + v
-		if i < len(m.data) {
+		if i+1 < len(m.data) {
 			entry += "\t"
 		}
 	}
@@ -136,9 +142,7 @@ func (m *Metadata) Encode() []byte {
 
 	// Calculate and set data length in encoded meta header.
 	dataLen := len(m.enc[headSize:])
-	m.enc[dataLenIdx1] = byte(dataLen >> 8)
-	m.enc[dataLenIdx2] = byte(dataLen)
-
+	binary.BigEndian.PutUint16(m.enc[dataLenIdx:dataLenIdx+2], uint16(dataLen))
 	return m.enc
 }
 

From 3364b4ea6f1ad5d6d19dd045c4767f645bf61a66 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 5 Feb 2019 13:11:11 +1030
Subject: [PATCH 58/83] stream/mts/meta.go: avoiding adding same key to order
 slice

---
 stream/mts/meta/meta.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go
index eaa0713e..a0e6e3e3 100644
--- a/stream/mts/meta/meta.go
+++ b/stream/mts/meta/meta.go
@@ -80,6 +80,12 @@ func New() *Metadata {
 func (m *Metadata) Add(key, val string) {
 	m.mu.Lock()
 	m.data[key] = val
+	for _, k := range m.order {
+		if k == key {
+			m.mu.Unlock()
+			return
+		}
+	}
 	m.order = append(m.order, key)
 	m.mu.Unlock()
 }

From 6425403fcb8ad5e3de656328edf29564cf0f52ac Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 5 Feb 2019 13:14:42 +1030
Subject: [PATCH 59/83] stream/mts/encoder.go: adding copyright meta first
 thing

---
 stream/mts/encoder.go | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index 093dbee5..6a0c55d3 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -88,9 +88,18 @@ const (
 	psiSndCnt = 7
 )
 
+const (
+	copyright = "copyright"
+	license   = "ausocean.org/license/content2019"
+)
+
 // global Meta
 var Meta = meta.New()
 
+func init() {
+	Meta.Add(copyright, license)
+}
+
 var (
 	patTable = standardPat.Bytes()
 	pmtTable = standardPmt.Bytes()

From dc46d9f0b28e088454d79c17dc75bca0fe93cb58 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 5 Feb 2019 23:18:05 +1030
Subject: [PATCH 60/83] stream/mts/encoder.go: not doing scoped conditional in
 writePSI when calling updateMeta

---
 stream/mts/encoder.go        |  4 ++--
 stream/mts/meta/meta_test.go | 10 ++++------
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index 6a0c55d3..0e52cb70 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -216,8 +216,8 @@ func (e *Encoder) writePSI() error {
 	if err != nil {
 		return err
 	}
-
-	if err = updateMeta(&pmtTable); err != nil {
+	err = updateMeta(&pmtTable)
+	if err != nil {
 		return err
 	}
 
diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go
index 15789ebf..afd38544 100644
--- a/stream/mts/meta/meta_test.go
+++ b/stream/mts/meta/meta_test.go
@@ -29,9 +29,9 @@ package meta
 
 import (
 	"bytes"
-	"errors"
 	"reflect"
 	"testing"
+	"encoding/binary"
 )
 
 const (
@@ -53,7 +53,6 @@ func TestAddAndGet(t *testing.T) {
 	meta := New()
 	meta.Add(tstKey1, tstData1)
 	meta.Add(tstKey2, tstData2)
-	errors.New("Trying to delete map entry that doesn't exist")
 	if data, err := meta.Get(tstKey1); err != nil {
 		t.Errorf("Could not get data for key: loc: %v", err.Error())
 		if data != tstData1 {
@@ -139,13 +138,12 @@ func TestEncode(t *testing.T) {
 	meta.Add(tstKey2, tstData2)
 
 	dataLen := len(tstKey1+tstData1+tstKey2+tstData2) + 3
-	expectedOut := []byte{
+	header := [4]byte{
 		0x00,
 		0x10,
-		byte(dataLen >> 8),
-		byte(dataLen),
 	}
-	expectedOut = append(expectedOut, []byte(
+	binary.BigEndian.PutUint16(header[2:4],uint16(dataLen))
+	expectedOut := append(header[:], []byte(
 		tstKey1+"="+tstData1+"\t"+
 			tstKey2+"="+tstData2)...)
 

From 6b32064a2bdab48752ab1007b3b11051bd82f6f6 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 5 Feb 2019 23:24:00 +1030
Subject: [PATCH 61/83] stream/mts/encoder.go: changed updateMeta signature to
 return bytes slice rather than passing in pointer to byte slice.

---
 stream/mts/encoder.go | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index 0e52cb70..7bda82c7 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -216,7 +216,7 @@ func (e *Encoder) writePSI() error {
 	if err != nil {
 		return err
 	}
-	err = updateMeta(&pmtTable)
+	pmtTable, err = updateMeta(pmtTable)
 	if err != nil {
 		return err
 	}
@@ -262,9 +262,8 @@ func (e *Encoder) ccFor(pid int) byte {
 
 // updateMeta adds/updates a metaData descriptor in the given psi bytes using data
 // contained in the global Meta struct.
-func updateMeta(b *[]byte) error {
-	p := psi.PSIBytes(*b)
+func updateMeta(b []byte) ([]byte, error) {
+	p := psi.PSIBytes(b)
 	err := p.AddDescriptor(psi.MetadataTag, Meta.Encode())
-	*b = []byte(p)
-	return err
+	return []byte(p), err
 }

From cf73c853b0734d201f5e2e573d75146782e078b3 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 5 Feb 2019 23:25:46 +1030
Subject: [PATCH 62/83] stream/mts/meta/meta.go: using defer for m.mu.Unlock()
 Meta.Add()

---
 stream/mts/meta/meta.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go
index a0e6e3e3..466cb8e8 100644
--- a/stream/mts/meta/meta.go
+++ b/stream/mts/meta/meta.go
@@ -79,15 +79,15 @@ func New() *Metadata {
 // Add adds metadata with key and val.
 func (m *Metadata) Add(key, val string) {
 	m.mu.Lock()
+	defer m.mu.Unlock()
 	m.data[key] = val
 	for _, k := range m.order {
 		if k == key {
-			m.mu.Unlock()
 			return
 		}
 	}
 	m.order = append(m.order, key)
-	m.mu.Unlock()
+	return
 }
 
 // All returns the a copy of the map containing the meta data.

From 89b05b3cb662bfcf6af315a2dba58433ad5789b5 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 5 Feb 2019 23:29:07 +1030
Subject: [PATCH 63/83] stream/mts/meta/meta.go: using copy instead of append
 to remove key from order slice in Meta.Delete()

---
 stream/mts/meta/meta.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go
index 466cb8e8..bc3a6da6 100644
--- a/stream/mts/meta/meta.go
+++ b/stream/mts/meta/meta.go
@@ -120,7 +120,8 @@ func (m *Metadata) Delete(key string) error {
 		delete(m.data, key)
 		for i, k := range m.order {
 			if k == key {
-				m.order = append(m.order[:i], m.order[i+1:]...)
+				copy(m.order[:i], m.order[i+1:])
+				m.order = m.order[:len(m.order)-1]
 				break
 			}
 		}

From 35c06c97aa56b3887d4ea70885841e95b49a1b46 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Tue, 5 Feb 2019 23:57:53 +1030
Subject: [PATCH 64/83] stream/mts/meta/meta_test.go: not using global vars for
 errors

---
 stream/mts/meta/meta_test.go | 30 ++++++++++++------------------
 1 file changed, 12 insertions(+), 18 deletions(-)

diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go
index afd38544..6d98fe0f 100644
--- a/stream/mts/meta/meta_test.go
+++ b/stream/mts/meta/meta_test.go
@@ -29,9 +29,9 @@ package meta
 
 import (
 	"bytes"
+	"encoding/binary"
 	"reflect"
 	"testing"
-	"encoding/binary"
 )
 
 const (
@@ -42,12 +42,6 @@ const (
 	tstData3 = "d,e,f"
 )
 
-var (
-	errNotExpectedOut = "Did not get expected out. \nGot : %v, \nwant: %v\n"
-	errUnexpectedErr  = "Unexpected err: %v\n"
-	errNotExpectedErr = "Not expected err: %v\n"
-)
-
 // TestAddAndGet ensures that we can add metadata and then successfully get it.
 func TestAddAndGet(t *testing.T) {
 	meta := New()
@@ -76,7 +70,7 @@ func TestUpdate(t *testing.T) {
 	meta.Add(tstKey1, tstData3)
 
 	if data, err := meta.Get(tstKey1); err != nil {
-		t.Errorf(errUnexpectedErr, err.Error())
+		t.Errorf("Unexpected err: %v\n", err.Error())
 		if data != tstData2 {
 			t.Errorf("Data did not correctly update for key \"loc\"")
 		}
@@ -106,7 +100,7 @@ func TestGetAbsentKey(t *testing.T) {
 	meta := New()
 
 	if _, err := meta.Get(tstKey1); err != errKeyAbsent {
-		t.Errorf(errNotExpectedErr, errKeyAbsent.Error())
+		t.Errorf("Not expected err: %v\n", errKeyAbsent.Error())
 	}
 }
 
@@ -115,10 +109,10 @@ func TestDelete(t *testing.T) {
 	meta := New()
 	meta.Add(tstKey1, tstData1)
 	if err := meta.Delete(tstKey1); err != nil {
-		t.Errorf(errUnexpectedErr, err.Error())
+		t.Errorf("Unexpected err: %v\n", err.Error())
 	}
 	if _, err := meta.Get(tstKey1); err != errKeyAbsent {
-		t.Errorf(errNotExpectedErr, errKeyAbsent.Error())
+		t.Errorf("Not expected err: %v\n", errKeyAbsent.Error())
 	}
 }
 
@@ -127,7 +121,7 @@ func TestDelete(t *testing.T) {
 func TestDeleteAbsentKey(t *testing.T) {
 	meta := New()
 	if err := meta.Delete(tstKey1); err != errKeyAbsent {
-		t.Errorf(errNotExpectedErr, errKeyAbsent.Error())
+		t.Errorf("Not expected err: %v\n", errKeyAbsent.Error())
 	}
 }
 
@@ -142,14 +136,14 @@ func TestEncode(t *testing.T) {
 		0x00,
 		0x10,
 	}
-	binary.BigEndian.PutUint16(header[2:4],uint16(dataLen))
+	binary.BigEndian.PutUint16(header[2:4], uint16(dataLen))
 	expectedOut := append(header[:], []byte(
 		tstKey1+"="+tstData1+"\t"+
 			tstKey2+"="+tstData2)...)
 
 	got := meta.Encode()
 	if !bytes.Equal(expectedOut, got) {
-		t.Errorf(errNotExpectedOut, got, expectedOut)
+		t.Errorf("Did not get expected out. \nGot : %v, \nwant: %v\n", got, expectedOut)
 	}
 }
 
@@ -157,18 +151,18 @@ func TestReadFrom(t *testing.T) {
 	tstMeta := append([]byte{0x00, 0x10, 0x00, 0x12}, "loc=a,b,c\tts=12345"...)
 	got, err := ReadFrom([]byte(tstMeta), "loc")
 	if err != nil {
-		t.Errorf(errUnexpectedErr, err.Error())
+		t.Errorf("Unexpected err: %v\n", err.Error())
 	}
 	want := "a,b,c"
 	if got != want {
-		t.Errorf(errNotExpectedOut, got, want)
+		t.Errorf("Did not get expected out. \nGot : %v, \nwant: %v\n", got, want)
 	}
 
 	if got, err = ReadFrom([]byte(tstMeta), "ts"); err != nil {
-		t.Errorf(errUnexpectedErr, err.Error())
+		t.Errorf("Unexpected err: %v\n", err.Error())
 	}
 	want = "12345"
 	if got != want {
-		t.Errorf(errNotExpectedOut, got, want)
+		t.Errorf("Did not get expected out. \nGot : %v, \nwant: %v\n", got, want)
 	}
 }

From 9b5eb558ef3e6c0f02bc2679850ac3d1b0007c95 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Wed, 6 Feb 2019 00:01:17 +1030
Subject: [PATCH 65/83] stream/mts/meta/meta_test.go: Updated func comment for
 TestReadFrom

---
 stream/mts/meta/meta_test.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go
index 6d98fe0f..700fda5b 100644
--- a/stream/mts/meta/meta_test.go
+++ b/stream/mts/meta/meta_test.go
@@ -147,6 +147,8 @@ func TestEncode(t *testing.T) {
 	}
 }
 
+// TestReadFrom checks that we can correctly obtain a value for a partiular key
+// from a string of metadata using the ReadFrom func.
 func TestReadFrom(t *testing.T) {
 	tstMeta := append([]byte{0x00, 0x10, 0x00, 0x12}, "loc=a,b,c\tts=12345"...)
 	got, err := ReadFrom([]byte(tstMeta), "loc")

From d5236776275492df202ff1fec459f088727f3b8c Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Wed, 6 Feb 2019 09:59:55 +1030
Subject: [PATCH 66/83] stream/mts/meta/meta.go: renamed Metadata struct to
 Data

---
 stream/mts/meta/meta.go | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go
index bc3a6da6..1e2182c0 100644
--- a/stream/mts/meta/meta.go
+++ b/stream/mts/meta/meta.go
@@ -56,7 +56,7 @@ var (
 
 // Metadata provides functionality for the storage and encoding of metadata
 // using a map.
-type Metadata struct {
+type Data struct {
 	mu    sync.RWMutex
 	data  map[string]string
 	order []string
@@ -64,8 +64,8 @@ type Metadata struct {
 }
 
 // New returns a pointer to a new Metadata.
-func New() *Metadata {
-	return &Metadata{
+func New() *Data {
+	return &Data{
 		data: make(map[string]string),
 		enc: []byte{
 			0x00,                   // Reserved byte
@@ -77,7 +77,7 @@ func New() *Metadata {
 }
 
 // Add adds metadata with key and val.
-func (m *Metadata) Add(key, val string) {
+func (m *Data) Add(key, val string) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	m.data[key] = val
@@ -91,7 +91,7 @@ func (m *Metadata) Add(key, val string) {
 }
 
 // All returns the a copy of the map containing the meta data.
-func (m *Metadata) All() map[string]string {
+func (m *Data) All() map[string]string {
 	m.mu.Lock()
 	cpy := make(map[string]string)
 	for k, v := range m.data {
@@ -102,7 +102,7 @@ func (m *Metadata) All() map[string]string {
 }
 
 // Get returns the meta data for the passed key.
-func (m *Metadata) Get(key string) (string, error) {
+func (m *Data) Get(key string) (string, error) {
 	m.mu.Lock()
 	val, ok := m.data[key]
 	m.mu.Unlock()
@@ -113,7 +113,7 @@ func (m *Metadata) Get(key string) (string, error) {
 }
 
 // Delete deletes a meta entry in the map and returns error if it doesn’t exist.
-func (m *Metadata) Delete(key string) error {
+func (m *Data) Delete(key string) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	if _, ok := m.data[key]; ok {
@@ -132,7 +132,7 @@ func (m *Metadata) Delete(key string) error {
 
 // Encode takes the meta data map and encodes into a byte slice with header
 // describing the version, length of data and data in TSV format.
-func (m *Metadata) Encode() []byte {
+func (m *Data) Encode() []byte {
 	m.enc = m.enc[:headSize]
 
 	// Iterate over map and append entries, only adding tab if we're not on the

From 2b7ab27763e882c2c4d10f02464bb325ee881e7a Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Wed, 6 Feb 2019 10:19:12 +1030
Subject: [PATCH 67/83] cmd/revid-cli & stream/mts/meta: added meta.NewWith
 function that creates a new meta.Data and adds an initial entry. Using
 meta.NewWith in revid-cli to initialise mts' global meta.Data with some
 preamble data.

---
 cmd/revid-cli/main.go   | 11 +++++++++++
 stream/mts/encoder.go   | 11 +----------
 stream/mts/meta/meta.go | 14 ++++++++++++++
 3 files changed, 26 insertions(+), 10 deletions(-)

diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go
index 5b826b91..d7f79721 100644
--- a/cmd/revid-cli/main.go
+++ b/cmd/revid-cli/main.go
@@ -37,6 +37,8 @@ import (
 	"time"
 
 	"bitbucket.org/ausocean/av/revid"
+	"bitbucket.org/ausocean/av/stream/mts"
+	"bitbucket.org/ausocean/av/stream/mts/meta"
 	"bitbucket.org/ausocean/iot/pi/netsender"
 	"bitbucket.org/ausocean/iot/pi/smartlogger"
 	"bitbucket.org/ausocean/utils/logger"
@@ -65,7 +67,16 @@ var canProfile = true
 // The logger that will be used throughout
 var log *logger.Logger
 
+var (
+	metaPreambleKey  = []byte{0x63, 0x6f, 0x70, 0x79, 0x72, 0x69, 0x67, 0x68, 0x74}
+	metaPreambleData = []byte{0x61, 0x75, 0x73, 0x6f, 0x63, 0x65, 0x61, 0x6e, 0x2e,
+		0x6f, 0x72, 0x67, 0x2f, 0x6c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x2f, 0x63,
+		0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x32, 0x30, 0x31, 0x39}
+)
+
 func main() {
+	mts.Meta = meta.NewWith(string(metaPreambleKey), string(metaPreambleData))
+
 	useNetsender := flag.Bool("NetSender", false, "Are we checking vars through netsender?")
 	runDurationPtr := flag.Duration("runDuration", defaultRunDuration, "How long do you want revid to run for?")
 
diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index 7bda82c7..2ebe5dc6 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -88,17 +88,8 @@ const (
 	psiSndCnt = 7
 )
 
-const (
-	copyright = "copyright"
-	license   = "ausocean.org/license/content2019"
-)
-
 // global Meta
-var Meta = meta.New()
-
-func init() {
-	Meta.Add(copyright, license)
-}
+var Meta *meta.Data
 
 var (
 	patTable = standardPat.Bytes()
diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go
index 1e2182c0..97725252 100644
--- a/stream/mts/meta/meta.go
+++ b/stream/mts/meta/meta.go
@@ -76,6 +76,20 @@ func New() *Data {
 	}
 }
 
+func NewWith(key, data string) *Data {
+	meta := &Data{
+		data: make(map[string]string),
+		enc: []byte{
+			0x00,                   // Reserved byte
+			(majVer << 4) | minVer, // MS and LS versions
+			0x00,                   // Data len byte1
+			0x00,                   // Data len byte2
+		},
+	}
+	meta.Add(key, data)
+	return meta
+}
+
 // Add adds metadata with key and val.
 func (m *Data) Add(key, val string) {
 	m.mu.Lock()

From 57d1dba2fb809d8f8874842138b773abdeb7267d Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Wed, 6 Feb 2019 12:16:44 +1030
Subject: [PATCH 68/83] stream/mts/meta/meta.go: updated meta.NewWith so that
 it just uses meta.New, and now ti can add an abitrary number of things to the
 map. It also overwrites keys that have been repeated

---
 stream/mts/meta/meta.go | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go
index 97725252..a2d56e1a 100644
--- a/stream/mts/meta/meta.go
+++ b/stream/mts/meta/meta.go
@@ -76,18 +76,18 @@ func New() *Data {
 	}
 }
 
-func NewWith(key, data string) *Data {
-	meta := &Data{
-		data: make(map[string]string),
-		enc: []byte{
-			0x00,                   // Reserved byte
-			(majVer << 4) | minVer, // MS and LS versions
-			0x00,                   // Data len byte1
-			0x00,                   // Data len byte2
-		},
+// NewWith creates a meta.Data and fills map with initial data given. If there
+// is repeated key, then the latter overwrites the prior.
+func NewWith(data [][2]string) *Data {
+	m := New()
+	m.order = make([]string, 0, len(data))
+	for _, d := range data {
+		if _, exists := m.data[d[0]]; !exists {
+			m.order = append(m.order, d[0])
+		}
+		m.data[d[0]] = d[1]
 	}
-	meta.Add(key, data)
-	return meta
+	return m
 }
 
 // Add adds metadata with key and val.

From e2bbc84c1ecdc976df5a3b215eb16bb4d4ce81cb Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Thu, 7 Feb 2019 08:25:37 +1030
Subject: [PATCH 69/83] cmd/revid-cli: using const strings for copyright
 metadata preamble

---
 cmd/revid-cli/main.go | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go
index d7f79721..d9c737e1 100644
--- a/cmd/revid-cli/main.go
+++ b/cmd/revid-cli/main.go
@@ -67,15 +67,13 @@ var canProfile = true
 // The logger that will be used throughout
 var log *logger.Logger
 
-var (
-	metaPreambleKey  = []byte{0x63, 0x6f, 0x70, 0x79, 0x72, 0x69, 0x67, 0x68, 0x74}
-	metaPreambleData = []byte{0x61, 0x75, 0x73, 0x6f, 0x63, 0x65, 0x61, 0x6e, 0x2e,
-		0x6f, 0x72, 0x67, 0x2f, 0x6c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x2f, 0x63,
-		0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x32, 0x30, 0x31, 0x39}
+const (
+	metaPreambleKey  = "copyright"
+	metaPreambleData = "ausocean.org/license/content2019"
 )
 
 func main() {
-	mts.Meta = meta.NewWith(string(metaPreambleKey), string(metaPreambleData))
+	mts.Meta = meta.NewWith([][2]string{{metaPreambleKey, metaPreambleData}})
 
 	useNetsender := flag.Bool("NetSender", false, "Are we checking vars through netsender?")
 	runDurationPtr := flag.Duration("runDuration", defaultRunDuration, "How long do you want revid to run for?")

From fb2217a1f90c5b75fd88c0613c93f07856d7e327 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Thu, 7 Feb 2019 08:27:37 +1030
Subject: [PATCH 70/83] stream/mts/encoder.go: wrote todo to make Meta
 meta.Data struct not be global

---
 stream/mts/encoder.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index 2ebe5dc6..29aa83fc 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -88,7 +88,7 @@ const (
 	psiSndCnt = 7
 )
 
-// global Meta
+// TODO: make this not global.
 var Meta *meta.Data
 
 var (

From 7cb58c81ebcb850ed598fe9a0643757620d27961 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Thu, 7 Feb 2019 08:29:51 +1030
Subject: [PATCH 71/83] stream/mts/encoder.go: added comment to Meta meta.Data
 global as it unexported.

---
 stream/mts/encoder.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index 29aa83fc..e6d10a61 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -89,6 +89,8 @@ const (
 )
 
 // TODO: make this not global.
+// Meta allows addition of metadata to encoded mts from outside of this pkg.
+// See meta pkg for usage.
 var Meta *meta.Data
 
 var (

From ad671b520c28f2d634eb7cfa04ee083e8307825f Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Thu, 7 Feb 2019 08:31:02 +1030
Subject: [PATCH 72/83] stream/mts/encoder.go: put todo under comment for Meta
 meta.Data global

---
 stream/mts/encoder.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go
index e6d10a61..6ff552b8 100644
--- a/stream/mts/encoder.go
+++ b/stream/mts/encoder.go
@@ -88,9 +88,10 @@ const (
 	psiSndCnt = 7
 )
 
-// TODO: make this not global.
 // Meta allows addition of metadata to encoded mts from outside of this pkg.
 // See meta pkg for usage.
+//
+// TODO: make this not global.
 var Meta *meta.Data
 
 var (

From f96f761b2f3eaabf5a58d3617f552c3e9418b241 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Thu, 7 Feb 2019 09:01:40 +1030
Subject: [PATCH 73/83] stream/mts/meta/meta_test.go: fixed up error logging in
 meta_test.go i.e. not doing err.Error() for %v format specifiers, and not
 using t.Errorf when we have no arguments

---
 stream/mts/meta/meta_test.go | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go
index 700fda5b..2be41b95 100644
--- a/stream/mts/meta/meta_test.go
+++ b/stream/mts/meta/meta_test.go
@@ -48,16 +48,16 @@ func TestAddAndGet(t *testing.T) {
 	meta.Add(tstKey1, tstData1)
 	meta.Add(tstKey2, tstData2)
 	if data, err := meta.Get(tstKey1); err != nil {
-		t.Errorf("Could not get data for key: loc: %v", err.Error())
+		t.Errorf("Could not get data for key: loc: %v", err)
 		if data != tstData1 {
-			t.Errorf("Did not get expected data")
+			t.Error("Did not get expected data")
 		}
 	}
 
 	if data, err := meta.Get(tstKey2); err != nil {
-		t.Errorf("Could not get data for key: ts: %v", err.Error())
+		t.Errorf("Could not get data for key: ts: %v", err)
 		if data != tstData2 {
-			t.Errorf("Did not get expected data")
+			t.Error("Did not get expected data")
 		}
 	}
 }
@@ -70,9 +70,9 @@ func TestUpdate(t *testing.T) {
 	meta.Add(tstKey1, tstData3)
 
 	if data, err := meta.Get(tstKey1); err != nil {
-		t.Errorf("Unexpected err: %v\n", err.Error())
+		t.Errorf("Unexpected err: %v\n", err)
 		if data != tstData2 {
-			t.Errorf("Data did not correctly update for key \"loc\"")
+			t.Error(`Data did not correctly update for key "loc"`)
 		}
 	}
 }
@@ -100,7 +100,7 @@ func TestGetAbsentKey(t *testing.T) {
 	meta := New()
 
 	if _, err := meta.Get(tstKey1); err != errKeyAbsent {
-		t.Errorf("Not expected err: %v\n", errKeyAbsent.Error())
+		t.Errorf("Not expected err: %v\n", errKeyAbsent)
 	}
 }
 
@@ -109,10 +109,10 @@ func TestDelete(t *testing.T) {
 	meta := New()
 	meta.Add(tstKey1, tstData1)
 	if err := meta.Delete(tstKey1); err != nil {
-		t.Errorf("Unexpected err: %v\n", err.Error())
+		t.Errorf("Unexpected err: %v\n", err)
 	}
 	if _, err := meta.Get(tstKey1); err != errKeyAbsent {
-		t.Errorf("Not expected err: %v\n", errKeyAbsent.Error())
+		t.Errorf("Not expected err. got: %v\n, want: $%v\n", err, errKeyAbsent)
 	}
 }
 
@@ -121,7 +121,7 @@ func TestDelete(t *testing.T) {
 func TestDeleteAbsentKey(t *testing.T) {
 	meta := New()
 	if err := meta.Delete(tstKey1); err != errKeyAbsent {
-		t.Errorf("Not expected err: %v\n", errKeyAbsent.Error())
+		t.Errorf("Not expected err: %v\n", errKeyAbsent)
 	}
 }
 
@@ -153,7 +153,7 @@ func TestReadFrom(t *testing.T) {
 	tstMeta := append([]byte{0x00, 0x10, 0x00, 0x12}, "loc=a,b,c\tts=12345"...)
 	got, err := ReadFrom([]byte(tstMeta), "loc")
 	if err != nil {
-		t.Errorf("Unexpected err: %v\n", err.Error())
+		t.Errorf("Unexpected err: %v\n", err)
 	}
 	want := "a,b,c"
 	if got != want {
@@ -161,7 +161,7 @@ func TestReadFrom(t *testing.T) {
 	}
 
 	if got, err = ReadFrom([]byte(tstMeta), "ts"); err != nil {
-		t.Errorf("Unexpected err: %v\n", err.Error())
+		t.Errorf("Unexpected err: %v\n", err)
 	}
 	want = "12345"
 	if got != want {

From 66a2325dcb659b179585f4ab34130654366f0250 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Thu, 7 Feb 2019 10:42:01 +1030
Subject: [PATCH 74/83] stream/mts/meta/meta_test.go: using table of tests for
 TestReadFrom

---
 stream/mts/meta/meta_test.go | 34 +++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go
index 2be41b95..002bbc8f 100644
--- a/stream/mts/meta/meta_test.go
+++ b/stream/mts/meta/meta_test.go
@@ -151,20 +151,28 @@ func TestEncode(t *testing.T) {
 // from a string of metadata using the ReadFrom func.
 func TestReadFrom(t *testing.T) {
 	tstMeta := append([]byte{0x00, 0x10, 0x00, 0x12}, "loc=a,b,c\tts=12345"...)
-	got, err := ReadFrom([]byte(tstMeta), "loc")
-	if err != nil {
-		t.Errorf("Unexpected err: %v\n", err)
-	}
-	want := "a,b,c"
-	if got != want {
-		t.Errorf("Did not get expected out. \nGot : %v, \nwant: %v\n", got, want)
+
+	tests := []struct {
+		key  string
+		want string
+	}{
+		{
+			"loc",
+			"a,b,c",
+		},
+		{
+			"ts",
+			"12345",
+		},
 	}
 
-	if got, err = ReadFrom([]byte(tstMeta), "ts"); err != nil {
-		t.Errorf("Unexpected err: %v\n", err)
-	}
-	want = "12345"
-	if got != want {
-		t.Errorf("Did not get expected out. \nGot : %v, \nwant: %v\n", got, want)
+	for _, test := range tests {
+		got, err := ReadFrom([]byte(tstMeta), test.key)
+		if err != nil {
+			t.Errorf("Unexpected err: %v\n", err)
+		}
+		if got != test.want {
+			t.Errorf("Did not get expected out. \nGot : %v, \nwant: %v\n", got, test.want)
+		}
 	}
 }

From bb032778c6b680211b6a1ecea2818743f8fa725f Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Thu, 7 Feb 2019 10:48:25 +1030
Subject: [PATCH 75/83] stream/mts/metaEncode_test.go: fixed metaEncode_test.go
 by initialising Meta global in tests

---
 stream/mts/metaEncode_test.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/stream/mts/metaEncode_test.go b/stream/mts/metaEncode_test.go
index d372f20a..c3c23131 100644
--- a/stream/mts/metaEncode_test.go
+++ b/stream/mts/metaEncode_test.go
@@ -31,6 +31,7 @@ import (
 	"bytes"
 	"testing"
 
+	"bitbucket.org/ausocean/av/stream/mts/meta"
 	"bitbucket.org/ausocean/av/stream/mts/psi"
 )
 
@@ -42,6 +43,7 @@ const (
 const fps = 25
 
 func TestMetaEncode1(t *testing.T) {
+	Meta = meta.New()
 	var b []byte
 	buf := bytes.NewBuffer(b)
 	e := NewEncoder(buf, fps)
@@ -67,6 +69,7 @@ func TestMetaEncode1(t *testing.T) {
 }
 
 func TestMetaEncode2(t *testing.T) {
+	Meta = meta.New()
 	var b []byte
 	buf := bytes.NewBuffer(b)
 	e := NewEncoder(buf, fps)

From 7a7be5580d74127366784606da7b28404e253770 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Thu, 7 Feb 2019 10:55:58 +1030
Subject: [PATCH 76/83] stream/mts/metaEncode_test.go: wrote function comments
 for tests

---
 stream/mts/metaEncode_test.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/stream/mts/metaEncode_test.go b/stream/mts/metaEncode_test.go
index c3c23131..e970b7c8 100644
--- a/stream/mts/metaEncode_test.go
+++ b/stream/mts/metaEncode_test.go
@@ -42,6 +42,9 @@ const (
 
 const fps = 25
 
+// TestMetaEncode1 checks that we can externally add a single metadata entry to
+// the mts global Meta meta.Data struct and then successfully have the mts encoder
+// write this to psi.
 func TestMetaEncode1(t *testing.T) {
 	Meta = meta.New()
 	var b []byte
@@ -68,6 +71,9 @@ func TestMetaEncode1(t *testing.T) {
 	}
 }
 
+// TestMetaEncode2 checks that we can externally add two metadata entries to the
+// Meta meta.Data global and then have the mts encoder successfully encode this
+// into psi.
 func TestMetaEncode2(t *testing.T) {
 	Meta = meta.New()
 	var b []byte

From 46b5acb48c4e9fd7b0e1f0afa430ed538253d839 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Thu, 7 Feb 2019 13:59:43 +1030
Subject: [PATCH 77/83] stream/mts/mpgets.go: removing unused consts

---
 stream/mts/mpegts.go | 24 ------------------------
 1 file changed, 24 deletions(-)

diff --git a/stream/mts/mpegts.go b/stream/mts/mpegts.go
index 71849513..ab23b9a7 100644
--- a/stream/mts/mpegts.go
+++ b/stream/mts/mpegts.go
@@ -60,30 +60,6 @@ const (
 	HasAdaptationField = 0x2
 )
 
-// Adaptation field body masks.
-const (
-	DiscontinuityIndicatorMask            = 0x80
-	RandomAccessIndicatorMask             = 0x40
-	ElementaryStreamPriorityIndicatorMask = 0x20
-	ProgramClockReferenceFlagMask         = 0x10
-	OriginalProgramClockReferenceFlagMask = 0x08
-	SplicingPointFlagMask                 = 0x04
-	TransportPrivateDataFlagMask          = 0x02
-	AdaptationFieldExtensionMask          = 0x01
-)
-
-// Adaptation field body indexes.
-const (
-	DiscontinuityIndicatorIdx            = AdaptationIdx + 1
-	RandomAccessIndicatorIdx             = AdaptationIdx + 1
-	ElementaryStreamPriorityIndicatorIdx = AdaptationIdx + 1
-	ProgramClockReferenceFlagIdx         = AdaptationIdx + 1
-	OriginalProgramClockReferenceFlagIdx = AdaptationIdx + 1
-	SplicingPointFlagIdx                 = AdaptationIdx + 1
-	TransportPrivateDataFlagIdx          = AdaptationIdx + 1
-	AdaptationFieldExtensionFlagIdx      = AdaptationIdx + 1
-)
-
 /*
 The below data struct encapsulates the fields of an MPEG-TS packet. Below is
 the formatting of an MPEG-TS packet for reference!

From b8fc6d7e9902fc6a32b29016807e1122ff1fd835 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Thu, 7 Feb 2019 14:50:31 +1030
Subject: [PATCH 78/83] stream/mts/meta/meta.go: ReadFrom to Extract

---
 stream/mts/meta/meta.go      | 2 +-
 stream/mts/meta/meta_test.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go
index a2d56e1a..bb8b1161 100644
--- a/stream/mts/meta/meta.go
+++ b/stream/mts/meta/meta.go
@@ -170,7 +170,7 @@ func (m *Data) Encode() []byte {
 // ReadFrom extracts a value from a metadata string d, for the given key. If the
 // key is not present in the metadata string, an error is returned. If the
 // metadata header is not present in the string, an error is returned.
-func ReadFrom(d []byte, key string) (string, error) {
+func Extract(key string, d []byte) (string, error) {
 	if d[0] != 0 {
 		return "", errNoHeader
 	} else if d[0] == 0 && binary.BigEndian.Uint16(d[2:headSize]) != uint16(len(d[headSize:])) {
diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go
index 002bbc8f..cb770883 100644
--- a/stream/mts/meta/meta_test.go
+++ b/stream/mts/meta/meta_test.go
@@ -167,7 +167,7 @@ func TestReadFrom(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		got, err := ReadFrom([]byte(tstMeta), test.key)
+		got, err := Extract(test.key, []byte(tstMeta))
 		if err != nil {
 			t.Errorf("Unexpected err: %v\n", err)
 		}

From 21265303d77ce3019dd99f0ce2e3881ddb51478a Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Thu, 7 Feb 2019 15:06:54 +1030
Subject: [PATCH 79/83] stream/mts/mpegts.go: reorganised exported constants
 and commented

---
 stream/mts/mpegts.go | 31 +++++++++++++++++++------------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/stream/mts/mpegts.go b/stream/mts/mpegts.go
index ab23b9a7..e18213b5 100644
--- a/stream/mts/mpegts.go
+++ b/stream/mts/mpegts.go
@@ -32,29 +32,36 @@ import (
 	"errors"
 )
 
+// General mpegts packet properties.
 const (
 	PacketSize  = 188
 	PayloadSize = 176
 )
 
+// Program ID for various types of ts packets.
 const (
-	SdtPid                = 17
-	PatPid                = 0
-	PmtPid                = 4096
-	VideoPid              = 256
-	StreamID              = 0xe0 // First video stream ID.
-	HeadSize              = 4
-	DefaultAdaptationSize = 2
+	SdtPid   = 17
+	PatPid   = 0
+	PmtPid   = 4096
+	VideoPid = 256
 )
 
+// StreamID is the id of the first stream.
+const StreamID = 0xe0
+
+// HeadSize is the size of an mpegts packet header.
+const HeadSize = 4
+
+// Consts relating to adaptation field.
 const (
-	AdaptationIdx             = 4
-	AdaptationControlIdx      = 3
-	AdaptationBodyIdx         = AdaptationIdx + 1
-	AdaptationControlMask     = 0x30
-	DefaultAdaptationBodySize = 1
+	AdaptationIdx         = 4                 // Index to the adaptation field (index of AFL).
+	AdaptationControlIdx  = 3                 // Index to octet with adaptation field control.
+	AdaptationFieldsIdx   = AdaptationIdx + 1 // Adaptation field index is the index of the adaptation fields.
+	DefaultAdaptationSize = 2                 // Default size of the adaptation field.
+	AdaptationControlMask = 0x30              // Mask for the adaptation field control in octet 3.
 )
 
+// TODO: make this better - currently doesn't make sense.
 const (
 	HasPayload         = 0x1
 	HasAdaptationField = 0x2

From b4393e5136be5bc53a9d6718eb292759ccc01043 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Thu, 7 Feb 2019 15:23:41 +1030
Subject: [PATCH 80/83] stream/mts/psi/psi.go: checking that we have enough
 space in psi before creating descriptor

---
 stream/mts/psi/psi.go | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index 91d8fade..cc9dc8f9 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -52,6 +52,8 @@ const (
 	pmtID = 0x02
 )
 
+const TotalSyntaxSecLen = 180
+
 // Consts relating to time description
 const (
 	TimeDescTag  = 234
@@ -251,8 +253,8 @@ func (p *PSIBytes) AddDescriptor(tag int, data []byte) error {
 
 	i, desc := p.HasDescriptor(tag)
 	if desc == nil {
-		p.createDescriptor(tag, data)
-		return nil
+		err := p.createDescriptor(tag, data)
+		return err
 	}
 
 	oldDescLen := desc.len()
@@ -308,9 +310,12 @@ func (p *PSIBytes) HasDescriptor(tag int) (int, Descriptor) {
 // createDescriptor creates a descriptor in a psi given a tag and data. It does so
 // by resizing the psi, shifting existing data down and copying in new descriptor
 // in new space.
-func (p *PSIBytes) createDescriptor(tag int, data []byte) {
+func (p *PSIBytes) createDescriptor(tag int, data []byte) error {
 	curProgLen := p.ProgramInfoLen()
 	oldSyntaxSectionLen := SyntaxSecLenFrom(*p)
+	if TotalSyntaxSecLen-(oldSyntaxSectionLen+2+len(data)) <= 0 {
+		return errors.New("Not enough space in psi to create descriptor.")
+	}
 	dataLen := len(data)
 	newDescIdx := DescriptorsIdx + curProgLen
 	newDescLen := dataLen + 2
@@ -332,6 +337,8 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) {
 	newSyntaxSectionLen := int(oldSyntaxSectionLen) + addedLen
 	p.setSectionLen(newSyntaxSectionLen)
 	UpdateCrc((*p)[1:])
+
+	return nil
 }
 
 // setProgInfoLen sets the program information length in a psi with a pmt.

From 330fdc6aa88862b5d17f75544bdca713b793f172 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Thu, 7 Feb 2019 15:48:35 +1030
Subject: [PATCH 81/83] stream/mts/psi/psi.go: cleaned up and comment consts

---
 stream/mts/psi/psi.go | 38 +++++++++++++++++---------------------
 1 file changed, 17 insertions(+), 21 deletions(-)

diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go
index cc9dc8f9..c93d3011 100644
--- a/stream/mts/psi/psi.go
+++ b/stream/mts/psi/psi.go
@@ -32,11 +32,10 @@ import (
 	"github.com/Comcast/gots/psi"
 )
 
-const (
-	PacketSize = 184 // packet size of a psi.
-)
+// PacketSize of psi (without mpegts header)
+const PacketSize = 184
 
-// Lengths of section definitions
+// Lengths of section definitions.
 const (
 	ESSDDefLen = 5
 	DescDefLen = 2
@@ -46,15 +45,14 @@ const (
 	PSIDefLen  = 3
 )
 
-// Table Type IDs
+// Table Type IDs.
 const (
 	patID = 0x00
 	pmtID = 0x02
 )
 
-const TotalSyntaxSecLen = 180
-
 // Consts relating to time description
+// TODO: remove this, we don't do metadata like this anymore.
 const (
 	TimeDescTag  = 234
 	TimeTagIndx  = 13
@@ -63,6 +61,7 @@ const (
 )
 
 // Consts relating to location description
+// TODO: remove this, we don't do metadata like this anymore.
 const (
 	LocationDescTag  = 235
 	LocationTagIndx  = 23
@@ -70,35 +69,32 @@ const (
 	LocationDataSize = 32 // bytes
 )
 
-// Other misc consts
+// crc hassh Size
+const crcSize = 4
+
+// Consts relating to syntax section.
 const (
+	TotalSyntaxSecLen = 180
 	SyntaxSecLenIdx1  = 2
 	SyntaxSecLenIdx2  = 3
 	SyntaxSecLenMask1 = 0x03
-	crcSize           = 4
-)
-
-const (
-	SectionLenIdx1 = 2
-	SectionLenIdx2 = 3
-)
-
-const (
-	SectionLenMask1 = 0x03
+	SectionLenMask1   = 0x03
 )
 
+// Consts relating to program info len.
 const (
 	ProgramInfoLenIdx1  = 11
 	ProgramInfoLenIdx2  = 12
 	ProgramInfoLenMask1 = 0x03
 )
 
-const (
-	DescriptorsIdx = ProgramInfoLenIdx2 + 1
-)
+// DescriptorsIdx is the index that the descriptors start at.
+const DescriptorsIdx = ProgramInfoLenIdx2 + 1
 
+// MetadataTag is the descriptor tag used for metadata.
 const MetadataTag = 0x26
 
+// TODO: get rid of these - not a good idea.
 type (
 	PSIBytes   []byte
 	Descriptor []byte

From a94bdbfe473e4b41137cfbb90cce0e6b51c1c721 Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Fri, 8 Feb 2019 10:25:57 +1030
Subject: [PATCH 82/83] stream/mts/meta: meta.Get now returns ok bool rather
 than error - updated usage accordingly

---
 stream/mts/meta/meta.go      |  9 +++------
 stream/mts/meta/meta_test.go | 20 ++++++++++----------
 2 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go
index bb8b1161..297b2fa2 100644
--- a/stream/mts/meta/meta.go
+++ b/stream/mts/meta/meta.go
@@ -116,14 +116,11 @@ func (m *Data) All() map[string]string {
 }
 
 // Get returns the meta data for the passed key.
-func (m *Data) Get(key string) (string, error) {
+func (m *Data) Get(key string) (val string, ok bool) {
 	m.mu.Lock()
-	val, ok := m.data[key]
+	val, ok = m.data[key]
 	m.mu.Unlock()
-	if !ok {
-		return "", errKeyAbsent
-	}
-	return val, nil
+	return
 }
 
 // Delete deletes a meta entry in the map and returns error if it doesn’t exist.
diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go
index cb770883..a97d925f 100644
--- a/stream/mts/meta/meta_test.go
+++ b/stream/mts/meta/meta_test.go
@@ -47,15 +47,15 @@ func TestAddAndGet(t *testing.T) {
 	meta := New()
 	meta.Add(tstKey1, tstData1)
 	meta.Add(tstKey2, tstData2)
-	if data, err := meta.Get(tstKey1); err != nil {
-		t.Errorf("Could not get data for key: loc: %v", err)
+	if data, ok := meta.Get(tstKey1); !ok {
+		t.Errorf("Could not get data for key: %v\n", tstKey1)
 		if data != tstData1 {
 			t.Error("Did not get expected data")
 		}
 	}
 
-	if data, err := meta.Get(tstKey2); err != nil {
-		t.Errorf("Could not get data for key: ts: %v", err)
+	if data, ok := meta.Get(tstKey2); !ok {
+		t.Errorf("Could not get data for key: %v", tstKey2)
 		if data != tstData2 {
 			t.Error("Did not get expected data")
 		}
@@ -69,8 +69,8 @@ func TestUpdate(t *testing.T) {
 	meta.Add(tstKey1, tstData1)
 	meta.Add(tstKey1, tstData3)
 
-	if data, err := meta.Get(tstKey1); err != nil {
-		t.Errorf("Unexpected err: %v\n", err)
+	if data, ok := meta.Get(tstKey1); !ok {
+		t.Errorf("Could not get data for key: %v\n", tstKey1)
 		if data != tstData2 {
 			t.Error(`Data did not correctly update for key "loc"`)
 		}
@@ -99,8 +99,8 @@ func TestAll(t *testing.T) {
 func TestGetAbsentKey(t *testing.T) {
 	meta := New()
 
-	if _, err := meta.Get(tstKey1); err != errKeyAbsent {
-		t.Errorf("Not expected err: %v\n", errKeyAbsent)
+	if _, ok := meta.Get(tstKey1); ok {
+		t.Error("Get for absent key incorrectly returned'ok'")
 	}
 }
 
@@ -111,8 +111,8 @@ func TestDelete(t *testing.T) {
 	if err := meta.Delete(tstKey1); err != nil {
 		t.Errorf("Unexpected err: %v\n", err)
 	}
-	if _, err := meta.Get(tstKey1); err != errKeyAbsent {
-		t.Errorf("Not expected err. got: %v\n, want: $%v\n", err, errKeyAbsent)
+	if _, ok := meta.Get(tstKey1); ok {
+		t.Error("Get incorrectly returned okay for absent key")
 	}
 }
 

From db3b34c10f1fe78f171f4d43f4ef4cefb5e71edb Mon Sep 17 00:00:00 2001
From: saxon <saxon.milton@gmail.com>
Date: Fri, 8 Feb 2019 10:56:19 +1030
Subject: [PATCH 83/83] stream/mts/meta: meta.Delete no longer returns error -
 updated code accordingly

---
 stream/mts/meta/meta.go      |  6 +++---
 stream/mts/meta/meta_test.go | 13 +------------
 2 files changed, 4 insertions(+), 15 deletions(-)

diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go
index 297b2fa2..66790315 100644
--- a/stream/mts/meta/meta.go
+++ b/stream/mts/meta/meta.go
@@ -124,7 +124,7 @@ func (m *Data) Get(key string) (val string, ok bool) {
 }
 
 // Delete deletes a meta entry in the map and returns error if it doesn’t exist.
-func (m *Data) Delete(key string) error {
+func (m *Data) Delete(key string) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	if _, ok := m.data[key]; ok {
@@ -136,9 +136,9 @@ func (m *Data) Delete(key string) error {
 				break
 			}
 		}
-		return nil
+		return
 	}
-	return errKeyAbsent
+	return
 }
 
 // Encode takes the meta data map and encodes into a byte slice with header
diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go
index a97d925f..e1f9f3b7 100644
--- a/stream/mts/meta/meta_test.go
+++ b/stream/mts/meta/meta_test.go
@@ -108,23 +108,12 @@ func TestGetAbsentKey(t *testing.T) {
 func TestDelete(t *testing.T) {
 	meta := New()
 	meta.Add(tstKey1, tstData1)
-	if err := meta.Delete(tstKey1); err != nil {
-		t.Errorf("Unexpected err: %v\n", err)
-	}
+	meta.Delete(tstKey1)
 	if _, ok := meta.Get(tstKey1); ok {
 		t.Error("Get incorrectly returned okay for absent key")
 	}
 }
 
-// TestDeleteAbsentKey checks that we get an expected error when we try to delete
-// an entry in the Meta map that doesn't exist.
-func TestDeleteAbsentKey(t *testing.T) {
-	meta := New()
-	if err := meta.Delete(tstKey1); err != errKeyAbsent {
-		t.Errorf("Not expected err: %v\n", errKeyAbsent)
-	}
-}
-
 // TestEncode checks that we're getting the correct byte slice from Meta.Encode().
 func TestEncode(t *testing.T) {
 	meta := New()