From 56422a366e7b4961780777f845c56631648ae0c6 Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 21 Jan 2019 14:57:40 +1030 Subject: [PATCH 001/208] av/stream/flac: added decode.go and flac_test.go --- stream/flac/decode.go | 0 stream/flac/flac_test.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 stream/flac/decode.go create mode 100644 stream/flac/flac_test.go diff --git a/stream/flac/decode.go b/stream/flac/decode.go new file mode 100644 index 00000000..e69de29b diff --git a/stream/flac/flac_test.go b/stream/flac/flac_test.go new file mode 100644 index 00000000..e69de29b From 44c79e225673f3f7fd74a420273d23f94227ec91 Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 21 Jan 2019 15:41:49 +1030 Subject: [PATCH 002/208] av/stream/flac: wrote decode function and test to see if we can get wav. --- stream/flac/decode.go | 108 +++++++++++++++++++++++++++++++++++++++ stream/flac/flac_test.go | 31 +++++++++++ 2 files changed, 139 insertions(+) diff --git a/stream/flac/decode.go b/stream/flac/decode.go index e69de29b..2ecfb737 100644 --- a/stream/flac/decode.go +++ b/stream/flac/decode.go @@ -0,0 +1,108 @@ +package flac + +import ( + "bytes" + "errors" + "io" + + "github.com/go-audio/audio" + "github.com/go-audio/wav" + "github.com/mewkiz/flac" +) + +const wavAudioFormat = 1 + +type buffer struct { + Buffer bytes.Buffer + Index int64 +} + +func (b *buffer) Bytes() []byte { + return b.Buffer.Bytes() +} + +func (b *buffer) Read(p []byte) (int, error) { + n, err := bytes.NewBuffer(b.Buffer.Bytes()[b.Index:]).Read(p) + + if err == nil { + if b.Index+int64(len(p)) < int64(b.Buffer.Len()) { + b.Index += int64(len(p)) + } else { + b.Index = int64(b.Buffer.Len()) + } + } + + return n, err +} + +func (b *buffer) Write(p []byte) (int, error) { + n, err := b.Buffer.Write(p) + + if err == nil { + b.Index = int64(b.Buffer.Len()) + } + + return n, err +} + +func (b *buffer) Seek(offset int64, whence int) (int64, error) { + var err error + var Index int64 = 0 + + switch whence { + case 0: + if offset >= int64(b.Buffer.Len()) || offset < 0 { + err = errors.New("Invalid Offset.") + } else { + b.Index = offset + Index = offset + } + default: + err = errors.New("Unsupported Seek Method.") + } + + return Index, err +} + +// Decode takes a slice of flac and decodes to wav +func Decode(buf []byte) ([]byte, error) { + r := bytes.NewReader(buf) + stream, err := flac.Parse(r) + if err != nil { + return nil, errors.New("Could not parse FLAC") + } + fb := &buffer{} + enc := wav.NewEncoder(fb, int(stream.Info.SampleRate), int(stream.Info.BitsPerSample), int(stream.Info.NChannels), wavAudioFormat) + defer enc.Close() + var data []int + for { + // Decode FLAC audio samples. + frame, err := stream.ParseNext() + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + + // Encode WAV audio samples. + data = data[:0] + for i := 0; i < frame.Subframes[0].NSamples; i++ { + for _, subframe := range frame.Subframes { + data = append(data, int(subframe.Samples[i])) + } + } + buf := &audio.IntBuffer{ + Format: &audio.Format{ + NumChannels: int(stream.Info.NChannels), + SampleRate: int(stream.Info.SampleRate), + }, + Data: data, + SourceBitDepth: int(stream.Info.BitsPerSample), + } + if err := enc.Write(buf); err != nil { + return nil, err + } + } + return fb.Bytes(), nil +} diff --git a/stream/flac/flac_test.go b/stream/flac/flac_test.go index e69de29b..763731d4 100644 --- a/stream/flac/flac_test.go +++ b/stream/flac/flac_test.go @@ -0,0 +1,31 @@ +package flac + +import ( + "io/ioutil" + "os" + "testing" +) + +const ( + testFile = "/home/saxon/Desktop/robot.flac" + outFile = "out.wav" +) + +func TestDecodeFlac(t *testing.T) { + b, err := ioutil.ReadFile(testFile) + if err != nil { + t.Fatalf("Could not read test file, failed with err: %v", err.Error()) + } + out, err := Decode(b) + if err != nil { + t.Errorf("Could not decode, failed with err: %v", err.Error()) + } + f, err := os.Create(outFile) + if err != nil { + t.Fatalf("Could not create output file, failed with err: %v", err.Error()) + } + _, err = f.Write(out) + if err != nil { + t.Fatalf("Could not write to output file, failed with err: %v", err.Error()) + } +} From 6fda0b3c3fe91515c792cac3e29124090f7dcf18 Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 21 Jan 2019 17:34:15 +1030 Subject: [PATCH 003/208] av/stream/flac: using writerseeker to pass to wav.NewEncoder because I don't want to give it a file, but it's not working --- stream/flac/decode.go | 68 ++++++++-------------------------------- stream/flac/flac_test.go | 2 +- 2 files changed, 14 insertions(+), 56 deletions(-) diff --git a/stream/flac/decode.go b/stream/flac/decode.go index 2ecfb737..941610e2 100644 --- a/stream/flac/decode.go +++ b/stream/flac/decode.go @@ -4,66 +4,16 @@ import ( "bytes" "errors" "io" + "io/ioutil" "github.com/go-audio/audio" "github.com/go-audio/wav" "github.com/mewkiz/flac" + "github.com/orcaman/writerseeker" ) const wavAudioFormat = 1 -type buffer struct { - Buffer bytes.Buffer - Index int64 -} - -func (b *buffer) Bytes() []byte { - return b.Buffer.Bytes() -} - -func (b *buffer) Read(p []byte) (int, error) { - n, err := bytes.NewBuffer(b.Buffer.Bytes()[b.Index:]).Read(p) - - if err == nil { - if b.Index+int64(len(p)) < int64(b.Buffer.Len()) { - b.Index += int64(len(p)) - } else { - b.Index = int64(b.Buffer.Len()) - } - } - - return n, err -} - -func (b *buffer) Write(p []byte) (int, error) { - n, err := b.Buffer.Write(p) - - if err == nil { - b.Index = int64(b.Buffer.Len()) - } - - return n, err -} - -func (b *buffer) Seek(offset int64, whence int) (int64, error) { - var err error - var Index int64 = 0 - - switch whence { - case 0: - if offset >= int64(b.Buffer.Len()) || offset < 0 { - err = errors.New("Invalid Offset.") - } else { - b.Index = offset - Index = offset - } - default: - err = errors.New("Unsupported Seek Method.") - } - - return Index, err -} - // Decode takes a slice of flac and decodes to wav func Decode(buf []byte) ([]byte, error) { r := bytes.NewReader(buf) @@ -71,10 +21,12 @@ func Decode(buf []byte) ([]byte, error) { if err != nil { return nil, errors.New("Could not parse FLAC") } - fb := &buffer{} - enc := wav.NewEncoder(fb, int(stream.Info.SampleRate), int(stream.Info.BitsPerSample), int(stream.Info.NChannels), wavAudioFormat) + ws := &writerseeker.WriterSeeker{} + enc := wav.NewEncoder(ws, int(stream.Info.SampleRate), int(stream.Info.BitsPerSample), int(stream.Info.NChannels), wavAudioFormat) defer enc.Close() var data []int + var out []byte + var d []byte for { // Decode FLAC audio samples. frame, err := stream.ParseNext() @@ -103,6 +55,12 @@ func Decode(buf []byte) ([]byte, error) { if err := enc.Write(buf); err != nil { return nil, err } + d, err = ioutil.ReadAll(ws.Reader()) + if err != nil { + return nil, err + } + out = append(out, d...) } - return fb.Bytes(), nil + + return d, nil } diff --git a/stream/flac/flac_test.go b/stream/flac/flac_test.go index 763731d4..13bef836 100644 --- a/stream/flac/flac_test.go +++ b/stream/flac/flac_test.go @@ -8,7 +8,7 @@ import ( const ( testFile = "/home/saxon/Desktop/robot.flac" - outFile = "out.wav" + outFile = "testOut.wav" ) func TestDecodeFlac(t *testing.T) { From 155134eeed1bd79ddc52c2bdd27c92fb9f527cef Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 21 Jan 2019 17:37:16 +1030 Subject: [PATCH 004/208] av/stream/flac: moved readAll to after loop --- stream/flac/decode.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/stream/flac/decode.go b/stream/flac/decode.go index 941610e2..6c8445e4 100644 --- a/stream/flac/decode.go +++ b/stream/flac/decode.go @@ -25,8 +25,6 @@ func Decode(buf []byte) ([]byte, error) { enc := wav.NewEncoder(ws, int(stream.Info.SampleRate), int(stream.Info.BitsPerSample), int(stream.Info.NChannels), wavAudioFormat) defer enc.Close() var data []int - var out []byte - var d []byte for { // Decode FLAC audio samples. frame, err := stream.ParseNext() @@ -55,12 +53,10 @@ func Decode(buf []byte) ([]byte, error) { if err := enc.Write(buf); err != nil { return nil, err } - d, err = ioutil.ReadAll(ws.Reader()) - if err != nil { - return nil, err - } - out = append(out, d...) } - + d, err := ioutil.ReadAll(ws.Reader()) + if err != nil { + return nil, err + } return d, nil } From 28e26cd151cc9866cf9e1072c2936528c2ed4554 Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 21 Jan 2019 17:50:09 +1030 Subject: [PATCH 005/208] av/stream/flc: using my own writeSeeker implementation - working --- stream/flac/decode.go | 52 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/stream/flac/decode.go b/stream/flac/decode.go index 6c8445e4..667fcf9c 100644 --- a/stream/flac/decode.go +++ b/stream/flac/decode.go @@ -4,16 +4,55 @@ import ( "bytes" "errors" "io" - "io/ioutil" "github.com/go-audio/audio" "github.com/go-audio/wav" "github.com/mewkiz/flac" - "github.com/orcaman/writerseeker" ) const wavAudioFormat = 1 +type WriterSeeker struct { + buf []byte + pos int +} + +func (ws *WriterSeeker) Bytes() []byte { + return ws.buf +} + +func (m *WriterSeeker) Write(p []byte) (n int, err error) { + minCap := m.pos + len(p) + if minCap > cap(m.buf) { // Make sure buf has enough capacity: + buf2 := make([]byte, len(m.buf), minCap+len(p)) // add some extra + copy(buf2, m.buf) + m.buf = buf2 + } + if minCap > len(m.buf) { + m.buf = m.buf[:minCap] + } + copy(m.buf[m.pos:], p) + m.pos += len(p) + return len(p), nil +} + +func (m *WriterSeeker) Seek(offset int64, whence int) (int64, error) { + newPos, offs := 0, int(offset) + switch whence { + case io.SeekStart: + newPos = offs + case io.SeekCurrent: + newPos = m.pos + offs + case io.SeekEnd: + newPos = len(m.buf) + offs + } + if newPos < 0 { + return 0, errors.New("negative result pos") + } + m.pos = newPos + return int64(newPos), nil +} + // Decode takes a slice of flac and decodes to wav func Decode(buf []byte) ([]byte, error) { r := bytes.NewReader(buf) @@ -21,7 +60,7 @@ func Decode(buf []byte) ([]byte, error) { if err != nil { return nil, errors.New("Could not parse FLAC") } - ws := &writerseeker.WriterSeeker{} + ws := &WriterSeeker{} enc := wav.NewEncoder(ws, int(stream.Info.SampleRate), int(stream.Info.BitsPerSample), int(stream.Info.NChannels), wavAudioFormat) defer enc.Close() var data []int @@ -54,9 +93,6 @@ func Decode(buf []byte) ([]byte, error) { return nil, err } } - d, err := ioutil.ReadAll(ws.Reader()) - if err != nil { - return nil, err - } - return d, nil + + return ws.Bytes(), nil } From 5f3bf33213926fee73fd1c2a8fee55735e93a12b Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 21 Jan 2019 22:52:17 +1030 Subject: [PATCH 006/208] av/stream/flac/decode.go: wrote func headers --- stream/flac/decode.go | 79 ++++++++++++++++++++++++++++++---------- stream/flac/flac_test.go | 26 +++++++++++++ 2 files changed, 85 insertions(+), 20 deletions(-) diff --git a/stream/flac/decode.go b/stream/flac/decode.go index 667fcf9c..42c4dace 100644 --- a/stream/flac/decode.go +++ b/stream/flac/decode.go @@ -1,3 +1,29 @@ +/* +NAME + decode.go + +DESCRIPTION + decode.go provides functionality for the decoding of FLAC compressed audio + +AUTHOR + Saxon Nelson-Milton + +LICENSE + decode.go is Copyright (C) 2017-2019 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ package flac import ( @@ -10,59 +36,72 @@ import ( "github.com/mewkiz/flac" ) -const wavAudioFormat = 1 +const wavFormat = 1 -type WriterSeeker struct { +// writeSeeker implements a memory based io.WriteSeeker. +type writeSeeker struct { buf []byte pos int } -func (ws *WriterSeeker) Bytes() []byte { +// Bytes returns the bytes contained in the writeSeekers buffer. +func (ws *writeSeeker) Bytes() []byte { return ws.buf } -func (m *WriterSeeker) Write(p []byte) (n int, err error) { - minCap := m.pos + len(p) - if minCap > cap(m.buf) { // Make sure buf has enough capacity: - buf2 := make([]byte, len(m.buf), minCap+len(p)) // add some extra - copy(buf2, m.buf) - m.buf = buf2 +// Write writes len(p) bytes from p to the writeSeeker's buf and returns the number +// of bytes written. If less than len(p) bytes are written, an error is returned. +func (ws *writeSeeker) Write(p []byte) (n int, err error) { + minCap := ws.pos + len(p) + if minCap > cap(ws.buf) { // Make sure buf has enough capacity: + buf2 := make([]byte, len(ws.buf), minCap+len(p)) // add some extra + copy(buf2, ws.buf) + ws.buf = buf2 } - if minCap > len(m.buf) { - m.buf = m.buf[:minCap] + if minCap > len(ws.buf) { + ws.buf = ws.buf[:minCap] } - copy(m.buf[m.pos:], p) - m.pos += len(p) + copy(ws.buf[ws.pos:], p) + ws.pos += len(p) return len(p), nil } -func (m *WriterSeeker) Seek(offset int64, whence int) (int64, error) { +// Seek sets the offset for the next Read or Write to offset, interpreted according +// to whence: SeekStart means relative to the start of the file, SeekCurrent means +// relative to the current offset, and SeekEnd means relative to the end. Seek returns +// the new offset relative to the start of the file and an error, if any. +func (ws *writeSeeker) Seek(offset int64, whence int) (int64, error) { newPos, offs := 0, int(offset) switch whence { case io.SeekStart: newPos = offs case io.SeekCurrent: - newPos = m.pos + offs + newPos = ws.pos + offs case io.SeekEnd: - newPos = len(m.buf) + offs + newPos = len(ws.buf) + offs } if newPos < 0 { return 0, errors.New("negative result pos") } - m.pos = newPos + ws.pos = newPos return int64(newPos), nil } -// Decode takes a slice of flac and decodes to wav +// Decode takes buf, a slice of FLAC, and decodes to WAV. If complete decoding +// fails, an error is returned. func Decode(buf []byte) ([]byte, error) { + // Lex and decode the FLAC into a stream to hold audio and properties. r := bytes.NewReader(buf) stream, err := flac.Parse(r) if err != nil { return nil, errors.New("Could not parse FLAC") } - ws := &WriterSeeker{} - enc := wav.NewEncoder(ws, int(stream.Info.SampleRate), int(stream.Info.BitsPerSample), int(stream.Info.NChannels), wavAudioFormat) + + // Create WAV encoder and pass writeSeeker that will store output WAV. + ws := &writeSeeker{} + enc := wav.NewEncoder(ws, int(stream.Info.SampleRate), int(stream.Info.BitsPerSample), int(stream.Info.NChannels), wavFormat) defer enc.Close() + var data []int for { // Decode FLAC audio samples. diff --git a/stream/flac/flac_test.go b/stream/flac/flac_test.go index 13bef836..d69c0494 100644 --- a/stream/flac/flac_test.go +++ b/stream/flac/flac_test.go @@ -1,3 +1,29 @@ +/* +NAME + flac_test.go + +DESCRIPTION + flac_test.go provides utilities to test FLAC audio decoding + +AUTHOR + Saxon Nelson-Milton + +LICENSE + flac_test.go is Copyright (C) 2017-2019 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ package flac import ( From 7628efdbc298a1e1527375fd4416c35f52349feb Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 22 Jan 2019 10:26:22 +1030 Subject: [PATCH 007/208] av/stream/flac: working on cleaning up decode code --- stream/flac/decode.go | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/stream/flac/decode.go b/stream/flac/decode.go index 42c4dace..5a470370 100644 --- a/stream/flac/decode.go +++ b/stream/flac/decode.go @@ -90,7 +90,8 @@ func (ws *writeSeeker) Seek(offset int64, whence int) (int64, error) { // Decode takes buf, a slice of FLAC, and decodes to WAV. If complete decoding // fails, an error is returned. func Decode(buf []byte) ([]byte, error) { - // Lex and decode the FLAC into a stream to hold audio and properties. + + // Lex the FLAC into a stream to hold audio and it's properties. r := bytes.NewReader(buf) stream, err := flac.Parse(r) if err != nil { @@ -99,17 +100,30 @@ func Decode(buf []byte) ([]byte, error) { // Create WAV encoder and pass writeSeeker that will store output WAV. ws := &writeSeeker{} - enc := wav.NewEncoder(ws, int(stream.Info.SampleRate), int(stream.Info.BitsPerSample), int(stream.Info.NChannels), wavFormat) + sr := int(stream.Info.SampleRate) + bps := int(stream.Info.BitsPerSample) + nc := int(stream.Info.NChannels) + enc := wav.NewEncoder(ws, sr, bps, nc, wavFormat) defer enc.Close() + // Decode FLAC into frames of samples + intBuf := &audio.IntBuffer{ + Format: &audio.Format{NumChannels: nc, SampleRate: sr}, + SourceBitDepth: bps, + } + return decodeFrames(stream, intBuf, enc, ws) +} + +// +func decodeFrames(s *flac.Stream, intBuf *audio.IntBuffer, e *wav.Encoder, ws *writeSeeker) ([]byte, error) { var data []int for { - // Decode FLAC audio samples. - frame, err := stream.ParseNext() - if err != nil { - if err == io.EOF { - break - } + frame, err := s.ParseNext() + + // If we've reached the end of the stream then we can output the writeSeeker's buffer. + if err == io.EOF { + return ws.Bytes(), nil + } else if err != nil { return nil, err } @@ -120,18 +134,9 @@ func Decode(buf []byte) ([]byte, error) { data = append(data, int(subframe.Samples[i])) } } - buf := &audio.IntBuffer{ - Format: &audio.Format{ - NumChannels: int(stream.Info.NChannels), - SampleRate: int(stream.Info.SampleRate), - }, - Data: data, - SourceBitDepth: int(stream.Info.BitsPerSample), - } - if err := enc.Write(buf); err != nil { + intBuf.Data = data + if err := e.Write(intBuf); err != nil { return nil, err } } - - return ws.Bytes(), nil } From 6f1767d152d98ca5128f4c91f56449fc6854b322 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 22 Jan 2019 10:40:40 +1030 Subject: [PATCH 008/208] av/stream/flac: finished cleaning up decode --- stream/flac/decode.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stream/flac/decode.go b/stream/flac/decode.go index 5a470370..34d42057 100644 --- a/stream/flac/decode.go +++ b/stream/flac/decode.go @@ -114,7 +114,9 @@ func Decode(buf []byte) ([]byte, error) { return decodeFrames(stream, intBuf, enc, ws) } -// +// decodeFrames parses frames from the stream and encodes them into WAV until +// the end of the stream is reached. The bytes from writeSeeker buffer are then +// returned. If any errors occur during encodeing, nil bytes and the error is returned. func decodeFrames(s *flac.Stream, intBuf *audio.IntBuffer, e *wav.Encoder, ws *writeSeeker) ([]byte, error) { var data []int for { From b5611bb2b42f8145dc364c1a6fbba0d96fc15757 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 22 Jan 2019 10:45:36 +1030 Subject: [PATCH 009/208] av/stream/flac: added writeseeker tests --- stream/flac/flac_test.go | 44 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/stream/flac/flac_test.go b/stream/flac/flac_test.go index d69c0494..9537d682 100644 --- a/stream/flac/flac_test.go +++ b/stream/flac/flac_test.go @@ -27,6 +27,7 @@ LICENSE package flac import ( + "io" "io/ioutil" "os" "testing" @@ -37,6 +38,49 @@ const ( outFile = "testOut.wav" ) +func TestWriteSeekerWrite(t *testing.T) { + writerSeeker := &writeSeeker{} + var ws io.WriteSeeker = writerSeeker + + ws.Write([]byte("hello")) + if string(writerSeeker.buf) != "hello" { + t.Fail() + } + + ws.Write([]byte(" world")) + if string(writerSeeker.buf) != "hello world" { + t.Fail() + } + +} + +func TestWriteSeekerSeek(t *testing.T) { + writerSeeker := &writeSeeker{} + var ws io.WriteSeeker = writerSeeker + + ws.Write([]byte("hello")) + if string(writerSeeker.buf) != "hello" { + t.Fail() + } + + ws.Write([]byte(" world")) + if string(writerSeeker.buf) != "hello world" { + t.Fail() + } + + ws.Seek(-2, io.SeekEnd) + ws.Write([]byte("k!")) + if string(writerSeeker.buf) != "hello work!" { + t.Fail() + } + + ws.Seek(6, io.SeekStart) + ws.Write([]byte("gopher")) + if string(writerSeeker.buf) != "hello gopher" { + t.Fail() + } +} + func TestDecodeFlac(t *testing.T) { b, err := ioutil.ReadFile(testFile) if err != nil { From 99b7f4a44bae180cd3f3cf288da5bca4d619581f Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 22 Jan 2019 11:15:39 +1030 Subject: [PATCH 010/208] av/stream/flac: saving progress --- stream/flac/flac_test.go | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/stream/flac/flac_test.go b/stream/flac/flac_test.go index 9537d682..79274819 100644 --- a/stream/flac/flac_test.go +++ b/stream/flac/flac_test.go @@ -38,45 +38,51 @@ const ( outFile = "testOut.wav" ) +// TestWriteSeekerWrite checks that basic writing to the ws works as expected. func TestWriteSeekerWrite(t *testing.T) { - writerSeeker := &writeSeeker{} - var ws io.WriteSeeker = writerSeeker + ws := &writeSeeker{} - ws.Write([]byte("hello")) - if string(writerSeeker.buf) != "hello" { - t.Fail() + const tstStr1 = "hello" + ws.Write([]byte(tstStr1)) + got := string(ws.buf) + if got != tstStr1 { + t.Errorf("Write failed, got: %v, want: %v", got, tstStr1) } - ws.Write([]byte(" world")) - if string(writerSeeker.buf) != "hello world" { - t.Fail() + const tstStr2 = " world" + const want = "hello world" + ws.Write([]byte(tstStr2)) + got = string(ws.buf) + if got != want { + t.Errorf("Second write failed, got: %v, want: %v", got, want) } - } +// TestWriteSeekerSeek checks that writing and seeking works as expected, i.e. we +// can write, then seek to a knew place in the buf, and write again, either replacing +// bytes, or appending bytes. func TestWriteSeekerSeek(t *testing.T) { - writerSeeker := &writeSeeker{} - var ws io.WriteSeeker = writerSeeker + ws := &writeSeeker{} ws.Write([]byte("hello")) - if string(writerSeeker.buf) != "hello" { + if string(ws.buf) != "hello" { t.Fail() } ws.Write([]byte(" world")) - if string(writerSeeker.buf) != "hello world" { + if string(ws.buf) != "hello world" { t.Fail() } ws.Seek(-2, io.SeekEnd) ws.Write([]byte("k!")) - if string(writerSeeker.buf) != "hello work!" { + if string(ws.buf) != "hello work!" { t.Fail() } ws.Seek(6, io.SeekStart) ws.Write([]byte("gopher")) - if string(writerSeeker.buf) != "hello gopher" { + if string(ws.buf) != "hello gopher" { t.Fail() } } From 9ffc5367cb01297c8aa696e24e440977353595ff Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 22 Jan 2019 12:14:40 +1030 Subject: [PATCH 011/208] av/stream/flac: cleaned up testing file --- stream/flac/flac_test.go | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/stream/flac/flac_test.go b/stream/flac/flac_test.go index 79274819..0d8079f7 100644 --- a/stream/flac/flac_test.go +++ b/stream/flac/flac_test.go @@ -64,29 +64,43 @@ func TestWriteSeekerWrite(t *testing.T) { func TestWriteSeekerSeek(t *testing.T) { ws := &writeSeeker{} - ws.Write([]byte("hello")) - if string(ws.buf) != "hello" { - t.Fail() + const tstStr1 = "hello" + want1 := tstStr1 + ws.Write([]byte(tstStr1)) + got := string(ws.buf) + if got != tstStr1 { + t.Errorf("Unexpected output, got: %v, want: %v", got, want1) } - ws.Write([]byte(" world")) - if string(ws.buf) != "hello world" { - t.Fail() + const tstStr2 = " world" + const want2 = tstStr1 + tstStr2 + ws.Write([]byte(tstStr2)) + got = string(ws.buf) + if got != want2 { + t.Errorf("Unexpected output, got: %v, want: %v", got, want2) } + const tstStr3 = "k!" + const want3 = "hello work!" ws.Seek(-2, io.SeekEnd) - ws.Write([]byte("k!")) - if string(ws.buf) != "hello work!" { - t.Fail() + ws.Write([]byte(tstStr3)) + got = string(ws.buf) + if got != want3 { + t.Errorf("Unexpected output, got: %v, want: %v", got, want3) } + const tstStr4 = "gopher" + const want4 = "hello gopher" ws.Seek(6, io.SeekStart) - ws.Write([]byte("gopher")) - if string(ws.buf) != "hello gopher" { - t.Fail() + ws.Write([]byte(tstStr4)) + got = string(ws.buf) + if got != want4 { + t.Errorf("Unexpected output, got: %v, want: %v", got, want4) } } +// TestDecodeFlac checks that we can load a flac file and decode to wav, writing +// to a wav file. func TestDecodeFlac(t *testing.T) { b, err := ioutil.ReadFile(testFile) if err != nil { From 4602a555d5d098471ad9ee6885a44786f2e36619 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 22 Jan 2019 12:27:52 +1030 Subject: [PATCH 012/208] av/stream/flac: updated test file directory --- stream/flac/flac_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stream/flac/flac_test.go b/stream/flac/flac_test.go index 0d8079f7..1f8019e5 100644 --- a/stream/flac/flac_test.go +++ b/stream/flac/flac_test.go @@ -34,7 +34,7 @@ import ( ) const ( - testFile = "/home/saxon/Desktop/robot.flac" + testFile = "../../../test/test-data/av/input/robot.flac" outFile = "testOut.wav" ) From 37bdb2cf8ea293b6d6ee8fdc8f8c60a08b9fed10 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 22 Jan 2019 14:31:42 +1030 Subject: [PATCH 013/208] av/revid: removed test commands that we're not using anymore --- revid/cmd/h264-file-to-flv-rtmp/main.go | 75 ---------------------- revid/cmd/h264-file-to-mpgets-file/main.go | 66 ------------------- 2 files changed, 141 deletions(-) delete mode 100644 revid/cmd/h264-file-to-flv-rtmp/main.go delete mode 100644 revid/cmd/h264-file-to-mpgets-file/main.go diff --git a/revid/cmd/h264-file-to-flv-rtmp/main.go b/revid/cmd/h264-file-to-flv-rtmp/main.go deleted file mode 100644 index 5f79df4a..00000000 --- a/revid/cmd/h264-file-to-flv-rtmp/main.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -NAME - main.go - -DESCRIPTION - See Readme.md - -AUTHOR - Saxon Nelson-Milton - -LICENSE - main.go is Copyright (C) 2017 the Australian Ocean Lab (AusOcean) - - It is free software: you can redistribute it and/or modify them - under the terms of the GNU General Public License as published by the - Free Software Foundation, either version 3 of the License, or (at your - option) any later version. - - It is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License - along with revid in gpl.txt. If not, see [GNU licenses](http://www.gnu.org/licenses). -*/ - -package main - -import ( - "flag" - "log" - "time" - - "bitbucket.org/ausocean/av/revid" - "bitbucket.org/ausocean/iot/pi/smartlogger" - "bitbucket.org/ausocean/utils/logger" -) - -const ( - inputFile = "../../../../test/test-data/av/input/betterInput.h264" - frameRate = "25" - runDuration = 120 * time.Second - logPath = "/var/log" -) - -// Test h264 inputfile to flv format into rtmp using librtmp c wrapper -func main() { - // Get the rtmp url from a cmd flag - rtmpUrlPtr := flag.String("rtmpUrl", "", "The rtmp url you would like to stream to.") - flag.Parse() - if *rtmpUrlPtr == "" { - log.Println("No RTMP url passed!") - return - } - - config := revid.Config{ - Input: revid.File, - InputFileName: inputFile, - InputCodec: revid.H264, - Output1: revid.Rtmp, - RtmpMethod: revid.LibRtmp, - RtmpUrl: *rtmpUrlPtr, - Packetization: revid.Flv, - Logger: logger.New(logger.Info, &smartlogger.New(logPath).LogRoller), - } - revidInst, err := revid.New(config, nil) - if err != nil { - config.Logger.Log(logger.Error, "Should not have got an error!: ", err.Error()) - return - } - revidInst.Start() - time.Sleep(runDuration) - revidInst.Stop() -} diff --git a/revid/cmd/h264-file-to-mpgets-file/main.go b/revid/cmd/h264-file-to-mpgets-file/main.go deleted file mode 100644 index 03a39fde..00000000 --- a/revid/cmd/h264-file-to-mpgets-file/main.go +++ /dev/null @@ -1,66 +0,0 @@ -/* -NAME - main.go - -DESCRIPTION - See Readme.md - -AUTHOR - Saxon Nelson-Milton - -LICENSE - main.go is Copyright (C) 2017 the Australian Ocean Lab (AusOcean) - - It is free software: you can redistribute it and/or modify them - under the terms of the GNU General Public License as published by the - Free Software Foundation, either version 3 of the License, or (at your - option) any later version. - - It is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License - along with revid in gpl.txt. If not, see [GNU licenses](http://www.gnu.org/licenses). -*/ - -package main - -import ( - "time" - - "bitbucket.org/ausocean/av/revid" - "bitbucket.org/ausocean/iot/pi/smartlogger" - "bitbucket.org/ausocean/utils/logger" -) - -const ( - inputFile = "../../../../test/test-data/av/input/betterInput.h264" - outputFile = "output.ts" - frameRate = "25" - runDuration = 120 * time.Second - logPath = "/var/log" -) - -// Test h264 inputfile to flv format into rtmp using librtmp c wrapper -func main() { - - config := revid.Config{ - Input: revid.File, - InputFileName: inputFile, - InputCodec: revid.H264, - Output1: revid.File, - OutputFileName: outputFile, - Packetization: revid.Mpegts, - Logger: logger.New(logger.Info, &smartlogger.New(logPath).LogRoller), - } - revidInst, err := revid.New(config, nil) - if err != nil { - config.Logger.Log(logger.Error, "Should not have got an error!:", err.Error()) - return - } - revidInst.Start() - time.Sleep(runDuration) - revidInst.Stop() -} From 5637d2b015c1bf3545fc6bba6ad7f1c060162bd2 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 22 Jan 2019 15:53:41 +1030 Subject: [PATCH 014/208] revid-cli: added sendRetry flag so that we can have the option to either keep sending on failure, or just drop data. --- cmd/revid-cli/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 7e375992..01429672 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -124,6 +124,7 @@ func handleFlags() revid.Config { rtpAddrPtr = flag.String("RtpAddr", "", "Rtp destination address: : (port is generally 6970-6999)") logPathPtr = flag.String("LogPath", defaultLogPath, "The log path") configFilePtr = flag.String("ConfigFile", "", "NetSender config file") + sendRetryPtr = flag.Bool("send-retry", false, "If true, we retry send on a failure, otherwise drop the data.") ) flag.Parse() @@ -254,6 +255,7 @@ func handleFlags() revid.Config { cfg.Quantization = *quantizationPtr cfg.IntraRefreshPeriod = *intraRefreshPeriodPtr cfg.RtpAddress = *rtpAddrPtr + cfg.SendRetry = *sendRetryPtr return cfg } From 42c9fb1d097abb012ba802b5792c16a870a49719 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 24 Jan 2019 14:33:22 +1030 Subject: [PATCH 015/208] stream/mts/encoder.go: writing psi based on time interval rather than number of packets interval --- stream/mts/encoder.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 60393285..8a4121a4 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -122,7 +122,7 @@ var ( ) const ( - psiSndCnt = 7 + psiInterval = 1 * time.Second ) // timeLocation holds time and location data @@ -199,9 +199,10 @@ type Encoder struct { tsSpace [PacketSize]byte pesSpace [pes.MaxPesSize]byte - psiCount int - continuity map[int]byte + + now time.Time + psiLastTime time.Time } // NewEncoder returns an Encoder with the specified frame rate. @@ -233,12 +234,15 @@ const ( // generate handles the incoming data and generates equivalent mpegts packets - // sending them to the output channel. func (e *Encoder) Encode(nalu []byte) error { - if e.psiCount <= 0 { + e.now = time.Now() + if e.now.Sub(e.psiLastTime) > psiInterval { err := e.writePSI() if err != nil { return err } + e.psiLastTime = e.now } + // Prepare PES data. pesPkt := pes.Packet{ StreamID: streamID, @@ -269,7 +273,6 @@ func (e *Encoder) Encode(nalu []byte) error { pusi = false } _, err := e.dst.Write(pkt.Bytes(e.tsSpace[:PacketSize])) - e.psiCount-- if err != nil { return err } @@ -318,7 +321,6 @@ func (e *Encoder) writePSI() error { if err != nil { return err } - e.psiCount = psiSndCnt return nil } From 31b9ec07e9be6b67393f8eda884556233705a685 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 24 Jan 2019 14:39:14 +1030 Subject: [PATCH 016/208] stream/mts/encoder.go: no need to have a now field to capture current time - this can be local to encode function --- stream/mts/encoder.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 8a4121a4..02761b91 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -201,7 +201,6 @@ type Encoder struct { continuity map[int]byte - now time.Time psiLastTime time.Time } @@ -234,13 +233,13 @@ const ( // generate handles the incoming data and generates equivalent mpegts packets - // sending them to the output channel. func (e *Encoder) Encode(nalu []byte) error { - e.now = time.Now() - if e.now.Sub(e.psiLastTime) > psiInterval { + now := time.Now() + if now.Sub(e.psiLastTime) > psiInterval { err := e.writePSI() if err != nil { return err } - e.psiLastTime = e.now + e.psiLastTime = now } // Prepare PES data. From ea1458014c3f971693d5efb373927947c24b0cff Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 24 Jan 2019 22:29:43 +1030 Subject: [PATCH 017/208] mts: fixing cc --- cmd/ts-repair/main.go | 82 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 cmd/ts-repair/main.go diff --git a/cmd/ts-repair/main.go b/cmd/ts-repair/main.go new file mode 100644 index 00000000..bcb1ebc1 --- /dev/null +++ b/cmd/ts-repair/main.go @@ -0,0 +1,82 @@ +package main + +import ( + "flag" + "io" + "os" + + "bitbucket.org/ausocean/av/stream/mts" + "github.com/Comcast/gots/packet" +) + +const ( + errBadInPath = "No file path provided, or file does not exist" + errCantCreateOut = "Can't create output file" + errCantGetPid = "Can't get pid from packet" + errReadFail = "Read failed" + errWriteFail = "Write to file failed" + usage = "The path to the file to be repaired" +) + +var ccMap = map[int]byte{ + mts.PatPid: 1, + mts.PmtPid: 1, + mts.VideoPid: 0, +} + +type Packet [mts.PacketSize]byte + +func (p *Packet) setCC(cc byte) { + (*p)[3] |= cc & 0xf +} + +func main() { + // Deal with input flags + inPtr := flag.String("path", "", usage) + outPtr := flag.String("out", "out.ts", usage) + flag.Parse() + + // Try and open the given input file, otherwise panic - we can't do anything + inFile, err := os.Open(*inPtr) + defer inFile.Close() + if err != nil { + panic(errBadInPath) + } + + // Try and create output file, otherwise panic - we can't do anything + outFile, err := os.Create(*outPtr) + defer outFile.Close() + if err != nil { + panic(errCantCreateOut) + } + + // Read each packet from the input file reader + var p Packet + for { + // If we get an end of file then return, otherwise we panic - can't do anything else + if _, err := inFile.Read(p[:mts.PacketSize]); err == io.EOF { + return + } else if err != nil { + panic(errReadFail + ": " + err.Error()) + } + + // Get the pid from the packet and set the cc based on this pid using our map + pid, err := packet.Pid((*packet.Packet)(&p)) + if err != nil { + panic(errCantGetPid) + } + p.setCC(ccFor(int(pid))) + + // Write this packet to the output file + if _, err := outFile.Write(p[:]); err != nil { + panic(errWriteFail + ": " + err.Error()) + } + } +} + +// ccFor gets the next cc for the given pid +func ccFor(pid int) byte { + cc := ccMap[pid] + ccMap[pid] = (cc + 1) & 0xf + return cc +} From 0fe5affc00221d7c4b978ecac9eb296cb6984d24 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 25 Jan 2019 15:03:57 +1030 Subject: [PATCH 018/208] cmd/ts-repair: got setting of discontinuity indicators working and also adding adaptation fields to pat and pmt for this reason. --- cmd/ts-repair/main.go | 125 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 115 insertions(+), 10 deletions(-) diff --git a/cmd/ts-repair/main.go b/cmd/ts-repair/main.go index bcb1ebc1..f34da350 100644 --- a/cmd/ts-repair/main.go +++ b/cmd/ts-repair/main.go @@ -2,6 +2,7 @@ package main import ( "flag" + "fmt" "io" "os" @@ -15,25 +16,94 @@ const ( errCantGetPid = "Can't get pid from packet" errReadFail = "Read failed" errWriteFail = "Write to file failed" - usage = "The path to the file to be repaired" + errBadMode = "Bad fix mode" +) + +const ( + inUsage = "The path to the file to be repaired" + outUsage = "Output file path" + modeUsage = "Fix mode: 0 = cc-shift, 1 = di-update" +) + +const ( + ccShift = iota + diUpdate ) var ccMap = map[int]byte{ - mts.PatPid: 1, - mts.PmtPid: 1, - mts.VideoPid: 0, + mts.PatPid: 16, + mts.PmtPid: 16, + mts.VideoPid: 16, } +var packetNo int + +type Option func(p *Packet) + type Packet [mts.PacketSize]byte +func (p *Packet) CC() byte { + return (*p)[3] & 0x0f +} + func (p *Packet) setCC(cc byte) { (*p)[3] |= cc & 0xf } +func (p *Packet) setDI(di bool) { + if di { + p[5] |= 0x80 + } else { + p[5] &= 0x7f + } +} + +func (p *Packet) addAdaptationField(options ...Option) { + // Create space for adaptation field + copy(p[mts.HeadSize+mts.DefaultAdaptationSize:], p[mts.HeadSize:len(p)-mts.DefaultAdaptationSize]) + + // TODO: seperate into own function + // Update adaptation field control + p[mts.AdaptationControlIdx] &= 0xff ^ mts.AdaptationControlMask + p[mts.AdaptationControlIdx] |= mts.AdaptationControlMask + // Default the adaptationfield + p.resetAdaptation() + + for _, option := range options { + option(p) + } +} + +func (p *Packet) resetAdaptation() { + p[mts.AdaptationIdx] = mts.DefaultAdaptationBodySize + p[mts.AdaptationBodyIdx] = 0x00 +} + +func (p *Packet) hasAdaptation() bool { + afc := p[mts.AdaptationControlIdx] & mts.AdaptationControlMask + if afc == 0x20 || afc == 0x30 { + return true + } else { + return false + } +} + +func DiscontinuityIndicator(f bool) Option { + return func(p *Packet) { + set := byte(mts.DiscontinuityIndicatorMask) + if !f { + set = 0x00 + } + p[mts.DiscontinuityIndicatorIdx] &= 0xff ^ mts.DiscontinuityIndicatorMask + p[mts.DiscontinuityIndicatorIdx] |= mts.DiscontinuityIndicatorMask & set + } +} + func main() { // Deal with input flags - inPtr := flag.String("path", "", usage) - outPtr := flag.String("out", "out.ts", usage) + inPtr := flag.String("in", "", inUsage) + outPtr := flag.String("out", "out.ts", outUsage) + modePtr := flag.Int("mode", diUpdate, modeUsage) flag.Parse() // Try and open the given input file, otherwise panic - we can't do anything @@ -59,14 +129,42 @@ func main() { } else if err != nil { panic(errReadFail + ": " + err.Error()) } - + packetNo++ // Get the pid from the packet and set the cc based on this pid using our map pid, err := packet.Pid((*packet.Packet)(&p)) if err != nil { panic(errCantGetPid) } - p.setCC(ccFor(int(pid))) + cc := p.CC() + expect, exists := expectedCC(int(pid)) + if !exists { + updateCCMap(int(pid), cc) + } else { + switch *modePtr { + case ccShift: + p.setCC(expect) + + case diUpdate: + + if cc != expect { + fmt.Printf("packetNo: %v pid: %v, cc: %v, expect: %v\n", packetNo, pid, cc, expect) + if p.hasAdaptation() { + fmt.Println("hasAdaptation") + p.setDI(true) + } else { + fmt.Println("doesn't have adaptation") + fmt.Println(p) + p.addAdaptationField(DiscontinuityIndicator(true)) + fmt.Println(p) + } + updateCCMap(int(pid), p.CC()) + } + + default: + panic(errBadMode) + } + } // Write this packet to the output file if _, err := outFile.Write(p[:]); err != nil { panic(errWriteFail + ": " + err.Error()) @@ -75,8 +173,15 @@ func main() { } // ccFor gets the next cc for the given pid -func ccFor(pid int) byte { +func expectedCC(pid int) (byte, bool) { cc := ccMap[pid] + if cc == 16 { + return 16, false + } + ccMap[pid] = (cc + 1) & 0xf + return cc, true +} + +func updateCCMap(pid int, cc byte) { ccMap[pid] = (cc + 1) & 0xf - return cc } From e6671c5772f84618c84a0c399ecf064c09bc3913 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 25 Jan 2019 15:18:27 +1030 Subject: [PATCH 019/208] cmd/ts-repair: got rid of debug prints --- cmd/ts-repair/main.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cmd/ts-repair/main.go b/cmd/ts-repair/main.go index f34da350..039ada9b 100644 --- a/cmd/ts-repair/main.go +++ b/cmd/ts-repair/main.go @@ -148,15 +148,11 @@ func main() { case diUpdate: if cc != expect { - fmt.Printf("packetNo: %v pid: %v, cc: %v, expect: %v\n", packetNo, pid, cc, expect) + fmt.Printf("***** Discontinuity found (packetNo: %v pid: %v, cc: %v, expect: %v)\n", packetNo, pid, cc, expect) if p.hasAdaptation() { - fmt.Println("hasAdaptation") p.setDI(true) } else { - fmt.Println("doesn't have adaptation") - fmt.Println(p) p.addAdaptationField(DiscontinuityIndicator(true)) - fmt.Println(p) } updateCCMap(int(pid), p.CC()) } From c4d68f15623688bd79f1aad8d8fed448417ad1c5 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 25 Jan 2019 16:10:13 +1030 Subject: [PATCH 020/208] cmd/ts-repair: updated comments and made funcs more robust --- cmd/ts-repair/main.go | 68 ++++++++++++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 17 deletions(-) diff --git a/cmd/ts-repair/main.go b/cmd/ts-repair/main.go index 039ada9b..f3f73bb2 100644 --- a/cmd/ts-repair/main.go +++ b/cmd/ts-repair/main.go @@ -1,6 +1,7 @@ package main import ( + "errors" "flag" "fmt" "io" @@ -10,21 +11,26 @@ import ( "github.com/Comcast/gots/packet" ) +// Various errors that we can encounter. const ( - errBadInPath = "No file path provided, or file does not exist" - errCantCreateOut = "Can't create output file" - errCantGetPid = "Can't get pid from packet" - errReadFail = "Read failed" - errWriteFail = "Write to file failed" - errBadMode = "Bad fix mode" + errBadInPath = "No file path provided, or file does not exist" + errCantCreateOut = "Can't create output file" + errCantGetPid = "Can't get pid from packet" + errReadFail = "Read failed" + errWriteFail = "Write to file failed" + errBadMode = "Bad fix mode" + errAdaptationPresent = "Adaptation field is already present in packet" + errNoAdaptationField = "No adaptation field in this packet" ) +// Consts describing flag usage. const ( inUsage = "The path to the file to be repaired" outUsage = "Output file path" modeUsage = "Fix mode: 0 = cc-shift, 1 = di-update" ) +// Repair modes. const ( ccShift = iota diUpdate @@ -36,20 +42,27 @@ var ccMap = map[int]byte{ mts.VideoPid: 16, } +// packetNo will keep track of the ts packet number for reference. var packetNo int +// Option defines a func that performs an action on p in order to change a ts option. type Option func(p *Packet) +// Packet is a byte array of size mts.PacketSize i.e. 188 bytes. We define this +// to allow us to write receiver funcs for the [mts.PacketSize]byte type. type Packet [mts.PacketSize]byte +// CC returns the CC of p. func (p *Packet) CC() byte { return (*p)[3] & 0x0f } +// setCC sets the CC of p. func (p *Packet) setCC(cc byte) { (*p)[3] |= cc & 0xf } +// setDI sets the discontinuity counter of p. func (p *Packet) setDI(di bool) { if di { p[5] |= 0x80 @@ -58,27 +71,41 @@ func (p *Packet) setDI(di bool) { } } -func (p *Packet) addAdaptationField(options ...Option) { - // Create space for adaptation field +// addAdaptationField adds an adaptation field to p, and applys the passed options to this field. +// TODO: this will probably break if we already have adaptation field. +func (p *Packet) addAdaptationField(options ...Option) error { + if p.hasAdaptation() { + return errors.New(errAdaptationPresent) + } + // Create space for adaptation field. copy(p[mts.HeadSize+mts.DefaultAdaptationSize:], p[mts.HeadSize:len(p)-mts.DefaultAdaptationSize]) // TODO: seperate into own function - // Update adaptation field control + // Update adaptation field control. p[mts.AdaptationControlIdx] &= 0xff ^ mts.AdaptationControlMask p[mts.AdaptationControlIdx] |= mts.AdaptationControlMask - // Default the adaptationfield + // Default the adaptationfield. p.resetAdaptation() + // Apply and options that have bee passed. for _, option := range options { option(p) } + return nil } -func (p *Packet) resetAdaptation() { +// resetAdaptation sets fields in ps adaptation field to 0 if the adaptation field +// exists, otherwise an error is returned. +func (p *Packet) resetAdaptation() error { + if !p.hasAdaptation() { + return errors.New(errNoAdaptationField) + } p[mts.AdaptationIdx] = mts.DefaultAdaptationBodySize p[mts.AdaptationBodyIdx] = 0x00 + return nil } +// hasAdaptation returns true if p has an adaptation field and false otherwise. func (p *Packet) hasAdaptation() bool { afc := p[mts.AdaptationControlIdx] & mts.AdaptationControlMask if afc == 0x20 || afc == 0x30 { @@ -88,6 +115,8 @@ func (p *Packet) hasAdaptation() bool { } } +// DiscontinuityIndicator returns and Option that will set p's discontinuity +// indicator according to f. func DiscontinuityIndicator(f bool) Option { return func(p *Packet) { set := byte(mts.DiscontinuityIndicatorMask) @@ -130,23 +159,26 @@ func main() { panic(errReadFail + ": " + err.Error()) } packetNo++ - // Get the pid from the packet and set the cc based on this pid using our map + + // Get the pid from the packet pid, err := packet.Pid((*packet.Packet)(&p)) if err != nil { panic(errCantGetPid) } + // Get the cc from the packet and also the expected cc (if exists) cc := p.CC() expect, exists := expectedCC(int(pid)) if !exists { updateCCMap(int(pid), cc) } else { switch *modePtr { + // ccShift mode shifts all CC regardless of presence of Discontinuities or not case ccShift: p.setCC(expect) - + // diUpdate mode finds discontinuities and sets the discontinuity indicator to true. + // If we have a pat or pmt then we need to add an adaptation field and then set the DI. case diUpdate: - if cc != expect { fmt.Printf("***** Discontinuity found (packetNo: %v pid: %v, cc: %v, expect: %v)\n", packetNo, pid, cc, expect) if p.hasAdaptation() { @@ -156,19 +188,20 @@ func main() { } updateCCMap(int(pid), p.CC()) } - default: panic(errBadMode) } } - // Write this packet to the output file + + // Write this packet to the output file. if _, err := outFile.Write(p[:]); err != nil { panic(errWriteFail + ": " + err.Error()) } } } -// ccFor gets the next cc for the given pid +// expectedCC returns the expected cc for the given pid. If the cc hasn't been +// used yet, then 16 and false is returned. func expectedCC(pid int) (byte, bool) { cc := ccMap[pid] if cc == 16 { @@ -178,6 +211,7 @@ func expectedCC(pid int) (byte, bool) { return cc, true } +// updateCCMap updates the cc for the passed pid. func updateCCMap(pid int, cc byte) { ccMap[pid] = (cc + 1) & 0xf } From aea41fb71064a6b67f502c92a960e3fccc842ae6 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 25 Jan 2019 16:25:01 +1030 Subject: [PATCH 021/208] stream/mts: adding some constants --- stream/mts/encoder.go | 39 ++++++++++++---------------------- stream/mts/mpegts.go | 49 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 27 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 60393285..53342624 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -171,14 +171,6 @@ var ( pmtTable = standardPmtTimeLocation.Bytes() ) -const ( - sdtPid = 17 - patPid = 0 - pmtPid = 4096 - videoPid = 256 - streamID = 0xe0 // First video stream ID. -) - // Time related constants. const ( // ptsOffset is the offset added to the clock to determine @@ -213,18 +205,13 @@ func NewEncoder(dst io.Writer, fps float64) *Encoder { ptsOffset: ptsOffset, continuity: map[int]byte{ - patPid: 0, - pmtPid: 0, - videoPid: 0, + PatPid: 0, + PmtPid: 0, + VideoPid: 0, }, } } -const ( - hasPayload = 0x1 - hasAdaptationField = 0x2 -) - const ( hasDTS = 0x1 hasPTS = 0x2 @@ -241,7 +228,7 @@ func (e *Encoder) Encode(nalu []byte) error { } // Prepare PES data. pesPkt := pes.Packet{ - StreamID: streamID, + StreamID: StreamID, PDI: hasPTS, PTS: e.pts(), Data: nalu, @@ -253,10 +240,10 @@ func (e *Encoder) Encode(nalu []byte) error { for len(buf) != 0 { pkt := Packet{ PUSI: pusi, - PID: videoPid, + PID: VideoPid, RAI: pusi, - CC: e.ccFor(videoPid), - AFC: hasAdaptationField | hasPayload, + CC: e.ccFor(VideoPid), + AFC: HasAdaptationField | HasPayload, PCRF: pusi, } n := pkt.FillPayload(buf) @@ -286,9 +273,9 @@ func (e *Encoder) writePSI() error { // Write PAT. patPkt := Packet{ PUSI: true, - PID: patPid, - CC: e.ccFor(patPid), - AFC: hasPayload, + PID: PatPid, + CC: e.ccFor(PatPid), + AFC: HasPayload, Payload: patTable, } _, err := e.dst.Write(patPkt.Bytes(e.tsSpace[:PacketSize])) @@ -309,9 +296,9 @@ func (e *Encoder) writePSI() error { // Create mts packet from pmt table. pmtPkt := Packet{ PUSI: true, - PID: pmtPid, - CC: e.ccFor(pmtPid), - AFC: hasPayload, + PID: PmtPid, + CC: e.ccFor(PmtPid), + AFC: HasPayload, Payload: pmtTable, } _, err = e.dst.Write(pmtPkt.Bytes(e.tsSpace[:PacketSize])) diff --git a/stream/mts/mpegts.go b/stream/mts/mpegts.go index 0bef80d2..71849513 100644 --- a/stream/mts/mpegts.go +++ b/stream/mts/mpegts.go @@ -37,6 +37,53 @@ const ( PayloadSize = 176 ) +const ( + SdtPid = 17 + PatPid = 0 + PmtPid = 4096 + VideoPid = 256 + StreamID = 0xe0 // First video stream ID. + HeadSize = 4 + DefaultAdaptationSize = 2 +) + +const ( + AdaptationIdx = 4 + AdaptationControlIdx = 3 + AdaptationBodyIdx = AdaptationIdx + 1 + AdaptationControlMask = 0x30 + DefaultAdaptationBodySize = 1 +) + +const ( + HasPayload = 0x1 + HasAdaptationField = 0x2 +) + +// Adaptation field body masks. +const ( + DiscontinuityIndicatorMask = 0x80 + RandomAccessIndicatorMask = 0x40 + ElementaryStreamPriorityIndicatorMask = 0x20 + ProgramClockReferenceFlagMask = 0x10 + OriginalProgramClockReferenceFlagMask = 0x08 + SplicingPointFlagMask = 0x04 + TransportPrivateDataFlagMask = 0x02 + AdaptationFieldExtensionMask = 0x01 +) + +// Adaptation field body indexes. +const ( + DiscontinuityIndicatorIdx = AdaptationIdx + 1 + RandomAccessIndicatorIdx = AdaptationIdx + 1 + ElementaryStreamPriorityIndicatorIdx = AdaptationIdx + 1 + ProgramClockReferenceFlagIdx = AdaptationIdx + 1 + OriginalProgramClockReferenceFlagIdx = AdaptationIdx + 1 + SplicingPointFlagIdx = AdaptationIdx + 1 + TransportPrivateDataFlagIdx = AdaptationIdx + 1 + AdaptationFieldExtensionFlagIdx = AdaptationIdx + 1 +) + /* The below data struct encapsulates the fields of an MPEG-TS packet. Below is the formatting of an MPEG-TS packet for reference! @@ -135,7 +182,7 @@ func FindPMT(d []byte) (p []byte, i int, err error) { } for i = 0; i < len(d); i += PacketSize { pid := (uint16(d[i+1]&0x1f) << 8) | uint16(d[i+2]) - if pid == pmtPid { + if pid == PmtPid { p = d[i+4 : i+PacketSize] return } From d10723122444e74e5201837c65477d99402fc74d Mon Sep 17 00:00:00 2001 From: saxon Date: Sat, 26 Jan 2019 18:04:21 +1030 Subject: [PATCH 022/208] stream/mts: starting to work out meta logistics in encoder.go and added incomplete AddDescriptor function in psi package --- stream/mts/encoder.go | 60 ++++++++++--------------------------------- stream/mts/psi/psi.go | 8 ++++++ 2 files changed, 21 insertions(+), 47 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 53342624..d4c2ecd6 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -82,43 +82,6 @@ var ( }, }, } - - // standardPmtTimeLocation is a standard PMT with time and location - // descriptors, but time and location fields zeroed out. - standardPmtTimeLocation = psi.PSI{ - Pf: 0x00, - Tid: 0x02, - Ssi: true, - Sl: 0x3e, - Tss: &psi.TSS{ - Tide: 0x01, - V: 0, - Cni: true, - Sn: 0, - Lsn: 0, - Sd: &psi.PMT{ - Pcrpid: 0x0100, - Pil: psi.PmtTimeLocationPil, - Pd: []psi.Desc{ - { - Dt: psi.TimeDescTag, - Dl: psi.TimeDataSize, - Dd: make([]byte, psi.TimeDataSize), - }, - { - Dt: psi.LocationDescTag, - Dl: psi.LocationDataSize, - Dd: make([]byte, psi.LocationDataSize), - }, - }, - Essd: &psi.ESSD{ - St: 0x1b, - Epid: 0x0100, - Esil: 0x00, - }, - }, - }, - } ) const ( @@ -162,13 +125,24 @@ func (tl *timeLocation) Location() string { return l } +func updateMeta(b []byte) error { + var p psi.PSIBytes + p = b + // TODO: get length of meta data and format appropriately + metaLen := 0 + metaData := "" + + p.AddDescriptor(psi.MetadataTag, metaLen, metaData) + return nil +} + // MetData will hold time and location data which may be set externally if // this data is available. It is then inserted into mpegts packets outputted. var MetaData timeLocation var ( patTable = standardPat.Bytes() - pmtTable = standardPmtTimeLocation.Bytes() + pmtTable = standardPmt.Bytes() ) // Time related constants. @@ -283,15 +257,7 @@ func (e *Encoder) writePSI() error { return err } - // Update pmt table time and location. - err = psi.UpdateTime(pmtTable, MetaData.TimeStamp()) - if err != nil { - return err - } - err = psi.UpdateLocation(pmtTable, MetaData.Location()) - if err != nil { - return nil - } + updateMeta(pmtTable) // Create mts packet from pmt table. pmtPkt := Packet{ diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 09028ba6..34063d0c 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -68,6 +68,14 @@ const ( crcSize = 4 ) +const MetadataTag = 0x26 + +type PSIBytes []byte + +func (p *PSIBytes) AddDescriptor(tag, len int, data string) { + +} + // Program specific information type PSI struct { Pf byte // Point field From 87ded6bf2ef84a3db52d6ba70625a561fcf1fb10 Mon Sep 17 00:00:00 2001 From: saxon Date: Sat, 26 Jan 2019 21:53:19 +1030 Subject: [PATCH 023/208] stream/mts/encoder.go: implemented metadata receiver functions: Add, Get, All and Delete --- stream/mts/encoder.go | 70 ++++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index d4c2ecd6..9909a0fe 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -29,6 +29,8 @@ LICENSE package mts import ( + "errors" + "fmt" "io" "sync" "time" @@ -88,43 +90,53 @@ const ( psiSndCnt = 7 ) -// timeLocation holds time and location data -type timeLocation struct { - mu sync.RWMutex - time uint64 - location string +// global Meta +var meta Meta + +type Meta struct { + mu sync.RWMutex + data map[string]string } -// SetTimeStamp sets the time field of a TimeLocation. -func (tl *timeLocation) SetTimeStamp(t uint64) { - tl.mu.Lock() - tl.time = t - tl.mu.Unlock() +// Add adds metadata with key and val, if already exists return error +func (m *Meta) Add(key, val string) error { + m.mu.Lock() + if _, exists := m.data[key]; !exists { + return errors.New(fmt.Sprintf("Metadata for: %v already exists", key)) + } + m.data[key] = val + m.mu.Unlock() + return nil } -// GetTimeStamp returns the location of a TimeLocation. -func (tl *timeLocation) TimeStamp() uint64 { - tl.mu.RLock() - t := tl.time - tl.mu.RUnlock() - return t +// All returns the a copy of the map containing the meta data +func (m *Meta) All() map[string]string { + var cpy map[string]string + for k, v := range m.data { + cpy[k] = v + } + return cpy } -// SetLocation sets the location of a TimeLocation. -func (tl *timeLocation) SetLocation(l string) { - tl.mu.Lock() - tl.location = l - tl.mu.Unlock() +// Get returns the meta data for the passed key +func (m *Meta) Get(key string) (string, error) { + val, ok := m.data[key] + if !ok { + return "", errors.New("Key does not exist in metadata map") + } + return val, nil } -// GetLocation returns the location of a TimeLocation. -func (tl *timeLocation) Location() string { - tl.mu.RLock() - l := tl.location - tl.mu.RUnlock() - return l +// Remove deletes a meta entry in the map and returns error if it doesn’t exist +func (m *Meta) Delete(key string) error { + if _, ok := m.data[key]; ok { + delete(m.data, key) + return nil + } + return errors.New("Trying to delete map entry that doesn't exist") } +// updateMeta ... func updateMeta(b []byte) error { var p psi.PSIBytes p = b @@ -136,10 +148,6 @@ func updateMeta(b []byte) error { return nil } -// MetData will hold time and location data which may be set externally if -// this data is available. It is then inserted into mpegts packets outputted. -var MetaData timeLocation - var ( patTable = standardPat.Bytes() pmtTable = standardPmt.Bytes() From 8f5a2352b294624bf00390880f82f0f44f70e211 Mon Sep 17 00:00:00 2001 From: saxon Date: Sat, 26 Jan 2019 21:57:14 +1030 Subject: [PATCH 024/208] stream/mts: added meta.go file to contain struct and methods relating to Metadata and operations --- stream/mts/encoder.go | 46 ------------------------------------------- 1 file changed, 46 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 9909a0fe..fb2afbed 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -29,10 +29,7 @@ LICENSE package mts import ( - "errors" - "fmt" "io" - "sync" "time" "bitbucket.org/ausocean/av/stream/mts/pes" @@ -93,49 +90,6 @@ const ( // global Meta var meta Meta -type Meta struct { - mu sync.RWMutex - data map[string]string -} - -// Add adds metadata with key and val, if already exists return error -func (m *Meta) Add(key, val string) error { - m.mu.Lock() - if _, exists := m.data[key]; !exists { - return errors.New(fmt.Sprintf("Metadata for: %v already exists", key)) - } - m.data[key] = val - m.mu.Unlock() - return nil -} - -// All returns the a copy of the map containing the meta data -func (m *Meta) All() map[string]string { - var cpy map[string]string - for k, v := range m.data { - cpy[k] = v - } - return cpy -} - -// Get returns the meta data for the passed key -func (m *Meta) Get(key string) (string, error) { - val, ok := m.data[key] - if !ok { - return "", errors.New("Key does not exist in metadata map") - } - return val, nil -} - -// Remove deletes a meta entry in the map and returns error if it doesn’t exist -func (m *Meta) Delete(key string) error { - if _, ok := m.data[key]; ok { - delete(m.data, key) - return nil - } - return errors.New("Trying to delete map entry that doesn't exist") -} - // updateMeta ... func updateMeta(b []byte) error { var p psi.PSIBytes From df07f3ff48a0be27598c5f81ef12aec731b8563c Mon Sep 17 00:00:00 2001 From: saxon Date: Sat, 26 Jan 2019 22:36:34 +1030 Subject: [PATCH 025/208] stream/mts/meta.go: added Format func, but need to complete later once specs are clarified --- stream/mts/meta.go | 82 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 stream/mts/meta.go diff --git a/stream/mts/meta.go b/stream/mts/meta.go new file mode 100644 index 00000000..8abdeb4f --- /dev/null +++ b/stream/mts/meta.go @@ -0,0 +1,82 @@ +/* +NAME + meta.go + +DESCRIPTION + See Readme.md + +AUTHOR + Saxon Nelson-Milton + +LICENSE + meta.go is Copyright (C) 2017-2019 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ + +package mts + +import ( + "errors" + "fmt" + "sync" +) + +type Meta struct { + mu sync.RWMutex + data map[string]string +} + +// Add adds metadata with key and val, if already exists return error +func (m *Meta) Add(key, val string) error { + m.mu.Lock() + if _, exists := m.data[key]; !exists { + return errors.New(fmt.Sprintf("Metadata for: %v already exists", key)) + } + m.data[key] = val + m.mu.Unlock() + return nil +} + +// All returns the a copy of the map containing the meta data +func (m *Meta) All() map[string]string { + var cpy map[string]string + for k, v := range m.data { + cpy[k] = v + } + return cpy +} + +// Get returns the meta data for the passed key +func (m *Meta) Get(key string) (string, error) { + val, ok := m.data[key] + if !ok { + return "", errors.New("Key does not exist in metadata map") + } + return val, nil +} + +// Remove deletes a meta entry in the map and returns error if it doesn’t exist +func (m *Meta) Delete(key string) error { + if _, ok := m.data[key]; ok { + delete(m.data, key) + return nil + } + return errors.New("Trying to delete map entry that doesn't exist") +} + +func (m *Meta) Format() []byte { + // TODO: complete this + return []byte("someData") +} From ecf7263bc190ed06b9246dc4216bf6ea3544a8a8 Mon Sep 17 00:00:00 2001 From: saxon Date: Sat, 26 Jan 2019 22:42:31 +1030 Subject: [PATCH 026/208] stream/mts/meta.go: started using mutex where I was supposed to --- stream/mts/encoder.go | 22 ++++++++++------------ stream/mts/meta.go | 8 ++++++++ stream/mts/psi/psi.go | 3 +-- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index fb2afbed..9ce254e7 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -90,18 +90,6 @@ const ( // global Meta var meta Meta -// updateMeta ... -func updateMeta(b []byte) error { - var p psi.PSIBytes - p = b - // TODO: get length of meta data and format appropriately - metaLen := 0 - metaData := "" - - p.AddDescriptor(psi.MetadataTag, metaLen, metaData) - return nil -} - var ( patTable = standardPat.Bytes() pmtTable = standardPmt.Bytes() @@ -259,3 +247,13 @@ func (e *Encoder) ccFor(pid int) byte { e.continuity[pid] = (cc + 1) & continuityCounterMask return cc } + +// updateMeta ... +func updateMeta(b []byte) error { + var p psi.PSIBytes + p = b + // TODO: get length of meta data and format appropriately + m := meta.Format() + p.AddDescriptor(psi.MetadataTag, len(m), m) + return nil +} diff --git a/stream/mts/meta.go b/stream/mts/meta.go index 8abdeb4f..c8d70ca2 100644 --- a/stream/mts/meta.go +++ b/stream/mts/meta.go @@ -51,24 +51,32 @@ func (m *Meta) Add(key, val string) error { // All returns the a copy of the map containing the meta data func (m *Meta) All() map[string]string { + m.mu.Lock() var cpy map[string]string for k, v := range m.data { cpy[k] = v } + m.mu.Unlock() return cpy } // Get returns the meta data for the passed key func (m *Meta) Get(key string) (string, error) { + m.mu.Lock() val, ok := m.data[key] + m.mu.Unlock() if !ok { return "", errors.New("Key does not exist in metadata map") } + return val, nil + } // Remove deletes a meta entry in the map and returns error if it doesn’t exist func (m *Meta) Delete(key string) error { + m.mu.Lock() + defer m.mu.Unlock() if _, ok := m.data[key]; ok { delete(m.data, key) return nil diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 34063d0c..2942fbfb 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -72,8 +72,7 @@ const MetadataTag = 0x26 type PSIBytes []byte -func (p *PSIBytes) AddDescriptor(tag, len int, data string) { - +func (p *PSIBytes) AddDescriptor(tag, len int, data []byte) { } // Program specific information From 6f421ab706962a0e29f1b994a63af867640242ec Mon Sep 17 00:00:00 2001 From: saxon Date: Sat, 26 Jan 2019 23:35:31 +1030 Subject: [PATCH 027/208] stream/mts/psi.go: added more to AddDescriptor and added signature for edistDesc --- stream/mts/encoder.go | 1 - stream/mts/psi/psi.go | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 9ce254e7..60fb6c07 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -252,7 +252,6 @@ func (e *Encoder) ccFor(pid int) byte { func updateMeta(b []byte) error { var p psi.PSIBytes p = b - // TODO: get length of meta data and format appropriately m := meta.Format() p.AddDescriptor(psi.MetadataTag, len(m), m) return nil diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 2942fbfb..7571fb03 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -26,6 +26,12 @@ LICENSE package psi +import ( + "errors" + + "github.com/Comcast/gots/psi" +) + const ( PacketSize = 184 // packet size of a psi. ) @@ -68,11 +74,39 @@ const ( crcSize = 4 ) +const ( + SectionLenIdx1 = 2 + SectionLenIdx2 = 3 +) + +const ( + SectionLenMask1 = 0x03 +) + const MetadataTag = 0x26 type PSIBytes []byte -func (p *PSIBytes) AddDescriptor(tag, len int, data []byte) { +func (p *PSIBytes) AddDescriptor(tag int, data []byte) error { + // first check that we actually have table syntax section + // NB: if table syntax section doesn't exist (unlikely I think) then we could + // just add it + + // Check that this is actually a PMT + if psi.TableID(*p) != pmtID { + return errors.New("trying to add descriptor, but not pmt") + } + + if !p.editDesc(tag, data) { + + } + // if exists update + // otherwise add + return nil +} + +func (p *PSIBytes) editDesc(tag int, data []byte) bool { + return true } // Program specific information From 9171b56d313d59ace1640a474c481516e739b29f Mon Sep 17 00:00:00 2001 From: saxon Date: Sun, 27 Jan 2019 16:55:00 +1030 Subject: [PATCH 028/208] stream/mts: created new type to represent Descriptor (typedef of []bytes) so that we can create receiver functions. Wrote AddDescriptor func to add or update a descriptor in a pmt. Wrote ProgramInfoLen func to return the program info length i.e. len of descriptors. Wrote HasDescriptor to check if descriptor exists, if so return the descriptor so that we can update. Wrote descriptors which returns []byte of all descriptors. Wrote create descriptor, which adds a descriptor to the existing if any i.e. shifts data downwards to accomodate new data. Wrote update func to update a descriptor. --- stream/mts/encoder.go | 2 +- stream/mts/psi/psi.go | 94 ++++++++++++++++++++++++++++++++----------- 2 files changed, 72 insertions(+), 24 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 60fb6c07..9fbf72a9 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -253,6 +253,6 @@ func updateMeta(b []byte) error { var p psi.PSIBytes p = b m := meta.Format() - p.AddDescriptor(psi.MetadataTag, len(m), m) + p.Descriptor(psi.MetadataTag, len(m), m) return nil } diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 7571fb03..167b2034 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -85,29 +85,10 @@ const ( const MetadataTag = 0x26 -type PSIBytes []byte - -func (p *PSIBytes) AddDescriptor(tag int, data []byte) error { - // first check that we actually have table syntax section - // NB: if table syntax section doesn't exist (unlikely I think) then we could - // just add it - - // Check that this is actually a PMT - if psi.TableID(*p) != pmtID { - return errors.New("trying to add descriptor, but not pmt") - } - - if !p.editDesc(tag, data) { - - } - // if exists update - // otherwise add - return nil -} - -func (p *PSIBytes) editDesc(tag int, data []byte) bool { - return true -} +type ( + PSIBytes []byte + Descriptor []byte +) // Program specific information type PSI struct { @@ -246,3 +227,70 @@ func asByte(b bool) byte { } return 0x00 } + +func (p *PSIBytes) AddDescriptor(tag int, data []byte) error { + if psi.TableID(*p) != pmtID { + return errors.New("trying to add descriptor, but not pmt") + } + + desc := p.HasDescriptor(tag) + if desc == nil { + p.createDescriptor(tag, data) + return nil + } + desc.update(data) + return nil +} + +func (p *PSIBytes) ProgramInfoLen() int { + return int((((*p)[ProgramInfoLenIdx1] & ProgramInfoLenMask1) << 8) | (*p)[ProgramInfoLenIdx2]) +} + +func (p *PSIBytes) HasDescriptor(int tag) Descriptor { + descs := p.descriptors() + if descs == nil { + return nil + } + for i := 0; i < len(descs); { + t := descs[i] + if t == tag { + return esc[i : i+int(descs[i+1])] + } + i += 1 + desc[i+1] + } + return nil +} + +func (p *PSIBytes) descriptors() []byte { + return p[DescriptorsIdx : DescriptorsIdx+p.ProgramInfoLen()] +} + +func (p *PSIBytes) createDescriptor(tag, data) { + curProgLen := p.ProgramInfoLen() + dataLen := len(data) + + // Calculate the new descriptors index and length. + newDescIdx := DescriptorsIdx + curProgLen + newDescLen := dLen + 2 + + // Copy data down from newDescIdx to create room for the new descriptor. + copy(p[newDescIdx+newDescLen:], p[newDescIdx:dataLen-newDescLen]) + + // Set the tag, data len and data of the new desriptor. + p[newDescIdx] = tag + p[newDescIdx+1] = dataLen + copy(p[newDescIdx+2:newDescIdx+2+dataLen], data) + + // Update the program info length to account for the new descriptor. + newProgInfoLen := curProgLen + dataLen + p[ProgramInfoLenIdx1] = newProgInfoLen >> 8 + p[ProgramInfoLenIdx2] = byte(newProgInfoLen) +} + +func (d *Descriptor) update(data) { + if len(data) > d[1] { + // TODO: implement resizing of descriptor + panic("Can't resize descriptor data") + } + +} From 1be7e08b9e5afa1b664bebd2a4e2c879ad7a38f4 Mon Sep 17 00:00:00 2001 From: saxon Date: Sun, 27 Jan 2019 16:56:14 +1030 Subject: [PATCH 029/208] stream/mts: renamed Format function for metadata to Encode --- stream/mts/meta.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stream/mts/meta.go b/stream/mts/meta.go index c8d70ca2..7ab9cd3a 100644 --- a/stream/mts/meta.go +++ b/stream/mts/meta.go @@ -84,7 +84,7 @@ func (m *Meta) Delete(key string) error { return errors.New("Trying to delete map entry that doesn't exist") } -func (m *Meta) Format() []byte { +func (m *Meta) Encode() []byte { // TODO: complete this return []byte("someData") } From 46b2bc45204ec078da79305805b451b25ded8a76 Mon Sep 17 00:00:00 2001 From: saxon Date: Sun, 27 Jan 2019 17:04:46 +1030 Subject: [PATCH 030/208] stream/mts: added some consts to describe indexes and masks, and fixed some syntax errors, so now it all builds --- stream/mts/psi/psi.go | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 167b2034..441a7221 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -83,6 +83,16 @@ const ( SectionLenMask1 = 0x03 ) +const ( + ProgramInfoLenIdx1 = 11 + ProgramInfoLenIdx2 = 12 + ProgramInfoLenMask1 = 0x03 +) + +const ( + DescriptorsIdx = ProgramInfoLenIdx2 + 1 +) + const MetadataTag = 0x26 type ( @@ -246,49 +256,49 @@ func (p *PSIBytes) ProgramInfoLen() int { return int((((*p)[ProgramInfoLenIdx1] & ProgramInfoLenMask1) << 8) | (*p)[ProgramInfoLenIdx2]) } -func (p *PSIBytes) HasDescriptor(int tag) Descriptor { +func (p *PSIBytes) HasDescriptor(tag int) Descriptor { descs := p.descriptors() if descs == nil { return nil } for i := 0; i < len(descs); { - t := descs[i] + t := int(descs[i]) if t == tag { - return esc[i : i+int(descs[i+1])] + return descs[i : i+int(descs[i+1])] } - i += 1 + desc[i+1] + i += 1 + int(descs[i+1]) } return nil } func (p *PSIBytes) descriptors() []byte { - return p[DescriptorsIdx : DescriptorsIdx+p.ProgramInfoLen()] + return (*p)[DescriptorsIdx : DescriptorsIdx+p.ProgramInfoLen()] } -func (p *PSIBytes) createDescriptor(tag, data) { +func (p *PSIBytes) createDescriptor(tag int, data []byte) { curProgLen := p.ProgramInfoLen() dataLen := len(data) // Calculate the new descriptors index and length. newDescIdx := DescriptorsIdx + curProgLen - newDescLen := dLen + 2 + newDescLen := dataLen + 2 // Copy data down from newDescIdx to create room for the new descriptor. - copy(p[newDescIdx+newDescLen:], p[newDescIdx:dataLen-newDescLen]) + copy((*p)[newDescIdx+newDescLen:], (*p)[newDescIdx:dataLen-newDescLen]) // Set the tag, data len and data of the new desriptor. - p[newDescIdx] = tag - p[newDescIdx+1] = dataLen - copy(p[newDescIdx+2:newDescIdx+2+dataLen], data) + (*p)[newDescIdx] = byte(tag) + (*p)[newDescIdx+1] = byte(dataLen) + copy((*p)[newDescIdx+2:newDescIdx+2+dataLen], data) // Update the program info length to account for the new descriptor. newProgInfoLen := curProgLen + dataLen - p[ProgramInfoLenIdx1] = newProgInfoLen >> 8 - p[ProgramInfoLenIdx2] = byte(newProgInfoLen) + (*p)[ProgramInfoLenIdx1] = byte(newProgInfoLen >> 8) + (*p)[ProgramInfoLenIdx2] = byte(newProgInfoLen) } -func (d *Descriptor) update(data) { - if len(data) > d[1] { +func (d *Descriptor) update(data []byte) { + if len(data) > int((*d)[1]) { // TODO: implement resizing of descriptor panic("Can't resize descriptor data") } From c547c8bd19e28384999fe13a86631f9414e669b1 Mon Sep 17 00:00:00 2001 From: saxon Date: Sun, 27 Jan 2019 17:16:44 +1030 Subject: [PATCH 031/208] stream/mts: added meta_test.go file and wrote todos for testing --- stream/mts/meta_test.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 stream/mts/meta_test.go diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go new file mode 100644 index 00000000..6796611f --- /dev/null +++ b/stream/mts/meta_test.go @@ -0,0 +1,36 @@ +/* +NAME + meta_test.go + +DESCRIPTION + See Readme.md + +AUTHOR + Saxon Nelson-Milton + +LICENSE + meta_test.go is Copyright (C) 2017-2019 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ + +package mts + +// TODO: test add when key doesn't exist +// TODO: test add when key does exist +// TODO: test all +// TODO: test get when key exists +// TODO: test get when key doesn't exist +// TODO: test delete when key exists +// TODO: test delete when key doesn't exist From 3bb1ca9379634250c570d645b11d27051c5978ff Mon Sep 17 00:00:00 2001 From: saxon Date: Sun, 27 Jan 2019 17:57:42 +1030 Subject: [PATCH 032/208] stream/mts: wrote NewMeta func to initialize map in the Meta struct. Also wrote two tests, one TestAddAndGet to see if we can add and get metadata, and also second TestUpdate to see if we can correctly update metadata with the same key using Meta.Add --- stream/mts/encoder.go | 4 ++-- stream/mts/meta.go | 12 +++++------- stream/mts/meta_test.go | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 9fbf72a9..8f46a15a 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -252,7 +252,7 @@ func (e *Encoder) ccFor(pid int) byte { func updateMeta(b []byte) error { var p psi.PSIBytes p = b - m := meta.Format() - p.Descriptor(psi.MetadataTag, len(m), m) + m := meta.Encode() + p.AddDescriptor(psi.MetadataTag, m) return nil } diff --git a/stream/mts/meta.go b/stream/mts/meta.go index 7ab9cd3a..57787452 100644 --- a/stream/mts/meta.go +++ b/stream/mts/meta.go @@ -29,7 +29,6 @@ package mts import ( "errors" - "fmt" "sync" ) @@ -38,15 +37,15 @@ type Meta struct { data map[string]string } +func NewMeta() *Meta { + return &Meta{data: make(map[string]string)} +} + // Add adds metadata with key and val, if already exists return error -func (m *Meta) Add(key, val string) error { +func (m *Meta) Add(key, val string) { m.mu.Lock() - if _, exists := m.data[key]; !exists { - return errors.New(fmt.Sprintf("Metadata for: %v already exists", key)) - } m.data[key] = val m.mu.Unlock() - return nil } // All returns the a copy of the map containing the meta data @@ -70,7 +69,6 @@ func (m *Meta) Get(key string) (string, error) { } return val, nil - } // Remove deletes a meta entry in the map and returns error if it doesn’t exist diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go index 6796611f..3b0662c9 100644 --- a/stream/mts/meta_test.go +++ b/stream/mts/meta_test.go @@ -27,8 +27,45 @@ LICENSE package mts +import ( + "testing" +) + // TODO: test add when key doesn't exist +func TestAddAndGet(t *testing.T) { + meta := NewMeta() + meta.Add("loc", "a,b,c") + meta.Add("ts", "12345678") + + if data, err := meta.Get("loc"); err != nil { + t.Errorf("Could not get data for key: loc: %v", err.Error()) + if data != "a,b,c" { + t.Errorf("Did not get expected data") + } + } + + if data, err := meta.Get("ts"); err != nil { + t.Errorf("Could not get data for key: ts: %v", err.Error()) + if data != "12345678" { + t.Errorf("Did not get expected data") + } + } +} + // TODO: test add when key does exist +func TestUpdate(t *testing.T) { + meta := NewMeta() + meta.Add("loc", "a,b,c") + meta.Add("loc", "d,e,f") + + if data, err := meta.Get("loc"); err != nil { + t.Errorf("Did not expect err: %v", err.Error()) + if data != "d,e,f" { + t.Errorf("Data did not correctly update for key \"loc\"") + } + } +} + // TODO: test all // TODO: test get when key exists // TODO: test get when key doesn't exist From 17d06f49f4063a9deea8262eaccc323f5519a0a1 Mon Sep 17 00:00:00 2001 From: saxon Date: Sun, 27 Jan 2019 18:02:51 +1030 Subject: [PATCH 033/208] stream/mts/meta_test.go: added TestAll func to make sure Meta.All is working correctly --- stream/mts/meta_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go index 3b0662c9..e5ae04e6 100644 --- a/stream/mts/meta_test.go +++ b/stream/mts/meta_test.go @@ -28,6 +28,7 @@ LICENSE package mts import ( + "reflect" "testing" ) @@ -66,6 +67,22 @@ func TestUpdate(t *testing.T) { } } +func TestAll(t *testing.T) { + meta := NewMeta() + tstMap := map[string]string{ + "loc": "a,b,c", + "ts": "12345678", + } + + meta.Add("loc", "a,b,c") + meta.Add("ts", "12345678") + metaMap := meta.All() + + if !reflect.DeepEqual(metaMap, tstMap) { + t.Errorf("Map not correct. Got: %v, want: %v", metaMap, tstMap) + } +} + // TODO: test all // TODO: test get when key exists // TODO: test get when key doesn't exist From bd54dd128ba21775264c3b81f89f41c66f94a27f Mon Sep 17 00:00:00 2001 From: saxon Date: Sun, 27 Jan 2019 18:15:22 +1030 Subject: [PATCH 034/208] stream/mts/meta_test.go: added TestGetAbsent key to check that we get an error when we try and get data with key that doesn't exist in metadata map --- stream/mts/meta.go | 8 ++++++-- stream/mts/meta_test.go | 10 ++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/stream/mts/meta.go b/stream/mts/meta.go index 57787452..c1e29ab0 100644 --- a/stream/mts/meta.go +++ b/stream/mts/meta.go @@ -32,6 +32,10 @@ import ( "sync" ) +var ( + errKeyAbsent = errors.New("Key does not exist in map") +) + type Meta struct { mu sync.RWMutex data map[string]string @@ -51,7 +55,7 @@ func (m *Meta) Add(key, val string) { // All returns the a copy of the map containing the meta data func (m *Meta) All() map[string]string { m.mu.Lock() - var cpy map[string]string + cpy := make(map[string]string) for k, v := range m.data { cpy[k] = v } @@ -65,7 +69,7 @@ func (m *Meta) Get(key string) (string, error) { val, ok := m.data[key] m.mu.Unlock() if !ok { - return "", errors.New("Key does not exist in metadata map") + return "", errKeyAbsent } return val, nil diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go index e5ae04e6..73727b5e 100644 --- a/stream/mts/meta_test.go +++ b/stream/mts/meta_test.go @@ -83,8 +83,14 @@ func TestAll(t *testing.T) { } } -// TODO: test all -// TODO: test get when key exists // TODO: test get when key doesn't exist +func TestGetAbsentKey(t *testing.T) { + meta := NewMeta() + + if _, err := meta.Get("loc"); err != errKeyAbsent { + t.Errorf("Expected error: %v", errKeyAbsent.Error()) + } +} + // TODO: test delete when key exists // TODO: test delete when key doesn't exist From c16f1443729b399a3034ee382355356fd2159c7a Mon Sep 17 00:00:00 2001 From: saxon Date: Sun, 27 Jan 2019 18:21:49 +1030 Subject: [PATCH 035/208] stream/mts/meta_test.go: added TestDelete to check that deleting a particular metadata entry by key works --- stream/mts/meta_test.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go index 73727b5e..70795a28 100644 --- a/stream/mts/meta_test.go +++ b/stream/mts/meta_test.go @@ -88,7 +88,18 @@ func TestGetAbsentKey(t *testing.T) { meta := NewMeta() if _, err := meta.Get("loc"); err != errKeyAbsent { - t.Errorf("Expected error: %v", errKeyAbsent.Error()) + t.Errorf("Did not get expected err: %v", errKeyAbsent.Error()) + } +} + +func TestDelete(t *testing.T) { + meta := NewMeta() + meta.Add("loc", "a,b,c") + if err := meta.Delete("loc"); err != nil { + t.Errorf("Did not expect error: %v", err.Error()) + } + if _, err := meta.Get("loc"); err != errKeyAbsent { + t.Errorf("Did not get expected err: %v", errKeyAbsent) } } From 7fc2b76e6b3230c8fa82d94d842d9d9006284a10 Mon Sep 17 00:00:00 2001 From: saxon Date: Sun, 27 Jan 2019 18:24:26 +1030 Subject: [PATCH 036/208] stream/mts/meta_test.go: added TestDeleteKeyAbsent to check that we get correct err when we try to delete data of key that doesn't exist --- stream/mts/meta.go | 2 +- stream/mts/meta_test.go | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/stream/mts/meta.go b/stream/mts/meta.go index c1e29ab0..1f482790 100644 --- a/stream/mts/meta.go +++ b/stream/mts/meta.go @@ -83,7 +83,7 @@ func (m *Meta) Delete(key string) error { delete(m.data, key) return nil } - return errors.New("Trying to delete map entry that doesn't exist") + return errKeyAbsent } func (m *Meta) Encode() []byte { diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go index 70795a28..6c81e4c5 100644 --- a/stream/mts/meta_test.go +++ b/stream/mts/meta_test.go @@ -99,7 +99,14 @@ func TestDelete(t *testing.T) { t.Errorf("Did not expect error: %v", err.Error()) } if _, err := meta.Get("loc"); err != errKeyAbsent { - t.Errorf("Did not get expected err: %v", errKeyAbsent) + t.Errorf("Did not get expected err: %v", errKeyAbsent.Error()) + } +} + +func TestDeleteAbsentKey(t *testing.T) { + meta := NewMeta() + if err := meta.Delete("loc"); err != errKeyAbsent { + t.Errorf("Did not get expected err: %v", errKeyAbsent.Error()) } } From e1ac1ac5d418734e1b89629bb92d2cd111ee5787 Mon Sep 17 00:00:00 2001 From: saxon Date: Sun, 27 Jan 2019 18:29:45 +1030 Subject: [PATCH 037/208] stream/mts/meta_test.go: using consts for commonly occuring test strings --- stream/mts/meta_test.go | 55 ++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go index 6c81e4c5..fa8e545c 100644 --- a/stream/mts/meta_test.go +++ b/stream/mts/meta_test.go @@ -28,40 +28,47 @@ LICENSE package mts import ( + "errors" "reflect" "testing" ) -// TODO: test add when key doesn't exist +const ( + tstKey1 = "loc" + tstData1 = "a,b,c" + tstKey2 = "ts" + tstData2 = "12345678" + tstData3 = "d,e,f" +) + func TestAddAndGet(t *testing.T) { meta := NewMeta() - meta.Add("loc", "a,b,c") - meta.Add("ts", "12345678") - - if data, err := meta.Get("loc"); err != nil { + meta.Add(tstKey1, tstData1) + meta.Add(tstKey2, tstData2) + errors.New("Trying to delete map entry that doesn't exist") + if data, err := meta.Get(tstKey1); err != nil { t.Errorf("Could not get data for key: loc: %v", err.Error()) - if data != "a,b,c" { + if data != tstData1 { t.Errorf("Did not get expected data") } } - if data, err := meta.Get("ts"); err != nil { + if data, err := meta.Get(tstKey2); err != nil { t.Errorf("Could not get data for key: ts: %v", err.Error()) - if data != "12345678" { + if data != tstData2 { t.Errorf("Did not get expected data") } } } -// TODO: test add when key does exist func TestUpdate(t *testing.T) { meta := NewMeta() - meta.Add("loc", "a,b,c") - meta.Add("loc", "d,e,f") + meta.Add(tstKey1, tstData1) + meta.Add(tstKey1, tstData3) - if data, err := meta.Get("loc"); err != nil { + if data, err := meta.Get(tstKey1); err != nil { t.Errorf("Did not expect err: %v", err.Error()) - if data != "d,e,f" { + if data != tstData2 { t.Errorf("Data did not correctly update for key \"loc\"") } } @@ -70,12 +77,12 @@ func TestUpdate(t *testing.T) { func TestAll(t *testing.T) { meta := NewMeta() tstMap := map[string]string{ - "loc": "a,b,c", - "ts": "12345678", + tstKey1: tstData1, + tstKey2: tstData2, } - meta.Add("loc", "a,b,c") - meta.Add("ts", "12345678") + meta.Add(tstKey1, tstData1) + meta.Add(tstKey2, tstData2) metaMap := meta.All() if !reflect.DeepEqual(metaMap, tstMap) { @@ -83,32 +90,28 @@ func TestAll(t *testing.T) { } } -// TODO: test get when key doesn't exist func TestGetAbsentKey(t *testing.T) { meta := NewMeta() - if _, err := meta.Get("loc"); err != errKeyAbsent { + if _, err := meta.Get(tstKey1); err != errKeyAbsent { t.Errorf("Did not get expected err: %v", errKeyAbsent.Error()) } } func TestDelete(t *testing.T) { meta := NewMeta() - meta.Add("loc", "a,b,c") - if err := meta.Delete("loc"); err != nil { + meta.Add(tstKey1, tstData1) + if err := meta.Delete(tstKey1); err != nil { t.Errorf("Did not expect error: %v", err.Error()) } - if _, err := meta.Get("loc"); err != errKeyAbsent { + if _, err := meta.Get(tstKey1); err != errKeyAbsent { t.Errorf("Did not get expected err: %v", errKeyAbsent.Error()) } } func TestDeleteAbsentKey(t *testing.T) { meta := NewMeta() - if err := meta.Delete("loc"); err != errKeyAbsent { + if err := meta.Delete(tstKey1); err != errKeyAbsent { t.Errorf("Did not get expected err: %v", errKeyAbsent.Error()) } } - -// TODO: test delete when key exists -// TODO: test delete when key doesn't exist From 601351b021bf128e7fe85ad1b9b1f0bbcb8d4cd5 Mon Sep 17 00:00:00 2001 From: saxon Date: Sun, 27 Jan 2019 18:33:15 +1030 Subject: [PATCH 038/208] stream/mts/meta_test.go: adding comments to meta_test.go testing functions --- stream/mts/meta_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go index fa8e545c..14adf278 100644 --- a/stream/mts/meta_test.go +++ b/stream/mts/meta_test.go @@ -41,6 +41,7 @@ const ( tstData3 = "d,e,f" ) +// TestAddAndGet ensures that we can add metadata and then successfully get it. func TestAddAndGet(t *testing.T) { meta := NewMeta() meta.Add(tstKey1, tstData1) @@ -61,6 +62,8 @@ func TestAddAndGet(t *testing.T) { } } +// TestUpdate checks that we can use Meta.Add to actually update metadata +// if it already exists in the Meta map. func TestUpdate(t *testing.T) { meta := NewMeta() meta.Add(tstKey1, tstData1) @@ -74,6 +77,7 @@ func TestUpdate(t *testing.T) { } } +// TestAll ensures we can get a correct map using Meta.All() after adding some data func TestAll(t *testing.T) { meta := NewMeta() tstMap := map[string]string{ @@ -90,6 +94,8 @@ func TestAll(t *testing.T) { } } +// TestGetAbsentKey ensures that we get the expected error when we try to get with +// key that does not yet exist in the Meta map. func TestGetAbsentKey(t *testing.T) { meta := NewMeta() @@ -98,6 +104,7 @@ func TestGetAbsentKey(t *testing.T) { } } +// TestDelete ensures we can remove a data entry in the Meta map. func TestDelete(t *testing.T) { meta := NewMeta() meta.Add(tstKey1, tstData1) @@ -109,6 +116,8 @@ func TestDelete(t *testing.T) { } } +// TestDeleteAbsentKey checks that we get an expected error when we try to delete +// an entry in the Meta map that doesn't exist. func TestDeleteAbsentKey(t *testing.T) { meta := NewMeta() if err := meta.Delete(tstKey1); err != errKeyAbsent { From 00816ecf84544c97178525ff9f72b1ae49e585b4 Mon Sep 17 00:00:00 2001 From: saxon Date: Sun, 27 Jan 2019 18:34:35 +1030 Subject: [PATCH 039/208] stream/mts/meta_test.go: added test to meta_test.go to remind me to add a test function once the Meta.Encode() function is complete --- stream/mts/meta_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go index 14adf278..8b871dcb 100644 --- a/stream/mts/meta_test.go +++ b/stream/mts/meta_test.go @@ -124,3 +124,5 @@ func TestDeleteAbsentKey(t *testing.T) { t.Errorf("Did not get expected err: %v", errKeyAbsent.Error()) } } + +// TODO: add test function for Encoding the Meta map data From 7d34fa1969007b4a531dd6fdeb8129eba082ba30 Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 28 Jan 2019 17:29:15 +1030 Subject: [PATCH 040/208] stream/mts/meta.go: completed Meta.Encode() function --- stream/mts/meta.go | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/stream/mts/meta.go b/stream/mts/meta.go index 1f482790..c25d3e85 100644 --- a/stream/mts/meta.go +++ b/stream/mts/meta.go @@ -32,6 +32,13 @@ import ( "sync" ) +const headSize = 4 + +const ( + majVer = 1 + minVer = 0 +) + var ( errKeyAbsent = errors.New("Key does not exist in map") ) @@ -39,10 +46,19 @@ var ( type Meta struct { mu sync.RWMutex data map[string]string + enc []byte } func NewMeta() *Meta { - return &Meta{data: make(map[string]string)} + return &Meta{ + data: make(map[string]string), + enc: []byte{ + 0x00, // Reserved byte + (majVer << 4) | minVer, // MS and LS versions + 0x00, // Data len byte1 + 0x00, // Data len byte2 + }, + } } // Add adds metadata with key and val, if already exists return error @@ -87,6 +103,12 @@ func (m *Meta) Delete(key string) error { } func (m *Meta) Encode() []byte { - // TODO: complete this - return []byte("someData") + m.enc = m.enc[:headSize] + for k, v := range m.data { + entry := k + "=" + v + "\t" + m.enc = append(m.enc, []byte(entry)...) + } + // Remove final tab + m.enc = m.enc[:len(m.enc)-1] + return m.enc } From 960c110acb561ef8ff496885a63e9e6f39b063d6 Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 28 Jan 2019 17:45:38 +1030 Subject: [PATCH 041/208] stream/mts/meta.go: fixed Meta.Encode() func so that it calculates data length correctly --- stream/mts/meta.go | 25 +++++++++++++++++++++---- stream/mts/meta_test.go | 4 +++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/stream/mts/meta.go b/stream/mts/meta.go index c25d3e85..c12998bd 100644 --- a/stream/mts/meta.go +++ b/stream/mts/meta.go @@ -39,6 +39,11 @@ const ( minVer = 0 ) +const ( + dataLenIdx1 = 2 + dataLenIdx2 = 3 +) + var ( errKeyAbsent = errors.New("Key does not exist in map") ) @@ -104,11 +109,23 @@ func (m *Meta) Delete(key string) error { func (m *Meta) Encode() []byte { m.enc = m.enc[:headSize] + + // Iterate over map and append entries, only adding tab if we're not on the last entry + var i int + var entry string for k, v := range m.data { - entry := k + "=" + v + "\t" - m.enc = append(m.enc, []byte(entry)...) + i++ + entry += k + "=" + v + if i < len(m.data) { + entry += "\t" + } } - // Remove final tab - m.enc = m.enc[:len(m.enc)-1] + m.enc = append(m.enc, []byte(entry)...) + + // Calculate and set data length in encoded meta header. + dataLen := len(m.enc[headSize:]) + m.enc[dataLenIdx1] = byte(dataLen >> 8) + m.enc[dataLenIdx2] = byte(dataLen) + return m.enc } diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go index 8b871dcb..07f0e5e2 100644 --- a/stream/mts/meta_test.go +++ b/stream/mts/meta_test.go @@ -125,4 +125,6 @@ func TestDeleteAbsentKey(t *testing.T) { } } -// TODO: add test function for Encoding the Meta map data +func TestEncode(t *testing.T) { + meta := NewMeta() +} From 3aa0efc16a128fcd4fe8b7c12eb257f89f02b7f7 Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 28 Jan 2019 18:07:34 +1030 Subject: [PATCH 042/208] stream/mts/meta_test.go: added TestEncode to test Meta.Encoding function --- stream/mts/meta_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go index 07f0e5e2..010c9b82 100644 --- a/stream/mts/meta_test.go +++ b/stream/mts/meta_test.go @@ -28,6 +28,7 @@ LICENSE package mts import ( + "bytes" "errors" "reflect" "testing" @@ -127,4 +128,23 @@ func TestDeleteAbsentKey(t *testing.T) { func TestEncode(t *testing.T) { meta := NewMeta() + meta.Add(tstKey1, tstData1) + meta.Add(tstKey2, tstData2) + + dataLen := len(tstKey1+tstData1+tstKey2+tstData2) + 3 + expectedOut := []byte{ + 0x00, + 0x10, + byte(dataLen >> 8), + byte(dataLen), + } + expectedOut = append(expectedOut, []byte( + tstKey1+"="+tstData1+"\t"+ + tstKey2+"="+tstData2)...) + + got := meta.Encode() + + if !bytes.Equal(expectedOut, got) { + t.Errorf("Did not get expected out. \nGot : %v \nwant: %v", got, expectedOut) + } } From 5aff27ac4dbebfc809d7b0b30cb3ba1433bc9e2b Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 29 Jan 2019 10:40:22 +1030 Subject: [PATCH 043/208] stream/mts/psi/descriptor_test.go: wrote test to check ProgramInfoLen func and descriptors func --- stream/mts/meta.go | 2 ++ stream/mts/meta_test.go | 1 + 2 files changed, 3 insertions(+) diff --git a/stream/mts/meta.go b/stream/mts/meta.go index c12998bd..465a537e 100644 --- a/stream/mts/meta.go +++ b/stream/mts/meta.go @@ -107,6 +107,8 @@ func (m *Meta) Delete(key string) error { return errKeyAbsent } +// Encode takes the meta data map and encods into a byte slice with header +// describing the version, length of data and data in TSV format. func (m *Meta) Encode() []byte { m.enc = m.enc[:headSize] diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go index 010c9b82..daff9038 100644 --- a/stream/mts/meta_test.go +++ b/stream/mts/meta_test.go @@ -126,6 +126,7 @@ func TestDeleteAbsentKey(t *testing.T) { } } +// TestEncode checks that we're getting the correct byte slice from Meta.Encode(). func TestEncode(t *testing.T) { meta := NewMeta() meta.Add(tstKey1, tstData1) From dce113d1e458556ad222f00e312cf74a0c366461 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 29 Jan 2019 11:28:40 +1030 Subject: [PATCH 044/208] stream/mts/psi/descriptor_test.go: wrote HasDescriptor test and fixed HasDescriptor function after bug was found --- stream/mts/psi/psi.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 441a7221..2f4ab5ad 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -264,9 +264,9 @@ func (p *PSIBytes) HasDescriptor(tag int) Descriptor { for i := 0; i < len(descs); { t := int(descs[i]) if t == tag { - return descs[i : i+int(descs[i+1])] + return descs[i : i+2+int(descs[i+1])] } - i += 1 + int(descs[i+1]) + i += 2 + int(descs[i+1]) } return nil } From 3a14b644802b5537edf14662e98306f7f5945031 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 29 Jan 2019 11:30:37 +1030 Subject: [PATCH 045/208] stream/mts/psi/psi.go: simplified HasDescriptor func --- stream/mts/psi/psi.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 2f4ab5ad..c2e30f9e 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -261,12 +261,10 @@ func (p *PSIBytes) HasDescriptor(tag int) Descriptor { if descs == nil { return nil } - for i := 0; i < len(descs); { - t := int(descs[i]) - if t == tag { + for i := 0; i < len(descs); i += 2 + int(descs[i+1]) { + if int(descs[i]) == tag { return descs[i : i+2+int(descs[i+1])] } - i += 2 + int(descs[i+1]) } return nil } From af239838d0038d556985453a42e27d5171731529 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 29 Jan 2019 14:16:33 +1030 Subject: [PATCH 046/208] stream/mts/psi: modified how update crc works so that we're only giving it data that's included in crc calc as well as the crc field itself. Added some consts to allow working with SyntaxSectionLen. Added trimPadding func so that we can easily get rid of any padding. Fixed createDescriptor func which was found to be broken from TestCreateDescriptor --- stream/mts/psi/crc.go | 4 ++-- stream/mts/psi/helpers.go | 9 ++++----- stream/mts/psi/psi.go | 32 +++++++++++++++++++++++++------- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/stream/mts/psi/crc.go b/stream/mts/psi/crc.go index a361307d..cd9a1199 100644 --- a/stream/mts/psi/crc.go +++ b/stream/mts/psi/crc.go @@ -37,13 +37,13 @@ import ( func addCrc(out []byte) []byte { t := make([]byte, len(out)+4) copy(t, out) - updateCrc(t) + updateCrc(t[1:]) return t } // updateCrc updates the crc of bytes slice, writing the checksum into the last four bytes. func updateCrc(b []byte) { - crc32 := crc32_Update(0xffffffff, crc32_MakeTable(bits.Reverse32(crc32.IEEE)), b[1:len(b)-4]) + crc32 := crc32_Update(0xffffffff, crc32_MakeTable(bits.Reverse32(crc32.IEEE)), b[:len(b)-4]) binary.BigEndian.PutUint32(b[len(b)-4:], crc32) } diff --git a/stream/mts/psi/helpers.go b/stream/mts/psi/helpers.go index 82d7ce72..174b289b 100644 --- a/stream/mts/psi/helpers.go +++ b/stream/mts/psi/helpers.go @@ -64,15 +64,14 @@ func UpdateTime(dst []byte, t uint64) error { for i := range dst[TimeDataIndx : TimeDataIndx+TimeDataSize] { dst[i+TimeDataIndx] = ts[i] } - updateCrc(dst) + updateCrc(dst[1:]) return nil } // SyntaxSecLenFrom takes a byte slice representation of a psi and extracts // it's syntax section length -func SyntaxSecLenFrom(p []byte) (l uint8) { - l = uint8(p[syntaxSecLenIndx]) - crcSize - return +func SyntaxSecLenFrom(p []byte) int { + return int(((p[SyntaxSecLenIdx1] & SyntaxSecLenMask1) << 8) | p[SyntaxSecLenIdx2]) } // TimeFrom takes a byte slice representation of a psi-pmt and extracts it's @@ -112,7 +111,7 @@ func UpdateLocation(d []byte, s string) error { for i := range loc { loc[i] = 0 } - updateCrc(d) + updateCrc(d[1:]) return nil } diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index c2e30f9e..6a4a9103 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -70,8 +70,10 @@ const ( // Other misc consts const ( - syntaxSecLenIndx = 3 - crcSize = 4 + SyntaxSecLenIdx1 = 2 + SyntaxSecLenIdx2 = 3 + SyntaxSecLenMask1 = 0x03 + crcSize = 4 ) const ( @@ -275,24 +277,40 @@ func (p *PSIBytes) descriptors() []byte { func (p *PSIBytes) createDescriptor(tag int, data []byte) { curProgLen := p.ProgramInfoLen() + oldSyntaxSectionLen := SyntaxSecLenFrom(*p) dataLen := len(data) // Calculate the new descriptors index and length. newDescIdx := DescriptorsIdx + curProgLen newDescLen := dataLen + 2 - // Copy data down from newDescIdx to create room for the new descriptor. - copy((*p)[newDescIdx+newDescLen:], (*p)[newDescIdx:dataLen-newDescLen]) - + copy((*p)[newDescIdx+newDescLen:], (*p)[newDescIdx:newDescIdx+newDescLen]) // Set the tag, data len and data of the new desriptor. (*p)[newDescIdx] = byte(tag) (*p)[newDescIdx+1] = byte(dataLen) copy((*p)[newDescIdx+2:newDescIdx+2+dataLen], data) // Update the program info length to account for the new descriptor. - newProgInfoLen := curProgLen + dataLen - (*p)[ProgramInfoLenIdx1] = byte(newProgInfoLen >> 8) + // TODO: put this in function set program info length + addedLen := dataLen + 2 + newProgInfoLen := curProgLen + addedLen + (*p)[ProgramInfoLenIdx1] &= 0xff ^ ProgramInfoLenMask1 + (*p)[ProgramInfoLenIdx1] |= byte(newProgInfoLen>>8) & ProgramInfoLenMask1 (*p)[ProgramInfoLenIdx2] = byte(newProgInfoLen) + + // set section length + // TODO: put this in func set program info length + newSyntaxSectionLen := int(oldSyntaxSectionLen) + addedLen + (*p)[SyntaxSecLenIdx1] &= 0xff ^ SyntaxSecLenMask1 + (*p)[SyntaxSecLenIdx1] |= byte(newSyntaxSectionLen>>8) & SyntaxSecLenMask1 + (*p)[SyntaxSecLenIdx2] = byte(newSyntaxSectionLen) +} + +func (p *PSIBytes) trimPadding() []byte { + sectionLength := SyntaxSecLenFrom(*p) + paddingIdx := (4 + sectionLength) + o := (*p)[:paddingIdx] + return o } func (d *Descriptor) update(data []byte) { From a61bdac2de12e51858264303b7151a4f01c22625 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 29 Jan 2019 14:21:25 +1030 Subject: [PATCH 047/208] stream/mts/psi: added testing file with tests --- stream/mts/psi/descriptor_test.go | 224 ++++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 stream/mts/psi/descriptor_test.go diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go new file mode 100644 index 00000000..cb25761c --- /dev/null +++ b/stream/mts/psi/descriptor_test.go @@ -0,0 +1,224 @@ +/* +NAME + descriptor_test.go + +DESCRIPTION + See Readme.md + +AUTHOR + Saxon Nelson-Milton + +LICENSE + descriptor_test.go is Copyright (C) 2017-2019 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ + +package psi + +import ( + "bytes" + "testing" +) + +const ( + errNotExpectedOut = "Did not get expected output: \ngot : %v, \nwant: %v" +) + +var tstPsi1 = PSI{ + Pf: 0x00, + Tid: 0x02, + Ssi: true, + Sl: 0x1c, + Tss: &TSS{ + Tide: 0x01, + V: 0, + Cni: true, + Sn: 0, + Lsn: 0, + Sd: &PMT{ + Pcrpid: 0x0100, // wrong + Pil: 10, + Pd: []Desc{ + { + Dt: TimeDescTag, + Dl: TimeDataSize, + Dd: make([]byte, TimeDataSize), + }, + }, + Essd: &ESSD{ + St: 0x1b, + Epid: 0x0100, + Esil: 0x00, + }, + }, + }, +} + +var tstPsi2 = PSI{ + Pf: 0x00, + Tid: 0x02, + Ssi: true, + Sl: 0x12, + Tss: &TSS{ + Tide: 0x01, + V: 0, + Cni: true, + Sn: 0, + Lsn: 0, + Sd: &PMT{ + Pcrpid: 0x0100, + Pil: 0, + Essd: &ESSD{ + St: 0x1b, + Epid: 0x0100, + Esil: 0x00, + }, + }, + }, +} + +var tstPsi3 = PSI{ + Pf: 0x00, + Tid: 0x02, + Ssi: true, + Sl: 0x3e, + Tss: &TSS{ + Tide: 0x01, + V: 0, + Cni: true, + Sn: 0, + Lsn: 0, + Sd: &PMT{ + Pcrpid: 0x0100, + Pil: PmtTimeLocationPil, + Pd: []Desc{ + { + Dt: TimeDescTag, + Dl: TimeDataSize, + Dd: make([]byte, TimeDataSize), + }, + { + Dt: LocationDescTag, + Dl: LocationDataSize, + Dd: make([]byte, LocationDataSize), + }, + }, + Essd: &ESSD{ + St: 0x1b, + Epid: 0x0100, + Esil: 0x00, + }, + }, + }, +} + +func TestHasDescriptor(t *testing.T) { + p := PSIBytes(tstPsi3.Bytes()) + + // Try getting descriptor that exists + got := p.HasDescriptor(LocationDescTag) + want := []byte{ + LocationDescTag, + LocationDataSize, + } + want = append(want, make([]byte, LocationDataSize)...) + + if !bytes.Equal(got, want) { + t.Errorf(errNotExpectedOut, got, want) + } + + // Try getting descriptor that doesnt exist + const fakeTag = 236 + got = p.HasDescriptor(fakeTag) + want = nil + + if !bytes.Equal(got, want) { + t.Errorf(errNotExpectedOut, got, want) + } + + // Try getting descriptor from psi that has no descriptors + p = PSIBytes(tstPsi2.Bytes()) + + got = p.HasDescriptor(LocationDescTag) + want = nil + + if !bytes.Equal(got, want) { + t.Errorf(errNotExpectedOut, got, want) + } +} + +func TestProgramInfoLen(t *testing.T) { + p := PSIBytes(tstPsi1.Bytes()) + got := p.ProgramInfoLen() + want := 10 + if got != want { + t.Errorf(errNotExpectedOut, got, want) + } +} + +func TestDescriptors(t *testing.T) { + // Try psi with descriptors + p := PSIBytes(tstPsi1.Bytes()) + got := p.descriptors() + want := []byte{ + TimeDescTag, + TimeDataSize, + } + want = append(want, make([]byte, TimeDataSize)...) + + if !bytes.Equal(got, want) { + t.Errorf(errNotExpectedOut, got, want) + } + + // Now try psi with empty descriptors + p = PSIBytes(tstPsi2.Bytes()) + got = p.descriptors() + want = nil + if !bytes.Equal(got, want) { + t.Errorf(errNotExpectedOut, got, want) + } +} + +func TestCreateDescriptor(t *testing.T) { + // Test with PSI containing no descriptors + p := PSIBytes(tstPsi2.Bytes()) + p.createDescriptor(TimeDescTag, make([]byte, TimeDataSize)) + // update crc + noPadding := p.trimPadding() + updateCrc(noPadding[1:]) + want := PSIBytes(tstPsi1.Bytes()) + got := p + + if !bytes.Equal(want, got) { + t.Errorf(errNotExpectedOut, got, want) + } + + // Test with psi containing descriptor already + p = PSIBytes(tstPsi1.Bytes()) + p.createDescriptor(LocationDescTag, make([]byte, LocationDataSize)) + // update crc + noPadding = p.trimPadding() + updateCrc(noPadding[1:]) + want = PSIBytes(tstPsi3.Bytes()) + got = p + + if !bytes.Equal(want, got) { + t.Errorf(errNotExpectedOut, got, want) + } +} + +func TestUpdateDescriptor(t *testing.T) { + +} From 4a3464252b78ecf3ee062ff03574ecda669ec746 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 29 Jan 2019 14:38:23 +1030 Subject: [PATCH 048/208] stream/mts/psi: changed HasDescriptor to also return index of descriptor in psi. Wrote func called SetProgInfoLen to set the program info length in a pmt. Started writing deleteDescriptor func to get rid of a descriptor. --- stream/mts/psi/descriptor_test.go | 10 +++++++--- stream/mts/psi/psi.go | 33 +++++++++++++++++++++++-------- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go index cb25761c..919653fa 100644 --- a/stream/mts/psi/descriptor_test.go +++ b/stream/mts/psi/descriptor_test.go @@ -128,7 +128,7 @@ func TestHasDescriptor(t *testing.T) { p := PSIBytes(tstPsi3.Bytes()) // Try getting descriptor that exists - got := p.HasDescriptor(LocationDescTag) + _, got := p.HasDescriptor(LocationDescTag) want := []byte{ LocationDescTag, LocationDataSize, @@ -141,7 +141,7 @@ func TestHasDescriptor(t *testing.T) { // Try getting descriptor that doesnt exist const fakeTag = 236 - got = p.HasDescriptor(fakeTag) + _, got = p.HasDescriptor(fakeTag) want = nil if !bytes.Equal(got, want) { @@ -151,7 +151,7 @@ func TestHasDescriptor(t *testing.T) { // Try getting descriptor from psi that has no descriptors p = PSIBytes(tstPsi2.Bytes()) - got = p.HasDescriptor(LocationDescTag) + _, got = p.HasDescriptor(LocationDescTag) want = nil if !bytes.Equal(got, want) { @@ -219,6 +219,10 @@ func TestCreateDescriptor(t *testing.T) { } } +func TestDeleteDescriptor(t *testing.T) { + +} + func TestUpdateDescriptor(t *testing.T) { } diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 6a4a9103..8832dfb4 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -245,7 +245,7 @@ func (p *PSIBytes) AddDescriptor(tag int, data []byte) error { return errors.New("trying to add descriptor, but not pmt") } - desc := p.HasDescriptor(tag) + _, desc := p.HasDescriptor(tag) if desc == nil { p.createDescriptor(tag, data) return nil @@ -258,17 +258,17 @@ func (p *PSIBytes) ProgramInfoLen() int { return int((((*p)[ProgramInfoLenIdx1] & ProgramInfoLenMask1) << 8) | (*p)[ProgramInfoLenIdx2]) } -func (p *PSIBytes) HasDescriptor(tag int) Descriptor { +func (p *PSIBytes) HasDescriptor(tag int) (int, Descriptor) { descs := p.descriptors() if descs == nil { - return nil + return -1, nil } for i := 0; i < len(descs); i += 2 + int(descs[i+1]) { if int(descs[i]) == tag { - return descs[i : i+2+int(descs[i+1])] + return i, descs[i : i+2+int(descs[i+1])] } } - return nil + return -1, nil } func (p *PSIBytes) descriptors() []byte { @@ -283,8 +283,10 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) { // Calculate the new descriptors index and length. newDescIdx := DescriptorsIdx + curProgLen newDescLen := dataLen + 2 + // Copy data down from newDescIdx to create room for the new descriptor. copy((*p)[newDescIdx+newDescLen:], (*p)[newDescIdx:newDescIdx+newDescLen]) + // Set the tag, data len and data of the new desriptor. (*p)[newDescIdx] = byte(tag) (*p)[newDescIdx+1] = byte(dataLen) @@ -294,9 +296,7 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) { // TODO: put this in function set program info length addedLen := dataLen + 2 newProgInfoLen := curProgLen + addedLen - (*p)[ProgramInfoLenIdx1] &= 0xff ^ ProgramInfoLenMask1 - (*p)[ProgramInfoLenIdx1] |= byte(newProgInfoLen>>8) & ProgramInfoLenMask1 - (*p)[ProgramInfoLenIdx2] = byte(newProgInfoLen) + p.SetProgInfoLen(newProgInfoLen) // set section length // TODO: put this in func set program info length @@ -306,6 +306,7 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) { (*p)[SyntaxSecLenIdx2] = byte(newSyntaxSectionLen) } +// TODO: make this safer. If no padding, does this index out of range ? func (p *PSIBytes) trimPadding() []byte { sectionLength := SyntaxSecLenFrom(*p) paddingIdx := (4 + sectionLength) @@ -313,6 +314,22 @@ func (p *PSIBytes) trimPadding() []byte { return o } +func (p *PSIBytes) SetProgInfoLen(l int) { + // TODO: check if pmt first + (*p)[ProgramInfoLenIdx1] &= 0xff ^ ProgramInfoLenMask1 + (*p)[ProgramInfoLenIdx1] |= byte(l>>8) & ProgramInfoLenMask1 + (*p)[ProgramInfoLenIdx2] = byte(l) +} + +func (p *PSIBytes) deleteDescriptor(tag int) error { + if i, desc := p.HasDescriptor(tag); desc != nil { + descLen := len(desc) + newProgInfoLen := p.ProgramInfoLen() - descLen + p.SetProgInfoLen(newProgInfoLen) + } + return errors.New("Descriptor doesn't exist") +} + func (d *Descriptor) update(data []byte) { if len(data) > int((*d)[1]) { // TODO: implement resizing of descriptor From 287238ddd170471cf6d476bc0c1c32642833d29e Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 29 Jan 2019 14:39:01 +1030 Subject: [PATCH 049/208] stream/mts/psi: removed deleteDescriptor func as we don't need it yet --- stream/mts/psi/psi.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 8832dfb4..9d0c1906 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -321,15 +321,6 @@ func (p *PSIBytes) SetProgInfoLen(l int) { (*p)[ProgramInfoLenIdx2] = byte(l) } -func (p *PSIBytes) deleteDescriptor(tag int) error { - if i, desc := p.HasDescriptor(tag); desc != nil { - descLen := len(desc) - newProgInfoLen := p.ProgramInfoLen() - descLen - p.SetProgInfoLen(newProgInfoLen) - } - return errors.New("Descriptor doesn't exist") -} - func (d *Descriptor) update(data []byte) { if len(data) > int((*d)[1]) { // TODO: implement resizing of descriptor From a5f7c5ad8750be4008cccc762556638a8e19aeac Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 29 Jan 2019 14:39:39 +1030 Subject: [PATCH 050/208] stream/mts/psi: removed TestDeleteDescriptor test as we don't need anymore --- stream/mts/psi/descriptor_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go index 919653fa..4381f896 100644 --- a/stream/mts/psi/descriptor_test.go +++ b/stream/mts/psi/descriptor_test.go @@ -219,10 +219,6 @@ func TestCreateDescriptor(t *testing.T) { } } -func TestDeleteDescriptor(t *testing.T) { - -} - func TestUpdateDescriptor(t *testing.T) { } From 2145db71d45f8f31285fed1e04c3cc6bae45d57f Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 29 Jan 2019 15:16:08 +1030 Subject: [PATCH 051/208] stream/mts/psi: wrote SetSectionLen func --- stream/mts/psi/descriptor_test.go | 2 +- stream/mts/psi/psi.go | 35 +++++++++++++++++++++---------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go index 4381f896..d11053d1 100644 --- a/stream/mts/psi/descriptor_test.go +++ b/stream/mts/psi/descriptor_test.go @@ -219,6 +219,6 @@ func TestCreateDescriptor(t *testing.T) { } } -func TestUpdateDescriptor(t *testing.T) { +func TestAddDescriptor(t *testing.T) { } diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 9d0c1906..7c7e150b 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -245,15 +245,32 @@ func (p *PSIBytes) AddDescriptor(tag int, data []byte) error { return errors.New("trying to add descriptor, but not pmt") } - _, desc := p.HasDescriptor(tag) + i, desc := p.HasDescriptor(tag) if desc == nil { p.createDescriptor(tag, data) return nil } - desc.update(data) + oldDescLen := desc.len() + // update the descriptor + oldDataLen := int(desc[1]) + newDataLen := len(data) + newDescLen := 2 + newDataLen + + // Shift data + copy(p[i+newDescLen:], data) + + deltaLen := desc.len() - oldDescLen + newProgInfoLen := p.ProgramInfoLen() + deltaLen + p.SetProgInfoLen(newProgInfoLen) + newSectionLen := int(psi.SectionLength(*p)) + deltaLen + p.SetSectionLen(newSectionLen) return nil } +func (d *Descriptor) len() int { + return int(2 + (*d)[1]) +} + func (p *PSIBytes) ProgramInfoLen() int { return int((((*p)[ProgramInfoLenIdx1] & ProgramInfoLenMask1) << 8) | (*p)[ProgramInfoLenIdx2]) } @@ -301,9 +318,7 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) { // set section length // TODO: put this in func set program info length newSyntaxSectionLen := int(oldSyntaxSectionLen) + addedLen - (*p)[SyntaxSecLenIdx1] &= 0xff ^ SyntaxSecLenMask1 - (*p)[SyntaxSecLenIdx1] |= byte(newSyntaxSectionLen>>8) & SyntaxSecLenMask1 - (*p)[SyntaxSecLenIdx2] = byte(newSyntaxSectionLen) + p.SetSectionLen(newSyntaxSectionLen) } // TODO: make this safer. If no padding, does this index out of range ? @@ -321,10 +336,8 @@ func (p *PSIBytes) SetProgInfoLen(l int) { (*p)[ProgramInfoLenIdx2] = byte(l) } -func (d *Descriptor) update(data []byte) { - if len(data) > int((*d)[1]) { - // TODO: implement resizing of descriptor - panic("Can't resize descriptor data") - } - +func (p *PSIBytes) SetSectionLen(l int) { + (*p)[SyntaxSecLenIdx1] &= 0xff ^ SyntaxSecLenMask1 + (*p)[SyntaxSecLenIdx1] |= byte(l>>8) & SyntaxSecLenMask1 + (*p)[SyntaxSecLenIdx2] = byte(l) } From 252e6680ed16e1885a22fdc3b797f34b844d78fa Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 29 Jan 2019 15:42:51 +1030 Subject: [PATCH 052/208] stream/mts/psi: modified way in which we add padding to psi - now we leave it up to the mts package to do this on creation of an ts packet. Also in the middle of writing AddDescriptor func, and finding issues, hence the mentioned change. --- stream/mts/psi/descriptor_test.go | 4 +++ stream/mts/psi/psi.go | 52 +++++++++++++++---------------- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go index d11053d1..b26fd007 100644 --- a/stream/mts/psi/descriptor_test.go +++ b/stream/mts/psi/descriptor_test.go @@ -168,6 +168,10 @@ func TestProgramInfoLen(t *testing.T) { } } +func TestSectionLen(t *testing.T) { + +} + func TestDescriptors(t *testing.T) { // Try psi with descriptors p := PSIBytes(tstPsi1.Bytes()) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 7c7e150b..c41e1d98 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -26,12 +26,6 @@ LICENSE package psi -import ( - "errors" - - "github.com/Comcast/gots/psi" -) - const ( PacketSize = 184 // packet size of a psi. ) @@ -170,7 +164,6 @@ func (p *PSI) Bytes() []byte { out[3] = byte(p.Sl) out = append(out, p.Tss.Bytes()...) out = addCrc(out) - out = addPadding(out) return out } @@ -241,29 +234,31 @@ func asByte(b bool) byte { } func (p *PSIBytes) AddDescriptor(tag int, data []byte) error { - if psi.TableID(*p) != pmtID { - return errors.New("trying to add descriptor, but not pmt") - } + /* + if psi.TableID(*p) != pmtID { + return errors.New("trying to add descriptor, but not pmt") + } - i, desc := p.HasDescriptor(tag) - if desc == nil { - p.createDescriptor(tag, data) - return nil - } - oldDescLen := desc.len() - // update the descriptor - oldDataLen := int(desc[1]) - newDataLen := len(data) - newDescLen := 2 + newDataLen + i, desc := p.HasDescriptor(tag) + if desc == nil { + p.createDescriptor(tag, data) + return nil + } + oldDescLen := desc.len() + // update the descriptor + oldDataLen := int(desc[1]) + newDataLen := len(data) + newDescLen := 2 + newDataLen - // Shift data - copy(p[i+newDescLen:], data) + // Shift data + // copy(p[i+newDescLen:], data) - deltaLen := desc.len() - oldDescLen - newProgInfoLen := p.ProgramInfoLen() + deltaLen - p.SetProgInfoLen(newProgInfoLen) - newSectionLen := int(psi.SectionLength(*p)) + deltaLen - p.SetSectionLen(newSectionLen) + deltaLen := desc.len() - oldDescLen + newProgInfoLen := p.ProgramInfoLen() + deltaLen + p.SetProgInfoLen(newProgInfoLen) + newSectionLen := int(psi.SectionLength(*p)) + deltaLen + p.SetSectionLen(newSectionLen) + */ return nil } @@ -302,6 +297,9 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) { newDescLen := dataLen + 2 // Copy data down from newDescIdx to create room for the new descriptor. + tmp := make([]byte, len(*p)+newDescLen) + copy(tmp, *p) + *p = tmp copy((*p)[newDescIdx+newDescLen:], (*p)[newDescIdx:newDescIdx+newDescLen]) // Set the tag, data len and data of the new desriptor. From 1786ed26614df36521eb4e08671add6790273f9a Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 29 Jan 2019 16:14:00 +1030 Subject: [PATCH 053/208] stream/mts/psi: finished writing AddDescriptor --- stream/mts/encoder.go | 20 ++++++------ stream/mts/psi/psi.go | 64 +++++++++++++++++++++++++------------- stream/mts/psi/psi_test.go | 2 +- 3 files changed, 53 insertions(+), 33 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 8f46a15a..33194ab8 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -196,12 +196,12 @@ func (e *Encoder) Encode(nalu []byte) error { func (e *Encoder) writePSI() error { // Write PAT. patPkt := Packet{ - PUSI: true, - PID: PatPid, - CC: e.ccFor(PatPid), - AFC: HasPayload, - Payload: patTable, + PUSI: true, + PID: PatPid, + CC: e.ccFor(PatPid), + AFC: HasPayload, } + patPkt.FillPayload(patTable) _, err := e.dst.Write(patPkt.Bytes(e.tsSpace[:PacketSize])) if err != nil { return err @@ -211,12 +211,12 @@ func (e *Encoder) writePSI() error { // Create mts packet from pmt table. pmtPkt := Packet{ - PUSI: true, - PID: PmtPid, - CC: e.ccFor(PmtPid), - AFC: HasPayload, - Payload: pmtTable, + PUSI: true, + PID: PmtPid, + CC: e.ccFor(PmtPid), + AFC: HasPayload, } + pmtPkt.FillPayload(pmtTable) _, err = e.dst.Write(pmtPkt.Bytes(e.tsSpace[:PacketSize])) if err != nil { return err diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index c41e1d98..9637594d 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -26,6 +26,12 @@ LICENSE package psi +import ( + "errors" + + "github.com/Comcast/gots/psi" +) + const ( PacketSize = 184 // packet size of a psi. ) @@ -234,31 +240,45 @@ func asByte(b bool) byte { } func (p *PSIBytes) AddDescriptor(tag int, data []byte) error { - /* - if psi.TableID(*p) != pmtID { - return errors.New("trying to add descriptor, but not pmt") - } + if psi.TableID(*p) != pmtID { + return errors.New("trying to add descriptor, but not pmt") + } - i, desc := p.HasDescriptor(tag) - if desc == nil { - p.createDescriptor(tag, data) - return nil - } - oldDescLen := desc.len() - // update the descriptor - oldDataLen := int(desc[1]) - newDataLen := len(data) - newDescLen := 2 + newDataLen + i, desc := p.HasDescriptor(tag) + if desc == nil { + p.createDescriptor(tag, data) + return nil + } - // Shift data - // copy(p[i+newDescLen:], data) + // Old lengths + oldDescLen := desc.len() + oldDataLen := int(desc[1]) + + // New lengths + newDataLen := len(data) + newDescLen := 2 + newDataLen + + delta := newDescLen - oldDescLen + + if oldDataLen > newDataLen { + copy((*p)[i+newDescLen:], (*p)[i+oldDescLen:]) + *p = (*p)[:len(*p)+delta] + + } else if oldDataLen < newDataLen { + tmp := make([]byte, len(*p)+delta) + copy(tmp, *p) + *p = tmp + copy((*p)[i+newDescLen:], (*p)[i+oldDescLen:]) + + } else { + copy((*p)[i+2:], data) + } + + newProgInfoLen := p.ProgramInfoLen() + delta + p.SetProgInfoLen(newProgInfoLen) + newSectionLen := int(psi.SectionLength(*p)) + delta + p.SetSectionLen(newSectionLen) - deltaLen := desc.len() - oldDescLen - newProgInfoLen := p.ProgramInfoLen() + deltaLen - p.SetProgInfoLen(newProgInfoLen) - newSectionLen := int(psi.SectionLength(*p)) + deltaLen - p.SetSectionLen(newSectionLen) - */ return nil } diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go index e437066e..61a740a4 100644 --- a/stream/mts/psi/psi_test.go +++ b/stream/mts/psi/psi_test.go @@ -282,7 +282,7 @@ var bytesTests = []struct { func TestBytes(t *testing.T) { for _, test := range bytesTests { got := test.input.Bytes() - if !bytes.Equal(got, addPadding(addCrc(test.want))) { + if !bytes.Equal(got, addCrc(test.want)) { t.Errorf("unexpected error for test %v: got:%v want:%v", test.name, got, test.want) } From c7d418ce81c4f54d70ed798ab449c59c90e8091e Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 29 Jan 2019 16:15:18 +1030 Subject: [PATCH 054/208] stream/mts/psi: deleted some todos as they have been addressed now --- stream/mts/psi/psi.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 9637594d..7d61f47b 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -328,13 +328,11 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) { copy((*p)[newDescIdx+2:newDescIdx+2+dataLen], data) // Update the program info length to account for the new descriptor. - // TODO: put this in function set program info length addedLen := dataLen + 2 newProgInfoLen := curProgLen + addedLen p.SetProgInfoLen(newProgInfoLen) // set section length - // TODO: put this in func set program info length newSyntaxSectionLen := int(oldSyntaxSectionLen) + addedLen p.SetSectionLen(newSyntaxSectionLen) } From 5cba86106350c4f9e7ff04232d4ba609c36f8b39 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 29 Jan 2019 16:16:37 +1030 Subject: [PATCH 055/208] stream/mts/psi: removed test that we don't need to do --- stream/mts/psi/descriptor_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go index b26fd007..d11053d1 100644 --- a/stream/mts/psi/descriptor_test.go +++ b/stream/mts/psi/descriptor_test.go @@ -168,10 +168,6 @@ func TestProgramInfoLen(t *testing.T) { } } -func TestSectionLen(t *testing.T) { - -} - func TestDescriptors(t *testing.T) { // Try psi with descriptors p := PSIBytes(tstPsi1.Bytes()) From a5e1763c9713999b32ffa72ba4f804f8f16e2bb1 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 29 Jan 2019 16:42:02 +1030 Subject: [PATCH 056/208] stream/mts/psi: finished writing AddDescriptor test -everything working fine now --- stream/mts/psi/descriptor_test.go | 25 +++++++++++++++++++++++++ stream/mts/psi/psi.go | 16 ++++++++++------ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go index d11053d1..99d005c2 100644 --- a/stream/mts/psi/descriptor_test.go +++ b/stream/mts/psi/descriptor_test.go @@ -34,6 +34,7 @@ import ( const ( errNotExpectedOut = "Did not get expected output: \ngot : %v, \nwant: %v" + errUnexpectedErr = "Unexpected error: %v\n" ) var tstPsi1 = PSI{ @@ -220,5 +221,29 @@ func TestCreateDescriptor(t *testing.T) { } func TestAddDescriptor(t *testing.T) { + // Add descriptor to psi without descriptors + got := PSIBytes(tstPsi2.Bytes()) + if err := got.AddDescriptor(TimeDescTag, make([]byte, TimeDataSize)); err != nil { + t.Errorf(errUnexpectedErr, err.Error()) + } + + want := PSIBytes(tstPsi1.Bytes()) + + if !bytes.Equal(got, want) { + t.Errorf(errNotExpectedOut, got, want) + } + + // Add to psi already with descriptor + got = PSIBytes(tstPsi1.Bytes()) + + if err := got.AddDescriptor(LocationDescTag, make([]byte, LocationDataSize)); err != nil { + t.Errorf(errUnexpectedErr, err.Error()) + } + + want = PSIBytes(tstPsi3.Bytes()) + + if !bytes.Equal(got, want) { + t.Errorf(errNotExpectedOut, got, want) + } } diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 7d61f47b..2bace251 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -275,9 +275,11 @@ func (p *PSIBytes) AddDescriptor(tag int, data []byte) error { } newProgInfoLen := p.ProgramInfoLen() + delta - p.SetProgInfoLen(newProgInfoLen) + p.setProgInfoLen(newProgInfoLen) newSectionLen := int(psi.SectionLength(*p)) + delta - p.SetSectionLen(newSectionLen) + p.setSectionLen(newSectionLen) + + updateCrc((*p)[1:]) return nil } @@ -330,11 +332,13 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) { // Update the program info length to account for the new descriptor. addedLen := dataLen + 2 newProgInfoLen := curProgLen + addedLen - p.SetProgInfoLen(newProgInfoLen) + p.setProgInfoLen(newProgInfoLen) // set section length newSyntaxSectionLen := int(oldSyntaxSectionLen) + addedLen - p.SetSectionLen(newSyntaxSectionLen) + p.setSectionLen(newSyntaxSectionLen) + + updateCrc((*p)[1:]) } // TODO: make this safer. If no padding, does this index out of range ? @@ -345,14 +349,14 @@ func (p *PSIBytes) trimPadding() []byte { return o } -func (p *PSIBytes) SetProgInfoLen(l int) { +func (p *PSIBytes) setProgInfoLen(l int) { // TODO: check if pmt first (*p)[ProgramInfoLenIdx1] &= 0xff ^ ProgramInfoLenMask1 (*p)[ProgramInfoLenIdx1] |= byte(l>>8) & ProgramInfoLenMask1 (*p)[ProgramInfoLenIdx2] = byte(l) } -func (p *PSIBytes) SetSectionLen(l int) { +func (p *PSIBytes) setSectionLen(l int) { (*p)[SyntaxSecLenIdx1] &= 0xff ^ SyntaxSecLenMask1 (*p)[SyntaxSecLenIdx1] |= byte(l>>8) & SyntaxSecLenMask1 (*p)[SyntaxSecLenIdx2] = byte(l) From 23e03eeddcae399ecbf349b3d0211f6e27258fa7 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 29 Jan 2019 16:47:55 +1030 Subject: [PATCH 057/208] cmd: removed ts-repair - which shouldn't have been on this branch --- cmd/ts-repair/main.go | 217 ------------------------------------------ 1 file changed, 217 deletions(-) delete mode 100644 cmd/ts-repair/main.go diff --git a/cmd/ts-repair/main.go b/cmd/ts-repair/main.go deleted file mode 100644 index f3f73bb2..00000000 --- a/cmd/ts-repair/main.go +++ /dev/null @@ -1,217 +0,0 @@ -package main - -import ( - "errors" - "flag" - "fmt" - "io" - "os" - - "bitbucket.org/ausocean/av/stream/mts" - "github.com/Comcast/gots/packet" -) - -// Various errors that we can encounter. -const ( - errBadInPath = "No file path provided, or file does not exist" - errCantCreateOut = "Can't create output file" - errCantGetPid = "Can't get pid from packet" - errReadFail = "Read failed" - errWriteFail = "Write to file failed" - errBadMode = "Bad fix mode" - errAdaptationPresent = "Adaptation field is already present in packet" - errNoAdaptationField = "No adaptation field in this packet" -) - -// Consts describing flag usage. -const ( - inUsage = "The path to the file to be repaired" - outUsage = "Output file path" - modeUsage = "Fix mode: 0 = cc-shift, 1 = di-update" -) - -// Repair modes. -const ( - ccShift = iota - diUpdate -) - -var ccMap = map[int]byte{ - mts.PatPid: 16, - mts.PmtPid: 16, - mts.VideoPid: 16, -} - -// packetNo will keep track of the ts packet number for reference. -var packetNo int - -// Option defines a func that performs an action on p in order to change a ts option. -type Option func(p *Packet) - -// Packet is a byte array of size mts.PacketSize i.e. 188 bytes. We define this -// to allow us to write receiver funcs for the [mts.PacketSize]byte type. -type Packet [mts.PacketSize]byte - -// CC returns the CC of p. -func (p *Packet) CC() byte { - return (*p)[3] & 0x0f -} - -// setCC sets the CC of p. -func (p *Packet) setCC(cc byte) { - (*p)[3] |= cc & 0xf -} - -// setDI sets the discontinuity counter of p. -func (p *Packet) setDI(di bool) { - if di { - p[5] |= 0x80 - } else { - p[5] &= 0x7f - } -} - -// addAdaptationField adds an adaptation field to p, and applys the passed options to this field. -// TODO: this will probably break if we already have adaptation field. -func (p *Packet) addAdaptationField(options ...Option) error { - if p.hasAdaptation() { - return errors.New(errAdaptationPresent) - } - // Create space for adaptation field. - copy(p[mts.HeadSize+mts.DefaultAdaptationSize:], p[mts.HeadSize:len(p)-mts.DefaultAdaptationSize]) - - // TODO: seperate into own function - // Update adaptation field control. - p[mts.AdaptationControlIdx] &= 0xff ^ mts.AdaptationControlMask - p[mts.AdaptationControlIdx] |= mts.AdaptationControlMask - // Default the adaptationfield. - p.resetAdaptation() - - // Apply and options that have bee passed. - for _, option := range options { - option(p) - } - return nil -} - -// resetAdaptation sets fields in ps adaptation field to 0 if the adaptation field -// exists, otherwise an error is returned. -func (p *Packet) resetAdaptation() error { - if !p.hasAdaptation() { - return errors.New(errNoAdaptationField) - } - p[mts.AdaptationIdx] = mts.DefaultAdaptationBodySize - p[mts.AdaptationBodyIdx] = 0x00 - return nil -} - -// hasAdaptation returns true if p has an adaptation field and false otherwise. -func (p *Packet) hasAdaptation() bool { - afc := p[mts.AdaptationControlIdx] & mts.AdaptationControlMask - if afc == 0x20 || afc == 0x30 { - return true - } else { - return false - } -} - -// DiscontinuityIndicator returns and Option that will set p's discontinuity -// indicator according to f. -func DiscontinuityIndicator(f bool) Option { - return func(p *Packet) { - set := byte(mts.DiscontinuityIndicatorMask) - if !f { - set = 0x00 - } - p[mts.DiscontinuityIndicatorIdx] &= 0xff ^ mts.DiscontinuityIndicatorMask - p[mts.DiscontinuityIndicatorIdx] |= mts.DiscontinuityIndicatorMask & set - } -} - -func main() { - // Deal with input flags - inPtr := flag.String("in", "", inUsage) - outPtr := flag.String("out", "out.ts", outUsage) - modePtr := flag.Int("mode", diUpdate, modeUsage) - flag.Parse() - - // Try and open the given input file, otherwise panic - we can't do anything - inFile, err := os.Open(*inPtr) - defer inFile.Close() - if err != nil { - panic(errBadInPath) - } - - // Try and create output file, otherwise panic - we can't do anything - outFile, err := os.Create(*outPtr) - defer outFile.Close() - if err != nil { - panic(errCantCreateOut) - } - - // Read each packet from the input file reader - var p Packet - for { - // If we get an end of file then return, otherwise we panic - can't do anything else - if _, err := inFile.Read(p[:mts.PacketSize]); err == io.EOF { - return - } else if err != nil { - panic(errReadFail + ": " + err.Error()) - } - packetNo++ - - // Get the pid from the packet - pid, err := packet.Pid((*packet.Packet)(&p)) - if err != nil { - panic(errCantGetPid) - } - - // Get the cc from the packet and also the expected cc (if exists) - cc := p.CC() - expect, exists := expectedCC(int(pid)) - if !exists { - updateCCMap(int(pid), cc) - } else { - switch *modePtr { - // ccShift mode shifts all CC regardless of presence of Discontinuities or not - case ccShift: - p.setCC(expect) - // diUpdate mode finds discontinuities and sets the discontinuity indicator to true. - // If we have a pat or pmt then we need to add an adaptation field and then set the DI. - case diUpdate: - if cc != expect { - fmt.Printf("***** Discontinuity found (packetNo: %v pid: %v, cc: %v, expect: %v)\n", packetNo, pid, cc, expect) - if p.hasAdaptation() { - p.setDI(true) - } else { - p.addAdaptationField(DiscontinuityIndicator(true)) - } - updateCCMap(int(pid), p.CC()) - } - default: - panic(errBadMode) - } - } - - // Write this packet to the output file. - if _, err := outFile.Write(p[:]); err != nil { - panic(errWriteFail + ": " + err.Error()) - } - } -} - -// expectedCC returns the expected cc for the given pid. If the cc hasn't been -// used yet, then 16 and false is returned. -func expectedCC(pid int) (byte, bool) { - cc := ccMap[pid] - if cc == 16 { - return 16, false - } - ccMap[pid] = (cc + 1) & 0xf - return cc, true -} - -// updateCCMap updates the cc for the passed pid. -func updateCCMap(pid int, cc byte) { - ccMap[pid] = (cc + 1) & 0xf -} From 568840f69180180380ddf9729108fc7085c07a1e Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 29 Jan 2019 18:22:57 +1030 Subject: [PATCH 058/208] stream/mts/psi: cleaned up descriptor_test.go --- stream/mts/psi/descriptor_test.go | 264 ++++++++++++++---------------- 1 file changed, 124 insertions(+), 140 deletions(-) diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go index 99d005c2..dd763b82 100644 --- a/stream/mts/psi/descriptor_test.go +++ b/stream/mts/psi/descriptor_test.go @@ -37,124 +37,123 @@ const ( errUnexpectedErr = "Unexpected error: %v\n" ) -var tstPsi1 = PSI{ - Pf: 0x00, - Tid: 0x02, - Ssi: true, - Sl: 0x1c, - Tss: &TSS{ - Tide: 0x01, - V: 0, - Cni: true, - Sn: 0, - Lsn: 0, - Sd: &PMT{ - Pcrpid: 0x0100, // wrong - Pil: 10, - Pd: []Desc{ - { - Dt: TimeDescTag, - Dl: TimeDataSize, - Dd: make([]byte, TimeDataSize), +var ( + tstPsi1 = PSI{ + Pf: 0x00, + Tid: 0x02, + Ssi: true, + Sl: 0x1c, + Tss: &TSS{ + Tide: 0x01, + V: 0, + Cni: true, + Sn: 0, + Lsn: 0, + Sd: &PMT{ + Pcrpid: 0x0100, // wrong + Pil: 10, + Pd: []Desc{ + { + Dt: TimeDescTag, + Dl: TimeDataSize, + Dd: make([]byte, TimeDataSize), + }, + }, + Essd: &ESSD{ + St: 0x1b, + Epid: 0x0100, + Esil: 0x00, }, }, - Essd: &ESSD{ - St: 0x1b, - Epid: 0x0100, - Esil: 0x00, - }, }, - }, -} + } -var tstPsi2 = PSI{ - Pf: 0x00, - Tid: 0x02, - Ssi: true, - Sl: 0x12, - Tss: &TSS{ - Tide: 0x01, - V: 0, - Cni: true, - Sn: 0, - Lsn: 0, - Sd: &PMT{ - Pcrpid: 0x0100, - Pil: 0, - Essd: &ESSD{ - St: 0x1b, - Epid: 0x0100, - Esil: 0x00, - }, - }, - }, -} - -var tstPsi3 = PSI{ - Pf: 0x00, - Tid: 0x02, - Ssi: true, - Sl: 0x3e, - Tss: &TSS{ - Tide: 0x01, - V: 0, - Cni: true, - Sn: 0, - Lsn: 0, - Sd: &PMT{ - Pcrpid: 0x0100, - Pil: PmtTimeLocationPil, - Pd: []Desc{ - { - Dt: TimeDescTag, - Dl: TimeDataSize, - Dd: make([]byte, TimeDataSize), - }, - { - Dt: LocationDescTag, - Dl: LocationDataSize, - Dd: make([]byte, LocationDataSize), + tstPsi2 = PSI{ + Pf: 0x00, + Tid: 0x02, + Ssi: true, + Sl: 0x12, + Tss: &TSS{ + Tide: 0x01, + V: 0, + Cni: true, + Sn: 0, + Lsn: 0, + Sd: &PMT{ + Pcrpid: 0x0100, + Pil: 0, + Essd: &ESSD{ + St: 0x1b, + Epid: 0x0100, + Esil: 0x00, }, }, - Essd: &ESSD{ - St: 0x1b, - Epid: 0x0100, - Esil: 0x00, + }, + } + + tstPsi3 = PSI{ + Pf: 0x00, + Tid: 0x02, + Ssi: true, + Sl: 0x3e, + Tss: &TSS{ + Tide: 0x01, + V: 0, + Cni: true, + Sn: 0, + Lsn: 0, + Sd: &PMT{ + Pcrpid: 0x0100, + Pil: PmtTimeLocationPil, + Pd: []Desc{ + { + Dt: TimeDescTag, + Dl: TimeDataSize, + Dd: make([]byte, TimeDataSize), + }, + { + Dt: LocationDescTag, + Dl: LocationDataSize, + Dd: make([]byte, LocationDataSize), + }, + }, + Essd: &ESSD{ + St: 0x1b, + Epid: 0x0100, + Esil: 0x00, + }, }, }, - }, -} + } +) -func TestHasDescriptor(t *testing.T) { +func TestHasDescriptorExists(t *testing.T) { p := PSIBytes(tstPsi3.Bytes()) - - // Try getting descriptor that exists _, got := p.HasDescriptor(LocationDescTag) want := []byte{ LocationDescTag, LocationDataSize, } want = append(want, make([]byte, LocationDataSize)...) - if !bytes.Equal(got, want) { t.Errorf(errNotExpectedOut, got, want) } +} - // Try getting descriptor that doesnt exist +func TestHasDescriptorAbsent(t *testing.T) { + p := PSIBytes(tstPsi3.Bytes()) const fakeTag = 236 - _, got = p.HasDescriptor(fakeTag) - want = nil - + _, got := p.HasDescriptor(fakeTag) + var want []byte if !bytes.Equal(got, want) { t.Errorf(errNotExpectedOut, got, want) } +} - // Try getting descriptor from psi that has no descriptors - p = PSIBytes(tstPsi2.Bytes()) - - _, got = p.HasDescriptor(LocationDescTag) - want = nil - +func TestHasDescriptorNone(t *testing.T) { + p := PSIBytes(tstPsi2.Bytes()) + _, got := p.HasDescriptor(LocationDescTag) + var want []byte if !bytes.Equal(got, want) { t.Errorf(errNotExpectedOut, got, want) } @@ -170,7 +169,6 @@ func TestProgramInfoLen(t *testing.T) { } func TestDescriptors(t *testing.T) { - // Try psi with descriptors p := PSIBytes(tstPsi1.Bytes()) got := p.descriptors() want := []byte{ @@ -178,71 +176,57 @@ func TestDescriptors(t *testing.T) { TimeDataSize, } want = append(want, make([]byte, TimeDataSize)...) - - if !bytes.Equal(got, want) { - t.Errorf(errNotExpectedOut, got, want) - } - - // Now try psi with empty descriptors - p = PSIBytes(tstPsi2.Bytes()) - got = p.descriptors() - want = nil if !bytes.Equal(got, want) { t.Errorf(errNotExpectedOut, got, want) } } -func TestCreateDescriptor(t *testing.T) { - // Test with PSI containing no descriptors +func TestDescriptorsNone(t *testing.T) { p := PSIBytes(tstPsi2.Bytes()) - p.createDescriptor(TimeDescTag, make([]byte, TimeDataSize)) - // update crc - noPadding := p.trimPadding() - updateCrc(noPadding[1:]) - want := PSIBytes(tstPsi1.Bytes()) - got := p - - if !bytes.Equal(want, got) { + got := p.descriptors() + var want []byte + if !bytes.Equal(got, want) { t.Errorf(errNotExpectedOut, got, want) } +} - // Test with psi containing descriptor already - p = PSIBytes(tstPsi1.Bytes()) - p.createDescriptor(LocationDescTag, make([]byte, LocationDataSize)) - // update crc - noPadding = p.trimPadding() - updateCrc(noPadding[1:]) - want = PSIBytes(tstPsi3.Bytes()) - got = p - +func TestCreateDescriptorEmpty(t *testing.T) { + got := PSIBytes(tstPsi2.Bytes()) + got.createDescriptor(TimeDescTag, make([]byte, TimeDataSize)) + updateCrc(got[1:]) + want := PSIBytes(tstPsi1.Bytes()) if !bytes.Equal(want, got) { t.Errorf(errNotExpectedOut, got, want) } } -func TestAddDescriptor(t *testing.T) { - // Add descriptor to psi without descriptors - got := PSIBytes(tstPsi2.Bytes()) +func TestCreateDescriptorNotEmpty(t *testing.T) { + got := PSIBytes(tstPsi1.Bytes()) + got.createDescriptor(LocationDescTag, make([]byte, LocationDataSize)) + updateCrc(got[1:]) + want := PSIBytes(tstPsi3.Bytes()) + if !bytes.Equal(want, got) { + t.Errorf(errNotExpectedOut, got, want) + } +} +func TestAddDescriptorEmpty(t *testing.T) { + got := PSIBytes(tstPsi2.Bytes()) if err := got.AddDescriptor(TimeDescTag, make([]byte, TimeDataSize)); err != nil { t.Errorf(errUnexpectedErr, err.Error()) } - want := PSIBytes(tstPsi1.Bytes()) - - if !bytes.Equal(got, want) { - t.Errorf(errNotExpectedOut, got, want) - } - - // Add to psi already with descriptor - got = PSIBytes(tstPsi1.Bytes()) - - if err := got.AddDescriptor(LocationDescTag, make([]byte, LocationDataSize)); err != nil { - t.Errorf(errUnexpectedErr, err.Error()) - } - - want = PSIBytes(tstPsi3.Bytes()) - + if !bytes.Equal(got, want) { + t.Errorf(errNotExpectedOut, got, want) + } +} + +func TestAddDescriptorNonEmpty(t *testing.T) { + got := PSIBytes(tstPsi1.Bytes()) + if err := got.AddDescriptor(LocationDescTag, make([]byte, LocationDataSize)); err != nil { + t.Errorf(errUnexpectedErr, err.Error()) + } + want := PSIBytes(tstPsi3.Bytes()) if !bytes.Equal(got, want) { t.Errorf(errNotExpectedOut, got, want) } From 7f8465f1adc241295a71d05e9b0d52b8f25fd668 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 29 Jan 2019 22:07:06 +1030 Subject: [PATCH 059/208] stream/mts/psi/descriptor_test.go: added comments to test funcs - as well as some todos for further testing. --- stream/mts/psi/descriptor_test.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go index dd763b82..cbb743a2 100644 --- a/stream/mts/psi/descriptor_test.go +++ b/stream/mts/psi/descriptor_test.go @@ -127,6 +127,10 @@ var ( } ) +// TestHasDescriptorExists checks that PSIBytes.HasDescriptor performs as expected +// when the PSI we're interested in has the descriptor of interest. HasDescriptor +// should return the descriptor bytes. +// TODO: HasDescriptor also returns index of descriptor - we should check this. func TestHasDescriptorExists(t *testing.T) { p := PSIBytes(tstPsi3.Bytes()) _, got := p.HasDescriptor(LocationDescTag) @@ -140,6 +144,10 @@ func TestHasDescriptorExists(t *testing.T) { } } +// TestHasDescriptorAbsent checks that PSIBytes.HasDescriptor performs as expected +// when the PSI does not have the descriptor of interest. HasDescriptor should +// return a nil slice and a negative index. +// TODO: check index here as well. func TestHasDescriptorAbsent(t *testing.T) { p := PSIBytes(tstPsi3.Bytes()) const fakeTag = 236 @@ -150,6 +158,10 @@ func TestHasDescriptorAbsent(t *testing.T) { } } +// TestHasDescriptorNone checks that PSIBytes.HasDescriptor behaves as expected +// when the PSI does not have any descriptors. HasDescriptor should return a nil +// slice. +// TODO: again check index here. func TestHasDescriptorNone(t *testing.T) { p := PSIBytes(tstPsi2.Bytes()) _, got := p.HasDescriptor(LocationDescTag) @@ -159,6 +171,8 @@ func TestHasDescriptorNone(t *testing.T) { } } +// TestProgramInfoLen checks that PSIBytes.ProgramInfoLen correctly extracts +// the program info length from a PSI. func TestProgramInfoLen(t *testing.T) { p := PSIBytes(tstPsi1.Bytes()) got := p.ProgramInfoLen() @@ -168,6 +182,8 @@ func TestProgramInfoLen(t *testing.T) { } } +// TestDescriptors checks that PSIBytes.descriptors correctly returns the descriptors +// from a PSI when descriptors exist. func TestDescriptors(t *testing.T) { p := PSIBytes(tstPsi1.Bytes()) got := p.descriptors() @@ -181,6 +197,8 @@ func TestDescriptors(t *testing.T) { } } +// TestDescriptors checks that PSIBYtes.desriptors correctly returns nil when +// we try to get descriptors from a psi without any descriptors. func TestDescriptorsNone(t *testing.T) { p := PSIBytes(tstPsi2.Bytes()) got := p.descriptors() @@ -190,6 +208,8 @@ func TestDescriptorsNone(t *testing.T) { } } +// TestCreateDescriptorEmpty checks that PSIBytes.createDescriptor correctly adds +// a descriptor to the descriptors list in a PSI when it has no descriptors already. func TestCreateDescriptorEmpty(t *testing.T) { got := PSIBytes(tstPsi2.Bytes()) got.createDescriptor(TimeDescTag, make([]byte, TimeDataSize)) @@ -200,6 +220,9 @@ func TestCreateDescriptorEmpty(t *testing.T) { } } +// TestCreateDescriptorNotEmpty checks that PSIBytes.createDescriptor correctly adds +// a descriptor to the descriptors list in a PSI when it already has one with +// a different tag. func TestCreateDescriptorNotEmpty(t *testing.T) { got := PSIBytes(tstPsi1.Bytes()) got.createDescriptor(LocationDescTag, make([]byte, LocationDataSize)) @@ -210,6 +233,8 @@ func TestCreateDescriptorNotEmpty(t *testing.T) { } } +// TestAddDescriptorEmpty checks that PSIBytes.AddDescriptor correctly adds a descriptor +// when there are no other descriptors present in the PSI. func TestAddDescriptorEmpty(t *testing.T) { got := PSIBytes(tstPsi2.Bytes()) if err := got.AddDescriptor(TimeDescTag, make([]byte, TimeDataSize)); err != nil { @@ -221,6 +246,8 @@ func TestAddDescriptorEmpty(t *testing.T) { } } +// TestAddDescriptorNonEmpty checks that PSIBytes.AddDescriptor correctly adds a +// descriptor when there is already a descriptor of a differing type in a PSI. func TestAddDescriptorNonEmpty(t *testing.T) { got := PSIBytes(tstPsi1.Bytes()) if err := got.AddDescriptor(LocationDescTag, make([]byte, LocationDataSize)); err != nil { @@ -231,3 +258,5 @@ func TestAddDescriptorNonEmpty(t *testing.T) { t.Errorf(errNotExpectedOut, got, want) } } + +// TODO: check that we can update descriptor with AddDescriptor From d49a8b8c6b38d31a6f3325d30db2e2c1a7cfcb26 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 29 Jan 2019 22:09:17 +1030 Subject: [PATCH 060/208] stream/mts/psi.go: removed trimPadding function as we don't need this anymore --- stream/mts/psi/psi.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 2bace251..11e36d38 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -341,14 +341,6 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) { updateCrc((*p)[1:]) } -// TODO: make this safer. If no padding, does this index out of range ? -func (p *PSIBytes) trimPadding() []byte { - sectionLength := SyntaxSecLenFrom(*p) - paddingIdx := (4 + sectionLength) - o := (*p)[:paddingIdx] - return o -} - func (p *PSIBytes) setProgInfoLen(l int) { // TODO: check if pmt first (*p)[ProgramInfoLenIdx1] &= 0xff ^ ProgramInfoLenMask1 From 55bee1532e15a60b75d67363171809bb49bebbea Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 30 Jan 2019 11:01:04 +1030 Subject: [PATCH 061/208] stream/mts/psi/psi.go: added function comments and improved commenting and layout inside functions --- stream/mts/psi/psi.go | 48 ++++++++++++++++++++++++++----------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 11e36d38..f1033be7 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -239,6 +239,11 @@ func asByte(b bool) byte { return 0x00 } +// AddDescriptor adds or updates a descriptor in a PSI given a descriptor tag +// and data. If the psi is not a pmt, then an error is returned. If a descriptor +// with the given tag is not found in the psi, room is made and a descriptor with +// given tag and data is created. If a descriptor with the tag is found, the +// descriptor is resized as required and the new data is copied in. func (p *PSIBytes) AddDescriptor(tag int, data []byte) error { if psi.TableID(*p) != pmtID { return errors.New("trying to add descriptor, but not pmt") @@ -250,27 +255,26 @@ func (p *PSIBytes) AddDescriptor(tag int, data []byte) error { return nil } - // Old lengths oldDescLen := desc.len() oldDataLen := int(desc[1]) - - // New lengths newDataLen := len(data) newDescLen := 2 + newDataLen - delta := newDescLen - oldDescLen - if oldDataLen > newDataLen { + // If the old data length is more than the new data length, we need shift data + // after descriptor up, and then trim the psi. If the oldDataLen is less than + // new data then we need reseize psi and shift data down. If data is same size + // just copy new data in. + switch { + case oldDataLen > newDataLen: copy((*p)[i+newDescLen:], (*p)[i+oldDescLen:]) *p = (*p)[:len(*p)+delta] - - } else if oldDataLen < newDataLen { + case oldDataLen < newDataLen: tmp := make([]byte, len(*p)+delta) copy(tmp, *p) *p = tmp copy((*p)[i+newDescLen:], (*p)[i+oldDescLen:]) - - } else { + default: copy((*p)[i+2:], data) } @@ -278,20 +282,27 @@ func (p *PSIBytes) AddDescriptor(tag int, data []byte) error { p.setProgInfoLen(newProgInfoLen) newSectionLen := int(psi.SectionLength(*p)) + delta p.setSectionLen(newSectionLen) - updateCrc((*p)[1:]) - return nil } +// len returns the length of a descriptor in bytes. func (d *Descriptor) len() int { return int(2 + (*d)[1]) } +// ProgramInfoLen returns the program info length of a PSI. +// +// TODO: check if pmt - if not return 0 ? or -1 ? func (p *PSIBytes) ProgramInfoLen() int { return int((((*p)[ProgramInfoLenIdx1] & ProgramInfoLenMask1) << 8) | (*p)[ProgramInfoLenIdx2]) } +// HasDescriptor checks if a descriptor of the given tag exists in a PSI. If the descriptor +// of the given tag exists, an index of this descriptor, as well as the Descriptor is returned. +// If the descriptor of the given tag cannot be found, -1 and a nil slice is returned. +// +// TODO: check if pmt, return error if not ? func (p *PSIBytes) HasDescriptor(tag int) (int, Descriptor) { descs := p.descriptors() if descs == nil { @@ -305,20 +316,21 @@ func (p *PSIBytes) HasDescriptor(tag int) (int, Descriptor) { return -1, nil } +// descriptors returns the descriptors in a psi if they exist, otherwise +// a nil slice is returned. func (p *PSIBytes) descriptors() []byte { return (*p)[DescriptorsIdx : DescriptorsIdx+p.ProgramInfoLen()] } +// createDescriptor creates a descriptor in a psi given a tag and data. func (p *PSIBytes) createDescriptor(tag int, data []byte) { curProgLen := p.ProgramInfoLen() oldSyntaxSectionLen := SyntaxSecLenFrom(*p) dataLen := len(data) - - // Calculate the new descriptors index and length. newDescIdx := DescriptorsIdx + curProgLen newDescLen := dataLen + 2 - // Copy data down from newDescIdx to create room for the new descriptor. + // Increase size of psi and copy data down to make room for new descriptor. tmp := make([]byte, len(*p)+newDescLen) copy(tmp, *p) *p = tmp @@ -329,25 +341,23 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) { (*p)[newDescIdx+1] = byte(dataLen) copy((*p)[newDescIdx+2:newDescIdx+2+dataLen], data) - // Update the program info length to account for the new descriptor. + // Set length fields and update the psi crc. addedLen := dataLen + 2 newProgInfoLen := curProgLen + addedLen p.setProgInfoLen(newProgInfoLen) - - // set section length newSyntaxSectionLen := int(oldSyntaxSectionLen) + addedLen p.setSectionLen(newSyntaxSectionLen) - updateCrc((*p)[1:]) } +// setProgInfoLen sets the program information length in a psi with a pmt. func (p *PSIBytes) setProgInfoLen(l int) { - // TODO: check if pmt first (*p)[ProgramInfoLenIdx1] &= 0xff ^ ProgramInfoLenMask1 (*p)[ProgramInfoLenIdx1] |= byte(l>>8) & ProgramInfoLenMask1 (*p)[ProgramInfoLenIdx2] = byte(l) } +// setSectionLen sets section length in a psi. func (p *PSIBytes) setSectionLen(l int) { (*p)[SyntaxSecLenIdx1] &= 0xff ^ SyntaxSecLenMask1 (*p)[SyntaxSecLenIdx1] |= byte(l>>8) & SyntaxSecLenMask1 From c2112e58ac3203a40377e5de8a68bf263a026f7f Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 30 Jan 2019 12:08:55 +1030 Subject: [PATCH 062/208] stream/mts/psi: added some more tests for AddDescriptor, and in the process fixed some bugs with AddDescriptor --- stream/mts/psi/descriptor_test.go | 55 ++++++++++++++++++++++++++++++- stream/mts/psi/psi.go | 51 ++++++++++++++-------------- 2 files changed, 81 insertions(+), 25 deletions(-) diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go index cbb743a2..1fed7b2b 100644 --- a/stream/mts/psi/descriptor_test.go +++ b/stream/mts/psi/descriptor_test.go @@ -127,6 +127,24 @@ var ( } ) +var ( + pmtTimeBytesResizedBigger = []byte{ + 0x00, 0x02, 0xb0, 0x1e, 0x00, 0x01, 0xc1, 0x00, 0x00, 0xe1, 0x00, 0xf0, 0x0c, + TimeDescTag, // Descriptor tag + 0x0a, // Length of bytes to follow + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, // timestamp + 0x1b, 0xe1, 0x00, 0xf0, 0x00, + } + + pmtTimeBytesResizedSmaller = []byte{ + 0x00, 0x02, 0xb0, 0x1a, 0x00, 0x01, 0xc1, 0x00, 0x00, 0xe1, 0x00, 0xf0, 0x08, + TimeDescTag, // Descriptor tag + 0x06, // Length of bytes to follow + 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, // timestamp + 0x1b, 0xe1, 0x00, 0xf0, 0x00, + } +) + // TestHasDescriptorExists checks that PSIBytes.HasDescriptor performs as expected // when the PSI we're interested in has the descriptor of interest. HasDescriptor // should return the descriptor bytes. @@ -259,4 +277,39 @@ func TestAddDescriptorNonEmpty(t *testing.T) { } } -// TODO: check that we can update descriptor with AddDescriptor +func TestAddDescriptorUpdateSame(t *testing.T) { + newData := [8]byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08} + want := PSIBytes(tstPsi2.Bytes()) + want.createDescriptor(TimeDescTag, newData[:]) + + got := PSIBytes(tstPsi1.Bytes()) + if err := got.AddDescriptor(TimeDescTag, newData[:]); err != nil { + t.Errorf(errUnexpectedErr, err.Error()) + } + + if !bytes.Equal(got, want) { + t.Errorf(errNotExpectedOut, got, want) + } +} + +func TestAddDescriptorUpdateBigger(t *testing.T) { + got := PSIBytes(tstPsi1.Bytes()) + if err := got.AddDescriptor(TimeDescTag, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a}); err != nil { + t.Errorf(errUnexpectedErr, err.Error()) + } + want := addCrc(pmtTimeBytesResizedBigger) + if !bytes.Equal(got, want) { + t.Errorf(errNotExpectedOut, got, want) + } +} + +func TestAddDescriptorUpdateSmaller(t *testing.T) { + got := PSIBytes(tstPsi1.Bytes()) + if err := got.AddDescriptor(TimeDescTag, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}); err != nil { + t.Errorf(errUnexpectedErr, err.Error()) + } + want := addCrc(pmtTimeBytesResizedSmaller) + if !bytes.Equal(got, want) { + t.Errorf(errNotExpectedOut, got, want) + } +} diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index f1033be7..dca9e57a 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -263,8 +263,7 @@ func (p *PSIBytes) AddDescriptor(tag int, data []byte) error { // If the old data length is more than the new data length, we need shift data // after descriptor up, and then trim the psi. If the oldDataLen is less than - // new data then we need reseize psi and shift data down. If data is same size - // just copy new data in. + // new data then we need reseize psi and shift data down. If same do nothing. switch { case oldDataLen > newDataLen: copy((*p)[i+newDescLen:], (*p)[i+oldDescLen:]) @@ -274,10 +273,12 @@ func (p *PSIBytes) AddDescriptor(tag int, data []byte) error { copy(tmp, *p) *p = tmp copy((*p)[i+newDescLen:], (*p)[i+oldDescLen:]) - default: - copy((*p)[i+2:], data) } + // Copy in new data + (*p)[i+1] = byte(newDataLen) + copy((*p)[i+2:], data) + newProgInfoLen := p.ProgramInfoLen() + delta p.setProgInfoLen(newProgInfoLen) newSectionLen := int(psi.SectionLength(*p)) + delta @@ -286,18 +287,6 @@ func (p *PSIBytes) AddDescriptor(tag int, data []byte) error { return nil } -// len returns the length of a descriptor in bytes. -func (d *Descriptor) len() int { - return int(2 + (*d)[1]) -} - -// ProgramInfoLen returns the program info length of a PSI. -// -// TODO: check if pmt - if not return 0 ? or -1 ? -func (p *PSIBytes) ProgramInfoLen() int { - return int((((*p)[ProgramInfoLenIdx1] & ProgramInfoLenMask1) << 8) | (*p)[ProgramInfoLenIdx2]) -} - // HasDescriptor checks if a descriptor of the given tag exists in a PSI. If the descriptor // of the given tag exists, an index of this descriptor, as well as the Descriptor is returned. // If the descriptor of the given tag cannot be found, -1 and a nil slice is returned. @@ -310,19 +299,15 @@ func (p *PSIBytes) HasDescriptor(tag int) (int, Descriptor) { } for i := 0; i < len(descs); i += 2 + int(descs[i+1]) { if int(descs[i]) == tag { - return i, descs[i : i+2+int(descs[i+1])] + return i + DescriptorsIdx, descs[i : i+2+int(descs[i+1])] } } return -1, nil } -// descriptors returns the descriptors in a psi if they exist, otherwise -// a nil slice is returned. -func (p *PSIBytes) descriptors() []byte { - return (*p)[DescriptorsIdx : DescriptorsIdx+p.ProgramInfoLen()] -} - -// createDescriptor creates a descriptor in a psi given a tag and data. +// createDescriptor creates a descriptor in a psi given a tag and data. It does so +// by resizing the psi, shifting existing data down and copying in new descriptor +// in new space. func (p *PSIBytes) createDescriptor(tag int, data []byte) { curProgLen := p.ProgramInfoLen() oldSyntaxSectionLen := SyntaxSecLenFrom(*p) @@ -363,3 +348,21 @@ func (p *PSIBytes) setSectionLen(l int) { (*p)[SyntaxSecLenIdx1] |= byte(l>>8) & SyntaxSecLenMask1 (*p)[SyntaxSecLenIdx2] = byte(l) } + +// descriptors returns the descriptors in a psi if they exist, otherwise +// a nil slice is returned. +func (p *PSIBytes) descriptors() []byte { + return (*p)[DescriptorsIdx : DescriptorsIdx+p.ProgramInfoLen()] +} + +// len returns the length of a descriptor in bytes. +func (d *Descriptor) len() int { + return int(2 + (*d)[1]) +} + +// ProgramInfoLen returns the program info length of a PSI. +// +// TODO: check if pmt - if not return 0 ? or -1 ? +func (p *PSIBytes) ProgramInfoLen() int { + return int((((*p)[ProgramInfoLenIdx1] & ProgramInfoLenMask1) << 8) | (*p)[ProgramInfoLenIdx2]) +} From 60c2eafd8cd7b18e4a29a0102cdd7f9df33a1798 Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 30 Jan 2019 12:13:29 +1030 Subject: [PATCH 063/208] stream/mts/psi/descriptor_test.go: added some function comments above new test funcs --- stream/mts/psi/descriptor_test.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go index 1fed7b2b..0f7d300e 100644 --- a/stream/mts/psi/descriptor_test.go +++ b/stream/mts/psi/descriptor_test.go @@ -277,21 +277,25 @@ func TestAddDescriptorNonEmpty(t *testing.T) { } } +// TestAddDescriptorUpdateSame checks that PSIBytes.AddDescriptor correctly updates data in a descriptor +// with the same given tag, with data being the same size. AddDescriptor should just copy new data into +// the descriptors data field. func TestAddDescriptorUpdateSame(t *testing.T) { newData := [8]byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08} want := PSIBytes(tstPsi2.Bytes()) want.createDescriptor(TimeDescTag, newData[:]) - got := PSIBytes(tstPsi1.Bytes()) if err := got.AddDescriptor(TimeDescTag, newData[:]); err != nil { t.Errorf(errUnexpectedErr, err.Error()) } - if !bytes.Equal(got, want) { t.Errorf(errNotExpectedOut, got, want) } } +// TestAddDescriptorUpdateBigger checks that PSIBytes.AddDescriptor correctly resizes descriptor with same given tag +// to a bigger size and copies in new data. AddDescriptor should find descriptor with same tag, increase size of psi, +// shift data to make room for update descriptor, and then copy in the new data. func TestAddDescriptorUpdateBigger(t *testing.T) { got := PSIBytes(tstPsi1.Bytes()) if err := got.AddDescriptor(TimeDescTag, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a}); err != nil { @@ -303,6 +307,9 @@ func TestAddDescriptorUpdateBigger(t *testing.T) { } } +// TestAddDescriptorUpdateSmaller checks that PSIBytes.AddDescriptor correctly resizes descriptor with same given tag +// in a psi to a smaller size and copies in new data. AddDescriptor should find tag with same descrtiptor, shift data +// after descriptor upwards, trim the psi to new size, and then copy in new data. func TestAddDescriptorUpdateSmaller(t *testing.T) { got := PSIBytes(tstPsi1.Bytes()) if err := got.AddDescriptor(TimeDescTag, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}); err != nil { From 83ac98fe84a336df3780c81baee2a1800f742268 Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 30 Jan 2019 12:26:51 +1030 Subject: [PATCH 064/208] stream/mts: added metaEncode_test.go a file that will contain tests that will use the Meta struct to actually encode metadata into psi --- stream/mts/metaEncode_test.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 stream/mts/metaEncode_test.go diff --git a/stream/mts/metaEncode_test.go b/stream/mts/metaEncode_test.go new file mode 100644 index 00000000..d422c700 --- /dev/null +++ b/stream/mts/metaEncode_test.go @@ -0,0 +1,34 @@ +/* +NAME + metaEncode_test.go + +DESCRIPTION + See Readme.md + +AUTHOR + Saxon Nelson-Milton + +LICENSE + metaEncode_test.go is Copyright (C) 2017-2019 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ + +package mts + +import "testing" + +func TestMetaEncode(t *testing.T) { + +} From 7dd1ce99e1454a7208cf12ab264f4738d97f6784 Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 30 Jan 2019 13:23:07 +1030 Subject: [PATCH 065/208] stream/mts: started writing metaEncode_test.go file --- stream/mts/encoder.go | 8 ++++++-- stream/mts/meta.go | 16 ++++++++-------- stream/mts/metaEncode_test.go | 24 ++++++++++++++++++++++-- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 33194ab8..4567aea2 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -88,7 +88,11 @@ const ( ) // global Meta -var meta Meta +var Meta *Metadata + +func init() { + Meta = NewMeta() +} var ( patTable = standardPat.Bytes() @@ -252,7 +256,7 @@ func (e *Encoder) ccFor(pid int) byte { func updateMeta(b []byte) error { var p psi.PSIBytes p = b - m := meta.Encode() + m := Meta.Encode() p.AddDescriptor(psi.MetadataTag, m) return nil } diff --git a/stream/mts/meta.go b/stream/mts/meta.go index 465a537e..62097d30 100644 --- a/stream/mts/meta.go +++ b/stream/mts/meta.go @@ -48,14 +48,14 @@ var ( errKeyAbsent = errors.New("Key does not exist in map") ) -type Meta struct { +type Metadata struct { mu sync.RWMutex data map[string]string enc []byte } -func NewMeta() *Meta { - return &Meta{ +func NewMeta() *Metadata { + return &Metadata{ data: make(map[string]string), enc: []byte{ 0x00, // Reserved byte @@ -67,14 +67,14 @@ func NewMeta() *Meta { } // Add adds metadata with key and val, if already exists return error -func (m *Meta) Add(key, val string) { +func (m *Metadata) Add(key, val string) { m.mu.Lock() m.data[key] = val m.mu.Unlock() } // All returns the a copy of the map containing the meta data -func (m *Meta) All() map[string]string { +func (m *Metadata) All() map[string]string { m.mu.Lock() cpy := make(map[string]string) for k, v := range m.data { @@ -85,7 +85,7 @@ func (m *Meta) All() map[string]string { } // Get returns the meta data for the passed key -func (m *Meta) Get(key string) (string, error) { +func (m *Metadata) Get(key string) (string, error) { m.mu.Lock() val, ok := m.data[key] m.mu.Unlock() @@ -97,7 +97,7 @@ func (m *Meta) Get(key string) (string, error) { } // Remove deletes a meta entry in the map and returns error if it doesn’t exist -func (m *Meta) Delete(key string) error { +func (m *Metadata) Delete(key string) error { m.mu.Lock() defer m.mu.Unlock() if _, ok := m.data[key]; ok { @@ -109,7 +109,7 @@ func (m *Meta) Delete(key string) error { // Encode takes the meta data map and encods into a byte slice with header // describing the version, length of data and data in TSV format. -func (m *Meta) Encode() []byte { +func (m *Metadata) Encode() []byte { m.enc = m.enc[:headSize] // Iterate over map and append entries, only adding tab if we're not on the last entry diff --git a/stream/mts/metaEncode_test.go b/stream/mts/metaEncode_test.go index d422c700..849f5c0d 100644 --- a/stream/mts/metaEncode_test.go +++ b/stream/mts/metaEncode_test.go @@ -27,8 +27,28 @@ LICENSE package mts -import "testing" +import ( + "bytes" + "testing" + + "bitbucket.org/ausocean/av/stream/mts/psi" +) + +const fps = 25 func TestMetaEncode(t *testing.T) { - + var b []byte + buf := bytes.NewBuffer(b) + e := NewEncoder(buf, fps) + Meta.Add("ts", "12345678") + e.writePSI() + out := buf.Bytes() + got := out[PacketSize:] + want := []byte{ + 0x00, 0x02, 0xb0, 0x12, 0x00, 0x01, 0xc1, 0x00, 0x00, 0xe1, 0x00, 0xf0, 0x0a, + psi.MetadataTag, // Descriptor tag + 0x08, // Length of bytes to follow + 0x00, 0x00, 0x00, 0x00, 0x49, 0xa2, 0x36, 0x0b, // timestamp + 0x1b, 0xe1, 0x00, 0xf0, 0x00, + } } From d373f85b850e32c864a9d20719c6f2ec6a853ea2 Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 30 Jan 2019 15:37:15 +1030 Subject: [PATCH 066/208] stream/mts: wrote test in metaEncode_test.go and found bug which was fixed --- stream/mts/encoder.go | 34 ++++++++++++++++--------------- stream/mts/metaEncode_test.go | 25 +++++++++++++++++------ stream/mts/meta_test.go | 1 - stream/mts/psi/crc.go | 6 +++--- stream/mts/psi/descriptor_test.go | 8 ++++---- stream/mts/psi/helpers.go | 6 +++--- stream/mts/psi/psi.go | 7 +++---- stream/mts/psi/psi_test.go | 6 +++--- 8 files changed, 53 insertions(+), 40 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 4567aea2..a1527e8d 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -200,27 +200,29 @@ func (e *Encoder) Encode(nalu []byte) error { func (e *Encoder) writePSI() error { // Write PAT. patPkt := Packet{ - PUSI: true, - PID: PatPid, - CC: e.ccFor(PatPid), - AFC: HasPayload, + PUSI: true, + PID: PatPid, + CC: e.ccFor(PatPid), + AFC: HasPayload, + Payload: psi.AddPadding(patTable), } - patPkt.FillPayload(patTable) _, err := e.dst.Write(patPkt.Bytes(e.tsSpace[:PacketSize])) if err != nil { return err } - updateMeta(pmtTable) + if err = updateMeta(&pmtTable); err != nil { + return err + } // Create mts packet from pmt table. pmtPkt := Packet{ - PUSI: true, - PID: PmtPid, - CC: e.ccFor(PmtPid), - AFC: HasPayload, + PUSI: true, + PID: PmtPid, + CC: e.ccFor(PmtPid), + AFC: HasPayload, + Payload: psi.AddPadding(pmtTable), } - pmtPkt.FillPayload(pmtTable) _, err = e.dst.Write(pmtPkt.Bytes(e.tsSpace[:PacketSize])) if err != nil { return err @@ -253,10 +255,10 @@ func (e *Encoder) ccFor(pid int) byte { } // updateMeta ... -func updateMeta(b []byte) error { - var p psi.PSIBytes - p = b +func updateMeta(b *[]byte) error { m := Meta.Encode() - p.AddDescriptor(psi.MetadataTag, m) - return nil + p := psi.PSIBytes(*b) + err := p.AddDescriptor(psi.MetadataTag, m) + *b = []byte(p) + return err } diff --git a/stream/mts/metaEncode_test.go b/stream/mts/metaEncode_test.go index 849f5c0d..a914da2b 100644 --- a/stream/mts/metaEncode_test.go +++ b/stream/mts/metaEncode_test.go @@ -34,6 +34,11 @@ import ( "bitbucket.org/ausocean/av/stream/mts/psi" ) +const ( + errNotExpectedOut = "Unexpected output. \n Got : %v\n, Want: %v\n" + errUnexpectedErr = "Unexpected error: %v\n" +) + const fps = 25 func TestMetaEncode(t *testing.T) { @@ -41,14 +46,22 @@ func TestMetaEncode(t *testing.T) { buf := bytes.NewBuffer(b) e := NewEncoder(buf, fps) Meta.Add("ts", "12345678") - e.writePSI() + if err := e.writePSI(); err != nil { + t.Errorf(errUnexpectedErr, err.Error()) + } out := buf.Bytes() - got := out[PacketSize:] + got := out[PacketSize+4:] + want := []byte{ - 0x00, 0x02, 0xb0, 0x12, 0x00, 0x01, 0xc1, 0x00, 0x00, 0xe1, 0x00, 0xf0, 0x0a, - psi.MetadataTag, // Descriptor tag - 0x08, // Length of bytes to follow - 0x00, 0x00, 0x00, 0x00, 0x49, 0xa2, 0x36, 0x0b, // timestamp + 0x00, 0x02, 0xb0, 0x23, 0x00, 0x01, 0xc1, 0x00, 0x00, 0xe1, 0x00, 0xf0, 0x11, + psi.MetadataTag, // Descriptor tag + 0x0f, // Length of bytes to follow + 0x00, 0x10, 0x00, 0x0b, 't', 's', '=', '1', '2', '3', '4', '5', '6', '7', '8', // timestamp 0x1b, 0xe1, 0x00, 0xf0, 0x00, } + want = psi.AddCrc(want) + want = psi.AddPadding(want) + if !bytes.Equal(got, want) { + t.Errorf(errNotExpectedOut, got, want) + } } diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go index daff9038..11003b9b 100644 --- a/stream/mts/meta_test.go +++ b/stream/mts/meta_test.go @@ -144,7 +144,6 @@ func TestEncode(t *testing.T) { tstKey2+"="+tstData2)...) got := meta.Encode() - if !bytes.Equal(expectedOut, got) { t.Errorf("Did not get expected out. \nGot : %v \nwant: %v", got, expectedOut) } diff --git a/stream/mts/psi/crc.go b/stream/mts/psi/crc.go index cd9a1199..e0ac7deb 100644 --- a/stream/mts/psi/crc.go +++ b/stream/mts/psi/crc.go @@ -34,15 +34,15 @@ import ( ) // addCrc appends a crc table to a given psi table in bytes -func addCrc(out []byte) []byte { +func AddCrc(out []byte) []byte { t := make([]byte, len(out)+4) copy(t, out) - updateCrc(t[1:]) + UpdateCrc(t[1:]) return t } // updateCrc updates the crc of bytes slice, writing the checksum into the last four bytes. -func updateCrc(b []byte) { +func UpdateCrc(b []byte) { crc32 := crc32_Update(0xffffffff, crc32_MakeTable(bits.Reverse32(crc32.IEEE)), b[:len(b)-4]) binary.BigEndian.PutUint32(b[len(b)-4:], crc32) } diff --git a/stream/mts/psi/descriptor_test.go b/stream/mts/psi/descriptor_test.go index 0f7d300e..94441277 100644 --- a/stream/mts/psi/descriptor_test.go +++ b/stream/mts/psi/descriptor_test.go @@ -231,7 +231,7 @@ func TestDescriptorsNone(t *testing.T) { func TestCreateDescriptorEmpty(t *testing.T) { got := PSIBytes(tstPsi2.Bytes()) got.createDescriptor(TimeDescTag, make([]byte, TimeDataSize)) - updateCrc(got[1:]) + UpdateCrc(got[1:]) want := PSIBytes(tstPsi1.Bytes()) if !bytes.Equal(want, got) { t.Errorf(errNotExpectedOut, got, want) @@ -244,7 +244,7 @@ func TestCreateDescriptorEmpty(t *testing.T) { func TestCreateDescriptorNotEmpty(t *testing.T) { got := PSIBytes(tstPsi1.Bytes()) got.createDescriptor(LocationDescTag, make([]byte, LocationDataSize)) - updateCrc(got[1:]) + UpdateCrc(got[1:]) want := PSIBytes(tstPsi3.Bytes()) if !bytes.Equal(want, got) { t.Errorf(errNotExpectedOut, got, want) @@ -301,7 +301,7 @@ func TestAddDescriptorUpdateBigger(t *testing.T) { if err := got.AddDescriptor(TimeDescTag, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a}); err != nil { t.Errorf(errUnexpectedErr, err.Error()) } - want := addCrc(pmtTimeBytesResizedBigger) + want := AddCrc(pmtTimeBytesResizedBigger) if !bytes.Equal(got, want) { t.Errorf(errNotExpectedOut, got, want) } @@ -315,7 +315,7 @@ func TestAddDescriptorUpdateSmaller(t *testing.T) { if err := got.AddDescriptor(TimeDescTag, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}); err != nil { t.Errorf(errUnexpectedErr, err.Error()) } - want := addCrc(pmtTimeBytesResizedSmaller) + want := AddCrc(pmtTimeBytesResizedSmaller) if !bytes.Equal(got, want) { t.Errorf(errNotExpectedOut, got, want) } diff --git a/stream/mts/psi/helpers.go b/stream/mts/psi/helpers.go index 174b289b..b8bab6b5 100644 --- a/stream/mts/psi/helpers.go +++ b/stream/mts/psi/helpers.go @@ -64,7 +64,7 @@ func UpdateTime(dst []byte, t uint64) error { for i := range dst[TimeDataIndx : TimeDataIndx+TimeDataSize] { dst[i+TimeDataIndx] = ts[i] } - updateCrc(dst[1:]) + UpdateCrc(dst[1:]) return nil } @@ -111,7 +111,7 @@ func UpdateLocation(d []byte, s string) error { for i := range loc { loc[i] = 0 } - updateCrc(d[1:]) + UpdateCrc(d[1:]) return nil } @@ -126,7 +126,7 @@ func trimTo(d []byte, t byte) []byte { // addPadding adds an appropriate amount of padding to a pat or pmt table for // addition to an mpegts packet -func addPadding(d []byte) []byte { +func AddPadding(d []byte) []byte { t := make([]byte, PacketSize) copy(t, d) padding := t[len(d):] diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index dca9e57a..91d8fade 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -169,7 +169,7 @@ func (p *PSI) Bytes() []byte { out[2] = 0x80 | 0x30 | (0x03 & byte(p.Sl>>8)) out[3] = byte(p.Sl) out = append(out, p.Tss.Bytes()...) - out = addCrc(out) + out = AddCrc(out) return out } @@ -283,7 +283,7 @@ func (p *PSIBytes) AddDescriptor(tag int, data []byte) error { p.setProgInfoLen(newProgInfoLen) newSectionLen := int(psi.SectionLength(*p)) + delta p.setSectionLen(newSectionLen) - updateCrc((*p)[1:]) + UpdateCrc((*p)[1:]) return nil } @@ -320,7 +320,6 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) { copy(tmp, *p) *p = tmp copy((*p)[newDescIdx+newDescLen:], (*p)[newDescIdx:newDescIdx+newDescLen]) - // Set the tag, data len and data of the new desriptor. (*p)[newDescIdx] = byte(tag) (*p)[newDescIdx+1] = byte(dataLen) @@ -332,7 +331,7 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) { p.setProgInfoLen(newProgInfoLen) newSyntaxSectionLen := int(oldSyntaxSectionLen) + addedLen p.setSectionLen(newSyntaxSectionLen) - updateCrc((*p)[1:]) + UpdateCrc((*p)[1:]) } // setProgInfoLen sets the program information length in a psi with a pmt. diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go index 61a740a4..7e3a3104 100644 --- a/stream/mts/psi/psi_test.go +++ b/stream/mts/psi/psi_test.go @@ -282,7 +282,7 @@ var bytesTests = []struct { func TestBytes(t *testing.T) { for _, test := range bytesTests { got := test.input.Bytes() - if !bytes.Equal(got, addCrc(test.want)) { + if !bytes.Equal(got, AddCrc(test.want)) { t.Errorf("unexpected error for test %v: got:%v want:%v", test.name, got, test.want) } @@ -301,7 +301,7 @@ func TestTimestampToBytes(t *testing.T) { func TestTimeUpdate(t *testing.T) { cpy := make([]byte, len(pmtTimeBytes1)) copy(cpy, pmtTimeBytes1) - cpy = addCrc(cpy) + cpy = AddCrc(cpy) err := UpdateTime(cpy, tstTime2) cpy = cpy[:len(cpy)-4] if err != nil { @@ -343,7 +343,7 @@ func TestLocationGet(t *testing.T) { func TestLocationUpdate(t *testing.T) { cpy := make([]byte, len(pmtWithMetaTst1)) copy(cpy, pmtWithMetaTst1) - cpy = addCrc(cpy) + cpy = AddCrc(cpy) err := UpdateLocation(cpy, locationTstStr2) cpy = cpy[:len(cpy)-4] if err != nil { From 1d9cb575053190a57fc65bfd9fc4fe228a0c6c78 Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 30 Jan 2019 16:22:57 +1030 Subject: [PATCH 067/208] stream/mts/metaEncode_test.go: added another test to check behaviour when more meta data is added. --- stream/mts/metaEncode_test.go | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/stream/mts/metaEncode_test.go b/stream/mts/metaEncode_test.go index a914da2b..d372f20a 100644 --- a/stream/mts/metaEncode_test.go +++ b/stream/mts/metaEncode_test.go @@ -41,7 +41,7 @@ const ( const fps = 25 -func TestMetaEncode(t *testing.T) { +func TestMetaEncode1(t *testing.T) { var b []byte buf := bytes.NewBuffer(b) e := NewEncoder(buf, fps) @@ -65,3 +65,29 @@ func TestMetaEncode(t *testing.T) { t.Errorf(errNotExpectedOut, got, want) } } + +func TestMetaEncode2(t *testing.T) { + var b []byte + buf := bytes.NewBuffer(b) + e := NewEncoder(buf, fps) + Meta.Add("ts", "12345678") + Meta.Add("loc", "1234,4321,1234") + if err := e.writePSI(); err != nil { + t.Errorf(errUnexpectedErr, err.Error()) + } + out := buf.Bytes() + got := out[PacketSize+4:] + want := []byte{ + 0x00, 0x02, 0xb0, 0x36, 0x00, 0x01, 0xc1, 0x00, 0x00, 0xe1, 0x00, 0xf0, 0x24, + psi.MetadataTag, // Descriptor tag + 0x22, // Length of bytes to follow + 0x00, 0x10, 0x00, 0x1e, 't', 's', '=', '1', '2', '3', '4', '5', '6', '7', '8', '\t', // timestamp + 'l', 'o', 'c', '=', '1', '2', '3', '4', ',', '4', '3', '2', '1', ',', '1', '2', '3', '4', // location + 0x1b, 0xe1, 0x00, 0xf0, 0x00, + } + want = psi.AddCrc(want) + want = psi.AddPadding(want) + if !bytes.Equal(got, want) { + t.Errorf(errNotExpectedOut, got, want) + } +} From 9011c090d9fa946b1ab73780841c1ebe27ce1252 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 31 Jan 2019 13:26:16 +1030 Subject: [PATCH 068/208] stream/mts/meta.go: wrote ReadFrom func - still need to write tests for it --- stream/mts/meta.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/stream/mts/meta.go b/stream/mts/meta.go index 62097d30..8bd5365b 100644 --- a/stream/mts/meta.go +++ b/stream/mts/meta.go @@ -29,6 +29,7 @@ package mts import ( "errors" + "strings" "sync" ) @@ -131,3 +132,14 @@ func (m *Metadata) Encode() []byte { return m.enc } + +func ReadFrom(d []byte, key string) (string, error) { + entries := strings.Split(string(d), "\t") + for _, entry := range entries { + kv := strings.Split(entry, "=") + if kv[0] == key { + return kv[1], nil + } + } + return "", errors.New("could not find key in metadata") +} From 10343375a30c5dcac68bc81ff905bd0561fc1612 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 31 Jan 2019 21:23:06 +1030 Subject: [PATCH 069/208] stream/mts: moved meta.go and meta_test.go into meta package --- stream/mts/meta/meta.go | 145 +++++++++++++++++++++++++++++++++ stream/mts/meta/meta_test.go | 150 +++++++++++++++++++++++++++++++++++ 2 files changed, 295 insertions(+) create mode 100644 stream/mts/meta/meta.go create mode 100644 stream/mts/meta/meta_test.go diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go new file mode 100644 index 00000000..8bd5365b --- /dev/null +++ b/stream/mts/meta/meta.go @@ -0,0 +1,145 @@ +/* +NAME + meta.go + +DESCRIPTION + See Readme.md + +AUTHOR + Saxon Nelson-Milton + +LICENSE + meta.go is Copyright (C) 2017-2019 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ + +package mts + +import ( + "errors" + "strings" + "sync" +) + +const headSize = 4 + +const ( + majVer = 1 + minVer = 0 +) + +const ( + dataLenIdx1 = 2 + dataLenIdx2 = 3 +) + +var ( + errKeyAbsent = errors.New("Key does not exist in map") +) + +type Metadata struct { + mu sync.RWMutex + data map[string]string + enc []byte +} + +func NewMeta() *Metadata { + return &Metadata{ + data: make(map[string]string), + enc: []byte{ + 0x00, // Reserved byte + (majVer << 4) | minVer, // MS and LS versions + 0x00, // Data len byte1 + 0x00, // Data len byte2 + }, + } +} + +// Add adds metadata with key and val, if already exists return error +func (m *Metadata) Add(key, val string) { + m.mu.Lock() + m.data[key] = val + m.mu.Unlock() +} + +// All returns the a copy of the map containing the meta data +func (m *Metadata) All() map[string]string { + m.mu.Lock() + cpy := make(map[string]string) + for k, v := range m.data { + cpy[k] = v + } + m.mu.Unlock() + return cpy +} + +// Get returns the meta data for the passed key +func (m *Metadata) Get(key string) (string, error) { + m.mu.Lock() + val, ok := m.data[key] + m.mu.Unlock() + if !ok { + return "", errKeyAbsent + } + + return val, nil +} + +// Remove deletes a meta entry in the map and returns error if it doesn’t exist +func (m *Metadata) Delete(key string) error { + m.mu.Lock() + defer m.mu.Unlock() + if _, ok := m.data[key]; ok { + delete(m.data, key) + return nil + } + return errKeyAbsent +} + +// Encode takes the meta data map and encods into a byte slice with header +// describing the version, length of data and data in TSV format. +func (m *Metadata) Encode() []byte { + m.enc = m.enc[:headSize] + + // Iterate over map and append entries, only adding tab if we're not on the last entry + var i int + var entry string + for k, v := range m.data { + i++ + entry += k + "=" + v + if i < len(m.data) { + entry += "\t" + } + } + m.enc = append(m.enc, []byte(entry)...) + + // Calculate and set data length in encoded meta header. + dataLen := len(m.enc[headSize:]) + m.enc[dataLenIdx1] = byte(dataLen >> 8) + m.enc[dataLenIdx2] = byte(dataLen) + + return m.enc +} + +func ReadFrom(d []byte, key string) (string, error) { + entries := strings.Split(string(d), "\t") + for _, entry := range entries { + kv := strings.Split(entry, "=") + if kv[0] == key { + return kv[1], nil + } + } + return "", errors.New("could not find key in metadata") +} diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go new file mode 100644 index 00000000..11003b9b --- /dev/null +++ b/stream/mts/meta/meta_test.go @@ -0,0 +1,150 @@ +/* +NAME + meta_test.go + +DESCRIPTION + See Readme.md + +AUTHOR + Saxon Nelson-Milton + +LICENSE + meta_test.go is Copyright (C) 2017-2019 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ + +package mts + +import ( + "bytes" + "errors" + "reflect" + "testing" +) + +const ( + tstKey1 = "loc" + tstData1 = "a,b,c" + tstKey2 = "ts" + tstData2 = "12345678" + tstData3 = "d,e,f" +) + +// TestAddAndGet ensures that we can add metadata and then successfully get it. +func TestAddAndGet(t *testing.T) { + meta := NewMeta() + meta.Add(tstKey1, tstData1) + meta.Add(tstKey2, tstData2) + errors.New("Trying to delete map entry that doesn't exist") + if data, err := meta.Get(tstKey1); err != nil { + t.Errorf("Could not get data for key: loc: %v", err.Error()) + if data != tstData1 { + t.Errorf("Did not get expected data") + } + } + + if data, err := meta.Get(tstKey2); err != nil { + t.Errorf("Could not get data for key: ts: %v", err.Error()) + if data != tstData2 { + t.Errorf("Did not get expected data") + } + } +} + +// TestUpdate checks that we can use Meta.Add to actually update metadata +// if it already exists in the Meta map. +func TestUpdate(t *testing.T) { + meta := NewMeta() + meta.Add(tstKey1, tstData1) + meta.Add(tstKey1, tstData3) + + if data, err := meta.Get(tstKey1); err != nil { + t.Errorf("Did not expect err: %v", err.Error()) + if data != tstData2 { + t.Errorf("Data did not correctly update for key \"loc\"") + } + } +} + +// TestAll ensures we can get a correct map using Meta.All() after adding some data +func TestAll(t *testing.T) { + meta := NewMeta() + tstMap := map[string]string{ + tstKey1: tstData1, + tstKey2: tstData2, + } + + meta.Add(tstKey1, tstData1) + meta.Add(tstKey2, tstData2) + metaMap := meta.All() + + if !reflect.DeepEqual(metaMap, tstMap) { + t.Errorf("Map not correct. Got: %v, want: %v", metaMap, tstMap) + } +} + +// TestGetAbsentKey ensures that we get the expected error when we try to get with +// key that does not yet exist in the Meta map. +func TestGetAbsentKey(t *testing.T) { + meta := NewMeta() + + if _, err := meta.Get(tstKey1); err != errKeyAbsent { + t.Errorf("Did not get expected err: %v", errKeyAbsent.Error()) + } +} + +// TestDelete ensures we can remove a data entry in the Meta map. +func TestDelete(t *testing.T) { + meta := NewMeta() + meta.Add(tstKey1, tstData1) + if err := meta.Delete(tstKey1); err != nil { + t.Errorf("Did not expect error: %v", err.Error()) + } + if _, err := meta.Get(tstKey1); err != errKeyAbsent { + t.Errorf("Did not get expected err: %v", errKeyAbsent.Error()) + } +} + +// TestDeleteAbsentKey checks that we get an expected error when we try to delete +// an entry in the Meta map that doesn't exist. +func TestDeleteAbsentKey(t *testing.T) { + meta := NewMeta() + if err := meta.Delete(tstKey1); err != errKeyAbsent { + t.Errorf("Did not get expected err: %v", errKeyAbsent.Error()) + } +} + +// TestEncode checks that we're getting the correct byte slice from Meta.Encode(). +func TestEncode(t *testing.T) { + meta := NewMeta() + meta.Add(tstKey1, tstData1) + meta.Add(tstKey2, tstData2) + + dataLen := len(tstKey1+tstData1+tstKey2+tstData2) + 3 + expectedOut := []byte{ + 0x00, + 0x10, + byte(dataLen >> 8), + byte(dataLen), + } + expectedOut = append(expectedOut, []byte( + tstKey1+"="+tstData1+"\t"+ + tstKey2+"="+tstData2)...) + + got := meta.Encode() + if !bytes.Equal(expectedOut, got) { + t.Errorf("Did not get expected out. \nGot : %v \nwant: %v", got, expectedOut) + } +} From 1f1546a28498d720da0959748fa848509a45b4f4 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 31 Jan 2019 21:30:08 +1030 Subject: [PATCH 070/208] revid: updated code in senders.go to work with meta changes --- revid/senders.go | 5 +- stream/mts/encoder.go | 5 +- stream/mts/meta.go | 145 --------------------------------- stream/mts/meta/meta.go | 6 +- stream/mts/meta/meta_test.go | 16 ++-- stream/mts/meta_test.go | 150 ----------------------------------- 6 files changed, 17 insertions(+), 310 deletions(-) delete mode 100644 stream/mts/meta.go delete mode 100644 stream/mts/meta_test.go diff --git a/revid/senders.go b/revid/senders.go index a73523d5..e86cb24b 100644 --- a/revid/senders.go +++ b/revid/senders.go @@ -35,6 +35,7 @@ import ( "net" "os" "os/exec" + "strconv" "bitbucket.org/ausocean/av/rtmp" "bitbucket.org/ausocean/av/stream/mts" @@ -172,7 +173,7 @@ func (s *httpSender) extractMeta(r string) error { s.log(logger.Warning, pkg+"No timestamp in reply") } else { s.log(logger.Debug, fmt.Sprintf("%v got timestamp: %v", pkg, t)) - mts.MetaData.SetTimeStamp(uint64(t)) + mts.Meta.Add("ts", strconv.Itoa(t)) } // Extract location from reply @@ -181,7 +182,7 @@ func (s *httpSender) extractMeta(r string) error { s.log(logger.Warning, pkg+"No location in reply") } else { s.log(logger.Debug, fmt.Sprintf("%v got location: %v", pkg, g)) - mts.MetaData.SetLocation(g) + mts.Meta.Add("loc", g) } return nil diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index a1527e8d..833c18d9 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -32,6 +32,7 @@ import ( "io" "time" + "bitbucket.org/ausocean/av/stream/mts/meta" "bitbucket.org/ausocean/av/stream/mts/pes" "bitbucket.org/ausocean/av/stream/mts/psi" ) @@ -88,10 +89,10 @@ const ( ) // global Meta -var Meta *Metadata +var Meta *meta.Metadata func init() { - Meta = NewMeta() + Meta = meta.New() } var ( diff --git a/stream/mts/meta.go b/stream/mts/meta.go deleted file mode 100644 index 8bd5365b..00000000 --- a/stream/mts/meta.go +++ /dev/null @@ -1,145 +0,0 @@ -/* -NAME - meta.go - -DESCRIPTION - See Readme.md - -AUTHOR - Saxon Nelson-Milton - -LICENSE - meta.go is Copyright (C) 2017-2019 the Australian Ocean Lab (AusOcean) - - It is free software: you can redistribute it and/or modify them - under the terms of the GNU General Public License as published by the - Free Software Foundation, either version 3 of the License, or (at your - option) any later version. - - It is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License - along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. -*/ - -package mts - -import ( - "errors" - "strings" - "sync" -) - -const headSize = 4 - -const ( - majVer = 1 - minVer = 0 -) - -const ( - dataLenIdx1 = 2 - dataLenIdx2 = 3 -) - -var ( - errKeyAbsent = errors.New("Key does not exist in map") -) - -type Metadata struct { - mu sync.RWMutex - data map[string]string - enc []byte -} - -func NewMeta() *Metadata { - return &Metadata{ - data: make(map[string]string), - enc: []byte{ - 0x00, // Reserved byte - (majVer << 4) | minVer, // MS and LS versions - 0x00, // Data len byte1 - 0x00, // Data len byte2 - }, - } -} - -// Add adds metadata with key and val, if already exists return error -func (m *Metadata) Add(key, val string) { - m.mu.Lock() - m.data[key] = val - m.mu.Unlock() -} - -// All returns the a copy of the map containing the meta data -func (m *Metadata) All() map[string]string { - m.mu.Lock() - cpy := make(map[string]string) - for k, v := range m.data { - cpy[k] = v - } - m.mu.Unlock() - return cpy -} - -// Get returns the meta data for the passed key -func (m *Metadata) Get(key string) (string, error) { - m.mu.Lock() - val, ok := m.data[key] - m.mu.Unlock() - if !ok { - return "", errKeyAbsent - } - - return val, nil -} - -// Remove deletes a meta entry in the map and returns error if it doesn’t exist -func (m *Metadata) Delete(key string) error { - m.mu.Lock() - defer m.mu.Unlock() - if _, ok := m.data[key]; ok { - delete(m.data, key) - return nil - } - return errKeyAbsent -} - -// Encode takes the meta data map and encods into a byte slice with header -// describing the version, length of data and data in TSV format. -func (m *Metadata) Encode() []byte { - m.enc = m.enc[:headSize] - - // Iterate over map and append entries, only adding tab if we're not on the last entry - var i int - var entry string - for k, v := range m.data { - i++ - entry += k + "=" + v - if i < len(m.data) { - entry += "\t" - } - } - m.enc = append(m.enc, []byte(entry)...) - - // Calculate and set data length in encoded meta header. - dataLen := len(m.enc[headSize:]) - m.enc[dataLenIdx1] = byte(dataLen >> 8) - m.enc[dataLenIdx2] = byte(dataLen) - - return m.enc -} - -func ReadFrom(d []byte, key string) (string, error) { - entries := strings.Split(string(d), "\t") - for _, entry := range entries { - kv := strings.Split(entry, "=") - if kv[0] == key { - return kv[1], nil - } - } - return "", errors.New("could not find key in metadata") -} diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go index 8bd5365b..2f26a797 100644 --- a/stream/mts/meta/meta.go +++ b/stream/mts/meta/meta.go @@ -25,7 +25,7 @@ LICENSE along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. */ -package mts +package meta import ( "errors" @@ -55,7 +55,7 @@ type Metadata struct { enc []byte } -func NewMeta() *Metadata { +func New() *Metadata { return &Metadata{ data: make(map[string]string), enc: []byte{ @@ -67,7 +67,7 @@ func NewMeta() *Metadata { } } -// Add adds metadata with key and val, if already exists return error +// Add adds metadata with key and val func (m *Metadata) Add(key, val string) { m.mu.Lock() m.data[key] = val diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go index 11003b9b..6df47e5c 100644 --- a/stream/mts/meta/meta_test.go +++ b/stream/mts/meta/meta_test.go @@ -25,7 +25,7 @@ LICENSE along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. */ -package mts +package meta import ( "bytes" @@ -44,7 +44,7 @@ const ( // TestAddAndGet ensures that we can add metadata and then successfully get it. func TestAddAndGet(t *testing.T) { - meta := NewMeta() + meta := New() meta.Add(tstKey1, tstData1) meta.Add(tstKey2, tstData2) errors.New("Trying to delete map entry that doesn't exist") @@ -66,7 +66,7 @@ func TestAddAndGet(t *testing.T) { // TestUpdate checks that we can use Meta.Add to actually update metadata // if it already exists in the Meta map. func TestUpdate(t *testing.T) { - meta := NewMeta() + meta := New() meta.Add(tstKey1, tstData1) meta.Add(tstKey1, tstData3) @@ -80,7 +80,7 @@ func TestUpdate(t *testing.T) { // TestAll ensures we can get a correct map using Meta.All() after adding some data func TestAll(t *testing.T) { - meta := NewMeta() + meta := New() tstMap := map[string]string{ tstKey1: tstData1, tstKey2: tstData2, @@ -98,7 +98,7 @@ func TestAll(t *testing.T) { // TestGetAbsentKey ensures that we get the expected error when we try to get with // key that does not yet exist in the Meta map. func TestGetAbsentKey(t *testing.T) { - meta := NewMeta() + meta := New() if _, err := meta.Get(tstKey1); err != errKeyAbsent { t.Errorf("Did not get expected err: %v", errKeyAbsent.Error()) @@ -107,7 +107,7 @@ func TestGetAbsentKey(t *testing.T) { // TestDelete ensures we can remove a data entry in the Meta map. func TestDelete(t *testing.T) { - meta := NewMeta() + meta := New() meta.Add(tstKey1, tstData1) if err := meta.Delete(tstKey1); err != nil { t.Errorf("Did not expect error: %v", err.Error()) @@ -120,7 +120,7 @@ func TestDelete(t *testing.T) { // TestDeleteAbsentKey checks that we get an expected error when we try to delete // an entry in the Meta map that doesn't exist. func TestDeleteAbsentKey(t *testing.T) { - meta := NewMeta() + meta := New() if err := meta.Delete(tstKey1); err != errKeyAbsent { t.Errorf("Did not get expected err: %v", errKeyAbsent.Error()) } @@ -128,7 +128,7 @@ func TestDeleteAbsentKey(t *testing.T) { // TestEncode checks that we're getting the correct byte slice from Meta.Encode(). func TestEncode(t *testing.T) { - meta := NewMeta() + meta := New() meta.Add(tstKey1, tstData1) meta.Add(tstKey2, tstData2) diff --git a/stream/mts/meta_test.go b/stream/mts/meta_test.go deleted file mode 100644 index 11003b9b..00000000 --- a/stream/mts/meta_test.go +++ /dev/null @@ -1,150 +0,0 @@ -/* -NAME - meta_test.go - -DESCRIPTION - See Readme.md - -AUTHOR - Saxon Nelson-Milton - -LICENSE - meta_test.go is Copyright (C) 2017-2019 the Australian Ocean Lab (AusOcean) - - It is free software: you can redistribute it and/or modify them - under the terms of the GNU General Public License as published by the - Free Software Foundation, either version 3 of the License, or (at your - option) any later version. - - It is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License - along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. -*/ - -package mts - -import ( - "bytes" - "errors" - "reflect" - "testing" -) - -const ( - tstKey1 = "loc" - tstData1 = "a,b,c" - tstKey2 = "ts" - tstData2 = "12345678" - tstData3 = "d,e,f" -) - -// TestAddAndGet ensures that we can add metadata and then successfully get it. -func TestAddAndGet(t *testing.T) { - meta := NewMeta() - meta.Add(tstKey1, tstData1) - meta.Add(tstKey2, tstData2) - errors.New("Trying to delete map entry that doesn't exist") - if data, err := meta.Get(tstKey1); err != nil { - t.Errorf("Could not get data for key: loc: %v", err.Error()) - if data != tstData1 { - t.Errorf("Did not get expected data") - } - } - - if data, err := meta.Get(tstKey2); err != nil { - t.Errorf("Could not get data for key: ts: %v", err.Error()) - if data != tstData2 { - t.Errorf("Did not get expected data") - } - } -} - -// TestUpdate checks that we can use Meta.Add to actually update metadata -// if it already exists in the Meta map. -func TestUpdate(t *testing.T) { - meta := NewMeta() - meta.Add(tstKey1, tstData1) - meta.Add(tstKey1, tstData3) - - if data, err := meta.Get(tstKey1); err != nil { - t.Errorf("Did not expect err: %v", err.Error()) - if data != tstData2 { - t.Errorf("Data did not correctly update for key \"loc\"") - } - } -} - -// TestAll ensures we can get a correct map using Meta.All() after adding some data -func TestAll(t *testing.T) { - meta := NewMeta() - tstMap := map[string]string{ - tstKey1: tstData1, - tstKey2: tstData2, - } - - meta.Add(tstKey1, tstData1) - meta.Add(tstKey2, tstData2) - metaMap := meta.All() - - if !reflect.DeepEqual(metaMap, tstMap) { - t.Errorf("Map not correct. Got: %v, want: %v", metaMap, tstMap) - } -} - -// TestGetAbsentKey ensures that we get the expected error when we try to get with -// key that does not yet exist in the Meta map. -func TestGetAbsentKey(t *testing.T) { - meta := NewMeta() - - if _, err := meta.Get(tstKey1); err != errKeyAbsent { - t.Errorf("Did not get expected err: %v", errKeyAbsent.Error()) - } -} - -// TestDelete ensures we can remove a data entry in the Meta map. -func TestDelete(t *testing.T) { - meta := NewMeta() - meta.Add(tstKey1, tstData1) - if err := meta.Delete(tstKey1); err != nil { - t.Errorf("Did not expect error: %v", err.Error()) - } - if _, err := meta.Get(tstKey1); err != errKeyAbsent { - t.Errorf("Did not get expected err: %v", errKeyAbsent.Error()) - } -} - -// TestDeleteAbsentKey checks that we get an expected error when we try to delete -// an entry in the Meta map that doesn't exist. -func TestDeleteAbsentKey(t *testing.T) { - meta := NewMeta() - if err := meta.Delete(tstKey1); err != errKeyAbsent { - t.Errorf("Did not get expected err: %v", errKeyAbsent.Error()) - } -} - -// TestEncode checks that we're getting the correct byte slice from Meta.Encode(). -func TestEncode(t *testing.T) { - meta := NewMeta() - meta.Add(tstKey1, tstData1) - meta.Add(tstKey2, tstData2) - - dataLen := len(tstKey1+tstData1+tstKey2+tstData2) + 3 - expectedOut := []byte{ - 0x00, - 0x10, - byte(dataLen >> 8), - byte(dataLen), - } - expectedOut = append(expectedOut, []byte( - tstKey1+"="+tstData1+"\t"+ - tstKey2+"="+tstData2)...) - - got := meta.Encode() - if !bytes.Equal(expectedOut, got) { - t.Errorf("Did not get expected out. \nGot : %v \nwant: %v", got, expectedOut) - } -} From 38d5d6f0fdea7985dc8726bf9148af3da13ea657 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 1 Feb 2019 09:27:43 +1030 Subject: [PATCH 071/208] stream/mts/meta/meta_test.go: adding test for ReadFrom --- stream/mts/meta/meta_test.go | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go index 6df47e5c..d40e26d5 100644 --- a/stream/mts/meta/meta_test.go +++ b/stream/mts/meta/meta_test.go @@ -42,6 +42,10 @@ const ( tstData3 = "d,e,f" ) +var ( + errNotExpectedOut = "Did not get expected out. \nGot : %v, \nwant: %v" +) + // TestAddAndGet ensures that we can add metadata and then successfully get it. func TestAddAndGet(t *testing.T) { meta := New() @@ -145,6 +149,26 @@ func TestEncode(t *testing.T) { got := meta.Encode() if !bytes.Equal(expectedOut, got) { - t.Errorf("Did not get expected out. \nGot : %v \nwant: %v", got, expectedOut) + t.Errorf(errNotExpectedOut, got, expectedOut) + } +} + +func TestReadFrom(t *testing.T) { + tstStr := "loc=a,b,c\tts=12345" + got, err := ReadFrom([]byte(tstStr), "loc") + if err != nil { + t.Errorf("Got unexpected error: %v", err.Error()) + } + want := "a,b,c" + if got != want { + t.Errorf(errNotExpectedOut, got, want) + } + + if got, err = ReadFrom([]byte(tstStr), "ts"); err != nil { + t.Errorf("Got unexpected error: %v", err.Error()) + } + want = "12345" + if got != want { + t.Errorf(errNotExpectedOut, got, want) } } From 4d4a8e04ec8b4a3019c4e451ae42cf4831c2c0e3 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 1 Feb 2019 11:17:53 +1030 Subject: [PATCH 072/208] stream/mts/meta/meta_test.go: created global err vars --- stream/mts/meta/meta_test.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go index d40e26d5..0fcda13b 100644 --- a/stream/mts/meta/meta_test.go +++ b/stream/mts/meta/meta_test.go @@ -43,7 +43,9 @@ const ( ) var ( - errNotExpectedOut = "Did not get expected out. \nGot : %v, \nwant: %v" + errNotExpectedOut = "Did not get expected out. \nGot : %v, \nwant: %v\n" + errUnexpectedErr = "Unexpected err: %v\n" + errNotExpectedErr = "Not expected err: %v\n" ) // TestAddAndGet ensures that we can add metadata and then successfully get it. @@ -75,7 +77,7 @@ func TestUpdate(t *testing.T) { meta.Add(tstKey1, tstData3) if data, err := meta.Get(tstKey1); err != nil { - t.Errorf("Did not expect err: %v", err.Error()) + t.Errorf(errUnexpectedErr, err.Error()) if data != tstData2 { t.Errorf("Data did not correctly update for key \"loc\"") } @@ -105,7 +107,7 @@ func TestGetAbsentKey(t *testing.T) { meta := New() if _, err := meta.Get(tstKey1); err != errKeyAbsent { - t.Errorf("Did not get expected err: %v", errKeyAbsent.Error()) + t.Errorf(errNotExpectedErr, errKeyAbsent.Error()) } } @@ -114,10 +116,10 @@ func TestDelete(t *testing.T) { meta := New() meta.Add(tstKey1, tstData1) if err := meta.Delete(tstKey1); err != nil { - t.Errorf("Did not expect error: %v", err.Error()) + t.Errorf(errUnexpectedErr, err.Error()) } if _, err := meta.Get(tstKey1); err != errKeyAbsent { - t.Errorf("Did not get expected err: %v", errKeyAbsent.Error()) + t.Errorf(errNotExpectedErr, errKeyAbsent.Error()) } } @@ -126,7 +128,7 @@ func TestDelete(t *testing.T) { func TestDeleteAbsentKey(t *testing.T) { meta := New() if err := meta.Delete(tstKey1); err != errKeyAbsent { - t.Errorf("Did not get expected err: %v", errKeyAbsent.Error()) + t.Errorf(errNotExpectedErr, errKeyAbsent.Error()) } } @@ -157,7 +159,7 @@ func TestReadFrom(t *testing.T) { tstStr := "loc=a,b,c\tts=12345" got, err := ReadFrom([]byte(tstStr), "loc") if err != nil { - t.Errorf("Got unexpected error: %v", err.Error()) + t.Errorf(errUnexpectedErr, err.Error()) } want := "a,b,c" if got != want { @@ -165,7 +167,7 @@ func TestReadFrom(t *testing.T) { } if got, err = ReadFrom([]byte(tstStr), "ts"); err != nil { - t.Errorf("Got unexpected error: %v", err.Error()) + t.Errorf(errUnexpectedErr, err.Error()) } want = "12345" if got != want { From eb5487140272059bc4d6d0102e9a68df6e3cddf2 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 1 Feb 2019 11:39:47 +1030 Subject: [PATCH 073/208] stream/mts/encoder.go: simplified updateMeta --- stream/mts/encoder.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 833c18d9..cc4a699a 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -257,9 +257,8 @@ func (e *Encoder) ccFor(pid int) byte { // updateMeta ... func updateMeta(b *[]byte) error { - m := Meta.Encode() p := psi.PSIBytes(*b) - err := p.AddDescriptor(psi.MetadataTag, m) + err := p.AddDescriptor(psi.MetadataTag, Meta.Encode()) *b = []byte(p) return err } From 1af4b250306d9240d097845672da2fcbc0c2413e Mon Sep 17 00:00:00 2001 From: saxon Date: Sun, 3 Feb 2019 21:17:44 +1030 Subject: [PATCH 074/208] cmd/revid-cli & revid: removed startRevid and stopRevid as shouldn't be required when we have revid.Start() and revid.Stop(). Created revid.Config() which returns copy of config safely using mutex. removed updateRevid in revid-cli and move to fun revid.Update() - as there's no reason why it can't just be a receiver func - even better considering we want to start moving alot of stuff from revid-cli to the revid-api anyways. --- cmd/revid-cli/main.go | 54 +++++--------------- revid/revid.go | 115 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 42 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 5b826b91..826cb2a4 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -72,23 +72,22 @@ func main() { cfg := handleFlags() if !*useNetsender { - // run revid for the specified duration - rv, _, err := startRevid(nil, cfg) + rv, err := revid.New(cfg, nil) if err != nil { + cfg.Logger.Log(logger.Fatal, pkg+"failed to initialiase revid", "error", err.Error()) + } + if err = rv.Start(); err != nil { cfg.Logger.Log(logger.Fatal, pkg+"failed to start revid", "error", err.Error()) } time.Sleep(*runDurationPtr) - err = stopRevid(rv) - if err != nil { + if err = rv.Stop(); err != nil { cfg.Logger.Log(logger.Error, pkg+"failed to stop revid before program termination", "error", err.Error()) } return } - err := run(nil, cfg) - if err != nil { + if err := run(cfg); err != nil { log.Log(logger.Fatal, pkg+"failed to run revid", "error", err.Error()) - os.Exit(1) } } @@ -244,27 +243,20 @@ func handleFlags() revid.Config { } // initialize then run the main NetSender client -func run(rv *revid.Revid, cfg revid.Config) error { +func run(cfg revid.Config) error { // initialize NetSender and use NetSender's logger - //config.Logger = netsender.Logger() log.Log(logger.Info, pkg+"running in NetSender mode") var ns netsender.Sender - err := ns.Init(log, nil, nil, nil) - if err != nil { + if err := ns.Init(log, nil, nil, nil); err != nil { return err } vars, _ := ns.Vars() vs := ns.VarSum() - paused := false - if vars["mode"] == "Paused" { - paused = true - } - if !paused { - rv, cfg, err = updateRevid(&ns, rv, cfg, vars, false) - if err != nil { - return err - } + + rv, cfg, err = updateRevid(&ns, rv, cfg, vars, false) + if err != nil { + return err } for { @@ -331,28 +323,6 @@ func send(ns *netsender.Sender, rv *revid.Revid) error { return nil } -// wrappers for stopping and starting revid -func startRevid(ns *netsender.Sender, cfg revid.Config) (*revid.Revid, revid.Config, error) { - rv, err := revid.New(cfg, ns) - if err != nil { - return nil, cfg, err - } - err = rv.Start() - return rv, cfg, err -} - -func stopRevid(rv *revid.Revid) error { - err := rv.Stop() - if err != nil { - return err - } - - // FIXME(kortschak): Is this waiting on completion of work? - // Use a wait group and Wait method if it is. - time.Sleep(revidStopTime) - return nil -} - func updateRevid(ns *netsender.Sender, rv *revid.Revid, cfg revid.Config, vars map[string]string, stop bool) (*revid.Revid, revid.Config, error) { if stop { err := stopRevid(rv) diff --git a/revid/revid.go b/revid/revid.go index 6833ec2b..81546ead 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -40,6 +40,7 @@ import ( "sync" "time" + "bitbucket.org/ausocean/av/revid" "bitbucket.org/ausocean/av/stream" "bitbucket.org/ausocean/av/stream/flv" "bitbucket.org/ausocean/av/stream/lex" @@ -326,6 +327,13 @@ func (r *Revid) IsRunning() bool { return ret } +func (r *Revid) Config() Config { + r.mu.Lock() + ret := r.config + r.mu.Unlock() + return ret +} + // setIsRunning sets revid.isRunning using b. func (r *Revid) setIsRunning(b bool) { r.mu.Lock() @@ -366,6 +374,113 @@ func (r *Revid) Stop() error { return nil } +func (r *Revid) Update(vars map[string]string) error { + if r.IsRunning() { + r.Stop() + } + //look through the vars and update revid where needed + for key, value := range vars { + switch key { + case "Output": + // FIXME(kortschak): There can be only one! + // How do we specify outputs after the first? + // + // Maybe we shouldn't be doing this! + switch value { + case "File": + cfg.Outputs[0] = revid.File + case "Http": + cfg.Outputs[0] = revid.Http + case "Rtmp": + cfg.Outputs[0] = revid.Rtmp + case "FfmpegRtmp": + cfg.Outputs[0] = revid.FfmpegRtmp + default: + log.Log(logger.Warning, pkg+"invalid Output1 param", "value", value) + continue + } + case "FramesPerClip": + f, err := strconv.ParseUint(value, 10, 0) + if err != nil { + log.Log(logger.Warning, pkg+"invalid framesperclip param", "value", value) + break + } + cfg.FramesPerClip = uint(f) + case "RtmpUrl": + cfg.RtmpUrl = value + case "Bitrate": + r, err := strconv.ParseUint(value, 10, 0) + if err != nil { + log.Log(logger.Warning, pkg+"invalid framerate param", "value", value) + break + } + cfg.Bitrate = uint(r) + case "OutputFileName": + cfg.OutputFileName = value + case "InputFileName": + cfg.InputFileName = value + case "Height": + h, err := strconv.ParseUint(value, 10, 0) + if err != nil { + log.Log(logger.Warning, pkg+"invalid height param", "value", value) + break + } + cfg.Height = uint(h) + case "Width": + w, err := strconv.ParseUint(value, 10, 0) + if err != nil { + log.Log(logger.Warning, pkg+"invalid width param", "value", value) + break + } + cfg.Width = uint(w) + case "FrameRate": + r, err := strconv.ParseUint(value, 10, 0) + if err != nil { + log.Log(logger.Warning, pkg+"invalid framerate param", "value", value) + break + } + cfg.FrameRate = uint(r) + case "HttpAddress": + cfg.HttpAddress = value + case "Quantization": + q, err := strconv.ParseUint(value, 10, 0) + if err != nil { + log.Log(logger.Warning, pkg+"invalid quantization param", "value", value) + break + } + cfg.Quantization = uint(q) + case "IntraRefreshPeriod": + p, err := strconv.ParseUint(value, 10, 0) + if err != nil { + log.Log(logger.Warning, pkg+"invalid intrarefreshperiod param", "value", value) + break + } + cfg.IntraRefreshPeriod = uint(p) + case "HorizontalFlip": + switch strings.ToLower(value) { + case "true": + cfg.FlipHorizontal = true + case "false": + cfg.FlipHorizontal = false + default: + log.Log(logger.Warning, pkg+"invalid HorizontalFlip param", "value", value) + } + case "VerticalFlip": + switch strings.ToLower(value) { + case "true": + cfg.FlipVertical = true + case "false": + cfg.FlipVertical = false + default: + log.Log(logger.Warning, pkg+"invalid VerticalFlip param", "value", value) + } + default: + } + } + + return startRevid(ns, cfg) +} + // outputClips takes the clips produced in the packClips method and outputs them // to the desired output defined in the revid config func (r *Revid) outputClips() { From 9095044e234e730c19ed6a4f468bb46f2ba5c944 Mon Sep 17 00:00:00 2001 From: saxon Date: Sun, 3 Feb 2019 21:55:40 +1030 Subject: [PATCH 075/208] revid: using waitgroups so that revid.Stop() is safer - we can wait until the input and output routines are done before we do anything, like touch the revid config. Also started modifying revid.Update() to remove errors introduced after the copy of updateRevid from revid-cli to revid.go in the previous commit. --- revid/revid.go | 59 ++++++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/revid/revid.go b/revid/revid.go index 81546ead..33d847ea 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -1,6 +1,6 @@ /* NAME - revid.go + r.go DESCRIPTION See Readme.md @@ -40,7 +40,6 @@ import ( "sync" "time" - "bitbucket.org/ausocean/av/revid" "bitbucket.org/ausocean/av/stream" "bitbucket.org/ausocean/av/stream/flv" "bitbucket.org/ausocean/av/stream/lex" @@ -120,10 +119,11 @@ type Revid struct { // bitrate hold the last send bitrate calculation result. bitrate int - // isRunning is a loaded and cocked foot-gun. mu sync.Mutex isRunning bool + wg sync.WaitGroup + err chan error } @@ -334,7 +334,7 @@ func (r *Revid) Config() Config { return ret } -// setIsRunning sets revid.isRunning using b. +// setIsRunning sets r.isRunning using b. func (r *Revid) setIsRunning(b bool) { r.mu.Lock() r.isRunning = b @@ -348,9 +348,11 @@ func (r *Revid) Start() error { return errors.New(pkg + "start called but revid is already running") } r.config.Logger.Log(logger.Info, pkg+"starting Revid") + // TODO: this doesn't need to be here r.config.Logger.Log(logger.Debug, pkg+"setting up output") r.setIsRunning(true) r.config.Logger.Log(logger.Info, pkg+"starting output routine") + r.wg.Add(1) go r.outputClips() r.config.Logger.Log(logger.Info, pkg+"setting up input and receiving content") err := r.setupInput() @@ -371,6 +373,7 @@ func (r *Revid) Stop() error { if r.cmd != nil && r.cmd.Process != nil { r.cmd.Process.Kill() } + r.wg.Wait() return nil } @@ -388,89 +391,89 @@ func (r *Revid) Update(vars map[string]string) error { // Maybe we shouldn't be doing this! switch value { case "File": - cfg.Outputs[0] = revid.File + r.config.Outputs[0] = File case "Http": - cfg.Outputs[0] = revid.Http + r.config.Outputs[0] = Http case "Rtmp": - cfg.Outputs[0] = revid.Rtmp + r.config.Outputs[0] = Rtmp case "FfmpegRtmp": - cfg.Outputs[0] = revid.FfmpegRtmp + r.config.Outputs[0] = FfmpegRtmp default: - log.Log(logger.Warning, pkg+"invalid Output1 param", "value", value) + r.config.Logger.Log(logger.Warning, pkg+"invalid Output1 param", "value", value) continue } case "FramesPerClip": f, err := strconv.ParseUint(value, 10, 0) if err != nil { - log.Log(logger.Warning, pkg+"invalid framesperclip param", "value", value) + .Log(logger.Warning, pkg+"invalid framesperclip param", "value", value) break } - cfg.FramesPerClip = uint(f) + r.config.FramesPerClip = uint(f) case "RtmpUrl": - cfg.RtmpUrl = value + r.config.RtmpUrl = value case "Bitrate": r, err := strconv.ParseUint(value, 10, 0) if err != nil { log.Log(logger.Warning, pkg+"invalid framerate param", "value", value) break } - cfg.Bitrate = uint(r) + r.config.Bitrate = uint(r) case "OutputFileName": - cfg.OutputFileName = value + r.config.OutputFileName = value case "InputFileName": - cfg.InputFileName = value + r.config.InputFileName = value case "Height": h, err := strconv.ParseUint(value, 10, 0) if err != nil { log.Log(logger.Warning, pkg+"invalid height param", "value", value) break } - cfg.Height = uint(h) + r.config.Height = uint(h) case "Width": w, err := strconv.ParseUint(value, 10, 0) if err != nil { log.Log(logger.Warning, pkg+"invalid width param", "value", value) break } - cfg.Width = uint(w) + r.config.Width = uint(w) case "FrameRate": r, err := strconv.ParseUint(value, 10, 0) if err != nil { log.Log(logger.Warning, pkg+"invalid framerate param", "value", value) break } - cfg.FrameRate = uint(r) + r.config.FrameRate = uint(r) case "HttpAddress": - cfg.HttpAddress = value + r.config.HttpAddress = value case "Quantization": q, err := strconv.ParseUint(value, 10, 0) if err != nil { log.Log(logger.Warning, pkg+"invalid quantization param", "value", value) break } - cfg.Quantization = uint(q) + r.config.Quantization = uint(q) case "IntraRefreshPeriod": p, err := strconv.ParseUint(value, 10, 0) if err != nil { log.Log(logger.Warning, pkg+"invalid intrarefreshperiod param", "value", value) break } - cfg.IntraRefreshPeriod = uint(p) + r.config.IntraRefreshPeriod = uint(p) case "HorizontalFlip": switch strings.ToLower(value) { case "true": - cfg.FlipHorizontal = true + r.config.FlipHorizontal = true case "false": - cfg.FlipHorizontal = false + r.config.FlipHorizontal = false default: log.Log(logger.Warning, pkg+"invalid HorizontalFlip param", "value", value) } case "VerticalFlip": switch strings.ToLower(value) { case "true": - cfg.FlipVertical = true + r.config.FlipVertical = true case "false": - cfg.FlipVertical = false + r.config.FlipVertical = false default: log.Log(logger.Warning, pkg+"invalid VerticalFlip param", "value", value) } @@ -478,7 +481,7 @@ func (r *Revid) Update(vars map[string]string) error { } } - return startRevid(ns, cfg) + return r.Start() } // outputClips takes the clips produced in the packClips method and outputs them @@ -622,6 +625,7 @@ func (r *Revid) startRaspivid() error { r.config.Logger.Log(logger.Fatal, pkg+"cannot start raspivid", "error", err.Error()) } + r.wg.Add(1) go r.processFrom(stdout, 0) return nil } @@ -670,6 +674,7 @@ func (r *Revid) startV4L() error { return err } + r.wg.Add(1) go r.processFrom(stdout, time.Duration(0)) return nil } @@ -685,6 +690,7 @@ func (r *Revid) setupInputForFile() error { defer f.Close() // TODO(kortschak): Maybe we want a context.Context-aware parser that we can stop. + r.wg.Add(1) go r.processFrom(f, time.Second/time.Duration(r.config.FrameRate)) return nil } @@ -693,4 +699,5 @@ func (r *Revid) processFrom(read io.Reader, delay time.Duration) { r.config.Logger.Log(logger.Info, pkg+"reading input data") r.err <- r.lexTo(r.encoder, read, delay) r.config.Logger.Log(logger.Info, pkg+"finished reading input data") + r.wg.Done() } From 1010721dd070882058660fff25f3e84db4ad446d Mon Sep 17 00:00:00 2001 From: saxon Date: Sun, 3 Feb 2019 23:37:38 +1030 Subject: [PATCH 076/208] cmd/revid-cli & revid: Checking revid mode differently - now using ns.Mode(), which should soon be an available feature. Also now using ns.SetMode() - which tells netreceiver that we've changed mode. --- cmd/revid-cli/main.go | 157 +++++++++--------------------------------- revid/revid.go | 26 +++---- 2 files changed, 45 insertions(+), 138 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 826cb2a4..58f8e507 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -50,6 +50,13 @@ const ( defaultLogVerbosity = logger.Debug ) +// Revid modes +const ( + normal = "Normal" + paused = "Paused" + burst = "Burst" +) + // Other misc consts const ( netSendRetryTime = 5 * time.Second @@ -244,9 +251,9 @@ func handleFlags() revid.Config { // initialize then run the main NetSender client func run(cfg revid.Config) error { - // initialize NetSender and use NetSender's logger log.Log(logger.Info, pkg+"running in NetSender mode") + // initialize NetSender and use NetSender's logger var ns netsender.Sender if err := ns.Init(log, nil, nil, nil); err != nil { return err @@ -254,8 +261,12 @@ func run(cfg revid.Config) error { vars, _ := ns.Vars() vs := ns.VarSum() - rv, cfg, err = updateRevid(&ns, rv, cfg, vars, false) + rv, err := revid.New(cfg, &ns) if err != nil { + log.Log(logger.Fatal, pkg+"could not initialise revid", "error", err.Error()) + } + + if err = rv.Update(vars); err != nil { return err } @@ -267,7 +278,6 @@ func run(cfg revid.Config) error { } if vs != ns.VarSum() { - // vars changed vars, err := ns.Vars() if err != nil { log.Log(logger.Error, pkg+"netSender failed to get vars", "error", err.Error()) @@ -275,24 +285,32 @@ func run(cfg revid.Config) error { continue } vs = ns.VarSum() - if vars["mode"] == "Paused" { - if !paused { + + switch ns.Mode() { + case paused: + if rv.IsRunning() { log.Log(logger.Info, pkg+"pausing revid") - err = stopRevid(rv) - if err != nil { - log.Log(logger.Error, pkg+"failed to stop revide", "error", err.Error()) + if err = rv.Stop(); err != nil { + log.Log(logger.Error, pkg+"failed to stop revid", "error", err.Error()) continue } - paused = true + ns.SetMode(paused) } - } else { - rv, cfg, err = updateRevid(&ns, rv, cfg, vars, !paused) - if err != nil { + case normal: + if err = rv.Update(vars); err != nil { return err } - if paused { - paused = false + ns.SetMode(normal) + case burst: + if err = rv.Start(); err != nil { + return err } + ns.SetMode(burst) + time.Sleep(rv.Config().BurstPeriod) + if err = rv.Stop(); err != nil { + return err + } + ns.SetMode(paused) } } sleepTime, _ := strconv.Atoi(ns.Param("mp")) @@ -323,117 +341,6 @@ func send(ns *netsender.Sender, rv *revid.Revid) error { return nil } -func updateRevid(ns *netsender.Sender, rv *revid.Revid, cfg revid.Config, vars map[string]string, stop bool) (*revid.Revid, revid.Config, error) { - if stop { - err := stopRevid(rv) - if err != nil { - return nil, cfg, err - } - } - - //look through the vars and update revid where needed - for key, value := range vars { - switch key { - case "Output": - // FIXME(kortschak): There can be only one! - // How do we specify outputs after the first? - // - // Maybe we shouldn't be doing this! - switch value { - case "File": - cfg.Outputs[0] = revid.File - case "Http": - cfg.Outputs[0] = revid.Http - case "Rtmp": - cfg.Outputs[0] = revid.Rtmp - case "FfmpegRtmp": - cfg.Outputs[0] = revid.FfmpegRtmp - default: - log.Log(logger.Warning, pkg+"invalid Output1 param", "value", value) - continue - } - case "FramesPerClip": - f, err := strconv.ParseUint(value, 10, 0) - if err != nil { - log.Log(logger.Warning, pkg+"invalid framesperclip param", "value", value) - break - } - cfg.FramesPerClip = uint(f) - case "RtmpUrl": - cfg.RtmpUrl = value - case "Bitrate": - r, err := strconv.ParseUint(value, 10, 0) - if err != nil { - log.Log(logger.Warning, pkg+"invalid framerate param", "value", value) - break - } - cfg.Bitrate = uint(r) - case "OutputFileName": - cfg.OutputFileName = value - case "InputFileName": - cfg.InputFileName = value - case "Height": - h, err := strconv.ParseUint(value, 10, 0) - if err != nil { - log.Log(logger.Warning, pkg+"invalid height param", "value", value) - break - } - cfg.Height = uint(h) - case "Width": - w, err := strconv.ParseUint(value, 10, 0) - if err != nil { - log.Log(logger.Warning, pkg+"invalid width param", "value", value) - break - } - cfg.Width = uint(w) - case "FrameRate": - r, err := strconv.ParseUint(value, 10, 0) - if err != nil { - log.Log(logger.Warning, pkg+"invalid framerate param", "value", value) - break - } - cfg.FrameRate = uint(r) - case "HttpAddress": - cfg.HttpAddress = value - case "Quantization": - q, err := strconv.ParseUint(value, 10, 0) - if err != nil { - log.Log(logger.Warning, pkg+"invalid quantization param", "value", value) - break - } - cfg.Quantization = uint(q) - case "IntraRefreshPeriod": - p, err := strconv.ParseUint(value, 10, 0) - if err != nil { - log.Log(logger.Warning, pkg+"invalid intrarefreshperiod param", "value", value) - break - } - cfg.IntraRefreshPeriod = uint(p) - case "HorizontalFlip": - switch strings.ToLower(value) { - case "true": - cfg.FlipHorizontal = true - case "false": - cfg.FlipHorizontal = false - default: - log.Log(logger.Warning, pkg+"invalid HorizontalFlip param", "value", value) - } - case "VerticalFlip": - switch strings.ToLower(value) { - case "true": - cfg.FlipVertical = true - case "false": - cfg.FlipVertical = false - default: - log.Log(logger.Warning, pkg+"invalid VerticalFlip param", "value", value) - } - default: - } - } - - return startRevid(ns, cfg) -} - // flagStrings implements an appending string set flag. type flagStrings []string diff --git a/revid/revid.go b/revid/revid.go index 33d847ea..c90ba8da 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -405,19 +405,19 @@ func (r *Revid) Update(vars map[string]string) error { case "FramesPerClip": f, err := strconv.ParseUint(value, 10, 0) if err != nil { - .Log(logger.Warning, pkg+"invalid framesperclip param", "value", value) + r.config.Logger.Log(logger.Warning, pkg+"invalid framesperclip param", "value", value) break } r.config.FramesPerClip = uint(f) case "RtmpUrl": r.config.RtmpUrl = value case "Bitrate": - r, err := strconv.ParseUint(value, 10, 0) + v, err := strconv.ParseUint(value, 10, 0) if err != nil { - log.Log(logger.Warning, pkg+"invalid framerate param", "value", value) + r.config.Logger.Log(logger.Warning, pkg+"invalid framerate param", "value", value) break } - r.config.Bitrate = uint(r) + r.config.Bitrate = uint(v) case "OutputFileName": r.config.OutputFileName = value case "InputFileName": @@ -425,37 +425,37 @@ func (r *Revid) Update(vars map[string]string) error { case "Height": h, err := strconv.ParseUint(value, 10, 0) if err != nil { - log.Log(logger.Warning, pkg+"invalid height param", "value", value) + r.config.Logger.Log(logger.Warning, pkg+"invalid height param", "value", value) break } r.config.Height = uint(h) case "Width": w, err := strconv.ParseUint(value, 10, 0) if err != nil { - log.Log(logger.Warning, pkg+"invalid width param", "value", value) + r.config.Logger.Log(logger.Warning, pkg+"invalid width param", "value", value) break } r.config.Width = uint(w) case "FrameRate": - r, err := strconv.ParseUint(value, 10, 0) + v, err := strconv.ParseUint(value, 10, 0) if err != nil { - log.Log(logger.Warning, pkg+"invalid framerate param", "value", value) + r.config.Logger.Log(logger.Warning, pkg+"invalid framerate param", "value", value) break } - r.config.FrameRate = uint(r) + r.config.FrameRate = uint(v) case "HttpAddress": r.config.HttpAddress = value case "Quantization": q, err := strconv.ParseUint(value, 10, 0) if err != nil { - log.Log(logger.Warning, pkg+"invalid quantization param", "value", value) + r.config.Logger.Log(logger.Warning, pkg+"invalid quantization param", "value", value) break } r.config.Quantization = uint(q) case "IntraRefreshPeriod": p, err := strconv.ParseUint(value, 10, 0) if err != nil { - log.Log(logger.Warning, pkg+"invalid intrarefreshperiod param", "value", value) + r.config.Logger.Log(logger.Warning, pkg+"invalid intrarefreshperiod param", "value", value) break } r.config.IntraRefreshPeriod = uint(p) @@ -466,7 +466,7 @@ func (r *Revid) Update(vars map[string]string) error { case "false": r.config.FlipHorizontal = false default: - log.Log(logger.Warning, pkg+"invalid HorizontalFlip param", "value", value) + r.config.Logger.Log(logger.Warning, pkg+"invalid HorizontalFlip param", "value", value) } case "VerticalFlip": switch strings.ToLower(value) { @@ -475,7 +475,7 @@ func (r *Revid) Update(vars map[string]string) error { case "false": r.config.FlipVertical = false default: - log.Log(logger.Warning, pkg+"invalid VerticalFlip param", "value", value) + r.config.Logger.Log(logger.Warning, pkg+"invalid VerticalFlip param", "value", value) } default: } From 6171c4e9994a87722733249a11e0d607e8e788b5 Mon Sep 17 00:00:00 2001 From: saxon Date: Sun, 3 Feb 2019 23:43:51 +1030 Subject: [PATCH 077/208] revid: added handling of burstPeriod to config --- revid/config.go | 7 +++++++ revid/revid.go | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/revid/config.go b/revid/config.go index dc9c5a8d..b0ba2bc4 100644 --- a/revid/config.go +++ b/revid/config.go @@ -68,6 +68,7 @@ type Config struct { RtpAddress string Logger Logger SendRetry bool + BurstPeriod uint } // Enums for config struct @@ -114,6 +115,7 @@ const ( defaultInputCodec = H264 defaultVerbosity = No // FIXME(kortschak): This makes no sense whatsoever. No is currently 15. defaultRtpAddr = "localhost:6970" + defaultBurstPeriod = 10 // Seconds ) // Validate checks for any errors in the config fields and defaults settings @@ -200,6 +202,11 @@ func (c *Config) Validate(r *Revid) error { } } + if c.BurstPeriod == 0 { + c.Logger.Log(logger.Warning, pkg+"no burst period defined, defaulting", "burstPeriod", defaultBurstPeriod) + c.BurstPeriod = defaultBurstPeriod + } + if c.FramesPerClip < 1 { c.Logger.Log(logger.Warning, pkg+"no FramesPerClip defined, defaulting", "framesPerClip", defaultFramesPerClip) diff --git a/revid/revid.go b/revid/revid.go index c90ba8da..9f1cc12c 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -477,6 +477,13 @@ func (r *Revid) Update(vars map[string]string) error { default: r.config.Logger.Log(logger.Warning, pkg+"invalid VerticalFlip param", "value", value) } + case "BurstPeriod": + v, err := strconv.ParseUint(value, 10, 0) + if err != nil { + r.config.Logger.Log(logger.Warning, pkg+"invalid BurstPeriod param", "value", value) + break + } + r.config.BurstPeriod = uint(v) default: } } From ee7eb84d26e29b3d9cab7005fddd48d1847a8211 Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 4 Feb 2019 13:25:37 +1030 Subject: [PATCH 078/208] revid-cli: correctly using ns.Mode() and ns.SetMode() --- cmd/revid-cli/main.go | 75 +++++++++++++++++++++++-------------------- revid/revid.go | 2 +- 2 files changed, 41 insertions(+), 36 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 58f8e507..ac8fff12 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -266,52 +266,57 @@ func run(cfg revid.Config) error { log.Log(logger.Fatal, pkg+"could not initialise revid", "error", err.Error()) } + // Update revid to get latest config settings from netreceiver. if err = rv.Update(vars); err != nil { return err } + // If mode on netreceiver isn't paused then we can start revid. + if ns.Mode() != paused { + if err = rv.Start(); err != nil { + return err + } + } + for { - if err := send(&ns, rv); err != nil { - log.Log(logger.Error, pkg+"polling failed", "error", err.Error()) + if err := ns.Run(); err != nil { + log.Log(logger.Error, pkg+"Run Failed. Retrying...") time.Sleep(netSendRetryTime) continue } - if vs != ns.VarSum() { - vars, err := ns.Vars() - if err != nil { - log.Log(logger.Error, pkg+"netSender failed to get vars", "error", err.Error()) - time.Sleep(netSendRetryTime) - continue - } - vs = ns.VarSum() + // If var sum hasn't change we continue + if vs == ns.VarSum() { + continue + } - switch ns.Mode() { - case paused: - if rv.IsRunning() { - log.Log(logger.Info, pkg+"pausing revid") - if err = rv.Stop(); err != nil { - log.Log(logger.Error, pkg+"failed to stop revid", "error", err.Error()) - continue - } - ns.SetMode(paused) - } - case normal: - if err = rv.Update(vars); err != nil { - return err - } - ns.SetMode(normal) - case burst: - if err = rv.Start(); err != nil { - return err - } - ns.SetMode(burst) - time.Sleep(rv.Config().BurstPeriod) - if err = rv.Stop(); err != nil { - return err - } - ns.SetMode(paused) + vars, err := ns.Vars() + if err != nil { + log.Log(logger.Error, pkg+"netSender failed to get vars", "error", err.Error()) + time.Sleep(netSendRetryTime) + continue + } + vs = ns.VarSum() + + if err = rv.Update(vars); err != nil { + return err + } + + switch ns.Mode() { + case paused: + case normal: + if err = rv.Start(); err != nil { + return err } + case burst: + if err = rv.Start(); err != nil { + return err + } + time.Sleep(time.Duration(rv.Config().BurstPeriod)) + if err = rv.Stop(); err != nil { + return err + } + ns.SetMode(paused, &vs) } sleepTime, _ := strconv.Atoi(ns.Param("mp")) time.Sleep(time.Duration(sleepTime) * time.Second) diff --git a/revid/revid.go b/revid/revid.go index 9f1cc12c..e4c7dbbe 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -488,7 +488,7 @@ func (r *Revid) Update(vars map[string]string) error { } } - return r.Start() + return nil } // outputClips takes the clips produced in the packClips method and outputs them From 93e3899725b8e6029c1cc80265f0ba1405eb4b7e Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 4 Feb 2019 17:04:49 +1030 Subject: [PATCH 079/208] cmd/revid-cli: using ns.Send() rather than ns.Run() to poll --- cmd/revid-cli/main.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index ac8fff12..bcabf40d 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -272,15 +272,19 @@ func run(cfg revid.Config) error { } // If mode on netreceiver isn't paused then we can start revid. - if ns.Mode() != paused { + if ns.Mode() != paused && ns.Mode() != burst { if err = rv.Start(); err != nil { return err } } + if ns.Mode() == burst { + ns.SetMode(normal, &vs) + } + for { - if err := ns.Run(); err != nil { - log.Log(logger.Error, pkg+"Run Failed. Retrying...") + if err := send(&ns, rv); err != nil { + log.Log(logger.Error, pkg+"Run Failed. Retrying...", "error", err.Error()) time.Sleep(netSendRetryTime) continue } @@ -309,10 +313,12 @@ func run(cfg revid.Config) error { return err } case burst: + log.Log(logger.Info, pkg+"Starting burst...") if err = rv.Start(); err != nil { return err } - time.Sleep(time.Duration(rv.Config().BurstPeriod)) + time.Sleep(time.Duration(rv.Config().BurstPeriod) * time.Second) + log.Log(logger.Info, pkg+"Stopping burst...") if err = rv.Stop(); err != nil { return err } From 8978f9edc59d050a1ee590edfd58f59f6cf2ace5 Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 4 Feb 2019 17:12:30 +1030 Subject: [PATCH 080/208] cmd/revid-cli & revid: using goto to sleep for monitor period, and using wg.Done() at the end of output routine. --- cmd/revid-cli/main.go | 10 +++++++--- revid/revid.go | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index bcabf40d..1eafaae9 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -253,12 +253,15 @@ func handleFlags() revid.Config { func run(cfg revid.Config) error { log.Log(logger.Info, pkg+"running in NetSender mode") + var err error + var vars map[string]string + // initialize NetSender and use NetSender's logger var ns netsender.Sender if err := ns.Init(log, nil, nil, nil); err != nil { return err } - vars, _ := ns.Vars() + vars, _ = ns.Vars() vs := ns.VarSum() rv, err := revid.New(cfg, &ns) @@ -291,10 +294,10 @@ func run(cfg revid.Config) error { // If var sum hasn't change we continue if vs == ns.VarSum() { - continue + goto sleep } - vars, err := ns.Vars() + vars, err = ns.Vars() if err != nil { log.Log(logger.Error, pkg+"netSender failed to get vars", "error", err.Error()) time.Sleep(netSendRetryTime) @@ -324,6 +327,7 @@ func run(cfg revid.Config) error { } ns.SetMode(paused, &vs) } + sleep: sleepTime, _ := strconv.Atoi(ns.Param("mp")) time.Sleep(time.Duration(sleepTime) * time.Second) } diff --git a/revid/revid.go b/revid/revid.go index e4c7dbbe..d0f3abf6 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -582,6 +582,7 @@ loop: r.config.Logger.Log(logger.Error, pkg+"failed to close output"+strconv.Itoa(i)+" destination", "error", err.Error()) } } + r.wg.Done() } // startRaspivid sets up things for input from raspivid i.e. starts From 1cdbfa2c66d2ccae05cf86ed9e81e8d6a4bae9c1 Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 4 Feb 2019 17:14:19 +1030 Subject: [PATCH 081/208] cmd/revid-cli: setting mode to paused if ns is in burst mode. --- cmd/revid-cli/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 1eafaae9..b33ef124 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -282,7 +282,7 @@ func run(cfg revid.Config) error { } if ns.Mode() == burst { - ns.SetMode(normal, &vs) + ns.SetMode(paused, &vs) } for { From bd2958ba4e0ccc62fe8c5af4d97cf7951bfcaa59 Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 4 Feb 2019 19:14:02 +1030 Subject: [PATCH 082/208] cmd/revid-cli & revid: added TODO for the use of Run() instead of send in cmd/revid-cli/main.go. Fixed filename in revid/revid.go file header. Renamed ret to cfg in revid.Config(). Catching error from call to revid.Stop() in revid.Udate() --- cmd/revid-cli/main.go | 1 + revid/revid.go | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index b33ef124..29191d3a 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -286,6 +286,7 @@ func run(cfg revid.Config) error { } for { + // TODO(saxon): replace this call with call to ns.Run(). if err := send(&ns, rv); err != nil { log.Log(logger.Error, pkg+"Run Failed. Retrying...", "error", err.Error()) time.Sleep(netSendRetryTime) diff --git a/revid/revid.go b/revid/revid.go index d0f3abf6..a3c10b66 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -1,6 +1,6 @@ /* NAME - r.go + revid.go DESCRIPTION See Readme.md @@ -329,9 +329,9 @@ func (r *Revid) IsRunning() bool { func (r *Revid) Config() Config { r.mu.Lock() - ret := r.config + cfg := r.config r.mu.Unlock() - return ret + return cfg } // setIsRunning sets r.isRunning using b. @@ -379,7 +379,9 @@ func (r *Revid) Stop() error { func (r *Revid) Update(vars map[string]string) error { if r.IsRunning() { - r.Stop() + if err := r.Stop(); err != nil { + return err + } } //look through the vars and update revid where needed for key, value := range vars { From 1570974ab7cfa84716be3879d75726cf3dec62ba Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 4 Feb 2019 22:14:17 +1030 Subject: [PATCH 083/208] stream/mts/encoder.go: removed init func where Meta was being initialised - just doing initialisation on same line --- stream/mts/encoder.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index cc4a699a..6b406536 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -89,11 +89,7 @@ const ( ) // global Meta -var Meta *meta.Metadata - -func init() { - Meta = meta.New() -} +var Meta = meta.New() var ( patTable = standardPat.Bytes() From 0a96d18a10a7c65b0b171f28c9e1fd07809a8b30 Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 4 Feb 2019 22:17:39 +1030 Subject: [PATCH 084/208] stream/mts/encoder.go: finished comment for updateMeta function --- stream/mts/encoder.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 6b406536..093dbee5 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -251,7 +251,8 @@ func (e *Encoder) ccFor(pid int) byte { return cc } -// updateMeta ... +// updateMeta adds/updates a metaData descriptor in the given psi bytes using data +// contained in the global Meta struct. func updateMeta(b *[]byte) error { p := psi.PSIBytes(*b) err := p.AddDescriptor(psi.MetadataTag, Meta.Encode()) From 953d363b3a50a1ee9d5c98da0d50e338b66a11c5 Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 4 Feb 2019 22:38:11 +1030 Subject: [PATCH 085/208] stream/mts/meta.go: improved const and function commenting --- stream/mts/meta/meta.go | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go index 2f26a797..1e4c72b8 100644 --- a/stream/mts/meta/meta.go +++ b/stream/mts/meta/meta.go @@ -33,6 +33,8 @@ import ( "sync" ) +// This is the headsize of our metadata string, +// which is encoded int the data body of a pmt descriptor. const headSize = 4 const ( @@ -40,6 +42,7 @@ const ( minVer = 0 ) +// Indices of bytes for uint16 metadata length. const ( dataLenIdx1 = 2 dataLenIdx2 = 3 @@ -49,12 +52,15 @@ var ( errKeyAbsent = errors.New("Key does not exist in map") ) +// Metadata provides functionality for the storage and encoding of metadata +// using a map. type Metadata struct { mu sync.RWMutex data map[string]string enc []byte } +// New returns a pointer to a new Metadata. func New() *Metadata { return &Metadata{ data: make(map[string]string), @@ -67,14 +73,14 @@ func New() *Metadata { } } -// Add adds metadata with key and val +// Add adds metadata with key and val. func (m *Metadata) Add(key, val string) { m.mu.Lock() m.data[key] = val m.mu.Unlock() } -// All returns the a copy of the map containing the meta data +// All returns the a copy of the map containing the meta data. func (m *Metadata) All() map[string]string { m.mu.Lock() cpy := make(map[string]string) @@ -85,7 +91,7 @@ func (m *Metadata) All() map[string]string { return cpy } -// Get returns the meta data for the passed key +// Get returns the meta data for the passed key. func (m *Metadata) Get(key string) (string, error) { m.mu.Lock() val, ok := m.data[key] @@ -93,11 +99,10 @@ func (m *Metadata) Get(key string) (string, error) { if !ok { return "", errKeyAbsent } - return val, nil } -// Remove deletes a meta entry in the map and returns error if it doesn’t exist +// Delete deletes a meta entry in the map and returns error if it doesn’t exist. func (m *Metadata) Delete(key string) error { m.mu.Lock() defer m.mu.Unlock() @@ -108,12 +113,13 @@ func (m *Metadata) Delete(key string) error { return errKeyAbsent } -// Encode takes the meta data map and encods into a byte slice with header +// Encode takes the meta data map and encodes into a byte slice with header // describing the version, length of data and data in TSV format. func (m *Metadata) Encode() []byte { m.enc = m.enc[:headSize] - // Iterate over map and append entries, only adding tab if we're not on the last entry + // Iterate over map and append entries, only adding tab if we're not on the + // last entry. var i int var entry string for k, v := range m.data { @@ -133,6 +139,9 @@ func (m *Metadata) Encode() []byte { return m.enc } +// ReadFrom extracts a value from a metadata string d, for the given key. If the +// key is not present in the metadata string, an error is returned. If the +// metadata header is not present in the string, an error is returned. func ReadFrom(d []byte, key string) (string, error) { entries := strings.Split(string(d), "\t") for _, entry := range entries { @@ -141,5 +150,5 @@ func ReadFrom(d []byte, key string) (string, error) { return kv[1], nil } } - return "", errors.New("could not find key in metadata") + return "", errKeyAbsent } From ad25785b77f6b06a83bbe34ad8d8fa60b3c9aaf5 Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 4 Feb 2019 22:48:51 +1030 Subject: [PATCH 086/208] stream/mts/meta/meta_test.go: improved ReadFrom by checking for valid header --- stream/mts/meta/meta.go | 11 ++++++++++- stream/mts/meta/meta_test.go | 6 +++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go index 1e4c72b8..7a863d0c 100644 --- a/stream/mts/meta/meta.go +++ b/stream/mts/meta/meta.go @@ -28,6 +28,7 @@ LICENSE package meta import ( + "encoding/binary" "errors" "strings" "sync" @@ -49,7 +50,9 @@ const ( ) var ( - errKeyAbsent = errors.New("Key does not exist in map") + errKeyAbsent = errors.New("Key does not exist in map") + errNoHeader = errors.New("Metadata string does not contain header") + errInvalidHeader = errors.New("Metadata string does not contain valid header") ) // Metadata provides functionality for the storage and encoding of metadata @@ -143,6 +146,12 @@ func (m *Metadata) Encode() []byte { // key is not present in the metadata string, an error is returned. If the // metadata header is not present in the string, an error is returned. func ReadFrom(d []byte, key string) (string, error) { + if d[0] != 0 { + return "", errNoHeader + } else if d[0] == 0 && binary.BigEndian.Uint16(d[2:headSize]) != uint16(len(d[headSize:])) { + return "", errInvalidHeader + } + d = d[headSize:] entries := strings.Split(string(d), "\t") for _, entry := range entries { kv := strings.Split(entry, "=") diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go index 0fcda13b..15789ebf 100644 --- a/stream/mts/meta/meta_test.go +++ b/stream/mts/meta/meta_test.go @@ -156,8 +156,8 @@ func TestEncode(t *testing.T) { } func TestReadFrom(t *testing.T) { - tstStr := "loc=a,b,c\tts=12345" - got, err := ReadFrom([]byte(tstStr), "loc") + tstMeta := append([]byte{0x00, 0x10, 0x00, 0x12}, "loc=a,b,c\tts=12345"...) + got, err := ReadFrom([]byte(tstMeta), "loc") if err != nil { t.Errorf(errUnexpectedErr, err.Error()) } @@ -166,7 +166,7 @@ func TestReadFrom(t *testing.T) { t.Errorf(errNotExpectedOut, got, want) } - if got, err = ReadFrom([]byte(tstStr), "ts"); err != nil { + if got, err = ReadFrom([]byte(tstMeta), "ts"); err != nil { t.Errorf(errUnexpectedErr, err.Error()) } want = "12345" From 35344402b848ec34a81260002e0394a7452fd225 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 5 Feb 2019 10:32:16 +1030 Subject: [PATCH 087/208] cmd/revid-cli/main.go: not using closed scope conditions anymore --- cmd/revid-cli/main.go | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 29191d3a..52bb283b 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -253,14 +253,15 @@ func handleFlags() revid.Config { func run(cfg revid.Config) error { log.Log(logger.Info, pkg+"running in NetSender mode") - var err error var vars map[string]string // initialize NetSender and use NetSender's logger var ns netsender.Sender - if err := ns.Init(log, nil, nil, nil); err != nil { + err := ns.Init(log, nil, nil, nil) + if err != nil { return err } + vars, _ = ns.Vars() vs := ns.VarSum() @@ -270,7 +271,8 @@ func run(cfg revid.Config) error { } // Update revid to get latest config settings from netreceiver. - if err = rv.Update(vars); err != nil { + err = rv.Update(vars) + if err != nil { return err } @@ -287,7 +289,8 @@ func run(cfg revid.Config) error { for { // TODO(saxon): replace this call with call to ns.Run(). - if err := send(&ns, rv); err != nil { + err = send(&ns, rv) + if err != nil { log.Log(logger.Error, pkg+"Run Failed. Retrying...", "error", err.Error()) time.Sleep(netSendRetryTime) continue @@ -306,24 +309,28 @@ func run(cfg revid.Config) error { } vs = ns.VarSum() - if err = rv.Update(vars); err != nil { + err = rv.Update(vars) + if err != nil { return err } switch ns.Mode() { case paused: case normal: - if err = rv.Start(); err != nil { + err = rv.Start() + if err != nil { return err } case burst: log.Log(logger.Info, pkg+"Starting burst...") - if err = rv.Start(); err != nil { + err = rv.Start() + if err != nil { return err } time.Sleep(time.Duration(rv.Config().BurstPeriod) * time.Second) log.Log(logger.Info, pkg+"Stopping burst...") - if err = rv.Stop(); err != nil { + err = rv.Stop() + if err != nil { return err } ns.SetMode(paused, &vs) From 4dcbd904499c4b51516cba0c90f3eebeef94df53 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 5 Feb 2019 10:40:08 +1030 Subject: [PATCH 088/208] cmd/revid-cli: removed another closed scope condition --- cmd/revid-cli/main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 52bb283b..f37144a2 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -278,7 +278,8 @@ func run(cfg revid.Config) error { // If mode on netreceiver isn't paused then we can start revid. if ns.Mode() != paused && ns.Mode() != burst { - if err = rv.Start(); err != nil { + err = rv.Start() + if err != nil { return err } } From ea8572a777a90b90bb3e19564402426337b73de5 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 5 Feb 2019 10:45:15 +1030 Subject: [PATCH 089/208] cmd/revid-cli: catching error in conversion of mp --- cmd/revid-cli/main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index f37144a2..1c2b9c90 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -337,7 +337,10 @@ func run(cfg revid.Config) error { ns.SetMode(paused, &vs) } sleep: - sleepTime, _ := strconv.Atoi(ns.Param("mp")) + sleepTime, err := strconv.Atoi(ns.Param("mp")) + if err != nil { + return err + } time.Sleep(time.Duration(sleepTime) * time.Second) } } From a4d179039b4086facc14cb5eb6b719d09bc0873c Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 5 Feb 2019 10:49:05 +1030 Subject: [PATCH 090/208] revid/revid.go: removed default case in switch with revid.Update() --- revid/revid.go | 1 - 1 file changed, 1 deletion(-) diff --git a/revid/revid.go b/revid/revid.go index a3c10b66..8c4b57ed 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -486,7 +486,6 @@ func (r *Revid) Update(vars map[string]string) error { break } r.config.BurstPeriod = uint(v) - default: } } From de4f471201bc370186924b21e955884066e1c1d3 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 5 Feb 2019 10:50:21 +1030 Subject: [PATCH 091/208] revid/revid.go: defer r.wg.Done() in revid.outputClips routine --- revid/revid.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/revid/revid.go b/revid/revid.go index 8c4b57ed..02656d03 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -495,6 +495,7 @@ func (r *Revid) Update(vars map[string]string) error { // outputClips takes the clips produced in the packClips method and outputs them // to the desired output defined in the revid config func (r *Revid) outputClips() { + defer r.wg.Done() lastTime := time.Now() var count int loop: @@ -583,7 +584,6 @@ loop: r.config.Logger.Log(logger.Error, pkg+"failed to close output"+strconv.Itoa(i)+" destination", "error", err.Error()) } } - r.wg.Done() } // startRaspivid sets up things for input from raspivid i.e. starts From 834625a9f75705b19878a9d6b280952712589b89 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 5 Feb 2019 12:55:32 +1030 Subject: [PATCH 092/208] stream/mts/meta: using order slice so that encoded order reamins consistent with order that we added metadata. And also now using binary.BigEndian.PutUint16() to encode len into metdata rather than doing ugly byte manipulation --- stream/mts/meta/meta.go | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go index 7a863d0c..eaa0713e 100644 --- a/stream/mts/meta/meta.go +++ b/stream/mts/meta/meta.go @@ -45,8 +45,7 @@ const ( // Indices of bytes for uint16 metadata length. const ( - dataLenIdx1 = 2 - dataLenIdx2 = 3 + dataLenIdx = 2 ) var ( @@ -58,9 +57,10 @@ var ( // Metadata provides functionality for the storage and encoding of metadata // using a map. type Metadata struct { - mu sync.RWMutex - data map[string]string - enc []byte + mu sync.RWMutex + data map[string]string + order []string + enc []byte } // New returns a pointer to a new Metadata. @@ -80,6 +80,7 @@ func New() *Metadata { func (m *Metadata) Add(key, val string) { m.mu.Lock() m.data[key] = val + m.order = append(m.order, key) m.mu.Unlock() } @@ -111,6 +112,12 @@ func (m *Metadata) Delete(key string) error { defer m.mu.Unlock() if _, ok := m.data[key]; ok { delete(m.data, key) + for i, k := range m.order { + if k == key { + m.order = append(m.order[:i], m.order[i+1:]...) + break + } + } return nil } return errKeyAbsent @@ -123,12 +130,11 @@ func (m *Metadata) Encode() []byte { // Iterate over map and append entries, only adding tab if we're not on the // last entry. - var i int var entry string - for k, v := range m.data { - i++ + for i, k := range m.order { + v := m.data[k] entry += k + "=" + v - if i < len(m.data) { + if i+1 < len(m.data) { entry += "\t" } } @@ -136,9 +142,7 @@ func (m *Metadata) Encode() []byte { // Calculate and set data length in encoded meta header. dataLen := len(m.enc[headSize:]) - m.enc[dataLenIdx1] = byte(dataLen >> 8) - m.enc[dataLenIdx2] = byte(dataLen) - + binary.BigEndian.PutUint16(m.enc[dataLenIdx:dataLenIdx+2], uint16(dataLen)) return m.enc } From 3364b4ea6f1ad5d6d19dd045c4767f645bf61a66 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 5 Feb 2019 13:11:11 +1030 Subject: [PATCH 093/208] stream/mts/meta.go: avoiding adding same key to order slice --- stream/mts/meta/meta.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go index eaa0713e..a0e6e3e3 100644 --- a/stream/mts/meta/meta.go +++ b/stream/mts/meta/meta.go @@ -80,6 +80,12 @@ func New() *Metadata { func (m *Metadata) Add(key, val string) { m.mu.Lock() m.data[key] = val + for _, k := range m.order { + if k == key { + m.mu.Unlock() + return + } + } m.order = append(m.order, key) m.mu.Unlock() } From 6425403fcb8ad5e3de656328edf29564cf0f52ac Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 5 Feb 2019 13:14:42 +1030 Subject: [PATCH 094/208] stream/mts/encoder.go: adding copyright meta first thing --- stream/mts/encoder.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 093dbee5..6a0c55d3 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -88,9 +88,18 @@ const ( psiSndCnt = 7 ) +const ( + copyright = "copyright" + license = "ausocean.org/license/content2019" +) + // global Meta var Meta = meta.New() +func init() { + Meta.Add(copyright, license) +} + var ( patTable = standardPat.Bytes() pmtTable = standardPmt.Bytes() From dc46d9f0b28e088454d79c17dc75bca0fe93cb58 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 5 Feb 2019 23:18:05 +1030 Subject: [PATCH 095/208] stream/mts/encoder.go: not doing scoped conditional in writePSI when calling updateMeta --- stream/mts/encoder.go | 4 ++-- stream/mts/meta/meta_test.go | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 6a0c55d3..0e52cb70 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -216,8 +216,8 @@ func (e *Encoder) writePSI() error { if err != nil { return err } - - if err = updateMeta(&pmtTable); err != nil { + err = updateMeta(&pmtTable) + if err != nil { return err } diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go index 15789ebf..afd38544 100644 --- a/stream/mts/meta/meta_test.go +++ b/stream/mts/meta/meta_test.go @@ -29,9 +29,9 @@ package meta import ( "bytes" - "errors" "reflect" "testing" + "encoding/binary" ) const ( @@ -53,7 +53,6 @@ func TestAddAndGet(t *testing.T) { meta := New() meta.Add(tstKey1, tstData1) meta.Add(tstKey2, tstData2) - errors.New("Trying to delete map entry that doesn't exist") if data, err := meta.Get(tstKey1); err != nil { t.Errorf("Could not get data for key: loc: %v", err.Error()) if data != tstData1 { @@ -139,13 +138,12 @@ func TestEncode(t *testing.T) { meta.Add(tstKey2, tstData2) dataLen := len(tstKey1+tstData1+tstKey2+tstData2) + 3 - expectedOut := []byte{ + header := [4]byte{ 0x00, 0x10, - byte(dataLen >> 8), - byte(dataLen), } - expectedOut = append(expectedOut, []byte( + binary.BigEndian.PutUint16(header[2:4],uint16(dataLen)) + expectedOut := append(header[:], []byte( tstKey1+"="+tstData1+"\t"+ tstKey2+"="+tstData2)...) From 6b32064a2bdab48752ab1007b3b11051bd82f6f6 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 5 Feb 2019 23:24:00 +1030 Subject: [PATCH 096/208] stream/mts/encoder.go: changed updateMeta signature to return bytes slice rather than passing in pointer to byte slice. --- stream/mts/encoder.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 0e52cb70..7bda82c7 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -216,7 +216,7 @@ func (e *Encoder) writePSI() error { if err != nil { return err } - err = updateMeta(&pmtTable) + pmtTable, err = updateMeta(pmtTable) if err != nil { return err } @@ -262,9 +262,8 @@ func (e *Encoder) ccFor(pid int) byte { // updateMeta adds/updates a metaData descriptor in the given psi bytes using data // contained in the global Meta struct. -func updateMeta(b *[]byte) error { - p := psi.PSIBytes(*b) +func updateMeta(b []byte) ([]byte, error) { + p := psi.PSIBytes(b) err := p.AddDescriptor(psi.MetadataTag, Meta.Encode()) - *b = []byte(p) - return err + return []byte(p), err } From cf73c853b0734d201f5e2e573d75146782e078b3 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 5 Feb 2019 23:25:46 +1030 Subject: [PATCH 097/208] stream/mts/meta/meta.go: using defer for m.mu.Unlock() Meta.Add() --- stream/mts/meta/meta.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go index a0e6e3e3..466cb8e8 100644 --- a/stream/mts/meta/meta.go +++ b/stream/mts/meta/meta.go @@ -79,15 +79,15 @@ func New() *Metadata { // Add adds metadata with key and val. func (m *Metadata) Add(key, val string) { m.mu.Lock() + defer m.mu.Unlock() m.data[key] = val for _, k := range m.order { if k == key { - m.mu.Unlock() return } } m.order = append(m.order, key) - m.mu.Unlock() + return } // All returns the a copy of the map containing the meta data. From 89b05b3cb662bfcf6af315a2dba58433ad5789b5 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 5 Feb 2019 23:29:07 +1030 Subject: [PATCH 098/208] stream/mts/meta/meta.go: using copy instead of append to remove key from order slice in Meta.Delete() --- stream/mts/meta/meta.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go index 466cb8e8..bc3a6da6 100644 --- a/stream/mts/meta/meta.go +++ b/stream/mts/meta/meta.go @@ -120,7 +120,8 @@ func (m *Metadata) Delete(key string) error { delete(m.data, key) for i, k := range m.order { if k == key { - m.order = append(m.order[:i], m.order[i+1:]...) + copy(m.order[:i], m.order[i+1:]) + m.order = m.order[:len(m.order)-1] break } } From 35c06c97aa56b3887d4ea70885841e95b49a1b46 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 5 Feb 2019 23:57:53 +1030 Subject: [PATCH 099/208] stream/mts/meta/meta_test.go: not using global vars for errors --- stream/mts/meta/meta_test.go | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go index afd38544..6d98fe0f 100644 --- a/stream/mts/meta/meta_test.go +++ b/stream/mts/meta/meta_test.go @@ -29,9 +29,9 @@ package meta import ( "bytes" + "encoding/binary" "reflect" "testing" - "encoding/binary" ) const ( @@ -42,12 +42,6 @@ const ( tstData3 = "d,e,f" ) -var ( - errNotExpectedOut = "Did not get expected out. \nGot : %v, \nwant: %v\n" - errUnexpectedErr = "Unexpected err: %v\n" - errNotExpectedErr = "Not expected err: %v\n" -) - // TestAddAndGet ensures that we can add metadata and then successfully get it. func TestAddAndGet(t *testing.T) { meta := New() @@ -76,7 +70,7 @@ func TestUpdate(t *testing.T) { meta.Add(tstKey1, tstData3) if data, err := meta.Get(tstKey1); err != nil { - t.Errorf(errUnexpectedErr, err.Error()) + t.Errorf("Unexpected err: %v\n", err.Error()) if data != tstData2 { t.Errorf("Data did not correctly update for key \"loc\"") } @@ -106,7 +100,7 @@ func TestGetAbsentKey(t *testing.T) { meta := New() if _, err := meta.Get(tstKey1); err != errKeyAbsent { - t.Errorf(errNotExpectedErr, errKeyAbsent.Error()) + t.Errorf("Not expected err: %v\n", errKeyAbsent.Error()) } } @@ -115,10 +109,10 @@ func TestDelete(t *testing.T) { meta := New() meta.Add(tstKey1, tstData1) if err := meta.Delete(tstKey1); err != nil { - t.Errorf(errUnexpectedErr, err.Error()) + t.Errorf("Unexpected err: %v\n", err.Error()) } if _, err := meta.Get(tstKey1); err != errKeyAbsent { - t.Errorf(errNotExpectedErr, errKeyAbsent.Error()) + t.Errorf("Not expected err: %v\n", errKeyAbsent.Error()) } } @@ -127,7 +121,7 @@ func TestDelete(t *testing.T) { func TestDeleteAbsentKey(t *testing.T) { meta := New() if err := meta.Delete(tstKey1); err != errKeyAbsent { - t.Errorf(errNotExpectedErr, errKeyAbsent.Error()) + t.Errorf("Not expected err: %v\n", errKeyAbsent.Error()) } } @@ -142,14 +136,14 @@ func TestEncode(t *testing.T) { 0x00, 0x10, } - binary.BigEndian.PutUint16(header[2:4],uint16(dataLen)) + binary.BigEndian.PutUint16(header[2:4], uint16(dataLen)) expectedOut := append(header[:], []byte( tstKey1+"="+tstData1+"\t"+ tstKey2+"="+tstData2)...) got := meta.Encode() if !bytes.Equal(expectedOut, got) { - t.Errorf(errNotExpectedOut, got, expectedOut) + t.Errorf("Did not get expected out. \nGot : %v, \nwant: %v\n", got, expectedOut) } } @@ -157,18 +151,18 @@ func TestReadFrom(t *testing.T) { tstMeta := append([]byte{0x00, 0x10, 0x00, 0x12}, "loc=a,b,c\tts=12345"...) got, err := ReadFrom([]byte(tstMeta), "loc") if err != nil { - t.Errorf(errUnexpectedErr, err.Error()) + t.Errorf("Unexpected err: %v\n", err.Error()) } want := "a,b,c" if got != want { - t.Errorf(errNotExpectedOut, got, want) + t.Errorf("Did not get expected out. \nGot : %v, \nwant: %v\n", got, want) } if got, err = ReadFrom([]byte(tstMeta), "ts"); err != nil { - t.Errorf(errUnexpectedErr, err.Error()) + t.Errorf("Unexpected err: %v\n", err.Error()) } want = "12345" if got != want { - t.Errorf(errNotExpectedOut, got, want) + t.Errorf("Did not get expected out. \nGot : %v, \nwant: %v\n", got, want) } } From 9b5eb558ef3e6c0f02bc2679850ac3d1b0007c95 Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 6 Feb 2019 00:01:17 +1030 Subject: [PATCH 100/208] stream/mts/meta/meta_test.go: Updated func comment for TestReadFrom --- stream/mts/meta/meta_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go index 6d98fe0f..700fda5b 100644 --- a/stream/mts/meta/meta_test.go +++ b/stream/mts/meta/meta_test.go @@ -147,6 +147,8 @@ func TestEncode(t *testing.T) { } } +// TestReadFrom checks that we can correctly obtain a value for a partiular key +// from a string of metadata using the ReadFrom func. func TestReadFrom(t *testing.T) { tstMeta := append([]byte{0x00, 0x10, 0x00, 0x12}, "loc=a,b,c\tts=12345"...) got, err := ReadFrom([]byte(tstMeta), "loc") From d5236776275492df202ff1fec459f088727f3b8c Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 6 Feb 2019 09:59:55 +1030 Subject: [PATCH 101/208] stream/mts/meta/meta.go: renamed Metadata struct to Data --- stream/mts/meta/meta.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go index bc3a6da6..1e2182c0 100644 --- a/stream/mts/meta/meta.go +++ b/stream/mts/meta/meta.go @@ -56,7 +56,7 @@ var ( // Metadata provides functionality for the storage and encoding of metadata // using a map. -type Metadata struct { +type Data struct { mu sync.RWMutex data map[string]string order []string @@ -64,8 +64,8 @@ type Metadata struct { } // New returns a pointer to a new Metadata. -func New() *Metadata { - return &Metadata{ +func New() *Data { + return &Data{ data: make(map[string]string), enc: []byte{ 0x00, // Reserved byte @@ -77,7 +77,7 @@ func New() *Metadata { } // Add adds metadata with key and val. -func (m *Metadata) Add(key, val string) { +func (m *Data) Add(key, val string) { m.mu.Lock() defer m.mu.Unlock() m.data[key] = val @@ -91,7 +91,7 @@ func (m *Metadata) Add(key, val string) { } // All returns the a copy of the map containing the meta data. -func (m *Metadata) All() map[string]string { +func (m *Data) All() map[string]string { m.mu.Lock() cpy := make(map[string]string) for k, v := range m.data { @@ -102,7 +102,7 @@ func (m *Metadata) All() map[string]string { } // Get returns the meta data for the passed key. -func (m *Metadata) Get(key string) (string, error) { +func (m *Data) Get(key string) (string, error) { m.mu.Lock() val, ok := m.data[key] m.mu.Unlock() @@ -113,7 +113,7 @@ func (m *Metadata) Get(key string) (string, error) { } // Delete deletes a meta entry in the map and returns error if it doesn’t exist. -func (m *Metadata) Delete(key string) error { +func (m *Data) Delete(key string) error { m.mu.Lock() defer m.mu.Unlock() if _, ok := m.data[key]; ok { @@ -132,7 +132,7 @@ func (m *Metadata) Delete(key string) error { // Encode takes the meta data map and encodes into a byte slice with header // describing the version, length of data and data in TSV format. -func (m *Metadata) Encode() []byte { +func (m *Data) Encode() []byte { m.enc = m.enc[:headSize] // Iterate over map and append entries, only adding tab if we're not on the From 2b7ab27763e882c2c4d10f02464bb325ee881e7a Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 6 Feb 2019 10:19:12 +1030 Subject: [PATCH 102/208] cmd/revid-cli & stream/mts/meta: added meta.NewWith function that creates a new meta.Data and adds an initial entry. Using meta.NewWith in revid-cli to initialise mts' global meta.Data with some preamble data. --- cmd/revid-cli/main.go | 11 +++++++++++ stream/mts/encoder.go | 11 +---------- stream/mts/meta/meta.go | 14 ++++++++++++++ 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 5b826b91..d7f79721 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -37,6 +37,8 @@ import ( "time" "bitbucket.org/ausocean/av/revid" + "bitbucket.org/ausocean/av/stream/mts" + "bitbucket.org/ausocean/av/stream/mts/meta" "bitbucket.org/ausocean/iot/pi/netsender" "bitbucket.org/ausocean/iot/pi/smartlogger" "bitbucket.org/ausocean/utils/logger" @@ -65,7 +67,16 @@ var canProfile = true // The logger that will be used throughout var log *logger.Logger +var ( + metaPreambleKey = []byte{0x63, 0x6f, 0x70, 0x79, 0x72, 0x69, 0x67, 0x68, 0x74} + metaPreambleData = []byte{0x61, 0x75, 0x73, 0x6f, 0x63, 0x65, 0x61, 0x6e, 0x2e, + 0x6f, 0x72, 0x67, 0x2f, 0x6c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x2f, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x32, 0x30, 0x31, 0x39} +) + func main() { + mts.Meta = meta.NewWith(string(metaPreambleKey), string(metaPreambleData)) + useNetsender := flag.Bool("NetSender", false, "Are we checking vars through netsender?") runDurationPtr := flag.Duration("runDuration", defaultRunDuration, "How long do you want revid to run for?") diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 7bda82c7..2ebe5dc6 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -88,17 +88,8 @@ const ( psiSndCnt = 7 ) -const ( - copyright = "copyright" - license = "ausocean.org/license/content2019" -) - // global Meta -var Meta = meta.New() - -func init() { - Meta.Add(copyright, license) -} +var Meta *meta.Data var ( patTable = standardPat.Bytes() diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go index 1e2182c0..97725252 100644 --- a/stream/mts/meta/meta.go +++ b/stream/mts/meta/meta.go @@ -76,6 +76,20 @@ func New() *Data { } } +func NewWith(key, data string) *Data { + meta := &Data{ + data: make(map[string]string), + enc: []byte{ + 0x00, // Reserved byte + (majVer << 4) | minVer, // MS and LS versions + 0x00, // Data len byte1 + 0x00, // Data len byte2 + }, + } + meta.Add(key, data) + return meta +} + // Add adds metadata with key and val. func (m *Data) Add(key, val string) { m.mu.Lock() From 57d1dba2fb809d8f8874842138b773abdeb7267d Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 6 Feb 2019 12:16:44 +1030 Subject: [PATCH 103/208] stream/mts/meta/meta.go: updated meta.NewWith so that it just uses meta.New, and now ti can add an abitrary number of things to the map. It also overwrites keys that have been repeated --- stream/mts/meta/meta.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go index 97725252..a2d56e1a 100644 --- a/stream/mts/meta/meta.go +++ b/stream/mts/meta/meta.go @@ -76,18 +76,18 @@ func New() *Data { } } -func NewWith(key, data string) *Data { - meta := &Data{ - data: make(map[string]string), - enc: []byte{ - 0x00, // Reserved byte - (majVer << 4) | minVer, // MS and LS versions - 0x00, // Data len byte1 - 0x00, // Data len byte2 - }, +// NewWith creates a meta.Data and fills map with initial data given. If there +// is repeated key, then the latter overwrites the prior. +func NewWith(data [][2]string) *Data { + m := New() + m.order = make([]string, 0, len(data)) + for _, d := range data { + if _, exists := m.data[d[0]]; !exists { + m.order = append(m.order, d[0]) + } + m.data[d[0]] = d[1] } - meta.Add(key, data) - return meta + return m } // Add adds metadata with key and val. From cdd670393042d7829bd71b5f92dcbabb5093d40d Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 6 Feb 2019 15:45:14 +1030 Subject: [PATCH 104/208] } --- revid/revid.go | 7 ++++++- stream/rtp/encoder.go | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/revid/revid.go b/revid/revid.go index 02656d03..bdf9ccf3 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -145,7 +145,12 @@ func (p *packer) Write(frame []byte) (int, error) { p.owner.config.Logger.Log(logger.Warning, pkg+"frame was too big", "frame size", len(frame)) return len(frame), nil } - n, err := p.owner.buffer.Write(frame) + var n int + var err error + if len(p.owner.destination) != 0 { + n, err = p.owner.buffer.Write(frame) + } + // If we have an rtp sender bypass ringbuffer and give straight to sender if p.owner.rtpSender != nil { err = p.owner.rtpSender.send(frame) diff --git a/stream/rtp/encoder.go b/stream/rtp/encoder.go index 20df9434..329a24c0 100644 --- a/stream/rtp/encoder.go +++ b/stream/rtp/encoder.go @@ -73,8 +73,8 @@ func NewEncoder(dst io.Writer, fps int) *Encoder { func (e *Encoder) Write(data []byte) (int, error) { e.buffer = append(e.buffer, data...) for len(e.buffer) >= sendLen { - e.Encode(e.buffer[:sendLen]) - e.buffer = e.buffer[sendLen:] + e.Encode(e.buffer) + e.buffer = e.buffer[:0] } return len(data), nil } From ef4aa8efd1018842536764b6fe877f6b15b91d92 Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 6 Feb 2019 15:52:59 +1030 Subject: [PATCH 105/208] revid/revid.go: reverted change regarding putting things in ringbuffer when we won't use it --- revid/revid.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/revid/revid.go b/revid/revid.go index bdf9ccf3..6948c0f9 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -145,11 +145,7 @@ func (p *packer) Write(frame []byte) (int, error) { p.owner.config.Logger.Log(logger.Warning, pkg+"frame was too big", "frame size", len(frame)) return len(frame), nil } - var n int - var err error - if len(p.owner.destination) != 0 { - n, err = p.owner.buffer.Write(frame) - } + n, err := p.owner.buffer.Write(frame) // If we have an rtp sender bypass ringbuffer and give straight to sender if p.owner.rtpSender != nil { From 87cb303935d7d974f2159ff1d835aab22eed152e Mon Sep 17 00:00:00 2001 From: Saxon Milton Date: Wed, 6 Feb 2019 05:27:35 +0000 Subject: [PATCH 106/208] revid/revid.go: again checking to see we actually have destinations before putting anything into the ring buffer --- revid/revid.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/revid/revid.go b/revid/revid.go index 6948c0f9..8968a1e5 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -145,7 +145,11 @@ func (p *packer) Write(frame []byte) (int, error) { p.owner.config.Logger.Log(logger.Warning, pkg+"frame was too big", "frame size", len(frame)) return len(frame), nil } - n, err := p.owner.buffer.Write(frame) + var n int + var err error + if len(p.owner.destination) != 0 { + n, err := p.owner.buffer.Write(frame) + } // If we have an rtp sender bypass ringbuffer and give straight to sender if p.owner.rtpSender != nil { From ccb8a52db169fe67151837e4a420bb1758e0585d Mon Sep 17 00:00:00 2001 From: Saxon Milton Date: Wed, 6 Feb 2019 05:28:57 +0000 Subject: [PATCH 107/208] revid/revid.go: removing shorthand initialisations for vars already declared --- revid/revid.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/revid/revid.go b/revid/revid.go index 8968a1e5..454a3b8a 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -148,7 +148,7 @@ func (p *packer) Write(frame []byte) (int, error) { var n int var err error if len(p.owner.destination) != 0 { - n, err := p.owner.buffer.Write(frame) + n, err = p.owner.buffer.Write(frame) } // If we have an rtp sender bypass ringbuffer and give straight to sender From e2bbc84c1ecdc976df5a3b215eb16bb4d4ce81cb Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 7 Feb 2019 08:25:37 +1030 Subject: [PATCH 108/208] cmd/revid-cli: using const strings for copyright metadata preamble --- cmd/revid-cli/main.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index d7f79721..d9c737e1 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -67,15 +67,13 @@ var canProfile = true // The logger that will be used throughout var log *logger.Logger -var ( - metaPreambleKey = []byte{0x63, 0x6f, 0x70, 0x79, 0x72, 0x69, 0x67, 0x68, 0x74} - metaPreambleData = []byte{0x61, 0x75, 0x73, 0x6f, 0x63, 0x65, 0x61, 0x6e, 0x2e, - 0x6f, 0x72, 0x67, 0x2f, 0x6c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x2f, 0x63, - 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x32, 0x30, 0x31, 0x39} +const ( + metaPreambleKey = "copyright" + metaPreambleData = "ausocean.org/license/content2019" ) func main() { - mts.Meta = meta.NewWith(string(metaPreambleKey), string(metaPreambleData)) + mts.Meta = meta.NewWith([][2]string{{metaPreambleKey, metaPreambleData}}) useNetsender := flag.Bool("NetSender", false, "Are we checking vars through netsender?") runDurationPtr := flag.Duration("runDuration", defaultRunDuration, "How long do you want revid to run for?") From fb2217a1f90c5b75fd88c0613c93f07856d7e327 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 7 Feb 2019 08:27:37 +1030 Subject: [PATCH 109/208] stream/mts/encoder.go: wrote todo to make Meta meta.Data struct not be global --- stream/mts/encoder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 2ebe5dc6..29aa83fc 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -88,7 +88,7 @@ const ( psiSndCnt = 7 ) -// global Meta +// TODO: make this not global. var Meta *meta.Data var ( From 7cb58c81ebcb850ed598fe9a0643757620d27961 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 7 Feb 2019 08:29:51 +1030 Subject: [PATCH 110/208] stream/mts/encoder.go: added comment to Meta meta.Data global as it unexported. --- stream/mts/encoder.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 29aa83fc..e6d10a61 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -89,6 +89,8 @@ const ( ) // TODO: make this not global. +// Meta allows addition of metadata to encoded mts from outside of this pkg. +// See meta pkg for usage. var Meta *meta.Data var ( From ad671b520c28f2d634eb7cfa04ee083e8307825f Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 7 Feb 2019 08:31:02 +1030 Subject: [PATCH 111/208] stream/mts/encoder.go: put todo under comment for Meta meta.Data global --- stream/mts/encoder.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index e6d10a61..6ff552b8 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -88,9 +88,10 @@ const ( psiSndCnt = 7 ) -// TODO: make this not global. // Meta allows addition of metadata to encoded mts from outside of this pkg. // See meta pkg for usage. +// +// TODO: make this not global. var Meta *meta.Data var ( From f96f761b2f3eaabf5a58d3617f552c3e9418b241 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 7 Feb 2019 09:01:40 +1030 Subject: [PATCH 112/208] stream/mts/meta/meta_test.go: fixed up error logging in meta_test.go i.e. not doing err.Error() for %v format specifiers, and not using t.Errorf when we have no arguments --- stream/mts/meta/meta_test.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go index 700fda5b..2be41b95 100644 --- a/stream/mts/meta/meta_test.go +++ b/stream/mts/meta/meta_test.go @@ -48,16 +48,16 @@ func TestAddAndGet(t *testing.T) { meta.Add(tstKey1, tstData1) meta.Add(tstKey2, tstData2) if data, err := meta.Get(tstKey1); err != nil { - t.Errorf("Could not get data for key: loc: %v", err.Error()) + t.Errorf("Could not get data for key: loc: %v", err) if data != tstData1 { - t.Errorf("Did not get expected data") + t.Error("Did not get expected data") } } if data, err := meta.Get(tstKey2); err != nil { - t.Errorf("Could not get data for key: ts: %v", err.Error()) + t.Errorf("Could not get data for key: ts: %v", err) if data != tstData2 { - t.Errorf("Did not get expected data") + t.Error("Did not get expected data") } } } @@ -70,9 +70,9 @@ func TestUpdate(t *testing.T) { meta.Add(tstKey1, tstData3) if data, err := meta.Get(tstKey1); err != nil { - t.Errorf("Unexpected err: %v\n", err.Error()) + t.Errorf("Unexpected err: %v\n", err) if data != tstData2 { - t.Errorf("Data did not correctly update for key \"loc\"") + t.Error(`Data did not correctly update for key "loc"`) } } } @@ -100,7 +100,7 @@ func TestGetAbsentKey(t *testing.T) { meta := New() if _, err := meta.Get(tstKey1); err != errKeyAbsent { - t.Errorf("Not expected err: %v\n", errKeyAbsent.Error()) + t.Errorf("Not expected err: %v\n", errKeyAbsent) } } @@ -109,10 +109,10 @@ func TestDelete(t *testing.T) { meta := New() meta.Add(tstKey1, tstData1) if err := meta.Delete(tstKey1); err != nil { - t.Errorf("Unexpected err: %v\n", err.Error()) + t.Errorf("Unexpected err: %v\n", err) } if _, err := meta.Get(tstKey1); err != errKeyAbsent { - t.Errorf("Not expected err: %v\n", errKeyAbsent.Error()) + t.Errorf("Not expected err. got: %v\n, want: $%v\n", err, errKeyAbsent) } } @@ -121,7 +121,7 @@ func TestDelete(t *testing.T) { func TestDeleteAbsentKey(t *testing.T) { meta := New() if err := meta.Delete(tstKey1); err != errKeyAbsent { - t.Errorf("Not expected err: %v\n", errKeyAbsent.Error()) + t.Errorf("Not expected err: %v\n", errKeyAbsent) } } @@ -153,7 +153,7 @@ func TestReadFrom(t *testing.T) { tstMeta := append([]byte{0x00, 0x10, 0x00, 0x12}, "loc=a,b,c\tts=12345"...) got, err := ReadFrom([]byte(tstMeta), "loc") if err != nil { - t.Errorf("Unexpected err: %v\n", err.Error()) + t.Errorf("Unexpected err: %v\n", err) } want := "a,b,c" if got != want { @@ -161,7 +161,7 @@ func TestReadFrom(t *testing.T) { } if got, err = ReadFrom([]byte(tstMeta), "ts"); err != nil { - t.Errorf("Unexpected err: %v\n", err.Error()) + t.Errorf("Unexpected err: %v\n", err) } want = "12345" if got != want { From 66a2325dcb659b179585f4ab34130654366f0250 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 7 Feb 2019 10:42:01 +1030 Subject: [PATCH 113/208] stream/mts/meta/meta_test.go: using table of tests for TestReadFrom --- stream/mts/meta/meta_test.go | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go index 2be41b95..002bbc8f 100644 --- a/stream/mts/meta/meta_test.go +++ b/stream/mts/meta/meta_test.go @@ -151,20 +151,28 @@ func TestEncode(t *testing.T) { // from a string of metadata using the ReadFrom func. func TestReadFrom(t *testing.T) { tstMeta := append([]byte{0x00, 0x10, 0x00, 0x12}, "loc=a,b,c\tts=12345"...) - got, err := ReadFrom([]byte(tstMeta), "loc") - if err != nil { - t.Errorf("Unexpected err: %v\n", err) - } - want := "a,b,c" - if got != want { - t.Errorf("Did not get expected out. \nGot : %v, \nwant: %v\n", got, want) + + tests := []struct { + key string + want string + }{ + { + "loc", + "a,b,c", + }, + { + "ts", + "12345", + }, } - if got, err = ReadFrom([]byte(tstMeta), "ts"); err != nil { - t.Errorf("Unexpected err: %v\n", err) - } - want = "12345" - if got != want { - t.Errorf("Did not get expected out. \nGot : %v, \nwant: %v\n", got, want) + for _, test := range tests { + got, err := ReadFrom([]byte(tstMeta), test.key) + if err != nil { + t.Errorf("Unexpected err: %v\n", err) + } + if got != test.want { + t.Errorf("Did not get expected out. \nGot : %v, \nwant: %v\n", got, test.want) + } } } From bb032778c6b680211b6a1ecea2818743f8fa725f Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 7 Feb 2019 10:48:25 +1030 Subject: [PATCH 114/208] stream/mts/metaEncode_test.go: fixed metaEncode_test.go by initialising Meta global in tests --- stream/mts/metaEncode_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stream/mts/metaEncode_test.go b/stream/mts/metaEncode_test.go index d372f20a..c3c23131 100644 --- a/stream/mts/metaEncode_test.go +++ b/stream/mts/metaEncode_test.go @@ -31,6 +31,7 @@ import ( "bytes" "testing" + "bitbucket.org/ausocean/av/stream/mts/meta" "bitbucket.org/ausocean/av/stream/mts/psi" ) @@ -42,6 +43,7 @@ const ( const fps = 25 func TestMetaEncode1(t *testing.T) { + Meta = meta.New() var b []byte buf := bytes.NewBuffer(b) e := NewEncoder(buf, fps) @@ -67,6 +69,7 @@ func TestMetaEncode1(t *testing.T) { } func TestMetaEncode2(t *testing.T) { + Meta = meta.New() var b []byte buf := bytes.NewBuffer(b) e := NewEncoder(buf, fps) From 7a7be5580d74127366784606da7b28404e253770 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 7 Feb 2019 10:55:58 +1030 Subject: [PATCH 115/208] stream/mts/metaEncode_test.go: wrote function comments for tests --- stream/mts/metaEncode_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/stream/mts/metaEncode_test.go b/stream/mts/metaEncode_test.go index c3c23131..e970b7c8 100644 --- a/stream/mts/metaEncode_test.go +++ b/stream/mts/metaEncode_test.go @@ -42,6 +42,9 @@ const ( const fps = 25 +// TestMetaEncode1 checks that we can externally add a single metadata entry to +// the mts global Meta meta.Data struct and then successfully have the mts encoder +// write this to psi. func TestMetaEncode1(t *testing.T) { Meta = meta.New() var b []byte @@ -68,6 +71,9 @@ func TestMetaEncode1(t *testing.T) { } } +// TestMetaEncode2 checks that we can externally add two metadata entries to the +// Meta meta.Data global and then have the mts encoder successfully encode this +// into psi. func TestMetaEncode2(t *testing.T) { Meta = meta.New() var b []byte From 46b5acb48c4e9fd7b0e1f0afa430ed538253d839 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 7 Feb 2019 13:59:43 +1030 Subject: [PATCH 116/208] stream/mts/mpgets.go: removing unused consts --- stream/mts/mpegts.go | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/stream/mts/mpegts.go b/stream/mts/mpegts.go index 71849513..ab23b9a7 100644 --- a/stream/mts/mpegts.go +++ b/stream/mts/mpegts.go @@ -60,30 +60,6 @@ const ( HasAdaptationField = 0x2 ) -// Adaptation field body masks. -const ( - DiscontinuityIndicatorMask = 0x80 - RandomAccessIndicatorMask = 0x40 - ElementaryStreamPriorityIndicatorMask = 0x20 - ProgramClockReferenceFlagMask = 0x10 - OriginalProgramClockReferenceFlagMask = 0x08 - SplicingPointFlagMask = 0x04 - TransportPrivateDataFlagMask = 0x02 - AdaptationFieldExtensionMask = 0x01 -) - -// Adaptation field body indexes. -const ( - DiscontinuityIndicatorIdx = AdaptationIdx + 1 - RandomAccessIndicatorIdx = AdaptationIdx + 1 - ElementaryStreamPriorityIndicatorIdx = AdaptationIdx + 1 - ProgramClockReferenceFlagIdx = AdaptationIdx + 1 - OriginalProgramClockReferenceFlagIdx = AdaptationIdx + 1 - SplicingPointFlagIdx = AdaptationIdx + 1 - TransportPrivateDataFlagIdx = AdaptationIdx + 1 - AdaptationFieldExtensionFlagIdx = AdaptationIdx + 1 -) - /* The below data struct encapsulates the fields of an MPEG-TS packet. Below is the formatting of an MPEG-TS packet for reference! From b8fc6d7e9902fc6a32b29016807e1122ff1fd835 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 7 Feb 2019 14:50:31 +1030 Subject: [PATCH 117/208] stream/mts/meta/meta.go: ReadFrom to Extract --- stream/mts/meta/meta.go | 2 +- stream/mts/meta/meta_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go index a2d56e1a..bb8b1161 100644 --- a/stream/mts/meta/meta.go +++ b/stream/mts/meta/meta.go @@ -170,7 +170,7 @@ func (m *Data) Encode() []byte { // ReadFrom extracts a value from a metadata string d, for the given key. If the // key is not present in the metadata string, an error is returned. If the // metadata header is not present in the string, an error is returned. -func ReadFrom(d []byte, key string) (string, error) { +func Extract(key string, d []byte) (string, error) { if d[0] != 0 { return "", errNoHeader } else if d[0] == 0 && binary.BigEndian.Uint16(d[2:headSize]) != uint16(len(d[headSize:])) { diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go index 002bbc8f..cb770883 100644 --- a/stream/mts/meta/meta_test.go +++ b/stream/mts/meta/meta_test.go @@ -167,7 +167,7 @@ func TestReadFrom(t *testing.T) { } for _, test := range tests { - got, err := ReadFrom([]byte(tstMeta), test.key) + got, err := Extract(test.key, []byte(tstMeta)) if err != nil { t.Errorf("Unexpected err: %v\n", err) } From 21265303d77ce3019dd99f0ce2e3881ddb51478a Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 7 Feb 2019 15:06:54 +1030 Subject: [PATCH 118/208] stream/mts/mpegts.go: reorganised exported constants and commented --- stream/mts/mpegts.go | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/stream/mts/mpegts.go b/stream/mts/mpegts.go index ab23b9a7..e18213b5 100644 --- a/stream/mts/mpegts.go +++ b/stream/mts/mpegts.go @@ -32,29 +32,36 @@ import ( "errors" ) +// General mpegts packet properties. const ( PacketSize = 188 PayloadSize = 176 ) +// Program ID for various types of ts packets. const ( - SdtPid = 17 - PatPid = 0 - PmtPid = 4096 - VideoPid = 256 - StreamID = 0xe0 // First video stream ID. - HeadSize = 4 - DefaultAdaptationSize = 2 + SdtPid = 17 + PatPid = 0 + PmtPid = 4096 + VideoPid = 256 ) +// StreamID is the id of the first stream. +const StreamID = 0xe0 + +// HeadSize is the size of an mpegts packet header. +const HeadSize = 4 + +// Consts relating to adaptation field. const ( - AdaptationIdx = 4 - AdaptationControlIdx = 3 - AdaptationBodyIdx = AdaptationIdx + 1 - AdaptationControlMask = 0x30 - DefaultAdaptationBodySize = 1 + AdaptationIdx = 4 // Index to the adaptation field (index of AFL). + AdaptationControlIdx = 3 // Index to octet with adaptation field control. + AdaptationFieldsIdx = AdaptationIdx + 1 // Adaptation field index is the index of the adaptation fields. + DefaultAdaptationSize = 2 // Default size of the adaptation field. + AdaptationControlMask = 0x30 // Mask for the adaptation field control in octet 3. ) +// TODO: make this better - currently doesn't make sense. const ( HasPayload = 0x1 HasAdaptationField = 0x2 From b4393e5136be5bc53a9d6718eb292759ccc01043 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 7 Feb 2019 15:23:41 +1030 Subject: [PATCH 119/208] stream/mts/psi/psi.go: checking that we have enough space in psi before creating descriptor --- stream/mts/psi/psi.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 91d8fade..cc9dc8f9 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -52,6 +52,8 @@ const ( pmtID = 0x02 ) +const TotalSyntaxSecLen = 180 + // Consts relating to time description const ( TimeDescTag = 234 @@ -251,8 +253,8 @@ func (p *PSIBytes) AddDescriptor(tag int, data []byte) error { i, desc := p.HasDescriptor(tag) if desc == nil { - p.createDescriptor(tag, data) - return nil + err := p.createDescriptor(tag, data) + return err } oldDescLen := desc.len() @@ -308,9 +310,12 @@ func (p *PSIBytes) HasDescriptor(tag int) (int, Descriptor) { // createDescriptor creates a descriptor in a psi given a tag and data. It does so // by resizing the psi, shifting existing data down and copying in new descriptor // in new space. -func (p *PSIBytes) createDescriptor(tag int, data []byte) { +func (p *PSIBytes) createDescriptor(tag int, data []byte) error { curProgLen := p.ProgramInfoLen() oldSyntaxSectionLen := SyntaxSecLenFrom(*p) + if TotalSyntaxSecLen-(oldSyntaxSectionLen+2+len(data)) <= 0 { + return errors.New("Not enough space in psi to create descriptor.") + } dataLen := len(data) newDescIdx := DescriptorsIdx + curProgLen newDescLen := dataLen + 2 @@ -332,6 +337,8 @@ func (p *PSIBytes) createDescriptor(tag int, data []byte) { newSyntaxSectionLen := int(oldSyntaxSectionLen) + addedLen p.setSectionLen(newSyntaxSectionLen) UpdateCrc((*p)[1:]) + + return nil } // setProgInfoLen sets the program information length in a psi with a pmt. From 330fdc6aa88862b5d17f75544bdca713b793f172 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 7 Feb 2019 15:48:35 +1030 Subject: [PATCH 120/208] stream/mts/psi/psi.go: cleaned up and comment consts --- stream/mts/psi/psi.go | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index cc9dc8f9..c93d3011 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -32,11 +32,10 @@ import ( "github.com/Comcast/gots/psi" ) -const ( - PacketSize = 184 // packet size of a psi. -) +// PacketSize of psi (without mpegts header) +const PacketSize = 184 -// Lengths of section definitions +// Lengths of section definitions. const ( ESSDDefLen = 5 DescDefLen = 2 @@ -46,15 +45,14 @@ const ( PSIDefLen = 3 ) -// Table Type IDs +// Table Type IDs. const ( patID = 0x00 pmtID = 0x02 ) -const TotalSyntaxSecLen = 180 - // Consts relating to time description +// TODO: remove this, we don't do metadata like this anymore. const ( TimeDescTag = 234 TimeTagIndx = 13 @@ -63,6 +61,7 @@ const ( ) // Consts relating to location description +// TODO: remove this, we don't do metadata like this anymore. const ( LocationDescTag = 235 LocationTagIndx = 23 @@ -70,35 +69,32 @@ const ( LocationDataSize = 32 // bytes ) -// Other misc consts +// crc hassh Size +const crcSize = 4 + +// Consts relating to syntax section. const ( + TotalSyntaxSecLen = 180 SyntaxSecLenIdx1 = 2 SyntaxSecLenIdx2 = 3 SyntaxSecLenMask1 = 0x03 - crcSize = 4 -) - -const ( - SectionLenIdx1 = 2 - SectionLenIdx2 = 3 -) - -const ( - SectionLenMask1 = 0x03 + SectionLenMask1 = 0x03 ) +// Consts relating to program info len. const ( ProgramInfoLenIdx1 = 11 ProgramInfoLenIdx2 = 12 ProgramInfoLenMask1 = 0x03 ) -const ( - DescriptorsIdx = ProgramInfoLenIdx2 + 1 -) +// DescriptorsIdx is the index that the descriptors start at. +const DescriptorsIdx = ProgramInfoLenIdx2 + 1 +// MetadataTag is the descriptor tag used for metadata. const MetadataTag = 0x26 +// TODO: get rid of these - not a good idea. type ( PSIBytes []byte Descriptor []byte From 628adee49a37072e7fe1ff33f93bf8f6a51a719c Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 7 Feb 2019 16:42:39 +1030 Subject: [PATCH 121/208] commenting out things i don't want to run --- revid/senders.go | 59 +++++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/revid/senders.go b/revid/senders.go index a73523d5..defdada0 100644 --- a/revid/senders.go +++ b/revid/senders.go @@ -126,37 +126,40 @@ func (s *httpSender) load(c *ring.Chunk) error { } func (s *httpSender) send() error { - if s.chunk == nil { - // Do not retry with httpSender, - // so just return without error - // if the chunk has been cleared. - return nil - } - // Only send if "V0" is configured as an input. - send := false - ip := s.client.Param("ip") - pins := netsender.MakePins(ip, "V") - for i, pin := range pins { - if pin.Name == "V0" { - send = true - pins[i].Value = s.chunk.Len() - pins[i].Data = s.chunk.Bytes() - pins[i].MimeType = "video/mp2t" - break + /* + if s.chunk == nil { + // Do not retry with httpSender, + // so just return without error + // if the chunk has been cleared. + return nil + } + // Only send if "V0" is configured as an input. + send := false + ip := s.client.Param("ip") + pins := netsender.MakePins(ip, "V") + for i, pin := range pins { + if pin.Name == "V0" { + send = true + pins[i].Value = s.chunk.Len() + pins[i].Data = s.chunk.Bytes() + pins[i].MimeType = "video/mp2t" + break + } } - } - if !send { - return nil - } - var err error - var reply string - reply, _, err = s.client.Send(netsender.RequestRecv, pins) - if err != nil { - return err - } + if !send { + return nil + } + var err error + var reply string + reply, _, err = s.client.Send(netsender.RequestRecv, pins) + if err != nil { + return err + } - return s.extractMeta(reply) + return s.extractMeta(reply) + */ + return nil } // extractMeta looks at a reply at extracts any time or location data - then used From 3f72e21970750cdac1227134272dcfe19c482552 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 7 Feb 2019 18:03:59 +1030 Subject: [PATCH 122/208] revid: reducing ring buffer size --- revid/revid.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/revid/revid.go b/revid/revid.go index 02656d03..d6620618 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -51,7 +51,7 @@ import ( // Ring buffer sizes and read/write timeouts. const ( - ringBufferSize = 10000 + ringBufferSize = 100 ringBufferElementSize = 150000 writeTimeout = 10 * time.Millisecond readTimeout = 10 * time.Millisecond From c7f1acac26276a9727c9d5a8d5172ebef2261bcb Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 7 Feb 2019 18:19:38 +1030 Subject: [PATCH 123/208] doesn't look like it worked --- revid/senders.go | 60 +++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/revid/senders.go b/revid/senders.go index defdada0..4be4c5f3 100644 --- a/revid/senders.go +++ b/revid/senders.go @@ -126,40 +126,38 @@ func (s *httpSender) load(c *ring.Chunk) error { } func (s *httpSender) send() error { - /* - if s.chunk == nil { - // Do not retry with httpSender, - // so just return without error - // if the chunk has been cleared. - return nil - } - // Only send if "V0" is configured as an input. - send := false - ip := s.client.Param("ip") - pins := netsender.MakePins(ip, "V") - for i, pin := range pins { - if pin.Name == "V0" { - send = true - pins[i].Value = s.chunk.Len() - pins[i].Data = s.chunk.Bytes() - pins[i].MimeType = "video/mp2t" - break - } - } - if !send { - return nil - } - var err error - var reply string - reply, _, err = s.client.Send(netsender.RequestRecv, pins) - if err != nil { - return err + if s.chunk == nil { + // Do not retry with httpSender, + // so just return without error + // if the chunk has been cleared. + return nil + } + // Only send if "V0" is configured as an input. + send := false + ip := s.client.Param("ip") + pins := netsender.MakePins(ip, "V") + for i, pin := range pins { + if pin.Name == "V0" { + send = true + pins[i].Value = s.chunk.Len() + pins[i].Data = s.chunk.Bytes() + pins[i].MimeType = "video/mp2t" + break } + } - return s.extractMeta(reply) - */ - return nil + if !send { + return nil + } + var err error + var reply string + reply, _, err = s.client.Send(netsender.RequestRecv, pins) + if err != nil { + return err + } + + return s.extractMeta(reply) } // extractMeta looks at a reply at extracts any time or location data - then used From 9ef5886d669c59b4a0dc69caa82a7867b6828af0 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 7 Feb 2019 19:58:08 +1030 Subject: [PATCH 124/208] created experimentation dir under av, and moved flac package here. created experimentation dir under av, and moved flac pkg here. experimentation/flac: removed wav file --- {stream => experimentation}/flac/decode.go | 0 {stream => experimentation}/flac/flac_test.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {stream => experimentation}/flac/decode.go (100%) rename {stream => experimentation}/flac/flac_test.go (100%) diff --git a/stream/flac/decode.go b/experimentation/flac/decode.go similarity index 100% rename from stream/flac/decode.go rename to experimentation/flac/decode.go diff --git a/stream/flac/flac_test.go b/experimentation/flac/flac_test.go similarity index 100% rename from stream/flac/flac_test.go rename to experimentation/flac/flac_test.go From 700328627dcdce15c04f32c705e8533d1733103e Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 8 Feb 2019 00:14:22 +1030 Subject: [PATCH 125/208] cmd/ts-repair: added required consts and undid changes to mts pkg --- cmd/ts-repair/main.go | 43 ++++++++++++++++++++++++------------- stream/mts/encoder.go | 39 ++++++++++++++++++++++------------ stream/mts/mpegts.go | 49 +------------------------------------------ 3 files changed, 56 insertions(+), 75 deletions(-) diff --git a/cmd/ts-repair/main.go b/cmd/ts-repair/main.go index f3f73bb2..f95d16fe 100644 --- a/cmd/ts-repair/main.go +++ b/cmd/ts-repair/main.go @@ -11,6 +11,21 @@ import ( "github.com/Comcast/gots/packet" ) +const ( + PatPid = 0 + PmtPid = 4096 + VideoPid = 256 + HeadSize = 4 + DefaultAdaptationSize = 2 + AdaptationIdx = 4 + AdaptationControlIdx = 3 + AdaptationBodyIdx = AdaptationIdx + 1 + AdaptationControlMask = 0x30 + DefaultAdaptationBodySize = 1 + DiscontinuityIndicatorMask = 0x80 + DiscontinuityIndicatorIdx = AdaptationIdx + 1 +) + // Various errors that we can encounter. const ( errBadInPath = "No file path provided, or file does not exist" @@ -37,9 +52,9 @@ const ( ) var ccMap = map[int]byte{ - mts.PatPid: 16, - mts.PmtPid: 16, - mts.VideoPid: 16, + PatPid: 16, + PmtPid: 16, + VideoPid: 16, } // packetNo will keep track of the ts packet number for reference. @@ -48,8 +63,8 @@ var packetNo int // Option defines a func that performs an action on p in order to change a ts option. type Option func(p *Packet) -// Packet is a byte array of size mts.PacketSize i.e. 188 bytes. We define this -// to allow us to write receiver funcs for the [mts.PacketSize]byte type. +// Packet is a byte array of size PacketSize i.e. 188 bytes. We define this +// to allow us to write receiver funcs for the [PacketSize]byte type. type Packet [mts.PacketSize]byte // CC returns the CC of p. @@ -78,12 +93,12 @@ func (p *Packet) addAdaptationField(options ...Option) error { return errors.New(errAdaptationPresent) } // Create space for adaptation field. - copy(p[mts.HeadSize+mts.DefaultAdaptationSize:], p[mts.HeadSize:len(p)-mts.DefaultAdaptationSize]) + copy(p[HeadSize+DefaultAdaptationSize:], p[HeadSize:len(p)-DefaultAdaptationSize]) // TODO: seperate into own function // Update adaptation field control. - p[mts.AdaptationControlIdx] &= 0xff ^ mts.AdaptationControlMask - p[mts.AdaptationControlIdx] |= mts.AdaptationControlMask + p[AdaptationControlIdx] &= 0xff ^ AdaptationControlMask + p[AdaptationControlIdx] |= AdaptationControlMask // Default the adaptationfield. p.resetAdaptation() @@ -100,14 +115,14 @@ func (p *Packet) resetAdaptation() error { if !p.hasAdaptation() { return errors.New(errNoAdaptationField) } - p[mts.AdaptationIdx] = mts.DefaultAdaptationBodySize - p[mts.AdaptationBodyIdx] = 0x00 + p[AdaptationIdx] = DefaultAdaptationBodySize + p[AdaptationBodyIdx] = 0x00 return nil } // hasAdaptation returns true if p has an adaptation field and false otherwise. func (p *Packet) hasAdaptation() bool { - afc := p[mts.AdaptationControlIdx] & mts.AdaptationControlMask + afc := p[AdaptationControlIdx] & AdaptationControlMask if afc == 0x20 || afc == 0x30 { return true } else { @@ -119,12 +134,12 @@ func (p *Packet) hasAdaptation() bool { // indicator according to f. func DiscontinuityIndicator(f bool) Option { return func(p *Packet) { - set := byte(mts.DiscontinuityIndicatorMask) + set := byte(DiscontinuityIndicatorMask) if !f { set = 0x00 } - p[mts.DiscontinuityIndicatorIdx] &= 0xff ^ mts.DiscontinuityIndicatorMask - p[mts.DiscontinuityIndicatorIdx] |= mts.DiscontinuityIndicatorMask & set + p[DiscontinuityIndicatorIdx] &= 0xff ^ DiscontinuityIndicatorMask + p[DiscontinuityIndicatorIdx] |= DiscontinuityIndicatorMask & set } } diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index e49d57b5..02761b91 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -171,6 +171,14 @@ var ( pmtTable = standardPmtTimeLocation.Bytes() ) +const ( + sdtPid = 17 + patPid = 0 + pmtPid = 4096 + videoPid = 256 + streamID = 0xe0 // First video stream ID. +) + // Time related constants. const ( // ptsOffset is the offset added to the clock to determine @@ -205,13 +213,18 @@ func NewEncoder(dst io.Writer, fps float64) *Encoder { ptsOffset: ptsOffset, continuity: map[int]byte{ - PatPid: 0, - PmtPid: 0, - VideoPid: 0, + patPid: 0, + pmtPid: 0, + videoPid: 0, }, } } +const ( + hasPayload = 0x1 + hasAdaptationField = 0x2 +) + const ( hasDTS = 0x1 hasPTS = 0x2 @@ -231,7 +244,7 @@ func (e *Encoder) Encode(nalu []byte) error { // Prepare PES data. pesPkt := pes.Packet{ - StreamID: StreamID, + StreamID: streamID, PDI: hasPTS, PTS: e.pts(), Data: nalu, @@ -243,10 +256,10 @@ func (e *Encoder) Encode(nalu []byte) error { for len(buf) != 0 { pkt := Packet{ PUSI: pusi, - PID: VideoPid, + PID: videoPid, RAI: pusi, - CC: e.ccFor(VideoPid), - AFC: HasAdaptationField | HasPayload, + CC: e.ccFor(videoPid), + AFC: hasAdaptationField | hasPayload, PCRF: pusi, } n := pkt.FillPayload(buf) @@ -275,9 +288,9 @@ func (e *Encoder) writePSI() error { // Write PAT. patPkt := Packet{ PUSI: true, - PID: PatPid, - CC: e.ccFor(PatPid), - AFC: HasPayload, + PID: patPid, + CC: e.ccFor(patPid), + AFC: hasPayload, Payload: patTable, } _, err := e.dst.Write(patPkt.Bytes(e.tsSpace[:PacketSize])) @@ -298,9 +311,9 @@ func (e *Encoder) writePSI() error { // Create mts packet from pmt table. pmtPkt := Packet{ PUSI: true, - PID: PmtPid, - CC: e.ccFor(PmtPid), - AFC: HasPayload, + PID: pmtPid, + CC: e.ccFor(pmtPid), + AFC: hasPayload, Payload: pmtTable, } _, err = e.dst.Write(pmtPkt.Bytes(e.tsSpace[:PacketSize])) diff --git a/stream/mts/mpegts.go b/stream/mts/mpegts.go index 71849513..0bef80d2 100644 --- a/stream/mts/mpegts.go +++ b/stream/mts/mpegts.go @@ -37,53 +37,6 @@ const ( PayloadSize = 176 ) -const ( - SdtPid = 17 - PatPid = 0 - PmtPid = 4096 - VideoPid = 256 - StreamID = 0xe0 // First video stream ID. - HeadSize = 4 - DefaultAdaptationSize = 2 -) - -const ( - AdaptationIdx = 4 - AdaptationControlIdx = 3 - AdaptationBodyIdx = AdaptationIdx + 1 - AdaptationControlMask = 0x30 - DefaultAdaptationBodySize = 1 -) - -const ( - HasPayload = 0x1 - HasAdaptationField = 0x2 -) - -// Adaptation field body masks. -const ( - DiscontinuityIndicatorMask = 0x80 - RandomAccessIndicatorMask = 0x40 - ElementaryStreamPriorityIndicatorMask = 0x20 - ProgramClockReferenceFlagMask = 0x10 - OriginalProgramClockReferenceFlagMask = 0x08 - SplicingPointFlagMask = 0x04 - TransportPrivateDataFlagMask = 0x02 - AdaptationFieldExtensionMask = 0x01 -) - -// Adaptation field body indexes. -const ( - DiscontinuityIndicatorIdx = AdaptationIdx + 1 - RandomAccessIndicatorIdx = AdaptationIdx + 1 - ElementaryStreamPriorityIndicatorIdx = AdaptationIdx + 1 - ProgramClockReferenceFlagIdx = AdaptationIdx + 1 - OriginalProgramClockReferenceFlagIdx = AdaptationIdx + 1 - SplicingPointFlagIdx = AdaptationIdx + 1 - TransportPrivateDataFlagIdx = AdaptationIdx + 1 - AdaptationFieldExtensionFlagIdx = AdaptationIdx + 1 -) - /* The below data struct encapsulates the fields of an MPEG-TS packet. Below is the formatting of an MPEG-TS packet for reference! @@ -182,7 +135,7 @@ func FindPMT(d []byte) (p []byte, i int, err error) { } for i = 0; i < len(d); i += PacketSize { pid := (uint16(d[i+1]&0x1f) << 8) | uint16(d[i+2]) - if pid == PmtPid { + if pid == pmtPid { p = d[i+4 : i+PacketSize] return } From 7d85b78b6d5b546e0defc7952654990fb71b2fcc Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 8 Feb 2019 00:16:19 +1030 Subject: [PATCH 126/208] moved cmd/ts-repair to experimentation --- {cmd => experimentation}/ts-repair/main.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {cmd => experimentation}/ts-repair/main.go (100%) diff --git a/cmd/ts-repair/main.go b/experimentation/ts-repair/main.go similarity index 100% rename from cmd/ts-repair/main.go rename to experimentation/ts-repair/main.go From b2150cf8ded9d84ed051092d13bfd91f9cbdefd9 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 8 Feb 2019 00:25:47 +1030 Subject: [PATCH 127/208] experimentation/ts-repair: added description to file header --- experimentation/ts-repair/main.go | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/experimentation/ts-repair/main.go b/experimentation/ts-repair/main.go index f95d16fe..bed81f19 100644 --- a/experimentation/ts-repair/main.go +++ b/experimentation/ts-repair/main.go @@ -1,3 +1,36 @@ +/* +NAME + ts-repair/main.go + +DESCRIPTION + This program attempts to repair mpegts discontinuities using one of two methods + as selected by the mode flag. Setting the mode flag to 0 will result in repair + by shifting all CCs such that they are continuous. Setting the mode flag to 1 + will result in repair through setting the discontinuity indicator to true at + packets where a discontinuity exists. + + Specify the input file with the in flag, and the output file with out flag. + +AUTHOR + Saxon A. Nelson-Milton + +LICENSE + mpegts.go is Copyright (C) 2017 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with revid in gpl.txt. If not, see [GNU licenses](http://www.gnu.org/licenses). +*/ + package main import ( From a94bdbfe473e4b41137cfbb90cce0e6b51c1c721 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 8 Feb 2019 10:25:57 +1030 Subject: [PATCH 128/208] stream/mts/meta: meta.Get now returns ok bool rather than error - updated usage accordingly --- stream/mts/meta/meta.go | 9 +++------ stream/mts/meta/meta_test.go | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go index bb8b1161..297b2fa2 100644 --- a/stream/mts/meta/meta.go +++ b/stream/mts/meta/meta.go @@ -116,14 +116,11 @@ func (m *Data) All() map[string]string { } // Get returns the meta data for the passed key. -func (m *Data) Get(key string) (string, error) { +func (m *Data) Get(key string) (val string, ok bool) { m.mu.Lock() - val, ok := m.data[key] + val, ok = m.data[key] m.mu.Unlock() - if !ok { - return "", errKeyAbsent - } - return val, nil + return } // Delete deletes a meta entry in the map and returns error if it doesn’t exist. diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go index cb770883..a97d925f 100644 --- a/stream/mts/meta/meta_test.go +++ b/stream/mts/meta/meta_test.go @@ -47,15 +47,15 @@ func TestAddAndGet(t *testing.T) { meta := New() meta.Add(tstKey1, tstData1) meta.Add(tstKey2, tstData2) - if data, err := meta.Get(tstKey1); err != nil { - t.Errorf("Could not get data for key: loc: %v", err) + if data, ok := meta.Get(tstKey1); !ok { + t.Errorf("Could not get data for key: %v\n", tstKey1) if data != tstData1 { t.Error("Did not get expected data") } } - if data, err := meta.Get(tstKey2); err != nil { - t.Errorf("Could not get data for key: ts: %v", err) + if data, ok := meta.Get(tstKey2); !ok { + t.Errorf("Could not get data for key: %v", tstKey2) if data != tstData2 { t.Error("Did not get expected data") } @@ -69,8 +69,8 @@ func TestUpdate(t *testing.T) { meta.Add(tstKey1, tstData1) meta.Add(tstKey1, tstData3) - if data, err := meta.Get(tstKey1); err != nil { - t.Errorf("Unexpected err: %v\n", err) + if data, ok := meta.Get(tstKey1); !ok { + t.Errorf("Could not get data for key: %v\n", tstKey1) if data != tstData2 { t.Error(`Data did not correctly update for key "loc"`) } @@ -99,8 +99,8 @@ func TestAll(t *testing.T) { func TestGetAbsentKey(t *testing.T) { meta := New() - if _, err := meta.Get(tstKey1); err != errKeyAbsent { - t.Errorf("Not expected err: %v\n", errKeyAbsent) + if _, ok := meta.Get(tstKey1); ok { + t.Error("Get for absent key incorrectly returned'ok'") } } @@ -111,8 +111,8 @@ func TestDelete(t *testing.T) { if err := meta.Delete(tstKey1); err != nil { t.Errorf("Unexpected err: %v\n", err) } - if _, err := meta.Get(tstKey1); err != errKeyAbsent { - t.Errorf("Not expected err. got: %v\n, want: $%v\n", err, errKeyAbsent) + if _, ok := meta.Get(tstKey1); ok { + t.Error("Get incorrectly returned okay for absent key") } } From db3b34c10f1fe78f171f4d43f4ef4cefb5e71edb Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 8 Feb 2019 10:56:19 +1030 Subject: [PATCH 129/208] stream/mts/meta: meta.Delete no longer returns error - updated code accordingly --- stream/mts/meta/meta.go | 6 +++--- stream/mts/meta/meta_test.go | 13 +------------ 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go index 297b2fa2..66790315 100644 --- a/stream/mts/meta/meta.go +++ b/stream/mts/meta/meta.go @@ -124,7 +124,7 @@ func (m *Data) Get(key string) (val string, ok bool) { } // Delete deletes a meta entry in the map and returns error if it doesn’t exist. -func (m *Data) Delete(key string) error { +func (m *Data) Delete(key string) { m.mu.Lock() defer m.mu.Unlock() if _, ok := m.data[key]; ok { @@ -136,9 +136,9 @@ func (m *Data) Delete(key string) error { break } } - return nil + return } - return errKeyAbsent + return } // Encode takes the meta data map and encodes into a byte slice with header diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go index a97d925f..e1f9f3b7 100644 --- a/stream/mts/meta/meta_test.go +++ b/stream/mts/meta/meta_test.go @@ -108,23 +108,12 @@ func TestGetAbsentKey(t *testing.T) { func TestDelete(t *testing.T) { meta := New() meta.Add(tstKey1, tstData1) - if err := meta.Delete(tstKey1); err != nil { - t.Errorf("Unexpected err: %v\n", err) - } + meta.Delete(tstKey1) if _, ok := meta.Get(tstKey1); ok { t.Error("Get incorrectly returned okay for absent key") } } -// TestDeleteAbsentKey checks that we get an expected error when we try to delete -// an entry in the Meta map that doesn't exist. -func TestDeleteAbsentKey(t *testing.T) { - meta := New() - if err := meta.Delete(tstKey1); err != errKeyAbsent { - t.Errorf("Not expected err: %v\n", errKeyAbsent) - } -} - // TestEncode checks that we're getting the correct byte slice from Meta.Encode(). func TestEncode(t *testing.T) { meta := New() From f0fc399d47cd105e14bbb9f301da2a98043226b3 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 8 Feb 2019 11:24:41 +1030 Subject: [PATCH 130/208] revid/revid.go: removed space --- revid/senders.go | 1 - 1 file changed, 1 deletion(-) diff --git a/revid/senders.go b/revid/senders.go index 4be4c5f3..a73523d5 100644 --- a/revid/senders.go +++ b/revid/senders.go @@ -126,7 +126,6 @@ func (s *httpSender) load(c *ring.Chunk) error { } func (s *httpSender) send() error { - if s.chunk == nil { // Do not retry with httpSender, // so just return without error From c7a9e0a06bfab6686caf24c483a6276e119cce98 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 24 Jan 2019 22:29:43 +1030 Subject: [PATCH 131/208] mts: fixing cc --- cmd/ts-repair/main.go | 82 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 cmd/ts-repair/main.go diff --git a/cmd/ts-repair/main.go b/cmd/ts-repair/main.go new file mode 100644 index 00000000..bcb1ebc1 --- /dev/null +++ b/cmd/ts-repair/main.go @@ -0,0 +1,82 @@ +package main + +import ( + "flag" + "io" + "os" + + "bitbucket.org/ausocean/av/stream/mts" + "github.com/Comcast/gots/packet" +) + +const ( + errBadInPath = "No file path provided, or file does not exist" + errCantCreateOut = "Can't create output file" + errCantGetPid = "Can't get pid from packet" + errReadFail = "Read failed" + errWriteFail = "Write to file failed" + usage = "The path to the file to be repaired" +) + +var ccMap = map[int]byte{ + mts.PatPid: 1, + mts.PmtPid: 1, + mts.VideoPid: 0, +} + +type Packet [mts.PacketSize]byte + +func (p *Packet) setCC(cc byte) { + (*p)[3] |= cc & 0xf +} + +func main() { + // Deal with input flags + inPtr := flag.String("path", "", usage) + outPtr := flag.String("out", "out.ts", usage) + flag.Parse() + + // Try and open the given input file, otherwise panic - we can't do anything + inFile, err := os.Open(*inPtr) + defer inFile.Close() + if err != nil { + panic(errBadInPath) + } + + // Try and create output file, otherwise panic - we can't do anything + outFile, err := os.Create(*outPtr) + defer outFile.Close() + if err != nil { + panic(errCantCreateOut) + } + + // Read each packet from the input file reader + var p Packet + for { + // If we get an end of file then return, otherwise we panic - can't do anything else + if _, err := inFile.Read(p[:mts.PacketSize]); err == io.EOF { + return + } else if err != nil { + panic(errReadFail + ": " + err.Error()) + } + + // Get the pid from the packet and set the cc based on this pid using our map + pid, err := packet.Pid((*packet.Packet)(&p)) + if err != nil { + panic(errCantGetPid) + } + p.setCC(ccFor(int(pid))) + + // Write this packet to the output file + if _, err := outFile.Write(p[:]); err != nil { + panic(errWriteFail + ": " + err.Error()) + } + } +} + +// ccFor gets the next cc for the given pid +func ccFor(pid int) byte { + cc := ccMap[pid] + ccMap[pid] = (cc + 1) & 0xf + return cc +} From 72f0087b9a6d22566a30e4bc0b01d915095c9406 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 25 Jan 2019 15:03:57 +1030 Subject: [PATCH 132/208] cmd/ts-repair: got setting of discontinuity indicators working and also adding adaptation fields to pat and pmt for this reason. --- cmd/ts-repair/main.go | 125 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 115 insertions(+), 10 deletions(-) diff --git a/cmd/ts-repair/main.go b/cmd/ts-repair/main.go index bcb1ebc1..f34da350 100644 --- a/cmd/ts-repair/main.go +++ b/cmd/ts-repair/main.go @@ -2,6 +2,7 @@ package main import ( "flag" + "fmt" "io" "os" @@ -15,25 +16,94 @@ const ( errCantGetPid = "Can't get pid from packet" errReadFail = "Read failed" errWriteFail = "Write to file failed" - usage = "The path to the file to be repaired" + errBadMode = "Bad fix mode" +) + +const ( + inUsage = "The path to the file to be repaired" + outUsage = "Output file path" + modeUsage = "Fix mode: 0 = cc-shift, 1 = di-update" +) + +const ( + ccShift = iota + diUpdate ) var ccMap = map[int]byte{ - mts.PatPid: 1, - mts.PmtPid: 1, - mts.VideoPid: 0, + mts.PatPid: 16, + mts.PmtPid: 16, + mts.VideoPid: 16, } +var packetNo int + +type Option func(p *Packet) + type Packet [mts.PacketSize]byte +func (p *Packet) CC() byte { + return (*p)[3] & 0x0f +} + func (p *Packet) setCC(cc byte) { (*p)[3] |= cc & 0xf } +func (p *Packet) setDI(di bool) { + if di { + p[5] |= 0x80 + } else { + p[5] &= 0x7f + } +} + +func (p *Packet) addAdaptationField(options ...Option) { + // Create space for adaptation field + copy(p[mts.HeadSize+mts.DefaultAdaptationSize:], p[mts.HeadSize:len(p)-mts.DefaultAdaptationSize]) + + // TODO: seperate into own function + // Update adaptation field control + p[mts.AdaptationControlIdx] &= 0xff ^ mts.AdaptationControlMask + p[mts.AdaptationControlIdx] |= mts.AdaptationControlMask + // Default the adaptationfield + p.resetAdaptation() + + for _, option := range options { + option(p) + } +} + +func (p *Packet) resetAdaptation() { + p[mts.AdaptationIdx] = mts.DefaultAdaptationBodySize + p[mts.AdaptationBodyIdx] = 0x00 +} + +func (p *Packet) hasAdaptation() bool { + afc := p[mts.AdaptationControlIdx] & mts.AdaptationControlMask + if afc == 0x20 || afc == 0x30 { + return true + } else { + return false + } +} + +func DiscontinuityIndicator(f bool) Option { + return func(p *Packet) { + set := byte(mts.DiscontinuityIndicatorMask) + if !f { + set = 0x00 + } + p[mts.DiscontinuityIndicatorIdx] &= 0xff ^ mts.DiscontinuityIndicatorMask + p[mts.DiscontinuityIndicatorIdx] |= mts.DiscontinuityIndicatorMask & set + } +} + func main() { // Deal with input flags - inPtr := flag.String("path", "", usage) - outPtr := flag.String("out", "out.ts", usage) + inPtr := flag.String("in", "", inUsage) + outPtr := flag.String("out", "out.ts", outUsage) + modePtr := flag.Int("mode", diUpdate, modeUsage) flag.Parse() // Try and open the given input file, otherwise panic - we can't do anything @@ -59,14 +129,42 @@ func main() { } else if err != nil { panic(errReadFail + ": " + err.Error()) } - + packetNo++ // Get the pid from the packet and set the cc based on this pid using our map pid, err := packet.Pid((*packet.Packet)(&p)) if err != nil { panic(errCantGetPid) } - p.setCC(ccFor(int(pid))) + cc := p.CC() + expect, exists := expectedCC(int(pid)) + if !exists { + updateCCMap(int(pid), cc) + } else { + switch *modePtr { + case ccShift: + p.setCC(expect) + + case diUpdate: + + if cc != expect { + fmt.Printf("packetNo: %v pid: %v, cc: %v, expect: %v\n", packetNo, pid, cc, expect) + if p.hasAdaptation() { + fmt.Println("hasAdaptation") + p.setDI(true) + } else { + fmt.Println("doesn't have adaptation") + fmt.Println(p) + p.addAdaptationField(DiscontinuityIndicator(true)) + fmt.Println(p) + } + updateCCMap(int(pid), p.CC()) + } + + default: + panic(errBadMode) + } + } // Write this packet to the output file if _, err := outFile.Write(p[:]); err != nil { panic(errWriteFail + ": " + err.Error()) @@ -75,8 +173,15 @@ func main() { } // ccFor gets the next cc for the given pid -func ccFor(pid int) byte { +func expectedCC(pid int) (byte, bool) { cc := ccMap[pid] + if cc == 16 { + return 16, false + } + ccMap[pid] = (cc + 1) & 0xf + return cc, true +} + +func updateCCMap(pid int, cc byte) { ccMap[pid] = (cc + 1) & 0xf - return cc } From cac718473758b686b3880af04e0bfa1e5558b9f0 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 25 Jan 2019 15:18:27 +1030 Subject: [PATCH 133/208] cmd/ts-repair: got rid of debug prints --- cmd/ts-repair/main.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cmd/ts-repair/main.go b/cmd/ts-repair/main.go index f34da350..039ada9b 100644 --- a/cmd/ts-repair/main.go +++ b/cmd/ts-repair/main.go @@ -148,15 +148,11 @@ func main() { case diUpdate: if cc != expect { - fmt.Printf("packetNo: %v pid: %v, cc: %v, expect: %v\n", packetNo, pid, cc, expect) + fmt.Printf("***** Discontinuity found (packetNo: %v pid: %v, cc: %v, expect: %v)\n", packetNo, pid, cc, expect) if p.hasAdaptation() { - fmt.Println("hasAdaptation") p.setDI(true) } else { - fmt.Println("doesn't have adaptation") - fmt.Println(p) p.addAdaptationField(DiscontinuityIndicator(true)) - fmt.Println(p) } updateCCMap(int(pid), p.CC()) } From 90a85fbfbb68731f664ebaacef39a0e6d23409c7 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 25 Jan 2019 16:10:13 +1030 Subject: [PATCH 134/208] cmd/ts-repair: updated comments and made funcs more robust --- cmd/ts-repair/main.go | 68 ++++++++++++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 17 deletions(-) diff --git a/cmd/ts-repair/main.go b/cmd/ts-repair/main.go index 039ada9b..f3f73bb2 100644 --- a/cmd/ts-repair/main.go +++ b/cmd/ts-repair/main.go @@ -1,6 +1,7 @@ package main import ( + "errors" "flag" "fmt" "io" @@ -10,21 +11,26 @@ import ( "github.com/Comcast/gots/packet" ) +// Various errors that we can encounter. const ( - errBadInPath = "No file path provided, or file does not exist" - errCantCreateOut = "Can't create output file" - errCantGetPid = "Can't get pid from packet" - errReadFail = "Read failed" - errWriteFail = "Write to file failed" - errBadMode = "Bad fix mode" + errBadInPath = "No file path provided, or file does not exist" + errCantCreateOut = "Can't create output file" + errCantGetPid = "Can't get pid from packet" + errReadFail = "Read failed" + errWriteFail = "Write to file failed" + errBadMode = "Bad fix mode" + errAdaptationPresent = "Adaptation field is already present in packet" + errNoAdaptationField = "No adaptation field in this packet" ) +// Consts describing flag usage. const ( inUsage = "The path to the file to be repaired" outUsage = "Output file path" modeUsage = "Fix mode: 0 = cc-shift, 1 = di-update" ) +// Repair modes. const ( ccShift = iota diUpdate @@ -36,20 +42,27 @@ var ccMap = map[int]byte{ mts.VideoPid: 16, } +// packetNo will keep track of the ts packet number for reference. var packetNo int +// Option defines a func that performs an action on p in order to change a ts option. type Option func(p *Packet) +// Packet is a byte array of size mts.PacketSize i.e. 188 bytes. We define this +// to allow us to write receiver funcs for the [mts.PacketSize]byte type. type Packet [mts.PacketSize]byte +// CC returns the CC of p. func (p *Packet) CC() byte { return (*p)[3] & 0x0f } +// setCC sets the CC of p. func (p *Packet) setCC(cc byte) { (*p)[3] |= cc & 0xf } +// setDI sets the discontinuity counter of p. func (p *Packet) setDI(di bool) { if di { p[5] |= 0x80 @@ -58,27 +71,41 @@ func (p *Packet) setDI(di bool) { } } -func (p *Packet) addAdaptationField(options ...Option) { - // Create space for adaptation field +// addAdaptationField adds an adaptation field to p, and applys the passed options to this field. +// TODO: this will probably break if we already have adaptation field. +func (p *Packet) addAdaptationField(options ...Option) error { + if p.hasAdaptation() { + return errors.New(errAdaptationPresent) + } + // Create space for adaptation field. copy(p[mts.HeadSize+mts.DefaultAdaptationSize:], p[mts.HeadSize:len(p)-mts.DefaultAdaptationSize]) // TODO: seperate into own function - // Update adaptation field control + // Update adaptation field control. p[mts.AdaptationControlIdx] &= 0xff ^ mts.AdaptationControlMask p[mts.AdaptationControlIdx] |= mts.AdaptationControlMask - // Default the adaptationfield + // Default the adaptationfield. p.resetAdaptation() + // Apply and options that have bee passed. for _, option := range options { option(p) } + return nil } -func (p *Packet) resetAdaptation() { +// resetAdaptation sets fields in ps adaptation field to 0 if the adaptation field +// exists, otherwise an error is returned. +func (p *Packet) resetAdaptation() error { + if !p.hasAdaptation() { + return errors.New(errNoAdaptationField) + } p[mts.AdaptationIdx] = mts.DefaultAdaptationBodySize p[mts.AdaptationBodyIdx] = 0x00 + return nil } +// hasAdaptation returns true if p has an adaptation field and false otherwise. func (p *Packet) hasAdaptation() bool { afc := p[mts.AdaptationControlIdx] & mts.AdaptationControlMask if afc == 0x20 || afc == 0x30 { @@ -88,6 +115,8 @@ func (p *Packet) hasAdaptation() bool { } } +// DiscontinuityIndicator returns and Option that will set p's discontinuity +// indicator according to f. func DiscontinuityIndicator(f bool) Option { return func(p *Packet) { set := byte(mts.DiscontinuityIndicatorMask) @@ -130,23 +159,26 @@ func main() { panic(errReadFail + ": " + err.Error()) } packetNo++ - // Get the pid from the packet and set the cc based on this pid using our map + + // Get the pid from the packet pid, err := packet.Pid((*packet.Packet)(&p)) if err != nil { panic(errCantGetPid) } + // Get the cc from the packet and also the expected cc (if exists) cc := p.CC() expect, exists := expectedCC(int(pid)) if !exists { updateCCMap(int(pid), cc) } else { switch *modePtr { + // ccShift mode shifts all CC regardless of presence of Discontinuities or not case ccShift: p.setCC(expect) - + // diUpdate mode finds discontinuities and sets the discontinuity indicator to true. + // If we have a pat or pmt then we need to add an adaptation field and then set the DI. case diUpdate: - if cc != expect { fmt.Printf("***** Discontinuity found (packetNo: %v pid: %v, cc: %v, expect: %v)\n", packetNo, pid, cc, expect) if p.hasAdaptation() { @@ -156,19 +188,20 @@ func main() { } updateCCMap(int(pid), p.CC()) } - default: panic(errBadMode) } } - // Write this packet to the output file + + // Write this packet to the output file. if _, err := outFile.Write(p[:]); err != nil { panic(errWriteFail + ": " + err.Error()) } } } -// ccFor gets the next cc for the given pid +// expectedCC returns the expected cc for the given pid. If the cc hasn't been +// used yet, then 16 and false is returned. func expectedCC(pid int) (byte, bool) { cc := ccMap[pid] if cc == 16 { @@ -178,6 +211,7 @@ func expectedCC(pid int) (byte, bool) { return cc, true } +// updateCCMap updates the cc for the passed pid. func updateCCMap(pid int, cc byte) { ccMap[pid] = (cc + 1) & 0xf } From a6d02a2e0289399cc69accdb033b33992d9a5617 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 25 Jan 2019 16:25:01 +1030 Subject: [PATCH 135/208] stream/mts: adding some constants --- stream/mts/encoder.go | 39 ++++++++++++---------------------- stream/mts/mpegts.go | 49 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 27 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 02761b91..e49d57b5 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -171,14 +171,6 @@ var ( pmtTable = standardPmtTimeLocation.Bytes() ) -const ( - sdtPid = 17 - patPid = 0 - pmtPid = 4096 - videoPid = 256 - streamID = 0xe0 // First video stream ID. -) - // Time related constants. const ( // ptsOffset is the offset added to the clock to determine @@ -213,18 +205,13 @@ func NewEncoder(dst io.Writer, fps float64) *Encoder { ptsOffset: ptsOffset, continuity: map[int]byte{ - patPid: 0, - pmtPid: 0, - videoPid: 0, + PatPid: 0, + PmtPid: 0, + VideoPid: 0, }, } } -const ( - hasPayload = 0x1 - hasAdaptationField = 0x2 -) - const ( hasDTS = 0x1 hasPTS = 0x2 @@ -244,7 +231,7 @@ func (e *Encoder) Encode(nalu []byte) error { // Prepare PES data. pesPkt := pes.Packet{ - StreamID: streamID, + StreamID: StreamID, PDI: hasPTS, PTS: e.pts(), Data: nalu, @@ -256,10 +243,10 @@ func (e *Encoder) Encode(nalu []byte) error { for len(buf) != 0 { pkt := Packet{ PUSI: pusi, - PID: videoPid, + PID: VideoPid, RAI: pusi, - CC: e.ccFor(videoPid), - AFC: hasAdaptationField | hasPayload, + CC: e.ccFor(VideoPid), + AFC: HasAdaptationField | HasPayload, PCRF: pusi, } n := pkt.FillPayload(buf) @@ -288,9 +275,9 @@ func (e *Encoder) writePSI() error { // Write PAT. patPkt := Packet{ PUSI: true, - PID: patPid, - CC: e.ccFor(patPid), - AFC: hasPayload, + PID: PatPid, + CC: e.ccFor(PatPid), + AFC: HasPayload, Payload: patTable, } _, err := e.dst.Write(patPkt.Bytes(e.tsSpace[:PacketSize])) @@ -311,9 +298,9 @@ func (e *Encoder) writePSI() error { // Create mts packet from pmt table. pmtPkt := Packet{ PUSI: true, - PID: pmtPid, - CC: e.ccFor(pmtPid), - AFC: hasPayload, + PID: PmtPid, + CC: e.ccFor(PmtPid), + AFC: HasPayload, Payload: pmtTable, } _, err = e.dst.Write(pmtPkt.Bytes(e.tsSpace[:PacketSize])) diff --git a/stream/mts/mpegts.go b/stream/mts/mpegts.go index 0bef80d2..71849513 100644 --- a/stream/mts/mpegts.go +++ b/stream/mts/mpegts.go @@ -37,6 +37,53 @@ const ( PayloadSize = 176 ) +const ( + SdtPid = 17 + PatPid = 0 + PmtPid = 4096 + VideoPid = 256 + StreamID = 0xe0 // First video stream ID. + HeadSize = 4 + DefaultAdaptationSize = 2 +) + +const ( + AdaptationIdx = 4 + AdaptationControlIdx = 3 + AdaptationBodyIdx = AdaptationIdx + 1 + AdaptationControlMask = 0x30 + DefaultAdaptationBodySize = 1 +) + +const ( + HasPayload = 0x1 + HasAdaptationField = 0x2 +) + +// Adaptation field body masks. +const ( + DiscontinuityIndicatorMask = 0x80 + RandomAccessIndicatorMask = 0x40 + ElementaryStreamPriorityIndicatorMask = 0x20 + ProgramClockReferenceFlagMask = 0x10 + OriginalProgramClockReferenceFlagMask = 0x08 + SplicingPointFlagMask = 0x04 + TransportPrivateDataFlagMask = 0x02 + AdaptationFieldExtensionMask = 0x01 +) + +// Adaptation field body indexes. +const ( + DiscontinuityIndicatorIdx = AdaptationIdx + 1 + RandomAccessIndicatorIdx = AdaptationIdx + 1 + ElementaryStreamPriorityIndicatorIdx = AdaptationIdx + 1 + ProgramClockReferenceFlagIdx = AdaptationIdx + 1 + OriginalProgramClockReferenceFlagIdx = AdaptationIdx + 1 + SplicingPointFlagIdx = AdaptationIdx + 1 + TransportPrivateDataFlagIdx = AdaptationIdx + 1 + AdaptationFieldExtensionFlagIdx = AdaptationIdx + 1 +) + /* The below data struct encapsulates the fields of an MPEG-TS packet. Below is the formatting of an MPEG-TS packet for reference! @@ -135,7 +182,7 @@ func FindPMT(d []byte) (p []byte, i int, err error) { } for i = 0; i < len(d); i += PacketSize { pid := (uint16(d[i+1]&0x1f) << 8) | uint16(d[i+2]) - if pid == pmtPid { + if pid == PmtPid { p = d[i+4 : i+PacketSize] return } From b28a75d665e275fa4faec7eacd47b566675ccdf5 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 22 Jan 2019 14:31:42 +1030 Subject: [PATCH 136/208] av/revid: removed test commands that we're not using anymore --- revid/cmd/h264-file-to-flv-rtmp/main.go | 75 ---------------------- revid/cmd/h264-file-to-mpgets-file/main.go | 66 ------------------- 2 files changed, 141 deletions(-) delete mode 100644 revid/cmd/h264-file-to-flv-rtmp/main.go delete mode 100644 revid/cmd/h264-file-to-mpgets-file/main.go diff --git a/revid/cmd/h264-file-to-flv-rtmp/main.go b/revid/cmd/h264-file-to-flv-rtmp/main.go deleted file mode 100644 index 4f7c9d7c..00000000 --- a/revid/cmd/h264-file-to-flv-rtmp/main.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -NAME - main.go - -DESCRIPTION - See Readme.md - -AUTHOR - Saxon Nelson-Milton - -LICENSE - main.go is Copyright (C) 2017 the Australian Ocean Lab (AusOcean) - - It is free software: you can redistribute it and/or modify them - under the terms of the GNU General Public License as published by the - Free Software Foundation, either version 3 of the License, or (at your - option) any later version. - - It is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License - along with revid in gpl.txt. If not, see [GNU licenses](http://www.gnu.org/licenses). -*/ - -package main - -import ( - "flag" - "log" - "time" - - "bitbucket.org/ausocean/av/revid" - "bitbucket.org/ausocean/iot/pi/smartlogger" - "bitbucket.org/ausocean/utils/logger" -) - -const ( - inputFile = "../../../../test/test-data/av/input/betterInput.h264" - frameRate = "25" - runDuration = 120 * time.Second - logPath = "/var/log" -) - -// Test h264 inputfile to flv format into rtmp using librtmp c wrapper -func main() { - // Get the rtmp url from a cmd flag - rtmpUrlPtr := flag.String("rtmpUrl", "", "The rtmp url you would like to stream to.") - flag.Parse() - if *rtmpUrlPtr == "" { - log.Println("No RTMP url passed!") - return - } - - config := revid.Config{ - Input: revid.File, - InputFileName: inputFile, - InputCodec: revid.H264, - Outputs: []byte{revid.Rtmp}, - RtmpMethod: revid.LibRtmp, - RtmpUrl: *rtmpUrlPtr, - Packetization: revid.Flv, - Logger: logger.New(logger.Info, &smartlogger.New(logPath).LogRoller), - } - revidInst, err := revid.New(config, nil) - if err != nil { - config.Logger.Log(logger.Error, "Should not have got an error!: ", err.Error()) - return - } - revidInst.Start() - time.Sleep(runDuration) - revidInst.Stop() -} diff --git a/revid/cmd/h264-file-to-mpgets-file/main.go b/revid/cmd/h264-file-to-mpgets-file/main.go deleted file mode 100644 index 768f560b..00000000 --- a/revid/cmd/h264-file-to-mpgets-file/main.go +++ /dev/null @@ -1,66 +0,0 @@ -/* -NAME - main.go - -DESCRIPTION - See Readme.md - -AUTHOR - Saxon Nelson-Milton - -LICENSE - main.go is Copyright (C) 2017 the Australian Ocean Lab (AusOcean) - - It is free software: you can redistribute it and/or modify them - under the terms of the GNU General Public License as published by the - Free Software Foundation, either version 3 of the License, or (at your - option) any later version. - - It is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License - along with revid in gpl.txt. If not, see [GNU licenses](http://www.gnu.org/licenses). -*/ - -package main - -import ( - "time" - - "bitbucket.org/ausocean/av/revid" - "bitbucket.org/ausocean/iot/pi/smartlogger" - "bitbucket.org/ausocean/utils/logger" -) - -const ( - inputFile = "../../../../test/test-data/av/input/betterInput.h264" - outputFile = "output.ts" - frameRate = "25" - runDuration = 120 * time.Second - logPath = "/var/log" -) - -// Test h264 inputfile to flv format into rtmp using librtmp c wrapper -func main() { - - config := revid.Config{ - Input: revid.File, - InputFileName: inputFile, - InputCodec: revid.H264, - Outputs: []byte{revid.File}, - OutputFileName: outputFile, - Packetization: revid.Mpegts, - Logger: logger.New(logger.Info, &smartlogger.New(logPath).LogRoller), - } - revidInst, err := revid.New(config, nil) - if err != nil { - config.Logger.Log(logger.Error, "Should not have got an error!:", err.Error()) - return - } - revidInst.Start() - time.Sleep(runDuration) - revidInst.Stop() -} From dda6602a475ad5f795776656e506c49acf25d7f6 Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 21 Jan 2019 14:57:40 +1030 Subject: [PATCH 137/208] av/stream/flac: added decode.go and flac_test.go --- stream/flac/decode.go | 0 stream/flac/flac_test.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 stream/flac/decode.go create mode 100644 stream/flac/flac_test.go diff --git a/stream/flac/decode.go b/stream/flac/decode.go new file mode 100644 index 00000000..e69de29b diff --git a/stream/flac/flac_test.go b/stream/flac/flac_test.go new file mode 100644 index 00000000..e69de29b From 631b3a27f2833cc89a0fa0fd89be408d98b68d6f Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 21 Jan 2019 15:41:49 +1030 Subject: [PATCH 138/208] av/stream/flac: wrote decode function and test to see if we can get wav. --- stream/flac/decode.go | 108 +++++++++++++++++++++++++++++++++++++++ stream/flac/flac_test.go | 31 +++++++++++ 2 files changed, 139 insertions(+) diff --git a/stream/flac/decode.go b/stream/flac/decode.go index e69de29b..2ecfb737 100644 --- a/stream/flac/decode.go +++ b/stream/flac/decode.go @@ -0,0 +1,108 @@ +package flac + +import ( + "bytes" + "errors" + "io" + + "github.com/go-audio/audio" + "github.com/go-audio/wav" + "github.com/mewkiz/flac" +) + +const wavAudioFormat = 1 + +type buffer struct { + Buffer bytes.Buffer + Index int64 +} + +func (b *buffer) Bytes() []byte { + return b.Buffer.Bytes() +} + +func (b *buffer) Read(p []byte) (int, error) { + n, err := bytes.NewBuffer(b.Buffer.Bytes()[b.Index:]).Read(p) + + if err == nil { + if b.Index+int64(len(p)) < int64(b.Buffer.Len()) { + b.Index += int64(len(p)) + } else { + b.Index = int64(b.Buffer.Len()) + } + } + + return n, err +} + +func (b *buffer) Write(p []byte) (int, error) { + n, err := b.Buffer.Write(p) + + if err == nil { + b.Index = int64(b.Buffer.Len()) + } + + return n, err +} + +func (b *buffer) Seek(offset int64, whence int) (int64, error) { + var err error + var Index int64 = 0 + + switch whence { + case 0: + if offset >= int64(b.Buffer.Len()) || offset < 0 { + err = errors.New("Invalid Offset.") + } else { + b.Index = offset + Index = offset + } + default: + err = errors.New("Unsupported Seek Method.") + } + + return Index, err +} + +// Decode takes a slice of flac and decodes to wav +func Decode(buf []byte) ([]byte, error) { + r := bytes.NewReader(buf) + stream, err := flac.Parse(r) + if err != nil { + return nil, errors.New("Could not parse FLAC") + } + fb := &buffer{} + enc := wav.NewEncoder(fb, int(stream.Info.SampleRate), int(stream.Info.BitsPerSample), int(stream.Info.NChannels), wavAudioFormat) + defer enc.Close() + var data []int + for { + // Decode FLAC audio samples. + frame, err := stream.ParseNext() + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + + // Encode WAV audio samples. + data = data[:0] + for i := 0; i < frame.Subframes[0].NSamples; i++ { + for _, subframe := range frame.Subframes { + data = append(data, int(subframe.Samples[i])) + } + } + buf := &audio.IntBuffer{ + Format: &audio.Format{ + NumChannels: int(stream.Info.NChannels), + SampleRate: int(stream.Info.SampleRate), + }, + Data: data, + SourceBitDepth: int(stream.Info.BitsPerSample), + } + if err := enc.Write(buf); err != nil { + return nil, err + } + } + return fb.Bytes(), nil +} diff --git a/stream/flac/flac_test.go b/stream/flac/flac_test.go index e69de29b..763731d4 100644 --- a/stream/flac/flac_test.go +++ b/stream/flac/flac_test.go @@ -0,0 +1,31 @@ +package flac + +import ( + "io/ioutil" + "os" + "testing" +) + +const ( + testFile = "/home/saxon/Desktop/robot.flac" + outFile = "out.wav" +) + +func TestDecodeFlac(t *testing.T) { + b, err := ioutil.ReadFile(testFile) + if err != nil { + t.Fatalf("Could not read test file, failed with err: %v", err.Error()) + } + out, err := Decode(b) + if err != nil { + t.Errorf("Could not decode, failed with err: %v", err.Error()) + } + f, err := os.Create(outFile) + if err != nil { + t.Fatalf("Could not create output file, failed with err: %v", err.Error()) + } + _, err = f.Write(out) + if err != nil { + t.Fatalf("Could not write to output file, failed with err: %v", err.Error()) + } +} From 05f5a967fee4c0f78d11f57aa11916f788c9ea46 Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 21 Jan 2019 17:34:15 +1030 Subject: [PATCH 139/208] av/stream/flac: using writerseeker to pass to wav.NewEncoder because I don't want to give it a file, but it's not working --- stream/flac/decode.go | 68 ++++++++-------------------------------- stream/flac/flac_test.go | 2 +- 2 files changed, 14 insertions(+), 56 deletions(-) diff --git a/stream/flac/decode.go b/stream/flac/decode.go index 2ecfb737..941610e2 100644 --- a/stream/flac/decode.go +++ b/stream/flac/decode.go @@ -4,66 +4,16 @@ import ( "bytes" "errors" "io" + "io/ioutil" "github.com/go-audio/audio" "github.com/go-audio/wav" "github.com/mewkiz/flac" + "github.com/orcaman/writerseeker" ) const wavAudioFormat = 1 -type buffer struct { - Buffer bytes.Buffer - Index int64 -} - -func (b *buffer) Bytes() []byte { - return b.Buffer.Bytes() -} - -func (b *buffer) Read(p []byte) (int, error) { - n, err := bytes.NewBuffer(b.Buffer.Bytes()[b.Index:]).Read(p) - - if err == nil { - if b.Index+int64(len(p)) < int64(b.Buffer.Len()) { - b.Index += int64(len(p)) - } else { - b.Index = int64(b.Buffer.Len()) - } - } - - return n, err -} - -func (b *buffer) Write(p []byte) (int, error) { - n, err := b.Buffer.Write(p) - - if err == nil { - b.Index = int64(b.Buffer.Len()) - } - - return n, err -} - -func (b *buffer) Seek(offset int64, whence int) (int64, error) { - var err error - var Index int64 = 0 - - switch whence { - case 0: - if offset >= int64(b.Buffer.Len()) || offset < 0 { - err = errors.New("Invalid Offset.") - } else { - b.Index = offset - Index = offset - } - default: - err = errors.New("Unsupported Seek Method.") - } - - return Index, err -} - // Decode takes a slice of flac and decodes to wav func Decode(buf []byte) ([]byte, error) { r := bytes.NewReader(buf) @@ -71,10 +21,12 @@ func Decode(buf []byte) ([]byte, error) { if err != nil { return nil, errors.New("Could not parse FLAC") } - fb := &buffer{} - enc := wav.NewEncoder(fb, int(stream.Info.SampleRate), int(stream.Info.BitsPerSample), int(stream.Info.NChannels), wavAudioFormat) + ws := &writerseeker.WriterSeeker{} + enc := wav.NewEncoder(ws, int(stream.Info.SampleRate), int(stream.Info.BitsPerSample), int(stream.Info.NChannels), wavAudioFormat) defer enc.Close() var data []int + var out []byte + var d []byte for { // Decode FLAC audio samples. frame, err := stream.ParseNext() @@ -103,6 +55,12 @@ func Decode(buf []byte) ([]byte, error) { if err := enc.Write(buf); err != nil { return nil, err } + d, err = ioutil.ReadAll(ws.Reader()) + if err != nil { + return nil, err + } + out = append(out, d...) } - return fb.Bytes(), nil + + return d, nil } diff --git a/stream/flac/flac_test.go b/stream/flac/flac_test.go index 763731d4..13bef836 100644 --- a/stream/flac/flac_test.go +++ b/stream/flac/flac_test.go @@ -8,7 +8,7 @@ import ( const ( testFile = "/home/saxon/Desktop/robot.flac" - outFile = "out.wav" + outFile = "testOut.wav" ) func TestDecodeFlac(t *testing.T) { From 670ebfeaf8526eb902e23879c1a432f89000b6c2 Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 21 Jan 2019 17:37:16 +1030 Subject: [PATCH 140/208] av/stream/flac: moved readAll to after loop --- stream/flac/decode.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/stream/flac/decode.go b/stream/flac/decode.go index 941610e2..6c8445e4 100644 --- a/stream/flac/decode.go +++ b/stream/flac/decode.go @@ -25,8 +25,6 @@ func Decode(buf []byte) ([]byte, error) { enc := wav.NewEncoder(ws, int(stream.Info.SampleRate), int(stream.Info.BitsPerSample), int(stream.Info.NChannels), wavAudioFormat) defer enc.Close() var data []int - var out []byte - var d []byte for { // Decode FLAC audio samples. frame, err := stream.ParseNext() @@ -55,12 +53,10 @@ func Decode(buf []byte) ([]byte, error) { if err := enc.Write(buf); err != nil { return nil, err } - d, err = ioutil.ReadAll(ws.Reader()) - if err != nil { - return nil, err - } - out = append(out, d...) } - + d, err := ioutil.ReadAll(ws.Reader()) + if err != nil { + return nil, err + } return d, nil } From 40ed9fcee17f13156789ac7d894b4e53b8e58ecd Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 21 Jan 2019 17:50:09 +1030 Subject: [PATCH 141/208] av/stream/flc: using my own writeSeeker implementation - working --- stream/flac/decode.go | 52 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/stream/flac/decode.go b/stream/flac/decode.go index 6c8445e4..667fcf9c 100644 --- a/stream/flac/decode.go +++ b/stream/flac/decode.go @@ -4,16 +4,55 @@ import ( "bytes" "errors" "io" - "io/ioutil" "github.com/go-audio/audio" "github.com/go-audio/wav" "github.com/mewkiz/flac" - "github.com/orcaman/writerseeker" ) const wavAudioFormat = 1 +type WriterSeeker struct { + buf []byte + pos int +} + +func (ws *WriterSeeker) Bytes() []byte { + return ws.buf +} + +func (m *WriterSeeker) Write(p []byte) (n int, err error) { + minCap := m.pos + len(p) + if minCap > cap(m.buf) { // Make sure buf has enough capacity: + buf2 := make([]byte, len(m.buf), minCap+len(p)) // add some extra + copy(buf2, m.buf) + m.buf = buf2 + } + if minCap > len(m.buf) { + m.buf = m.buf[:minCap] + } + copy(m.buf[m.pos:], p) + m.pos += len(p) + return len(p), nil +} + +func (m *WriterSeeker) Seek(offset int64, whence int) (int64, error) { + newPos, offs := 0, int(offset) + switch whence { + case io.SeekStart: + newPos = offs + case io.SeekCurrent: + newPos = m.pos + offs + case io.SeekEnd: + newPos = len(m.buf) + offs + } + if newPos < 0 { + return 0, errors.New("negative result pos") + } + m.pos = newPos + return int64(newPos), nil +} + // Decode takes a slice of flac and decodes to wav func Decode(buf []byte) ([]byte, error) { r := bytes.NewReader(buf) @@ -21,7 +60,7 @@ func Decode(buf []byte) ([]byte, error) { if err != nil { return nil, errors.New("Could not parse FLAC") } - ws := &writerseeker.WriterSeeker{} + ws := &WriterSeeker{} enc := wav.NewEncoder(ws, int(stream.Info.SampleRate), int(stream.Info.BitsPerSample), int(stream.Info.NChannels), wavAudioFormat) defer enc.Close() var data []int @@ -54,9 +93,6 @@ func Decode(buf []byte) ([]byte, error) { return nil, err } } - d, err := ioutil.ReadAll(ws.Reader()) - if err != nil { - return nil, err - } - return d, nil + + return ws.Bytes(), nil } From 3773fc79fa90474bcbd9b207e59807dc2871815d Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 21 Jan 2019 22:52:17 +1030 Subject: [PATCH 142/208] av/stream/flac/decode.go: wrote func headers --- stream/flac/decode.go | 79 ++++++++++++++++++++++++++++++---------- stream/flac/flac_test.go | 26 +++++++++++++ 2 files changed, 85 insertions(+), 20 deletions(-) diff --git a/stream/flac/decode.go b/stream/flac/decode.go index 667fcf9c..42c4dace 100644 --- a/stream/flac/decode.go +++ b/stream/flac/decode.go @@ -1,3 +1,29 @@ +/* +NAME + decode.go + +DESCRIPTION + decode.go provides functionality for the decoding of FLAC compressed audio + +AUTHOR + Saxon Nelson-Milton + +LICENSE + decode.go is Copyright (C) 2017-2019 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ package flac import ( @@ -10,59 +36,72 @@ import ( "github.com/mewkiz/flac" ) -const wavAudioFormat = 1 +const wavFormat = 1 -type WriterSeeker struct { +// writeSeeker implements a memory based io.WriteSeeker. +type writeSeeker struct { buf []byte pos int } -func (ws *WriterSeeker) Bytes() []byte { +// Bytes returns the bytes contained in the writeSeekers buffer. +func (ws *writeSeeker) Bytes() []byte { return ws.buf } -func (m *WriterSeeker) Write(p []byte) (n int, err error) { - minCap := m.pos + len(p) - if minCap > cap(m.buf) { // Make sure buf has enough capacity: - buf2 := make([]byte, len(m.buf), minCap+len(p)) // add some extra - copy(buf2, m.buf) - m.buf = buf2 +// Write writes len(p) bytes from p to the writeSeeker's buf and returns the number +// of bytes written. If less than len(p) bytes are written, an error is returned. +func (ws *writeSeeker) Write(p []byte) (n int, err error) { + minCap := ws.pos + len(p) + if minCap > cap(ws.buf) { // Make sure buf has enough capacity: + buf2 := make([]byte, len(ws.buf), minCap+len(p)) // add some extra + copy(buf2, ws.buf) + ws.buf = buf2 } - if minCap > len(m.buf) { - m.buf = m.buf[:minCap] + if minCap > len(ws.buf) { + ws.buf = ws.buf[:minCap] } - copy(m.buf[m.pos:], p) - m.pos += len(p) + copy(ws.buf[ws.pos:], p) + ws.pos += len(p) return len(p), nil } -func (m *WriterSeeker) Seek(offset int64, whence int) (int64, error) { +// Seek sets the offset for the next Read or Write to offset, interpreted according +// to whence: SeekStart means relative to the start of the file, SeekCurrent means +// relative to the current offset, and SeekEnd means relative to the end. Seek returns +// the new offset relative to the start of the file and an error, if any. +func (ws *writeSeeker) Seek(offset int64, whence int) (int64, error) { newPos, offs := 0, int(offset) switch whence { case io.SeekStart: newPos = offs case io.SeekCurrent: - newPos = m.pos + offs + newPos = ws.pos + offs case io.SeekEnd: - newPos = len(m.buf) + offs + newPos = len(ws.buf) + offs } if newPos < 0 { return 0, errors.New("negative result pos") } - m.pos = newPos + ws.pos = newPos return int64(newPos), nil } -// Decode takes a slice of flac and decodes to wav +// Decode takes buf, a slice of FLAC, and decodes to WAV. If complete decoding +// fails, an error is returned. func Decode(buf []byte) ([]byte, error) { + // Lex and decode the FLAC into a stream to hold audio and properties. r := bytes.NewReader(buf) stream, err := flac.Parse(r) if err != nil { return nil, errors.New("Could not parse FLAC") } - ws := &WriterSeeker{} - enc := wav.NewEncoder(ws, int(stream.Info.SampleRate), int(stream.Info.BitsPerSample), int(stream.Info.NChannels), wavAudioFormat) + + // Create WAV encoder and pass writeSeeker that will store output WAV. + ws := &writeSeeker{} + enc := wav.NewEncoder(ws, int(stream.Info.SampleRate), int(stream.Info.BitsPerSample), int(stream.Info.NChannels), wavFormat) defer enc.Close() + var data []int for { // Decode FLAC audio samples. diff --git a/stream/flac/flac_test.go b/stream/flac/flac_test.go index 13bef836..d69c0494 100644 --- a/stream/flac/flac_test.go +++ b/stream/flac/flac_test.go @@ -1,3 +1,29 @@ +/* +NAME + flac_test.go + +DESCRIPTION + flac_test.go provides utilities to test FLAC audio decoding + +AUTHOR + Saxon Nelson-Milton + +LICENSE + flac_test.go is Copyright (C) 2017-2019 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ package flac import ( From 3ee56bff1a196d67e5338ef3de0e6f98ce1b0475 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 22 Jan 2019 10:26:22 +1030 Subject: [PATCH 143/208] av/stream/flac: working on cleaning up decode code --- stream/flac/decode.go | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/stream/flac/decode.go b/stream/flac/decode.go index 42c4dace..5a470370 100644 --- a/stream/flac/decode.go +++ b/stream/flac/decode.go @@ -90,7 +90,8 @@ func (ws *writeSeeker) Seek(offset int64, whence int) (int64, error) { // Decode takes buf, a slice of FLAC, and decodes to WAV. If complete decoding // fails, an error is returned. func Decode(buf []byte) ([]byte, error) { - // Lex and decode the FLAC into a stream to hold audio and properties. + + // Lex the FLAC into a stream to hold audio and it's properties. r := bytes.NewReader(buf) stream, err := flac.Parse(r) if err != nil { @@ -99,17 +100,30 @@ func Decode(buf []byte) ([]byte, error) { // Create WAV encoder and pass writeSeeker that will store output WAV. ws := &writeSeeker{} - enc := wav.NewEncoder(ws, int(stream.Info.SampleRate), int(stream.Info.BitsPerSample), int(stream.Info.NChannels), wavFormat) + sr := int(stream.Info.SampleRate) + bps := int(stream.Info.BitsPerSample) + nc := int(stream.Info.NChannels) + enc := wav.NewEncoder(ws, sr, bps, nc, wavFormat) defer enc.Close() + // Decode FLAC into frames of samples + intBuf := &audio.IntBuffer{ + Format: &audio.Format{NumChannels: nc, SampleRate: sr}, + SourceBitDepth: bps, + } + return decodeFrames(stream, intBuf, enc, ws) +} + +// +func decodeFrames(s *flac.Stream, intBuf *audio.IntBuffer, e *wav.Encoder, ws *writeSeeker) ([]byte, error) { var data []int for { - // Decode FLAC audio samples. - frame, err := stream.ParseNext() - if err != nil { - if err == io.EOF { - break - } + frame, err := s.ParseNext() + + // If we've reached the end of the stream then we can output the writeSeeker's buffer. + if err == io.EOF { + return ws.Bytes(), nil + } else if err != nil { return nil, err } @@ -120,18 +134,9 @@ func Decode(buf []byte) ([]byte, error) { data = append(data, int(subframe.Samples[i])) } } - buf := &audio.IntBuffer{ - Format: &audio.Format{ - NumChannels: int(stream.Info.NChannels), - SampleRate: int(stream.Info.SampleRate), - }, - Data: data, - SourceBitDepth: int(stream.Info.BitsPerSample), - } - if err := enc.Write(buf); err != nil { + intBuf.Data = data + if err := e.Write(intBuf); err != nil { return nil, err } } - - return ws.Bytes(), nil } From da5e13bb5dbcf8c3300e8d3791e2ba039657e287 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 22 Jan 2019 10:40:40 +1030 Subject: [PATCH 144/208] av/stream/flac: finished cleaning up decode --- stream/flac/decode.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stream/flac/decode.go b/stream/flac/decode.go index 5a470370..34d42057 100644 --- a/stream/flac/decode.go +++ b/stream/flac/decode.go @@ -114,7 +114,9 @@ func Decode(buf []byte) ([]byte, error) { return decodeFrames(stream, intBuf, enc, ws) } -// +// decodeFrames parses frames from the stream and encodes them into WAV until +// the end of the stream is reached. The bytes from writeSeeker buffer are then +// returned. If any errors occur during encodeing, nil bytes and the error is returned. func decodeFrames(s *flac.Stream, intBuf *audio.IntBuffer, e *wav.Encoder, ws *writeSeeker) ([]byte, error) { var data []int for { From e557734c832731a87fcb3fc327926b0c926a2013 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 22 Jan 2019 10:45:36 +1030 Subject: [PATCH 145/208] av/stream/flac: added writeseeker tests --- stream/flac/flac_test.go | 44 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/stream/flac/flac_test.go b/stream/flac/flac_test.go index d69c0494..9537d682 100644 --- a/stream/flac/flac_test.go +++ b/stream/flac/flac_test.go @@ -27,6 +27,7 @@ LICENSE package flac import ( + "io" "io/ioutil" "os" "testing" @@ -37,6 +38,49 @@ const ( outFile = "testOut.wav" ) +func TestWriteSeekerWrite(t *testing.T) { + writerSeeker := &writeSeeker{} + var ws io.WriteSeeker = writerSeeker + + ws.Write([]byte("hello")) + if string(writerSeeker.buf) != "hello" { + t.Fail() + } + + ws.Write([]byte(" world")) + if string(writerSeeker.buf) != "hello world" { + t.Fail() + } + +} + +func TestWriteSeekerSeek(t *testing.T) { + writerSeeker := &writeSeeker{} + var ws io.WriteSeeker = writerSeeker + + ws.Write([]byte("hello")) + if string(writerSeeker.buf) != "hello" { + t.Fail() + } + + ws.Write([]byte(" world")) + if string(writerSeeker.buf) != "hello world" { + t.Fail() + } + + ws.Seek(-2, io.SeekEnd) + ws.Write([]byte("k!")) + if string(writerSeeker.buf) != "hello work!" { + t.Fail() + } + + ws.Seek(6, io.SeekStart) + ws.Write([]byte("gopher")) + if string(writerSeeker.buf) != "hello gopher" { + t.Fail() + } +} + func TestDecodeFlac(t *testing.T) { b, err := ioutil.ReadFile(testFile) if err != nil { From 1104c96a2fb1ac5e3f224d9ee4f53dc94a51f0c7 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 22 Jan 2019 11:15:39 +1030 Subject: [PATCH 146/208] av/stream/flac: saving progress --- stream/flac/flac_test.go | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/stream/flac/flac_test.go b/stream/flac/flac_test.go index 9537d682..79274819 100644 --- a/stream/flac/flac_test.go +++ b/stream/flac/flac_test.go @@ -38,45 +38,51 @@ const ( outFile = "testOut.wav" ) +// TestWriteSeekerWrite checks that basic writing to the ws works as expected. func TestWriteSeekerWrite(t *testing.T) { - writerSeeker := &writeSeeker{} - var ws io.WriteSeeker = writerSeeker + ws := &writeSeeker{} - ws.Write([]byte("hello")) - if string(writerSeeker.buf) != "hello" { - t.Fail() + const tstStr1 = "hello" + ws.Write([]byte(tstStr1)) + got := string(ws.buf) + if got != tstStr1 { + t.Errorf("Write failed, got: %v, want: %v", got, tstStr1) } - ws.Write([]byte(" world")) - if string(writerSeeker.buf) != "hello world" { - t.Fail() + const tstStr2 = " world" + const want = "hello world" + ws.Write([]byte(tstStr2)) + got = string(ws.buf) + if got != want { + t.Errorf("Second write failed, got: %v, want: %v", got, want) } - } +// TestWriteSeekerSeek checks that writing and seeking works as expected, i.e. we +// can write, then seek to a knew place in the buf, and write again, either replacing +// bytes, or appending bytes. func TestWriteSeekerSeek(t *testing.T) { - writerSeeker := &writeSeeker{} - var ws io.WriteSeeker = writerSeeker + ws := &writeSeeker{} ws.Write([]byte("hello")) - if string(writerSeeker.buf) != "hello" { + if string(ws.buf) != "hello" { t.Fail() } ws.Write([]byte(" world")) - if string(writerSeeker.buf) != "hello world" { + if string(ws.buf) != "hello world" { t.Fail() } ws.Seek(-2, io.SeekEnd) ws.Write([]byte("k!")) - if string(writerSeeker.buf) != "hello work!" { + if string(ws.buf) != "hello work!" { t.Fail() } ws.Seek(6, io.SeekStart) ws.Write([]byte("gopher")) - if string(writerSeeker.buf) != "hello gopher" { + if string(ws.buf) != "hello gopher" { t.Fail() } } From 3f41c7b72bb7db6d76dab53b34f55d568c99c80b Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 22 Jan 2019 12:14:40 +1030 Subject: [PATCH 147/208] av/stream/flac: cleaned up testing file --- stream/flac/flac_test.go | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/stream/flac/flac_test.go b/stream/flac/flac_test.go index 79274819..0d8079f7 100644 --- a/stream/flac/flac_test.go +++ b/stream/flac/flac_test.go @@ -64,29 +64,43 @@ func TestWriteSeekerWrite(t *testing.T) { func TestWriteSeekerSeek(t *testing.T) { ws := &writeSeeker{} - ws.Write([]byte("hello")) - if string(ws.buf) != "hello" { - t.Fail() + const tstStr1 = "hello" + want1 := tstStr1 + ws.Write([]byte(tstStr1)) + got := string(ws.buf) + if got != tstStr1 { + t.Errorf("Unexpected output, got: %v, want: %v", got, want1) } - ws.Write([]byte(" world")) - if string(ws.buf) != "hello world" { - t.Fail() + const tstStr2 = " world" + const want2 = tstStr1 + tstStr2 + ws.Write([]byte(tstStr2)) + got = string(ws.buf) + if got != want2 { + t.Errorf("Unexpected output, got: %v, want: %v", got, want2) } + const tstStr3 = "k!" + const want3 = "hello work!" ws.Seek(-2, io.SeekEnd) - ws.Write([]byte("k!")) - if string(ws.buf) != "hello work!" { - t.Fail() + ws.Write([]byte(tstStr3)) + got = string(ws.buf) + if got != want3 { + t.Errorf("Unexpected output, got: %v, want: %v", got, want3) } + const tstStr4 = "gopher" + const want4 = "hello gopher" ws.Seek(6, io.SeekStart) - ws.Write([]byte("gopher")) - if string(ws.buf) != "hello gopher" { - t.Fail() + ws.Write([]byte(tstStr4)) + got = string(ws.buf) + if got != want4 { + t.Errorf("Unexpected output, got: %v, want: %v", got, want4) } } +// TestDecodeFlac checks that we can load a flac file and decode to wav, writing +// to a wav file. func TestDecodeFlac(t *testing.T) { b, err := ioutil.ReadFile(testFile) if err != nil { From 5b131c80f6a65eb060add7611032c63cc7eb9279 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 22 Jan 2019 12:27:52 +1030 Subject: [PATCH 148/208] av/stream/flac: updated test file directory --- stream/flac/flac_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stream/flac/flac_test.go b/stream/flac/flac_test.go index 0d8079f7..1f8019e5 100644 --- a/stream/flac/flac_test.go +++ b/stream/flac/flac_test.go @@ -34,7 +34,7 @@ import ( ) const ( - testFile = "/home/saxon/Desktop/robot.flac" + testFile = "../../../test/test-data/av/input/robot.flac" outFile = "testOut.wav" ) From e98d7bb62e74f32b5d7790a05710d5ebb146bfcf Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 7 Feb 2019 19:58:08 +1030 Subject: [PATCH 149/208] created experimentation dir under av, and moved flac package here. created experimentation dir under av, and moved flac pkg here. experimentation/flac: removed wav file --- {stream => experimentation}/flac/decode.go | 0 {stream => experimentation}/flac/flac_test.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {stream => experimentation}/flac/decode.go (100%) rename {stream => experimentation}/flac/flac_test.go (100%) diff --git a/stream/flac/decode.go b/experimentation/flac/decode.go similarity index 100% rename from stream/flac/decode.go rename to experimentation/flac/decode.go diff --git a/stream/flac/flac_test.go b/experimentation/flac/flac_test.go similarity index 100% rename from stream/flac/flac_test.go rename to experimentation/flac/flac_test.go From 19821553e869f0a4f353132dd9f3640b4ba8d9aa Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 8 Feb 2019 00:14:22 +1030 Subject: [PATCH 150/208] cmd/ts-repair: added required consts and undid changes to mts pkg --- cmd/ts-repair/main.go | 43 ++++++++++++++++++++++++------------- stream/mts/encoder.go | 39 ++++++++++++++++++++++------------ stream/mts/mpegts.go | 49 +------------------------------------------ 3 files changed, 56 insertions(+), 75 deletions(-) diff --git a/cmd/ts-repair/main.go b/cmd/ts-repair/main.go index f3f73bb2..f95d16fe 100644 --- a/cmd/ts-repair/main.go +++ b/cmd/ts-repair/main.go @@ -11,6 +11,21 @@ import ( "github.com/Comcast/gots/packet" ) +const ( + PatPid = 0 + PmtPid = 4096 + VideoPid = 256 + HeadSize = 4 + DefaultAdaptationSize = 2 + AdaptationIdx = 4 + AdaptationControlIdx = 3 + AdaptationBodyIdx = AdaptationIdx + 1 + AdaptationControlMask = 0x30 + DefaultAdaptationBodySize = 1 + DiscontinuityIndicatorMask = 0x80 + DiscontinuityIndicatorIdx = AdaptationIdx + 1 +) + // Various errors that we can encounter. const ( errBadInPath = "No file path provided, or file does not exist" @@ -37,9 +52,9 @@ const ( ) var ccMap = map[int]byte{ - mts.PatPid: 16, - mts.PmtPid: 16, - mts.VideoPid: 16, + PatPid: 16, + PmtPid: 16, + VideoPid: 16, } // packetNo will keep track of the ts packet number for reference. @@ -48,8 +63,8 @@ var packetNo int // Option defines a func that performs an action on p in order to change a ts option. type Option func(p *Packet) -// Packet is a byte array of size mts.PacketSize i.e. 188 bytes. We define this -// to allow us to write receiver funcs for the [mts.PacketSize]byte type. +// Packet is a byte array of size PacketSize i.e. 188 bytes. We define this +// to allow us to write receiver funcs for the [PacketSize]byte type. type Packet [mts.PacketSize]byte // CC returns the CC of p. @@ -78,12 +93,12 @@ func (p *Packet) addAdaptationField(options ...Option) error { return errors.New(errAdaptationPresent) } // Create space for adaptation field. - copy(p[mts.HeadSize+mts.DefaultAdaptationSize:], p[mts.HeadSize:len(p)-mts.DefaultAdaptationSize]) + copy(p[HeadSize+DefaultAdaptationSize:], p[HeadSize:len(p)-DefaultAdaptationSize]) // TODO: seperate into own function // Update adaptation field control. - p[mts.AdaptationControlIdx] &= 0xff ^ mts.AdaptationControlMask - p[mts.AdaptationControlIdx] |= mts.AdaptationControlMask + p[AdaptationControlIdx] &= 0xff ^ AdaptationControlMask + p[AdaptationControlIdx] |= AdaptationControlMask // Default the adaptationfield. p.resetAdaptation() @@ -100,14 +115,14 @@ func (p *Packet) resetAdaptation() error { if !p.hasAdaptation() { return errors.New(errNoAdaptationField) } - p[mts.AdaptationIdx] = mts.DefaultAdaptationBodySize - p[mts.AdaptationBodyIdx] = 0x00 + p[AdaptationIdx] = DefaultAdaptationBodySize + p[AdaptationBodyIdx] = 0x00 return nil } // hasAdaptation returns true if p has an adaptation field and false otherwise. func (p *Packet) hasAdaptation() bool { - afc := p[mts.AdaptationControlIdx] & mts.AdaptationControlMask + afc := p[AdaptationControlIdx] & AdaptationControlMask if afc == 0x20 || afc == 0x30 { return true } else { @@ -119,12 +134,12 @@ func (p *Packet) hasAdaptation() bool { // indicator according to f. func DiscontinuityIndicator(f bool) Option { return func(p *Packet) { - set := byte(mts.DiscontinuityIndicatorMask) + set := byte(DiscontinuityIndicatorMask) if !f { set = 0x00 } - p[mts.DiscontinuityIndicatorIdx] &= 0xff ^ mts.DiscontinuityIndicatorMask - p[mts.DiscontinuityIndicatorIdx] |= mts.DiscontinuityIndicatorMask & set + p[DiscontinuityIndicatorIdx] &= 0xff ^ DiscontinuityIndicatorMask + p[DiscontinuityIndicatorIdx] |= DiscontinuityIndicatorMask & set } } diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index e49d57b5..02761b91 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -171,6 +171,14 @@ var ( pmtTable = standardPmtTimeLocation.Bytes() ) +const ( + sdtPid = 17 + patPid = 0 + pmtPid = 4096 + videoPid = 256 + streamID = 0xe0 // First video stream ID. +) + // Time related constants. const ( // ptsOffset is the offset added to the clock to determine @@ -205,13 +213,18 @@ func NewEncoder(dst io.Writer, fps float64) *Encoder { ptsOffset: ptsOffset, continuity: map[int]byte{ - PatPid: 0, - PmtPid: 0, - VideoPid: 0, + patPid: 0, + pmtPid: 0, + videoPid: 0, }, } } +const ( + hasPayload = 0x1 + hasAdaptationField = 0x2 +) + const ( hasDTS = 0x1 hasPTS = 0x2 @@ -231,7 +244,7 @@ func (e *Encoder) Encode(nalu []byte) error { // Prepare PES data. pesPkt := pes.Packet{ - StreamID: StreamID, + StreamID: streamID, PDI: hasPTS, PTS: e.pts(), Data: nalu, @@ -243,10 +256,10 @@ func (e *Encoder) Encode(nalu []byte) error { for len(buf) != 0 { pkt := Packet{ PUSI: pusi, - PID: VideoPid, + PID: videoPid, RAI: pusi, - CC: e.ccFor(VideoPid), - AFC: HasAdaptationField | HasPayload, + CC: e.ccFor(videoPid), + AFC: hasAdaptationField | hasPayload, PCRF: pusi, } n := pkt.FillPayload(buf) @@ -275,9 +288,9 @@ func (e *Encoder) writePSI() error { // Write PAT. patPkt := Packet{ PUSI: true, - PID: PatPid, - CC: e.ccFor(PatPid), - AFC: HasPayload, + PID: patPid, + CC: e.ccFor(patPid), + AFC: hasPayload, Payload: patTable, } _, err := e.dst.Write(patPkt.Bytes(e.tsSpace[:PacketSize])) @@ -298,9 +311,9 @@ func (e *Encoder) writePSI() error { // Create mts packet from pmt table. pmtPkt := Packet{ PUSI: true, - PID: PmtPid, - CC: e.ccFor(PmtPid), - AFC: HasPayload, + PID: pmtPid, + CC: e.ccFor(pmtPid), + AFC: hasPayload, Payload: pmtTable, } _, err = e.dst.Write(pmtPkt.Bytes(e.tsSpace[:PacketSize])) diff --git a/stream/mts/mpegts.go b/stream/mts/mpegts.go index 71849513..0bef80d2 100644 --- a/stream/mts/mpegts.go +++ b/stream/mts/mpegts.go @@ -37,53 +37,6 @@ const ( PayloadSize = 176 ) -const ( - SdtPid = 17 - PatPid = 0 - PmtPid = 4096 - VideoPid = 256 - StreamID = 0xe0 // First video stream ID. - HeadSize = 4 - DefaultAdaptationSize = 2 -) - -const ( - AdaptationIdx = 4 - AdaptationControlIdx = 3 - AdaptationBodyIdx = AdaptationIdx + 1 - AdaptationControlMask = 0x30 - DefaultAdaptationBodySize = 1 -) - -const ( - HasPayload = 0x1 - HasAdaptationField = 0x2 -) - -// Adaptation field body masks. -const ( - DiscontinuityIndicatorMask = 0x80 - RandomAccessIndicatorMask = 0x40 - ElementaryStreamPriorityIndicatorMask = 0x20 - ProgramClockReferenceFlagMask = 0x10 - OriginalProgramClockReferenceFlagMask = 0x08 - SplicingPointFlagMask = 0x04 - TransportPrivateDataFlagMask = 0x02 - AdaptationFieldExtensionMask = 0x01 -) - -// Adaptation field body indexes. -const ( - DiscontinuityIndicatorIdx = AdaptationIdx + 1 - RandomAccessIndicatorIdx = AdaptationIdx + 1 - ElementaryStreamPriorityIndicatorIdx = AdaptationIdx + 1 - ProgramClockReferenceFlagIdx = AdaptationIdx + 1 - OriginalProgramClockReferenceFlagIdx = AdaptationIdx + 1 - SplicingPointFlagIdx = AdaptationIdx + 1 - TransportPrivateDataFlagIdx = AdaptationIdx + 1 - AdaptationFieldExtensionFlagIdx = AdaptationIdx + 1 -) - /* The below data struct encapsulates the fields of an MPEG-TS packet. Below is the formatting of an MPEG-TS packet for reference! @@ -182,7 +135,7 @@ func FindPMT(d []byte) (p []byte, i int, err error) { } for i = 0; i < len(d); i += PacketSize { pid := (uint16(d[i+1]&0x1f) << 8) | uint16(d[i+2]) - if pid == PmtPid { + if pid == pmtPid { p = d[i+4 : i+PacketSize] return } From 7b789aed29939b29533df36c9b951f6e0163b6f1 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 8 Feb 2019 00:16:19 +1030 Subject: [PATCH 151/208] moved cmd/ts-repair to experimentation --- {cmd => experimentation}/ts-repair/main.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {cmd => experimentation}/ts-repair/main.go (100%) diff --git a/cmd/ts-repair/main.go b/experimentation/ts-repair/main.go similarity index 100% rename from cmd/ts-repair/main.go rename to experimentation/ts-repair/main.go From 383b2962af0a7440184813068602456e17055b86 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 8 Feb 2019 00:25:47 +1030 Subject: [PATCH 152/208] experimentation/ts-repair: added description to file header --- experimentation/ts-repair/main.go | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/experimentation/ts-repair/main.go b/experimentation/ts-repair/main.go index f95d16fe..bed81f19 100644 --- a/experimentation/ts-repair/main.go +++ b/experimentation/ts-repair/main.go @@ -1,3 +1,36 @@ +/* +NAME + ts-repair/main.go + +DESCRIPTION + This program attempts to repair mpegts discontinuities using one of two methods + as selected by the mode flag. Setting the mode flag to 0 will result in repair + by shifting all CCs such that they are continuous. Setting the mode flag to 1 + will result in repair through setting the discontinuity indicator to true at + packets where a discontinuity exists. + + Specify the input file with the in flag, and the output file with out flag. + +AUTHOR + Saxon A. Nelson-Milton + +LICENSE + mpegts.go is Copyright (C) 2017 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with revid in gpl.txt. If not, see [GNU licenses](http://www.gnu.org/licenses). +*/ + package main import ( From 795157577139d0fc37d43ed0eae78bf933a4257b Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 8 Feb 2019 18:00:23 +1030 Subject: [PATCH 153/208] stream/mts: undo changes to encoder.go stream/rtp/encoder.go: undoing changes --- stream/mts/encoder.go | 118 +++++++++--------------------------------- stream/rtp/encoder.go | 4 +- 2 files changed, 26 insertions(+), 96 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 02761b91..a309eaf9 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -30,9 +30,9 @@ package mts import ( "io" - "sync" "time" + "bitbucket.org/ausocean/av/stream/mts/meta" "bitbucket.org/ausocean/av/stream/mts/pes" "bitbucket.org/ausocean/av/stream/mts/psi" ) @@ -82,93 +82,21 @@ var ( }, }, } - - // standardPmtTimeLocation is a standard PMT with time and location - // descriptors, but time and location fields zeroed out. - standardPmtTimeLocation = psi.PSI{ - Pf: 0x00, - Tid: 0x02, - Ssi: true, - Sl: 0x3e, - Tss: &psi.TSS{ - Tide: 0x01, - V: 0, - Cni: true, - Sn: 0, - Lsn: 0, - Sd: &psi.PMT{ - Pcrpid: 0x0100, - Pil: psi.PmtTimeLocationPil, - Pd: []psi.Desc{ - { - Dt: psi.TimeDescTag, - Dl: psi.TimeDataSize, - Dd: make([]byte, psi.TimeDataSize), - }, - { - Dt: psi.LocationDescTag, - Dl: psi.LocationDataSize, - Dd: make([]byte, psi.LocationDataSize), - }, - }, - Essd: &psi.ESSD{ - St: 0x1b, - Epid: 0x0100, - Esil: 0x00, - }, - }, - }, - } ) const ( psiInterval = 1 * time.Second ) -// timeLocation holds time and location data -type timeLocation struct { - mu sync.RWMutex - time uint64 - location string -} - -// SetTimeStamp sets the time field of a TimeLocation. -func (tl *timeLocation) SetTimeStamp(t uint64) { - tl.mu.Lock() - tl.time = t - tl.mu.Unlock() -} - -// GetTimeStamp returns the location of a TimeLocation. -func (tl *timeLocation) TimeStamp() uint64 { - tl.mu.RLock() - t := tl.time - tl.mu.RUnlock() - return t -} - -// SetLocation sets the location of a TimeLocation. -func (tl *timeLocation) SetLocation(l string) { - tl.mu.Lock() - tl.location = l - tl.mu.Unlock() -} - -// GetLocation returns the location of a TimeLocation. -func (tl *timeLocation) Location() string { - tl.mu.RLock() - l := tl.location - tl.mu.RUnlock() - return l -} - -// MetData will hold time and location data which may be set externally if -// this data is available. It is then inserted into mpegts packets outputted. -var MetaData timeLocation +// Meta allows addition of metadata to encoded mts from outside of this pkg. +// See meta pkg for usage. +// +// TODO: make this not global. +var Meta *meta.Data var ( patTable = standardPat.Bytes() - pmtTable = standardPmtTimeLocation.Bytes() + pmtTable = standardPmt.Bytes() ) const ( @@ -288,33 +216,27 @@ func (e *Encoder) writePSI() error { // Write PAT. patPkt := Packet{ PUSI: true, - PID: patPid, - CC: e.ccFor(patPid), - AFC: hasPayload, - Payload: patTable, + PID: PatPid, + CC: e.ccFor(PatPid), + AFC: HasPayload, + Payload: psi.AddPadding(patTable), } _, err := e.dst.Write(patPkt.Bytes(e.tsSpace[:PacketSize])) if err != nil { return err } - - // Update pmt table time and location. - err = psi.UpdateTime(pmtTable, MetaData.TimeStamp()) + pmtTable, err = updateMeta(pmtTable) if err != nil { return err } - err = psi.UpdateLocation(pmtTable, MetaData.Location()) - if err != nil { - return nil - } // Create mts packet from pmt table. pmtPkt := Packet{ PUSI: true, - PID: pmtPid, - CC: e.ccFor(pmtPid), - AFC: hasPayload, - Payload: pmtTable, + PID: PmtPid, + CC: e.ccFor(PmtPid), + AFC: HasPayload, + Payload: psi.AddPadding(pmtTable), } _, err = e.dst.Write(pmtPkt.Bytes(e.tsSpace[:PacketSize])) if err != nil { @@ -345,3 +267,11 @@ func (e *Encoder) ccFor(pid int) byte { e.continuity[pid] = (cc + 1) & continuityCounterMask return cc } + +// updateMeta adds/updates a metaData descriptor in the given psi bytes using data +// contained in the global Meta struct. +func updateMeta(b []byte) ([]byte, error) { + p := psi.PSIBytes(b) + err := p.AddDescriptor(psi.MetadataTag, Meta.Encode()) + return []byte(p), err +} diff --git a/stream/rtp/encoder.go b/stream/rtp/encoder.go index 329a24c0..20df9434 100644 --- a/stream/rtp/encoder.go +++ b/stream/rtp/encoder.go @@ -73,8 +73,8 @@ func NewEncoder(dst io.Writer, fps int) *Encoder { func (e *Encoder) Write(data []byte) (int, error) { e.buffer = append(e.buffer, data...) for len(e.buffer) >= sendLen { - e.Encode(e.buffer) - e.buffer = e.buffer[:0] + e.Encode(e.buffer[:sendLen]) + e.buffer = e.buffer[sendLen:] } return len(data), nil } From 020428db620cebb1d0072bdc2129d4a5aae2f4cc Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 8 Feb 2019 18:12:03 +1030 Subject: [PATCH 154/208] revid/revid.go: checking err from ring buffer write within destinations length check --- revid/revid.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/revid/revid.go b/revid/revid.go index 454a3b8a..aa81e8fd 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -145,10 +145,17 @@ func (p *packer) Write(frame []byte) (int, error) { p.owner.config.Logger.Log(logger.Warning, pkg+"frame was too big", "frame size", len(frame)) return len(frame), nil } - var n int - var err error + if len(p.owner.destination) != 0 { - n, err = p.owner.buffer.Write(frame) + n, err := p.owner.buffer.Write(frame) + if err != nil { + if err == ring.ErrDropped { + p.owner.config.Logger.Log(logger.Warning, pkg+"dropped frame", "frame size", len(frame)) + return len(frame), nil + } + p.owner.config.Logger.Log(logger.Error, pkg+"unexpected ring buffer write error", "error", err.Error()) + return n, err + } } // If we have an rtp sender bypass ringbuffer and give straight to sender @@ -158,14 +165,7 @@ func (p *packer) Write(frame []byte) (int, error) { p.owner.config.Logger.Log(logger.Error, pkg+"rtp send failed with error", "error", err.Error()) } } - if err != nil { - if err == ring.ErrDropped { - p.owner.config.Logger.Log(logger.Warning, pkg+"dropped frame", "frame size", len(frame)) - return len(frame), nil - } - p.owner.config.Logger.Log(logger.Error, pkg+"unexpected ring buffer write error", "error", err.Error()) - return n, err - } + p.packetCount++ var hasRtmp bool for _, d := range p.owner.config.Outputs { From ad04893432ea2c90ad73c5185fbe9b6ba2c94c60 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 8 Feb 2019 18:14:33 +1030 Subject: [PATCH 155/208] revid/revid.go: intialising err from rtp send --- revid/revid.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/revid/revid.go b/revid/revid.go index 6d99ac76..e426272a 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -160,7 +160,7 @@ func (p *packer) Write(frame []byte) (int, error) { // If we have an rtp sender bypass ringbuffer and give straight to sender if p.owner.rtpSender != nil { - err = p.owner.rtpSender.send(frame) + err := p.owner.rtpSender.send(frame) if err != nil { p.owner.config.Logger.Log(logger.Error, pkg+"rtp send failed with error", "error", err.Error()) } From 5090056972a242fa4a80c4fdb6f0acc34700cd5e Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 8 Feb 2019 18:19:44 +1030 Subject: [PATCH 156/208] cmd/revid-cli: send-retry cmd line flag is now retry --- cmd/revid-cli/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 577896a8..6ca6ba68 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -137,7 +137,7 @@ func handleFlags() revid.Config { rtpAddrPtr = flag.String("RtpAddr", "", "Rtp destination address: : (port is generally 6970-6999)") logPathPtr = flag.String("LogPath", defaultLogPath, "The log path") configFilePtr = flag.String("ConfigFile", "", "NetSender config file") - sendRetryPtr = flag.Bool("send-retry", false, "If true, we retry send on a failure, otherwise drop the data.") + sendRetryPtr = flag.Bool("retry", false, "Specify whether a failed send should be retried.") ) var outputs flagStrings From d5a26a98ff60cdb1951f3532d26190b746b29e49 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 8 Feb 2019 20:20:19 +1030 Subject: [PATCH 157/208] stream/rtp/encoder.go: more sensible use of encoder buffer --- stream/rtp/encoder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stream/rtp/encoder.go b/stream/rtp/encoder.go index 20df9434..329a24c0 100644 --- a/stream/rtp/encoder.go +++ b/stream/rtp/encoder.go @@ -73,8 +73,8 @@ func NewEncoder(dst io.Writer, fps int) *Encoder { func (e *Encoder) Write(data []byte) (int, error) { e.buffer = append(e.buffer, data...) for len(e.buffer) >= sendLen { - e.Encode(e.buffer[:sendLen]) - e.buffer = e.buffer[sendLen:] + e.Encode(e.buffer) + e.buffer = e.buffer[:0] } return len(data), nil } From fadc1fed1b706148359d9811884b845b46a14268 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 8 Feb 2019 21:31:00 +1030 Subject: [PATCH 158/208] stream/mts/meta: added ExtractAll func and added testing utilities --- stream/mts/meta/meta.go | 45 ++++++++++++++++++++++++++++++------ stream/mts/meta/meta_test.go | 23 ++++++++++++++++++ 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go index 66790315..a4db5da4 100644 --- a/stream/mts/meta/meta.go +++ b/stream/mts/meta/meta.go @@ -49,9 +49,10 @@ const ( ) var ( - errKeyAbsent = errors.New("Key does not exist in map") - errNoHeader = errors.New("Metadata string does not contain header") - errInvalidHeader = errors.New("Metadata string does not contain valid header") + errKeyAbsent = errors.New("Key does not exist in map") + errNoHeader = errors.New("Metadata string does not contain header") + errInvalidHeader = errors.New("Metadata string does not contain valid header") + errUnexpectedMetaFormat = errors.New("Unexpected meta format") ) // Metadata provides functionality for the storage and encoding of metadata @@ -168,10 +169,9 @@ func (m *Data) Encode() []byte { // key is not present in the metadata string, an error is returned. If the // metadata header is not present in the string, an error is returned. func Extract(key string, d []byte) (string, error) { - if d[0] != 0 { - return "", errNoHeader - } else if d[0] == 0 && binary.BigEndian.Uint16(d[2:headSize]) != uint16(len(d[headSize:])) { - return "", errInvalidHeader + err := checkHeader(d) + if err != nil { + return "", err } d = d[headSize:] entries := strings.Split(string(d), "\t") @@ -183,3 +183,34 @@ func Extract(key string, d []byte) (string, error) { } return "", errKeyAbsent } + +// ExtractAll extracts all metadata entries from given data. An Error is returned +// if the metadata does not have a valid header, or if the meta format is unexpected. +func ExtractAll(d []byte) ([][2]string, error) { + err := checkHeader(d) + if err != nil { + return nil, err + } + d = d[headSize:] + entries := strings.Split(string(d), "\t") + all := make([][2]string, len(entries)) + for i, entry := range entries { + kv := strings.Split(entry, "=") + if len(kv) != 2 { + return nil, errUnexpectedMetaFormat + } + copy(all[i][:], kv) + } + return all, nil +} + +// checkHeader checks that a valid metadata header exists in the given data. An +// error is returned if the header is absent, or if the header is not valid. +func checkHeader(d []byte) error { + if d[0] != 0 { + return errNoHeader + } else if d[0] == 0 && binary.BigEndian.Uint16(d[2:headSize]) != uint16(len(d[headSize:])) { + return errInvalidHeader + } + return nil +} diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go index e1f9f3b7..f809b172 100644 --- a/stream/mts/meta/meta_test.go +++ b/stream/mts/meta/meta_test.go @@ -165,3 +165,26 @@ func TestReadFrom(t *testing.T) { } } } + +// TestExtractAll checks that meta.ExtractAll can correctly extract all metadata +// from descriptor data. +func TestExtractAll(t *testing.T) { + tstMeta := append([]byte{0x00, 0x10, 0x00, 0x12}, "loc=a,b,c\tts=12345"...) + want := [][2]string{ + { + "loc", + "a,b,c", + }, + { + "ts", + "12345", + }, + } + got, err := ExtractAll(tstMeta) + if err != nil { + t.Errorf("Unexpected error: %v\n", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("Did not get expected out. \nGot : %v, \nWant: %v\n", got, want) + } +} From f9d8accdae77de44239c7348a5225e725576496f Mon Sep 17 00:00:00 2001 From: saxon Date: Sat, 9 Feb 2019 12:14:32 +1030 Subject: [PATCH 159/208] stream/mts/meta/meta.go: Extract and ExtractAll to Get and GetAll --- stream/mts/meta/meta.go | 4 ++-- stream/mts/meta/meta_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go index a4db5da4..52e81465 100644 --- a/stream/mts/meta/meta.go +++ b/stream/mts/meta/meta.go @@ -168,7 +168,7 @@ func (m *Data) Encode() []byte { // ReadFrom extracts a value from a metadata string d, for the given key. If the // key is not present in the metadata string, an error is returned. If the // metadata header is not present in the string, an error is returned. -func Extract(key string, d []byte) (string, error) { +func Get(key string, d []byte) (string, error) { err := checkHeader(d) if err != nil { return "", err @@ -186,7 +186,7 @@ func Extract(key string, d []byte) (string, error) { // ExtractAll extracts all metadata entries from given data. An Error is returned // if the metadata does not have a valid header, or if the meta format is unexpected. -func ExtractAll(d []byte) ([][2]string, error) { +func GetAll(d []byte) ([][2]string, error) { err := checkHeader(d) if err != nil { return nil, err diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go index f809b172..e5f6a4ff 100644 --- a/stream/mts/meta/meta_test.go +++ b/stream/mts/meta/meta_test.go @@ -156,7 +156,7 @@ func TestReadFrom(t *testing.T) { } for _, test := range tests { - got, err := Extract(test.key, []byte(tstMeta)) + got, err := Get(test.key, []byte(tstMeta)) if err != nil { t.Errorf("Unexpected err: %v\n", err) } @@ -180,7 +180,7 @@ func TestExtractAll(t *testing.T) { "12345", }, } - got, err := ExtractAll(tstMeta) + got, err := GetAll(tstMeta) if err != nil { t.Errorf("Unexpected error: %v\n", err) } From e796a5a3b7dae48b9291c800d1663ff4f5fa4128 Mon Sep 17 00:00:00 2001 From: saxon Date: Sat, 9 Feb 2019 12:16:57 +1030 Subject: [PATCH 160/208] stream/mts/meta: updating function comments and test function names according to Extract->Get and ExtractAll->GetAll change --- stream/mts/meta/meta.go | 4 ++-- stream/mts/meta/meta_test.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go index 52e81465..29ed0a06 100644 --- a/stream/mts/meta/meta.go +++ b/stream/mts/meta/meta.go @@ -165,7 +165,7 @@ func (m *Data) Encode() []byte { return m.enc } -// ReadFrom extracts a value from a metadata string d, for the given key. If the +// ReadFrom gets a value from a metadata string d, for the given key. If the // key is not present in the metadata string, an error is returned. If the // metadata header is not present in the string, an error is returned. func Get(key string, d []byte) (string, error) { @@ -184,7 +184,7 @@ func Get(key string, d []byte) (string, error) { return "", errKeyAbsent } -// ExtractAll extracts all metadata entries from given data. An Error is returned +// GetAll gets all metadata entries from given data. An Error is returned // if the metadata does not have a valid header, or if the meta format is unexpected. func GetAll(d []byte) ([][2]string, error) { err := checkHeader(d) diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go index e5f6a4ff..455a829c 100644 --- a/stream/mts/meta/meta_test.go +++ b/stream/mts/meta/meta_test.go @@ -138,7 +138,7 @@ func TestEncode(t *testing.T) { // TestReadFrom checks that we can correctly obtain a value for a partiular key // from a string of metadata using the ReadFrom func. -func TestReadFrom(t *testing.T) { +func TestGetFrom(t *testing.T) { tstMeta := append([]byte{0x00, 0x10, 0x00, 0x12}, "loc=a,b,c\tts=12345"...) tests := []struct { @@ -166,9 +166,9 @@ func TestReadFrom(t *testing.T) { } } -// TestExtractAll checks that meta.ExtractAll can correctly extract all metadata +// TestGetAll checks that meta.GetAll can correctly get all metadata // from descriptor data. -func TestExtractAll(t *testing.T) { +func TestGetAll(t *testing.T) { tstMeta := append([]byte{0x00, 0x10, 0x00, 0x12}, "loc=a,b,c\tts=12345"...) want := [][2]string{ { From 01513fbb3ff2917602a18639fb3060cc9b54ecd6 Mon Sep 17 00:00:00 2001 From: saxon Date: Sat, 9 Feb 2019 13:22:18 +1030 Subject: [PATCH 161/208] stream/rtp/encoder.go: correct encoder.Write to work if an abitrary number of bytes are written --- stream/rtp/encoder.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/stream/rtp/encoder.go b/stream/rtp/encoder.go index 329a24c0..b9453409 100644 --- a/stream/rtp/encoder.go +++ b/stream/rtp/encoder.go @@ -72,13 +72,26 @@ func NewEncoder(dst io.Writer, fps int) *Encoder { // so that multiple layers of packetization can occur. func (e *Encoder) Write(data []byte) (int, error) { e.buffer = append(e.buffer, data...) - for len(e.buffer) >= sendLen { - e.Encode(e.buffer) - e.buffer = e.buffer[:0] + if len(e.buffer) < sendLen { // sendSize + return len(data), nil } + buf := e.buffer + for len(buf) != 0 { + l := min(sendLen, len(buf)) // sendSize + e.Encode(buf[:l]) + buf = buf[l:] + } + e.buffer = e.buffer[:0] return len(data), nil } +func min(a, b int) int { + if a < b { + return a + } + return b +} + // Encode takes a nalu unit and encodes it into an rtp packet and // writes to the io.Writer given in NewEncoder func (e *Encoder) Encode(payload []byte) error { From 716a92a72cf5f05dde7a9410444ef419ec4da3e5 Mon Sep 17 00:00:00 2001 From: saxon Date: Sat, 9 Feb 2019 19:31:15 +1030 Subject: [PATCH 162/208] stream/rtp/encoder.go: sendLen to sendSize and capturing error from e.Encode() --- stream/rtp/encoder.go | 11 +++++++---- stream/rtp/rtp.go | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/stream/rtp/encoder.go b/stream/rtp/encoder.go index b9453409..9cda9c00 100644 --- a/stream/rtp/encoder.go +++ b/stream/rtp/encoder.go @@ -40,7 +40,7 @@ const ( timestampFreq = 90000 // Hz mtsSize = 188 bufferSize = 1000 - sendLen = 7 * 188 + sendSize = 7 * 188 ) // Encoder implements io writer and provides functionality to wrap data into @@ -72,13 +72,16 @@ func NewEncoder(dst io.Writer, fps int) *Encoder { // so that multiple layers of packetization can occur. func (e *Encoder) Write(data []byte) (int, error) { e.buffer = append(e.buffer, data...) - if len(e.buffer) < sendLen { // sendSize + if len(e.buffer) < sendSize { return len(data), nil } buf := e.buffer for len(buf) != 0 { - l := min(sendLen, len(buf)) // sendSize - e.Encode(buf[:l]) + l := min(sendSize, len(buf)) + err := e.Encode(buf[:l]) + if err != nil { + return len(data), err + } buf = buf[l:] } e.buffer = e.buffer[:0] diff --git a/stream/rtp/rtp.go b/stream/rtp/rtp.go index 47f4a91b..92192294 100644 --- a/stream/rtp/rtp.go +++ b/stream/rtp/rtp.go @@ -35,7 +35,7 @@ package rtp const ( rtpVer = 2 headSize = 3 * 4 // Header size of an rtp packet. - defPayloadSize = sendLen // Default payload size for the rtp packet. + defPayloadSize = sendSize // Default payload size for the rtp packet. defPktSize = headSize + defPayloadSize // Default packet size is header size + payload size. ) From 50575270b9da50936e52bc55d554235005fddb41 Mon Sep 17 00:00:00 2001 From: saxon Date: Sat, 9 Feb 2019 21:35:35 +1030 Subject: [PATCH 163/208] stream/mts/meta: checking if given slice is nil or empty and returning error if either. Also updated some func comments --- stream/mts/meta/meta.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go index 29ed0a06..d69a2b60 100644 --- a/stream/mts/meta/meta.go +++ b/stream/mts/meta/meta.go @@ -165,9 +165,7 @@ func (m *Data) Encode() []byte { return m.enc } -// ReadFrom gets a value from a metadata string d, for the given key. If the -// key is not present in the metadata string, an error is returned. If the -// metadata header is not present in the string, an error is returned. +// Get returns the value for the given key in d. func Get(key string, d []byte) (string, error) { err := checkHeader(d) if err != nil { @@ -184,9 +182,14 @@ func Get(key string, d []byte) (string, error) { return "", errKeyAbsent } -// GetAll gets all metadata entries from given data. An Error is returned -// if the metadata does not have a valid header, or if the meta format is unexpected. +// GetAll returns metadata keys and values from d. func GetAll(d []byte) ([][2]string, error) { + if d == nil { + return nil, errors.New("nil slice given") + } + if len(d) == 0 { + return nil, errors.New("empty slice given") + } err := checkHeader(d) if err != nil { return nil, err @@ -204,8 +207,7 @@ func GetAll(d []byte) ([][2]string, error) { return all, nil } -// checkHeader checks that a valid metadata header exists in the given data. An -// error is returned if the header is absent, or if the header is not valid. +// checkHeader checks that a valid metadata header exists in the given data. func checkHeader(d []byte) error { if d[0] != 0 { return errNoHeader From 46ab4b200ea16167842a814308bcd3160a62f1e8 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Sat, 9 Feb 2019 22:12:55 +1030 Subject: [PATCH 164/208] exp/ts-repair: fix assignment --- experimentation/ts-repair/main.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/experimentation/ts-repair/main.go b/experimentation/ts-repair/main.go index bed81f19..97c350f6 100644 --- a/experimentation/ts-repair/main.go +++ b/experimentation/ts-repair/main.go @@ -209,10 +209,7 @@ func main() { packetNo++ // Get the pid from the packet - pid, err := packet.Pid((*packet.Packet)(&p)) - if err != nil { - panic(errCantGetPid) - } + pid := packet.Pid((*packet.Packet)(&p)) // Get the cc from the packet and also the expected cc (if exists) cc := p.CC() From b24e72caa547f9a6b1ed3e0de07c2578144c78b5 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Sat, 9 Feb 2019 22:14:21 +1030 Subject: [PATCH 165/208] experimentation: rename to exp --- {experimentation => exp}/flac/decode.go | 0 {experimentation => exp}/flac/flac_test.go | 0 {experimentation => exp}/ts-repair/main.go | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {experimentation => exp}/flac/decode.go (100%) rename {experimentation => exp}/flac/flac_test.go (100%) rename {experimentation => exp}/ts-repair/main.go (100%) diff --git a/experimentation/flac/decode.go b/exp/flac/decode.go similarity index 100% rename from experimentation/flac/decode.go rename to exp/flac/decode.go diff --git a/experimentation/flac/flac_test.go b/exp/flac/flac_test.go similarity index 100% rename from experimentation/flac/flac_test.go rename to exp/flac/flac_test.go diff --git a/experimentation/ts-repair/main.go b/exp/ts-repair/main.go similarity index 100% rename from experimentation/ts-repair/main.go rename to exp/ts-repair/main.go From cc0becf58cd0d5916b87770370fee7bef1e927d1 Mon Sep 17 00:00:00 2001 From: saxon Date: Sat, 9 Feb 2019 22:25:36 +1030 Subject: [PATCH 166/208] stream/mts/meta: simplified meta checking for Get and GetAll --- stream/mts/meta/meta.go | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go index d69a2b60..4b5cf8e9 100644 --- a/stream/mts/meta/meta.go +++ b/stream/mts/meta/meta.go @@ -50,8 +50,7 @@ const ( var ( errKeyAbsent = errors.New("Key does not exist in map") - errNoHeader = errors.New("Metadata string does not contain header") - errInvalidHeader = errors.New("Metadata string does not contain valid header") + errInvalidMeta = errors.New("Invalid metadata given") errUnexpectedMetaFormat = errors.New("Unexpected meta format") ) @@ -167,7 +166,7 @@ func (m *Data) Encode() []byte { // Get returns the value for the given key in d. func Get(key string, d []byte) (string, error) { - err := checkHeader(d) + err := checkMeta(d) if err != nil { return "", err } @@ -184,13 +183,7 @@ func Get(key string, d []byte) (string, error) { // GetAll returns metadata keys and values from d. func GetAll(d []byte) ([][2]string, error) { - if d == nil { - return nil, errors.New("nil slice given") - } - if len(d) == 0 { - return nil, errors.New("empty slice given") - } - err := checkHeader(d) + err := checkMeta(d) if err != nil { return nil, err } @@ -208,11 +201,9 @@ func GetAll(d []byte) ([][2]string, error) { } // checkHeader checks that a valid metadata header exists in the given data. -func checkHeader(d []byte) error { - if d[0] != 0 { - return errNoHeader - } else if d[0] == 0 && binary.BigEndian.Uint16(d[2:headSize]) != uint16(len(d[headSize:])) { - return errInvalidHeader +func checkMeta(d []byte) error { + if len(d) == 0 || d[0] != 0 || binary.BigEndian.Uint16(d[2:headSize]) != uint16(len(d[headSize:])) { + return errInvalidMeta } return nil } From 1f3d34b6bb0e038add0eb9c929c2209598e9700f Mon Sep 17 00:00:00 2001 From: saxon Date: Sat, 9 Feb 2019 22:27:25 +1030 Subject: [PATCH 167/208] stream/mts/meta/meta_test.go: corrected func comment for TestGetFrom --- stream/mts/meta/meta_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go index 455a829c..459b9912 100644 --- a/stream/mts/meta/meta_test.go +++ b/stream/mts/meta/meta_test.go @@ -136,7 +136,7 @@ func TestEncode(t *testing.T) { } } -// TestReadFrom checks that we can correctly obtain a value for a partiular key +// TestGetFrom checks that we can correctly obtain a value for a partiular key // from a string of metadata using the ReadFrom func. func TestGetFrom(t *testing.T) { tstMeta := append([]byte{0x00, 0x10, 0x00, 0x12}, "loc=a,b,c\tts=12345"...) From ce92dd37d8ecf4c334db4a8e948662aad62a8376 Mon Sep 17 00:00:00 2001 From: saxon Date: Sun, 10 Feb 2019 08:58:38 +1030 Subject: [PATCH 168/208] stream/mts/meta: added Keys() func and appropriate testing A meta.Keys(d []byte) []string, error func has been added that will extract the keys of a metadata string. A test has also been added to test that this function performs as expected. --- stream/mts/meta/meta.go | 13 +++++++++++++ stream/mts/meta/meta_test.go | 14 ++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go index 4b5cf8e9..b0d0fc2f 100644 --- a/stream/mts/meta/meta.go +++ b/stream/mts/meta/meta.go @@ -164,6 +164,19 @@ func (m *Data) Encode() []byte { return m.enc } +// Keys returns all keys in a slice of metadata d. +func Keys(d []byte) ([]string, error) { + tmp, err := GetAll(d) + if err != nil { + return nil, err + } + keys := make([]string, len(tmp)) + for i, entry := range tmp { + keys[i] = entry[0] + } + return keys, nil +} + // Get returns the value for the given key in d. func Get(key string, d []byte) (string, error) { err := checkMeta(d) diff --git a/stream/mts/meta/meta_test.go b/stream/mts/meta/meta_test.go index 459b9912..38e4dbb6 100644 --- a/stream/mts/meta/meta_test.go +++ b/stream/mts/meta/meta_test.go @@ -188,3 +188,17 @@ func TestGetAll(t *testing.T) { t.Errorf("Did not get expected out. \nGot : %v, \nWant: %v\n", got, want) } } + +// TestKeys checks that we can successfully get keys from some metadata using +// the meta.Keys method. +func TestKeys(t *testing.T) { + tstMeta := append([]byte{0x00, 0x10, 0x00, 0x12}, "loc=a,b,c\tts=12345"...) + want := []string{"loc", "ts"} + got, err := Keys(tstMeta) + if err != nil { + t.Errorf("Unexpected error: %v\n", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("Did not get expected out. \nGot : %v, \nWant: %v\n", got, want) + } +} From 14ca8e8b27e02e18516176cd07f74c088277dad7 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Sun, 10 Feb 2019 09:08:00 +1030 Subject: [PATCH 169/208] circle-ci: make test data available --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index af93f767..b1ae37f2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,6 +13,8 @@ jobs: steps: - checkout + - run: git clone --depth=1 https://bitbucket.org/ausocean/test.git ${GOPATH}/src/bitbucket.org/ausocean/test + - restore_cache: keys: - v1-pkg-cache From dad70b37b40c6b3005a3050e7fab44cdc5fa0c92 Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 11 Feb 2019 14:30:37 +1030 Subject: [PATCH 170/208] stream/mts/meta/meta.go: reduced verbosity of local vars in meta.Keys --- stream/mts/meta/meta.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/stream/mts/meta/meta.go b/stream/mts/meta/meta.go index b0d0fc2f..481b5ae5 100644 --- a/stream/mts/meta/meta.go +++ b/stream/mts/meta/meta.go @@ -166,15 +166,15 @@ func (m *Data) Encode() []byte { // Keys returns all keys in a slice of metadata d. func Keys(d []byte) ([]string, error) { - tmp, err := GetAll(d) + m, err := GetAll(d) if err != nil { return nil, err } - keys := make([]string, len(tmp)) - for i, entry := range tmp { - keys[i] = entry[0] + k := make([]string, len(m)) + for i, kv := range m { + k[i] = kv[0] } - return keys, nil + return k, nil } // Get returns the value for the given key in d. From f67fb1ec8a12aa14cc5033d6b6ec429f8842e470 Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 11 Feb 2019 16:49:28 +1030 Subject: [PATCH 171/208] cmd/revid-cli: replaced use of send() with netsender.Run() and implemented readPin func. --- cmd/revid-cli/main.go | 38 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 6ca6ba68..ea093ce1 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -40,6 +40,7 @@ import ( "bitbucket.org/ausocean/av/stream/mts" "bitbucket.org/ausocean/av/stream/mts/meta" "bitbucket.org/ausocean/iot/pi/netsender" + "bitbucket.org/ausocean/iot/pi/sds" "bitbucket.org/ausocean/iot/pi/smartlogger" "bitbucket.org/ausocean/utils/logger" ) @@ -267,8 +268,7 @@ func run(cfg revid.Config) error { var vars map[string]string // initialize NetSender and use NetSender's logger - var ns netsender.Sender - err := ns.Init(log, nil, nil, nil) + ns, err := netsender.New(log, nil, readPin, nil) if err != nil { return err } @@ -276,7 +276,7 @@ func run(cfg revid.Config) error { vars, _ = ns.Vars() vs := ns.VarSum() - rv, err := revid.New(cfg, &ns) + rv, err := revid.New(cfg, ns) if err != nil { log.Log(logger.Fatal, pkg+"could not initialise revid", "error", err.Error()) } @@ -300,8 +300,7 @@ func run(cfg revid.Config) error { } for { - // TODO(saxon): replace this call with call to ns.Run(). - err = send(&ns, rv) + err = ns.Run() if err != nil { log.Log(logger.Error, pkg+"Run Failed. Retrying...", "error", err.Error()) time.Sleep(netSendRetryTime) @@ -356,27 +355,16 @@ func run(cfg revid.Config) error { } } -// send implements our main NetSender client and handles NetReceiver configuration -// (as distinct from httpSender which just sends video data). -func send(ns *netsender.Sender, rv *revid.Revid) error { - // populate input values, if any - inputs := netsender.MakePins(ns.Param("ip"), "X") - if rv != nil { - for i, pin := range inputs { - if pin.Name == "X23" { - inputs[i].Value = rv.Bitrate() - } - } +func readPin(pin *netsender.Pin) error { + switch { + case pin.Name == "X23": + //pin.Value = rv.Bitrate() + case pin.Name[0] == 'X': + return sds.ReadSystem(pin) + default: + pin.Value = -1 } - - _, reconfig, err := ns.Send(netsender.RequestPoll, inputs) - if err != nil { - return err - } - if reconfig { - return ns.Config() - } - return nil + return nil // Return error only if we want NetSender to generate an error } // flagStrings implements an appending string set flag. From 4e1c8e7c45db1150898076cc6448cf8b30ade7a6 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 12 Feb 2019 10:39:23 +1030 Subject: [PATCH 172/208] cmd/revid-cli: made Revid global so that revid.Bitrate() can be accessed in readPin --- cmd/revid-cli/main.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index ea093ce1..edce23e2 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -75,6 +75,8 @@ var canProfile = true // The logger that will be used throughout var log *logger.Logger +var rv *revid.Revid + const ( metaPreambleKey = "copyright" metaPreambleData = "ausocean.org/license/content2019" @@ -87,9 +89,9 @@ func main() { runDurationPtr := flag.Duration("runDuration", defaultRunDuration, "How long do you want revid to run for?") cfg := handleFlags() - + var err error if !*useNetsender { - rv, err := revid.New(cfg, nil) + rv, err = revid.New(cfg, nil) if err != nil { cfg.Logger.Log(logger.Fatal, pkg+"failed to initialiase revid", "error", err.Error()) } @@ -276,7 +278,7 @@ func run(cfg revid.Config) error { vars, _ = ns.Vars() vs := ns.VarSum() - rv, err := revid.New(cfg, ns) + rv, err = revid.New(cfg, ns) if err != nil { log.Log(logger.Fatal, pkg+"could not initialise revid", "error", err.Error()) } @@ -358,7 +360,7 @@ func run(cfg revid.Config) error { func readPin(pin *netsender.Pin) error { switch { case pin.Name == "X23": - //pin.Value = rv.Bitrate() + pin.Value = rv.Bitrate() case pin.Name[0] == 'X': return sds.ReadSystem(pin) default: From b9c53791d8c7a14e68b5e4cafa3611ee2a945ea3 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 12 Feb 2019 10:42:26 +1030 Subject: [PATCH 173/208] cmd/revid-cli: updated readPin func comment --- cmd/revid-cli/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index edce23e2..f34b77c6 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -357,6 +357,8 @@ func run(cfg revid.Config) error { } } +// readPin is callback func for netsender so that pins can be appropriately set +// for this circumstance. func readPin(pin *netsender.Pin) error { switch { case pin.Name == "X23": From 69029889fe0d34c7bfbf4cba700676d0e45e0832 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 12 Feb 2019 11:32:02 +1030 Subject: [PATCH 174/208] cmd/revid-cli: using closure for readPin func so that we don't have to have global revid --- cmd/revid-cli/main.go | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index f34b77c6..28be6fa2 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -75,8 +75,6 @@ var canProfile = true // The logger that will be used throughout var log *logger.Logger -var rv *revid.Revid - const ( metaPreambleKey = "copyright" metaPreambleData = "ausocean.org/license/content2019" @@ -89,9 +87,8 @@ func main() { runDurationPtr := flag.Duration("runDuration", defaultRunDuration, "How long do you want revid to run for?") cfg := handleFlags() - var err error if !*useNetsender { - rv, err = revid.New(cfg, nil) + rv, err := revid.New(cfg, nil) if err != nil { cfg.Logger.Log(logger.Fatal, pkg+"failed to initialiase revid", "error", err.Error()) } @@ -268,21 +265,34 @@ func run(cfg revid.Config) error { log.Log(logger.Info, pkg+"running in NetSender mode") var vars map[string]string + var rv *revid.Revid // initialize NetSender and use NetSender's logger + readPin := func(pin *netsender.Pin) error { + switch { + case pin.Name == "X23": + pin.Value = rv.Bitrate() + case pin.Name[0] == 'X': + return sds.ReadSystem(pin) + default: + pin.Value = -1 + } + return nil // Return error only if we want NetSender to generate an error + } + ns, err := netsender.New(log, nil, readPin, nil) if err != nil { return err } - vars, _ = ns.Vars() - vs := ns.VarSum() - rv, err = revid.New(cfg, ns) if err != nil { log.Log(logger.Fatal, pkg+"could not initialise revid", "error", err.Error()) } + vars, _ = ns.Vars() + vs := ns.VarSum() + // Update revid to get latest config settings from netreceiver. err = rv.Update(vars) if err != nil { @@ -357,20 +367,6 @@ func run(cfg revid.Config) error { } } -// readPin is callback func for netsender so that pins can be appropriately set -// for this circumstance. -func readPin(pin *netsender.Pin) error { - switch { - case pin.Name == "X23": - pin.Value = rv.Bitrate() - case pin.Name[0] == 'X': - return sds.ReadSystem(pin) - default: - pin.Value = -1 - } - return nil // Return error only if we want NetSender to generate an error -} - // flagStrings implements an appending string set flag. type flagStrings []string From e2d80b423c4db23680b6b91490fce880f264580f Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 12 Feb 2019 11:33:48 +1030 Subject: [PATCH 175/208] cmd/revid-cli: removed comment we don't need anymore --- cmd/revid-cli/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 28be6fa2..265e7227 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -267,7 +267,6 @@ func run(cfg revid.Config) error { var vars map[string]string var rv *revid.Revid - // initialize NetSender and use NetSender's logger readPin := func(pin *netsender.Pin) error { switch { case pin.Name == "X23": From 2a61f2d08dee559f2d08c1c53b2acb1bda32ef00 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 12 Feb 2019 11:37:50 +1030 Subject: [PATCH 176/208] cmd/revid-cli: added space before rv declaration --- cmd/revid-cli/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 265e7227..81c2e55a 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -265,6 +265,7 @@ func run(cfg revid.Config) error { log.Log(logger.Info, pkg+"running in NetSender mode") var vars map[string]string + var rv *revid.Revid readPin := func(pin *netsender.Pin) error { From 3ee2683ca9fd85454b70e34def1304d1f3278e7a Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 12 Feb 2019 18:22:46 +1030 Subject: [PATCH 177/208] cmd/revid-cli & revid: outputFileNamePtr => outputPathPtr, config.OutputFileName=>config.OutputPath, inputFileNamePtr=>inputPathPtr and finally, config.InputFileName=>config.InputPath --- cmd/revid-cli/main.go | 8 ++++---- revid/config.go | 4 ++-- revid/revid.go | 18 +++++++++--------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 6ca6ba68..e690a579 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -124,8 +124,8 @@ func handleFlags() revid.Config { framesPerClipPtr = flag.Uint("FramesPerClip", 0, "Number of frames per clip sent") rtmpUrlPtr = flag.String("RtmpUrl", "", "Url of rtmp endpoint") bitratePtr = flag.Uint("Bitrate", 0, "Bitrate of recorded video") - outputFileNamePtr = flag.String("OutputFileName", "", "The directory of the output file") - inputFileNamePtr = flag.String("InputFileName", "", "The directory of the input file") + outputPathPtr = flag.String("OutputPath", "", "The directory of the output file") + inputFilePtr = flag.String("InputPath", "", "The directory of the input file") heightPtr = flag.Uint("Height", 0, "Height in pixels") widthPtr = flag.Uint("Width", 0, "Width in pixels") frameRatePtr = flag.Uint("FrameRate", 0, "Frame rate of captured video") @@ -246,8 +246,8 @@ func handleFlags() revid.Config { cfg.FramesPerClip = *framesPerClipPtr cfg.RtmpUrl = *rtmpUrlPtr cfg.Bitrate = *bitratePtr - cfg.OutputFileName = *outputFileNamePtr - cfg.InputFileName = *inputFileNamePtr + cfg.OutputPath = *outputPathPtr + cfg.InputPath = *inputFilePtr cfg.Height = *heightPtr cfg.Width = *widthPtr cfg.FrameRate = *frameRatePtr diff --git a/revid/config.go b/revid/config.go index b0ba2bc4..1e1cc02c 100644 --- a/revid/config.go +++ b/revid/config.go @@ -57,8 +57,8 @@ type Config struct { FramesPerClip uint RtmpUrl string Bitrate uint - OutputFileName string - InputFileName string + OutputPath string + InputPath string Height uint Width uint FrameRate uint diff --git a/revid/revid.go b/revid/revid.go index e426272a..b4edd4f1 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -243,7 +243,7 @@ func (r *Revid) reset(config Config) error { for _, typ := range r.config.Outputs { switch typ { case File: - s, err := newFileSender(config.OutputFileName) + s, err := newFileSender(config.OutputPath) if err != nil { return err } @@ -425,10 +425,10 @@ func (r *Revid) Update(vars map[string]string) error { break } r.config.Bitrate = uint(v) - case "OutputFileName": - r.config.OutputFileName = value - case "InputFileName": - r.config.InputFileName = value + case "OutputPath": + r.config.OutputPath = value + case "InputPath": + r.config.InputPath = value case "Height": h, err := strconv.ParseUint(value, 10, 0) if err != nil { @@ -648,13 +648,13 @@ func (r *Revid) startV4L() error { const defaultVideo = "/dev/video0" r.config.Logger.Log(logger.Info, pkg+"starting webcam") - if r.config.InputFileName == "" { + if r.config.InputPath == "" { r.config.Logger.Log(logger.Info, pkg+"using default video device", "device", defaultVideo) - r.config.InputFileName = defaultVideo + r.config.InputPath = defaultVideo } args := []string{ - "-i", r.config.InputFileName, + "-i", r.config.InputPath, "-f", "h264", "-r", fmt.Sprint(r.config.FrameRate), } @@ -695,7 +695,7 @@ func (r *Revid) startV4L() error { // setupInputForFile sets things up for getting input from a file func (r *Revid) setupInputForFile() error { - f, err := os.Open(r.config.InputFileName) + f, err := os.Open(r.config.InputPath) if err != nil { r.config.Logger.Log(logger.Error, err.Error()) r.Stop() From c3743174da97ecf6b2bea074a76a1137d4983145 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 12 Feb 2019 18:37:15 +1030 Subject: [PATCH 178/208] cmd/revid-cli: initialisation of logger actually usses what was given as revid-cli verbosity flag input --- cmd/revid-cli/main.go | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 6ca6ba68..a53d08a0 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -145,7 +145,24 @@ func handleFlags() revid.Config { flag.Parse() - log = logger.New(defaultLogVerbosity, &smartlogger.New(*logPathPtr).LogRoller) + switch *verbosityPtr { + case "Debug": + cfg.LogLevel = logger.Debug + case "Info": + cfg.LogLevel = logger.Info + case "Warning": + cfg.LogLevel = logger.Warning + case "Error": + cfg.LogLevel = logger.Error + case "Fatal": + cfg.LogLevel = logger.Fatal + case "": + cfg.LogLevel = defaultLogVerbosity + default: + log.Log(logger.Error, pkg+"bad verbosity argument") + } + + log = logger.New(cfg.LogLevel, &smartlogger.New(*logPathPtr).LogRoller) cfg.Logger = log @@ -225,17 +242,6 @@ func handleFlags() revid.Config { log.Log(logger.Error, pkg+"bad packetization argument") } - switch *verbosityPtr { - case "No": - cfg.LogLevel = logger.Fatal - case "Debug": - cfg.LogLevel = logger.Debug - //logger.SetLevel(logger.Debug) - case "": - default: - log.Log(logger.Error, pkg+"bad verbosity argument") - } - if *configFilePtr != "" { netsender.ConfigFile = *configFilePtr } From b96df6d3a70293ec5d4895fb20476ae1e247bcf6 Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 13 Feb 2019 14:40:58 +1030 Subject: [PATCH 179/208] stream/mts: added general FindPID func and FindPAT func. --- stream/mts/mpegts.go | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/stream/mts/mpegts.go b/stream/mts/mpegts.go index 08791af7..012a912c 100644 --- a/stream/mts/mpegts.go +++ b/stream/mts/mpegts.go @@ -159,18 +159,30 @@ type Packet struct { // FindPMT will take a clip of mpegts and try to find a PMT table - if one // is found, then it is returned along with its index, otherwise nil, -1 and an error is returned. -func FindPMT(d []byte) (p []byte, i int, err error) { +func FindPMT(d []byte) ([]byte, int, error) { + return FindPID(d, PmtPid) +} + +// FindPAT will take a clip of mpegts and try to find a PAT table - if one +// is found, then it is returned along with its index, otherwise nil, -1 and an error is returned. +func FindPAT(d []byte) ([]byte, int, error) { + return FindPID(d, PatPid) +} + +// FindPID will take a clip of mpegts and try to find a packet with given PID - if one +// is found, then it is returned along with its index, otherwise nil, -1 and an error is returned. +func FindPID(d []byte, pid uint16) (pkt []byte, i int, err error) { if len(d) < PacketSize { return nil, -1, errors.New("Mmpegts data not of valid length") } for i = 0; i < len(d); i += PacketSize { - pid := (uint16(d[i+1]&0x1f) << 8) | uint16(d[i+2]) - if pid == pmtPid { - p = d[i+4 : i+PacketSize] + p := (uint16(d[i+1]&0x1f) << 8) | uint16(d[i+2]) + if p == pid { + pkt = d[i+4 : i+PacketSize] return } } - return nil, -1, errors.New("Could not find pmt table in mpegts data") + return nil, -1, errors.New("Could not find packet with given pid in mpegts data") } // FillPayload takes a channel and fills the packets Payload field until the From da5fdccaf321753dece70be6c8212684b9874b88 Mon Sep 17 00:00:00 2001 From: saxon Date: Sat, 16 Feb 2019 09:09:54 +1030 Subject: [PATCH 180/208] stream/mts & stream/rtp: fixed timing calculations Currently time calculations in the mts encoder are based on the premise that each mpegts packet is a 'frame', and in the rtp encoder that each encoding of multiple packets into one rtp packet is a 'frame', these are both false. Mpegts encoding and rtp encoding should do timestamp calculations using a reasonable approximation of time at encoding/send. --- stream/mts/encoder.go | 5 ++++- stream/rtp/encoder.go | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index a309eaf9..e63c15bc 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -122,6 +122,7 @@ type Encoder struct { dst io.Writer clock time.Duration + lastTime time.Time frameInterval time.Duration ptsOffset time.Duration tsSpace [PacketSize]byte @@ -247,7 +248,9 @@ func (e *Encoder) writePSI() error { // tick advances the clock one frame interval. func (e *Encoder) tick() { - e.clock += e.frameInterval + now := time.Now() + e.clock += now.Sub(e.lastTime) + e.lastTime = now } // pts retuns the current presentation timestamp. diff --git a/stream/rtp/encoder.go b/stream/rtp/encoder.go index 9cda9c00..28119b43 100644 --- a/stream/rtp/encoder.go +++ b/stream/rtp/encoder.go @@ -51,6 +51,7 @@ type Encoder struct { seqNo uint16 clock time.Duration frameInterval time.Duration + lastTime time.Time fps int buffer []byte pktSpace [defPktSize]byte @@ -120,7 +121,9 @@ func (e *Encoder) Encode(payload []byte) error { // tick advances the clock one frame interval. func (e *Encoder) tick() { - e.clock += e.frameInterval + now := time.Now() + e.clock += now.Sub(e.lastTime) + e.lastTime = now } // nxtTimestamp gets the next timestamp From c58b207419509937104a59443889c049eef02e2d Mon Sep 17 00:00:00 2001 From: saxon Date: Sun, 17 Feb 2019 13:35:55 +1030 Subject: [PATCH 181/208] stream: reverted timing changes from most recent PR --- stream/mts/encoder.go | 4 +--- stream/rtp/encoder.go | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index e63c15bc..6ac11cc3 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -248,9 +248,7 @@ func (e *Encoder) writePSI() error { // tick advances the clock one frame interval. func (e *Encoder) tick() { - now := time.Now() - e.clock += now.Sub(e.lastTime) - e.lastTime = now + e.clock += e.frameInterval } // pts retuns the current presentation timestamp. diff --git a/stream/rtp/encoder.go b/stream/rtp/encoder.go index 28119b43..26016a23 100644 --- a/stream/rtp/encoder.go +++ b/stream/rtp/encoder.go @@ -121,9 +121,7 @@ func (e *Encoder) Encode(payload []byte) error { // tick advances the clock one frame interval. func (e *Encoder) tick() { - now := time.Now() - e.clock += now.Sub(e.lastTime) - e.lastTime = now + e.clock += e.frameInterval } // nxtTimestamp gets the next timestamp From 01d85af133a988f6d5b9434689081d1b4c4dcad8 Mon Sep 17 00:00:00 2001 From: Alan Noble Date: Fri, 22 Feb 2019 20:45:13 +0000 Subject: [PATCH 182/208] Removed incorrect Close upon Dial error. --- revid/senders.go | 1 - 1 file changed, 1 deletion(-) diff --git a/revid/senders.go b/revid/senders.go index e86cb24b..9d326471 100644 --- a/revid/senders.go +++ b/revid/senders.go @@ -273,7 +273,6 @@ func newRtmpSender(url string, timeout uint, retries int, log func(lvl int8, msg break } log(logger.Error, err.Error()) - conn.Close() if n < retries-1 { log(logger.Info, pkg+"retry rtmp connection") } From 37850e8350ddfc08b4ec550648e4514addfbac47 Mon Sep 17 00:00:00 2001 From: Saxon Date: Mon, 25 Feb 2019 12:59:57 +1030 Subject: [PATCH 183/208] revid-cli: initialising config.Output slice when no outputs are defined so that a default output can still be set --- cmd/revid-cli/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 6ca6ba68..05de1acf 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -184,6 +184,10 @@ func handleFlags() revid.Config { log.Log(logger.Error, pkg+"bad input codec argument") } + if len(outputs) == 0 { + cfg.Outputs = make([]uint8, 1) + } + for _, o := range outputs { switch o { case "File": From e118a639d4e0cd5b11fd049bce2f7ec14f8fdbba Mon Sep 17 00:00:00 2001 From: Saxon Date: Mon, 25 Feb 2019 15:28:19 +1030 Subject: [PATCH 184/208] revid: outputting config when we update --- revid/revid.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/revid/revid.go b/revid/revid.go index e426272a..bec75d7f 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -493,7 +493,7 @@ func (r *Revid) Update(vars map[string]string) error { r.config.BurstPeriod = uint(v) } } - + r.config.Logger.Log(logger.Info, pkg+"revid config changed", "config", fmt.Sprint("%+v", r.config)) return nil } From 5a2d89ef8fc2e40cdace32294ab7249a06aaa6bf Mon Sep 17 00:00:00 2001 From: Saxon Date: Mon, 25 Feb 2019 15:31:42 +1030 Subject: [PATCH 185/208] revid: using only one output when we're getting config from netreceiver --- revid/revid.go | 1 + 1 file changed, 1 insertion(+) diff --git a/revid/revid.go b/revid/revid.go index bec75d7f..92102143 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -392,6 +392,7 @@ func (r *Revid) Update(vars map[string]string) error { for key, value := range vars { switch key { case "Output": + r.config.Outputs = make([]uint8, 1) // FIXME(kortschak): There can be only one! // How do we specify outputs after the first? // From cc4d683ba610ca7ba4f22b8598808e022e7f6c7d Mon Sep 17 00:00:00 2001 From: Saxon Date: Mon, 25 Feb 2019 15:47:43 +1030 Subject: [PATCH 186/208] revid: calling reset inside revid.Update once config is updated --- revid/revid.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/revid/revid.go b/revid/revid.go index 92102143..2a843b3f 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -495,7 +495,7 @@ func (r *Revid) Update(vars map[string]string) error { } } r.config.Logger.Log(logger.Info, pkg+"revid config changed", "config", fmt.Sprint("%+v", r.config)) - return nil + return r.reset(r.config) } // outputClips takes the clips produced in the packClips method and outputs them From be6e16877430e7e2f33b03d6c81e73d46bfa15e1 Mon Sep 17 00:00:00 2001 From: Saxon Date: Mon, 25 Feb 2019 15:56:51 +1030 Subject: [PATCH 187/208] revid: fixed quantization and bitrate logic in config.Validate() --- revid/config.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/revid/config.go b/revid/config.go index b0ba2bc4..6f682028 100644 --- a/revid/config.go +++ b/revid/config.go @@ -148,11 +148,9 @@ func (c *Config) Validate(r *Revid) error { // Configuration really needs to be rethought here. if c.Quantize && c.Quantization == 0 { c.Quantization = defaultQuantization - } else { - c.Bitrate = defaultBitrate } - if (c.Bitrate > 0 && c.Quantization > 0) || (c.Bitrate == 0 && c.Quantization == 0) { + if (c.Bitrate > 0 && c.Quantize) || (c.Bitrate == 0 && !c.Quantize) { return errors.New("bad bitrate and quantization combination for H264 input") } From b7d1bd17a72f8f6a5685469bc11ab6313e349b19 Mon Sep 17 00:00:00 2001 From: Saxon Date: Mon, 25 Feb 2019 16:03:28 +1030 Subject: [PATCH 188/208] revid: initialising ring buffer in different location --- revid/revid.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/revid/revid.go b/revid/revid.go index 2a843b3f..be02e366 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -187,7 +187,6 @@ func (p *packer) Write(frame []byte) (int, error) { // an error if construction of the new instance was not successful. func New(c Config, ns *netsender.Sender) (*Revid, error) { r := Revid{ns: ns, err: make(chan error)} - r.buffer = ring.NewBuffer(ringBufferSize, ringBufferElementSize, writeTimeout) r.packer.owner = &r err := r.reset(c) if err != nil { @@ -230,6 +229,8 @@ func (r *Revid) reset(config Config) error { } r.config = config + r.buffer = ring.NewBuffer(ringBufferSize, ringBufferElementSize, writeTimeout) + for _, dest := range r.destination { if dest != nil { err = dest.close() From 7f2d53e77872382c3a0450d0d62bceb8b4792d5b Mon Sep 17 00:00:00 2001 From: Saxon Date: Mon, 25 Feb 2019 16:29:44 +1030 Subject: [PATCH 189/208] revid: now checking for Packetization var from netreceiver --- revid/revid.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/revid/revid.go b/revid/revid.go index be02e366..02174da2 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -411,6 +411,17 @@ func (r *Revid) Update(vars map[string]string) error { r.config.Logger.Log(logger.Warning, pkg+"invalid Output1 param", "value", value) continue } + + case "Packetization": + switch value { + case "Mpegts": + r.config.Packetization = Mpegts + case "Flv": + r.config.Packetization = Flv + default: + r.config.Logger.Log(logger.Warning, pkg+"invalid packetization param", "value", value) + continue + } case "FramesPerClip": f, err := strconv.ParseUint(value, 10, 0) if err != nil { From 8340143622b4c65e8692d2574567b4b805bfa46c Mon Sep 17 00:00:00 2001 From: Saxon Date: Mon, 25 Feb 2019 16:56:37 +1030 Subject: [PATCH 190/208] rtmp: not logging conn close message unless we do have a connection --- rtmp/conn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rtmp/conn.go b/rtmp/conn.go index 7e1b3b15..4249a637 100644 --- a/rtmp/conn.go +++ b/rtmp/conn.go @@ -134,10 +134,10 @@ func Dial(url string, timeout uint, log Log) (*Conn, error) { // Close terminates the RTMP connection. // NB: Close is idempotent and the connection value is cleared completely. func (c *Conn) Close() error { - c.log(DebugLevel, pkg+"Conn.Close") if !c.isConnected() { return errNotConnected } + c.log(DebugLevel, pkg+"Conn.Close") if c.streamID > 0 { if c.link.protocol&featureWrite != 0 { sendFCUnpublish(c) From 5d5af5601f064b9ed9ea9db45b868670e01a9f42 Mon Sep 17 00:00:00 2001 From: Saxon Date: Mon, 25 Feb 2019 17:00:31 +1030 Subject: [PATCH 191/208] revid: not closing destinations on call to revid.Reset() anymore --- revid/revid.go | 9 --------- rtmp/conn.go | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/revid/revid.go b/revid/revid.go index 02174da2..a5efa44d 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -231,15 +231,6 @@ func (r *Revid) reset(config Config) error { r.buffer = ring.NewBuffer(ringBufferSize, ringBufferElementSize, writeTimeout) - for _, dest := range r.destination { - if dest != nil { - err = dest.close() - if err != nil { - return err - } - } - } - r.destination = r.destination[:0] for _, typ := range r.config.Outputs { switch typ { diff --git a/rtmp/conn.go b/rtmp/conn.go index 4249a637..7e1b3b15 100644 --- a/rtmp/conn.go +++ b/rtmp/conn.go @@ -134,10 +134,10 @@ func Dial(url string, timeout uint, log Log) (*Conn, error) { // Close terminates the RTMP connection. // NB: Close is idempotent and the connection value is cleared completely. func (c *Conn) Close() error { + c.log(DebugLevel, pkg+"Conn.Close") if !c.isConnected() { return errNotConnected } - c.log(DebugLevel, pkg+"Conn.Close") if c.streamID > 0 { if c.link.protocol&featureWrite != 0 { sendFCUnpublish(c) From 95303b4e71f12130e35689ba2e5b785b6d643835 Mon Sep 17 00:00:00 2001 From: Alan Noble Date: Mon, 25 Feb 2019 06:31:12 +0000 Subject: [PATCH 192/208] Set default log level to Info (to match the comment). --- cmd/revid-cli/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 81c2e55a..1898613c 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -50,7 +50,7 @@ const ( progName = "revid-cli" // Logging is set to INFO level. - defaultLogVerbosity = logger.Debug + defaultLogVerbosity = logger.Info ) // Revid modes From a90b35b319274f6e62e8cd129156fe02025a22fe Mon Sep 17 00:00:00 2001 From: Alan Noble Date: Mon, 25 Feb 2019 07:14:30 +0000 Subject: [PATCH 193/208] Only log a debug message when ring.ErrTimeout occurs, not a warning. --- revid/revid.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/revid/revid.go b/revid/revid.go index a5efa44d..25a48854 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -515,7 +515,7 @@ loop: case nil: // Do nothing. case ring.ErrTimeout: - r.config.Logger.Log(logger.Warning, pkg+"ring buffer read timeout") + r.config.Logger.Log(logger.Debug, pkg+"ring buffer read timeout") continue default: r.config.Logger.Log(logger.Error, pkg+"unexpected error", "error", err.Error()) From 8240fea9ab1001513cab273094bc09f32f12ba85 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Mon, 25 Feb 2019 18:39:01 +1030 Subject: [PATCH 194/208] revid: fix build --- revid/revid.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/revid/revid.go b/revid/revid.go index 25a48854..8c202901 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -497,7 +497,7 @@ func (r *Revid) Update(vars map[string]string) error { r.config.BurstPeriod = uint(v) } } - r.config.Logger.Log(logger.Info, pkg+"revid config changed", "config", fmt.Sprint("%+v", r.config)) + r.config.Logger.Log(logger.Info, pkg+"revid config changed", "config", fmt.Sprintf("%+v", r.config)) return r.reset(r.config) } From 6143baed0bd155551a7298e1fa9bc33d505376b8 Mon Sep 17 00:00:00 2001 From: Saxon Date: Tue, 26 Feb 2019 09:06:25 +1030 Subject: [PATCH 195/208] rtmp: doing conn.close log message after we actually check that there is still a connection, otherwise we might have a nil logger and cause a panic --- rtmp/conn.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rtmp/conn.go b/rtmp/conn.go index 7e1b3b15..4249a637 100644 --- a/rtmp/conn.go +++ b/rtmp/conn.go @@ -134,10 +134,10 @@ func Dial(url string, timeout uint, log Log) (*Conn, error) { // Close terminates the RTMP connection. // NB: Close is idempotent and the connection value is cleared completely. func (c *Conn) Close() error { - c.log(DebugLevel, pkg+"Conn.Close") if !c.isConnected() { return errNotConnected } + c.log(DebugLevel, pkg+"Conn.Close") if c.streamID > 0 { if c.link.protocol&featureWrite != 0 { sendFCUnpublish(c) From 8a13bb70072d54b553a7ddcec15cdf985cbbbc8e Mon Sep 17 00:00:00 2001 From: Alan Noble Date: Wed, 27 Feb 2019 05:53:49 +0000 Subject: [PATCH 196/208] Temporary fix: Make ringBufferSize size larger for RTMP to work over flaky networks. --- revid/revid.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/revid/revid.go b/revid/revid.go index 8c202901..26ace6a6 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -51,7 +51,7 @@ import ( // Ring buffer sizes and read/write timeouts. const ( - ringBufferSize = 100 + ringBufferSize = 1000 ringBufferElementSize = 150000 writeTimeout = 10 * time.Millisecond readTimeout = 10 * time.Millisecond From 947f818bc626f3a1430f50b7749335148076998e Mon Sep 17 00:00:00 2001 From: Saxon Date: Wed, 27 Feb 2019 16:36:59 +1030 Subject: [PATCH 197/208] stream/mts: modified error in FindPid in the case that we can't a packet with given pid so that we show pid --- stream/mts/mpegts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stream/mts/mpegts.go b/stream/mts/mpegts.go index 012a912c..6d51a0b0 100644 --- a/stream/mts/mpegts.go +++ b/stream/mts/mpegts.go @@ -182,7 +182,7 @@ func FindPID(d []byte, pid uint16) (pkt []byte, i int, err error) { return } } - return nil, -1, errors.New("Could not find packet with given pid in mpegts data") + return nil, -1, errors.New(fmt.Sprintf("could not find packet with pid: %v",pid)) } // FillPayload takes a channel and fills the packets Payload field until the From 1e9fcda47c377c4f88dab30a2a40fc26f15d2800 Mon Sep 17 00:00:00 2001 From: Saxon Date: Wed, 27 Feb 2019 17:12:52 +1030 Subject: [PATCH 198/208] stream/mts: %v to %d in fmt.Sprintf usage in error message --- stream/mts/mpegts.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stream/mts/mpegts.go b/stream/mts/mpegts.go index 6d51a0b0..8173d2ce 100644 --- a/stream/mts/mpegts.go +++ b/stream/mts/mpegts.go @@ -30,6 +30,7 @@ package mts import ( "errors" + "fmt" ) // General mpegts packet properties. @@ -182,7 +183,7 @@ func FindPID(d []byte, pid uint16) (pkt []byte, i int, err error) { return } } - return nil, -1, errors.New(fmt.Sprintf("could not find packet with pid: %v",pid)) + return nil, -1, errors.New(fmt.Sprintf("could not find packet with pid: %d",pid)) } // FillPayload takes a channel and fills the packets Payload field until the From c1f7497078d0468fefdb5daf5ee3d3f9772d9042 Mon Sep 17 00:00:00 2001 From: Saxon Date: Wed, 27 Feb 2019 17:16:15 +1030 Subject: [PATCH 199/208] stream/mts: using fmt.Errorf instead of fmt.Sprintf inside errors.New --- stream/mts/mpegts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stream/mts/mpegts.go b/stream/mts/mpegts.go index 8173d2ce..e9bbf0e1 100644 --- a/stream/mts/mpegts.go +++ b/stream/mts/mpegts.go @@ -183,7 +183,7 @@ func FindPID(d []byte, pid uint16) (pkt []byte, i int, err error) { return } } - return nil, -1, errors.New(fmt.Sprintf("could not find packet with pid: %d",pid)) + return nil, -1, fmt.Errorf("could not find packet with pid: %d",pid) } // FillPayload takes a channel and fills the packets Payload field until the From 1e9b6c25c9d4f1757edc7b8abd210727e229bef0 Mon Sep 17 00:00:00 2001 From: Saxon Date: Thu, 28 Feb 2019 11:04:40 +1030 Subject: [PATCH 200/208] stream/mts: lowercase PID PAT and PMT in func names --- stream/mts/mpegts.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/stream/mts/mpegts.go b/stream/mts/mpegts.go index e9bbf0e1..c3c0d390 100644 --- a/stream/mts/mpegts.go +++ b/stream/mts/mpegts.go @@ -160,19 +160,19 @@ type Packet struct { // FindPMT will take a clip of mpegts and try to find a PMT table - if one // is found, then it is returned along with its index, otherwise nil, -1 and an error is returned. -func FindPMT(d []byte) ([]byte, int, error) { - return FindPID(d, PmtPid) +func FindPmt(d []byte) ([]byte, int, error) { + return FindPid(d, PmtPid) } // FindPAT will take a clip of mpegts and try to find a PAT table - if one // is found, then it is returned along with its index, otherwise nil, -1 and an error is returned. -func FindPAT(d []byte) ([]byte, int, error) { - return FindPID(d, PatPid) +func FindPat(d []byte) ([]byte, int, error) { + return FindPid(d, PatPid) } // FindPID will take a clip of mpegts and try to find a packet with given PID - if one // is found, then it is returned along with its index, otherwise nil, -1 and an error is returned. -func FindPID(d []byte, pid uint16) (pkt []byte, i int, err error) { +func FindPid(d []byte, pid uint16) (pkt []byte, i int, err error) { if len(d) < PacketSize { return nil, -1, errors.New("Mmpegts data not of valid length") } @@ -183,7 +183,7 @@ func FindPID(d []byte, pid uint16) (pkt []byte, i int, err error) { return } } - return nil, -1, fmt.Errorf("could not find packet with pid: %d",pid) + return nil, -1, fmt.Errorf("could not find packet with pid: %d", pid) } // FillPayload takes a channel and fills the packets Payload field until the From a7989955ca84d713a59f99fe40c8e60f8fc9f48a Mon Sep 17 00:00:00 2001 From: Saxon Date: Thu, 28 Feb 2019 11:06:27 +1030 Subject: [PATCH 201/208] stream/mts: corrected func comments for FindPat, FindPmt and FindPid --- stream/mts/mpegts.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stream/mts/mpegts.go b/stream/mts/mpegts.go index c3c0d390..705f88de 100644 --- a/stream/mts/mpegts.go +++ b/stream/mts/mpegts.go @@ -158,19 +158,19 @@ type Packet struct { Payload []byte // Mpeg ts Payload } -// FindPMT will take a clip of mpegts and try to find a PMT table - if one +// FindPmt will take a clip of mpegts and try to find a PMT table - if one // is found, then it is returned along with its index, otherwise nil, -1 and an error is returned. func FindPmt(d []byte) ([]byte, int, error) { return FindPid(d, PmtPid) } -// FindPAT will take a clip of mpegts and try to find a PAT table - if one +// FindPat will take a clip of mpegts and try to find a PAT table - if one // is found, then it is returned along with its index, otherwise nil, -1 and an error is returned. func FindPat(d []byte) ([]byte, int, error) { return FindPid(d, PatPid) } -// FindPID will take a clip of mpegts and try to find a packet with given PID - if one +// FindPid will take a clip of mpegts and try to find a packet with given PID - if one // is found, then it is returned along with its index, otherwise nil, -1 and an error is returned. func FindPid(d []byte, pid uint16) (pkt []byte, i int, err error) { if len(d) < PacketSize { From 589d922ddb2d81d58491d29f327d258511b765f7 Mon Sep 17 00:00:00 2001 From: Saxon Date: Thu, 28 Feb 2019 14:12:44 +1030 Subject: [PATCH 202/208] cmd/revid-cli: verbosity flag default to Info --- cmd/revid-cli/main.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 11f410b7..60dd4246 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -156,8 +156,6 @@ func handleFlags() revid.Config { cfg.LogLevel = logger.Error case "Fatal": cfg.LogLevel = logger.Fatal - case "": - cfg.LogLevel = defaultLogVerbosity default: log.Log(logger.Error, pkg+"bad verbosity argument") } From b9bccaabdde89243b261766f598afe27195f6813 Mon Sep 17 00:00:00 2001 From: Saxon Date: Thu, 28 Feb 2019 16:53:06 +1030 Subject: [PATCH 203/208] cmd/revid-cli: fixed default value for log verbosity flag --- cmd/revid-cli/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 60dd4246..064be699 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -120,7 +120,7 @@ func handleFlags() revid.Config { rtmpMethodPtr = flag.String("RtmpMethod", "", "The method used to send over rtmp: Ffmpeg, Librtmp") packetizationPtr = flag.String("Packetization", "", "The method of data packetisation: Flv, Mpegts, None") quantizePtr = flag.Bool("Quantize", false, "Quantize input (non-variable bitrate)") - verbosityPtr = flag.String("Verbosity", "", "Verbosity: Info, Warning, Error, Fatal") + verbosityPtr = flag.String("Verbosity", "Info", "Verbosity: Info, Warning, Error, Fatal") framesPerClipPtr = flag.Uint("FramesPerClip", 0, "Number of frames per clip sent") rtmpUrlPtr = flag.String("RtmpUrl", "", "Url of rtmp endpoint") bitratePtr = flag.Uint("Bitrate", 0, "Bitrate of recorded video") @@ -157,7 +157,7 @@ func handleFlags() revid.Config { case "Fatal": cfg.LogLevel = logger.Fatal default: - log.Log(logger.Error, pkg+"bad verbosity argument") + cfg.LogLevel = defaultLogVerbosity } log = logger.New(cfg.LogLevel, &smartlogger.New(*logPathPtr).LogRoller) From abe972235028ca45c8cbaf408c8cfe8052c47486 Mon Sep 17 00:00:00 2001 From: Saxon Date: Fri, 1 Mar 2019 21:01:33 +1030 Subject: [PATCH 204/208] revid: relaxed error handling in handleErrors routine --- revid/revid.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/revid/revid.go b/revid/revid.go index f3030b3a..48e35a87 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -204,11 +204,11 @@ func (r *Revid) handleErrors() { r.config.Logger.Log(logger.Error, pkg+"async error", "error", err.Error()) err = r.Stop() if err != nil { - r.config.Logger.Log(logger.Fatal, pkg+"failed to stop", "error", err.Error()) + r.config.Logger.Log(logger.Error, pkg+"failed to stop in response to error", "error", err.Error()) } err = r.Start() if err != nil { - r.config.Logger.Log(logger.Fatal, pkg+"failed to restart", "error", err.Error()) + r.config.Logger.Log(logger.Error, pkg+"failed to restart in response to error", "error", err.Error()) } } } @@ -359,7 +359,7 @@ func (r *Revid) Start() error { // Stop halts any processing of video data from a camera or file func (r *Revid) Stop() error { if !r.IsRunning() { - return errors.New(pkg + "stop called but revid is already stopped") + return errors.New(pkg + "stop called but revid isn't running") } r.config.Logger.Log(logger.Info, pkg+"stopping revid") From c9996cbae56e93899bff055a0607d68e9df56e98 Mon Sep 17 00:00:00 2001 From: Saxon Date: Sat, 2 Mar 2019 12:42:36 +1030 Subject: [PATCH 205/208] revid: stop() no longer returns error, just logs if stop is called when revid is not running. --- revid/revid.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/revid/revid.go b/revid/revid.go index 48e35a87..edbfc3d0 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -357,9 +357,10 @@ func (r *Revid) Start() error { } // Stop halts any processing of video data from a camera or file -func (r *Revid) Stop() error { +func (r *Revid) Stop() { if !r.IsRunning() { - return errors.New(pkg + "stop called but revid isn't running") + r.config.Logger.Log(logger.Warning, pkg+"stop called but revid isn't running") + return } r.config.Logger.Log(logger.Info, pkg+"stopping revid") @@ -371,7 +372,6 @@ func (r *Revid) Stop() error { r.cmd.Process.Kill() } r.wg.Wait() - return nil } func (r *Revid) Update(vars map[string]string) error { From 4e1ceb140dd68c0be62ec49a8e06559011273511 Mon Sep 17 00:00:00 2001 From: Saxon Date: Sat, 2 Mar 2019 12:49:09 +1030 Subject: [PATCH 206/208] revid: if start is called when revid is already running an error is no longer returned --- revid/revid.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/revid/revid.go b/revid/revid.go index edbfc3d0..ee6bc992 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -202,10 +202,7 @@ func (r *Revid) handleErrors() { err := <-r.err if err != nil { r.config.Logger.Log(logger.Error, pkg+"async error", "error", err.Error()) - err = r.Stop() - if err != nil { - r.config.Logger.Log(logger.Error, pkg+"failed to stop in response to error", "error", err.Error()) - } + r.Stop() err = r.Start() if err != nil { r.config.Logger.Log(logger.Error, pkg+"failed to restart in response to error", "error", err.Error()) @@ -342,7 +339,8 @@ func (r *Revid) setIsRunning(b bool) { // and packetising (if theres packetization) to a defined output. func (r *Revid) Start() error { if r.IsRunning() { - return errors.New(pkg + "start called but revid is already running") + r.config.Logger.Log(logger.Warning, pkg+"start called, but revid already running") + return nil } r.config.Logger.Log(logger.Info, pkg+"starting Revid") // TODO: this doesn't need to be here @@ -376,9 +374,7 @@ func (r *Revid) Stop() { func (r *Revid) Update(vars map[string]string) error { if r.IsRunning() { - if err := r.Stop(); err != nil { - return err - } + r.Stop() } //look through the vars and update revid where needed for key, value := range vars { From c3f846379d65b90db777cd04c7d430b8c6b15d76 Mon Sep 17 00:00:00 2001 From: Saxon Date: Sat, 2 Mar 2019 12:50:43 +1030 Subject: [PATCH 207/208] revid: removed in response to error in log message in handleErrors if failed to start() revid --- revid/revid.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/revid/revid.go b/revid/revid.go index ee6bc992..474280ef 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -205,7 +205,7 @@ func (r *Revid) handleErrors() { r.Stop() err = r.Start() if err != nil { - r.config.Logger.Log(logger.Error, pkg+"failed to restart in response to error", "error", err.Error()) + r.config.Logger.Log(logger.Error, pkg+"failed to restart revid", "error", err.Error()) } } } From 3c273adf4c866fbab4ee7d7a6206bc7c239b23d2 Mon Sep 17 00:00:00 2001 From: Saxon Date: Sat, 2 Mar 2019 13:30:46 +1030 Subject: [PATCH 208/208] cmd/revid-cli: not expecting error from revid.Stop() in revid-cli anymore: --- cmd/revid-cli/main.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index 064be699..056687de 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -96,9 +96,7 @@ func main() { cfg.Logger.Log(logger.Fatal, pkg+"failed to start revid", "error", err.Error()) } time.Sleep(*runDurationPtr) - if err = rv.Stop(); err != nil { - cfg.Logger.Log(logger.Error, pkg+"failed to stop revid before program termination", "error", err.Error()) - } + rv.Stop() return } @@ -360,10 +358,7 @@ func run(cfg revid.Config) error { } time.Sleep(time.Duration(rv.Config().BurstPeriod) * time.Second) log.Log(logger.Info, pkg+"Stopping burst...") - err = rv.Stop() - if err != nil { - return err - } + rv.Stop() ns.SetMode(paused, &vs) } sleep: