From 5f0bef93650d023913d3403e554a881228aa7312 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 4 Dec 2018 16:03:04 +1030 Subject: [PATCH 01/54] psi: writing new data strcutres to make things neater and more usable --- stream/mts/psi/pmt.go | 16 +++++++++++++++- stream/mts/psi/psi.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 stream/mts/psi/psi.go diff --git a/stream/mts/psi/pmt.go b/stream/mts/psi/pmt.go index 2971e8c0..9def4e60 100644 --- a/stream/mts/psi/pmt.go +++ b/stream/mts/psi/pmt.go @@ -27,6 +27,18 @@ LICENSE package psi +// Bitmasks +const ( + // bitmasks used for second byte + SYNTAX_INDICATOR = 0x80 + PRIVATE_BIT = 0x40 + RESERVED_BITS = 0x30 + SECTION_LENGTH_1 = 0x03 // first two bits of the section length + + // bitmasks used for third byte + SECTION_LENGTH_2 = 0xff +) + type PMT struct { PF byte // Point field PFB []byte // pointer filler bytes @@ -39,9 +51,11 @@ type PMT struct { CNI bool // Current/next indicator Section byte // Section number LSN byte // Last section number - } +func ReadPMT(pmtData []byte) (PMT, error) { + return PMT{}, nil +} func (p *PMT) ToByteSlice() (output []byte) { return } diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go new file mode 100644 index 00000000..13816f15 --- /dev/null +++ b/stream/mts/psi/psi.go @@ -0,0 +1,36 @@ +package psi + +// Program specific information +type PSI struct { + PF byte // Point field + PFB []byte // pointer filler bytes + TableID byte // Table ID + SSI bool // Sectiopn syntax indicator (1 for PAT, PMT, CAT) + PB bool // Private bit (0 for PAT, PMT, CAT) + SL uint16 // Section length + tss *TSS // Table syntax section (length defined by SL) if length 0 then nil +} + +// Table syntax section +type TSS struct { + TIE uint16 // Table ID extension + Version byte // Version number + CNI bool // Current/next indicator + Section byte // Section number + LSN byte // Last section number + table SD // specific data PAT/PMT +} + +// Specific Data, which could be PAT or PMT +type SD interface { + Read(psiData []byte) (PSI, error) + ToByte() []byte +} + +func (p *PSI) Read(psiData []byte) (PSI, error) { + return PSI{}, nil +} + +func (p *PSI) ToByte() []byte { + return nil +} From 731285d665b56b4366a911e260f2af8a186c827b Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 5 Dec 2018 19:31:29 +1030 Subject: [PATCH 02/54] psi: restructuring data structures and files --- stream/mts/psi/pat.go | 51 --------------------------------- stream/mts/psi/pmt.go | 61 ---------------------------------------- stream/mts/psi/psi.go | 65 ++++++++++++++++++++++++++++++------------- 3 files changed, 46 insertions(+), 131 deletions(-) delete mode 100644 stream/mts/psi/pat.go delete mode 100644 stream/mts/psi/pmt.go diff --git a/stream/mts/psi/pat.go b/stream/mts/psi/pat.go deleted file mode 100644 index d090a0d7..00000000 --- a/stream/mts/psi/pat.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -NAME - revid - a testbed for re-muxing and re-directing video streams as MPEG-TS over various protocols. - -DESCRIPTION - See Readme.md - -AUTHOR - Alan Noble - -LICENSE - revid is Copyright (C) 2017 Alan Noble. - - It is free software: you can redistribute it and/or modify them - under the terms of the GNU General Public License as published by the - Free Software Foundation, either version 3 of the License, or (at your - option) any later version. - - It is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License - along with revid in gpl.txt. If not, see [GNU licenses](http://www.gnu.org/licenses). -*/ - -package psi - -type PAT struct { - PF byte // Point field - PFB []byte // pointer filler bytes - TableID byte // Table ID - SSI bool // Sectiopn syntax indicator (1 for PAT, PMT, CAT) - PB bool // Private bit (0 for PAT, PMT, CAT) - SL uint16 // Section length - TIE uint16 // Table ID extension - Version byte // Version number - CNI bool // Current/next indicator - Section byte // Section number - LSN byte // Last section number - DT byte // Descriptor tag - DL byte // Descriptor length - Program uint16 // Program number - PMPID uint16 // Program map PID - CRC32 uint32 // Checksum of table -} - -func (p *PAT) ToByteSlice() (output []byte) { - return -} diff --git a/stream/mts/psi/pmt.go b/stream/mts/psi/pmt.go deleted file mode 100644 index 9def4e60..00000000 --- a/stream/mts/psi/pmt.go +++ /dev/null @@ -1,61 +0,0 @@ -/* -NAME - revid - a testbed for re-muxing and re-directing video streams as MPEG-TS over various protocols. - -DESCRIPTION - See Readme.md - -AUTHOR - Alan Noble - -LICENSE - revid is Copyright (C) 2017 Alan Noble. - - It is free software: you can redistribute it and/or modify them - under the terms of the GNU General Public License as published by the - Free Software Foundation, either version 3 of the License, or (at your - option) any later version. - - It is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License - along with revid in gpl.txt. If not, see [GNU licenses](http://www.gnu.org/licenses). -*/ - -package psi - -// Bitmasks -const ( - // bitmasks used for second byte - SYNTAX_INDICATOR = 0x80 - PRIVATE_BIT = 0x40 - RESERVED_BITS = 0x30 - SECTION_LENGTH_1 = 0x03 // first two bits of the section length - - // bitmasks used for third byte - SECTION_LENGTH_2 = 0xff -) - -type PMT struct { - PF byte // Point field - PFB []byte // pointer filler bytes - TableID byte // Table ID - SSI bool // Sectiopn syntax indicator (1 for PAT, PMT, CAT) - PB bool // Private bit (0 for PAT, PMT, CAT) - SL uint16 // Section length - TIE uint16 // Table ID extension - Version byte // Version number - CNI bool // Current/next indicator - Section byte // Section number - LSN byte // Last section number -} - -func ReadPMT(pmtData []byte) (PMT, error) { - return PMT{}, nil -} -func (p *PMT) ToByteSlice() (output []byte) { - return -} diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 13816f15..31393c00 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -2,35 +2,62 @@ package psi // Program specific information type PSI struct { - PF byte // Point field - PFB []byte // pointer filler bytes - TableID byte // Table ID - SSI bool // Sectiopn syntax indicator (1 for PAT, PMT, CAT) - PB bool // Private bit (0 for PAT, PMT, CAT) - SL uint16 // Section length - tss *TSS // Table syntax section (length defined by SL) if length 0 then nil + pf byte // Point field + pfb []byte // pointer filler bytes + tid byte // Table ID + ssi bool // Sectiopn syntax indicator (1 for PAT, PMT, CAT) + pb bool // Private bit (0 for PAT, PMT, CAT) + sl uint16 // Section length + tss *TSS // Table syntax section (length defined by SL) if length 0 then nil } // Table syntax section type TSS struct { - TIE uint16 // Table ID extension - Version byte // Version number - CNI bool // Current/next indicator - Section byte // Section number - LSN byte // Last section number - table SD // specific data PAT/PMT + tie uint16 // Table ID extension + v byte // Version number + cni bool // Current/next indicator + sn byte // Section number + lsn byte // Last section number + sd SD // specific data PAT/PMT + crc []byte // crc32 of entire table excluding pointer field, pointer filler bytes and the trailing CRC32 } -// Specific Data, which could be PAT or PMT +// Specific Data, (could be PAT or PMT) type SD interface { - Read(psiData []byte) (PSI, error) - ToByte() []byte + Bytes() []byte } -func (p *PSI) Read(psiData []byte) (PSI, error) { - return PSI{}, nil +// Program association table, implements SD +type PAT struct { + pn uint16 // Program Number + pmpid uint16 // program map PID } -func (p *PSI) ToByte() []byte { +type Descriptor struct { + dt byte // Descriptor tag + dl byte // Descriptor length + dd []byte // Descriptor data +} + +// Program mapping table, implements SD +type PMT struct { + pcrpid uint16 // program clock reference pid + PIL uint16 // program info length + PD []Descriptor // program descriptors +} + +func ReadPSI(data []byte) *PAT { + return nil +} + +func (p *PSI) Bytes() (output []byte) { + return nil +} + +func (p *PAT) Bytes() (output []byte) { + return nil +} + +func (p *PMT) Bytes() (output []byte) { return nil } From fe11ce6f08a3bb320c166f89fb7da7438f3b6616 Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 5 Dec 2018 21:46:06 +1030 Subject: [PATCH 03/54] psi: added essd data structure, isolated space check to minimise repetition --- stream/mts/psi/psi.go | 73 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 15 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 31393c00..5eb9ce6e 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -1,9 +1,14 @@ package psi +const ( + ESSDHeadLen = 5 + DescriptorHeadLen = 2 +) + // Program specific information type PSI struct { pf byte // Point field - pfb []byte // pointer filler bytes + pfb []byte // Pointer filler bytes tid byte // Table ID ssi bool // Sectiopn syntax indicator (1 for PAT, PMT, CAT) pb bool // Private bit (0 for PAT, PMT, CAT) @@ -18,7 +23,7 @@ type TSS struct { cni bool // Current/next indicator sn byte // Section number lsn byte // Last section number - sd SD // specific data PAT/PMT + sd SD // Specific data PAT/PMT crc []byte // crc32 of entire table excluding pointer field, pointer filler bytes and the trailing CRC32 } @@ -30,34 +35,72 @@ type SD interface { // Program association table, implements SD type PAT struct { pn uint16 // Program Number - pmpid uint16 // program map PID + pmpid uint16 // Program map PID } -type Descriptor struct { +// Program mapping table, implements SD +type PMT struct { + pcrpid uint16 // Program clock reference pid + pil uint16 // Program info length + pd []Descriptor // Program descriptors + essd []ESSD // Elementary stream specific data +} + +// Elementary stream specific data +type ESSD struct { + st byte // Stream type + epid byte // Elementary pid + esil uint16 // Elementary stream + esd []Descriptor // Elementary stream desriptors +} + +// Descriptor +type Desc struct { dt byte // Descriptor tag dl byte // Descriptor length dd []byte // Descriptor data } -// Program mapping table, implements SD -type PMT struct { - pcrpid uint16 // program clock reference pid - PIL uint16 // program info length - PD []Descriptor // program descriptors -} - -func ReadPSI(data []byte) *PAT { +func ReadPSI(data []byte) *PSI { return nil } -func (p *PSI) Bytes() (output []byte) { +func (p *PSI) Bytes() (out []byte) { return nil } -func (p *PAT) Bytes() (output []byte) { +func (t *TSS) Fill(space []byte) error { return nil } -func (p *PMT) Bytes() (output []byte) { +func (p *PAT) Fill(space []byte) error { return nil } + +func (p *PMT) Fill(space []byte) error { + checkSpace(space) + return nil +} + +func (d *Descriptor) Fill(space []byte) error { + checkSpace(space, DescriptorHeadLen+int(d.dl)) + space[0] = d.dt + space[1] = d.dl + copy(space[2:], d.dd) + return out +} + +func (e *ESSD) Fill(space []byte) error { + checkSpace(space, ESSDHeadLen+e.esil) + return nil +} + +func checkSpace(s []byte, c uint) { + if s == nil { + panic("Slice provided is nil") + } else if len(s) != 0 { + panic("Slice provided already has something in it") + } else if cap(s) != c { + panic("Slice provided has wrong capacity") + } +} From 2a589be6bf9af6c3f94e20414408fce36a38077e Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 5 Dec 2018 21:47:16 +1030 Subject: [PATCH 04/54] psi: using Desc instead of Descriptor --- stream/mts/psi/psi.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 5eb9ce6e..ba1daf58 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -1,8 +1,8 @@ package psi const ( - ESSDHeadLen = 5 - DescriptorHeadLen = 2 + ESSDHeadLen = 5 + DescHeadLen = 2 ) // Program specific information @@ -40,18 +40,18 @@ type PAT struct { // Program mapping table, implements SD type PMT struct { - pcrpid uint16 // Program clock reference pid - pil uint16 // Program info length - pd []Descriptor // Program descriptors - essd []ESSD // Elementary stream specific data + pcrpid uint16 // Program clock reference pid + pil uint16 // Program info length + pd []Desc // Program descriptors + essd []ESSD // Elementary stream specific data } // Elementary stream specific data type ESSD struct { - st byte // Stream type - epid byte // Elementary pid - esil uint16 // Elementary stream - esd []Descriptor // Elementary stream desriptors + st byte // Stream type + epid byte // Elementary pid + esil uint16 // Elementary stream + esd []Desc // Elementary stream desriptors } // Descriptor @@ -82,8 +82,8 @@ func (p *PMT) Fill(space []byte) error { return nil } -func (d *Descriptor) Fill(space []byte) error { - checkSpace(space, DescriptorHeadLen+int(d.dl)) +func (d *Desc) Fill(space []byte) error { + checkSpace(space, DescHeadLen+int(d.dl)) space[0] = d.dt space[1] = d.dl copy(space[2:], d.dd) From e2a5e6a16a4cd63a4e18b422d3b12992b8900e9c Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 6 Dec 2018 00:04:19 +1030 Subject: [PATCH 05/54] psi: re-wrote bytes for desc and essd to make less bug prone, though at a cost for performance - but optimisation can happen later --- stream/mts/psi/psi.go | 53 ++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index ba1daf58..409858f7 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -10,7 +10,7 @@ type PSI struct { pf byte // Point field pfb []byte // Pointer filler bytes tid byte // Table ID - ssi bool // Sectiopn syntax indicator (1 for PAT, PMT, CAT) + ssi bool // Section syntax indicator (1 for PAT, PMT, CAT) pb bool // Private bit (0 for PAT, PMT, CAT) sl uint16 // Section length tss *TSS // Table syntax section (length defined by SL) if length 0 then nil @@ -69,38 +69,33 @@ func (p *PSI) Bytes() (out []byte) { return nil } -func (t *TSS) Fill(space []byte) error { +func (t *TSS) Fill(s []byte) { +} + +func (p *PAT) Fill(s []byte) { +} + +func (p *PMT) Bytes() (out []byte) { return nil } -func (p *PAT) Fill(space []byte) error { - return nil +func (d *Desc) Bytes() (out []byte) { + out = make([]byte, DescHeadLen) + out[0] = d.dt + out[1] = d.dl + out = append(out, d.dd...) + return } -func (p *PMT) Fill(space []byte) error { - checkSpace(space) - return nil -} - -func (d *Desc) Fill(space []byte) error { - checkSpace(space, DescHeadLen+int(d.dl)) - space[0] = d.dt - space[1] = d.dl - copy(space[2:], d.dd) - return out -} - -func (e *ESSD) Fill(space []byte) error { - checkSpace(space, ESSDHeadLen+e.esil) - return nil -} - -func checkSpace(s []byte, c uint) { - if s == nil { - panic("Slice provided is nil") - } else if len(s) != 0 { - panic("Slice provided already has something in it") - } else if cap(s) != c { - panic("Slice provided has wrong capacity") +func (e *ESSD) Bytes() (out []byte) { + out = make([]byte, ESSDHeadLen) + out[0] = e.st + out[1] = 0xe0 | e.epid>>3 + out[2] = e.epid + out[3] = 0xf0 | byte(e.esil>>6) + out[4] = byte(e.esil) + for _, d := range e.esd { + out = append(out, d.Bytes()...) } + return } From 1fd9bed1e4d1caae9c72432df18361ddcf3abfae Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 6 Dec 2018 00:34:29 +1030 Subject: [PATCH 06/54] psi: wrote bytes() for pmt --- stream/mts/psi/psi.go | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 409858f7..ec56e8aa 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -3,6 +3,7 @@ package psi const ( ESSDHeadLen = 5 DescHeadLen = 2 + PMTHeadLen = 2 ) // Program specific information @@ -49,7 +50,7 @@ type PMT struct { // Elementary stream specific data type ESSD struct { st byte // Stream type - epid byte // Elementary pid + epid uint16 // Elementary pid esil uint16 // Elementary stream esd []Desc // Elementary stream desriptors } @@ -75,27 +76,38 @@ func (t *TSS) Fill(s []byte) { func (p *PAT) Fill(s []byte) { } -func (p *PMT) Bytes() (out []byte) { - return nil +func (p *PMT) Bytes() []byte { + out := make([]byte, PMTHeadLen) + out[0] = 0xe0 | (0x1f & byte(p.pcrpid>>8)) + out[1] = byte(p.pcrpid) + out[2] = 0xf0 | (0x03 & byte(p.pil>>8)) + out[3] = byte(p.pil) + for _, d := range p.pd { + out = append(out, d.Bytes()...) + } + for _, e := range p.essd { + out = append(out, e.Bytes()...) + } + return out } -func (d *Desc) Bytes() (out []byte) { - out = make([]byte, DescHeadLen) +func (d *Desc) Bytes() []byte { + out := make([]byte, DescHeadLen) out[0] = d.dt out[1] = d.dl out = append(out, d.dd...) - return + return out } -func (e *ESSD) Bytes() (out []byte) { - out = make([]byte, ESSDHeadLen) +func (e *ESSD) Bytes() []byte { + out := make([]byte, ESSDHeadLen) out[0] = e.st - out[1] = 0xe0 | e.epid>>3 - out[2] = e.epid - out[3] = 0xf0 | byte(e.esil>>6) + out[1] = 0xe0 | (0x1f & byte(e.epid>>8)) + out[2] = byte(e.epid) + out[3] = 0xf0 | (0x03 & byte(e.esil>>8)) out[4] = byte(e.esil) for _, d := range e.esd { out = append(out, d.Bytes()...) } - return + return out } From fa0498bc52b5a88efce87a5006893c4b95d9ab66 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 6 Dec 2018 00:41:53 +1030 Subject: [PATCH 07/54] psi: wrote bytes() for pat --- stream/mts/psi/psi.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index ec56e8aa..3047106f 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -1,9 +1,10 @@ package psi const ( - ESSDHeadLen = 5 - DescHeadLen = 2 - PMTHeadLen = 2 + ESSDDefLen = 5 + DescDefLen = 2 + PMTDefLen = 4 + PATLen = 4 ) // Program specific information @@ -73,11 +74,17 @@ func (p *PSI) Bytes() (out []byte) { func (t *TSS) Fill(s []byte) { } -func (p *PAT) Fill(s []byte) { +func (p *PAT) Bytes() []byte { + out := make([]byte, PATLen) + out[0] = byte(p.pn >> 8) + out[1] = byte(p.pn) + out[2] = 0xe0 | (0x1f & byte(p.pmpid>>8)) + out[3] = byte(p.pmpid) + return out } func (p *PMT) Bytes() []byte { - out := make([]byte, PMTHeadLen) + out := make([]byte, PMTDefLen) out[0] = 0xe0 | (0x1f & byte(p.pcrpid>>8)) out[1] = byte(p.pcrpid) out[2] = 0xf0 | (0x03 & byte(p.pil>>8)) @@ -92,7 +99,7 @@ func (p *PMT) Bytes() []byte { } func (d *Desc) Bytes() []byte { - out := make([]byte, DescHeadLen) + out := make([]byte, DescDefLen) out[0] = d.dt out[1] = d.dl out = append(out, d.dd...) @@ -100,7 +107,7 @@ func (d *Desc) Bytes() []byte { } func (e *ESSD) Bytes() []byte { - out := make([]byte, ESSDHeadLen) + out := make([]byte, ESSDDefLen) out[0] = e.st out[1] = 0xe0 | (0x1f & byte(e.epid>>8)) out[2] = byte(e.epid) From 42038a8cb9d3016abc96805e8f1dc292d98a71d4 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 6 Dec 2018 01:28:14 +1030 Subject: [PATCH 08/54] psi: wrote Bytes() for TSS and almost done writing for PSI --- stream/mts/psi/psi.go | 47 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 3047106f..05cd2c3e 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -5,6 +5,8 @@ const ( DescDefLen = 2 PMTDefLen = 4 PATLen = 4 + TSSDefLen = 5 + PSIDefLen = 3 ) // Program specific information @@ -16,17 +18,17 @@ type PSI struct { pb bool // Private bit (0 for PAT, PMT, CAT) sl uint16 // Section length tss *TSS // Table syntax section (length defined by SL) if length 0 then nil + crc uint32 // crc32 of entire table excluding pointer field, pointer filler bytes and the trailing CRC32 } // Table syntax section type TSS struct { - tie uint16 // Table ID extension - v byte // Version number - cni bool // Current/next indicator - sn byte // Section number - lsn byte // Last section number - sd SD // Specific data PAT/PMT - crc []byte // crc32 of entire table excluding pointer field, pointer filler bytes and the trailing CRC32 + tide uint16 // Table ID extension + v byte // Version number + cni bool // Current/next indicator + sn byte // Section number + lsn byte // Last section number + sd SD // Specific data PAT/PMT } // Specific Data, (could be PAT or PMT) @@ -63,15 +65,35 @@ type Desc struct { dd []byte // Descriptor data } +// TODO: Implement this func ReadPSI(data []byte) *PSI { return nil } -func (p *PSI) Bytes() (out []byte) { +func (p *PSI) Bytes() []byte { + l := 1 + len(p.pfb) + out := make([]byte, l+PSIDefLen) + out[0] = p.pf + for i, b := range p.pfb { + out[1+i] = b + } + out[l] = p.tid + out[l+1] = 0x80 | 0x40 | 0x30 | (0x03 & byte(p.sl>>8)) + out[l+2] = byte(p.sl) + out = append(out, p.tss.Bytes()...) + crc32 := crc32_op() return nil } -func (t *TSS) Fill(s []byte) { +func (t *TSS) Bytes() []byte { + out := make([]byte, TSSDefLen) + out[0] = byte(t.tide >> 8) + out[1] = byte(t.tide) + out[2] = 0xc0 | (0x3e & (t.v << 1)) | (0x01 & boolToByte(t.cni)) + out[3] = t.sn + out[4] = t.lsn + out = append(out, t.sd.Bytes()...) + return out } func (p *PAT) Bytes() []byte { @@ -118,3 +140,10 @@ func (e *ESSD) Bytes() []byte { } return out } + +func boolToByte(b bool) byte { + if b { + return 0x01 + } + return 0x00 +} From ba35615964e0515bfe0802799332a544167780af Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 6 Dec 2018 12:16:26 +1030 Subject: [PATCH 09/54] psi: completed writing Bytes() for PSI table --- stream/mts/psi/psi.go | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 05cd2c3e..608aedec 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -1,5 +1,10 @@ package psi +import ( + "hash/crc32" + "math/bits" +) + const ( ESSDDefLen = 5 DescDefLen = 2 @@ -81,8 +86,13 @@ func (p *PSI) Bytes() []byte { out[l+1] = 0x80 | 0x40 | 0x30 | (0x03 & byte(p.sl>>8)) out[l+2] = byte(p.sl) out = append(out, p.tss.Bytes()...) - crc32 := crc32_op() - return nil + crc32 := crc32_Update(0xffffffff, crc32_MakeTable(bits.Reverse32(crc32.IEEE)), out[l:]) + out = append(out, make([]byte, 0, 4)...) + out = append(out, byte(crc32>>24)) + out = append(out, byte(crc32>>16)) + out = append(out, byte(crc32>>8)) + out = append(out, byte(crc32)) + return out } func (t *TSS) Bytes() []byte { @@ -147,3 +157,26 @@ func boolToByte(b bool) byte { } return 0x00 } + +func crc32_MakeTable(poly uint32) *crc32.Table { + var t crc32.Table + for i := range t { + crc := uint32(i) << 24 + for j := 0; j < 8; j++ { + if crc&0x80000000 != 0 { + crc = (crc << 1) ^ poly + } else { + crc <<= 1 + } + } + t[i] = crc + } + return &t +} + +func crc32_Update(crc uint32, tab *crc32.Table, p []byte) uint32 { + for _, v := range p { + crc = tab[byte(crc>>24)^v] ^ (crc << 8) + } + return crc +} From 2ffa0f9b7ba1cdf46723fd9e21b6cd899f2280d4 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 6 Dec 2018 13:50:17 +1030 Subject: [PATCH 10/54] psi: starting to write read functions for psi and tss --- stream/mts/psi/psi.go | 182 ++++++++++++++++++++++++++++-------------- 1 file changed, 122 insertions(+), 60 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 608aedec..3fb57c86 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -5,6 +5,7 @@ import ( "math/bits" ) +// Some common lengths const ( ESSDDefLen = 5 DescDefLen = 2 @@ -14,26 +15,32 @@ const ( PSIDefLen = 3 ) +// Table Type IDs +const ( + PATTableID = 0x00 + PMTTableID = 0x02 +) + // Program specific information type PSI struct { - pf byte // Point field - pfb []byte // Pointer filler bytes - tid byte // Table ID - ssi bool // Section syntax indicator (1 for PAT, PMT, CAT) - pb bool // Private bit (0 for PAT, PMT, CAT) - sl uint16 // Section length - tss *TSS // Table syntax section (length defined by SL) if length 0 then nil - crc uint32 // crc32 of entire table excluding pointer field, pointer filler bytes and the trailing CRC32 + Pf byte // Point field + Pfb []byte // Pointer filler bytes + Tid byte // Table ID + Ssi bool // Section syntax indicator (1 for PAT, PMT, CAT) + Pb bool // Private bit (0 for PAT, PMT, CAT) + Sl uint16 // Section length + Tss *TSS // Table syntax section (length defined by SL) if length 0 then nil + Crc uint32 // crc32 of entire table excluding pointer field, pointer filler bytes and the trailing CRC32 } // Table syntax section type TSS struct { - tide uint16 // Table ID extension - v byte // Version number - cni bool // Current/next indicator - sn byte // Section number - lsn byte // Last section number - sd SD // Specific data PAT/PMT + Tide uint16 // Table ID extension + V byte // Version number + Cni bool // Current/next indicator + Sn byte // Section number + Lsn byte // Last section number + Sd SD // Specific data PAT/PMT } // Specific Data, (could be PAT or PMT) @@ -43,49 +50,92 @@ type SD interface { // Program association table, implements SD type PAT struct { - pn uint16 // Program Number - pmpid uint16 // Program map PID + Pn uint16 // Program Number + Pmpid uint16 // Program map PID } // Program mapping table, implements SD type PMT struct { - pcrpid uint16 // Program clock reference pid - pil uint16 // Program info length - pd []Desc // Program descriptors - essd []ESSD // Elementary stream specific data + Pcrpid uint16 // Program clock reference pid + Pil uint16 // Program info length + Pd []Desc // Program descriptors + Essd []ESSD // Elementary stream specific data } // Elementary stream specific data type ESSD struct { - st byte // Stream type - epid uint16 // Elementary pid - esil uint16 // Elementary stream - esd []Desc // Elementary stream desriptors + St byte // Stream type + Epid uint16 // Elementary pid + Esil uint16 // Elementary stream + Esd []Desc // Elementary stream desriptors } // Descriptor type Desc struct { - dt byte // Descriptor tag - dl byte // Descriptor length - dd []byte // Descriptor data + Dt byte // Descriptor tag + Dl byte // Descriptor length + Dd []byte // Descriptor data } -// TODO: Implement this +// ReadPSI creates a PSI data structure from a given byte slice that represents a PSI func ReadPSI(data []byte) *PSI { + psi := PSI{} + pos := 0 + psi.Pf = data[pos] + if psi.Pf != 0 { + psi.Pfb = make([]byte, 0, psi.Pf) + pos++ + for i := 0; i < int(psi.Pf); i++ { + psi.Pfb = append(psi.Pfb, data[pos]) + pos++ + } + } + psi.Tid = data[pos] + pos++ + psi.Ssi = byteToBool(data[pos] & 0x80) + psi.Pb = byteToBool(data[pos] & 0x40) + psi.Sl = uint16(data[pos]&0x03)<<8 | uint16(data[pos+1]) + pos += 2 + psi.Tss = readTSS(data[pos:], &psi) + return &psi +} + +// ReadTSS creates a TSS data structure from a given byte slice that represents a TSS +func readTSS(data []byte, p *PSI) *TSS { + tss := TSS{} + pos := 0 + tss.Tide = uint16(data[pos])<<8 | uint16(data[pos+1]) + pos += 2 + tss.V = (data[pos] & 0x3e) >> 1 + tss.Cni = byteToBool(data[pos] & 0x01) + pos++ + tss.Sn = data[pos] + pos++ + tss.Lsn = data[pos] + pos++ + if p.Tid == PATTableID { + tss.Sd = readPAT(data, &tss) + } else if p.Tid == PMTTableID { + tss.Sd = readPMT(data, &tss) + } else { + panic("Can't yet deal with tables that are not PAT or PMT") + } + return nil } +// Bytes outputs a byte slice representation of the PSI func (p *PSI) Bytes() []byte { - l := 1 + len(p.pfb) + l := 1 + len(p.Pfb) out := make([]byte, l+PSIDefLen) - out[0] = p.pf - for i, b := range p.pfb { + out[0] = p.Pf + for i, b := range p.Pfb { out[1+i] = b } - out[l] = p.tid - out[l+1] = 0x80 | 0x40 | 0x30 | (0x03 & byte(p.sl>>8)) - out[l+2] = byte(p.sl) - out = append(out, p.tss.Bytes()...) + out[l] = p.Tid + out[l+1] = 0x80 | 0x40 | 0x30 | (0x03 & byte(p.Sl>>8)) + out[l+2] = byte(p.Sl) + out = append(out, p.Tss.Bytes()...) crc32 := crc32_Update(0xffffffff, crc32_MakeTable(bits.Reverse32(crc32.IEEE)), out[l:]) out = append(out, make([]byte, 0, 4)...) out = append(out, byte(crc32>>24)) @@ -95,57 +145,62 @@ func (p *PSI) Bytes() []byte { return out } +// Bytes outputs a byte slice representation of the TSS func (t *TSS) Bytes() []byte { out := make([]byte, TSSDefLen) - out[0] = byte(t.tide >> 8) - out[1] = byte(t.tide) - out[2] = 0xc0 | (0x3e & (t.v << 1)) | (0x01 & boolToByte(t.cni)) - out[3] = t.sn - out[4] = t.lsn - out = append(out, t.sd.Bytes()...) + out[0] = byte(t.Tide >> 8) + out[1] = byte(t.Tide) + out[2] = 0xc0 | (0x3e & (t.V << 1)) | (0x01 & boolToByte(t.Cni)) + out[3] = t.Sn + out[4] = t.Lsn + out = append(out, t.Sd.Bytes()...) return out } +// Bytes outputs a byte slice representation of the PAT func (p *PAT) Bytes() []byte { out := make([]byte, PATLen) - out[0] = byte(p.pn >> 8) - out[1] = byte(p.pn) - out[2] = 0xe0 | (0x1f & byte(p.pmpid>>8)) - out[3] = byte(p.pmpid) + out[0] = byte(p.Pn >> 8) + out[1] = byte(p.Pn) + out[2] = 0xe0 | (0x1f & byte(p.Pmpid>>8)) + out[3] = byte(p.Pmpid) return out } +// Bytes outputs a byte slice representation of the PMT func (p *PMT) Bytes() []byte { out := make([]byte, PMTDefLen) - out[0] = 0xe0 | (0x1f & byte(p.pcrpid>>8)) - out[1] = byte(p.pcrpid) - out[2] = 0xf0 | (0x03 & byte(p.pil>>8)) - out[3] = byte(p.pil) - for _, d := range p.pd { + out[0] = 0xe0 | (0x1f & byte(p.Pcrpid>>8)) + out[1] = byte(p.Pcrpid) + out[2] = 0xf0 | (0x03 & byte(p.Pil>>8)) + out[3] = byte(p.Pil) + for _, d := range p.Pd { out = append(out, d.Bytes()...) } - for _, e := range p.essd { + for _, e := range p.Essd { out = append(out, e.Bytes()...) } return out } +// Bytes outputs a byte slice representation of the Desc func (d *Desc) Bytes() []byte { out := make([]byte, DescDefLen) - out[0] = d.dt - out[1] = d.dl - out = append(out, d.dd...) + out[0] = d.Dt + out[1] = d.Dl + out = append(out, d.Dd...) return out } +// Bytes outputs a byte slice representation of the ESSD func (e *ESSD) Bytes() []byte { out := make([]byte, ESSDDefLen) - out[0] = e.st - out[1] = 0xe0 | (0x1f & byte(e.epid>>8)) - out[2] = byte(e.epid) - out[3] = 0xf0 | (0x03 & byte(e.esil>>8)) - out[4] = byte(e.esil) - for _, d := range e.esd { + out[0] = e.St + out[1] = 0xe0 | (0x1f & byte(e.Epid>>8)) + out[2] = byte(e.Epid) + out[3] = 0xf0 | (0x03 & byte(e.Esil>>8)) + out[4] = byte(e.Esil) + for _, d := range e.Esd { out = append(out, d.Bytes()...) } return out @@ -158,6 +213,13 @@ func boolToByte(b bool) byte { return 0x00 } +func byteToBool(b byte) bool { + if b == 0 { + return false + } + return true +} + func crc32_MakeTable(poly uint32) *crc32.Table { var t crc32.Table for i := range t { From 94cefe848acbdfa4b166154725ffe5b1bb7f3af0 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 6 Dec 2018 16:02:40 +1030 Subject: [PATCH 11/54] psi: wrote read func for pat, pmt, desc and started writing for essd --- stream/mts/psi/psi.go | 85 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 69 insertions(+), 16 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 3fb57c86..7366ee3b 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -21,6 +21,10 @@ const ( PMTTableID = 0x02 ) +const ( + timestampDescTag = 234 +) + // Program specific information type PSI struct { Pf byte // Point field @@ -59,7 +63,7 @@ type PMT struct { Pcrpid uint16 // Program clock reference pid Pil uint16 // Program info length Pd []Desc // Program descriptors - Essd []ESSD // Elementary stream specific data + Essd ESSD // Elementary stream specific data } // Elementary stream specific data @@ -113,17 +117,68 @@ func readTSS(data []byte, p *PSI) *TSS { pos++ tss.Lsn = data[pos] pos++ - if p.Tid == PATTableID { - tss.Sd = readPAT(data, &tss) - } else if p.Tid == PMTTableID { - tss.Sd = readPMT(data, &tss) - } else { + switch p.Tid { + case PATTableID: + tss.Sd = readPAT(data[pos:], &tss) + case PMTTableID: + tss.Sd = readPMT(data[pos:], &tss) + default: panic("Can't yet deal with tables that are not PAT or PMT") } + return &tss +} +func readPAT(data []byte, p *TSS) *PAT { + pat := PAT{} + pos := 0 + pat.Pn = uint16(data[pos])<<8 | uint16(data[pos+1]) + pos += 2 + pat.Pmpid = uint16(data[pos]&0x1f)<<8 | uint16(data[pos+1]) + return &pat +} + +func readPMT(data []byte, p *TSS) *PAT { + pmt := PMT{} + pos := 0 + pmt.Pcrpid = uint16(data[pos]&0x1f)<<8 | uint16(data[pos+1]) + pos += 2 + pmt.Pil = uint16(data[pos]&0x03)<<8 | uint16(data[pos+1]) + pos += 2 + if pmt.Pil != 0 { + pmt.Pd = readDescs(data[pos:], int(pmt.Pil)) + } + pos += int(pmt.Pil) + // TODO Read ES stuff + pmt.Essd = readEssd(data[pos:]) return nil } +// readDescs reads provides a slice of Descs given a byte slice that represents Descs +// and the no of bytes that the descs accumilate +func readDescs(data []byte, descLen int) (o []Desc) { + pos := 0 + o = make([]Desc, 1) + o[0].Dt = data[pos] + pos++ + o[0].Dl = data[pos] + pos++ + o[0].Dd = make([]byte, o[0].Dl) + for i := 0; i < int(o[0].Dl); i++ { + o[0].Dd[i] = data[pos] + pos++ + } + if 2+len(o[0].Dd) != descLen { + panic("No support for reading more than one descriptor") + } + return +} + +func readEssd(data []byte) *ESSD { + essd := ESSD{} + pos := 0 + essd.St = data[pos] +} + // Bytes outputs a byte slice representation of the PSI func (p *PSI) Bytes() []byte { l := 1 + len(p.Pfb) @@ -177,9 +232,7 @@ func (p *PMT) Bytes() []byte { for _, d := range p.Pd { out = append(out, d.Bytes()...) } - for _, e := range p.Essd { - out = append(out, e.Bytes()...) - } + out = append(out, p.Essd.Bytes()...) return out } @@ -206,18 +259,18 @@ func (e *ESSD) Bytes() []byte { return out } -func boolToByte(b bool) byte { +func boolToByte(b bool) (o byte) { if b { - return 0x01 + o = 0x01 } - return 0x00 + return } -func byteToBool(b byte) bool { - if b == 0 { - return false +func byteToBool(b byte) (o bool) { + if b != 0 { + o = true } - return true + return } func crc32_MakeTable(poly uint32) *crc32.Table { From 8fe430f5f5ef9d1798fc342a3c38b553f6b0b3df Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 7 Dec 2018 15:53:38 +1030 Subject: [PATCH 12/54] psi: start writing testing file --- stream/mts/psi/psi.go | 19 ++++++++++++++++++- stream/mts/psi/psi_test.go | 9 +++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 stream/mts/psi/psi_test.go diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 7366ee3b..7f9569e6 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -1,6 +1,7 @@ package psi import ( + "errors" "hash/crc32" "math/bits" ) @@ -63,7 +64,7 @@ type PMT struct { Pcrpid uint16 // Program clock reference pid Pil uint16 // Program info length Pd []Desc // Program descriptors - Essd ESSD // Elementary stream specific data + Essd *ESSD // Elementary stream specific data } // Elementary stream specific data @@ -177,6 +178,13 @@ func readEssd(data []byte) *ESSD { essd := ESSD{} pos := 0 essd.St = data[pos] + pos++ + essd.Epid = uint16(data[pos]&0x1f)<<8 | uint16(data[pos+1]) + pos += 2 + essd.Esil = uint16(data[pos]&0x03)<<8 | uint16(data[pos+1]) + pos += 2 + essd.Esd = readDescs(data[pos:], int(essd.Esil)) + return &essd } // Bytes outputs a byte slice representation of the PSI @@ -295,3 +303,12 @@ func crc32_Update(crc uint32, tab *crc32.Table, p []byte) uint32 { } return crc } + +// UpdateTimestamp +func UpdateTimestamp(data []byte, t int) error { + psi := ReadPSI(data) + if psi.Tid != PATTableID { + return errors.New("Timestamp update failed because psi is not a PMT") + } + return nil +} diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go new file mode 100644 index 00000000..7ec1a3b2 --- /dev/null +++ b/stream/mts/psi/psi_test.go @@ -0,0 +1,9 @@ +package psi + +import ( + "testing" +) + +func TestReadPSI1(t *testing.T) { + +} From 9e7b65ac6a6e45c47f70bd11313a3038a00286be Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 10 Dec 2018 17:42:45 +1030 Subject: [PATCH 13/54] psi: working on psi test file --- stream/mts/psi/psi.go | 53 ++++++++++++++++++++++++++++++++++++++ stream/mts/psi/psi_test.go | 43 +++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 7f9569e6..ee874b6c 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -6,6 +6,59 @@ import ( "math/bits" ) +// TODO: Finish off mts/psi so that we can create pat and pmt tables instead +// of hardcoding. +var ( + StdPat = []byte{ + 0x00, // pointer + + // ---- section included in data sent to CRC32 during check + // table header + 0x00, // table id + 0xb0, // section syntax indicator:1|private bit:1|reserved:2|section length:2|more bytes...:2 + 0x0d, // more bytes... + + // syntax section + 0x00, 0x01, // table id extension + 0xc1, // reserved bits:3|version:5|use now:1 1100 0001 + 0x00, // section number + 0x00, // last section number + // table data + 0x00, 0x01, // Program number + 0xf0, 0x00, // reserved:3|program map PID:13 + + // 0x2a, 0xb1, 0x04, 0xb2, // CRC + // ---- + } + StdPmt = []byte{ + 0x00, // pointer + + // ---- section included in data sent to CRC32 during check + // table header + 0x02, // table id + 0xb0, // section syntax indicator:1|private bit:1|reserved:2|section length:2|more bytes...:2 + 0x12, // more bytes... + + // syntax section + 0x00, 0x01, // table id extension + 0xc1, // reserved bits:3|version:5|use now:1 + 0x00, // section number + 0x00, // last section number + // table data + 0xe1, 0x00, // reserved:3|PCR PID:13 + 0xf0, 0x00, // reserved:4|unused:2|program info length:10 + // No program descriptors since program info length is 0. + // elementary stream info data + 0x1b, // stream type + 0xe1, 0x00, // reserved:3|elementary PID:13 + 0xf0, 0x00, // reserved:4|unused:2|ES info length:10 + // No elementary stream descriptors since ES info length is 0. + + // 0x15, 0xbd, 0x4d, 0x56, // CRC + // ---- + } +) + // Some common lengths const ( ESSDDefLen = 5 diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go index 7ec1a3b2..0ca5e2e9 100644 --- a/stream/mts/psi/psi_test.go +++ b/stream/mts/psi/psi_test.go @@ -4,6 +4,49 @@ import ( "testing" ) +var ( + patPsi = PSI{ + Pf: 0x00, + Tid: 0x00, + Ssi: true, + Pb: false, + Sl: uint16(0x0d), + Tss: &TSS{ + Tide: uint16(0x01), + V: 0, + Cni: true, + Sn: 0, + Lsn: 0, + Sd: &PAT{ + Pn: uint16(0x01), + Pmpid: 16, + }, + }, + } + pmtPsi = PSI{ + Pf: 0x00, + Tid: 0x02, + Ssi: true, + Sl: uint16(0x12), + Tss: & TSS{ + Tide: uint16(0x01), + V:0, + Cni: true, + Sn: 0, + Lsn: 0, + Sd: &PMT{ + Pcrpid: 16, // wrong + Pil: 0, + Essd: &ESSD{ + St: 0x1b, + + } + } + } + } + } +) + func TestReadPSI1(t *testing.T) { } From 43abed9522640f6f4dee8a3f5475bd929657c87c Mon Sep 17 00:00:00 2001 From: saxon Date: Mon, 10 Dec 2018 17:44:56 +1030 Subject: [PATCH 14/54] psi: starting to remove psi stuff from encoder.go and put into psi package --- stream/mts/encoder.go | 63 ++++++------------------------------------- 1 file changed, 8 insertions(+), 55 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 8c6bce27..566f7457 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -36,6 +36,12 @@ import ( "time" "bitbucket.org/ausocean/av/stream/mts/pes" + "bitbucket.org/ausocean/av/stream/mts/psi" +) + +var ( + patTable []byte + pmtTable []byte ) const ( @@ -43,66 +49,13 @@ const ( psiSendCount = 100 ) -// TODO: Finish off mts/psi so that we can create pat and pmt tables instead -// of hardcoding. -var ( - patTable = []byte{ - 0x00, // pointer - - // ---- section included in data sent to CRC32 during check - // table header - 0x00, // table id - 0xb0, // section syntax indicator:1|private bit:1|reserved:2|section length:2|more bytes...:2 - 0x0d, // more bytes... - - // syntax section - 0x00, 0x01, // table id extension - 0xc1, // reserved bits:3|version:5|use now:1 - 0x00, // section number - 0x00, // last section number - // table data - 0x00, 0x01, // Program number - 0xf0, 0x00, // reserved:3|program map PID:13 - - // 0x2a, 0xb1, 0x04, 0xb2, // CRC - // ---- - } - pmtTable = []byte{ - 0x00, // pointer - - // ---- section included in data sent to CRC32 during check - // table header - 0x02, // table id - 0xb0, // section syntax indicator:1|private bit:1|reserved:2|section length:2|more bytes...:2 - 0x12, // more bytes... - - // syntax section - 0x00, 0x01, // table id extension - 0xc1, // reserved bits:3|version:5|use now:1 - 0x00, // section number - 0x00, // last section number - // table data - 0xe1, 0x00, // reserved:3|PCR PID:13 - 0xf0, 0x00, // reserved:4|unused:2|program info length:10 - // No program descriptors since program info length is 0. - // elementary stream info data - 0x1b, // stream type - 0xe1, 0x00, // reserved:3|elementary PID:13 - 0xf0, 0x00, // reserved:4|unused:2|ES info length:10 - // No elementary stream descriptors since ES info length is 0. - - // 0x15, 0xbd, 0x4d, 0x56, // CRC - // ---- - } -) - func init() { // Generate IEEE polynomial table // for the big-endian algorithm. crcTable := crc32_MakeTable(bits.Reverse32(crc32.IEEE)) - patTable = completePSI(patTable, crcTable) - pmtTable = completePSI(pmtTable, crcTable) + patTable = completePSI(psi.StdPat, crcTable) + pmtTable = completePSI(psi.StdPmt, crcTable) } func completePSI(psi []byte, tab *crc32.Table) []byte { From a5fa6bed5fb2c2609e1e5126c9dea2ef7df12f68 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 11 Dec 2018 14:52:18 +1030 Subject: [PATCH 15/54] psi: wrote first tests, for simple pat and pmt tables. Pat writing seems to be working --- stream/mts/psi/psi.go | 24 +++++++----------- stream/mts/psi/psi_test.go | 51 ++++++++++++++++++++++++-------------- 2 files changed, 42 insertions(+), 33 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index ee874b6c..6a88fc50 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -20,7 +20,7 @@ var ( // syntax section 0x00, 0x01, // table id extension - 0xc1, // reserved bits:3|version:5|use now:1 1100 0001 + 0xc1, // reserved bits:2|version:5|use now:1 1100 0001 0x00, // section number 0x00, // last section number // table data @@ -141,12 +141,7 @@ func ReadPSI(data []byte) *PSI { pos := 0 psi.Pf = data[pos] if psi.Pf != 0 { - psi.Pfb = make([]byte, 0, psi.Pf) - pos++ - for i := 0; i < int(psi.Pf); i++ { - psi.Pfb = append(psi.Pfb, data[pos]) - pos++ - } + panic("No support for pointer filler bytes") } psi.Tid = data[pos] pos++ @@ -242,17 +237,16 @@ func readEssd(data []byte) *ESSD { // Bytes outputs a byte slice representation of the PSI func (p *PSI) Bytes() []byte { - l := 1 + len(p.Pfb) - out := make([]byte, l+PSIDefLen) + out := make([]byte, 4) out[0] = p.Pf - for i, b := range p.Pfb { - out[1+i] = b + if p.Pf != 0 { + panic("No support for pointer filler bytes") } - out[l] = p.Tid - out[l+1] = 0x80 | 0x40 | 0x30 | (0x03 & byte(p.Sl>>8)) - out[l+2] = byte(p.Sl) + out[1] = p.Tid + out[2] = 0x80 | 0x30 | (0x03 & byte(p.Sl>>8)) + out[3] = byte(p.Sl) out = append(out, p.Tss.Bytes()...) - crc32 := crc32_Update(0xffffffff, crc32_MakeTable(bits.Reverse32(crc32.IEEE)), out[l:]) + crc32 := crc32_Update(0xffffffff, crc32_MakeTable(bits.Reverse32(crc32.IEEE)), out[3:]) out = append(out, make([]byte, 0, 4)...) out = append(out, byte(crc32>>24)) out = append(out, byte(crc32>>16)) diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go index 0ca5e2e9..4d09dd6c 100644 --- a/stream/mts/psi/psi_test.go +++ b/stream/mts/psi/psi_test.go @@ -1,6 +1,7 @@ package psi import ( + "bytes" "testing" ) @@ -19,34 +20,48 @@ var ( Lsn: 0, Sd: &PAT{ Pn: uint16(0x01), - Pmpid: 16, + Pmpid: uint16(0x1000), }, }, } pmtPsi = PSI{ - Pf: 0x00, + Pf: 0x00, Tid: 0x02, Ssi: true, - Sl: uint16(0x12), - Tss: & TSS{ + Sl: uint16(0x12), + Tss: &TSS{ Tide: uint16(0x01), - V:0, - Cni: true, - Sn: 0, - Lsn: 0, + V: 0, + Cni: true, + Sn: 0, + Lsn: 0, Sd: &PMT{ - Pcrpid: 16, // wrong - Pil: 0, + Pcrpid: 0x1100, // wrong + Pil: 0, Essd: &ESSD{ - St: 0x1b, - - } - } - } - } + St: 0x1b, + Epid: 0x0100, + Esil: 0x00, + }, + }, + }, } ) -func TestReadPSI1(t *testing.T) { - +func TestWritePAT1(t *testing.T) { + got := patPsi.Bytes() + // Remove crc32 + got = got[:len(got)-4] + if !bytes.Equal(StdPat, got[:len(got)-4]) { + t.Errorf("Incorrect output, wanted: %v, got: %v", StdPat, got) + } +} + +func TestWritePMT1(t *testing.T) { + got := pmtPsi.Bytes() + // Remove crc32 + got = got[:len(got)-4] + if !bytes.Equal(StdPmt, got[:len(got)-4]) { + t.Errorf("Incorrect output, wanted: %v, got: %v", StdPmt, got) + } } From c6cf3e72446eeae8e12aab84922fa7602ae1cf7f Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 11 Dec 2018 16:02:57 +1030 Subject: [PATCH 16/54] psi: wrote test for timestampToBytes and also for pmt with timestamp to bytes - everything working as expected --- stream/mts/psi/psi.go | 9 +++- stream/mts/psi/psi_test.go | 106 +++++++++++++++++++++++++++++++++---- 2 files changed, 105 insertions(+), 10 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 6a88fc50..31360b25 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -1,6 +1,7 @@ package psi import ( + "encoding/binary" "errors" "hash/crc32" "math/bits" @@ -280,7 +281,7 @@ func (p *PAT) Bytes() []byte { // Bytes outputs a byte slice representation of the PMT func (p *PMT) Bytes() []byte { out := make([]byte, PMTDefLen) - out[0] = 0xe0 | (0x1f & byte(p.Pcrpid>>8)) + out[0] = 0xe0 | (0x1f & byte(p.Pcrpid>>8)) // byte 10 out[1] = byte(p.Pcrpid) out[2] = 0xf0 | (0x03 & byte(p.Pil>>8)) out[3] = byte(p.Pil) @@ -351,6 +352,12 @@ func crc32_Update(crc uint32, tab *crc32.Table, p []byte) uint32 { return crc } +func TimeToBytes(time uint64) []byte { + s := make([]byte, 8) + binary.BigEndian.PutUint64(s, time) + return s +} + // UpdateTimestamp func UpdateTimestamp(data []byte, t int) error { psi := ReadPSI(data) diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go index 4d09dd6c..448b52c2 100644 --- a/stream/mts/psi/psi_test.go +++ b/stream/mts/psi/psi_test.go @@ -5,8 +5,15 @@ import ( "testing" ) +const ( + testTime = 1235367435 // hex = 49A2360B +) + var ( - patPsi = PSI{ + timeSlice = []byte{ + 0x00, 0x00, 0x00, 0x00, 0x49, 0xA2, 0x36, 0x0B, + } + patPsi1 = PSI{ Pf: 0x00, Tid: 0x00, Ssi: true, @@ -24,7 +31,7 @@ var ( }, }, } - pmtPsi = PSI{ + pmtPsi1 = PSI{ Pf: 0x00, Tid: 0x02, Ssi: true, @@ -36,7 +43,7 @@ var ( Sn: 0, Lsn: 0, Sd: &PMT{ - Pcrpid: 0x1100, // wrong + Pcrpid: 0x0100, // wrong Pil: 0, Essd: &ESSD{ St: 0x1b, @@ -46,22 +53,103 @@ var ( }, }, } + // pmt with descriptor in it + StdPmtTimestamp = []byte{ + 0x00, // pointer + + // ---- section included in data sent to CRC32 during check + // table header + 0x02, // table id + 0xb0, // section syntax indicator:1|private bit:1|reserved:2|section length:2|more bytes...:2 + 0x12, // more bytes... + + // syntax section + 0x00, 0x01, // table id extension + 0xc1, // reserved bits:3|version:5|use now:1 + 0x00, // section number + 0x00, // last section number + // table data + 0xe1, 0x00, // reserved:3|PCR PID:13 + 0xf0, 0x0a, // reserved:4|unused:2|program info length:10 + // Desriptor + byte(timestampDescTag), // Descriptor tag + byte(8), // Length of bytes to follow + 0x00, 0x00, 0x00, 0x00, 0x49, 0xA2, 0x36, 0x0B, // timestamp + // No program descriptors since program info length is 0. + // elementary stream info data + 0x1b, // stream type + 0xe1, 0x00, // reserved:3|elementary PID:13 + 0xf0, 0x00, // reserved:4|unused:2|ES info length:10 + // No elementary stream descriptors since ES info length is 0. + + // 0x15, 0xbd, 0x4d, 0x56, // CRC + // ---- + } + pmtPsiTimestamped = PSI{ + Pf: 0x00, + Tid: 0x02, + Ssi: true, + Sl: uint16(0x12), + Tss: &TSS{ + Tide: uint16(0x01), + V: 0, + Cni: true, + Sn: 0, + Lsn: 0, + Sd: &PMT{ + Pcrpid: 0x0100, // wrong + Pil: 10, + Pd: []Desc{ + Desc{ + Dt: byte(timestampDescTag), + Dl: byte(8), + Dd: TimeToBytes(testTime), + }, + }, + Essd: &ESSD{ + St: 0x1b, + Epid: 0x0100, + Esil: 0x00, + }, + }, + }, + } ) -func TestWritePAT1(t *testing.T) { - got := patPsi.Bytes() +// Test Bytes for a standard pat +func TestBytesPAT1(t *testing.T) { + got := patPsi1.Bytes() // Remove crc32 got = got[:len(got)-4] - if !bytes.Equal(StdPat, got[:len(got)-4]) { + if !bytes.Equal(StdPat, got) { t.Errorf("Incorrect output, wanted: %v, got: %v", StdPat, got) } } -func TestWritePMT1(t *testing.T) { - got := pmtPsi.Bytes() +// Test Bytes for a standard pmt +func TestBytesPMT1(t *testing.T) { + got := pmtPsi1.Bytes() // Remove crc32 got = got[:len(got)-4] - if !bytes.Equal(StdPmt, got[:len(got)-4]) { + if !bytes.Equal(StdPmt, got) { t.Errorf("Incorrect output, wanted: %v, got: %v", StdPmt, got) } } + +// A quick sanity check of the int64 time to []byte func +func TestTimestampToBytes(t *testing.T) { + timeBytes := TimeToBytes(testTime) + if !bytes.Equal(timeSlice, timeBytes) { + t.Errorf("Incorrect results, wanted: %v, got: %v", timeSlice, timeBytes) + } +} + +// Test Bytes for a a standard PMT with a descripot containing a timestamp +func TestBytesPmt2(t *testing.T) { + got := pmtPsiTimestamped.Bytes() + // Remove crc32 + got = got[:len(got)-4] + if !bytes.Equal(StdPmtTimestamp, got) { + t.Errorf("Incorrect output, wanted: %v, got: %v", StdPmtTimestamp, got) + } +} From 68eec9de988d0e340d377b53dbefbf033fb6d76e Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 11 Dec 2018 17:05:40 +1030 Subject: [PATCH 17/54] psi: wrote func for update timstamp - as well as test (passes). Also wrote func for get timestamp - as well as test (passes) --- stream/mts/psi/psi.go | 36 +++++++++++++--- stream/mts/psi/psi_test.go | 85 ++++++++++++++++++++++++++++++++------ 2 files changed, 102 insertions(+), 19 deletions(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 31360b25..ef3801be 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -77,7 +77,9 @@ const ( ) const ( - timestampDescTag = 234 + timeDescTag = 234 + descTagIndx = 13 + timeIndx = 15 ) // Program specific information @@ -358,11 +360,33 @@ func TimeToBytes(time uint64) []byte { return s } -// UpdateTimestamp -func UpdateTimestamp(data []byte, t int) error { - psi := ReadPSI(data) - if psi.Tid != PATTableID { - return errors.New("Timestamp update failed because psi is not a PMT") +func chkTime(d []byte) error { + if d[descTagIndx] != timeDescTag { + return errors.New("PSI does not contain a time descriptor, cannot update") } return nil } + +// Updatetime +func UpdateTime(d []byte, t int) error { + err := chkTime(d) + if err != nil { + return err + } + ts := TimeToBytes(uint64(t)) + for i := range d[timeIndx : timeIndx+8] { + d[i+timeIndx] = ts[i] + } + return nil +} + +func GetTime(d []byte) (t uint64, err error) { + err = chkTime(d) + if err != nil { + return 0, err + } + for i := range d[timeIndx : timeIndx+8] { + t |= uint64(d[i+timeIndx]) << uint(56-i*8) + } + return t, nil +} diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go index 448b52c2..88408bb1 100644 --- a/stream/mts/psi/psi_test.go +++ b/stream/mts/psi/psi_test.go @@ -6,7 +6,12 @@ import ( ) const ( - testTime = 1235367435 // hex = 49A2360B + tstTime = 1235367435 // hex = 49A2360B + tstTime2 = 1735357535 // hex = 676F745F +) + +const ( + errCmp = "Incorrect output, for: %v wanted: %v, got: %v" ) var ( @@ -54,7 +59,7 @@ var ( }, } // pmt with descriptor in it - StdPmtTimestamp = []byte{ + StdPmtTime = []byte{ 0x00, // pointer // ---- section included in data sent to CRC32 during check @@ -72,7 +77,7 @@ var ( 0xe1, 0x00, // reserved:3|PCR PID:13 0xf0, 0x0a, // reserved:4|unused:2|program info length:10 // Desriptor - byte(timestampDescTag), // Descriptor tag + byte(timeDescTag), // Descriptor tag byte(8), // Length of bytes to follow 0x00, 0x00, 0x00, 0x00, 0x49, 0xA2, 0x36, 0x0B, // timestamp // No program descriptors since program info length is 0. @@ -85,7 +90,7 @@ var ( // 0x15, 0xbd, 0x4d, 0x56, // CRC // ---- } - pmtPsiTimestamped = PSI{ + pmtPsiTime = PSI{ Pf: 0x00, Tid: 0x02, Ssi: true, @@ -101,9 +106,9 @@ var ( Pil: 10, Pd: []Desc{ Desc{ - Dt: byte(timestampDescTag), + Dt: byte(timeDescTag), Dl: byte(8), - Dd: TimeToBytes(testTime), + Dd: TimeToBytes(tstTime), }, }, Essd: &ESSD{ @@ -114,6 +119,38 @@ var ( }, }, } + // pmt with descriptor in it + StdPmtTime2 = []byte{ + 0x00, // pointer + + // ---- section included in data sent to CRC32 during check + // table header + 0x02, // table id + 0xb0, // section syntax indicator:1|private bit:1|reserved:2|section length:2|more bytes...:2 + 0x12, // more bytes... + + // syntax section + 0x00, 0x01, // table id extension + 0xc1, // reserved bits:3|version:5|use now:1 + 0x00, // section number + 0x00, // last section number + // table data + 0xe1, 0x00, // reserved:3|PCR PID:13 + 0xf0, 0x0a, // reserved:4|unused:2|program info length:10 + // Desriptor + byte(timeDescTag), // Descriptor tag + byte(8), // Length of bytes to follow + 0x00, 0x00, 0x00, 0x00, 0x67, 0x6F, 0x74, 0x5F, // timestamp + // No program descriptors since program info length is 0. + // elementary stream info data + 0x1b, // stream type + 0xe1, 0x00, // reserved:3|elementary PID:13 + 0xf0, 0x00, // reserved:4|unused:2|ES info length:10 + // No elementary stream descriptors since ES info length is 0. + + // 0x15, 0xbd, 0x4d, 0x56, // CRC + // ---- + } ) // Test Bytes for a standard pat @@ -122,7 +159,7 @@ func TestBytesPAT1(t *testing.T) { // Remove crc32 got = got[:len(got)-4] if !bytes.Equal(StdPat, got) { - t.Errorf("Incorrect output, wanted: %v, got: %v", StdPat, got) + t.Errorf(errCmp, "TestBytesPAT1", StdPat, got) } } @@ -132,24 +169,46 @@ func TestBytesPMT1(t *testing.T) { // Remove crc32 got = got[:len(got)-4] if !bytes.Equal(StdPmt, got) { - t.Errorf("Incorrect output, wanted: %v, got: %v", StdPmt, got) + t.Errorf(errCmp, "TestBytesPMT1", StdPmt, got) } } // A quick sanity check of the int64 time to []byte func func TestTimestampToBytes(t *testing.T) { - timeBytes := TimeToBytes(testTime) + timeBytes := TimeToBytes(tstTime) if !bytes.Equal(timeSlice, timeBytes) { - t.Errorf("Incorrect results, wanted: %v, got: %v", timeSlice, timeBytes) + t.Errorf(errCmp, "testTimeStampToBytes", timeSlice, timeBytes) } } // Test Bytes for a a standard PMT with a descripot containing a timestamp func TestBytesPmt2(t *testing.T) { - got := pmtPsiTimestamped.Bytes() + got := pmtPsiTime.Bytes() // Remove crc32 got = got[:len(got)-4] - if !bytes.Equal(StdPmtTimestamp, got) { - t.Errorf("Incorrect output, wanted: %v, got: %v", StdPmtTimestamp, got) + if !bytes.Equal(StdPmtTime, got) { + t.Errorf(errCmp, "testBytesPmt2", StdPmtTime, got) + } +} + +func TestTimeUpdate(t *testing.T) { + cpy := make([]byte, len(StdPmtTime)) + copy(cpy, StdPmtTime) + err := UpdateTime(cpy, tstTime2) + if err != nil { + t.Errorf("Update time returned err: %v", err) + } + if !bytes.Equal(StdPmtTime2, cpy) { + t.Errorf(errCmp, "TestTimeUpdate", StdPmtTime2, cpy) + } +} + +func TestTimeGet(t *testing.T) { + s, err := GetTime(StdPmtTime) + if err != nil { + t.Errorf("Getting timestamp failed with err: %v", err) + } + if s != tstTime { + t.Errorf(errCmp, "TestTimeGet", tstTime, s) } } From d8c46eefaa9b35aaec9e9863bb8a582781ffdea0 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 11 Dec 2018 17:16:26 +1030 Subject: [PATCH 18/54] psi: restructuring to make neater --- stream/mts/psi/op.go | 43 +++++++++++++++++ stream/mts/psi/psi.go | 95 +------------------------------------- stream/mts/psi/psi_test.go | 2 +- stream/mts/psi/std.go | 54 ++++++++++++++++++++++ 4 files changed, 100 insertions(+), 94 deletions(-) create mode 100644 stream/mts/psi/op.go create mode 100644 stream/mts/psi/std.go diff --git a/stream/mts/psi/op.go b/stream/mts/psi/op.go new file mode 100644 index 00000000..55ceb7d4 --- /dev/null +++ b/stream/mts/psi/op.go @@ -0,0 +1,43 @@ +package psi + +import ( + "encoding/binary" + "errors" +) + +func TimeToBytes(time uint64) []byte { + s := make([]byte, 8) + binary.BigEndian.PutUint64(s, time) + return s +} + +func chkTime(d []byte) error { + if d[descTagIndx] != timeDescTag { + return errors.New("PSI does not contain a time descriptor, cannot update") + } + return nil +} + +// Updatetime +func UpdateTime(d []byte, t int) error { + err := chkTime(d) + if err != nil { + return err + } + ts := TimeToBytes(uint64(t)) + for i := range d[timeIndx : timeIndx+8] { + d[i+timeIndx] = ts[i] + } + return nil +} + +func TimeOf(d []byte) (t uint64, err error) { + err = chkTime(d) + if err != nil { + return 0, err + } + for i := range d[timeIndx : timeIndx+8] { + t |= uint64(d[i+timeIndx]) << uint(56-i*8) + } + return t, nil +} diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index ef3801be..2358fb0a 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -1,66 +1,11 @@ package psi import ( - "encoding/binary" - "errors" "hash/crc32" "math/bits" ) -// TODO: Finish off mts/psi so that we can create pat and pmt tables instead -// of hardcoding. -var ( - StdPat = []byte{ - 0x00, // pointer - - // ---- section included in data sent to CRC32 during check - // table header - 0x00, // table id - 0xb0, // section syntax indicator:1|private bit:1|reserved:2|section length:2|more bytes...:2 - 0x0d, // more bytes... - - // syntax section - 0x00, 0x01, // table id extension - 0xc1, // reserved bits:2|version:5|use now:1 1100 0001 - 0x00, // section number - 0x00, // last section number - // table data - 0x00, 0x01, // Program number - 0xf0, 0x00, // reserved:3|program map PID:13 - - // 0x2a, 0xb1, 0x04, 0xb2, // CRC - // ---- - } - StdPmt = []byte{ - 0x00, // pointer - - // ---- section included in data sent to CRC32 during check - // table header - 0x02, // table id - 0xb0, // section syntax indicator:1|private bit:1|reserved:2|section length:2|more bytes...:2 - 0x12, // more bytes... - - // syntax section - 0x00, 0x01, // table id extension - 0xc1, // reserved bits:3|version:5|use now:1 - 0x00, // section number - 0x00, // last section number - // table data - 0xe1, 0x00, // reserved:3|PCR PID:13 - 0xf0, 0x00, // reserved:4|unused:2|program info length:10 - // No program descriptors since program info length is 0. - // elementary stream info data - 0x1b, // stream type - 0xe1, 0x00, // reserved:3|elementary PID:13 - 0xf0, 0x00, // reserved:4|unused:2|ES info length:10 - // No elementary stream descriptors since ES info length is 0. - - // 0x15, 0xbd, 0x4d, 0x56, // CRC - // ---- - } -) - -// Some common lengths +// Lengths of section definitions const ( ESSDDefLen = 5 DescDefLen = 2 @@ -76,6 +21,7 @@ const ( PMTTableID = 0x02 ) +// Consts relating to time description const ( timeDescTag = 234 descTagIndx = 13 @@ -353,40 +299,3 @@ func crc32_Update(crc uint32, tab *crc32.Table, p []byte) uint32 { } return crc } - -func TimeToBytes(time uint64) []byte { - s := make([]byte, 8) - binary.BigEndian.PutUint64(s, time) - return s -} - -func chkTime(d []byte) error { - if d[descTagIndx] != timeDescTag { - return errors.New("PSI does not contain a time descriptor, cannot update") - } - return nil -} - -// Updatetime -func UpdateTime(d []byte, t int) error { - err := chkTime(d) - if err != nil { - return err - } - ts := TimeToBytes(uint64(t)) - for i := range d[timeIndx : timeIndx+8] { - d[i+timeIndx] = ts[i] - } - return nil -} - -func GetTime(d []byte) (t uint64, err error) { - err = chkTime(d) - if err != nil { - return 0, err - } - for i := range d[timeIndx : timeIndx+8] { - t |= uint64(d[i+timeIndx]) << uint(56-i*8) - } - return t, nil -} diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go index 88408bb1..3063e4d1 100644 --- a/stream/mts/psi/psi_test.go +++ b/stream/mts/psi/psi_test.go @@ -204,7 +204,7 @@ func TestTimeUpdate(t *testing.T) { } func TestTimeGet(t *testing.T) { - s, err := GetTime(StdPmtTime) + s, err := TimeOf(StdPmtTime) if err != nil { t.Errorf("Getting timestamp failed with err: %v", err) } diff --git a/stream/mts/psi/std.go b/stream/mts/psi/std.go new file mode 100644 index 00000000..fee80347 --- /dev/null +++ b/stream/mts/psi/std.go @@ -0,0 +1,54 @@ +package psi + +// TODO: Finish off mts/psi so that we can create pat and pmt tables instead +// of hardcoding. +var ( + StdPat = []byte{ + 0x00, // pointer + + // ---- section included in data sent to CRC32 during check + // table header + 0x00, // table id + 0xb0, // section syntax indicator:1|private bit:1|reserved:2|section length:2|more bytes...:2 + 0x0d, // more bytes... + + // syntax section + 0x00, 0x01, // table id extension + 0xc1, // reserved bits:2|version:5|use now:1 1100 0001 + 0x00, // section number + 0x00, // last section number + // table data + 0x00, 0x01, // Program number + 0xf0, 0x00, // reserved:3|program map PID:13 + + // 0x2a, 0xb1, 0x04, 0xb2, // CRC + // ---- + } + StdPmt = []byte{ + 0x00, // pointer + + // ---- section included in data sent to CRC32 during check + // table header + 0x02, // table id + 0xb0, // section syntax indicator:1|private bit:1|reserved:2|section length:2|more bytes...:2 + 0x12, // more bytes... + + // syntax section + 0x00, 0x01, // table id extension + 0xc1, // reserved bits:3|version:5|use now:1 + 0x00, // section number + 0x00, // last section number + // table data + 0xe1, 0x00, // reserved:3|PCR PID:13 + 0xf0, 0x00, // reserved:4|unused:2|program info length:10 + // No program descriptors since program info length is 0. + // elementary stream info data + 0x1b, // stream type + 0xe1, 0x00, // reserved:3|elementary PID:13 + 0xf0, 0x00, // reserved:4|unused:2|ES info length:10 + // No elementary stream descriptors since ES info length is 0. + + // 0x15, 0xbd, 0x4d, 0x56, // CRC + // ---- + } +) From 4598d51e7990381922d53ba7474466f5d62ebe67 Mon Sep 17 00:00:00 2001 From: saxon Date: Tue, 11 Dec 2018 17:20:01 +1030 Subject: [PATCH 19/54] psi: started commenting and restructuring psi_test.go to make neater --- stream/mts/psi/psi_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go index 3063e4d1..2714db72 100644 --- a/stream/mts/psi/psi_test.go +++ b/stream/mts/psi/psi_test.go @@ -6,14 +6,16 @@ import ( ) const ( - tstTime = 1235367435 // hex = 49A2360B - tstTime2 = 1735357535 // hex = 676F745F + tstTime = 1235367435 // 0x49A2360B + tstTime2 = 1735357535 // 0x676F745F ) const ( errCmp = "Incorrect output, for: %v wanted: %v, got: %v" ) +// Frist write tests + var ( timeSlice = []byte{ 0x00, 0x00, 0x00, 0x00, 0x49, 0xA2, 0x36, 0x0B, From 14e5676f6fb598b1db878b52520cf57a89add8bb Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 12 Dec 2018 10:18:05 +1030 Subject: [PATCH 20/54] psi: added file headers --- stream/mts/psi/op.go | 27 +++++++++++++++++++++++++++ stream/mts/psi/psi.go | 26 ++++++++++++++++++++++++++ stream/mts/psi/psi_test.go | 26 ++++++++++++++++++++++++++ stream/mts/psi/std.go | 26 ++++++++++++++++++++++++++ 4 files changed, 105 insertions(+) diff --git a/stream/mts/psi/op.go b/stream/mts/psi/op.go index 55ceb7d4..5abb96a4 100644 --- a/stream/mts/psi/op.go +++ b/stream/mts/psi/op.go @@ -1,3 +1,30 @@ +/* +NAME + op.go +DESCRIPTION + op.go provides functionality for editing and reading bytes slices + directly in order to insert/read timestamp and gps data in psi. + +AUTHOR + Saxon Milton + +LICENSE + op.go is Copyright (C) 2018 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ + package psi import ( diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 2358fb0a..294b4d21 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -1,3 +1,29 @@ +/* +NAME + psi.go +DESCRIPTION + See Readme.md + +AUTHOR + Saxon Milton + +LICENSE + psi.go is Copyright (C) 2018 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ + package psi import ( diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go index 2714db72..10eb773f 100644 --- a/stream/mts/psi/psi_test.go +++ b/stream/mts/psi/psi_test.go @@ -1,3 +1,29 @@ +/* +NAME + psi_test.go +DESCRIPTION + See Readme.md + +AUTHOR + Saxon Milton + +LICENSE + psi_test.go is Copyright (C) 2018 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ + package psi import ( diff --git a/stream/mts/psi/std.go b/stream/mts/psi/std.go index fee80347..6beed1c8 100644 --- a/stream/mts/psi/std.go +++ b/stream/mts/psi/std.go @@ -1,3 +1,29 @@ +/* +NAME + std.go +DESCRIPTION + See Readme.md + +AUTHOR + Saxon Milton + +LICENSE + std.go is Copyright (C) 2018 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with revid in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ + package psi // TODO: Finish off mts/psi so that we can create pat and pmt tables instead From 3cf6c0099145395d36959b43c3789db411310a26 Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 12 Dec 2018 16:26:59 +1030 Subject: [PATCH 21/54] psi: restructured psi_test.go and started writing tests for gps --- stream/mts/psi/op.go | 16 +- stream/mts/psi/psi.go | 11 +- stream/mts/psi/psi_test.go | 348 +++++++++++++++++++------------------ stream/mts/psi/std.go | 52 +++++- 4 files changed, 249 insertions(+), 178 deletions(-) diff --git a/stream/mts/psi/op.go b/stream/mts/psi/op.go index 5abb96a4..a2ce053a 100644 --- a/stream/mts/psi/op.go +++ b/stream/mts/psi/op.go @@ -32,14 +32,14 @@ import ( "errors" ) -func TimeToBytes(time uint64) []byte { - s := make([]byte, 8) - binary.BigEndian.PutUint64(s, time) +func TimeBytes(time uint64) (s []byte) { + s = make([]byte, timeSize) + binary.BigEndian.PutUint64(s[:], time) return s } func chkTime(d []byte) error { - if d[descTagIndx] != timeDescTag { + if d[timeTagIndx] != timeDescTag { return errors.New("PSI does not contain a time descriptor, cannot update") } return nil @@ -51,7 +51,7 @@ func UpdateTime(d []byte, t int) error { if err != nil { return err } - ts := TimeToBytes(uint64(t)) + ts := TimeBytes(uint64(t)) for i := range d[timeIndx : timeIndx+8] { d[i+timeIndx] = ts[i] } @@ -68,3 +68,9 @@ func TimeOf(d []byte) (t uint64, err error) { } return t, nil } + +func GpsStrBytes(l string) (out []byte) { + out = make([]byte, gpsSize) + copy(out, []byte(l)) + return +} diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 294b4d21..52f1dcd1 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -50,8 +50,17 @@ const ( // Consts relating to time description const ( timeDescTag = 234 - descTagIndx = 13 + timeTagIndx = 13 timeIndx = 15 + timeSize = 8 +) + +// Consts relating to gps description +const ( + gpsDescTag = 235 + gpsTagIndx = 23 + gpsIndx = 25 + gpsSize = 32 // bytes ) // Program specific information diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go index 10eb773f..b0edd7ee 100644 --- a/stream/mts/psi/psi_test.go +++ b/stream/mts/psi/psi_test.go @@ -31,212 +31,224 @@ import ( "testing" ) +// Times as ints for testing const ( - tstTime = 1235367435 // 0x49A2360B + tstTime1 = 1235367435 // 0x49A2360B tstTime2 = 1735357535 // 0x676F745F ) +// GPS string for testing +const ( + gpsTstStr1 = "$GPGGA,123519,4807.038,N,01131.000,E,1,08,0.9,545.4,M,46.9,M,,*47" + gpsTstStr2 = "$GPGGA,183710,4902.048,N,02171.020,E,1,09,0.5,547.2,M,43.4,M,,*42" +) + +// err message const ( errCmp = "Incorrect output, for: %v wanted: %v, got: %v" ) -// Frist write tests - +// Test time to bytes test Data var ( timeSlice = []byte{ 0x00, 0x00, 0x00, 0x00, 0x49, 0xA2, 0x36, 0x0B, } - patPsi1 = PSI{ - Pf: 0x00, - Tid: 0x00, - Ssi: true, - Pb: false, - Sl: uint16(0x0d), - Tss: &TSS{ - Tide: uint16(0x01), - V: 0, - Cni: true, - Sn: 0, - Lsn: 0, - Sd: &PAT{ - Pn: uint16(0x01), - Pmpid: uint16(0x1000), - }, - }, +) + +// Parts to construct bytes of pmt with time and bytes +var ( + pmtTimeGpsBytesPart1 = []byte{ + 0x00, 0x02, 0xb0, 0x12, 0x00, 0x01, 0xc1, 0x00, 0x00, 0xe1, 0x00, 0xf0, 0x0a, + byte(timeDescTag), // Descriptor tag for timestamp + byte(timeSize), // Length of bytes to follow + 0x00, 0x00, 0x00, 0x00, 0x67, 0x6F, 0x74, 0x5F, // Timestamp data + byte(gpsDescTag), // Descriptor tag for gps + byte(gpsSize), // Length of bytes to follow } - pmtPsi1 = PSI{ - Pf: 0x00, - Tid: 0x02, - Ssi: true, - Sl: uint16(0x12), - Tss: &TSS{ - Tide: uint16(0x01), - V: 0, - Cni: true, - Sn: 0, - Lsn: 0, - Sd: &PMT{ - Pcrpid: 0x0100, // wrong - Pil: 0, - Essd: &ESSD{ - St: 0x1b, - Epid: 0x0100, - Esil: 0x00, - }, - }, - }, - } - // pmt with descriptor in it - StdPmtTime = []byte{ - 0x00, // pointer - - // ---- section included in data sent to CRC32 during check - // table header - 0x02, // table id - 0xb0, // section syntax indicator:1|private bit:1|reserved:2|section length:2|more bytes...:2 - 0x12, // more bytes... - - // syntax section - 0x00, 0x01, // table id extension - 0xc1, // reserved bits:3|version:5|use now:1 - 0x00, // section number - 0x00, // last section number - // table data - 0xe1, 0x00, // reserved:3|PCR PID:13 - 0xf0, 0x0a, // reserved:4|unused:2|program info length:10 - // Desriptor - byte(timeDescTag), // Descriptor tag - byte(8), // Length of bytes to follow - 0x00, 0x00, 0x00, 0x00, 0x49, 0xA2, 0x36, 0x0B, // timestamp - // No program descriptors since program info length is 0. - // elementary stream info data - 0x1b, // stream type - 0xe1, 0x00, // reserved:3|elementary PID:13 - 0xf0, 0x00, // reserved:4|unused:2|ES info length:10 - // No elementary stream descriptors since ES info length is 0. - - // 0x15, 0xbd, 0x4d, 0x56, // CRC - // ---- - } - pmtPsiTime = PSI{ - Pf: 0x00, - Tid: 0x02, - Ssi: true, - Sl: uint16(0x12), - Tss: &TSS{ - Tide: uint16(0x01), - V: 0, - Cni: true, - Sn: 0, - Lsn: 0, - Sd: &PMT{ - Pcrpid: 0x0100, // wrong - Pil: 10, - Pd: []Desc{ - Desc{ - Dt: byte(timeDescTag), - Dl: byte(8), - Dd: TimeToBytes(tstTime), - }, - }, - Essd: &ESSD{ - St: 0x1b, - Epid: 0x0100, - Esil: 0x00, - }, - }, - }, - } - // pmt with descriptor in it - StdPmtTime2 = []byte{ - 0x00, // pointer - - // ---- section included in data sent to CRC32 during check - // table header - 0x02, // table id - 0xb0, // section syntax indicator:1|private bit:1|reserved:2|section length:2|more bytes...:2 - 0x12, // more bytes... - - // syntax section - 0x00, 0x01, // table id extension - 0xc1, // reserved bits:3|version:5|use now:1 - 0x00, // section number - 0x00, // last section number - // table data - 0xe1, 0x00, // reserved:3|PCR PID:13 - 0xf0, 0x0a, // reserved:4|unused:2|program info length:10 - // Desriptor - byte(timeDescTag), // Descriptor tag - byte(8), // Length of bytes to follow - 0x00, 0x00, 0x00, 0x00, 0x67, 0x6F, 0x74, 0x5F, // timestamp - // No program descriptors since program info length is 0. - // elementary stream info data - 0x1b, // stream type - 0xe1, 0x00, // reserved:3|elementary PID:13 - 0xf0, 0x00, // reserved:4|unused:2|ES info length:10 - // No elementary stream descriptors since ES info length is 0. - - // 0x15, 0xbd, 0x4d, 0x56, // CRC - // ---- + pmtTimeGpsBytesPart2 = []byte{ + 0x1b, 0xe1, 0x00, 0xf0, 0x00, } ) -// Test Bytes for a standard pat -func TestBytesPAT1(t *testing.T) { - got := patPsi1.Bytes() - // Remove crc32 - got = got[:len(got)-4] - if !bytes.Equal(StdPat, got) { - t.Errorf(errCmp, "TestBytesPAT1", StdPat, got) +var ( + // Bytes representing pmt with tstTime1 + pmtTimeBytes1 = []byte{ + 0x00, 0x02, 0xb0, 0x12, 0x00, 0x01, 0xc1, 0x00, 0x00, 0xe1, 0x00, 0xf0, 0x0a, + byte(timeDescTag), // Descriptor tag + byte(timeSize), // Length of bytes to follow + 0x00, 0x00, 0x00, 0x00, 0x49, 0xA2, 0x36, 0x0B, // timestamp + 0x1b, 0xe1, 0x00, 0xf0, 0x00, } + + // Bytes representing pmt with tstTime 2 + pmtTimeBytes2 = []byte{ + 0x00, 0x02, 0xb0, 0x12, 0x00, 0x01, 0xc1, 0x00, 0x00, 0xe1, 0x00, 0xf0, 0x0a, + byte(timeDescTag), // Descriptor tag + byte(timeSize), // Length of bytes to follow + 0x00, 0x00, 0x00, 0x00, 0x67, 0x6F, 0x74, 0x5F, // timestamp + 0x1b, 0xe1, 0x00, 0xf0, 0x00, + } + + // Bytes representing pmt with time1 and gps1 + pmtTimeGpsBytes1 = buildPmtTimeGpsBytes(gpsTstStr1) + + // bytes representing pmt with with time1 and gps 2 + pmtTimeGpsBytes2 = buildPmtTimeGpsBytes(gpsTstStr2) +) + +// bytesTests contains data for testing the Bytes() funcs for the PSI data struct +var bytesTests = []struct { + name string + input PSI + want []byte +}{ + // Pat test + { + name: "pat Bytes()", + input: stdPat, + want: stdPatBytes, + }, + + // Pmt test data no descriptor + { + name: "pmt to Bytes() without descriptors", + input: stdPmt, + want: stdPmtBytes, + }, + + // Pmt with time descriptor + { + name: "pmt to Bytes() with time descriptor", + input: PSI{ + Pf: 0x00, + Tid: 0x02, + Ssi: true, + Sl: uint16(0x12), + Tss: &TSS{ + Tide: uint16(0x01), + V: 0, + Cni: true, + Sn: 0, + Lsn: 0, + Sd: &PMT{ + Pcrpid: 0x0100, // wrong + Pil: 10, + Pd: []Desc{ + Desc{ + Dt: byte(timeDescTag), + Dl: byte(timeSize), + Dd: TimeBytes(tstTime1), + }, + }, + Essd: &ESSD{ + St: 0x1b, + Epid: 0x0100, + Esil: 0x00, + }, + }, + }, + }, + want: pmtTimeBytes1, + }, + + // Pmt with time and gps + { + name: "pmt Bytes() with time and gps", + input: PSI{ + Pf: 0x00, + Tid: 0x02, + Ssi: true, + Sl: uint16(0x12), + Tss: &TSS{ + Tide: uint16(0x01), + V: 0, + Cni: true, + Sn: 0, + Lsn: 0, + Sd: &PMT{ + Pcrpid: 0x0100, // wrong + Pil: 10, + Pd: []Desc{ + Desc{ + Dt: byte(timeDescTag), + Dl: byte(timeSize), + Dd: TimeBytes(tstTime2), + }, + Desc{ + Dt: byte(gpsDescTag), + Dl: byte(gpsSize), + Dd: GpsStrBytes(gpsTstStr1), + }, + }, + Essd: &ESSD{ + St: 0x1b, + Epid: 0x0100, + Esil: 0x00, + }, + }, + }, + }, + want: buildPmtTimeGpsBytes(gpsTstStr1), + }, } -// Test Bytes for a standard pmt -func TestBytesPMT1(t *testing.T) { - got := pmtPsi1.Bytes() - // Remove crc32 - got = got[:len(got)-4] - if !bytes.Equal(StdPmt, got) { - t.Errorf(errCmp, "TestBytesPMT1", StdPmt, got) +// TestBytes ensures that the Bytes() funcs are working correctly to take PSI +// structs and converting them to byte slices +func TestBytes(t *testing.T) { + for _, test := range bytesTests { + got := test.input.Bytes() + // Remove crc32s + got = got[:len(got)-4] + if !bytes.Equal(got, test.want) { + t.Errorf("unexpected error for test %v: got:%v want:%v", test.name, got, + test.want) + } } } // A quick sanity check of the int64 time to []byte func func TestTimestampToBytes(t *testing.T) { - timeBytes := TimeToBytes(tstTime) - if !bytes.Equal(timeSlice, timeBytes) { - t.Errorf(errCmp, "testTimeStampToBytes", timeSlice, timeBytes) - } -} - -// Test Bytes for a a standard PMT with a descripot containing a timestamp -func TestBytesPmt2(t *testing.T) { - got := pmtPsiTime.Bytes() - // Remove crc32 - got = got[:len(got)-4] - if !bytes.Equal(StdPmtTime, got) { - t.Errorf(errCmp, "testBytesPmt2", StdPmtTime, got) + tb := TimeBytes(tstTime1) + if !bytes.Equal(timeSlice, tb) { + t.Errorf(errCmp, "testTimeStampToBytes", timeSlice, tb) } } func TestTimeUpdate(t *testing.T) { - cpy := make([]byte, len(StdPmtTime)) - copy(cpy, StdPmtTime) + cpy := make([]byte, len(pmtTimeBytes1)) + copy(cpy, pmtTimeBytes1) err := UpdateTime(cpy, tstTime2) if err != nil { t.Errorf("Update time returned err: %v", err) } - if !bytes.Equal(StdPmtTime2, cpy) { - t.Errorf(errCmp, "TestTimeUpdate", StdPmtTime2, cpy) + if !bytes.Equal(pmtTimeBytes2, cpy) { + t.Errorf(errCmp, "TestTimeUpdate", pmtTimeBytes2, cpy) } } func TestTimeGet(t *testing.T) { - s, err := TimeOf(StdPmtTime) + s, err := TimeOf(pmtTimeBytes1) if err != nil { t.Errorf("Getting timestamp failed with err: %v", err) } - if s != tstTime { - t.Errorf(errCmp, "TestTimeGet", tstTime, s) + if s != tstTime1 { + t.Errorf(errCmp, "TestTimeGet", tstTime1, s) } } + +func TestGpsUpdate(t *testing.T) { + cpy := make([]byte, len(pmtTimeGpsBytes1)) + copy(cpy, pmtTimeGpsBytes1) + err := UpdateGps(cpy, tstTime2) + if err != nil { + t.Errorf("Update time returned err: %v", err) + } + if !bytes.Equal(pmtTimeGpsBytes2, cpy) { + t.Errorf(errCmp, "TestGpsUpdate", pmtTimeGpsBytes2, cpy) + } +} + +func buildPmtTimeGpsBytes(tstStr string) []byte { + return append(append(append(make([]byte, 0), pmtTimeGpsBytesPart1...), + GpsStrBytes(tstStr)...), pmtTimeGpsBytesPart2...) +} diff --git a/stream/mts/psi/std.go b/stream/mts/psi/std.go index 6beed1c8..f3448a6a 100644 --- a/stream/mts/psi/std.go +++ b/stream/mts/psi/std.go @@ -26,10 +26,54 @@ LICENSE package psi -// TODO: Finish off mts/psi so that we can create pat and pmt tables instead -// of hardcoding. +// Std PSI in struct form without descriptor var ( - StdPat = []byte{ + stdPat = PSI{ + Pf: 0x00, + Tid: 0x00, + Ssi: true, + Pb: false, + Sl: uint16(0x0d), + Tss: &TSS{ + Tide: uint16(0x01), + V: 0, + Cni: true, + Sn: 0, + Lsn: 0, + Sd: &PAT{ + Pn: uint16(0x01), + Pmpid: uint16(0x1000), + }, + }, + } + + stdPmt = PSI{ + Pf: 0x00, + Tid: 0x02, + Ssi: true, + Sl: uint16(0x12), + Tss: &TSS{ + Tide: uint16(0x01), + V: 0, + Cni: true, + Sn: 0, + Lsn: 0, + Sd: &PMT{ + Pcrpid: 0x0100, // wrong + Pil: 0, + Essd: &ESSD{ + St: 0x1b, + Epid: 0x0100, + Esil: 0x00, + }, + }, + }, + } +) + +// Std PSI in bytes form +var ( + stdPatBytes = []byte{ 0x00, // pointer // ---- section included in data sent to CRC32 during check @@ -50,7 +94,7 @@ var ( // 0x2a, 0xb1, 0x04, 0xb2, // CRC // ---- } - StdPmt = []byte{ + stdPmtBytes = []byte{ 0x00, // pointer // ---- section included in data sent to CRC32 during check From f320746b5d60ffd9d0e735edf042a171d7ce4fa1 Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 12 Dec 2018 17:02:03 +1030 Subject: [PATCH 22/54] psi: wrote test for gpsUpdate - appears to be working --- stream/mts/psi/op.go | 32 ++++++++++++++++++++++++++------ stream/mts/psi/psi.go | 16 ++++++++-------- stream/mts/psi/psi_test.go | 18 +++++++++--------- 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/stream/mts/psi/op.go b/stream/mts/psi/op.go index a2ce053a..26f33c87 100644 --- a/stream/mts/psi/op.go +++ b/stream/mts/psi/op.go @@ -33,7 +33,7 @@ import ( ) func TimeBytes(time uint64) (s []byte) { - s = make([]byte, timeSize) + s = make([]byte, timeDataSize) binary.BigEndian.PutUint64(s[:], time) return s } @@ -45,6 +45,13 @@ func chkTime(d []byte) error { return nil } +func chkGps(d []byte) error { + if d[gpsTagIndx] != gpsDescTag { + return errors.New("PSI does not contain a gps descriptor, cannot update") + } + return nil +} + // Updatetime func UpdateTime(d []byte, t int) error { err := chkTime(d) @@ -52,8 +59,8 @@ func UpdateTime(d []byte, t int) error { return err } ts := TimeBytes(uint64(t)) - for i := range d[timeIndx : timeIndx+8] { - d[i+timeIndx] = ts[i] + for i := range d[timeDataIndx : timeDataIndx+timeDataSize] { + d[i+timeDataIndx] = ts[i] } return nil } @@ -63,14 +70,27 @@ func TimeOf(d []byte) (t uint64, err error) { if err != nil { return 0, err } - for i := range d[timeIndx : timeIndx+8] { - t |= uint64(d[i+timeIndx]) << uint(56-i*8) + for i := range d[timeDataIndx : timeDataIndx+timeDataSize] { + t |= uint64(d[i+timeDataIndx]) << uint(56-i*timeDataSize) } return t, nil } func GpsStrBytes(l string) (out []byte) { - out = make([]byte, gpsSize) + out = make([]byte, gpsDataSize) copy(out, []byte(l)) return } + +func UpdateGps(d []byte, s string) error { + // First check if there is a GPS descriptor + err := chkGps(d) + if err != nil { + return err + } + gb := GpsStrBytes(s) + for i := range d[gpsDataIndx : gpsDataIndx+gpsDataSize] { + d[i+gpsDataIndx] = gb[i] + } + return nil +} diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 52f1dcd1..6abc3a51 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -49,18 +49,18 @@ const ( // Consts relating to time description const ( - timeDescTag = 234 - timeTagIndx = 13 - timeIndx = 15 - timeSize = 8 + timeDescTag = 234 + timeTagIndx = 13 + timeDataIndx = 15 + timeDataSize = 8 ) // Consts relating to gps description const ( - gpsDescTag = 235 - gpsTagIndx = 23 - gpsIndx = 25 - gpsSize = 32 // bytes + gpsDescTag = 235 + gpsTagIndx = 23 + gpsDataIndx = 25 + gpsDataSize = 32 // bytes ) // Program specific information diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go index b0edd7ee..0c189021 100644 --- a/stream/mts/psi/psi_test.go +++ b/stream/mts/psi/psi_test.go @@ -60,10 +60,10 @@ var ( pmtTimeGpsBytesPart1 = []byte{ 0x00, 0x02, 0xb0, 0x12, 0x00, 0x01, 0xc1, 0x00, 0x00, 0xe1, 0x00, 0xf0, 0x0a, byte(timeDescTag), // Descriptor tag for timestamp - byte(timeSize), // Length of bytes to follow + byte(timeDataSize), // Length of bytes to follow 0x00, 0x00, 0x00, 0x00, 0x67, 0x6F, 0x74, 0x5F, // Timestamp data - byte(gpsDescTag), // Descriptor tag for gps - byte(gpsSize), // Length of bytes to follow + byte(gpsDescTag), // Descriptor tag for gps + byte(gpsDataSize), // Length of bytes to follow } pmtTimeGpsBytesPart2 = []byte{ 0x1b, 0xe1, 0x00, 0xf0, 0x00, @@ -75,7 +75,7 @@ var ( pmtTimeBytes1 = []byte{ 0x00, 0x02, 0xb0, 0x12, 0x00, 0x01, 0xc1, 0x00, 0x00, 0xe1, 0x00, 0xf0, 0x0a, byte(timeDescTag), // Descriptor tag - byte(timeSize), // Length of bytes to follow + byte(timeDataSize), // Length of bytes to follow 0x00, 0x00, 0x00, 0x00, 0x49, 0xA2, 0x36, 0x0B, // timestamp 0x1b, 0xe1, 0x00, 0xf0, 0x00, } @@ -84,7 +84,7 @@ var ( pmtTimeBytes2 = []byte{ 0x00, 0x02, 0xb0, 0x12, 0x00, 0x01, 0xc1, 0x00, 0x00, 0xe1, 0x00, 0xf0, 0x0a, byte(timeDescTag), // Descriptor tag - byte(timeSize), // Length of bytes to follow + byte(timeDataSize), // Length of bytes to follow 0x00, 0x00, 0x00, 0x00, 0x67, 0x6F, 0x74, 0x5F, // timestamp 0x1b, 0xe1, 0x00, 0xf0, 0x00, } @@ -136,7 +136,7 @@ var bytesTests = []struct { Pd: []Desc{ Desc{ Dt: byte(timeDescTag), - Dl: byte(timeSize), + Dl: byte(timeDataSize), Dd: TimeBytes(tstTime1), }, }, @@ -171,12 +171,12 @@ var bytesTests = []struct { Pd: []Desc{ Desc{ Dt: byte(timeDescTag), - Dl: byte(timeSize), + Dl: byte(timeDataSize), Dd: TimeBytes(tstTime2), }, Desc{ Dt: byte(gpsDescTag), - Dl: byte(gpsSize), + Dl: byte(gpsDataSize), Dd: GpsStrBytes(gpsTstStr1), }, }, @@ -239,7 +239,7 @@ func TestTimeGet(t *testing.T) { func TestGpsUpdate(t *testing.T) { cpy := make([]byte, len(pmtTimeGpsBytes1)) copy(cpy, pmtTimeGpsBytes1) - err := UpdateGps(cpy, tstTime2) + err := UpdateGps(cpy, gpsTstStr2) if err != nil { t.Errorf("Update time returned err: %v", err) } From 41d98b4dcd1132bdf0082d1f4e028996d589f5a1 Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 12 Dec 2018 17:06:01 +1030 Subject: [PATCH 23/54] psi: added some commenting above testing funcs --- stream/mts/psi/psi_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go index 0c189021..3d3a74a7 100644 --- a/stream/mts/psi/psi_test.go +++ b/stream/mts/psi/psi_test.go @@ -193,7 +193,7 @@ var bytesTests = []struct { } // TestBytes ensures that the Bytes() funcs are working correctly to take PSI -// structs and converting them to byte slices +// structs and convert them to byte slices func TestBytes(t *testing.T) { for _, test := range bytesTests { got := test.input.Bytes() @@ -206,7 +206,7 @@ func TestBytes(t *testing.T) { } } -// A quick sanity check of the int64 time to []byte func +// TestTimestampToBytes is a quick sanity check of the int64 time to []byte func func TestTimestampToBytes(t *testing.T) { tb := TimeBytes(tstTime1) if !bytes.Equal(timeSlice, tb) { @@ -214,6 +214,7 @@ func TestTimestampToBytes(t *testing.T) { } } +// TestTimeUpdate checks to see if we can correctly update the timstamp in pmt func TestTimeUpdate(t *testing.T) { cpy := make([]byte, len(pmtTimeBytes1)) copy(cpy, pmtTimeBytes1) @@ -226,6 +227,7 @@ func TestTimeUpdate(t *testing.T) { } } +// TestTimeGet tsts to see if we can correctly get the timestamp from a pmt func TestTimeGet(t *testing.T) { s, err := TimeOf(pmtTimeBytes1) if err != nil { @@ -236,6 +238,7 @@ func TestTimeGet(t *testing.T) { } } +// TestGpsUpdate checks to see if we can update the gps string in a pmt correctly func TestGpsUpdate(t *testing.T) { cpy := make([]byte, len(pmtTimeGpsBytes1)) copy(cpy, pmtTimeGpsBytes1) @@ -248,6 +251,9 @@ func TestGpsUpdate(t *testing.T) { } } +// buildPmtTimeGpsBytes is a helper function to help construct the byte slices +// for pmts with time and gps, as the gps data field is 32 bytes, i.e. quite large +// to type out func buildPmtTimeGpsBytes(tstStr string) []byte { return append(append(append(make([]byte, 0), pmtTimeGpsBytesPart1...), GpsStrBytes(tstStr)...), pmtTimeGpsBytesPart2...) From 283689ea6c97cf862dce554bd025564a1ed4a70e Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 12 Dec 2018 17:13:43 +1030 Subject: [PATCH 24/54] psi: added a std template PSI struct for pmt with time and gps descriptors to make things easier --- stream/mts/psi/std.go | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/stream/mts/psi/std.go b/stream/mts/psi/std.go index f3448a6a..228f4493 100644 --- a/stream/mts/psi/std.go +++ b/stream/mts/psi/std.go @@ -26,8 +26,9 @@ LICENSE package psi -// Std PSI in struct form without descriptor +// Some common manifestations of PSI var ( + // PSI struct to represent basic pat stdPat = PSI{ Pf: 0x00, Tid: 0x00, @@ -47,6 +48,7 @@ var ( }, } + // PSI struct to represent basic pmt without descriptors for time and gps stdPmt = PSI{ Pf: 0x00, Tid: 0x02, @@ -69,6 +71,42 @@ var ( }, }, } + + // Std pmt with time and gps descriptors, time and gps fields are zerod out + stdPmtTimeGps = PSI{ + Pf: 0x00, + Tid: 0x02, + Ssi: true, + Sl: uint16(0x12), + Tss: &TSS{ + Tide: uint16(0x01), + V: 0, + Cni: true, + Sn: 0, + Lsn: 0, + Sd: &PMT{ + Pcrpid: 0x0100, // wrong + Pil: 10, + Pd: []Desc{ + Desc{ + Dt: byte(timeDescTag), + Dl: byte(timeDataSize), + Dd: make([]byte, timeDataSize), + }, + Desc{ + Dt: byte(gpsDescTag), + Dl: byte(gpsDataSize), + Dd: make([]byte, gpsDataSize), + }, + }, + Essd: &ESSD{ + St: 0x1b, + Epid: 0x0100, + Esil: 0x00, + }, + }, + }, + } ) // Std PSI in bytes form From fe2c5d103361709d59b4bb166351c5db43735c8b Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 12 Dec 2018 17:17:06 +1030 Subject: [PATCH 25/54] psi: created const for pmt with time and gps program inormation length --- stream/mts/psi/std.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/stream/mts/psi/std.go b/stream/mts/psi/std.go index 228f4493..ce87b42b 100644 --- a/stream/mts/psi/std.go +++ b/stream/mts/psi/std.go @@ -26,6 +26,10 @@ LICENSE package psi +const ( + pmtTimeGpsPil = 44 +) + // Some common manifestations of PSI var ( // PSI struct to represent basic pat @@ -85,8 +89,8 @@ var ( Sn: 0, Lsn: 0, Sd: &PMT{ - Pcrpid: 0x0100, // wrong - Pil: 10, + Pcrpid: 0x0100, + Pil: pmtTimeGpsPil, Pd: []Desc{ Desc{ Dt: byte(timeDescTag), From 2ca393c276d358ed7d96e3ac0e40e7e5baf5f85e Mon Sep 17 00:00:00 2001 From: saxon Date: Wed, 12 Dec 2018 23:17:28 +1030 Subject: [PATCH 26/54] psi: improved commenting in op.go --- stream/mts/psi/op.go | 45 ++++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/stream/mts/psi/op.go b/stream/mts/psi/op.go index 26f33c87..46e0db77 100644 --- a/stream/mts/psi/op.go +++ b/stream/mts/psi/op.go @@ -32,29 +32,37 @@ import ( "errors" ) -func TimeBytes(time uint64) (s []byte) { +// timeBytes takes a timestamp as an uint64 and converts to an 8 byte slice - +// allows for updating of timestamp in pmt time descriptor. +func TimeBytes(t uint64) (s []byte) { s = make([]byte, timeDataSize) - binary.BigEndian.PutUint64(s[:], time) + binary.BigEndian.PutUint64(s[:], t) return s } -func chkTime(d []byte) error { - if d[timeTagIndx] != timeDescTag { +// ChkTime takes a psi as a byte slice and checks to see if it has a time descriptor +// - if so return nil, otherwise return error +func ChkTime(p []byte) error { + if p[timeTagIndx] != timeDescTag { return errors.New("PSI does not contain a time descriptor, cannot update") } return nil } -func chkGps(d []byte) error { - if d[gpsTagIndx] != gpsDescTag { +// ChkGps takes a psi as a byte slice and checks to see if it has a gps descriptor +// - if so return nil, otherwise return error +func ChkGps(p []byte) error { + if p[gpsTagIndx] != gpsDescTag { return errors.New("PSI does not contain a gps descriptor, cannot update") } return nil } -// Updatetime +// UpdateTime takes the byte slice representation of a psi-pmt as well as a time +// as an integer and attempts to update the time descriptor in the pmt with the +// given time if the time descriptor exists, otherwise an error is returned func UpdateTime(d []byte, t int) error { - err := chkTime(d) + err := ChkTime(d) if err != nil { return err } @@ -65,26 +73,35 @@ func UpdateTime(d []byte, t int) error { return nil } -func TimeOf(d []byte) (t uint64, err error) { - err = chkTime(d) +// TimeOf takes a byte slice representation of a psi-pmt and extracts it's +// timestamp, returning as a uint64 if it exists, otherwise returning 0 and nil +// if it does not exist +func TimeOf(p []byte) (t uint64, err error) { + err = ChkTime(p) if err != nil { return 0, err } - for i := range d[timeDataIndx : timeDataIndx+timeDataSize] { - t |= uint64(d[i+timeDataIndx]) << uint(56-i*timeDataSize) + // TODO: could probably write a byte slice to int func, so that this would + // be replaced by extraction of time data slice and then call to conversion func + for i := range p[timeDataIndx : timeDataIndx+timeDataSize] { + t |= uint64(p[i+timeDataIndx]) << uint(56-i*timeDataSize) } return t, nil } +// GpsStrBytes take a string of gps data and converts to a 32 byte slice - +// easy update of slice representation of a pmt with gps descriptor func GpsStrBytes(l string) (out []byte) { out = make([]byte, gpsDataSize) copy(out, []byte(l)) return } +// UpdateGps takes a byte slice representation of a psi-pmt containing a gps +// descriptor and attempts to update the gps data value with the passed string. +// If the psi does not contain a gps descriptor, and error is returned. func UpdateGps(d []byte, s string) error { - // First check if there is a GPS descriptor - err := chkGps(d) + err := ChkGps(d) if err != nil { return err } From 9a7d7a9ab38d423dc26480d524acda95228c247a Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 13 Dec 2018 12:22:06 +1030 Subject: [PATCH 27/54] revid: modified http sender to look at reply of send and get time and gps data to mts package --- revid/senders.go | 20 ++++++++++- stream/mts/encoder.go | 74 +++++++++++++++----------------------- stream/mts/psi/op.go | 4 +-- stream/mts/psi/psi_test.go | 8 ++--- stream/mts/psi/std.go | 10 +++--- 5 files changed, 58 insertions(+), 58 deletions(-) diff --git a/revid/senders.go b/revid/senders.go index c4ab9cfc..fdf5fe4c 100644 --- a/revid/senders.go +++ b/revid/senders.go @@ -35,6 +35,7 @@ import ( "os/exec" "bitbucket.org/ausocean/av/rtmp" + "bitbucket.org/ausocean/av/stream/mts" "bitbucket.org/ausocean/av/stream/rtp" "bitbucket.org/ausocean/iot/pi/netsender" "bitbucket.org/ausocean/utils/ring" @@ -143,9 +144,26 @@ func (s *httpSender) send() error { } } var err error + var reply string if send { - _, _, err = s.client.Send(netsender.RequestRecv, pins) + reply, _, err = s.client.Send(netsender.RequestPoll, pins) } + // Extract time from reply + t, err := netsender.ExtractJsonInt(reply, "ts") + if err != nil { + s.log(smartlogger.Warning, pkg+"No timestamp in reply") + } else { + mts.TimeMeta(uint64(t)) + } + + // Extract gps from reply + g, err := netsender.ExtractJsonString(reply, "ll") + if err != nil { + s.log(smartlogger.Warning, pkg+"No gps in reply") + } else { + mts.GpsMeta(g) + } + return err } diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 566f7457..69e027f7 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -29,10 +29,7 @@ LICENSE package mts import ( - "encoding/binary" - "hash/crc32" "io" - "math/bits" "time" "bitbucket.org/ausocean/av/stream/mts/pes" @@ -49,49 +46,24 @@ const ( psiSendCount = 100 ) +type MetaData struct { + time uint64 + gps string +} + +var metaData = MetaData{time: 0, gps: ""} + +func TimeMeta(t uint64) { + metaData.time = t +} + +func GpsMeta(g string) { + metaData.gps = g +} + func init() { - // Generate IEEE polynomial table - // for the big-endian algorithm. - crcTable := crc32_MakeTable(bits.Reverse32(crc32.IEEE)) - - patTable = completePSI(psi.StdPat, crcTable) - pmtTable = completePSI(psi.StdPmt, crcTable) -} - -func completePSI(psi []byte, tab *crc32.Table) []byte { - var buf [4]byte - crc := crc32_Update(0xffffffff, tab, psi[1:]) - binary.BigEndian.PutUint32(buf[:], crc) - dst := make([]byte, len(psi), psiPacketSize) - copy(dst, psi) - dst = append(dst, buf[:]...) - for len(dst) < cap(dst) { - dst = append(dst, 0xff) - } - return dst -} - -func crc32_MakeTable(poly uint32) *crc32.Table { - var t crc32.Table - for i := range t { - crc := uint32(i) << 24 - for j := 0; j < 8; j++ { - if crc&0x80000000 != 0 { - crc = (crc << 1) ^ poly - } else { - crc <<= 1 - } - } - t[i] = crc - } - return &t -} - -func crc32_Update(crc uint32, tab *crc32.Table, p []byte) uint32 { - for _, v := range p { - crc = tab[byte(crc>>24)^v] ^ (crc << 8) - } - return crc + patTable = psi.StdPat.Bytes() + pmtTable = psi.StdPmtTimeGps.Bytes() } const ( @@ -215,7 +187,17 @@ func (e *Encoder) writePSI() error { return err } - // Write PMT. + // Update pmt table time and gps + err = psi.UpdateTime(pmtTable, metaData.time) + if err != nil { + return err + } + err = psi.UpdateGps(pmtTable, metaData.gps) + if err != nil { + return nil + } + + // Create mts packet from pmt table pmtPkt := Packet{ PUSI: true, PID: pmtPid, diff --git a/stream/mts/psi/op.go b/stream/mts/psi/op.go index 46e0db77..79db4764 100644 --- a/stream/mts/psi/op.go +++ b/stream/mts/psi/op.go @@ -1,6 +1,6 @@ /* NAME - op.go + op.go DESCRIPTION op.go provides functionality for editing and reading bytes slices directly in order to insert/read timestamp and gps data in psi. @@ -61,7 +61,7 @@ func ChkGps(p []byte) error { // UpdateTime takes the byte slice representation of a psi-pmt as well as a time // as an integer and attempts to update the time descriptor in the pmt with the // given time if the time descriptor exists, otherwise an error is returned -func UpdateTime(d []byte, t int) error { +func UpdateTime(d []byte, t uint64) error { err := ChkTime(d) if err != nil { return err diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go index 3d3a74a7..977a29e4 100644 --- a/stream/mts/psi/psi_test.go +++ b/stream/mts/psi/psi_test.go @@ -105,15 +105,15 @@ var bytesTests = []struct { // Pat test { name: "pat Bytes()", - input: stdPat, - want: stdPatBytes, + input: StdPat, + want: StdPatBytes, }, // Pmt test data no descriptor { name: "pmt to Bytes() without descriptors", - input: stdPmt, - want: stdPmtBytes, + input: StdPmt, + want: StdPmtBytes, }, // Pmt with time descriptor diff --git a/stream/mts/psi/std.go b/stream/mts/psi/std.go index ce87b42b..34647821 100644 --- a/stream/mts/psi/std.go +++ b/stream/mts/psi/std.go @@ -33,7 +33,7 @@ const ( // Some common manifestations of PSI var ( // PSI struct to represent basic pat - stdPat = PSI{ + StdPat = PSI{ Pf: 0x00, Tid: 0x00, Ssi: true, @@ -53,7 +53,7 @@ var ( } // PSI struct to represent basic pmt without descriptors for time and gps - stdPmt = PSI{ + StdPmt = PSI{ Pf: 0x00, Tid: 0x02, Ssi: true, @@ -77,7 +77,7 @@ var ( } // Std pmt with time and gps descriptors, time and gps fields are zerod out - stdPmtTimeGps = PSI{ + StdPmtTimeGps = PSI{ Pf: 0x00, Tid: 0x02, Ssi: true, @@ -115,7 +115,7 @@ var ( // Std PSI in bytes form var ( - stdPatBytes = []byte{ + StdPatBytes = []byte{ 0x00, // pointer // ---- section included in data sent to CRC32 during check @@ -136,7 +136,7 @@ var ( // 0x2a, 0xb1, 0x04, 0xb2, // CRC // ---- } - stdPmtBytes = []byte{ + StdPmtBytes = []byte{ 0x00, // pointer // ---- section included in data sent to CRC32 during check From ce036abf8be68d1baef6ee22152af3bcd1ff2229 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 13 Dec 2018 12:34:21 +1030 Subject: [PATCH 28/54] revid: changed request time to recv --- revid/senders.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/revid/senders.go b/revid/senders.go index fdf5fe4c..97276793 100644 --- a/revid/senders.go +++ b/revid/senders.go @@ -146,7 +146,7 @@ func (s *httpSender) send() error { var err error var reply string if send { - reply, _, err = s.client.Send(netsender.RequestPoll, pins) + reply, _, err = s.client.Send(netsender.RequestRecv, pins) } // Extract time from reply t, err := netsender.ExtractJsonInt(reply, "ts") From 35d86b559d0ba1f4a421f8ea59b498f5aa3cd878 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 13 Dec 2018 15:09:23 +1030 Subject: [PATCH 29/54] revid & psi: fixed playback issues... added padding to pat/pmt tables --- stream/mts/encoder.go | 33 +++++++++++++++++++++------------ stream/mts/psi/op.go | 18 ++++++++++++++++++ stream/mts/psi/psi.go | 8 +------- stream/mts/psi/psi_test.go | 4 ++++ 4 files changed, 44 insertions(+), 19 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 69e027f7..e6f35728 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -63,7 +63,7 @@ func GpsMeta(g string) { func init() { patTable = psi.StdPat.Bytes() - pmtTable = psi.StdPmtTimeGps.Bytes() + pmtTable = psi.StdPmt.Bytes() } const ( @@ -180,22 +180,24 @@ func (e *Encoder) writePSI() error { PID: patPid, CC: e.ccFor(patPid), AFC: hasPayload, - Payload: patTable, + Payload: addPadding(patTable), } _, err := e.dst.Write(patPkt.Bytes()) if err != nil { return err } - // Update pmt table time and gps - err = psi.UpdateTime(pmtTable, metaData.time) - if err != nil { - return err - } - err = psi.UpdateGps(pmtTable, metaData.gps) - if err != nil { - return nil - } + /* + // Update pmt table time and gps + err = psi.UpdateTime(pmtTable, metaData.time) + if err != nil { + return err + } + err = psi.UpdateGps(pmtTable, metaData.gps) + if err != nil { + return nil + } + */ // Create mts packet from pmt table pmtPkt := Packet{ @@ -203,7 +205,7 @@ func (e *Encoder) writePSI() error { PID: pmtPid, CC: e.ccFor(pmtPid), AFC: hasPayload, - Payload: pmtTable, + Payload: addPadding(pmtTable), } _, err = e.dst.Write(pmtPkt.Bytes()) if err != nil { @@ -213,6 +215,13 @@ func (e *Encoder) writePSI() error { return nil } +func addPadding(d []byte) []byte { + for len(d) < psiPacketSize { + d = append(d, 0xff) + } + return d +} + // tick advances the clock one frame interval. func (e *Encoder) tick() { e.clock += e.frameInterval diff --git a/stream/mts/psi/op.go b/stream/mts/psi/op.go index 79db4764..7c38af2a 100644 --- a/stream/mts/psi/op.go +++ b/stream/mts/psi/op.go @@ -30,6 +30,8 @@ package psi import ( "encoding/binary" "errors" + "hash/crc32" + "math/bits" ) // timeBytes takes a timestamp as an uint64 and converts to an 8 byte slice - @@ -70,6 +72,7 @@ func UpdateTime(d []byte, t uint64) error { for i := range d[timeDataIndx : timeDataIndx+timeDataSize] { d[i+timeDataIndx] = ts[i] } + d = updateCrc(d) return nil } @@ -109,5 +112,20 @@ func UpdateGps(d []byte, s string) error { for i := range d[gpsDataIndx : gpsDataIndx+gpsDataSize] { d[i+gpsDataIndx] = gb[i] } + d = updateCrc(d) return nil } + +func addCrc(out []byte) []byte { + out = append(out, make([]byte, 4)...) + out = updateCrc(out) + return out +} +func updateCrc(out []byte) []byte { + crc32 := crc32_Update(0xffffffff, crc32_MakeTable(bits.Reverse32(crc32.IEEE)), out[1:len(out)-4]) + out[len(out)-4] = byte(crc32 >> 24) + out[len(out)-3] = byte(crc32 >> 16) + out[len(out)-2] = byte(crc32 >> 8) + out[len(out)-1] = byte(crc32) + return out +} diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 6abc3a51..a2c34046 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -28,7 +28,6 @@ package psi import ( "hash/crc32" - "math/bits" ) // Lengths of section definitions @@ -230,12 +229,7 @@ func (p *PSI) Bytes() []byte { out[2] = 0x80 | 0x30 | (0x03 & byte(p.Sl>>8)) out[3] = byte(p.Sl) out = append(out, p.Tss.Bytes()...) - crc32 := crc32_Update(0xffffffff, crc32_MakeTable(bits.Reverse32(crc32.IEEE)), out[3:]) - out = append(out, make([]byte, 0, 4)...) - out = append(out, byte(crc32>>24)) - out = append(out, byte(crc32>>16)) - out = append(out, byte(crc32>>8)) - out = append(out, byte(crc32)) + out = addCrc(out) return out } diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go index 977a29e4..cba8247f 100644 --- a/stream/mts/psi/psi_test.go +++ b/stream/mts/psi/psi_test.go @@ -218,7 +218,9 @@ func TestTimestampToBytes(t *testing.T) { func TestTimeUpdate(t *testing.T) { cpy := make([]byte, len(pmtTimeBytes1)) copy(cpy, pmtTimeBytes1) + cpy = addCrc(cpy) err := UpdateTime(cpy, tstTime2) + cpy = cpy[:len(cpy)-4] if err != nil { t.Errorf("Update time returned err: %v", err) } @@ -242,7 +244,9 @@ func TestTimeGet(t *testing.T) { func TestGpsUpdate(t *testing.T) { cpy := make([]byte, len(pmtTimeGpsBytes1)) copy(cpy, pmtTimeGpsBytes1) + cpy = addCrc(cpy) err := UpdateGps(cpy, gpsTstStr2) + cpy = cpy[:len(cpy)-4] if err != nil { t.Errorf("Update time returned err: %v", err) } From b78904ac72b145107217d73d68228784646f3419 Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 13 Dec 2018 15:51:51 +1030 Subject: [PATCH 30/54] revid + psi: getting metadata from replies now --- revid/senders.go | 20 +++++++++++++++++--- stream/mts/psi/op.go | 1 + 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/revid/senders.go b/revid/senders.go index 97276793..64eb7967 100644 --- a/revid/senders.go +++ b/revid/senders.go @@ -148,8 +148,22 @@ func (s *httpSender) send() error { if send { reply, _, err = s.client.Send(netsender.RequestRecv, pins) } + + err = extractMeta(reply, s) + if err != nil { + return err + } + + return err +} + +func extractMeta(r string, s *httpSender) error { + dec, err := netsender.NewJsonDecoder(r) + if err != nil { + s.log(smartlogger.Warning, pkg+"Could not decode JSON to get metadata") + } // Extract time from reply - t, err := netsender.ExtractJsonInt(reply, "ts") + t, err := dec.Int("ts") if err != nil { s.log(smartlogger.Warning, pkg+"No timestamp in reply") } else { @@ -157,14 +171,14 @@ func (s *httpSender) send() error { } // Extract gps from reply - g, err := netsender.ExtractJsonString(reply, "ll") + g, err := dec.String("ll") if err != nil { s.log(smartlogger.Warning, pkg+"No gps in reply") } else { mts.GpsMeta(g) } - return err + return nil } func (s *httpSender) release() { diff --git a/stream/mts/psi/op.go b/stream/mts/psi/op.go index 7c38af2a..e8d2f04e 100644 --- a/stream/mts/psi/op.go +++ b/stream/mts/psi/op.go @@ -121,6 +121,7 @@ func addCrc(out []byte) []byte { out = updateCrc(out) return out } + func updateCrc(out []byte) []byte { crc32 := crc32_Update(0xffffffff, crc32_MakeTable(bits.Reverse32(crc32.IEEE)), out[1:len(out)-4]) out[len(out)-4] = byte(crc32 >> 24) From 05fd7c37d1d4bdf67ba3e63ebcb076a6e01add7e Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 13 Dec 2018 17:06:39 +1030 Subject: [PATCH 31/54] psi: fixed issue with std psi --- stream/mts/encoder.go | 7 +++++-- stream/mts/psi/std.go | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index e6f35728..95dbb441 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -34,6 +34,7 @@ import ( "bitbucket.org/ausocean/av/stream/mts/pes" "bitbucket.org/ausocean/av/stream/mts/psi" + "fmt" ) var ( @@ -63,7 +64,7 @@ func GpsMeta(g string) { func init() { patTable = psi.StdPat.Bytes() - pmtTable = psi.StdPmt.Bytes() + pmtTable = psi.StdPmtTimeGps.Bytes() } const ( @@ -199,13 +200,15 @@ func (e *Encoder) writePSI() error { } */ + packet := addPadding(pmtTable) + fmt.Println(len(pmtTable)) // Create mts packet from pmt table pmtPkt := Packet{ PUSI: true, PID: pmtPid, CC: e.ccFor(pmtPid), AFC: hasPayload, - Payload: addPadding(pmtTable), + Payload: packet, } _, err = e.dst.Write(pmtPkt.Bytes()) if err != nil { diff --git a/stream/mts/psi/std.go b/stream/mts/psi/std.go index 34647821..95aaf806 100644 --- a/stream/mts/psi/std.go +++ b/stream/mts/psi/std.go @@ -81,7 +81,7 @@ var ( Pf: 0x00, Tid: 0x02, Ssi: true, - Sl: uint16(0x12), + Sl: uint16(0x3e), Tss: &TSS{ Tide: uint16(0x01), V: 0, From afc7c1f086002bc9a6c0775ec4d48ab69237045d Mon Sep 17 00:00:00 2001 From: saxon Date: Thu, 13 Dec 2018 17:13:34 +1030 Subject: [PATCH 32/54] mts: actually updating time and gps now --- stream/mts/encoder.go | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 95dbb441..78cbf2b2 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -34,7 +34,6 @@ import ( "bitbucket.org/ausocean/av/stream/mts/pes" "bitbucket.org/ausocean/av/stream/mts/psi" - "fmt" ) var ( @@ -188,27 +187,23 @@ func (e *Encoder) writePSI() error { return err } - /* - // Update pmt table time and gps - err = psi.UpdateTime(pmtTable, metaData.time) - if err != nil { - return err - } - err = psi.UpdateGps(pmtTable, metaData.gps) - if err != nil { - return nil - } - */ + // Update pmt table time and gps + err = psi.UpdateTime(pmtTable, metaData.time) + if err != nil { + return err + } + err = psi.UpdateGps(pmtTable, metaData.gps) + if err != nil { + return nil + } - packet := addPadding(pmtTable) - fmt.Println(len(pmtTable)) // Create mts packet from pmt table pmtPkt := Packet{ PUSI: true, PID: pmtPid, CC: e.ccFor(pmtPid), AFC: hasPayload, - Payload: packet, + Payload: addPadding(pmtTable), } _, err = e.dst.Write(pmtPkt.Bytes()) if err != nil { From 7fa245bca320674106f0c1c262f20049960644ce Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 11:15:49 +1030 Subject: [PATCH 33/54] psi: wrote testing for get gps from pmt byte slice --- stream/mts/psi/op.go | 13 +++++++++++++ stream/mts/psi/psi_test.go | 19 +++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/stream/mts/psi/op.go b/stream/mts/psi/op.go index e8d2f04e..0d4f607c 100644 --- a/stream/mts/psi/op.go +++ b/stream/mts/psi/op.go @@ -92,6 +92,19 @@ func TimeOf(p []byte) (t uint64, err error) { return t, nil } +// TimeOf takes a byte slice representation of a psi-pmt and extracts it's +// timestamp, returning as a uint64 if it exists, otherwise returning 0 and nil +// if it does not exist +func GpsOf(p []byte) (g string, err error) { + err = ChkGps(p) + if err != nil { + return "", err + } + gBytes := p[gpsDataIndx : gpsDataIndx+gpsDataSize] + g = string(gBytes) + return g, nil +} + // GpsStrBytes take a string of gps data and converts to a 32 byte slice - // easy update of slice representation of a pmt with gps descriptor func GpsStrBytes(l string) (out []byte) { diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go index cba8247f..12f0ea5b 100644 --- a/stream/mts/psi/psi_test.go +++ b/stream/mts/psi/psi_test.go @@ -39,8 +39,8 @@ const ( // GPS string for testing const ( - gpsTstStr1 = "$GPGGA,123519,4807.038,N,01131.000,E,1,08,0.9,545.4,M,46.9,M,,*47" - gpsTstStr2 = "$GPGGA,183710,4902.048,N,02171.020,E,1,09,0.5,547.2,M,43.4,M,,*42" + gpsTstStr1 = "$GPGGA,123519,4807.038,N,01131.0" + gpsTstStr2 = "$GPGGA,183710,4902.048,N,02171.0" ) // err message @@ -240,6 +240,21 @@ func TestTimeGet(t *testing.T) { } } +func TestGpsGet(t *testing.T) { + pb := StdPmtTimeGps.Bytes() + err := UpdateGps(pb, gpsTstStr1) + if err != nil { + t.Errorf("Error for TestGpsGet UpdateGps(pb, gpsTstStr1): %v", err) + } + g, err := GpsOf(pb) + if err != nil { + t.Errorf("Error for TestGpsGet GpsOf(pb): %v", err) + } + if g != gpsTstStr1 { + t.Errorf(errCmp, "TestGpsGet", gpsTstStr1, g) + } +} + // TestGpsUpdate checks to see if we can update the gps string in a pmt correctly func TestGpsUpdate(t *testing.T) { cpy := make([]byte, len(pmtTimeGpsBytes1)) From 798add533bcd3e156e863362c80c0ec035c20f61 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 13:35:56 +1030 Subject: [PATCH 34/54] revid + mts +psi: wrote func to find pmt in byte slice, wrote func to get gps, changed the way in which psi are insterted, based no of mpgets packets to send on time, i.e. per second --- cmd/revid-cli/main.go | 5 +++-- revid/revid.go | 7 ++++++- revid/senders.go | 5 ++++- stream/mts/encoder.go | 19 ++++++++++--------- stream/mts/mpegts.go | 17 +++++++++++++++++ 5 files changed, 40 insertions(+), 13 deletions(-) diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index f50fd367..32b18be6 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -45,7 +45,7 @@ const ( progName = "revid-cli" // Logging is set to INFO level. - defaultLogVerbosity = smartlogger.Info + defaultLogVerbosity = smartlogger.Debug ) // Other misc consts @@ -231,8 +231,9 @@ func handleFlags() revid.Config { switch *verbosityPtr { case "No": cfg.LogLevel = smartlogger.Fatal - case "Yes": + case "Debug": cfg.LogLevel = smartlogger.Debug + //logger.SetLevel(smartlogger.Debug) case "": default: logger.Log(smartlogger.Error, pkg+"bad verbosity argument") diff --git a/revid/revid.go b/revid/revid.go index 5baf69ed..f7dbc9ad 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -125,6 +125,9 @@ type Revid struct { isRunning bool } +var now = time.Now() +var prevTime = now + // packer takes data segments and packs them into clips // of the number frames specified in the owners config. type packer struct { @@ -153,9 +156,11 @@ func (p *packer) Write(frame []byte) (int, error) { return n, err } p.packetCount++ - if p.packetCount >= p.owner.config.FramesPerClip { + now = time.Now() + if now.Sub(prevTime) > clipDuration && p.packetCount%7 == 0 { p.owner.buffer.Flush() p.packetCount = 0 + prevTime = now } return len(frame), nil } diff --git a/revid/senders.go b/revid/senders.go index 64eb7967..d2172b09 100644 --- a/revid/senders.go +++ b/revid/senders.go @@ -29,6 +29,7 @@ LICENSE package revid import ( + "fmt" "io" "net" "os" @@ -160,13 +161,14 @@ func (s *httpSender) send() error { func extractMeta(r string, s *httpSender) error { dec, err := netsender.NewJsonDecoder(r) if err != nil { - s.log(smartlogger.Warning, pkg+"Could not decode JSON to get metadata") + return nil } // Extract time from reply t, err := dec.Int("ts") if err != nil { s.log(smartlogger.Warning, pkg+"No timestamp in reply") } else { + s.log(smartlogger.Debug, fmt.Sprintf("%v got timestamp: %v", pkg, t)) mts.TimeMeta(uint64(t)) } @@ -175,6 +177,7 @@ func extractMeta(r string, s *httpSender) error { if err != nil { s.log(smartlogger.Warning, pkg+"No gps in reply") } else { + s.log(smartlogger.Debug, fmt.Sprintf("%v got gps: %v", pkg, g)) mts.GpsMeta(g) } diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 78cbf2b2..fc1d2ef1 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -43,7 +43,7 @@ var ( const ( psiPacketSize = 184 - psiSendCount = 100 + psiSendCount = 21 ) type MetaData struct { @@ -92,7 +92,7 @@ type Encoder struct { frameInterval time.Duration ptsOffset time.Duration - psiCount uint + psiCount int continuity map[int]byte } @@ -126,13 +126,7 @@ const ( // generate handles the incoming data and generates equivalent mpegts packets - // sending them to the output channel func (e *Encoder) Encode(nalu []byte) error { - if e.psiCount <= 0 { - err := e.writePSI() - if err != nil { - return err - } - } - e.psiCount-- + // Prepare PES data. pesPkt := pes.Packet{ StreamID: streamID, @@ -162,6 +156,13 @@ func (e *Encoder) Encode(nalu []byte) error { pkt.PCR = e.pcr() pusi = false } + if e.psiCount <= 0 { + err := e.writePSI() + if err != nil { + return err + } + } + e.psiCount-- _, err := e.dst.Write(pkt.Bytes()) if err != nil { return err diff --git a/stream/mts/mpegts.go b/stream/mts/mpegts.go index eb1dce6f..d8bd0ed9 100644 --- a/stream/mts/mpegts.go +++ b/stream/mts/mpegts.go @@ -28,6 +28,10 @@ LICENSE package mts +import ( + "errors" +) + const ( mpegTsSize = 188 mpegtsPayloadSize = 176 @@ -123,6 +127,19 @@ type Packet struct { Payload []byte // Mpeg ts Payload } +// FindPMT will take a clip of mpegts and try to find a PMT table - if one +// is found, then it is returned, otherwise nil and an error is returned. +func FindPMT(d []byte) (p []byte, err error) { + for i := 0; i < len(d); i += mpegTsSize { + pid := (uint16(d[i+1]&0x1f) << 8) | uint16(d[i+2]) + if pid == pmtPid { + p = d[i+4 : i+mpegTsSize] + return + } + } + return nil, errors.New("Could not find pmt table in mpegts data") +} + // FillPayload takes a channel and fills the packets Payload field until the // channel is empty or we've the packet reaches capacity func (p *Packet) FillPayload(data []byte) int { From dc6964d41c664a1a1a09687422a8975eaea132dc Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 13:46:10 +1030 Subject: [PATCH 35/54] psi: trimming excess bytes in gps data from pmt --- stream/mts/psi/op.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stream/mts/psi/op.go b/stream/mts/psi/op.go index 0d4f607c..91e0c88f 100644 --- a/stream/mts/psi/op.go +++ b/stream/mts/psi/op.go @@ -28,6 +28,7 @@ LICENSE package psi import ( + "bytes" "encoding/binary" "errors" "hash/crc32" @@ -101,6 +102,7 @@ func GpsOf(p []byte) (g string, err error) { return "", err } gBytes := p[gpsDataIndx : gpsDataIndx+gpsDataSize] + gBytes = bytes.Trim(gBytes, "\x00") g = string(gBytes) return g, nil } From f1ee09ad7b3f367ff1f26b6f777effd000e118b3 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 15:59:44 +1030 Subject: [PATCH 36/54] revid: improved naming of encoder globals for location and timestamp --- revid/senders.go | 14 ++++++-------- stream/mts/encoder.go | 4 ++-- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/revid/senders.go b/revid/senders.go index d2172b09..9d9edcc5 100644 --- a/revid/senders.go +++ b/revid/senders.go @@ -148,14 +148,12 @@ func (s *httpSender) send() error { var reply string if send { reply, _, err = s.client.Send(netsender.RequestRecv, pins) + if err != nil { + return err + } } - err = extractMeta(reply, s) - if err != nil { - return err - } - - return err + return extractMeta(reply, s) } func extractMeta(r string, s *httpSender) error { @@ -169,7 +167,7 @@ func extractMeta(r string, s *httpSender) error { s.log(smartlogger.Warning, pkg+"No timestamp in reply") } else { s.log(smartlogger.Debug, fmt.Sprintf("%v got timestamp: %v", pkg, t)) - mts.TimeMeta(uint64(t)) + mts.SetTimeStamp(uint64(t)) } // Extract gps from reply @@ -178,7 +176,7 @@ func extractMeta(r string, s *httpSender) error { s.log(smartlogger.Warning, pkg+"No gps in reply") } else { s.log(smartlogger.Debug, fmt.Sprintf("%v got gps: %v", pkg, g)) - mts.GpsMeta(g) + mts.SetLocation(g) } return nil diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index fc1d2ef1..d824a84c 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -53,11 +53,11 @@ type MetaData struct { var metaData = MetaData{time: 0, gps: ""} -func TimeMeta(t uint64) { +func SetTimeStamp(t uint64) { metaData.time = t } -func GpsMeta(g string) { +func SetLocation(g string) { metaData.gps = g } From e79f6d191db5faf5332752318979a313dd761ba1 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 16:01:10 +1030 Subject: [PATCH 37/54] revid: made extractMeta a function of httpSender --- revid/senders.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/revid/senders.go b/revid/senders.go index 9d9edcc5..6742a6b5 100644 --- a/revid/senders.go +++ b/revid/senders.go @@ -153,10 +153,10 @@ func (s *httpSender) send() error { } } - return extractMeta(reply, s) + return s.extractMeta(reply) } -func extractMeta(r string, s *httpSender) error { +func (s *httpSender) extractMeta(r string) error { dec, err := netsender.NewJsonDecoder(r) if err != nil { return nil From 21dd2f4b701a93937453a7a09e95b8ca9850325a Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 16:02:47 +1030 Subject: [PATCH 38/54] revid: using location instead of gps in names --- revid/senders.go | 6 ++-- stream/mts/encoder.go | 14 ++++---- stream/mts/psi/op.go | 40 ++++++++++----------- stream/mts/psi/psi.go | 10 +++--- stream/mts/psi/psi_test.go | 72 +++++++++++++++++++------------------- stream/mts/psi/std.go | 16 ++++----- 6 files changed, 79 insertions(+), 79 deletions(-) diff --git a/revid/senders.go b/revid/senders.go index 6742a6b5..549ce30d 100644 --- a/revid/senders.go +++ b/revid/senders.go @@ -170,12 +170,12 @@ func (s *httpSender) extractMeta(r string) error { mts.SetTimeStamp(uint64(t)) } - // Extract gps from reply + // Extract location from reply g, err := dec.String("ll") if err != nil { - s.log(smartlogger.Warning, pkg+"No gps in reply") + s.log(smartlogger.Warning, pkg+"No location in reply") } else { - s.log(smartlogger.Debug, fmt.Sprintf("%v got gps: %v", pkg, g)) + s.log(smartlogger.Debug, fmt.Sprintf("%v got location: %v", pkg, g)) mts.SetLocation(g) } diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index d824a84c..3e9359b9 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -47,23 +47,23 @@ const ( ) type MetaData struct { - time uint64 - gps string + time uint64 + location string } -var metaData = MetaData{time: 0, gps: ""} +var metaData = MetaData{time: 0, location: ""} func SetTimeStamp(t uint64) { metaData.time = t } func SetLocation(g string) { - metaData.gps = g + metaData.location = g } func init() { patTable = psi.StdPat.Bytes() - pmtTable = psi.StdPmtTimeGps.Bytes() + pmtTable = psi.StdPmtTimeLocation.Bytes() } const ( @@ -188,12 +188,12 @@ func (e *Encoder) writePSI() error { return err } - // Update pmt table time and gps + // Update pmt table time and location err = psi.UpdateTime(pmtTable, metaData.time) if err != nil { return err } - err = psi.UpdateGps(pmtTable, metaData.gps) + err = psi.UpdateLocation(pmtTable, metaData.location) if err != nil { return nil } diff --git a/stream/mts/psi/op.go b/stream/mts/psi/op.go index 91e0c88f..d2483e2f 100644 --- a/stream/mts/psi/op.go +++ b/stream/mts/psi/op.go @@ -3,7 +3,7 @@ NAME op.go DESCRIPTION op.go provides functionality for editing and reading bytes slices - directly in order to insert/read timestamp and gps data in psi. + directly in order to insert/read timestamp and location data in psi. AUTHOR Saxon Milton @@ -52,11 +52,11 @@ func ChkTime(p []byte) error { return nil } -// ChkGps takes a psi as a byte slice and checks to see if it has a gps descriptor +// ChkLocation takes a psi as a byte slice and checks to see if it has a location descriptor // - if so return nil, otherwise return error -func ChkGps(p []byte) error { - if p[gpsTagIndx] != gpsDescTag { - return errors.New("PSI does not contain a gps descriptor, cannot update") +func ChkLocation(p []byte) error { + if p[locationTagIndx] != locationDescTag { + return errors.New("PSI does not contain a location descriptor, cannot update") } return nil } @@ -96,36 +96,36 @@ func TimeOf(p []byte) (t uint64, err error) { // TimeOf takes a byte slice representation of a psi-pmt and extracts it's // timestamp, returning as a uint64 if it exists, otherwise returning 0 and nil // if it does not exist -func GpsOf(p []byte) (g string, err error) { - err = ChkGps(p) +func LocationOf(p []byte) (g string, err error) { + err = ChkLocation(p) if err != nil { return "", err } - gBytes := p[gpsDataIndx : gpsDataIndx+gpsDataSize] + gBytes := p[locationDataIndx : locationDataIndx+locationDataSize] gBytes = bytes.Trim(gBytes, "\x00") g = string(gBytes) return g, nil } -// GpsStrBytes take a string of gps data and converts to a 32 byte slice - -// easy update of slice representation of a pmt with gps descriptor -func GpsStrBytes(l string) (out []byte) { - out = make([]byte, gpsDataSize) +// LocationStrBytes take a string of location data and converts to a 32 byte slice - +// easy update of slice representation of a pmt with location descriptor +func LocationStrBytes(l string) (out []byte) { + out = make([]byte, locationDataSize) copy(out, []byte(l)) return } -// UpdateGps takes a byte slice representation of a psi-pmt containing a gps -// descriptor and attempts to update the gps data value with the passed string. -// If the psi does not contain a gps descriptor, and error is returned. -func UpdateGps(d []byte, s string) error { - err := ChkGps(d) +// UpdateLocation takes a byte slice representation of a psi-pmt containing a location +// descriptor and attempts to update the location data value with the passed string. +// If the psi does not contain a location descriptor, and error is returned. +func UpdateLocation(d []byte, s string) error { + err := ChkLocation(d) if err != nil { return err } - gb := GpsStrBytes(s) - for i := range d[gpsDataIndx : gpsDataIndx+gpsDataSize] { - d[i+gpsDataIndx] = gb[i] + gb := LocationStrBytes(s) + for i := range d[locationDataIndx : locationDataIndx+locationDataSize] { + d[i+locationDataIndx] = gb[i] } d = updateCrc(d) return nil diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index a2c34046..cf1ea440 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -54,12 +54,12 @@ const ( timeDataSize = 8 ) -// Consts relating to gps description +// Consts relating to location description const ( - gpsDescTag = 235 - gpsTagIndx = 23 - gpsDataIndx = 25 - gpsDataSize = 32 // bytes + locationDescTag = 235 + locationTagIndx = 23 + locationDataIndx = 25 + locationDataSize = 32 // bytes ) // Program specific information diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go index 12f0ea5b..83df4bcc 100644 --- a/stream/mts/psi/psi_test.go +++ b/stream/mts/psi/psi_test.go @@ -39,8 +39,8 @@ const ( // GPS string for testing const ( - gpsTstStr1 = "$GPGGA,123519,4807.038,N,01131.0" - gpsTstStr2 = "$GPGGA,183710,4902.048,N,02171.0" + locationTstStr1 = "$GPGGA,123519,4807.038,N,01131.0" + locationTstStr2 = "$GPGGA,183710,4902.048,N,02171.0" ) // err message @@ -57,15 +57,15 @@ var ( // Parts to construct bytes of pmt with time and bytes var ( - pmtTimeGpsBytesPart1 = []byte{ + pmtTimeLocationBytesPart1 = []byte{ 0x00, 0x02, 0xb0, 0x12, 0x00, 0x01, 0xc1, 0x00, 0x00, 0xe1, 0x00, 0xf0, 0x0a, byte(timeDescTag), // Descriptor tag for timestamp byte(timeDataSize), // Length of bytes to follow 0x00, 0x00, 0x00, 0x00, 0x67, 0x6F, 0x74, 0x5F, // Timestamp data - byte(gpsDescTag), // Descriptor tag for gps - byte(gpsDataSize), // Length of bytes to follow + byte(locationDescTag), // Descriptor tag for location + byte(locationDataSize), // Length of bytes to follow } - pmtTimeGpsBytesPart2 = []byte{ + pmtTimeLocationBytesPart2 = []byte{ 0x1b, 0xe1, 0x00, 0xf0, 0x00, } ) @@ -89,11 +89,11 @@ var ( 0x1b, 0xe1, 0x00, 0xf0, 0x00, } - // Bytes representing pmt with time1 and gps1 - pmtTimeGpsBytes1 = buildPmtTimeGpsBytes(gpsTstStr1) + // Bytes representing pmt with time1 and location1 + pmtTimeLocationBytes1 = buildPmtTimeLocationBytes(locationTstStr1) - // bytes representing pmt with with time1 and gps 2 - pmtTimeGpsBytes2 = buildPmtTimeGpsBytes(gpsTstStr2) + // bytes representing pmt with with time1 and location 2 + pmtTimeLocationBytes2 = buildPmtTimeLocationBytes(locationTstStr2) ) // bytesTests contains data for testing the Bytes() funcs for the PSI data struct @@ -151,9 +151,9 @@ var bytesTests = []struct { want: pmtTimeBytes1, }, - // Pmt with time and gps + // Pmt with time and location { - name: "pmt Bytes() with time and gps", + name: "pmt Bytes() with time and location", input: PSI{ Pf: 0x00, Tid: 0x02, @@ -175,9 +175,9 @@ var bytesTests = []struct { Dd: TimeBytes(tstTime2), }, Desc{ - Dt: byte(gpsDescTag), - Dl: byte(gpsDataSize), - Dd: GpsStrBytes(gpsTstStr1), + Dt: byte(locationDescTag), + Dl: byte(locationDataSize), + Dd: LocationStrBytes(locationTstStr1), }, }, Essd: &ESSD{ @@ -188,7 +188,7 @@ var bytesTests = []struct { }, }, }, - want: buildPmtTimeGpsBytes(gpsTstStr1), + want: buildPmtTimeLocationBytes(locationTstStr1), }, } @@ -240,40 +240,40 @@ func TestTimeGet(t *testing.T) { } } -func TestGpsGet(t *testing.T) { - pb := StdPmtTimeGps.Bytes() - err := UpdateGps(pb, gpsTstStr1) +func TestLocationGet(t *testing.T) { + pb := StdPmtTimeLocation.Bytes() + err := UpdateLocation(pb, locationTstStr1) if err != nil { - t.Errorf("Error for TestGpsGet UpdateGps(pb, gpsTstStr1): %v", err) + t.Errorf("Error for TestLocationGet UpdateLocation(pb, locationTstStr1): %v", err) } - g, err := GpsOf(pb) + g, err := LocationOf(pb) if err != nil { - t.Errorf("Error for TestGpsGet GpsOf(pb): %v", err) + t.Errorf("Error for TestLocationGet LocationOf(pb): %v", err) } - if g != gpsTstStr1 { - t.Errorf(errCmp, "TestGpsGet", gpsTstStr1, g) + if g != locationTstStr1 { + t.Errorf(errCmp, "TestLocationGet", locationTstStr1, g) } } -// TestGpsUpdate checks to see if we can update the gps string in a pmt correctly -func TestGpsUpdate(t *testing.T) { - cpy := make([]byte, len(pmtTimeGpsBytes1)) - copy(cpy, pmtTimeGpsBytes1) +// TestLocationUpdate checks to see if we can update the location string in a pmt correctly +func TestLocationUpdate(t *testing.T) { + cpy := make([]byte, len(pmtTimeLocationBytes1)) + copy(cpy, pmtTimeLocationBytes1) cpy = addCrc(cpy) - err := UpdateGps(cpy, gpsTstStr2) + err := UpdateLocation(cpy, locationTstStr2) cpy = cpy[:len(cpy)-4] if err != nil { t.Errorf("Update time returned err: %v", err) } - if !bytes.Equal(pmtTimeGpsBytes2, cpy) { - t.Errorf(errCmp, "TestGpsUpdate", pmtTimeGpsBytes2, cpy) + if !bytes.Equal(pmtTimeLocationBytes2, cpy) { + t.Errorf(errCmp, "TestLocationUpdate", pmtTimeLocationBytes2, cpy) } } -// buildPmtTimeGpsBytes is a helper function to help construct the byte slices -// for pmts with time and gps, as the gps data field is 32 bytes, i.e. quite large +// buildPmtTimeLocationBytes is a helper function to help construct the byte slices +// for pmts with time and location, as the location data field is 32 bytes, i.e. quite large // to type out -func buildPmtTimeGpsBytes(tstStr string) []byte { - return append(append(append(make([]byte, 0), pmtTimeGpsBytesPart1...), - GpsStrBytes(tstStr)...), pmtTimeGpsBytesPart2...) +func buildPmtTimeLocationBytes(tstStr string) []byte { + return append(append(append(make([]byte, 0), pmtTimeLocationBytesPart1...), + LocationStrBytes(tstStr)...), pmtTimeLocationBytesPart2...) } diff --git a/stream/mts/psi/std.go b/stream/mts/psi/std.go index 95aaf806..5596805d 100644 --- a/stream/mts/psi/std.go +++ b/stream/mts/psi/std.go @@ -27,7 +27,7 @@ LICENSE package psi const ( - pmtTimeGpsPil = 44 + pmtTimeLocationPil = 44 ) // Some common manifestations of PSI @@ -52,7 +52,7 @@ var ( }, } - // PSI struct to represent basic pmt without descriptors for time and gps + // PSI struct to represent basic pmt without descriptors for time and location StdPmt = PSI{ Pf: 0x00, Tid: 0x02, @@ -76,8 +76,8 @@ var ( }, } - // Std pmt with time and gps descriptors, time and gps fields are zerod out - StdPmtTimeGps = PSI{ + // Std pmt with time and location descriptors, time and location fields are zerod out + StdPmtTimeLocation = PSI{ Pf: 0x00, Tid: 0x02, Ssi: true, @@ -90,7 +90,7 @@ var ( Lsn: 0, Sd: &PMT{ Pcrpid: 0x0100, - Pil: pmtTimeGpsPil, + Pil: pmtTimeLocationPil, Pd: []Desc{ Desc{ Dt: byte(timeDescTag), @@ -98,9 +98,9 @@ var ( Dd: make([]byte, timeDataSize), }, Desc{ - Dt: byte(gpsDescTag), - Dl: byte(gpsDataSize), - Dd: make([]byte, gpsDataSize), + Dt: byte(locationDescTag), + Dl: byte(locationDataSize), + Dd: make([]byte, locationDataSize), }, }, Essd: &ESSD{ From cf4c44f4d5979d6bec04c69142daa3d74277838b Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 16:06:01 +1030 Subject: [PATCH 39/54] psi: using HasLocation or HasTime instead of ChkLocation or ChkTime --- stream/mts/psi/op.go | 10 +++++----- stream/mts/psi/psi_test.go | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/stream/mts/psi/op.go b/stream/mts/psi/op.go index d2483e2f..56fed186 100644 --- a/stream/mts/psi/op.go +++ b/stream/mts/psi/op.go @@ -54,7 +54,7 @@ func ChkTime(p []byte) error { // ChkLocation takes a psi as a byte slice and checks to see if it has a location descriptor // - if so return nil, otherwise return error -func ChkLocation(p []byte) error { +func HasLocation(p []byte) error { if p[locationTagIndx] != locationDescTag { return errors.New("PSI does not contain a location descriptor, cannot update") } @@ -80,7 +80,7 @@ func UpdateTime(d []byte, t uint64) error { // TimeOf takes a byte slice representation of a psi-pmt and extracts it's // timestamp, returning as a uint64 if it exists, otherwise returning 0 and nil // if it does not exist -func TimeOf(p []byte) (t uint64, err error) { +func TimeFrom(p []byte) (t uint64, err error) { err = ChkTime(p) if err != nil { return 0, err @@ -96,8 +96,8 @@ func TimeOf(p []byte) (t uint64, err error) { // TimeOf takes a byte slice representation of a psi-pmt and extracts it's // timestamp, returning as a uint64 if it exists, otherwise returning 0 and nil // if it does not exist -func LocationOf(p []byte) (g string, err error) { - err = ChkLocation(p) +func LocationFrom(p []byte) (g string, err error) { + err = HasLocation(p) if err != nil { return "", err } @@ -119,7 +119,7 @@ func LocationStrBytes(l string) (out []byte) { // descriptor and attempts to update the location data value with the passed string. // If the psi does not contain a location descriptor, and error is returned. func UpdateLocation(d []byte, s string) error { - err := ChkLocation(d) + err := HasLocation(d) if err != nil { return err } diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go index 83df4bcc..eccf057a 100644 --- a/stream/mts/psi/psi_test.go +++ b/stream/mts/psi/psi_test.go @@ -231,7 +231,7 @@ func TestTimeUpdate(t *testing.T) { // TestTimeGet tsts to see if we can correctly get the timestamp from a pmt func TestTimeGet(t *testing.T) { - s, err := TimeOf(pmtTimeBytes1) + s, err := TimeFrom(pmtTimeBytes1) if err != nil { t.Errorf("Getting timestamp failed with err: %v", err) } @@ -246,7 +246,7 @@ func TestLocationGet(t *testing.T) { if err != nil { t.Errorf("Error for TestLocationGet UpdateLocation(pb, locationTstStr1): %v", err) } - g, err := LocationOf(pb) + g, err := LocationFrom(pb) if err != nil { t.Errorf("Error for TestLocationGet LocationOf(pb): %v", err) } From ac11b281c5696a1e5d95fecfa79f45abab1dd2a8 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 16:09:53 +1030 Subject: [PATCH 40/54] mts: patTable and pmtTable in var block instead of init func now --- stream/mts/encoder.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 3e9359b9..719ccd86 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -36,11 +36,6 @@ import ( "bitbucket.org/ausocean/av/stream/mts/psi" ) -var ( - patTable []byte - pmtTable []byte -) - const ( psiPacketSize = 184 psiSendCount = 21 @@ -61,10 +56,10 @@ func SetLocation(g string) { metaData.location = g } -func init() { +var ( patTable = psi.StdPat.Bytes() pmtTable = psi.StdPmtTimeLocation.Bytes() -} +) const ( sdtPid = 17 From 0b9f0f49fe08d5c4a0fc21e671c71dc316f549c8 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 16:11:45 +1030 Subject: [PATCH 41/54] psi: HasTime rather than chkTime --- stream/mts/psi/op.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stream/mts/psi/op.go b/stream/mts/psi/op.go index 56fed186..78730fc2 100644 --- a/stream/mts/psi/op.go +++ b/stream/mts/psi/op.go @@ -45,7 +45,7 @@ func TimeBytes(t uint64) (s []byte) { // ChkTime takes a psi as a byte slice and checks to see if it has a time descriptor // - if so return nil, otherwise return error -func ChkTime(p []byte) error { +func HasTime(p []byte) error { if p[timeTagIndx] != timeDescTag { return errors.New("PSI does not contain a time descriptor, cannot update") } @@ -65,7 +65,7 @@ func HasLocation(p []byte) error { // as an integer and attempts to update the time descriptor in the pmt with the // given time if the time descriptor exists, otherwise an error is returned func UpdateTime(d []byte, t uint64) error { - err := ChkTime(d) + err := HasTime(d) if err != nil { return err } @@ -81,7 +81,7 @@ func UpdateTime(d []byte, t uint64) error { // timestamp, returning as a uint64 if it exists, otherwise returning 0 and nil // if it does not exist func TimeFrom(p []byte) (t uint64, err error) { - err = ChkTime(p) + err = HasTime(p) if err != nil { return 0, err } From 24f0be991725f0603e3cf42ce841cbeba519abb2 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 16:15:02 +1030 Subject: [PATCH 42/54] mts: FindPmt checks validity of data length --- stream/mts/mpegts.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stream/mts/mpegts.go b/stream/mts/mpegts.go index d8bd0ed9..36b4d2fe 100644 --- a/stream/mts/mpegts.go +++ b/stream/mts/mpegts.go @@ -130,6 +130,9 @@ type Packet struct { // FindPMT will take a clip of mpegts and try to find a PMT table - if one // is found, then it is returned, otherwise nil and an error is returned. func FindPMT(d []byte) (p []byte, err error) { + if len(d) < mpegTsSize { + return nil, errors.New("Mmpegts data not of valid length") + } for i := 0; i < len(d); i += mpegTsSize { pid := (uint16(d[i+1]&0x1f) << 8) | uint16(d[i+2]) if pid == pmtPid { From e50a8e6995952270a94a322217e7da3b9ad6c8d8 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 16:18:18 +1030 Subject: [PATCH 43/54] revid: zerod to zeroed in comment --- stream/mts/psi/std.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stream/mts/psi/std.go b/stream/mts/psi/std.go index 5596805d..de28d44a 100644 --- a/stream/mts/psi/std.go +++ b/stream/mts/psi/std.go @@ -76,7 +76,7 @@ var ( }, } - // Std pmt with time and location descriptors, time and location fields are zerod out + // Std pmt with time and location descriptors, time and location fields are zeroed out StdPmtTimeLocation = PSI{ Pf: 0x00, Tid: 0x02, From d961cf172d92018c344a1cfcc318410afbed0c6b Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 16:20:20 +1030 Subject: [PATCH 44/54] psi: indicated that time data size const is in bytes to be consistent with storage of uint64 --- stream/mts/psi/psi.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index cf1ea440..259fa95c 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -51,7 +51,7 @@ const ( timeDescTag = 234 timeTagIndx = 13 timeDataIndx = 15 - timeDataSize = 8 + timeDataSize = 8 // bytes, because time is stored in uint64 ) // Consts relating to location description From 404a2b6c9935ffcf09e1c301b14272946b2f42f4 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 16:21:56 +1030 Subject: [PATCH 45/54] psi: d to dst in updateTime --- stream/mts/psi/op.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/stream/mts/psi/op.go b/stream/mts/psi/op.go index 78730fc2..18fcd42c 100644 --- a/stream/mts/psi/op.go +++ b/stream/mts/psi/op.go @@ -64,16 +64,16 @@ func HasLocation(p []byte) error { // UpdateTime takes the byte slice representation of a psi-pmt as well as a time // as an integer and attempts to update the time descriptor in the pmt with the // given time if the time descriptor exists, otherwise an error is returned -func UpdateTime(d []byte, t uint64) error { - err := HasTime(d) +func UpdateTime(dst []byte, t uint64) error { + err := HasTime(dst) if err != nil { return err } ts := TimeBytes(uint64(t)) - for i := range d[timeDataIndx : timeDataIndx+timeDataSize] { - d[i+timeDataIndx] = ts[i] + for i := range dst[timeDataIndx : timeDataIndx+timeDataSize] { + dst[i+timeDataIndx] = ts[i] } - d = updateCrc(d) + dst = updateCrc(dst) return nil } From f7991cd0afc01e509d8c685086736e38bb4f7417 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 16:22:48 +1030 Subject: [PATCH 46/54] psi: using array in TimeBytes() --- stream/mts/psi/op.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stream/mts/psi/op.go b/stream/mts/psi/op.go index 18fcd42c..cebea13a 100644 --- a/stream/mts/psi/op.go +++ b/stream/mts/psi/op.go @@ -37,10 +37,10 @@ import ( // timeBytes takes a timestamp as an uint64 and converts to an 8 byte slice - // allows for updating of timestamp in pmt time descriptor. -func TimeBytes(t uint64) (s []byte) { - s = make([]byte, timeDataSize) +func TimeBytes(t uint64) []byte { + var s [timeDataSize]byte binary.BigEndian.PutUint64(s[:], t) - return s + return s[:] } // ChkTime takes a psi as a byte slice and checks to see if it has a time descriptor From 7c24c7928728fa0ef03a18c6ed3ebbc8992dc24c Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 16:31:13 +1030 Subject: [PATCH 47/54] psi: using binary.BigEndian.Uint64 for TimeFrom --- stream/mts/psi/op.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/stream/mts/psi/op.go b/stream/mts/psi/op.go index cebea13a..55f9df8d 100644 --- a/stream/mts/psi/op.go +++ b/stream/mts/psi/op.go @@ -85,11 +85,7 @@ func TimeFrom(p []byte) (t uint64, err error) { if err != nil { return 0, err } - // TODO: could probably write a byte slice to int func, so that this would - // be replaced by extraction of time data slice and then call to conversion func - for i := range p[timeDataIndx : timeDataIndx+timeDataSize] { - t |= uint64(p[i+timeDataIndx]) << uint(56-i*timeDataSize) - } + t = binary.BigEndian.Uint64(p[timeDataIndx : timeDataIndx+timeDataSize]) return t, nil } From 239b52ad84c02379347e609baa341e1ca1a42678 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 16:37:13 +1030 Subject: [PATCH 48/54] psi: simplified LocationStrBytes --- stream/mts/psi/op.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/stream/mts/psi/op.go b/stream/mts/psi/op.go index 55f9df8d..331019b7 100644 --- a/stream/mts/psi/op.go +++ b/stream/mts/psi/op.go @@ -105,10 +105,13 @@ func LocationFrom(p []byte) (g string, err error) { // LocationStrBytes take a string of location data and converts to a 32 byte slice - // easy update of slice representation of a pmt with location descriptor -func LocationStrBytes(l string) (out []byte) { - out = make([]byte, locationDataSize) - copy(out, []byte(l)) - return +func LocationStrBytes(s string) []byte { + if len(s) != locationDataSize { + panic("Location string not the right size") + } + var b [locationDataSize]byte + copy(b[:], s) + return b[:] } // UpdateLocation takes a byte slice representation of a psi-pmt containing a location From fbdce669e77d88355d5be946a9dafb6469ee6869 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 16:46:36 +1030 Subject: [PATCH 49/54] revid: cleaned and added commenting --- revid/senders.go | 2 ++ stream/mts/encoder.go | 4 ++++ stream/mts/psi/op.go | 12 +++++++----- stream/mts/psi/psi.go | 3 +++ stream/mts/psi/psi_test.go | 2 ++ 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/revid/senders.go b/revid/senders.go index 549ce30d..3db3a982 100644 --- a/revid/senders.go +++ b/revid/senders.go @@ -156,6 +156,8 @@ func (s *httpSender) send() error { return s.extractMeta(reply) } +// extractMeta looks at a reply at extracts any time or location data - then used +// to update time and location information in the mpegts encoder. func (s *httpSender) extractMeta(r string) error { dec, err := netsender.NewJsonDecoder(r) if err != nil { diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 719ccd86..3236c95b 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -169,6 +169,8 @@ func (e *Encoder) Encode(nalu []byte) error { return nil } +// writePSI creates mpegts with pat and pmt tables - with pmt table having updated +// location and time data. func (e *Encoder) writePSI() error { // Write PAT patPkt := Packet{ @@ -209,6 +211,8 @@ func (e *Encoder) writePSI() error { return nil } +// addPadding adds an appropriate amount of padding to a pat or pmt table for +// addition to an mpegts packet func addPadding(d []byte) []byte { for len(d) < psiPacketSize { d = append(d, 0xff) diff --git a/stream/mts/psi/op.go b/stream/mts/psi/op.go index 331019b7..e08d64d5 100644 --- a/stream/mts/psi/op.go +++ b/stream/mts/psi/op.go @@ -35,7 +35,7 @@ import ( "math/bits" ) -// timeBytes takes a timestamp as an uint64 and converts to an 8 byte slice - +// TimeBytes takes a timestamp as an uint64 and converts to an 8 byte slice - // allows for updating of timestamp in pmt time descriptor. func TimeBytes(t uint64) []byte { var s [timeDataSize]byte @@ -43,7 +43,7 @@ func TimeBytes(t uint64) []byte { return s[:] } -// ChkTime takes a psi as a byte slice and checks to see if it has a time descriptor +// HasTime takes a psi as a byte slice and checks to see if it has a time descriptor // - if so return nil, otherwise return error func HasTime(p []byte) error { if p[timeTagIndx] != timeDescTag { @@ -52,7 +52,7 @@ func HasTime(p []byte) error { return nil } -// ChkLocation takes a psi as a byte slice and checks to see if it has a location descriptor +// HasLocation takes a psi as a byte slice and checks to see if it has a location descriptor // - if so return nil, otherwise return error func HasLocation(p []byte) error { if p[locationTagIndx] != locationDescTag { @@ -77,7 +77,7 @@ func UpdateTime(dst []byte, t uint64) error { return nil } -// TimeOf takes a byte slice representation of a psi-pmt and extracts it's +// TimeFrom takes a byte slice representation of a psi-pmt and extracts it's // timestamp, returning as a uint64 if it exists, otherwise returning 0 and nil // if it does not exist func TimeFrom(p []byte) (t uint64, err error) { @@ -89,7 +89,7 @@ func TimeFrom(p []byte) (t uint64, err error) { return t, nil } -// TimeOf takes a byte slice representation of a psi-pmt and extracts it's +// LocationFrom takes a byte slice representation of a psi-pmt and extracts it's // timestamp, returning as a uint64 if it exists, otherwise returning 0 and nil // if it does not exist func LocationFrom(p []byte) (g string, err error) { @@ -130,12 +130,14 @@ func UpdateLocation(d []byte, s string) error { return nil } +// addCrc appends a crc table to a given psi table in bytes func addCrc(out []byte) []byte { out = append(out, make([]byte, 4)...) out = updateCrc(out) return out } +// updateCrc updates the crc of psi bytes slice that may have been modified func updateCrc(out []byte) []byte { crc32 := crc32_Update(0xffffffff, crc32_MakeTable(bits.Reverse32(crc32.IEEE)), out[1:len(out)-4]) out[len(out)-4] = byte(crc32 >> 24) diff --git a/stream/mts/psi/psi.go b/stream/mts/psi/psi.go index 259fa95c..56abc260 100644 --- a/stream/mts/psi/psi.go +++ b/stream/mts/psi/psi.go @@ -160,6 +160,7 @@ func readTSS(data []byte, p *PSI) *TSS { return &tss } +// readPAT creates a pat struct based on a bytes slice representing a pat func readPAT(data []byte, p *TSS) *PAT { pat := PAT{} pos := 0 @@ -169,6 +170,7 @@ func readPAT(data []byte, p *TSS) *PAT { return &pat } +// readPMT creates a pmt struct based on a bytes slice that represents a pmt func readPMT(data []byte, p *TSS) *PAT { pmt := PMT{} pos := 0 @@ -205,6 +207,7 @@ func readDescs(data []byte, descLen int) (o []Desc) { return } +// readEESD creates an ESSD struct based on a bytes slice that represents ESSD func readEssd(data []byte) *ESSD { essd := ESSD{} pos := 0 diff --git a/stream/mts/psi/psi_test.go b/stream/mts/psi/psi_test.go index eccf057a..86c19f60 100644 --- a/stream/mts/psi/psi_test.go +++ b/stream/mts/psi/psi_test.go @@ -38,6 +38,7 @@ const ( ) // GPS string for testing +// TODO: make these realistic const ( locationTstStr1 = "$GPGGA,123519,4807.038,N,01131.0" locationTstStr2 = "$GPGGA,183710,4902.048,N,02171.0" @@ -240,6 +241,7 @@ func TestTimeGet(t *testing.T) { } } +// TestLocationGet checks that we can correctly get location data from a pmt table func TestLocationGet(t *testing.T) { pb := StdPmtTimeLocation.Bytes() err := UpdateLocation(pb, locationTstStr1) From c7f55c3c4354b374fb771e99c43ffe401e4b98fb Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 16:58:02 +1030 Subject: [PATCH 50/54] revid: using NewJSONDecoder rather than newJsonDecoder to be consistent with changes in iot --- revid/senders.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/revid/senders.go b/revid/senders.go index 3db3a982..9309e940 100644 --- a/revid/senders.go +++ b/revid/senders.go @@ -159,7 +159,7 @@ func (s *httpSender) send() error { // extractMeta looks at a reply at extracts any time or location data - then used // to update time and location information in the mpegts encoder. func (s *httpSender) extractMeta(r string) error { - dec, err := netsender.NewJsonDecoder(r) + dec, err := netsender.NewJSONDecoder(r) if err != nil { return nil } From d8587bdbe29c6e972ae41586e650278b7e2aa3ee Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 18:46:19 +1030 Subject: [PATCH 51/54] psi: don't panic when LocationStrBytes get's non 32 byte str --- stream/mts/psi/op.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/stream/mts/psi/op.go b/stream/mts/psi/op.go index e08d64d5..e78cdac7 100644 --- a/stream/mts/psi/op.go +++ b/stream/mts/psi/op.go @@ -106,9 +106,6 @@ func LocationFrom(p []byte) (g string, err error) { // LocationStrBytes take a string of location data and converts to a 32 byte slice - // easy update of slice representation of a pmt with location descriptor func LocationStrBytes(s string) []byte { - if len(s) != locationDataSize { - panic("Location string not the right size") - } var b [locationDataSize]byte copy(b[:], s) return b[:] From 618fadd90bb6acfdfb1110570c52d7dacb28246e Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 20:05:10 +1030 Subject: [PATCH 52/54] revid: trying to fix jumpy rtp --- revid/config.go | 3 +-- stream/mts/encoder.go | 2 +- stream/rtp/encoder.go | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/revid/config.go b/revid/config.go index 72165f60..b6402860 100644 --- a/revid/config.go +++ b/revid/config.go @@ -188,7 +188,6 @@ func (c *Config) Validate(r *Revid) error { switch c.Output1 { case File: - case Rtp: case Udp: case Rtmp, FfmpegRtmp: if c.RtmpUrl == "" { @@ -204,7 +203,7 @@ func (c *Config) Validate(r *Revid) error { defaultOutput) c.Output1 = defaultOutput fallthrough - case Http: + case Http, Rtp: c.Logger.Log(smartlogger.Info, pkg+"defaulting frames per clip for http out", "framesPerClip", httpFramesPerClip) c.FramesPerClip = httpFramesPerClip diff --git a/stream/mts/encoder.go b/stream/mts/encoder.go index 3236c95b..31fe4923 100644 --- a/stream/mts/encoder.go +++ b/stream/mts/encoder.go @@ -38,7 +38,7 @@ import ( const ( psiPacketSize = 184 - psiSendCount = 21 + psiSendCount = 7 ) type MetaData struct { diff --git a/stream/rtp/encoder.go b/stream/rtp/encoder.go index c8750e95..e20be442 100644 --- a/stream/rtp/encoder.go +++ b/stream/rtp/encoder.go @@ -63,7 +63,7 @@ func NewEncoder(dst io.Writer, fps int) *Encoder { ssrc: rand.Uint32(), frameInterval: time.Duration(float64(time.Second) / float64(fps)), fps: fps, - buffer: make([]byte, 0, sendLength), + buffer: make([]byte, 0), } } @@ -72,8 +72,8 @@ func NewEncoder(dst io.Writer, fps int) *Encoder { func (e *Encoder) Write(data []byte) (int, error) { e.buffer = append(e.buffer, data...) for len(e.buffer) >= sendLength { - e.Encode(e.buffer) - e.buffer = e.buffer[:0] + e.Encode(e.buffer[:sendLength]) + e.buffer = e.buffer[sendLength:] } return len(data), nil } From ab643f2b974af71d8299eb70cdb54675170b0dbc Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 20:38:53 +1030 Subject: [PATCH 53/54] revid: set default bitrate to 400000 --- revid/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/revid/config.go b/revid/config.go index b6402860..5b9b8ac3 100644 --- a/revid/config.go +++ b/revid/config.go @@ -103,7 +103,7 @@ const ( defaultIntraRefreshPeriod = "100" defaultTimeout = "0" defaultQuantization = "40" - defaultBitrate = "500000" + defaultBitrate = "400000" defaultQuantizationMode = QuantizationOff defaultFramesPerClip = 1 defaultVerticalFlip = No From 1095f32ae412a7c90f6ba31bba347eb706efd210 Mon Sep 17 00:00:00 2001 From: saxon Date: Fri, 14 Dec 2018 20:50:26 +1030 Subject: [PATCH 54/54] revid: added send retry flag --- revid/config.go | 1 + revid/revid.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/revid/config.go b/revid/config.go index 5b9b8ac3..06a4b8da 100644 --- a/revid/config.go +++ b/revid/config.go @@ -65,6 +65,7 @@ type Config struct { IntraRefreshPeriod string RtpAddress string Logger Logger + SendRetry bool } // Enums for config struct diff --git a/revid/revid.go b/revid/revid.go index f7dbc9ad..93bcdf38 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -379,6 +379,8 @@ loop: err = dest.send() if err == nil { r.config.Logger.Log(smartlogger.Debug, pkg+"sent clip to output "+strconv.Itoa(i)) + } else if r.config.SendRetry == false { + r.config.Logger.Log(smartlogger.Warning, pkg+"send to output "+strconv.Itoa(i)+"failed", "error", err.Error()) } else { r.config.Logger.Log(smartlogger.Error, pkg+"send to output "+strconv.Itoa(i)+ "failed, trying again", "error", err.Error())