From c1930baf400461d4787fbb8f9faeff5f9d19ea3f Mon Sep 17 00:00:00 2001 From: Trek H Date: Thu, 12 Dec 2019 13:56:50 +1030 Subject: [PATCH 1/7] mjpeg-player: added player code and hlsjs modifications --- cmd/mjpeg-player/favicon.ico | Bin 0 -> 15406 bytes cmd/mjpeg-player/index.html | 35 +++++++++++++ cmd/mjpeg-player/lex-mjpeg.js | 63 ++++++++++++++++++++++++ cmd/mjpeg-player/main.js | 75 ++++++++++++++++++++++++++++ cmd/mjpeg-player/player.js | 89 ++++++++++++++++++++++++++++++++++ 5 files changed, 262 insertions(+) create mode 100644 cmd/mjpeg-player/favicon.ico create mode 100644 cmd/mjpeg-player/index.html create mode 100644 cmd/mjpeg-player/lex-mjpeg.js create mode 100644 cmd/mjpeg-player/main.js create mode 100644 cmd/mjpeg-player/player.js diff --git a/cmd/mjpeg-player/favicon.ico b/cmd/mjpeg-player/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..465b02c257d3a5388ce1a7744355834fbd37fa82 GIT binary patch literal 15406 zcmeHO32>9w5f(`rkPoo2&0#KI_`sKBUAAobKJh78zp*UIwk+SqMihrYN+u3zVIXPK z5E?oyB?U5cI-zMMkf8xmAS6Ii8X%;!C8VSrA($)J@olu-C+n7sEL)b%6wU1Xzx4m_ z?f3TW+ugVOM59fiJxTNMpyAbzR{s=@#--6{ettvWLp^D<9rzoAp?w}qqn+@f(L(VZ zY=SL@e#gtOA4yJVQi)*B-V(mo1xe!U%MT+eZss{j(!3p&`H_SLtuiM(U&L|iDTpZWS99@AZ$On5m%1_JWHS zV&Kkg4ieXbZnv{x*RO)GkExn|gltzRV!42b<9?a&-n=#t)aobz7@n?Q4MS{nc)npC zoIA|`Wv-XL{maiSerQ|L@BQy1;BTKq!r)*u3=Bj;b)J3gjj>Hc`_7I?lra+?JczQu zzj*mP%psSJZC3!{=HLO9f;{h=Oen`nk#6FGWBbR5os%i?;6?x2c zva|)iSLQD{Sd+ix=vb&c)BoLytdMOD(u}6njlwV@7b{q%D5*GZS$e=BA;+yt7(=&P z?lxqsM%yqTj`MC;(K#D^>GgFq@1cGSlxKzQ(_vVp z6fIgNnENI6HSWCicoBm%J!Eroa`IDE1&p>L7JX3v%uatX+IHetuiLE+#Zi(J|Ffg* zV=V1Z9|r465?f0Iz5`?FXDn^yX$#-f`7T{QLKYqEsVgjtcSCZwgWNRjh`-hKW$Rjz z8+OpZe=1OM9LE?oMz04dQ95@0(X?0h2}ZS@SqFyxaTh5_T6qsGs-ycCIk zf@#y|82e1)d*bW+`l8|2n?tP1fcTN(*qQogC2ymdGThEvBwK^$wC z2O^J+tE1rTX)ZjY^s~+%{oL^n6K%#DIzA7(o;w`?C2@41V%^|{mc?-9GZysrFoF2n zwRMZeo&WhWLGY7$e;f}Rx_)5kWlG`zXYlXrWI|^LThCV~D`)Z7@iV_Z=I@;OHNO@U% zkH~xXwg}t@80Z?mA|eze_m?`|~w^#B(t zvwieDjQy+m{gD4N_0wXWD@ar3f-K$NZjz?X0Wr^IM7#+WNy>sFx;%k7HfMZS zNeyUs1}{qo<%MYjiRYnD#0Dr6!C_s;R13;Kg) zf;pF!MXaK>Hrf+9{^UwgOP+3&q~=#-1s^C8%)jqQS>U)Hpsro5E@Ez3(NGYA?4}s= z{(rtFc@j*~(khWgnkT_mH6Jo(t5=u$)|Frx#N%Nb0YHMqM zw6causuprSB6%(??qO|}wtuHS$9(D_=0vVGE=ycvHijpfWm(Z!SNIF&XL~hcc2cg$ zPC2MC_8Kfr@cl-S&#jYZMrm}3_?oIb#%7!&;_D_#S4KLQ8hxVTXPqT}stwn4i!75b zNm+EB=$W1J7%5MOzbC@=pva4UU8Sn@XcWh<7joQ*Uz_Z*H^>2HElUqL)l`umuE=Md zB)R=ZMfO^pYOKZGuA>sNIPYjlPy5ZowbFGCJ|@GkewYTKI1j7Xp6d4 z6&ag`@}Z`Cts$Ny-|Ggll<7z_kUShjGoUO9>uIrA6CH!h&`-A`MSpUr(ruQYYxZc`ad#Tu^*tZR&b zbAL~S&Q2B#V7_KxfZH|kU z;A!&RBRdpv@Xg#hhYUj9e!LMhV^c)!)0!a zU3cH#7Xb&~TL@)|2Hha_it60i@Y?!dICeM@Zr@_V_xBj^+J*?MJ(!Gh82n2RKhHV% z`y=4cz6BcmBrhgQ@q`U)m~i442kza)HK)y$c6LS~{_shO|M2?@p(fuOo^J|-gZntp z{#~3!j(4)+9~g*)bDu}T{=Gc-&sS{d>1Jrodzjmd$%=ny4Ktt*YlGyRiaBoGI+?8a zb@*&uALDnt53?n68UIlH=7O=A8?;D#;SW1_aPgm9 z97hb+c87GzKKxFOpSAHP*}n+cSbI$LfHgIX;hjAk`1-4OO-$6$G4l9RB@0HK&4jZPySFUFvS0wabPShN%DkM6+!Uc+dPctS?~+o$JmVjQ3i?}Up%+W z#u z+w$_v--u&7zZAsm#uC2sLN9Puhz@blUI~yHWu?d?;!q1ss1t`P4)XgRh1jitjJj+&j>k+ z|L>Ww^V)XKy;{WcxT`GStZQqlb!n@ub*T}>HsiVX8;`&okpD~c;@SKIN+CZ2i`bg; zBG1)|<~GVww&Qc3X=}Z37DeG7J0E`@OTd8=_#Y2!$T$E1 literal 0 HcmV?d00001 diff --git a/cmd/mjpeg-player/index.html b/cmd/mjpeg-player/index.html new file mode 100644 index 00000000..c2e08128 --- /dev/null +++ b/cmd/mjpeg-player/index.html @@ -0,0 +1,35 @@ + + + + + + Mjpeg Player + + + + + + +
+
+
+
+ +
+
+
+ Frame Rate: fps +
+
+ +
+
+
+
+
+ ©2019 Australian Ocean Laboratory Limited (AusOcean) (License) +
+
+ + + \ No newline at end of file diff --git a/cmd/mjpeg-player/lex-mjpeg.js b/cmd/mjpeg-player/lex-mjpeg.js new file mode 100644 index 00000000..36c128f5 --- /dev/null +++ b/cmd/mjpeg-player/lex-mjpeg.js @@ -0,0 +1,63 @@ +/* +NAME + lex-mjpeg.js + +AUTHOR + Trek Hopton + +LICENSE + This file is Copyright (C) 2019 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see [GNU licenses](http://www.gnu.org/licenses). +*/ + +// MJPEGLexer lexes a byte array containing MJPEG into individual JPEGs. +class MJPEGLexer{ + constructor(src){ + this.src = src; + this.off = 0; + } + + // read returns the next single frame. + read(){ + // Check if the src can contain at least the start and end flags (4B). + if(this.off+4 > this.src.length){ + return null; + } + // Iterate through bytes until the start flag is found. + while(this.src[this.off] != 0xff || this.src[this.off+1] != 0xd8){ + this.off++; + if(this.off+4 > this.src.length){ + return null; + } + } + // Start after the start flag and loop until the end flag is found. + let end = this.off+2; + while(true){ + if(end+2 > this.src.length){ + return null; + } + if(this.src[end] == 0xff && this.src[end+1] == 0xd9){ + break; + } + end++; + } + // Copy the frame's bytes to a new array to return. + // Note: optimally this would return a reference but since we are in a worker thread, + // the main thread doesn't have access to the ArrayBuffer that we are working with. + let frame = this.src.slice(this.off, end+2); + this.off = end+2 + return frame; + } +} \ No newline at end of file diff --git a/cmd/mjpeg-player/main.js b/cmd/mjpeg-player/main.js new file mode 100644 index 00000000..dcc68f0d --- /dev/null +++ b/cmd/mjpeg-player/main.js @@ -0,0 +1,75 @@ +/* +NAME + main.js + +AUTHOR + Trek Hopton + +LICENSE + This file is Copyright (C) 2019 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see [GNU licenses](http://www.gnu.org/licenses). +*/ + +// play will process and play the chosen target file. +function play() { + const viewer = document.getElementById('viewer'); + const input = event.target.files[0]; + const reader = new FileReader(); + + reader.onload = event => { + const player = new Worker("player.js"); + + let rate = document.getElementById('rate'); + if(rate.value && rate.value > 0){ + player.postMessage({msg:"setFrameRate", data: rate.value}); + } + + player.onmessage = e => { + switch(e.data.msg){ + case "frame": + const blob = new Blob([new Uint8Array(e.data.data)], { + type: 'video/x-motion-jpeg' + }); + const url = URL.createObjectURL(blob); + viewer.src = url; + break; + case "log": + console.log(e.data.data); + break; + case "stop": + break; + default: + console.error("unknown message from player"); + break; + } + }; + + switch(input.name.split('.')[1]){ + case "mjpeg": + case "mjpg": + player.postMessage({msg:"loadMjpeg", data:event.target.result}, [event.target.result]); + break; + case "ts": + player.postMessage({msg:"loadMtsMjpeg", data:event.target.result}, [event.target.result]); + break; + default: + console.error("unknown file format"); + break; + } + }; + reader.onerror = error => reject(error); + reader.readAsArrayBuffer(input); + +} diff --git a/cmd/mjpeg-player/player.js b/cmd/mjpeg-player/player.js new file mode 100644 index 00000000..3ad0c811 --- /dev/null +++ b/cmd/mjpeg-player/player.js @@ -0,0 +1,89 @@ +/* +NAME + player.js + +AUTHOR + Trek Hopton + +LICENSE + This file is Copyright (C) 2019 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see [GNU licenses](http://www.gnu.org/licenses). +*/ + +let frameRate = 30; + +onmessage = e => { + switch(e.data.msg){ + case "setFrameRate": + frameRate = e.data.data; + break; + case "loadMjpeg": + self.importScripts('./lex-mjpeg.js'); + let mjpeg = new Uint8Array(e.data.data); + let lex = new MJPEGLexer(mjpeg); + player = new Player(lex); + player.setFrameRate(frameRate); + player.start(); + break; + case "loadMtsMjpeg": + self.importScripts('./hlsjs/mts-demuxer.js'); + let mtsMjpeg = new Uint8Array(e.data.data); + let demux = new MTSDemuxer(); + let tracks = demux._getTracks(); + demux.append(mtsMjpeg); + buf = new FrameBuffer(tracks.video.data); + player = new Player(buf); + player.setFrameRate(frameRate); //TODO: read frame rate from metadata. + player.start(); + break; + default: + console.error("unknown message from main thread"); + break; + } +}; + +class Player{ + constructor(buffer){ + this.buffer = buffer; + this.frameRate = frameRate; + } + + setFrameRate(rate){ + this.frameRate = rate; + } + + start(){ + let frame = this.buffer.read(); + if(frame != null){ + postMessage({msg:"frame", data:frame.buffer}, [frame.buffer]); + setTimeout(() => { this.start();}, 1000/this.frameRate); + } else { + postMessage({msg:"stop"}); + } + } +} + +// FrameBuffer allows an array of subarrays (MJPEG frames) to be read one at a time. +class FrameBuffer{ + constructor(src){ + this.src = src; + this.off = 0; + } + + // read returns the next single frame. + read(){ + return this.src[this.off++]; + } +} \ No newline at end of file From a0a5496d445187d196ff7edf4e989ed18a580093 Mon Sep 17 00:00:00 2001 From: Trek H Date: Thu, 12 Dec 2019 14:00:21 +1030 Subject: [PATCH 2/7] mjpeg-player: demux mods added --- cmd/mjpeg-player/hlsjs/mts-demuxer.js | 1083 +++++-------------------- 1 file changed, 181 insertions(+), 902 deletions(-) diff --git a/cmd/mjpeg-player/hlsjs/mts-demuxer.js b/cmd/mjpeg-player/hlsjs/mts-demuxer.js index 9095935e..a1ba5fd9 100644 --- a/cmd/mjpeg-player/hlsjs/mts-demuxer.js +++ b/cmd/mjpeg-player/hlsjs/mts-demuxer.js @@ -1,74 +1,49 @@ -/** - * highly optimized TS demuxer: - * parse PAT, PMT - * extract PES packet from audio and video PIDs - * extract AVC/H264 NAL units and AAC/ADTS samples from PES packet - * trigger the remuxer upon parsing completion - * it also tries to workaround as best as it can audio codec switch (HE-AAC to AAC and vice versa), without having to restart the MediaSource. - * it also controls the remuxing process : - * upon discontinuity or level switch detection, it will also notifies the remuxer so that it can reset its state. -*/ - -import * as ADTS from './adts'; -import MpegAudio from './mpegaudio'; -import Event from '../events'; -import ExpGolomb from './exp-golomb'; -import SampleAesDecrypter from './sample-aes'; -// import Hex from '../utils/hex'; -import { logger } from '../utils/logger'; -import { ErrorTypes, ErrorDetails } from '../errors'; - -// We are using fixed track IDs for driving the MP4 remuxer -// instead of following the TS PIDs. -// There is no reason not to do this and some browsers/SourceBuffer-demuxers -// may not like if there are TrackID "switches" -// See https://github.com/video-dev/hls.js/issues/1331 -// Here we are mapping our internal track types to constant MP4 track IDs -// With MSE currently one can only have one track of each, and we are muxing -// whatever video/audio rendition in them. -const RemuxerTrackIdConfig = { - video: 1, - audio: 2, - id3: 3, - text: 4 -}; - -class TSDemuxer { - constructor (observer, remuxer, config, typeSupported) { - this.observer = observer; - this.config = config; - this.typeSupported = typeSupported; - this.remuxer = remuxer; - this.sampleAes = null; +// MTSDemuxer demultiplexes an MPEG-TS stream into its individual streams. +// While it is possible that the MPEG-TS stream may contain many streams, +// this demuxer will result in at most one stream of each type ie. video, audio, id3 metadata. +class MTSDemuxer{ + constructor(){ + this.init(); + } + + // init initialises MTSDemuxer's state. It can be used to reset an MTSDemuxer instance. + init() { + this.pmtParsed = false; + this._pmtId = -1; + this._videoTrack = MTSDemuxer.createTrack('video'); + this._audioTrack = MTSDemuxer.createTrack('audio'); + this._id3Track = MTSDemuxer.createTrack('id3'); } - setDecryptData (decryptdata) { - if ((decryptdata != null) && (decryptdata.key != null) && (decryptdata.method === 'SAMPLE-AES')) { - this.sampleAes = new SampleAesDecrypter(this.observer, this.config, decryptdata, this.discardEPB); - } else { - this.sampleAes = null; - } + // createTrack creates and returns a track model. + /** + * @param {string} type 'audio' | 'video' | 'id3' | 'text' + * @return {object} MTSDemuxer's internal track model. + */ + static createTrack (type) { + return { + type, + pid: -1, + data: [] // This will contain Uint8Arrays representing each PES packet's payload for this track. + }; } - static probe (data) { - const syncOffset = TSDemuxer._syncOffset(data); - if (syncOffset < 0) { - return false; - } else { - if (syncOffset) { - logger.warn(`MPEG2-TS detected but first sync word found @ offset ${syncOffset}, junk ahead ?`); - } - - return true; - } + // _getTracks returns this MTSDemuxer's tracks. + _getTracks(){ + return { + video: this._videoTrack, + audio: this._audioTrack, + id3: this._id3Track + }; } + // _syncOffset scans the first 1000 bytes and returns an offset to the beginning of the first three MTS packets, + // or -1 if three are not found. + // A TS fragment should contain at least 3 TS packets, a PAT, a PMT, and one PID, each starting with 0x47. static _syncOffset (data) { - // scan 1000 first bytes const scanwindow = Math.min(1000, data.length - 3 * 188); let i = 0; while (i < scanwindow) { - // a TS fragment should contain at least 3 TS packets, a PAT, a PMT, and one PID, each starting with 0x47 if (data[i] === 0x47 && data[i + 188] === 0x47 && data[i + 2 * 188] === 0x47) { return i; } else { @@ -78,102 +53,45 @@ class TSDemuxer { return -1; } - /** - * Creates a track model internal to demuxer used to drive remuxing input - * - * @param {string} type 'audio' | 'video' | 'id3' | 'text' - * @param {number} duration - * @return {object} TSDemuxer's internal track model - */ - static createTrack (type, duration) { - return { - container: type === 'video' || type === 'audio' ? 'video/mp2t' : undefined, - type, - id: RemuxerTrackIdConfig[type], - pid: -1, - inputTimeScale: 90000, - sequenceNumber: 0, - samples: [], - dropped: type === 'video' ? 0 : undefined, - isAAC: type === 'audio' ? true : undefined, - duration: type === 'audio' ? duration : undefined - }; - } + append(data) { + let videoTrack = this._videoTrack; + let videoData = videoTrack.pesData; + let videoId = videoTrack.pid; - /** - * Initializes a new init segment on the demuxer/remuxer interface. Needed for discontinuities/track-switches (or at stream start) - * Resets all internal track instances of the demuxer. - * - * @override Implements generic demuxing/remuxing interface (see DemuxerInline) - * @param {object} initSegment - * @param {string} audioCodec - * @param {string} videoCodec - * @param {number} duration (in TS timescale = 90kHz) - */ - resetInitSegment (initSegment, audioCodec, videoCodec, duration) { - this.pmtParsed = false; - this._pmtId = -1; + let audioTrack = this._audioTrack; + let audioData = audioTrack.pesData; + let audioId = audioTrack.pid; - this._avcTrack = TSDemuxer.createTrack('video', duration); - this._audioTrack = TSDemuxer.createTrack('audio', duration); - this._id3Track = TSDemuxer.createTrack('id3', duration); - this._txtTrack = TSDemuxer.createTrack('text', duration); + let id3Track = this._id3Track; + let id3Data = id3Track.pesData; + let id3Id = id3Track.pid; - // flush any partial content - this.aacOverFlow = null; - this.aacLastPTS = null; - this.avcSample = null; - this.audioCodec = audioCodec; - this.videoCodec = videoCodec; - this._duration = duration; - } + let pmtId = this._pmtId; + let pmtParsed = this.pmtParsed; + let parsePAT = this._parsePAT; + let parsePMT = this._parsePMT; + let parsePES = this._parsePES; - /** - * - * @override - */ - resetTimeStamp () {} - - // feed incoming data to the front of the parsing pipeline - append (data, timeOffset, contiguous, accurateTimeOffset) { - let start, len = data.length, stt, pid, atf, offset, pes, - unknownPIDs = false; - this.contiguous = contiguous; - let pmtParsed = this.pmtParsed, - avcTrack = this._avcTrack, - audioTrack = this._audioTrack, - id3Track = this._id3Track, - avcId = avcTrack.pid, - audioId = audioTrack.pid, - id3Id = id3Track.pid, - pmtId = this._pmtId, - avcData = avcTrack.pesData, - audioData = audioTrack.pesData, - id3Data = id3Track.pesData, - parsePAT = this._parsePAT, - parsePMT = this._parsePMT, - parsePES = this._parsePES, - parseAVCPES = this._parseAVCPES.bind(this), - parseAACPES = this._parseAACPES.bind(this), - parseMPEGPES = this._parseMPEGPES.bind(this), - parseID3PES = this._parseID3PES.bind(this); - - const syncOffset = TSDemuxer._syncOffset(data); - - // don't parse last TS packet if incomplete + let len = data.length; + let unknownPIDs = false; + + const syncOffset = MTSDemuxer._syncOffset(data); + + // Don't parse last TS packet if incomplete. len -= (len + syncOffset) % 188; - - // loop through TS packets + + // Loop through TS packets. + let start, offset, pusi, pid, afc, pes; for (start = syncOffset; start < len; start += 188) { if (data[start] === 0x47) { - stt = !!(data[start + 1] & 0x40); - // pid is a 13-bit field starting at the last bit of TS[1] + pusi = !!(data[start + 1] & 0x40); + // pid is a 13-bit field starting at the last bit of TS[1]. pid = ((data[start + 1] & 0x1f) << 8) + data[start + 2]; - atf = (data[start + 3] & 0x30) >> 4; - // if an adaption field is present, its length is specified by the fifth byte of the TS packet header. - if (atf > 1) { + afc = (data[start + 3] & 0x30) >> 4; + // If an adaption field is present, its length is specified by the fifth byte of the TS packet header. + if (afc > 1) { offset = start + 5 + data[start + 4]; - // continue if there is only adaptation field + // Continue if there is only adaptation field. if (offset === (start + 188)) { continue; } @@ -181,27 +99,61 @@ class TSDemuxer { offset = start + 4; } switch (pid) { - case avcId: - if (stt) { - if (avcData && (pes = parsePES(avcData)) && pes.pts !== undefined) { - parseAVCPES(pes, false); - } - - avcData = { data: [], size: 0 }; + case 0: + if (pusi) { + offset += data[offset] + 1; } - if (avcData) { - avcData.data.push(data.subarray(offset, start + 188)); - avcData.size += start + 188 - offset; + + pmtId = this._pmtId = parsePAT(data, offset); + break; + case pmtId: + if (pusi) { + offset += data[offset] + 1; + } + + let parsedPIDs = parsePMT(data, offset); + + // Only update track id if track PID found while parsing PMT. + // This is to avoid resetting the PID to -1 in case track PID transiently disappears from the stream, + // this could happen in case of transient missing audio samples for example. + videoId = parsedPIDs.video; + if (videoId > 0) { + videoTrack.pid = videoId; + } + audioId = parsedPIDs.audio; + if (audioId > 0) { + audioTrack.pid = audioId; + } + id3Id = parsedPIDs.id3; + if (id3Id > 0) { + id3Track.pid = id3Id; + } + + if (unknownPIDs && !pmtParsed) { + // Reparse from beginning. + unknownPIDs = false; + // We set it to -188, the += 188 in the for loop will reset start to 0. + start = syncOffset - 188; + } + pmtParsed = this.pmtParsed = true; + break; + case videoId: + if (pusi) { + if (videoData && (pes = parsePES(videoData)) && pes.pts !== undefined) { + videoTrack.data.push(pes.data); + // TODO: here pes contains data, pts, dts and len. Are all these needed? + } + videoData = { data: [], size: 0 }; + } + if (videoData) { + videoData.data.push(data.subarray(offset, start + 188)); + videoData.size += start + 188 - offset; } break; case audioId: - if (stt) { + if (pusi) { if (audioData && (pes = parsePES(audioData)) && pes.pts !== undefined) { - if (audioTrack.isAAC) { - parseAACPES(pes); - } else { - parseMPEGPES(pes); - } + audioTrack.data.push(pes.data); } audioData = { data: [], size: 0 }; } @@ -211,11 +163,10 @@ class TSDemuxer { } break; case id3Id: - if (stt) { + if (pusi) { if (id3Data && (pes = parsePES(id3Data)) && pes.pts !== undefined) { - parseID3PES(pes); + id3Track.data.push(pes.data); } - id3Data = { data: [], size: 0 }; } if (id3Data) { @@ -223,210 +174,88 @@ class TSDemuxer { id3Data.size += start + 188 - offset; } break; - case 0: - if (stt) { - offset += data[offset] + 1; - } - - pmtId = this._pmtId = parsePAT(data, offset); - break; - case pmtId: - if (stt) { - offset += data[offset] + 1; - } - - let parsedPIDs = parsePMT(data, offset, this.typeSupported.mpeg === true || this.typeSupported.mp3 === true, this.sampleAes != null); - - // only update track id if track PID found while parsing PMT - // this is to avoid resetting the PID to -1 in case - // track PID transiently disappears from the stream - // this could happen in case of transient missing audio samples for example - // NOTE this is only the PID of the track as found in TS, - // but we are not using this for MP4 track IDs. - avcId = parsedPIDs.avc; - if (avcId > 0) { - avcTrack.pid = avcId; - } - - audioId = parsedPIDs.audio; - if (audioId > 0) { - audioTrack.pid = audioId; - audioTrack.isAAC = parsedPIDs.isAAC; - } - id3Id = parsedPIDs.id3; - if (id3Id > 0) { - id3Track.pid = id3Id; - } - - if (unknownPIDs && !pmtParsed) { - logger.log('reparse from beginning'); - unknownPIDs = false; - // we set it to -188, the += 188 in the for loop will reset start to 0 - start = syncOffset - 188; - } - pmtParsed = this.pmtParsed = true; - break; - case 17: - case 0x1fff: - break; default: - unknownPIDs = true; + unknownPIDs = true; break; } } else { - this.observer.trigger(Event.ERROR, { type: ErrorTypes.MEDIA_ERROR, details: ErrorDetails.FRAG_PARSING_ERROR, fatal: false, reason: 'TS packet did not start with 0x47' }); + console.error('TS packet did not start with 0x47'); } } - // try to parse last PES packets - if (avcData && (pes = parsePES(avcData)) && pes.pts !== undefined) { - parseAVCPES(pes, true); - avcTrack.pesData = null; + + // Try to parse last PES packets. + if (videoData && (pes = parsePES(videoData)) && pes.pts !== undefined) { + videoTrack.data.push(pes.data); + videoTrack.pesData = null; } else { - // either avcData null or PES truncated, keep it for next frag parsing - avcTrack.pesData = avcData; + // Either pesPkts null or PES truncated, keep it for next frag parsing. + videoTrack.pesData = videoData; } - + if (audioData && (pes = parsePES(audioData)) && pes.pts !== undefined) { - if (audioTrack.isAAC) { - parseAACPES(pes); - } else { - parseMPEGPES(pes); - } - + audioTrack.data.push(pes.data); audioTrack.pesData = null; } else { - if (audioData && audioData.size) { - logger.log('last AAC PES packet truncated,might overlap between fragments'); - } - - // either audioData null or PES truncated, keep it for next frag parsing + // Either pesPkts null or PES truncated, keep it for next frag parsing. audioTrack.pesData = audioData; } - + if (id3Data && (pes = parsePES(id3Data)) && pes.pts !== undefined) { - parseID3PES(pes); + id3Track.data.push(pes.data); id3Track.pesData = null; } else { - // either id3Data null or PES truncated, keep it for next frag parsing + // Either pesPkts null or PES truncated, keep it for next frag parsing. id3Track.pesData = id3Data; } - - if (this.sampleAes == null) { - this.remuxer.remux(audioTrack, avcTrack, id3Track, this._txtTrack, timeOffset, contiguous, accurateTimeOffset); - } else { - this.decryptAndRemux(audioTrack, avcTrack, id3Track, this._txtTrack, timeOffset, contiguous, accurateTimeOffset); - } - } - - decryptAndRemux (audioTrack, videoTrack, id3Track, textTrack, timeOffset, contiguous, accurateTimeOffset) { - if (audioTrack.samples && audioTrack.isAAC) { - let localthis = this; - this.sampleAes.decryptAacSamples(audioTrack.samples, 0, function () { - localthis.decryptAndRemuxAvc(audioTrack, videoTrack, id3Track, textTrack, timeOffset, contiguous, accurateTimeOffset); - }); - } else { - this.decryptAndRemuxAvc(audioTrack, videoTrack, id3Track, textTrack, timeOffset, contiguous, accurateTimeOffset); - } - } - - decryptAndRemuxAvc (audioTrack, videoTrack, id3Track, textTrack, timeOffset, contiguous, accurateTimeOffset) { - if (videoTrack.samples) { - let localthis = this; - this.sampleAes.decryptAvcSamples(videoTrack.samples, 0, 0, function () { - localthis.remuxer.remux(audioTrack, videoTrack, id3Track, textTrack, timeOffset, contiguous, accurateTimeOffset); - }); - } else { - this.remuxer.remux(audioTrack, videoTrack, id3Track, textTrack, timeOffset, contiguous, accurateTimeOffset); - } - } - - destroy () { - this._initPTS = this._initDTS = undefined; - this._duration = 0; } _parsePAT (data, offset) { - // skip the PSI header and parse the first PMT entry + // Skip the PSI header and parse the first PMT entry. return (data[offset + 10] & 0x1F) << 8 | data[offset + 11]; - // logger.log('PMT PID:' + this._pmtId); + // console.log('PMT PID:' + this._pmtId); } - _parsePMT (data, offset, mpegSupported, isSampleAes) { - let sectionLength, tableEnd, programInfoLength, pid, result = { audio: -1, avc: -1, id3: -1, isAAC: true }; - sectionLength = (data[offset + 1] & 0x0f) << 8 | data[offset + 2]; - tableEnd = offset + 3 + sectionLength - 4; - // to determine where the table is, we have to figure out how - // long the program info descriptors are - programInfoLength = (data[offset + 10] & 0x0f) << 8 | data[offset + 11]; - // advance the offset to the first entry in the mapping table + _parsePMT (data, offset) { + let pid; + let result = { audio: -1, video: -1, id3: -1}; + let sectionLength = (data[offset + 1] & 0x0f) << 8 | data[offset + 2]; + let tableEnd = offset + 3 + sectionLength - 4; + // To determine where the table is, we have to figure out how long the program info descriptors are. + let programInfoLength = (data[offset + 10] & 0x0f) << 8 | data[offset + 11]; + // Advance the offset to the first entry in the mapping table. offset += 12 + programInfoLength; while (offset < tableEnd) { pid = (data[offset + 1] & 0x1F) << 8 | data[offset + 2]; switch (data[offset]) { - case 0xcf: // SAMPLE-AES AAC - if (!isSampleAes) { - logger.log('unknown stream type:' + data[offset]); - break; + case 0x1c: // MJPEG + case 0xdb: // SAMPLE-AES AVC. + case 0x1b: // ITU-T Rec. H.264 and ISO/IEC 14496-10 (lower bit-rate video). + if (result.video === -1) { + result.video = pid; } - /* falls through */ - - // ISO/IEC 13818-7 ADTS AAC (MPEG-2 lower bit-rate audio) - case 0x0f: - // logger.log('AAC PID:' + pid); + break; + case 0xcf: // SAMPLE-AES AAC. + case 0x0f: // ISO/IEC 13818-7 ADTS AAC (MPEG-2 lower bit-rate audio). + case 0xd2: // ADPCM audio. + case 0x03: // ISO/IEC 11172-3 (MPEG-1 audio). + case 0x24: + // console.warn('HEVC stream type found, not supported for now'); + case 0x04: // or ISO/IEC 13818-3 (MPEG-2 halved sample rate audio). if (result.audio === -1) { result.audio = pid; } - break; - - // Packetized metadata (ID3) - case 0x15: - // logger.log('ID3 PID:' + pid); + case 0x15: // Packetized metadata (ID3) + // console.log('ID3 PID:' + pid); if (result.id3 === -1) { result.id3 = pid; } - break; - - case 0xdb: // SAMPLE-AES AVC - if (!isSampleAes) { - logger.log('unknown stream type:' + data[offset]); - break; - } - /* falls through */ - - // ITU-T Rec. H.264 and ISO/IEC 14496-10 (lower bit-rate video) - case 0x1b: - // logger.log('AVC PID:' + pid); - if (result.avc === -1) { - result.avc = pid; - } - - break; - - // ISO/IEC 11172-3 (MPEG-1 audio) - // or ISO/IEC 13818-3 (MPEG-2 halved sample rate audio) - case 0x03: - case 0x04: - // logger.log('MPEG PID:' + pid); - if (!mpegSupported) { - logger.log('MPEG audio found, not supported in this browser for now'); - } else if (result.audio === -1) { - result.audio = pid; - result.isAAC = false; - } - break; - - case 0x24: - logger.warn('HEVC stream type found, not supported for now'); - break; - default: - logger.log('unknown stream type:' + data[offset]); + // console.log('unknown stream type:' + data[offset]); break; } - // move to the next table entry - // skip past the elementary stream descriptors, if present + // Move to the next table entry, skip past the elementary stream descriptors, if present. offset += ((data[offset + 3] & 0x0F) << 8 | data[offset + 4]) + 5; } return result; @@ -434,14 +263,14 @@ class TSDemuxer { _parsePES (stream) { let i = 0, frag, pesFlags, pesPrefix, pesLen, pesHdrLen, pesData, pesPts, pesDts, payloadStartOffset, data = stream.data; - // safety check + // Safety check. if (!stream || stream.size === 0) { return null; } - // we might need up to 19 bytes to read PES header - // if first chunk of data is less than 19 bytes, let's merge it with following ones until we get 19 bytes - // usually only one merge is needed (and this is rare ...) + // We might need up to 19 bytes to read PES header. + // If first chunk of data is less than 19 bytes, let's merge it with following ones until we get 19 bytes. + // Usually only one merge is needed (and this is rare ...). while (data[0].length < 19 && data.length > 1) { let newData = new Uint8Array(data[0].length + data[1].length); newData.set(data[0]); @@ -449,30 +278,30 @@ class TSDemuxer { data[0] = newData; data.splice(1, 1); } - // retrieve PTS/DTS from first fragment + // Retrieve PTS/DTS from first fragment. frag = data[0]; pesPrefix = (frag[0] << 16) + (frag[1] << 8) + frag[2]; if (pesPrefix === 1) { pesLen = (frag[4] << 8) + frag[5]; - // if PES parsed length is not zero and greater than total received length, stop parsing. PES might be truncated - // minus 6 : PES header size + // If PES parsed length is not zero and greater than total received length, stop parsing. PES might be truncated. + // Minus 6 : PES header size. if (pesLen && pesLen > stream.size - 6) { return null; } pesFlags = frag[7]; if (pesFlags & 0xC0) { - /* PES header described here : http://dvd.sourceforge.net/dvdinfo/pes-hdr.html - as PTS / DTS is 33 bit we cannot use bitwise operator in JS, - as Bitwise operators treat their operands as a sequence of 32 bits */ + // PES header described here : http://dvd.sourceforge.net/dvdinfo/pes-hdr.html + // As PTS / DTS is 33 bit we cannot use bitwise operator in JS, + // as Bitwise operators treat their operands as a sequence of 32 bits. pesPts = (frag[9] & 0x0E) * 536870912 +// 1 << 29 (frag[10] & 0xFF) * 4194304 +// 1 << 22 (frag[11] & 0xFE) * 16384 +// 1 << 14 (frag[12] & 0xFF) * 128 +// 1 << 7 (frag[13] & 0xFE) / 2; - // check if greater than 2^32 -1 + // Check if greater than 2^32 -1. if (pesPts > 4294967295) { - // decrement 2^33 + // Decrement 2^33. pesPts -= 8589934592; } if (pesFlags & 0x40) { @@ -481,13 +310,13 @@ class TSDemuxer { (frag[16] & 0xFE) * 16384 +// 1 << 14 (frag[17] & 0xFF) * 128 +// 1 << 7 (frag[18] & 0xFE) / 2; - // check if greater than 2^32 -1 + // Check if greater than 2^32 -1. if (pesDts > 4294967295) { - // decrement 2^33 + // Decrement 2^33. pesDts -= 8589934592; } if (pesPts - pesDts > 60 * 90000) { - logger.warn(`${Math.round((pesPts - pesDts) / 90000)}s delta between PTS and DTS, align them`); + // console.warn(`${Math.round((pesPts - pesDts) / 90000)}s delta between PTS and DTS, align them`); pesPts = pesDts; } } else { @@ -495,22 +324,22 @@ class TSDemuxer { } } pesHdrLen = frag[8]; - // 9 bytes : 6 bytes for PES header + 3 bytes for PES extension + // 9 bytes : 6 bytes for PES header + 3 bytes for PES extension. payloadStartOffset = pesHdrLen + 9; stream.size -= payloadStartOffset; - // reassemble PES packet + // Reassemble PES packet. pesData = new Uint8Array(stream.size); for (let j = 0, dataLen = data.length; j < dataLen; j++) { frag = data[j]; let len = frag.byteLength; if (payloadStartOffset) { if (payloadStartOffset > len) { - // trim full frag if PES header bigger than frag + // Trim full frag if PES header bigger than frag. payloadStartOffset -= len; continue; } else { - // trim partial frag if PES header smaller than frag + // Trim partial frag if PES header smaller than frag. frag = frag.subarray(payloadStartOffset); len -= payloadStartOffset; payloadStartOffset = 0; @@ -520,7 +349,7 @@ class TSDemuxer { i += len; } if (pesLen) { - // payload size : remove PES header + PES extension + // Payload size : remove PES header + PES extension. pesLen -= pesHdrLen + 3; } return { data: pesData, pts: pesPts, dts: pesDts, len: pesLen }; @@ -528,554 +357,4 @@ class TSDemuxer { return null; } } - - pushAccesUnit (avcSample, avcTrack) { - if (avcSample.units.length && avcSample.frame) { - const samples = avcTrack.samples; - const nbSamples = samples.length; - // only push AVC sample if starting with a keyframe is not mandatory OR - // if keyframe already found in this fragment OR - // keyframe found in last fragment (track.sps) AND - // samples already appended (we already found a keyframe in this fragment) OR fragment is contiguous - if (!this.config.forceKeyFrameOnDiscontinuity || - avcSample.key === true || - (avcTrack.sps && (nbSamples || this.contiguous))) { - avcSample.id = nbSamples; - samples.push(avcSample); - } else { - // dropped samples, track it - avcTrack.dropped++; - } - } - if (avcSample.debug.length) { - logger.log(avcSample.pts + '/' + avcSample.dts + ':' + avcSample.debug); - } - } - - _parseAVCPES (pes, last) { - // logger.log('parse new PES'); - let track = this._avcTrack, - units = this._parseAVCNALu(pes.data), - debug = false, - expGolombDecoder, - avcSample = this.avcSample, - push, - spsfound = false, - i, - pushAccesUnit = this.pushAccesUnit.bind(this), - createAVCSample = function (key, pts, dts, debug) { - return { key: key, pts: pts, dts: dts, units: [], debug: debug }; - }; - // free pes.data to save up some memory - pes.data = null; - - // if new NAL units found and last sample still there, let's push ... - // this helps parsing streams with missing AUD (only do this if AUD never found) - if (avcSample && units.length && !track.audFound) { - pushAccesUnit(avcSample, track); - avcSample = this.avcSample = createAVCSample(false, pes.pts, pes.dts, ''); - } - - units.forEach(unit => { - switch (unit.type) { - // NDR - case 1: - push = true; - if (!avcSample) { - avcSample = this.avcSample = createAVCSample(true, pes.pts, pes.dts, ''); - } - - if (debug) { - avcSample.debug += 'NDR '; - } - - avcSample.frame = true; - let data = unit.data; - // only check slice type to detect KF in case SPS found in same packet (any keyframe is preceded by SPS ...) - if (spsfound && data.length > 4) { - // retrieve slice type by parsing beginning of NAL unit (follow H264 spec, slice_header definition) to detect keyframe embedded in NDR - let sliceType = new ExpGolomb(data).readSliceType(); - // 2 : I slice, 4 : SI slice, 7 : I slice, 9: SI slice - // SI slice : A slice that is coded using intra prediction only and using quantisation of the prediction samples. - // An SI slice can be coded such that its decoded samples can be constructed identically to an SP slice. - // I slice: A slice that is not an SI slice that is decoded using intra prediction only. - // if (sliceType === 2 || sliceType === 7) { - if (sliceType === 2 || sliceType === 4 || sliceType === 7 || sliceType === 9) { - avcSample.key = true; - } - } - break; - // IDR - case 5: - push = true; - // handle PES not starting with AUD - if (!avcSample) { - avcSample = this.avcSample = createAVCSample(true, pes.pts, pes.dts, ''); - } - - if (debug) { - avcSample.debug += 'IDR '; - } - - avcSample.key = true; - avcSample.frame = true; - break; - // SEI - case 6: - push = true; - if (debug && avcSample) { - avcSample.debug += 'SEI '; - } - - expGolombDecoder = new ExpGolomb(this.discardEPB(unit.data)); - - // skip frameType - expGolombDecoder.readUByte(); - - var payloadType = 0; - var payloadSize = 0; - var endOfCaptions = false; - var b = 0; - - while (!endOfCaptions && expGolombDecoder.bytesAvailable > 1) { - payloadType = 0; - do { - b = expGolombDecoder.readUByte(); - payloadType += b; - } while (b === 0xFF); - - // Parse payload size. - payloadSize = 0; - do { - b = expGolombDecoder.readUByte(); - payloadSize += b; - } while (b === 0xFF); - - // TODO: there can be more than one payload in an SEI packet... - // TODO: need to read type and size in a while loop to get them all - if (payloadType === 4 && expGolombDecoder.bytesAvailable !== 0) { - endOfCaptions = true; - - let countryCode = expGolombDecoder.readUByte(); - - if (countryCode === 181) { - let providerCode = expGolombDecoder.readUShort(); - - if (providerCode === 49) { - let userStructure = expGolombDecoder.readUInt(); - - if (userStructure === 0x47413934) { - let userDataType = expGolombDecoder.readUByte(); - - // Raw CEA-608 bytes wrapped in CEA-708 packet - if (userDataType === 3) { - let firstByte = expGolombDecoder.readUByte(); - let secondByte = expGolombDecoder.readUByte(); - - let totalCCs = 31 & firstByte; - let byteArray = [firstByte, secondByte]; - - for (i = 0; i < totalCCs; i++) { - // 3 bytes per CC - byteArray.push(expGolombDecoder.readUByte()); - byteArray.push(expGolombDecoder.readUByte()); - byteArray.push(expGolombDecoder.readUByte()); - } - - this._insertSampleInOrder(this._txtTrack.samples, { type: 3, pts: pes.pts, bytes: byteArray }); - } - } - } - } - } else if (payloadType === 5 && expGolombDecoder.bytesAvailable !== 0) { - endOfCaptions = true; - - if (payloadSize > 16) { - let uuidStrArray = []; - let userDataPayloadBytes = []; - - for (i = 0; i < 16; i++) { - uuidStrArray.push(expGolombDecoder.readUByte().toString(16)); - - if (i === 3 || i === 5 || i === 7 || i === 9) { - uuidStrArray.push('-'); - } - } - - for (i = 16; i < payloadSize; i++) { - userDataPayloadBytes.push(expGolombDecoder.readUByte()); - } - - this._insertSampleInOrder(this._txtTrack.samples, { - pts: pes.pts, - payloadType: payloadType, - uuid: uuidStrArray.join(''), - userData: String.fromCharCode.apply(null, userDataPayloadBytes), - userDataBytes: userDataPayloadBytes - }); - } - } else if (payloadSize < expGolombDecoder.bytesAvailable) { - for (i = 0; i < payloadSize; i++) { - expGolombDecoder.readUByte(); - } - } - } - break; - // SPS - case 7: - push = true; - spsfound = true; - if (debug && avcSample) { - avcSample.debug += 'SPS '; - } - - if (!track.sps) { - expGolombDecoder = new ExpGolomb(unit.data); - let config = expGolombDecoder.readSPS(); - track.width = config.width; - track.height = config.height; - track.pixelRatio = config.pixelRatio; - track.sps = [unit.data]; - track.duration = this._duration; - let codecarray = unit.data.subarray(1, 4); - let codecstring = 'avc1.'; - for (i = 0; i < 3; i++) { - let h = codecarray[i].toString(16); - if (h.length < 2) { - h = '0' + h; - } - - codecstring += h; - } - track.codec = codecstring; - } - break; - // PPS - case 8: - push = true; - if (debug && avcSample) { - avcSample.debug += 'PPS '; - } - - if (!track.pps) { - track.pps = [unit.data]; - } - - break; - // AUD - case 9: - push = false; - track.audFound = true; - if (avcSample) { - pushAccesUnit(avcSample, track); - } - - avcSample = this.avcSample = createAVCSample(false, pes.pts, pes.dts, debug ? 'AUD ' : ''); - break; - // Filler Data - case 12: - push = false; - break; - default: - push = false; - if (avcSample) { - avcSample.debug += 'unknown NAL ' + unit.type + ' '; - } - - break; - } - if (avcSample && push) { - let units = avcSample.units; - units.push(unit); - } - }); - // if last PES packet, push samples - if (last && avcSample) { - pushAccesUnit(avcSample, track); - this.avcSample = null; - } - } - - _insertSampleInOrder (arr, data) { - let len = arr.length; - if (len > 0) { - if (data.pts >= arr[len - 1].pts) { - arr.push(data); - } else { - for (let pos = len - 1; pos >= 0; pos--) { - if (data.pts < arr[pos].pts) { - arr.splice(pos, 0, data); - break; - } - } - } - } else { - arr.push(data); - } - } - - _getLastNalUnit () { - let avcSample = this.avcSample, lastUnit; - // try to fallback to previous sample if current one is empty - if (!avcSample || avcSample.units.length === 0) { - let track = this._avcTrack, samples = track.samples; - avcSample = samples[samples.length - 1]; - } - if (avcSample) { - let units = avcSample.units; - lastUnit = units[units.length - 1]; - } - return lastUnit; - } - - _parseAVCNALu (array) { - let i = 0, len = array.byteLength, value, overflow, track = this._avcTrack, state = track.naluState || 0, lastState = state; - let units = [], unit, unitType, lastUnitStart = -1, lastUnitType; - // logger.log('PES:' + Hex.hexDump(array)); - - if (state === -1) { - // special use case where we found 3 or 4-byte start codes exactly at the end of previous PES packet - lastUnitStart = 0; - // NALu type is value read from offset 0 - lastUnitType = array[0] & 0x1f; - state = 0; - i = 1; - } - - while (i < len) { - value = array[i++]; - // optimization. state 0 and 1 are the predominant case. let's handle them outside of the switch/case - if (!state) { - state = value ? 0 : 1; - continue; - } - if (state === 1) { - state = value ? 0 : 2; - continue; - } - // here we have state either equal to 2 or 3 - if (!value) { - state = 3; - } else if (value === 1) { - if (lastUnitStart >= 0) { - unit = { data: array.subarray(lastUnitStart, i - state - 1), type: lastUnitType }; - // logger.log('pushing NALU, type/size:' + unit.type + '/' + unit.data.byteLength); - units.push(unit); - } else { - // lastUnitStart is undefined => this is the first start code found in this PES packet - // first check if start code delimiter is overlapping between 2 PES packets, - // ie it started in last packet (lastState not zero) - // and ended at the beginning of this PES packet (i <= 4 - lastState) - let lastUnit = this._getLastNalUnit(); - if (lastUnit) { - if (lastState && (i <= 4 - lastState)) { - // start delimiter overlapping between PES packets - // strip start delimiter bytes from the end of last NAL unit - // check if lastUnit had a state different from zero - if (lastUnit.state) { - // strip last bytes - lastUnit.data = lastUnit.data.subarray(0, lastUnit.data.byteLength - lastState); - } - } - // If NAL units are not starting right at the beginning of the PES packet, push preceding data into previous NAL unit. - overflow = i - state - 1; - if (overflow > 0) { - // logger.log('first NALU found with overflow:' + overflow); - let tmp = new Uint8Array(lastUnit.data.byteLength + overflow); - tmp.set(lastUnit.data, 0); - tmp.set(array.subarray(0, overflow), lastUnit.data.byteLength); - lastUnit.data = tmp; - } - } - } - // check if we can read unit type - if (i < len) { - unitType = array[i] & 0x1f; - // logger.log('find NALU @ offset:' + i + ',type:' + unitType); - lastUnitStart = i; - lastUnitType = unitType; - state = 0; - } else { - // not enough byte to read unit type. let's read it on next PES parsing - state = -1; - } - } else { - state = 0; - } - } - if (lastUnitStart >= 0 && state >= 0) { - unit = { data: array.subarray(lastUnitStart, len), type: lastUnitType, state: state }; - units.push(unit); - // logger.log('pushing NALU, type/size/state:' + unit.type + '/' + unit.data.byteLength + '/' + state); - } - // no NALu found - if (units.length === 0) { - // append pes.data to previous NAL unit - let lastUnit = this._getLastNalUnit(); - if (lastUnit) { - let tmp = new Uint8Array(lastUnit.data.byteLength + array.byteLength); - tmp.set(lastUnit.data, 0); - tmp.set(array, lastUnit.data.byteLength); - lastUnit.data = tmp; - } - } - track.naluState = state; - return units; - } - - /** - * remove Emulation Prevention bytes from a RBSP - */ - discardEPB (data) { - let length = data.byteLength, - EPBPositions = [], - i = 1, - newLength, newData; - - // Find all `Emulation Prevention Bytes` - while (i < length - 2) { - if (data[i] === 0 && - data[i + 1] === 0 && - data[i + 2] === 0x03) { - EPBPositions.push(i + 2); - i += 2; - } else { - i++; - } - } - - // If no Emulation Prevention Bytes were found just return the original - // array - if (EPBPositions.length === 0) { - return data; - } - - // Create a new array to hold the NAL unit data - newLength = length - EPBPositions.length; - newData = new Uint8Array(newLength); - let sourceIndex = 0; - - for (i = 0; i < newLength; sourceIndex++, i++) { - if (sourceIndex === EPBPositions[0]) { - // Skip this byte - sourceIndex++; - // Remove this position index - EPBPositions.shift(); - } - newData[i] = data[sourceIndex]; - } - return newData; - } - - _parseAACPES (pes) { - let track = this._audioTrack, - data = pes.data, - pts = pes.pts, - startOffset = 0, - aacOverFlow = this.aacOverFlow, - aacLastPTS = this.aacLastPTS, - frameDuration, frameIndex, offset, stamp, len; - if (aacOverFlow) { - let tmp = new Uint8Array(aacOverFlow.byteLength + data.byteLength); - tmp.set(aacOverFlow, 0); - tmp.set(data, aacOverFlow.byteLength); - // logger.log(`AAC: append overflowing ${aacOverFlow.byteLength} bytes to beginning of new PES`); - data = tmp; - } - // look for ADTS header (0xFFFx) - for (offset = startOffset, len = data.length; offset < len - 1; offset++) { - if (ADTS.isHeader(data, offset)) { - break; - } - } - // if ADTS header does not start straight from the beginning of the PES payload, raise an error - if (offset) { - let reason, fatal; - if (offset < len - 1) { - reason = `AAC PES did not start with ADTS header,offset:${offset}`; - fatal = false; - } else { - reason = 'no ADTS header found in AAC PES'; - fatal = true; - } - logger.warn(`parsing error:${reason}`); - this.observer.trigger(Event.ERROR, { type: ErrorTypes.MEDIA_ERROR, details: ErrorDetails.FRAG_PARSING_ERROR, fatal: fatal, reason: reason }); - if (fatal) { - return; - } - } - - ADTS.initTrackConfig(track, this.observer, data, offset, this.audioCodec); - frameIndex = 0; - frameDuration = ADTS.getFrameDuration(track.samplerate); - - // if last AAC frame is overflowing, we should ensure timestamps are contiguous: - // first sample PTS should be equal to last sample PTS + frameDuration - if (aacOverFlow && aacLastPTS) { - let newPTS = aacLastPTS + frameDuration; - if (Math.abs(newPTS - pts) > 1) { - logger.log(`AAC: align PTS for overlapping frames by ${Math.round((newPTS - pts) / 90)}`); - pts = newPTS; - } - } - - // scan for aac samples - while (offset < len) { - if (ADTS.isHeader(data, offset) && (offset + 5) < len) { - let frame = ADTS.appendFrame(track, data, offset, pts, frameIndex); - if (frame) { - // logger.log(`${Math.round(frame.sample.pts)} : AAC`); - offset += frame.length; - stamp = frame.sample.pts; - frameIndex++; - } else { - // logger.log('Unable to parse AAC frame'); - break; - } - } else { - // nothing found, keep looking - offset++; - } - } - - if (offset < len) { - aacOverFlow = data.subarray(offset, len); - // logger.log(`AAC: overflow detected:${len-offset}`); - } else { - aacOverFlow = null; - } - - this.aacOverFlow = aacOverFlow; - this.aacLastPTS = stamp; - } - - _parseMPEGPES (pes) { - let data = pes.data; - let length = data.length; - let frameIndex = 0; - let offset = 0; - let pts = pes.pts; - - while (offset < length) { - if (MpegAudio.isHeader(data, offset)) { - let frame = MpegAudio.appendFrame(this._audioTrack, data, offset, pts, frameIndex); - if (frame) { - offset += frame.length; - frameIndex++; - } else { - // logger.log('Unable to parse Mpeg audio frame'); - break; - } - } else { - // nothing found, keep looking - offset++; - } - } - } - - _parseID3PES (pes) { - this._id3Track.samples.push(pes); - } -} - -export default TSDemuxer; +} \ No newline at end of file From 47d91303850f056061c3617c61719be4971c7150 Mon Sep 17 00:00:00 2001 From: Trek H Date: Thu, 12 Dec 2019 14:16:52 +1030 Subject: [PATCH 3/7] mjpeg-player: added header to mts-demuxer --- cmd/mjpeg-player/hlsjs/mts-demuxer.js | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/cmd/mjpeg-player/hlsjs/mts-demuxer.js b/cmd/mjpeg-player/hlsjs/mts-demuxer.js index a1ba5fd9..0eecfdc0 100644 --- a/cmd/mjpeg-player/hlsjs/mts-demuxer.js +++ b/cmd/mjpeg-player/hlsjs/mts-demuxer.js @@ -1,3 +1,29 @@ +/* +NAME + mts-demuxer.js + +AUTHOR + Trek Hopton + +LICENSE + This file is Copyright (C) 2019 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see [GNU licenses](http://www.gnu.org/licenses). + + For hls.js Copyright notice and license, see LICENSE file. +*/ + // MTSDemuxer demultiplexes an MPEG-TS stream into its individual streams. // While it is possible that the MPEG-TS stream may contain many streams, // this demuxer will result in at most one stream of each type ie. video, audio, id3 metadata. From 9ba9eb14af348d1d6b7be6ae5461fddfef7d56ad Mon Sep 17 00:00:00 2001 From: Trek H Date: Fri, 13 Dec 2019 13:53:52 +1030 Subject: [PATCH 4/7] mjpeg-player: cosmetic changes to minimise diff --- cmd/mjpeg-player/hlsjs/mts-demuxer.js | 92 +++++++++++++-------------- 1 file changed, 44 insertions(+), 48 deletions(-) diff --git a/cmd/mjpeg-player/hlsjs/mts-demuxer.js b/cmd/mjpeg-player/hlsjs/mts-demuxer.js index 0eecfdc0..11c1f5d6 100644 --- a/cmd/mjpeg-player/hlsjs/mts-demuxer.js +++ b/cmd/mjpeg-player/hlsjs/mts-demuxer.js @@ -36,6 +36,7 @@ class MTSDemuxer{ init() { this.pmtParsed = false; this._pmtId = -1; + this._videoTrack = MTSDemuxer.createTrack('video'); this._audioTrack = MTSDemuxer.createTrack('audio'); this._id3Track = MTSDemuxer.createTrack('id3'); @@ -80,34 +81,29 @@ class MTSDemuxer{ } append(data) { - let videoTrack = this._videoTrack; - let videoData = videoTrack.pesData; - let videoId = videoTrack.pid; + let start, len = data.length, pusi, pid, afc, offset, pes, + unknownPIDs = false; + let pmtParsed = this.pmtParsed, + videoTrack = this._videoTrack, + audioTrack = this._audioTrack, + id3Track = this._id3Track, + videoId = videoTrack.pid, + audioId = audioTrack.pid, + id3Id = id3Track.pid, + pmtId = this._pmtId, + videoData = videoTrack.pesData, + audioData = audioTrack.pesData, + id3Data = id3Track.pesData, + parsePAT = this._parsePAT, + parsePMT = this._parsePMT, + parsePES = this._parsePES; - let audioTrack = this._audioTrack; - let audioData = audioTrack.pesData; - let audioId = audioTrack.pid; - - let id3Track = this._id3Track; - let id3Data = id3Track.pesData; - let id3Id = id3Track.pid; - - let pmtId = this._pmtId; - let pmtParsed = this.pmtParsed; - let parsePAT = this._parsePAT; - let parsePMT = this._parsePMT; - let parsePES = this._parsePES; - - let len = data.length; - let unknownPIDs = false; - const syncOffset = MTSDemuxer._syncOffset(data); // Don't parse last TS packet if incomplete. len -= (len + syncOffset) % 188; // Loop through TS packets. - let start, offset, pusi, pid, afc, pes; for (start = syncOffset; start < len; start += 188) { if (data[start] === 0x47) { pusi = !!(data[start + 1] & 0x40); @@ -242,12 +238,12 @@ class MTSDemuxer{ } _parsePMT (data, offset) { - let pid; - let result = { audio: -1, video: -1, id3: -1}; - let sectionLength = (data[offset + 1] & 0x0f) << 8 | data[offset + 2]; - let tableEnd = offset + 3 + sectionLength - 4; - // To determine where the table is, we have to figure out how long the program info descriptors are. - let programInfoLength = (data[offset + 10] & 0x0f) << 8 | data[offset + 11]; + let programInfoLength, pid, result = { audio: -1, video: -1, id3: -1}, + sectionLength = (data[offset + 1] & 0x0f) << 8 | data[offset + 2], + tableEnd = offset + 3 + sectionLength - 4; + // To determine where the table is, we have to figure out how + // long the program info descriptors are. + programInfoLength = (data[offset + 10] & 0x0f) << 8 | data[offset + 11]; // Advance the offset to the first entry in the mapping table. offset += 12 + programInfoLength; while (offset < tableEnd) { @@ -259,27 +255,27 @@ class MTSDemuxer{ if (result.video === -1) { result.video = pid; } - break; - case 0xcf: // SAMPLE-AES AAC. - case 0x0f: // ISO/IEC 13818-7 ADTS AAC (MPEG-2 lower bit-rate audio). - case 0xd2: // ADPCM audio. - case 0x03: // ISO/IEC 11172-3 (MPEG-1 audio). - case 0x24: - // console.warn('HEVC stream type found, not supported for now'); - case 0x04: // or ISO/IEC 13818-3 (MPEG-2 halved sample rate audio). - if (result.audio === -1) { - result.audio = pid; - } - break; - case 0x15: // Packetized metadata (ID3) - // console.log('ID3 PID:' + pid); - if (result.id3 === -1) { - result.id3 = pid; - } - break; - default: - // console.log('unknown stream type:' + data[offset]); - break; + break; + case 0xcf: // SAMPLE-AES AAC. + case 0x0f: // ISO/IEC 13818-7 ADTS AAC (MPEG-2 lower bit-rate audio). + case 0xd2: // ADPCM audio. + case 0x03: // ISO/IEC 11172-3 (MPEG-1 audio). + case 0x24: + // console.warn('HEVC stream type found, not supported for now'); + case 0x04: // or ISO/IEC 13818-3 (MPEG-2 halved sample rate audio). + if (result.audio === -1) { + result.audio = pid; + } + break; + case 0x15: // Packetized metadata (ID3) + // console.log('ID3 PID:' + pid); + if (result.id3 === -1) { + result.id3 = pid; + } + break; + default: + // console.log('unknown stream type:' + data[offset]); + break; } // Move to the next table entry, skip past the elementary stream descriptors, if present. offset += ((data[offset + 3] & 0x0F) << 8 | data[offset + 4]) + 5; From ee4b18817538e9e259f6657c46868a69f2951a08 Mon Sep 17 00:00:00 2001 From: Trek H Date: Fri, 13 Dec 2019 14:03:39 +1030 Subject: [PATCH 5/7] mjpeg-player: revert reorder of switch cases to minimise diff --- cmd/mjpeg-player/hlsjs/mts-demuxer.js | 74 +++++++++++++-------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/cmd/mjpeg-player/hlsjs/mts-demuxer.js b/cmd/mjpeg-player/hlsjs/mts-demuxer.js index 11c1f5d6..415b057c 100644 --- a/cmd/mjpeg-player/hlsjs/mts-demuxer.js +++ b/cmd/mjpeg-player/hlsjs/mts-demuxer.js @@ -121,6 +121,43 @@ class MTSDemuxer{ offset = start + 4; } switch (pid) { + case videoId: + if (pusi) { + if (videoData && (pes = parsePES(videoData)) && pes.pts !== undefined) { + videoTrack.data.push(pes.data); + // TODO: here pes contains data, pts, dts and len. Are all these needed? + } + videoData = { data: [], size: 0 }; + } + if (videoData) { + videoData.data.push(data.subarray(offset, start + 188)); + videoData.size += start + 188 - offset; + } + break; + case audioId: + if (pusi) { + if (audioData && (pes = parsePES(audioData)) && pes.pts !== undefined) { + audioTrack.data.push(pes.data); + } + audioData = { data: [], size: 0 }; + } + if (audioData) { + audioData.data.push(data.subarray(offset, start + 188)); + audioData.size += start + 188 - offset; + } + break; + case id3Id: + if (pusi) { + if (id3Data && (pes = parsePES(id3Data)) && pes.pts !== undefined) { + id3Track.data.push(pes.data); + } + id3Data = { data: [], size: 0 }; + } + if (id3Data) { + id3Data.data.push(data.subarray(offset, start + 188)); + id3Data.size += start + 188 - offset; + } + break; case 0: if (pusi) { offset += data[offset] + 1; @@ -159,43 +196,6 @@ class MTSDemuxer{ } pmtParsed = this.pmtParsed = true; break; - case videoId: - if (pusi) { - if (videoData && (pes = parsePES(videoData)) && pes.pts !== undefined) { - videoTrack.data.push(pes.data); - // TODO: here pes contains data, pts, dts and len. Are all these needed? - } - videoData = { data: [], size: 0 }; - } - if (videoData) { - videoData.data.push(data.subarray(offset, start + 188)); - videoData.size += start + 188 - offset; - } - break; - case audioId: - if (pusi) { - if (audioData && (pes = parsePES(audioData)) && pes.pts !== undefined) { - audioTrack.data.push(pes.data); - } - audioData = { data: [], size: 0 }; - } - if (audioData) { - audioData.data.push(data.subarray(offset, start + 188)); - audioData.size += start + 188 - offset; - } - break; - case id3Id: - if (pusi) { - if (id3Data && (pes = parsePES(id3Data)) && pes.pts !== undefined) { - id3Track.data.push(pes.data); - } - id3Data = { data: [], size: 0 }; - } - if (id3Data) { - id3Data.data.push(data.subarray(offset, start + 188)); - id3Data.size += start + 188 - offset; - } - break; default: unknownPIDs = true; break; From fb11dde938f7ece76eda79643504c75241917ad5 Mon Sep 17 00:00:00 2001 From: Trek H Date: Wed, 25 Dec 2019 09:34:29 +1030 Subject: [PATCH 6/7] mjpeg-player: make maxScanWindow a const an document it. --- cmd/mjpeg-player/hlsjs/mts-demuxer.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/mjpeg-player/hlsjs/mts-demuxer.js b/cmd/mjpeg-player/hlsjs/mts-demuxer.js index 415b057c..443d3bcc 100644 --- a/cmd/mjpeg-player/hlsjs/mts-demuxer.js +++ b/cmd/mjpeg-player/hlsjs/mts-demuxer.js @@ -64,13 +64,14 @@ class MTSDemuxer{ }; } - // _syncOffset scans the first 1000 bytes and returns an offset to the beginning of the first three MTS packets, + // _syncOffset scans the first 'maxScanWindow' bytes and returns an offset to the beginning of the first three MTS packets, // or -1 if three are not found. // A TS fragment should contain at least 3 TS packets, a PAT, a PMT, and one PID, each starting with 0x47. static _syncOffset (data) { - const scanwindow = Math.min(1000, data.length - 3 * 188); + const maxScanWindow = 1000; // 1000 is a reasonable number of bytes to search for the first MTS packets. + const scanWindow = Math.min(maxScanWindow, data.length - 3 * 188); let i = 0; - while (i < scanwindow) { + while (i < scanWindow) { if (data[i] === 0x47 && data[i + 188] === 0x47 && data[i + 2 * 188] === 0x47) { return i; } else { From 372865132bd3f173839b6e1c54aa1a849ac3b417 Mon Sep 17 00:00:00 2001 From: Trek H Date: Thu, 2 Jan 2020 11:35:43 +1030 Subject: [PATCH 7/7] mjpeg-player: fomatting and removed bootstrap. --- cmd/mjpeg-player/hlsjs/mts-demuxer.js | 188 +++++++++++++------------- cmd/mjpeg-player/index.html | 1 - cmd/mjpeg-player/lex-mjpeg.js | 26 ++-- cmd/mjpeg-player/main.js | 30 ++-- cmd/mjpeg-player/player.js | 26 ++-- go.mod | 1 - go.sum | 2 - 7 files changed, 135 insertions(+), 139 deletions(-) diff --git a/cmd/mjpeg-player/hlsjs/mts-demuxer.js b/cmd/mjpeg-player/hlsjs/mts-demuxer.js index 443d3bcc..22ac59ca 100644 --- a/cmd/mjpeg-player/hlsjs/mts-demuxer.js +++ b/cmd/mjpeg-player/hlsjs/mts-demuxer.js @@ -19,7 +19,7 @@ LICENSE for more details. You should have received a copy of the GNU General Public License in gpl.txt. - If not, see [GNU licenses](http://www.gnu.org/licenses). + If not, see http://www.gnu.org/licenses. For hls.js Copyright notice and license, see LICENSE file. */ @@ -27,11 +27,11 @@ LICENSE // MTSDemuxer demultiplexes an MPEG-TS stream into its individual streams. // While it is possible that the MPEG-TS stream may contain many streams, // this demuxer will result in at most one stream of each type ie. video, audio, id3 metadata. -class MTSDemuxer{ - constructor(){ +class MTSDemuxer { + constructor() { this.init(); } - + // init initialises MTSDemuxer's state. It can be used to reset an MTSDemuxer instance. init() { this.pmtParsed = false; @@ -47,7 +47,7 @@ class MTSDemuxer{ * @param {string} type 'audio' | 'video' | 'id3' | 'text' * @return {object} MTSDemuxer's internal track model. */ - static createTrack (type) { + static createTrack(type) { return { type, pid: -1, @@ -56,10 +56,10 @@ class MTSDemuxer{ } // _getTracks returns this MTSDemuxer's tracks. - _getTracks(){ + _getTracks() { return { - video: this._videoTrack, - audio: this._audioTrack, + video: this._videoTrack, + audio: this._audioTrack, id3: this._id3Track }; } @@ -67,7 +67,7 @@ class MTSDemuxer{ // _syncOffset scans the first 'maxScanWindow' bytes and returns an offset to the beginning of the first three MTS packets, // or -1 if three are not found. // A TS fragment should contain at least 3 TS packets, a PAT, a PMT, and one PID, each starting with 0x47. - static _syncOffset (data) { + static _syncOffset(data) { const maxScanWindow = 1000; // 1000 is a reasonable number of bytes to search for the first MTS packets. const scanWindow = Math.min(maxScanWindow, data.length - 3 * 188); let i = 0; @@ -100,10 +100,10 @@ class MTSDemuxer{ parsePES = this._parsePES; const syncOffset = MTSDemuxer._syncOffset(data); - + // Don't parse last TS packet if incomplete. len -= (len + syncOffset) % 188; - + // Loop through TS packets. for (start = syncOffset; start < len; start += 188) { if (data[start] === 0x47) { @@ -122,84 +122,84 @@ class MTSDemuxer{ offset = start + 4; } switch (pid) { - case videoId: - if (pusi) { - if (videoData && (pes = parsePES(videoData)) && pes.pts !== undefined) { - videoTrack.data.push(pes.data); - // TODO: here pes contains data, pts, dts and len. Are all these needed? + case videoId: + if (pusi) { + if (videoData && (pes = parsePES(videoData)) && pes.pts !== undefined) { + videoTrack.data.push(pes.data); + // TODO: here pes contains data, pts, dts and len. Are all these needed? + } + videoData = { data: [], size: 0 }; } - videoData = { data: [], size: 0 }; - } - if (videoData) { - videoData.data.push(data.subarray(offset, start + 188)); - videoData.size += start + 188 - offset; - } - break; - case audioId: - if (pusi) { - if (audioData && (pes = parsePES(audioData)) && pes.pts !== undefined) { - audioTrack.data.push(pes.data); + if (videoData) { + videoData.data.push(data.subarray(offset, start + 188)); + videoData.size += start + 188 - offset; } - audioData = { data: [], size: 0 }; - } - if (audioData) { - audioData.data.push(data.subarray(offset, start + 188)); - audioData.size += start + 188 - offset; - } - break; - case id3Id: - if (pusi) { - if (id3Data && (pes = parsePES(id3Data)) && pes.pts !== undefined) { - id3Track.data.push(pes.data); + break; + case audioId: + if (pusi) { + if (audioData && (pes = parsePES(audioData)) && pes.pts !== undefined) { + audioTrack.data.push(pes.data); + } + audioData = { data: [], size: 0 }; + } + if (audioData) { + audioData.data.push(data.subarray(offset, start + 188)); + audioData.size += start + 188 - offset; + } + break; + case id3Id: + if (pusi) { + if (id3Data && (pes = parsePES(id3Data)) && pes.pts !== undefined) { + id3Track.data.push(pes.data); + } + id3Data = { data: [], size: 0 }; + } + if (id3Data) { + id3Data.data.push(data.subarray(offset, start + 188)); + id3Data.size += start + 188 - offset; + } + break; + case 0: + if (pusi) { + offset += data[offset] + 1; } - id3Data = { data: [], size: 0 }; - } - if (id3Data) { - id3Data.data.push(data.subarray(offset, start + 188)); - id3Data.size += start + 188 - offset; - } - break; - case 0: - if (pusi) { - offset += data[offset] + 1; - } - pmtId = this._pmtId = parsePAT(data, offset); - break; - case pmtId: - if (pusi) { - offset += data[offset] + 1; - } + pmtId = this._pmtId = parsePAT(data, offset); + break; + case pmtId: + if (pusi) { + offset += data[offset] + 1; + } - let parsedPIDs = parsePMT(data, offset); + let parsedPIDs = parsePMT(data, offset); - // Only update track id if track PID found while parsing PMT. - // This is to avoid resetting the PID to -1 in case track PID transiently disappears from the stream, - // this could happen in case of transient missing audio samples for example. - videoId = parsedPIDs.video; - if (videoId > 0) { - videoTrack.pid = videoId; - } - audioId = parsedPIDs.audio; - if (audioId > 0) { - audioTrack.pid = audioId; - } - id3Id = parsedPIDs.id3; - if (id3Id > 0) { - id3Track.pid = id3Id; - } + // Only update track id if track PID found while parsing PMT. + // This is to avoid resetting the PID to -1 in case track PID transiently disappears from the stream, + // this could happen in case of transient missing audio samples for example. + videoId = parsedPIDs.video; + if (videoId > 0) { + videoTrack.pid = videoId; + } + audioId = parsedPIDs.audio; + if (audioId > 0) { + audioTrack.pid = audioId; + } + id3Id = parsedPIDs.id3; + if (id3Id > 0) { + id3Track.pid = id3Id; + } - if (unknownPIDs && !pmtParsed) { - // Reparse from beginning. - unknownPIDs = false; - // We set it to -188, the += 188 in the for loop will reset start to 0. - start = syncOffset - 188; - } - pmtParsed = this.pmtParsed = true; - break; - default: - unknownPIDs = true; - break; + if (unknownPIDs && !pmtParsed) { + // Reparse from beginning. + unknownPIDs = false; + // We set it to -188, the += 188 in the for loop will reset start to 0. + start = syncOffset - 188; + } + pmtParsed = this.pmtParsed = true; + break; + default: + unknownPIDs = true; + break; } } else { console.error('TS packet did not start with 0x47'); @@ -214,7 +214,7 @@ class MTSDemuxer{ // Either pesPkts null or PES truncated, keep it for next frag parsing. videoTrack.pesData = videoData; } - + if (audioData && (pes = parsePES(audioData)) && pes.pts !== undefined) { audioTrack.data.push(pes.data); audioTrack.pesData = null; @@ -222,7 +222,7 @@ class MTSDemuxer{ // Either pesPkts null or PES truncated, keep it for next frag parsing. audioTrack.pesData = audioData; } - + if (id3Data && (pes = parsePES(id3Data)) && pes.pts !== undefined) { id3Track.data.push(pes.data); id3Track.pesData = null; @@ -232,16 +232,16 @@ class MTSDemuxer{ } } - _parsePAT (data, offset) { + _parsePAT(data, offset) { // Skip the PSI header and parse the first PMT entry. return (data[offset + 10] & 0x1F) << 8 | data[offset + 11]; // console.log('PMT PID:' + this._pmtId); } - _parsePMT (data, offset) { - let programInfoLength, pid, result = { audio: -1, video: -1, id3: -1}, - sectionLength = (data[offset + 1] & 0x0f) << 8 | data[offset + 2], - tableEnd = offset + 3 + sectionLength - 4; + _parsePMT(data, offset) { + let programInfoLength, pid, result = { audio: -1, video: -1, id3: -1 }, + sectionLength = (data[offset + 1] & 0x0f) << 8 | data[offset + 2], + tableEnd = offset + 3 + sectionLength - 4; // To determine where the table is, we have to figure out how // long the program info descriptors are. programInfoLength = (data[offset + 10] & 0x0f) << 8 | data[offset + 11]; @@ -253,16 +253,16 @@ class MTSDemuxer{ case 0x1c: // MJPEG case 0xdb: // SAMPLE-AES AVC. case 0x1b: // ITU-T Rec. H.264 and ISO/IEC 14496-10 (lower bit-rate video). - if (result.video === -1) { - result.video = pid; - } + if (result.video === -1) { + result.video = pid; + } break; case 0xcf: // SAMPLE-AES AAC. case 0x0f: // ISO/IEC 13818-7 ADTS AAC (MPEG-2 lower bit-rate audio). case 0xd2: // ADPCM audio. case 0x03: // ISO/IEC 11172-3 (MPEG-1 audio). case 0x24: - // console.warn('HEVC stream type found, not supported for now'); + // console.warn('HEVC stream type found, not supported for now'); case 0x04: // or ISO/IEC 13818-3 (MPEG-2 halved sample rate audio). if (result.audio === -1) { result.audio = pid; @@ -284,7 +284,7 @@ class MTSDemuxer{ return result; } - _parsePES (stream) { + _parsePES(stream) { let i = 0, frag, pesFlags, pesPrefix, pesLen, pesHdrLen, pesData, pesPts, pesDts, payloadStartOffset, data = stream.data; // Safety check. if (!stream || stream.size === 0) { diff --git a/cmd/mjpeg-player/index.html b/cmd/mjpeg-player/index.html index c2e08128..064f436c 100644 --- a/cmd/mjpeg-player/index.html +++ b/cmd/mjpeg-player/index.html @@ -6,7 +6,6 @@ Mjpeg Player - diff --git a/cmd/mjpeg-player/lex-mjpeg.js b/cmd/mjpeg-player/lex-mjpeg.js index 36c128f5..4d5be7e9 100644 --- a/cmd/mjpeg-player/lex-mjpeg.js +++ b/cmd/mjpeg-player/lex-mjpeg.js @@ -19,36 +19,36 @@ LICENSE for more details. You should have received a copy of the GNU General Public License in gpl.txt. - If not, see [GNU licenses](http://www.gnu.org/licenses). + If not, see http://www.gnu.org/licenses. */ // MJPEGLexer lexes a byte array containing MJPEG into individual JPEGs. -class MJPEGLexer{ - constructor(src){ +class MJPEGLexer { + constructor(src) { this.src = src; this.off = 0; } // read returns the next single frame. - read(){ + read() { // Check if the src can contain at least the start and end flags (4B). - if(this.off+4 > this.src.length){ + if (this.off + 4 > this.src.length) { return null; } // Iterate through bytes until the start flag is found. - while(this.src[this.off] != 0xff || this.src[this.off+1] != 0xd8){ + while (this.src[this.off] != 0xff || this.src[this.off + 1] != 0xd8) { this.off++; - if(this.off+4 > this.src.length){ + if (this.off + 4 > this.src.length) { return null; } } // Start after the start flag and loop until the end flag is found. - let end = this.off+2; - while(true){ - if(end+2 > this.src.length){ + let end = this.off + 2; + while (true) { + if (end + 2 > this.src.length) { return null; } - if(this.src[end] == 0xff && this.src[end+1] == 0xd9){ + if (this.src[end] == 0xff && this.src[end + 1] == 0xd9) { break; } end++; @@ -56,8 +56,8 @@ class MJPEGLexer{ // Copy the frame's bytes to a new array to return. // Note: optimally this would return a reference but since we are in a worker thread, // the main thread doesn't have access to the ArrayBuffer that we are working with. - let frame = this.src.slice(this.off, end+2); - this.off = end+2 + let frame = this.src.slice(this.off, end + 2); + this.off = end + 2 return frame; } } \ No newline at end of file diff --git a/cmd/mjpeg-player/main.js b/cmd/mjpeg-player/main.js index dcc68f0d..037b31ed 100644 --- a/cmd/mjpeg-player/main.js +++ b/cmd/mjpeg-player/main.js @@ -19,7 +19,7 @@ LICENSE for more details. You should have received a copy of the GNU General Public License in gpl.txt. - If not, see [GNU licenses](http://www.gnu.org/licenses). + If not, see http://www.gnu.org/licenses. */ // play will process and play the chosen target file. @@ -32,12 +32,12 @@ function play() { const player = new Worker("player.js"); let rate = document.getElementById('rate'); - if(rate.value && rate.value > 0){ - player.postMessage({msg:"setFrameRate", data: rate.value}); + if (rate.value && rate.value > 0) { + player.postMessage({ msg: "setFrameRate", data: rate.value }); } player.onmessage = e => { - switch(e.data.msg){ + switch (e.data.msg) { case "frame": const blob = new Blob([new Uint8Array(e.data.data)], { type: 'video/x-motion-jpeg' @@ -56,17 +56,17 @@ function play() { } }; - switch(input.name.split('.')[1]){ - case "mjpeg": - case "mjpg": - player.postMessage({msg:"loadMjpeg", data:event.target.result}, [event.target.result]); - break; - case "ts": - player.postMessage({msg:"loadMtsMjpeg", data:event.target.result}, [event.target.result]); - break; - default: - console.error("unknown file format"); - break; + switch (input.name.split('.')[1]) { + case "mjpeg": + case "mjpg": + player.postMessage({ msg: "loadMjpeg", data: event.target.result }, [event.target.result]); + break; + case "ts": + player.postMessage({ msg: "loadMtsMjpeg", data: event.target.result }, [event.target.result]); + break; + default: + console.error("unknown file format"); + break; } }; reader.onerror = error => reject(error); diff --git a/cmd/mjpeg-player/player.js b/cmd/mjpeg-player/player.js index 3ad0c811..34ff75ab 100644 --- a/cmd/mjpeg-player/player.js +++ b/cmd/mjpeg-player/player.js @@ -19,13 +19,13 @@ LICENSE for more details. You should have received a copy of the GNU General Public License in gpl.txt. - If not, see [GNU licenses](http://www.gnu.org/licenses). + If not, see http://www.gnu.org/licenses. */ let frameRate = 30; onmessage = e => { - switch(e.data.msg){ + switch (e.data.msg) { case "setFrameRate": frameRate = e.data.data; break; @@ -54,36 +54,36 @@ onmessage = e => { } }; -class Player{ - constructor(buffer){ +class Player { + constructor(buffer) { this.buffer = buffer; this.frameRate = frameRate; } - setFrameRate(rate){ + setFrameRate(rate) { this.frameRate = rate; } - start(){ + start() { let frame = this.buffer.read(); - if(frame != null){ - postMessage({msg:"frame", data:frame.buffer}, [frame.buffer]); - setTimeout(() => { this.start();}, 1000/this.frameRate); + if (frame != null) { + postMessage({ msg: "frame", data: frame.buffer }, [frame.buffer]); + setTimeout(() => { this.start(); }, 1000 / this.frameRate); } else { - postMessage({msg:"stop"}); + postMessage({ msg: "stop" }); } } } // FrameBuffer allows an array of subarrays (MJPEG frames) to be read one at a time. -class FrameBuffer{ - constructor(src){ +class FrameBuffer { + constructor(src) { this.src = src; this.off = 0; } // read returns the next single frame. - read(){ + read() { return this.src[this.off++]; } } \ No newline at end of file diff --git a/go.mod b/go.mod index 1ab98014..67c14597 100644 --- a/go.mod +++ b/go.mod @@ -12,5 +12,4 @@ require ( github.com/pkg/errors v0.8.1 github.com/yobert/alsa v0.0.0-20180630182551-d38d89fa843e gocv.io/x/gocv v0.21.0 - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect ) diff --git a/go.sum b/go.sum index e6eee428..26f1493b 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -bitbucket.org/ausocean/iot v1.2.8 h1:m2UhfAbG/6RKPBzY4OJ5S7zLxTmuzyMKPNbu0qaxFIw= -bitbucket.org/ausocean/iot v1.2.8/go.mod h1:wCLOYeEDCxDquneSZ/zTEcKGXZ6uan+6sgZyTMlNVDo= bitbucket.org/ausocean/iot v1.2.9 h1:3tzgiekH+Z0yXhkwnqBzxxe8qQJ2O7YTkz4s0T6stgw= bitbucket.org/ausocean/iot v1.2.9/go.mod h1:Q5FwaOKnCty3dVeVtki6DLwYa5vhNpOaeu1lwLyPCg8= bitbucket.org/ausocean/utils v1.2.11 h1:zA0FOaPjN960ryp8PKCkV5y50uWBYrIxCVnXjwbvPqg=