From 9476440e211bba4a7ff4f8999312be11eb0b11e0 Mon Sep 17 00:00:00 2001 From: Trek H Date: Thu, 12 Dec 2019 13:41:44 +1030 Subject: [PATCH] mjpeg-player: added original hls.js code for mts demuxer --- cmd/mjpeg-player/hlsjs/LICENSE | 28 + cmd/mjpeg-player/hlsjs/mts-demuxer.js | 1081 +++++++++++++++++++++++++ 2 files changed, 1109 insertions(+) create mode 100644 cmd/mjpeg-player/hlsjs/LICENSE create mode 100644 cmd/mjpeg-player/hlsjs/mts-demuxer.js diff --git a/cmd/mjpeg-player/hlsjs/LICENSE b/cmd/mjpeg-player/hlsjs/LICENSE new file mode 100644 index 00000000..8f263a03 --- /dev/null +++ b/cmd/mjpeg-player/hlsjs/LICENSE @@ -0,0 +1,28 @@ +Copyright (c) 2017 Dailymotion (http://www.dailymotion.com) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +src/remux/mp4-generator.js and src/demux/exp-golomb.js implementation in this project +are derived from the HLS library for video.js (https://github.com/videojs/videojs-contrib-hls) + +That work is also covered by the Apache 2 License, following copyright: +Copyright (c) 2013-2015 Brightcove + + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/cmd/mjpeg-player/hlsjs/mts-demuxer.js b/cmd/mjpeg-player/hlsjs/mts-demuxer.js new file mode 100644 index 00000000..9095935e --- /dev/null +++ b/cmd/mjpeg-player/hlsjs/mts-demuxer.js @@ -0,0 +1,1081 @@ +/** + * highly optimized TS demuxer: + * parse PAT, PMT + * extract PES packet from audio and video PIDs + * extract AVC/H264 NAL units and AAC/ADTS samples from PES packet + * trigger the remuxer upon parsing completion + * it also tries to workaround as best as it can audio codec switch (HE-AAC to AAC and vice versa), without having to restart the MediaSource. + * it also controls the remuxing process : + * upon discontinuity or level switch detection, it will also notifies the remuxer so that it can reset its state. +*/ + +import * as ADTS from './adts'; +import MpegAudio from './mpegaudio'; +import Event from '../events'; +import ExpGolomb from './exp-golomb'; +import SampleAesDecrypter from './sample-aes'; +// import Hex from '../utils/hex'; +import { logger } from '../utils/logger'; +import { ErrorTypes, ErrorDetails } from '../errors'; + +// We are using fixed track IDs for driving the MP4 remuxer +// instead of following the TS PIDs. +// There is no reason not to do this and some browsers/SourceBuffer-demuxers +// may not like if there are TrackID "switches" +// See https://github.com/video-dev/hls.js/issues/1331 +// Here we are mapping our internal track types to constant MP4 track IDs +// With MSE currently one can only have one track of each, and we are muxing +// whatever video/audio rendition in them. +const RemuxerTrackIdConfig = { + video: 1, + audio: 2, + id3: 3, + text: 4 +}; + +class TSDemuxer { + constructor (observer, remuxer, config, typeSupported) { + this.observer = observer; + this.config = config; + this.typeSupported = typeSupported; + this.remuxer = remuxer; + this.sampleAes = null; + } + + setDecryptData (decryptdata) { + if ((decryptdata != null) && (decryptdata.key != null) && (decryptdata.method === 'SAMPLE-AES')) { + this.sampleAes = new SampleAesDecrypter(this.observer, this.config, decryptdata, this.discardEPB); + } else { + this.sampleAes = null; + } + } + + static probe (data) { + const syncOffset = TSDemuxer._syncOffset(data); + if (syncOffset < 0) { + return false; + } else { + if (syncOffset) { + logger.warn(`MPEG2-TS detected but first sync word found @ offset ${syncOffset}, junk ahead ?`); + } + + return true; + } + } + + static _syncOffset (data) { + // scan 1000 first bytes + const scanwindow = Math.min(1000, data.length - 3 * 188); + let i = 0; + while (i < scanwindow) { + // a TS fragment should contain at least 3 TS packets, a PAT, a PMT, and one PID, each starting with 0x47 + if (data[i] === 0x47 && data[i + 188] === 0x47 && data[i + 2 * 188] === 0x47) { + return i; + } else { + i++; + } + } + return -1; + } + + /** + * Creates a track model internal to demuxer used to drive remuxing input + * + * @param {string} type 'audio' | 'video' | 'id3' | 'text' + * @param {number} duration + * @return {object} TSDemuxer's internal track model + */ + static createTrack (type, duration) { + return { + container: type === 'video' || type === 'audio' ? 'video/mp2t' : undefined, + type, + id: RemuxerTrackIdConfig[type], + pid: -1, + inputTimeScale: 90000, + sequenceNumber: 0, + samples: [], + dropped: type === 'video' ? 0 : undefined, + isAAC: type === 'audio' ? true : undefined, + duration: type === 'audio' ? duration : undefined + }; + } + + /** + * Initializes a new init segment on the demuxer/remuxer interface. Needed for discontinuities/track-switches (or at stream start) + * Resets all internal track instances of the demuxer. + * + * @override Implements generic demuxing/remuxing interface (see DemuxerInline) + * @param {object} initSegment + * @param {string} audioCodec + * @param {string} videoCodec + * @param {number} duration (in TS timescale = 90kHz) + */ + resetInitSegment (initSegment, audioCodec, videoCodec, duration) { + this.pmtParsed = false; + this._pmtId = -1; + + this._avcTrack = TSDemuxer.createTrack('video', duration); + this._audioTrack = TSDemuxer.createTrack('audio', duration); + this._id3Track = TSDemuxer.createTrack('id3', duration); + this._txtTrack = TSDemuxer.createTrack('text', duration); + + // flush any partial content + this.aacOverFlow = null; + this.aacLastPTS = null; + this.avcSample = null; + this.audioCodec = audioCodec; + this.videoCodec = videoCodec; + this._duration = duration; + } + + /** + * + * @override + */ + resetTimeStamp () {} + + // feed incoming data to the front of the parsing pipeline + append (data, timeOffset, contiguous, accurateTimeOffset) { + let start, len = data.length, stt, pid, atf, offset, pes, + unknownPIDs = false; + this.contiguous = contiguous; + let pmtParsed = this.pmtParsed, + avcTrack = this._avcTrack, + audioTrack = this._audioTrack, + id3Track = this._id3Track, + avcId = avcTrack.pid, + audioId = audioTrack.pid, + id3Id = id3Track.pid, + pmtId = this._pmtId, + avcData = avcTrack.pesData, + audioData = audioTrack.pesData, + id3Data = id3Track.pesData, + parsePAT = this._parsePAT, + parsePMT = this._parsePMT, + parsePES = this._parsePES, + parseAVCPES = this._parseAVCPES.bind(this), + parseAACPES = this._parseAACPES.bind(this), + parseMPEGPES = this._parseMPEGPES.bind(this), + parseID3PES = this._parseID3PES.bind(this); + + const syncOffset = TSDemuxer._syncOffset(data); + + // don't parse last TS packet if incomplete + len -= (len + syncOffset) % 188; + + // loop through TS packets + for (start = syncOffset; start < len; start += 188) { + if (data[start] === 0x47) { + stt = !!(data[start + 1] & 0x40); + // pid is a 13-bit field starting at the last bit of TS[1] + pid = ((data[start + 1] & 0x1f) << 8) + data[start + 2]; + atf = (data[start + 3] & 0x30) >> 4; + // if an adaption field is present, its length is specified by the fifth byte of the TS packet header. + if (atf > 1) { + offset = start + 5 + data[start + 4]; + // continue if there is only adaptation field + if (offset === (start + 188)) { + continue; + } + } else { + offset = start + 4; + } + switch (pid) { + case avcId: + if (stt) { + if (avcData && (pes = parsePES(avcData)) && pes.pts !== undefined) { + parseAVCPES(pes, false); + } + + avcData = { data: [], size: 0 }; + } + if (avcData) { + avcData.data.push(data.subarray(offset, start + 188)); + avcData.size += start + 188 - offset; + } + break; + case audioId: + if (stt) { + if (audioData && (pes = parsePES(audioData)) && pes.pts !== undefined) { + if (audioTrack.isAAC) { + parseAACPES(pes); + } else { + parseMPEGPES(pes); + } + } + audioData = { data: [], size: 0 }; + } + if (audioData) { + audioData.data.push(data.subarray(offset, start + 188)); + audioData.size += start + 188 - offset; + } + break; + case id3Id: + if (stt) { + if (id3Data && (pes = parsePES(id3Data)) && pes.pts !== undefined) { + parseID3PES(pes); + } + + id3Data = { data: [], size: 0 }; + } + if (id3Data) { + id3Data.data.push(data.subarray(offset, start + 188)); + id3Data.size += start + 188 - offset; + } + break; + case 0: + if (stt) { + offset += data[offset] + 1; + } + + pmtId = this._pmtId = parsePAT(data, offset); + break; + case pmtId: + if (stt) { + offset += data[offset] + 1; + } + + let parsedPIDs = parsePMT(data, offset, this.typeSupported.mpeg === true || this.typeSupported.mp3 === true, this.sampleAes != null); + + // only update track id if track PID found while parsing PMT + // this is to avoid resetting the PID to -1 in case + // track PID transiently disappears from the stream + // this could happen in case of transient missing audio samples for example + // NOTE this is only the PID of the track as found in TS, + // but we are not using this for MP4 track IDs. + avcId = parsedPIDs.avc; + if (avcId > 0) { + avcTrack.pid = avcId; + } + + audioId = parsedPIDs.audio; + if (audioId > 0) { + audioTrack.pid = audioId; + audioTrack.isAAC = parsedPIDs.isAAC; + } + id3Id = parsedPIDs.id3; + if (id3Id > 0) { + id3Track.pid = id3Id; + } + + if (unknownPIDs && !pmtParsed) { + logger.log('reparse from beginning'); + unknownPIDs = false; + // we set it to -188, the += 188 in the for loop will reset start to 0 + start = syncOffset - 188; + } + pmtParsed = this.pmtParsed = true; + break; + case 17: + case 0x1fff: + break; + default: + unknownPIDs = true; + break; + } + } else { + this.observer.trigger(Event.ERROR, { type: ErrorTypes.MEDIA_ERROR, details: ErrorDetails.FRAG_PARSING_ERROR, fatal: false, reason: 'TS packet did not start with 0x47' }); + } + } + // try to parse last PES packets + if (avcData && (pes = parsePES(avcData)) && pes.pts !== undefined) { + parseAVCPES(pes, true); + avcTrack.pesData = null; + } else { + // either avcData null or PES truncated, keep it for next frag parsing + avcTrack.pesData = avcData; + } + + if (audioData && (pes = parsePES(audioData)) && pes.pts !== undefined) { + if (audioTrack.isAAC) { + parseAACPES(pes); + } else { + parseMPEGPES(pes); + } + + audioTrack.pesData = null; + } else { + if (audioData && audioData.size) { + logger.log('last AAC PES packet truncated,might overlap between fragments'); + } + + // either audioData null or PES truncated, keep it for next frag parsing + audioTrack.pesData = audioData; + } + + if (id3Data && (pes = parsePES(id3Data)) && pes.pts !== undefined) { + parseID3PES(pes); + id3Track.pesData = null; + } else { + // either id3Data null or PES truncated, keep it for next frag parsing + id3Track.pesData = id3Data; + } + + if (this.sampleAes == null) { + this.remuxer.remux(audioTrack, avcTrack, id3Track, this._txtTrack, timeOffset, contiguous, accurateTimeOffset); + } else { + this.decryptAndRemux(audioTrack, avcTrack, id3Track, this._txtTrack, timeOffset, contiguous, accurateTimeOffset); + } + } + + decryptAndRemux (audioTrack, videoTrack, id3Track, textTrack, timeOffset, contiguous, accurateTimeOffset) { + if (audioTrack.samples && audioTrack.isAAC) { + let localthis = this; + this.sampleAes.decryptAacSamples(audioTrack.samples, 0, function () { + localthis.decryptAndRemuxAvc(audioTrack, videoTrack, id3Track, textTrack, timeOffset, contiguous, accurateTimeOffset); + }); + } else { + this.decryptAndRemuxAvc(audioTrack, videoTrack, id3Track, textTrack, timeOffset, contiguous, accurateTimeOffset); + } + } + + decryptAndRemuxAvc (audioTrack, videoTrack, id3Track, textTrack, timeOffset, contiguous, accurateTimeOffset) { + if (videoTrack.samples) { + let localthis = this; + this.sampleAes.decryptAvcSamples(videoTrack.samples, 0, 0, function () { + localthis.remuxer.remux(audioTrack, videoTrack, id3Track, textTrack, timeOffset, contiguous, accurateTimeOffset); + }); + } else { + this.remuxer.remux(audioTrack, videoTrack, id3Track, textTrack, timeOffset, contiguous, accurateTimeOffset); + } + } + + destroy () { + this._initPTS = this._initDTS = undefined; + this._duration = 0; + } + + _parsePAT (data, offset) { + // skip the PSI header and parse the first PMT entry + return (data[offset + 10] & 0x1F) << 8 | data[offset + 11]; + // logger.log('PMT PID:' + this._pmtId); + } + + _parsePMT (data, offset, mpegSupported, isSampleAes) { + let sectionLength, tableEnd, programInfoLength, pid, result = { audio: -1, avc: -1, id3: -1, isAAC: true }; + sectionLength = (data[offset + 1] & 0x0f) << 8 | data[offset + 2]; + tableEnd = offset + 3 + sectionLength - 4; + // to determine where the table is, we have to figure out how + // long the program info descriptors are + programInfoLength = (data[offset + 10] & 0x0f) << 8 | data[offset + 11]; + // advance the offset to the first entry in the mapping table + offset += 12 + programInfoLength; + while (offset < tableEnd) { + pid = (data[offset + 1] & 0x1F) << 8 | data[offset + 2]; + switch (data[offset]) { + case 0xcf: // SAMPLE-AES AAC + if (!isSampleAes) { + logger.log('unknown stream type:' + data[offset]); + break; + } + /* falls through */ + + // ISO/IEC 13818-7 ADTS AAC (MPEG-2 lower bit-rate audio) + case 0x0f: + // logger.log('AAC PID:' + pid); + if (result.audio === -1) { + result.audio = pid; + } + + break; + + // Packetized metadata (ID3) + case 0x15: + // logger.log('ID3 PID:' + pid); + if (result.id3 === -1) { + result.id3 = pid; + } + + break; + + case 0xdb: // SAMPLE-AES AVC + if (!isSampleAes) { + logger.log('unknown stream type:' + data[offset]); + break; + } + /* falls through */ + + // ITU-T Rec. H.264 and ISO/IEC 14496-10 (lower bit-rate video) + case 0x1b: + // logger.log('AVC PID:' + pid); + if (result.avc === -1) { + result.avc = pid; + } + + break; + + // ISO/IEC 11172-3 (MPEG-1 audio) + // or ISO/IEC 13818-3 (MPEG-2 halved sample rate audio) + case 0x03: + case 0x04: + // logger.log('MPEG PID:' + pid); + if (!mpegSupported) { + logger.log('MPEG audio found, not supported in this browser for now'); + } else if (result.audio === -1) { + result.audio = pid; + result.isAAC = false; + } + break; + + case 0x24: + logger.warn('HEVC stream type found, not supported for now'); + break; + + default: + logger.log('unknown stream type:' + data[offset]); + break; + } + // move to the next table entry + // skip past the elementary stream descriptors, if present + offset += ((data[offset + 3] & 0x0F) << 8 | data[offset + 4]) + 5; + } + return result; + } + + _parsePES (stream) { + let i = 0, frag, pesFlags, pesPrefix, pesLen, pesHdrLen, pesData, pesPts, pesDts, payloadStartOffset, data = stream.data; + // safety check + if (!stream || stream.size === 0) { + return null; + } + + // we might need up to 19 bytes to read PES header + // if first chunk of data is less than 19 bytes, let's merge it with following ones until we get 19 bytes + // usually only one merge is needed (and this is rare ...) + while (data[0].length < 19 && data.length > 1) { + let newData = new Uint8Array(data[0].length + data[1].length); + newData.set(data[0]); + newData.set(data[1], data[0].length); + data[0] = newData; + data.splice(1, 1); + } + // retrieve PTS/DTS from first fragment + frag = data[0]; + pesPrefix = (frag[0] << 16) + (frag[1] << 8) + frag[2]; + if (pesPrefix === 1) { + pesLen = (frag[4] << 8) + frag[5]; + // if PES parsed length is not zero and greater than total received length, stop parsing. PES might be truncated + // minus 6 : PES header size + if (pesLen && pesLen > stream.size - 6) { + return null; + } + + pesFlags = frag[7]; + if (pesFlags & 0xC0) { + /* PES header described here : http://dvd.sourceforge.net/dvdinfo/pes-hdr.html + as PTS / DTS is 33 bit we cannot use bitwise operator in JS, + as Bitwise operators treat their operands as a sequence of 32 bits */ + pesPts = (frag[9] & 0x0E) * 536870912 +// 1 << 29 + (frag[10] & 0xFF) * 4194304 +// 1 << 22 + (frag[11] & 0xFE) * 16384 +// 1 << 14 + (frag[12] & 0xFF) * 128 +// 1 << 7 + (frag[13] & 0xFE) / 2; + // check if greater than 2^32 -1 + if (pesPts > 4294967295) { + // decrement 2^33 + pesPts -= 8589934592; + } + if (pesFlags & 0x40) { + pesDts = (frag[14] & 0x0E) * 536870912 +// 1 << 29 + (frag[15] & 0xFF) * 4194304 +// 1 << 22 + (frag[16] & 0xFE) * 16384 +// 1 << 14 + (frag[17] & 0xFF) * 128 +// 1 << 7 + (frag[18] & 0xFE) / 2; + // check if greater than 2^32 -1 + if (pesDts > 4294967295) { + // decrement 2^33 + pesDts -= 8589934592; + } + if (pesPts - pesDts > 60 * 90000) { + logger.warn(`${Math.round((pesPts - pesDts) / 90000)}s delta between PTS and DTS, align them`); + pesPts = pesDts; + } + } else { + pesDts = pesPts; + } + } + pesHdrLen = frag[8]; + // 9 bytes : 6 bytes for PES header + 3 bytes for PES extension + payloadStartOffset = pesHdrLen + 9; + + stream.size -= payloadStartOffset; + // reassemble PES packet + pesData = new Uint8Array(stream.size); + for (let j = 0, dataLen = data.length; j < dataLen; j++) { + frag = data[j]; + let len = frag.byteLength; + if (payloadStartOffset) { + if (payloadStartOffset > len) { + // trim full frag if PES header bigger than frag + payloadStartOffset -= len; + continue; + } else { + // trim partial frag if PES header smaller than frag + frag = frag.subarray(payloadStartOffset); + len -= payloadStartOffset; + payloadStartOffset = 0; + } + } + pesData.set(frag, i); + i += len; + } + if (pesLen) { + // payload size : remove PES header + PES extension + pesLen -= pesHdrLen + 3; + } + return { data: pesData, pts: pesPts, dts: pesDts, len: pesLen }; + } else { + return null; + } + } + + pushAccesUnit (avcSample, avcTrack) { + if (avcSample.units.length && avcSample.frame) { + const samples = avcTrack.samples; + const nbSamples = samples.length; + // only push AVC sample if starting with a keyframe is not mandatory OR + // if keyframe already found in this fragment OR + // keyframe found in last fragment (track.sps) AND + // samples already appended (we already found a keyframe in this fragment) OR fragment is contiguous + if (!this.config.forceKeyFrameOnDiscontinuity || + avcSample.key === true || + (avcTrack.sps && (nbSamples || this.contiguous))) { + avcSample.id = nbSamples; + samples.push(avcSample); + } else { + // dropped samples, track it + avcTrack.dropped++; + } + } + if (avcSample.debug.length) { + logger.log(avcSample.pts + '/' + avcSample.dts + ':' + avcSample.debug); + } + } + + _parseAVCPES (pes, last) { + // logger.log('parse new PES'); + let track = this._avcTrack, + units = this._parseAVCNALu(pes.data), + debug = false, + expGolombDecoder, + avcSample = this.avcSample, + push, + spsfound = false, + i, + pushAccesUnit = this.pushAccesUnit.bind(this), + createAVCSample = function (key, pts, dts, debug) { + return { key: key, pts: pts, dts: dts, units: [], debug: debug }; + }; + // free pes.data to save up some memory + pes.data = null; + + // if new NAL units found and last sample still there, let's push ... + // this helps parsing streams with missing AUD (only do this if AUD never found) + if (avcSample && units.length && !track.audFound) { + pushAccesUnit(avcSample, track); + avcSample = this.avcSample = createAVCSample(false, pes.pts, pes.dts, ''); + } + + units.forEach(unit => { + switch (unit.type) { + // NDR + case 1: + push = true; + if (!avcSample) { + avcSample = this.avcSample = createAVCSample(true, pes.pts, pes.dts, ''); + } + + if (debug) { + avcSample.debug += 'NDR '; + } + + avcSample.frame = true; + let data = unit.data; + // only check slice type to detect KF in case SPS found in same packet (any keyframe is preceded by SPS ...) + if (spsfound && data.length > 4) { + // retrieve slice type by parsing beginning of NAL unit (follow H264 spec, slice_header definition) to detect keyframe embedded in NDR + let sliceType = new ExpGolomb(data).readSliceType(); + // 2 : I slice, 4 : SI slice, 7 : I slice, 9: SI slice + // SI slice : A slice that is coded using intra prediction only and using quantisation of the prediction samples. + // An SI slice can be coded such that its decoded samples can be constructed identically to an SP slice. + // I slice: A slice that is not an SI slice that is decoded using intra prediction only. + // if (sliceType === 2 || sliceType === 7) { + if (sliceType === 2 || sliceType === 4 || sliceType === 7 || sliceType === 9) { + avcSample.key = true; + } + } + break; + // IDR + case 5: + push = true; + // handle PES not starting with AUD + if (!avcSample) { + avcSample = this.avcSample = createAVCSample(true, pes.pts, pes.dts, ''); + } + + if (debug) { + avcSample.debug += 'IDR '; + } + + avcSample.key = true; + avcSample.frame = true; + break; + // SEI + case 6: + push = true; + if (debug && avcSample) { + avcSample.debug += 'SEI '; + } + + expGolombDecoder = new ExpGolomb(this.discardEPB(unit.data)); + + // skip frameType + expGolombDecoder.readUByte(); + + var payloadType = 0; + var payloadSize = 0; + var endOfCaptions = false; + var b = 0; + + while (!endOfCaptions && expGolombDecoder.bytesAvailable > 1) { + payloadType = 0; + do { + b = expGolombDecoder.readUByte(); + payloadType += b; + } while (b === 0xFF); + + // Parse payload size. + payloadSize = 0; + do { + b = expGolombDecoder.readUByte(); + payloadSize += b; + } while (b === 0xFF); + + // TODO: there can be more than one payload in an SEI packet... + // TODO: need to read type and size in a while loop to get them all + if (payloadType === 4 && expGolombDecoder.bytesAvailable !== 0) { + endOfCaptions = true; + + let countryCode = expGolombDecoder.readUByte(); + + if (countryCode === 181) { + let providerCode = expGolombDecoder.readUShort(); + + if (providerCode === 49) { + let userStructure = expGolombDecoder.readUInt(); + + if (userStructure === 0x47413934) { + let userDataType = expGolombDecoder.readUByte(); + + // Raw CEA-608 bytes wrapped in CEA-708 packet + if (userDataType === 3) { + let firstByte = expGolombDecoder.readUByte(); + let secondByte = expGolombDecoder.readUByte(); + + let totalCCs = 31 & firstByte; + let byteArray = [firstByte, secondByte]; + + for (i = 0; i < totalCCs; i++) { + // 3 bytes per CC + byteArray.push(expGolombDecoder.readUByte()); + byteArray.push(expGolombDecoder.readUByte()); + byteArray.push(expGolombDecoder.readUByte()); + } + + this._insertSampleInOrder(this._txtTrack.samples, { type: 3, pts: pes.pts, bytes: byteArray }); + } + } + } + } + } else if (payloadType === 5 && expGolombDecoder.bytesAvailable !== 0) { + endOfCaptions = true; + + if (payloadSize > 16) { + let uuidStrArray = []; + let userDataPayloadBytes = []; + + for (i = 0; i < 16; i++) { + uuidStrArray.push(expGolombDecoder.readUByte().toString(16)); + + if (i === 3 || i === 5 || i === 7 || i === 9) { + uuidStrArray.push('-'); + } + } + + for (i = 16; i < payloadSize; i++) { + userDataPayloadBytes.push(expGolombDecoder.readUByte()); + } + + this._insertSampleInOrder(this._txtTrack.samples, { + pts: pes.pts, + payloadType: payloadType, + uuid: uuidStrArray.join(''), + userData: String.fromCharCode.apply(null, userDataPayloadBytes), + userDataBytes: userDataPayloadBytes + }); + } + } else if (payloadSize < expGolombDecoder.bytesAvailable) { + for (i = 0; i < payloadSize; i++) { + expGolombDecoder.readUByte(); + } + } + } + break; + // SPS + case 7: + push = true; + spsfound = true; + if (debug && avcSample) { + avcSample.debug += 'SPS '; + } + + if (!track.sps) { + expGolombDecoder = new ExpGolomb(unit.data); + let config = expGolombDecoder.readSPS(); + track.width = config.width; + track.height = config.height; + track.pixelRatio = config.pixelRatio; + track.sps = [unit.data]; + track.duration = this._duration; + let codecarray = unit.data.subarray(1, 4); + let codecstring = 'avc1.'; + for (i = 0; i < 3; i++) { + let h = codecarray[i].toString(16); + if (h.length < 2) { + h = '0' + h; + } + + codecstring += h; + } + track.codec = codecstring; + } + break; + // PPS + case 8: + push = true; + if (debug && avcSample) { + avcSample.debug += 'PPS '; + } + + if (!track.pps) { + track.pps = [unit.data]; + } + + break; + // AUD + case 9: + push = false; + track.audFound = true; + if (avcSample) { + pushAccesUnit(avcSample, track); + } + + avcSample = this.avcSample = createAVCSample(false, pes.pts, pes.dts, debug ? 'AUD ' : ''); + break; + // Filler Data + case 12: + push = false; + break; + default: + push = false; + if (avcSample) { + avcSample.debug += 'unknown NAL ' + unit.type + ' '; + } + + break; + } + if (avcSample && push) { + let units = avcSample.units; + units.push(unit); + } + }); + // if last PES packet, push samples + if (last && avcSample) { + pushAccesUnit(avcSample, track); + this.avcSample = null; + } + } + + _insertSampleInOrder (arr, data) { + let len = arr.length; + if (len > 0) { + if (data.pts >= arr[len - 1].pts) { + arr.push(data); + } else { + for (let pos = len - 1; pos >= 0; pos--) { + if (data.pts < arr[pos].pts) { + arr.splice(pos, 0, data); + break; + } + } + } + } else { + arr.push(data); + } + } + + _getLastNalUnit () { + let avcSample = this.avcSample, lastUnit; + // try to fallback to previous sample if current one is empty + if (!avcSample || avcSample.units.length === 0) { + let track = this._avcTrack, samples = track.samples; + avcSample = samples[samples.length - 1]; + } + if (avcSample) { + let units = avcSample.units; + lastUnit = units[units.length - 1]; + } + return lastUnit; + } + + _parseAVCNALu (array) { + let i = 0, len = array.byteLength, value, overflow, track = this._avcTrack, state = track.naluState || 0, lastState = state; + let units = [], unit, unitType, lastUnitStart = -1, lastUnitType; + // logger.log('PES:' + Hex.hexDump(array)); + + if (state === -1) { + // special use case where we found 3 or 4-byte start codes exactly at the end of previous PES packet + lastUnitStart = 0; + // NALu type is value read from offset 0 + lastUnitType = array[0] & 0x1f; + state = 0; + i = 1; + } + + while (i < len) { + value = array[i++]; + // optimization. state 0 and 1 are the predominant case. let's handle them outside of the switch/case + if (!state) { + state = value ? 0 : 1; + continue; + } + if (state === 1) { + state = value ? 0 : 2; + continue; + } + // here we have state either equal to 2 or 3 + if (!value) { + state = 3; + } else if (value === 1) { + if (lastUnitStart >= 0) { + unit = { data: array.subarray(lastUnitStart, i - state - 1), type: lastUnitType }; + // logger.log('pushing NALU, type/size:' + unit.type + '/' + unit.data.byteLength); + units.push(unit); + } else { + // lastUnitStart is undefined => this is the first start code found in this PES packet + // first check if start code delimiter is overlapping between 2 PES packets, + // ie it started in last packet (lastState not zero) + // and ended at the beginning of this PES packet (i <= 4 - lastState) + let lastUnit = this._getLastNalUnit(); + if (lastUnit) { + if (lastState && (i <= 4 - lastState)) { + // start delimiter overlapping between PES packets + // strip start delimiter bytes from the end of last NAL unit + // check if lastUnit had a state different from zero + if (lastUnit.state) { + // strip last bytes + lastUnit.data = lastUnit.data.subarray(0, lastUnit.data.byteLength - lastState); + } + } + // If NAL units are not starting right at the beginning of the PES packet, push preceding data into previous NAL unit. + overflow = i - state - 1; + if (overflow > 0) { + // logger.log('first NALU found with overflow:' + overflow); + let tmp = new Uint8Array(lastUnit.data.byteLength + overflow); + tmp.set(lastUnit.data, 0); + tmp.set(array.subarray(0, overflow), lastUnit.data.byteLength); + lastUnit.data = tmp; + } + } + } + // check if we can read unit type + if (i < len) { + unitType = array[i] & 0x1f; + // logger.log('find NALU @ offset:' + i + ',type:' + unitType); + lastUnitStart = i; + lastUnitType = unitType; + state = 0; + } else { + // not enough byte to read unit type. let's read it on next PES parsing + state = -1; + } + } else { + state = 0; + } + } + if (lastUnitStart >= 0 && state >= 0) { + unit = { data: array.subarray(lastUnitStart, len), type: lastUnitType, state: state }; + units.push(unit); + // logger.log('pushing NALU, type/size/state:' + unit.type + '/' + unit.data.byteLength + '/' + state); + } + // no NALu found + if (units.length === 0) { + // append pes.data to previous NAL unit + let lastUnit = this._getLastNalUnit(); + if (lastUnit) { + let tmp = new Uint8Array(lastUnit.data.byteLength + array.byteLength); + tmp.set(lastUnit.data, 0); + tmp.set(array, lastUnit.data.byteLength); + lastUnit.data = tmp; + } + } + track.naluState = state; + return units; + } + + /** + * remove Emulation Prevention bytes from a RBSP + */ + discardEPB (data) { + let length = data.byteLength, + EPBPositions = [], + i = 1, + newLength, newData; + + // Find all `Emulation Prevention Bytes` + while (i < length - 2) { + if (data[i] === 0 && + data[i + 1] === 0 && + data[i + 2] === 0x03) { + EPBPositions.push(i + 2); + i += 2; + } else { + i++; + } + } + + // If no Emulation Prevention Bytes were found just return the original + // array + if (EPBPositions.length === 0) { + return data; + } + + // Create a new array to hold the NAL unit data + newLength = length - EPBPositions.length; + newData = new Uint8Array(newLength); + let sourceIndex = 0; + + for (i = 0; i < newLength; sourceIndex++, i++) { + if (sourceIndex === EPBPositions[0]) { + // Skip this byte + sourceIndex++; + // Remove this position index + EPBPositions.shift(); + } + newData[i] = data[sourceIndex]; + } + return newData; + } + + _parseAACPES (pes) { + let track = this._audioTrack, + data = pes.data, + pts = pes.pts, + startOffset = 0, + aacOverFlow = this.aacOverFlow, + aacLastPTS = this.aacLastPTS, + frameDuration, frameIndex, offset, stamp, len; + if (aacOverFlow) { + let tmp = new Uint8Array(aacOverFlow.byteLength + data.byteLength); + tmp.set(aacOverFlow, 0); + tmp.set(data, aacOverFlow.byteLength); + // logger.log(`AAC: append overflowing ${aacOverFlow.byteLength} bytes to beginning of new PES`); + data = tmp; + } + // look for ADTS header (0xFFFx) + for (offset = startOffset, len = data.length; offset < len - 1; offset++) { + if (ADTS.isHeader(data, offset)) { + break; + } + } + // if ADTS header does not start straight from the beginning of the PES payload, raise an error + if (offset) { + let reason, fatal; + if (offset < len - 1) { + reason = `AAC PES did not start with ADTS header,offset:${offset}`; + fatal = false; + } else { + reason = 'no ADTS header found in AAC PES'; + fatal = true; + } + logger.warn(`parsing error:${reason}`); + this.observer.trigger(Event.ERROR, { type: ErrorTypes.MEDIA_ERROR, details: ErrorDetails.FRAG_PARSING_ERROR, fatal: fatal, reason: reason }); + if (fatal) { + return; + } + } + + ADTS.initTrackConfig(track, this.observer, data, offset, this.audioCodec); + frameIndex = 0; + frameDuration = ADTS.getFrameDuration(track.samplerate); + + // if last AAC frame is overflowing, we should ensure timestamps are contiguous: + // first sample PTS should be equal to last sample PTS + frameDuration + if (aacOverFlow && aacLastPTS) { + let newPTS = aacLastPTS + frameDuration; + if (Math.abs(newPTS - pts) > 1) { + logger.log(`AAC: align PTS for overlapping frames by ${Math.round((newPTS - pts) / 90)}`); + pts = newPTS; + } + } + + // scan for aac samples + while (offset < len) { + if (ADTS.isHeader(data, offset) && (offset + 5) < len) { + let frame = ADTS.appendFrame(track, data, offset, pts, frameIndex); + if (frame) { + // logger.log(`${Math.round(frame.sample.pts)} : AAC`); + offset += frame.length; + stamp = frame.sample.pts; + frameIndex++; + } else { + // logger.log('Unable to parse AAC frame'); + break; + } + } else { + // nothing found, keep looking + offset++; + } + } + + if (offset < len) { + aacOverFlow = data.subarray(offset, len); + // logger.log(`AAC: overflow detected:${len-offset}`); + } else { + aacOverFlow = null; + } + + this.aacOverFlow = aacOverFlow; + this.aacLastPTS = stamp; + } + + _parseMPEGPES (pes) { + let data = pes.data; + let length = data.length; + let frameIndex = 0; + let offset = 0; + let pts = pes.pts; + + while (offset < length) { + if (MpegAudio.isHeader(data, offset)) { + let frame = MpegAudio.appendFrame(this._audioTrack, data, offset, pts, frameIndex); + if (frame) { + offset += frame.length; + frameIndex++; + } else { + // logger.log('Unable to parse Mpeg audio frame'); + break; + } + } else { + // nothing found, keep looking + offset++; + } + } + } + + _parseID3PES (pes) { + this._id3Track.samples.push(pes); + } +} + +export default TSDemuxer;