diff --git a/cmd/mjpeg-player/hlsjs/mts-demuxer.js b/cmd/mjpeg-player/hlsjs/mts-demuxer.js index 9095935e..a1ba5fd9 100644 --- a/cmd/mjpeg-player/hlsjs/mts-demuxer.js +++ b/cmd/mjpeg-player/hlsjs/mts-demuxer.js @@ -1,74 +1,49 @@ -/** - * highly optimized TS demuxer: - * parse PAT, PMT - * extract PES packet from audio and video PIDs - * extract AVC/H264 NAL units and AAC/ADTS samples from PES packet - * trigger the remuxer upon parsing completion - * it also tries to workaround as best as it can audio codec switch (HE-AAC to AAC and vice versa), without having to restart the MediaSource. - * it also controls the remuxing process : - * upon discontinuity or level switch detection, it will also notifies the remuxer so that it can reset its state. -*/ - -import * as ADTS from './adts'; -import MpegAudio from './mpegaudio'; -import Event from '../events'; -import ExpGolomb from './exp-golomb'; -import SampleAesDecrypter from './sample-aes'; -// import Hex from '../utils/hex'; -import { logger } from '../utils/logger'; -import { ErrorTypes, ErrorDetails } from '../errors'; - -// We are using fixed track IDs for driving the MP4 remuxer -// instead of following the TS PIDs. -// There is no reason not to do this and some browsers/SourceBuffer-demuxers -// may not like if there are TrackID "switches" -// See https://github.com/video-dev/hls.js/issues/1331 -// Here we are mapping our internal track types to constant MP4 track IDs -// With MSE currently one can only have one track of each, and we are muxing -// whatever video/audio rendition in them. -const RemuxerTrackIdConfig = { - video: 1, - audio: 2, - id3: 3, - text: 4 -}; - -class TSDemuxer { - constructor (observer, remuxer, config, typeSupported) { - this.observer = observer; - this.config = config; - this.typeSupported = typeSupported; - this.remuxer = remuxer; - this.sampleAes = null; +// MTSDemuxer demultiplexes an MPEG-TS stream into its individual streams. +// While it is possible that the MPEG-TS stream may contain many streams, +// this demuxer will result in at most one stream of each type ie. video, audio, id3 metadata. +class MTSDemuxer{ + constructor(){ + this.init(); + } + + // init initialises MTSDemuxer's state. It can be used to reset an MTSDemuxer instance. + init() { + this.pmtParsed = false; + this._pmtId = -1; + this._videoTrack = MTSDemuxer.createTrack('video'); + this._audioTrack = MTSDemuxer.createTrack('audio'); + this._id3Track = MTSDemuxer.createTrack('id3'); } - setDecryptData (decryptdata) { - if ((decryptdata != null) && (decryptdata.key != null) && (decryptdata.method === 'SAMPLE-AES')) { - this.sampleAes = new SampleAesDecrypter(this.observer, this.config, decryptdata, this.discardEPB); - } else { - this.sampleAes = null; - } + // createTrack creates and returns a track model. + /** + * @param {string} type 'audio' | 'video' | 'id3' | 'text' + * @return {object} MTSDemuxer's internal track model. + */ + static createTrack (type) { + return { + type, + pid: -1, + data: [] // This will contain Uint8Arrays representing each PES packet's payload for this track. + }; } - static probe (data) { - const syncOffset = TSDemuxer._syncOffset(data); - if (syncOffset < 0) { - return false; - } else { - if (syncOffset) { - logger.warn(`MPEG2-TS detected but first sync word found @ offset ${syncOffset}, junk ahead ?`); - } - - return true; - } + // _getTracks returns this MTSDemuxer's tracks. + _getTracks(){ + return { + video: this._videoTrack, + audio: this._audioTrack, + id3: this._id3Track + }; } + // _syncOffset scans the first 1000 bytes and returns an offset to the beginning of the first three MTS packets, + // or -1 if three are not found. + // A TS fragment should contain at least 3 TS packets, a PAT, a PMT, and one PID, each starting with 0x47. static _syncOffset (data) { - // scan 1000 first bytes const scanwindow = Math.min(1000, data.length - 3 * 188); let i = 0; while (i < scanwindow) { - // a TS fragment should contain at least 3 TS packets, a PAT, a PMT, and one PID, each starting with 0x47 if (data[i] === 0x47 && data[i + 188] === 0x47 && data[i + 2 * 188] === 0x47) { return i; } else { @@ -78,102 +53,45 @@ class TSDemuxer { return -1; } - /** - * Creates a track model internal to demuxer used to drive remuxing input - * - * @param {string} type 'audio' | 'video' | 'id3' | 'text' - * @param {number} duration - * @return {object} TSDemuxer's internal track model - */ - static createTrack (type, duration) { - return { - container: type === 'video' || type === 'audio' ? 'video/mp2t' : undefined, - type, - id: RemuxerTrackIdConfig[type], - pid: -1, - inputTimeScale: 90000, - sequenceNumber: 0, - samples: [], - dropped: type === 'video' ? 0 : undefined, - isAAC: type === 'audio' ? true : undefined, - duration: type === 'audio' ? duration : undefined - }; - } + append(data) { + let videoTrack = this._videoTrack; + let videoData = videoTrack.pesData; + let videoId = videoTrack.pid; - /** - * Initializes a new init segment on the demuxer/remuxer interface. Needed for discontinuities/track-switches (or at stream start) - * Resets all internal track instances of the demuxer. - * - * @override Implements generic demuxing/remuxing interface (see DemuxerInline) - * @param {object} initSegment - * @param {string} audioCodec - * @param {string} videoCodec - * @param {number} duration (in TS timescale = 90kHz) - */ - resetInitSegment (initSegment, audioCodec, videoCodec, duration) { - this.pmtParsed = false; - this._pmtId = -1; + let audioTrack = this._audioTrack; + let audioData = audioTrack.pesData; + let audioId = audioTrack.pid; - this._avcTrack = TSDemuxer.createTrack('video', duration); - this._audioTrack = TSDemuxer.createTrack('audio', duration); - this._id3Track = TSDemuxer.createTrack('id3', duration); - this._txtTrack = TSDemuxer.createTrack('text', duration); + let id3Track = this._id3Track; + let id3Data = id3Track.pesData; + let id3Id = id3Track.pid; - // flush any partial content - this.aacOverFlow = null; - this.aacLastPTS = null; - this.avcSample = null; - this.audioCodec = audioCodec; - this.videoCodec = videoCodec; - this._duration = duration; - } + let pmtId = this._pmtId; + let pmtParsed = this.pmtParsed; + let parsePAT = this._parsePAT; + let parsePMT = this._parsePMT; + let parsePES = this._parsePES; - /** - * - * @override - */ - resetTimeStamp () {} - - // feed incoming data to the front of the parsing pipeline - append (data, timeOffset, contiguous, accurateTimeOffset) { - let start, len = data.length, stt, pid, atf, offset, pes, - unknownPIDs = false; - this.contiguous = contiguous; - let pmtParsed = this.pmtParsed, - avcTrack = this._avcTrack, - audioTrack = this._audioTrack, - id3Track = this._id3Track, - avcId = avcTrack.pid, - audioId = audioTrack.pid, - id3Id = id3Track.pid, - pmtId = this._pmtId, - avcData = avcTrack.pesData, - audioData = audioTrack.pesData, - id3Data = id3Track.pesData, - parsePAT = this._parsePAT, - parsePMT = this._parsePMT, - parsePES = this._parsePES, - parseAVCPES = this._parseAVCPES.bind(this), - parseAACPES = this._parseAACPES.bind(this), - parseMPEGPES = this._parseMPEGPES.bind(this), - parseID3PES = this._parseID3PES.bind(this); - - const syncOffset = TSDemuxer._syncOffset(data); - - // don't parse last TS packet if incomplete + let len = data.length; + let unknownPIDs = false; + + const syncOffset = MTSDemuxer._syncOffset(data); + + // Don't parse last TS packet if incomplete. len -= (len + syncOffset) % 188; - - // loop through TS packets + + // Loop through TS packets. + let start, offset, pusi, pid, afc, pes; for (start = syncOffset; start < len; start += 188) { if (data[start] === 0x47) { - stt = !!(data[start + 1] & 0x40); - // pid is a 13-bit field starting at the last bit of TS[1] + pusi = !!(data[start + 1] & 0x40); + // pid is a 13-bit field starting at the last bit of TS[1]. pid = ((data[start + 1] & 0x1f) << 8) + data[start + 2]; - atf = (data[start + 3] & 0x30) >> 4; - // if an adaption field is present, its length is specified by the fifth byte of the TS packet header. - if (atf > 1) { + afc = (data[start + 3] & 0x30) >> 4; + // If an adaption field is present, its length is specified by the fifth byte of the TS packet header. + if (afc > 1) { offset = start + 5 + data[start + 4]; - // continue if there is only adaptation field + // Continue if there is only adaptation field. if (offset === (start + 188)) { continue; } @@ -181,27 +99,61 @@ class TSDemuxer { offset = start + 4; } switch (pid) { - case avcId: - if (stt) { - if (avcData && (pes = parsePES(avcData)) && pes.pts !== undefined) { - parseAVCPES(pes, false); - } - - avcData = { data: [], size: 0 }; + case 0: + if (pusi) { + offset += data[offset] + 1; } - if (avcData) { - avcData.data.push(data.subarray(offset, start + 188)); - avcData.size += start + 188 - offset; + + pmtId = this._pmtId = parsePAT(data, offset); + break; + case pmtId: + if (pusi) { + offset += data[offset] + 1; + } + + let parsedPIDs = parsePMT(data, offset); + + // Only update track id if track PID found while parsing PMT. + // This is to avoid resetting the PID to -1 in case track PID transiently disappears from the stream, + // this could happen in case of transient missing audio samples for example. + videoId = parsedPIDs.video; + if (videoId > 0) { + videoTrack.pid = videoId; + } + audioId = parsedPIDs.audio; + if (audioId > 0) { + audioTrack.pid = audioId; + } + id3Id = parsedPIDs.id3; + if (id3Id > 0) { + id3Track.pid = id3Id; + } + + if (unknownPIDs && !pmtParsed) { + // Reparse from beginning. + unknownPIDs = false; + // We set it to -188, the += 188 in the for loop will reset start to 0. + start = syncOffset - 188; + } + pmtParsed = this.pmtParsed = true; + break; + case videoId: + if (pusi) { + if (videoData && (pes = parsePES(videoData)) && pes.pts !== undefined) { + videoTrack.data.push(pes.data); + // TODO: here pes contains data, pts, dts and len. Are all these needed? + } + videoData = { data: [], size: 0 }; + } + if (videoData) { + videoData.data.push(data.subarray(offset, start + 188)); + videoData.size += start + 188 - offset; } break; case audioId: - if (stt) { + if (pusi) { if (audioData && (pes = parsePES(audioData)) && pes.pts !== undefined) { - if (audioTrack.isAAC) { - parseAACPES(pes); - } else { - parseMPEGPES(pes); - } + audioTrack.data.push(pes.data); } audioData = { data: [], size: 0 }; } @@ -211,11 +163,10 @@ class TSDemuxer { } break; case id3Id: - if (stt) { + if (pusi) { if (id3Data && (pes = parsePES(id3Data)) && pes.pts !== undefined) { - parseID3PES(pes); + id3Track.data.push(pes.data); } - id3Data = { data: [], size: 0 }; } if (id3Data) { @@ -223,210 +174,88 @@ class TSDemuxer { id3Data.size += start + 188 - offset; } break; - case 0: - if (stt) { - offset += data[offset] + 1; - } - - pmtId = this._pmtId = parsePAT(data, offset); - break; - case pmtId: - if (stt) { - offset += data[offset] + 1; - } - - let parsedPIDs = parsePMT(data, offset, this.typeSupported.mpeg === true || this.typeSupported.mp3 === true, this.sampleAes != null); - - // only update track id if track PID found while parsing PMT - // this is to avoid resetting the PID to -1 in case - // track PID transiently disappears from the stream - // this could happen in case of transient missing audio samples for example - // NOTE this is only the PID of the track as found in TS, - // but we are not using this for MP4 track IDs. - avcId = parsedPIDs.avc; - if (avcId > 0) { - avcTrack.pid = avcId; - } - - audioId = parsedPIDs.audio; - if (audioId > 0) { - audioTrack.pid = audioId; - audioTrack.isAAC = parsedPIDs.isAAC; - } - id3Id = parsedPIDs.id3; - if (id3Id > 0) { - id3Track.pid = id3Id; - } - - if (unknownPIDs && !pmtParsed) { - logger.log('reparse from beginning'); - unknownPIDs = false; - // we set it to -188, the += 188 in the for loop will reset start to 0 - start = syncOffset - 188; - } - pmtParsed = this.pmtParsed = true; - break; - case 17: - case 0x1fff: - break; default: - unknownPIDs = true; + unknownPIDs = true; break; } } else { - this.observer.trigger(Event.ERROR, { type: ErrorTypes.MEDIA_ERROR, details: ErrorDetails.FRAG_PARSING_ERROR, fatal: false, reason: 'TS packet did not start with 0x47' }); + console.error('TS packet did not start with 0x47'); } } - // try to parse last PES packets - if (avcData && (pes = parsePES(avcData)) && pes.pts !== undefined) { - parseAVCPES(pes, true); - avcTrack.pesData = null; + + // Try to parse last PES packets. + if (videoData && (pes = parsePES(videoData)) && pes.pts !== undefined) { + videoTrack.data.push(pes.data); + videoTrack.pesData = null; } else { - // either avcData null or PES truncated, keep it for next frag parsing - avcTrack.pesData = avcData; + // Either pesPkts null or PES truncated, keep it for next frag parsing. + videoTrack.pesData = videoData; } - + if (audioData && (pes = parsePES(audioData)) && pes.pts !== undefined) { - if (audioTrack.isAAC) { - parseAACPES(pes); - } else { - parseMPEGPES(pes); - } - + audioTrack.data.push(pes.data); audioTrack.pesData = null; } else { - if (audioData && audioData.size) { - logger.log('last AAC PES packet truncated,might overlap between fragments'); - } - - // either audioData null or PES truncated, keep it for next frag parsing + // Either pesPkts null or PES truncated, keep it for next frag parsing. audioTrack.pesData = audioData; } - + if (id3Data && (pes = parsePES(id3Data)) && pes.pts !== undefined) { - parseID3PES(pes); + id3Track.data.push(pes.data); id3Track.pesData = null; } else { - // either id3Data null or PES truncated, keep it for next frag parsing + // Either pesPkts null or PES truncated, keep it for next frag parsing. id3Track.pesData = id3Data; } - - if (this.sampleAes == null) { - this.remuxer.remux(audioTrack, avcTrack, id3Track, this._txtTrack, timeOffset, contiguous, accurateTimeOffset); - } else { - this.decryptAndRemux(audioTrack, avcTrack, id3Track, this._txtTrack, timeOffset, contiguous, accurateTimeOffset); - } - } - - decryptAndRemux (audioTrack, videoTrack, id3Track, textTrack, timeOffset, contiguous, accurateTimeOffset) { - if (audioTrack.samples && audioTrack.isAAC) { - let localthis = this; - this.sampleAes.decryptAacSamples(audioTrack.samples, 0, function () { - localthis.decryptAndRemuxAvc(audioTrack, videoTrack, id3Track, textTrack, timeOffset, contiguous, accurateTimeOffset); - }); - } else { - this.decryptAndRemuxAvc(audioTrack, videoTrack, id3Track, textTrack, timeOffset, contiguous, accurateTimeOffset); - } - } - - decryptAndRemuxAvc (audioTrack, videoTrack, id3Track, textTrack, timeOffset, contiguous, accurateTimeOffset) { - if (videoTrack.samples) { - let localthis = this; - this.sampleAes.decryptAvcSamples(videoTrack.samples, 0, 0, function () { - localthis.remuxer.remux(audioTrack, videoTrack, id3Track, textTrack, timeOffset, contiguous, accurateTimeOffset); - }); - } else { - this.remuxer.remux(audioTrack, videoTrack, id3Track, textTrack, timeOffset, contiguous, accurateTimeOffset); - } - } - - destroy () { - this._initPTS = this._initDTS = undefined; - this._duration = 0; } _parsePAT (data, offset) { - // skip the PSI header and parse the first PMT entry + // Skip the PSI header and parse the first PMT entry. return (data[offset + 10] & 0x1F) << 8 | data[offset + 11]; - // logger.log('PMT PID:' + this._pmtId); + // console.log('PMT PID:' + this._pmtId); } - _parsePMT (data, offset, mpegSupported, isSampleAes) { - let sectionLength, tableEnd, programInfoLength, pid, result = { audio: -1, avc: -1, id3: -1, isAAC: true }; - sectionLength = (data[offset + 1] & 0x0f) << 8 | data[offset + 2]; - tableEnd = offset + 3 + sectionLength - 4; - // to determine where the table is, we have to figure out how - // long the program info descriptors are - programInfoLength = (data[offset + 10] & 0x0f) << 8 | data[offset + 11]; - // advance the offset to the first entry in the mapping table + _parsePMT (data, offset) { + let pid; + let result = { audio: -1, video: -1, id3: -1}; + let sectionLength = (data[offset + 1] & 0x0f) << 8 | data[offset + 2]; + let tableEnd = offset + 3 + sectionLength - 4; + // To determine where the table is, we have to figure out how long the program info descriptors are. + let programInfoLength = (data[offset + 10] & 0x0f) << 8 | data[offset + 11]; + // Advance the offset to the first entry in the mapping table. offset += 12 + programInfoLength; while (offset < tableEnd) { pid = (data[offset + 1] & 0x1F) << 8 | data[offset + 2]; switch (data[offset]) { - case 0xcf: // SAMPLE-AES AAC - if (!isSampleAes) { - logger.log('unknown stream type:' + data[offset]); - break; + case 0x1c: // MJPEG + case 0xdb: // SAMPLE-AES AVC. + case 0x1b: // ITU-T Rec. H.264 and ISO/IEC 14496-10 (lower bit-rate video). + if (result.video === -1) { + result.video = pid; } - /* falls through */ - - // ISO/IEC 13818-7 ADTS AAC (MPEG-2 lower bit-rate audio) - case 0x0f: - // logger.log('AAC PID:' + pid); + break; + case 0xcf: // SAMPLE-AES AAC. + case 0x0f: // ISO/IEC 13818-7 ADTS AAC (MPEG-2 lower bit-rate audio). + case 0xd2: // ADPCM audio. + case 0x03: // ISO/IEC 11172-3 (MPEG-1 audio). + case 0x24: + // console.warn('HEVC stream type found, not supported for now'); + case 0x04: // or ISO/IEC 13818-3 (MPEG-2 halved sample rate audio). if (result.audio === -1) { result.audio = pid; } - break; - - // Packetized metadata (ID3) - case 0x15: - // logger.log('ID3 PID:' + pid); + case 0x15: // Packetized metadata (ID3) + // console.log('ID3 PID:' + pid); if (result.id3 === -1) { result.id3 = pid; } - break; - - case 0xdb: // SAMPLE-AES AVC - if (!isSampleAes) { - logger.log('unknown stream type:' + data[offset]); - break; - } - /* falls through */ - - // ITU-T Rec. H.264 and ISO/IEC 14496-10 (lower bit-rate video) - case 0x1b: - // logger.log('AVC PID:' + pid); - if (result.avc === -1) { - result.avc = pid; - } - - break; - - // ISO/IEC 11172-3 (MPEG-1 audio) - // or ISO/IEC 13818-3 (MPEG-2 halved sample rate audio) - case 0x03: - case 0x04: - // logger.log('MPEG PID:' + pid); - if (!mpegSupported) { - logger.log('MPEG audio found, not supported in this browser for now'); - } else if (result.audio === -1) { - result.audio = pid; - result.isAAC = false; - } - break; - - case 0x24: - logger.warn('HEVC stream type found, not supported for now'); - break; - default: - logger.log('unknown stream type:' + data[offset]); + // console.log('unknown stream type:' + data[offset]); break; } - // move to the next table entry - // skip past the elementary stream descriptors, if present + // Move to the next table entry, skip past the elementary stream descriptors, if present. offset += ((data[offset + 3] & 0x0F) << 8 | data[offset + 4]) + 5; } return result; @@ -434,14 +263,14 @@ class TSDemuxer { _parsePES (stream) { let i = 0, frag, pesFlags, pesPrefix, pesLen, pesHdrLen, pesData, pesPts, pesDts, payloadStartOffset, data = stream.data; - // safety check + // Safety check. if (!stream || stream.size === 0) { return null; } - // we might need up to 19 bytes to read PES header - // if first chunk of data is less than 19 bytes, let's merge it with following ones until we get 19 bytes - // usually only one merge is needed (and this is rare ...) + // We might need up to 19 bytes to read PES header. + // If first chunk of data is less than 19 bytes, let's merge it with following ones until we get 19 bytes. + // Usually only one merge is needed (and this is rare ...). while (data[0].length < 19 && data.length > 1) { let newData = new Uint8Array(data[0].length + data[1].length); newData.set(data[0]); @@ -449,30 +278,30 @@ class TSDemuxer { data[0] = newData; data.splice(1, 1); } - // retrieve PTS/DTS from first fragment + // Retrieve PTS/DTS from first fragment. frag = data[0]; pesPrefix = (frag[0] << 16) + (frag[1] << 8) + frag[2]; if (pesPrefix === 1) { pesLen = (frag[4] << 8) + frag[5]; - // if PES parsed length is not zero and greater than total received length, stop parsing. PES might be truncated - // minus 6 : PES header size + // If PES parsed length is not zero and greater than total received length, stop parsing. PES might be truncated. + // Minus 6 : PES header size. if (pesLen && pesLen > stream.size - 6) { return null; } pesFlags = frag[7]; if (pesFlags & 0xC0) { - /* PES header described here : http://dvd.sourceforge.net/dvdinfo/pes-hdr.html - as PTS / DTS is 33 bit we cannot use bitwise operator in JS, - as Bitwise operators treat their operands as a sequence of 32 bits */ + // PES header described here : http://dvd.sourceforge.net/dvdinfo/pes-hdr.html + // As PTS / DTS is 33 bit we cannot use bitwise operator in JS, + // as Bitwise operators treat their operands as a sequence of 32 bits. pesPts = (frag[9] & 0x0E) * 536870912 +// 1 << 29 (frag[10] & 0xFF) * 4194304 +// 1 << 22 (frag[11] & 0xFE) * 16384 +// 1 << 14 (frag[12] & 0xFF) * 128 +// 1 << 7 (frag[13] & 0xFE) / 2; - // check if greater than 2^32 -1 + // Check if greater than 2^32 -1. if (pesPts > 4294967295) { - // decrement 2^33 + // Decrement 2^33. pesPts -= 8589934592; } if (pesFlags & 0x40) { @@ -481,13 +310,13 @@ class TSDemuxer { (frag[16] & 0xFE) * 16384 +// 1 << 14 (frag[17] & 0xFF) * 128 +// 1 << 7 (frag[18] & 0xFE) / 2; - // check if greater than 2^32 -1 + // Check if greater than 2^32 -1. if (pesDts > 4294967295) { - // decrement 2^33 + // Decrement 2^33. pesDts -= 8589934592; } if (pesPts - pesDts > 60 * 90000) { - logger.warn(`${Math.round((pesPts - pesDts) / 90000)}s delta between PTS and DTS, align them`); + // console.warn(`${Math.round((pesPts - pesDts) / 90000)}s delta between PTS and DTS, align them`); pesPts = pesDts; } } else { @@ -495,22 +324,22 @@ class TSDemuxer { } } pesHdrLen = frag[8]; - // 9 bytes : 6 bytes for PES header + 3 bytes for PES extension + // 9 bytes : 6 bytes for PES header + 3 bytes for PES extension. payloadStartOffset = pesHdrLen + 9; stream.size -= payloadStartOffset; - // reassemble PES packet + // Reassemble PES packet. pesData = new Uint8Array(stream.size); for (let j = 0, dataLen = data.length; j < dataLen; j++) { frag = data[j]; let len = frag.byteLength; if (payloadStartOffset) { if (payloadStartOffset > len) { - // trim full frag if PES header bigger than frag + // Trim full frag if PES header bigger than frag. payloadStartOffset -= len; continue; } else { - // trim partial frag if PES header smaller than frag + // Trim partial frag if PES header smaller than frag. frag = frag.subarray(payloadStartOffset); len -= payloadStartOffset; payloadStartOffset = 0; @@ -520,7 +349,7 @@ class TSDemuxer { i += len; } if (pesLen) { - // payload size : remove PES header + PES extension + // Payload size : remove PES header + PES extension. pesLen -= pesHdrLen + 3; } return { data: pesData, pts: pesPts, dts: pesDts, len: pesLen }; @@ -528,554 +357,4 @@ class TSDemuxer { return null; } } - - pushAccesUnit (avcSample, avcTrack) { - if (avcSample.units.length && avcSample.frame) { - const samples = avcTrack.samples; - const nbSamples = samples.length; - // only push AVC sample if starting with a keyframe is not mandatory OR - // if keyframe already found in this fragment OR - // keyframe found in last fragment (track.sps) AND - // samples already appended (we already found a keyframe in this fragment) OR fragment is contiguous - if (!this.config.forceKeyFrameOnDiscontinuity || - avcSample.key === true || - (avcTrack.sps && (nbSamples || this.contiguous))) { - avcSample.id = nbSamples; - samples.push(avcSample); - } else { - // dropped samples, track it - avcTrack.dropped++; - } - } - if (avcSample.debug.length) { - logger.log(avcSample.pts + '/' + avcSample.dts + ':' + avcSample.debug); - } - } - - _parseAVCPES (pes, last) { - // logger.log('parse new PES'); - let track = this._avcTrack, - units = this._parseAVCNALu(pes.data), - debug = false, - expGolombDecoder, - avcSample = this.avcSample, - push, - spsfound = false, - i, - pushAccesUnit = this.pushAccesUnit.bind(this), - createAVCSample = function (key, pts, dts, debug) { - return { key: key, pts: pts, dts: dts, units: [], debug: debug }; - }; - // free pes.data to save up some memory - pes.data = null; - - // if new NAL units found and last sample still there, let's push ... - // this helps parsing streams with missing AUD (only do this if AUD never found) - if (avcSample && units.length && !track.audFound) { - pushAccesUnit(avcSample, track); - avcSample = this.avcSample = createAVCSample(false, pes.pts, pes.dts, ''); - } - - units.forEach(unit => { - switch (unit.type) { - // NDR - case 1: - push = true; - if (!avcSample) { - avcSample = this.avcSample = createAVCSample(true, pes.pts, pes.dts, ''); - } - - if (debug) { - avcSample.debug += 'NDR '; - } - - avcSample.frame = true; - let data = unit.data; - // only check slice type to detect KF in case SPS found in same packet (any keyframe is preceded by SPS ...) - if (spsfound && data.length > 4) { - // retrieve slice type by parsing beginning of NAL unit (follow H264 spec, slice_header definition) to detect keyframe embedded in NDR - let sliceType = new ExpGolomb(data).readSliceType(); - // 2 : I slice, 4 : SI slice, 7 : I slice, 9: SI slice - // SI slice : A slice that is coded using intra prediction only and using quantisation of the prediction samples. - // An SI slice can be coded such that its decoded samples can be constructed identically to an SP slice. - // I slice: A slice that is not an SI slice that is decoded using intra prediction only. - // if (sliceType === 2 || sliceType === 7) { - if (sliceType === 2 || sliceType === 4 || sliceType === 7 || sliceType === 9) { - avcSample.key = true; - } - } - break; - // IDR - case 5: - push = true; - // handle PES not starting with AUD - if (!avcSample) { - avcSample = this.avcSample = createAVCSample(true, pes.pts, pes.dts, ''); - } - - if (debug) { - avcSample.debug += 'IDR '; - } - - avcSample.key = true; - avcSample.frame = true; - break; - // SEI - case 6: - push = true; - if (debug && avcSample) { - avcSample.debug += 'SEI '; - } - - expGolombDecoder = new ExpGolomb(this.discardEPB(unit.data)); - - // skip frameType - expGolombDecoder.readUByte(); - - var payloadType = 0; - var payloadSize = 0; - var endOfCaptions = false; - var b = 0; - - while (!endOfCaptions && expGolombDecoder.bytesAvailable > 1) { - payloadType = 0; - do { - b = expGolombDecoder.readUByte(); - payloadType += b; - } while (b === 0xFF); - - // Parse payload size. - payloadSize = 0; - do { - b = expGolombDecoder.readUByte(); - payloadSize += b; - } while (b === 0xFF); - - // TODO: there can be more than one payload in an SEI packet... - // TODO: need to read type and size in a while loop to get them all - if (payloadType === 4 && expGolombDecoder.bytesAvailable !== 0) { - endOfCaptions = true; - - let countryCode = expGolombDecoder.readUByte(); - - if (countryCode === 181) { - let providerCode = expGolombDecoder.readUShort(); - - if (providerCode === 49) { - let userStructure = expGolombDecoder.readUInt(); - - if (userStructure === 0x47413934) { - let userDataType = expGolombDecoder.readUByte(); - - // Raw CEA-608 bytes wrapped in CEA-708 packet - if (userDataType === 3) { - let firstByte = expGolombDecoder.readUByte(); - let secondByte = expGolombDecoder.readUByte(); - - let totalCCs = 31 & firstByte; - let byteArray = [firstByte, secondByte]; - - for (i = 0; i < totalCCs; i++) { - // 3 bytes per CC - byteArray.push(expGolombDecoder.readUByte()); - byteArray.push(expGolombDecoder.readUByte()); - byteArray.push(expGolombDecoder.readUByte()); - } - - this._insertSampleInOrder(this._txtTrack.samples, { type: 3, pts: pes.pts, bytes: byteArray }); - } - } - } - } - } else if (payloadType === 5 && expGolombDecoder.bytesAvailable !== 0) { - endOfCaptions = true; - - if (payloadSize > 16) { - let uuidStrArray = []; - let userDataPayloadBytes = []; - - for (i = 0; i < 16; i++) { - uuidStrArray.push(expGolombDecoder.readUByte().toString(16)); - - if (i === 3 || i === 5 || i === 7 || i === 9) { - uuidStrArray.push('-'); - } - } - - for (i = 16; i < payloadSize; i++) { - userDataPayloadBytes.push(expGolombDecoder.readUByte()); - } - - this._insertSampleInOrder(this._txtTrack.samples, { - pts: pes.pts, - payloadType: payloadType, - uuid: uuidStrArray.join(''), - userData: String.fromCharCode.apply(null, userDataPayloadBytes), - userDataBytes: userDataPayloadBytes - }); - } - } else if (payloadSize < expGolombDecoder.bytesAvailable) { - for (i = 0; i < payloadSize; i++) { - expGolombDecoder.readUByte(); - } - } - } - break; - // SPS - case 7: - push = true; - spsfound = true; - if (debug && avcSample) { - avcSample.debug += 'SPS '; - } - - if (!track.sps) { - expGolombDecoder = new ExpGolomb(unit.data); - let config = expGolombDecoder.readSPS(); - track.width = config.width; - track.height = config.height; - track.pixelRatio = config.pixelRatio; - track.sps = [unit.data]; - track.duration = this._duration; - let codecarray = unit.data.subarray(1, 4); - let codecstring = 'avc1.'; - for (i = 0; i < 3; i++) { - let h = codecarray[i].toString(16); - if (h.length < 2) { - h = '0' + h; - } - - codecstring += h; - } - track.codec = codecstring; - } - break; - // PPS - case 8: - push = true; - if (debug && avcSample) { - avcSample.debug += 'PPS '; - } - - if (!track.pps) { - track.pps = [unit.data]; - } - - break; - // AUD - case 9: - push = false; - track.audFound = true; - if (avcSample) { - pushAccesUnit(avcSample, track); - } - - avcSample = this.avcSample = createAVCSample(false, pes.pts, pes.dts, debug ? 'AUD ' : ''); - break; - // Filler Data - case 12: - push = false; - break; - default: - push = false; - if (avcSample) { - avcSample.debug += 'unknown NAL ' + unit.type + ' '; - } - - break; - } - if (avcSample && push) { - let units = avcSample.units; - units.push(unit); - } - }); - // if last PES packet, push samples - if (last && avcSample) { - pushAccesUnit(avcSample, track); - this.avcSample = null; - } - } - - _insertSampleInOrder (arr, data) { - let len = arr.length; - if (len > 0) { - if (data.pts >= arr[len - 1].pts) { - arr.push(data); - } else { - for (let pos = len - 1; pos >= 0; pos--) { - if (data.pts < arr[pos].pts) { - arr.splice(pos, 0, data); - break; - } - } - } - } else { - arr.push(data); - } - } - - _getLastNalUnit () { - let avcSample = this.avcSample, lastUnit; - // try to fallback to previous sample if current one is empty - if (!avcSample || avcSample.units.length === 0) { - let track = this._avcTrack, samples = track.samples; - avcSample = samples[samples.length - 1]; - } - if (avcSample) { - let units = avcSample.units; - lastUnit = units[units.length - 1]; - } - return lastUnit; - } - - _parseAVCNALu (array) { - let i = 0, len = array.byteLength, value, overflow, track = this._avcTrack, state = track.naluState || 0, lastState = state; - let units = [], unit, unitType, lastUnitStart = -1, lastUnitType; - // logger.log('PES:' + Hex.hexDump(array)); - - if (state === -1) { - // special use case where we found 3 or 4-byte start codes exactly at the end of previous PES packet - lastUnitStart = 0; - // NALu type is value read from offset 0 - lastUnitType = array[0] & 0x1f; - state = 0; - i = 1; - } - - while (i < len) { - value = array[i++]; - // optimization. state 0 and 1 are the predominant case. let's handle them outside of the switch/case - if (!state) { - state = value ? 0 : 1; - continue; - } - if (state === 1) { - state = value ? 0 : 2; - continue; - } - // here we have state either equal to 2 or 3 - if (!value) { - state = 3; - } else if (value === 1) { - if (lastUnitStart >= 0) { - unit = { data: array.subarray(lastUnitStart, i - state - 1), type: lastUnitType }; - // logger.log('pushing NALU, type/size:' + unit.type + '/' + unit.data.byteLength); - units.push(unit); - } else { - // lastUnitStart is undefined => this is the first start code found in this PES packet - // first check if start code delimiter is overlapping between 2 PES packets, - // ie it started in last packet (lastState not zero) - // and ended at the beginning of this PES packet (i <= 4 - lastState) - let lastUnit = this._getLastNalUnit(); - if (lastUnit) { - if (lastState && (i <= 4 - lastState)) { - // start delimiter overlapping between PES packets - // strip start delimiter bytes from the end of last NAL unit - // check if lastUnit had a state different from zero - if (lastUnit.state) { - // strip last bytes - lastUnit.data = lastUnit.data.subarray(0, lastUnit.data.byteLength - lastState); - } - } - // If NAL units are not starting right at the beginning of the PES packet, push preceding data into previous NAL unit. - overflow = i - state - 1; - if (overflow > 0) { - // logger.log('first NALU found with overflow:' + overflow); - let tmp = new Uint8Array(lastUnit.data.byteLength + overflow); - tmp.set(lastUnit.data, 0); - tmp.set(array.subarray(0, overflow), lastUnit.data.byteLength); - lastUnit.data = tmp; - } - } - } - // check if we can read unit type - if (i < len) { - unitType = array[i] & 0x1f; - // logger.log('find NALU @ offset:' + i + ',type:' + unitType); - lastUnitStart = i; - lastUnitType = unitType; - state = 0; - } else { - // not enough byte to read unit type. let's read it on next PES parsing - state = -1; - } - } else { - state = 0; - } - } - if (lastUnitStart >= 0 && state >= 0) { - unit = { data: array.subarray(lastUnitStart, len), type: lastUnitType, state: state }; - units.push(unit); - // logger.log('pushing NALU, type/size/state:' + unit.type + '/' + unit.data.byteLength + '/' + state); - } - // no NALu found - if (units.length === 0) { - // append pes.data to previous NAL unit - let lastUnit = this._getLastNalUnit(); - if (lastUnit) { - let tmp = new Uint8Array(lastUnit.data.byteLength + array.byteLength); - tmp.set(lastUnit.data, 0); - tmp.set(array, lastUnit.data.byteLength); - lastUnit.data = tmp; - } - } - track.naluState = state; - return units; - } - - /** - * remove Emulation Prevention bytes from a RBSP - */ - discardEPB (data) { - let length = data.byteLength, - EPBPositions = [], - i = 1, - newLength, newData; - - // Find all `Emulation Prevention Bytes` - while (i < length - 2) { - if (data[i] === 0 && - data[i + 1] === 0 && - data[i + 2] === 0x03) { - EPBPositions.push(i + 2); - i += 2; - } else { - i++; - } - } - - // If no Emulation Prevention Bytes were found just return the original - // array - if (EPBPositions.length === 0) { - return data; - } - - // Create a new array to hold the NAL unit data - newLength = length - EPBPositions.length; - newData = new Uint8Array(newLength); - let sourceIndex = 0; - - for (i = 0; i < newLength; sourceIndex++, i++) { - if (sourceIndex === EPBPositions[0]) { - // Skip this byte - sourceIndex++; - // Remove this position index - EPBPositions.shift(); - } - newData[i] = data[sourceIndex]; - } - return newData; - } - - _parseAACPES (pes) { - let track = this._audioTrack, - data = pes.data, - pts = pes.pts, - startOffset = 0, - aacOverFlow = this.aacOverFlow, - aacLastPTS = this.aacLastPTS, - frameDuration, frameIndex, offset, stamp, len; - if (aacOverFlow) { - let tmp = new Uint8Array(aacOverFlow.byteLength + data.byteLength); - tmp.set(aacOverFlow, 0); - tmp.set(data, aacOverFlow.byteLength); - // logger.log(`AAC: append overflowing ${aacOverFlow.byteLength} bytes to beginning of new PES`); - data = tmp; - } - // look for ADTS header (0xFFFx) - for (offset = startOffset, len = data.length; offset < len - 1; offset++) { - if (ADTS.isHeader(data, offset)) { - break; - } - } - // if ADTS header does not start straight from the beginning of the PES payload, raise an error - if (offset) { - let reason, fatal; - if (offset < len - 1) { - reason = `AAC PES did not start with ADTS header,offset:${offset}`; - fatal = false; - } else { - reason = 'no ADTS header found in AAC PES'; - fatal = true; - } - logger.warn(`parsing error:${reason}`); - this.observer.trigger(Event.ERROR, { type: ErrorTypes.MEDIA_ERROR, details: ErrorDetails.FRAG_PARSING_ERROR, fatal: fatal, reason: reason }); - if (fatal) { - return; - } - } - - ADTS.initTrackConfig(track, this.observer, data, offset, this.audioCodec); - frameIndex = 0; - frameDuration = ADTS.getFrameDuration(track.samplerate); - - // if last AAC frame is overflowing, we should ensure timestamps are contiguous: - // first sample PTS should be equal to last sample PTS + frameDuration - if (aacOverFlow && aacLastPTS) { - let newPTS = aacLastPTS + frameDuration; - if (Math.abs(newPTS - pts) > 1) { - logger.log(`AAC: align PTS for overlapping frames by ${Math.round((newPTS - pts) / 90)}`); - pts = newPTS; - } - } - - // scan for aac samples - while (offset < len) { - if (ADTS.isHeader(data, offset) && (offset + 5) < len) { - let frame = ADTS.appendFrame(track, data, offset, pts, frameIndex); - if (frame) { - // logger.log(`${Math.round(frame.sample.pts)} : AAC`); - offset += frame.length; - stamp = frame.sample.pts; - frameIndex++; - } else { - // logger.log('Unable to parse AAC frame'); - break; - } - } else { - // nothing found, keep looking - offset++; - } - } - - if (offset < len) { - aacOverFlow = data.subarray(offset, len); - // logger.log(`AAC: overflow detected:${len-offset}`); - } else { - aacOverFlow = null; - } - - this.aacOverFlow = aacOverFlow; - this.aacLastPTS = stamp; - } - - _parseMPEGPES (pes) { - let data = pes.data; - let length = data.length; - let frameIndex = 0; - let offset = 0; - let pts = pes.pts; - - while (offset < length) { - if (MpegAudio.isHeader(data, offset)) { - let frame = MpegAudio.appendFrame(this._audioTrack, data, offset, pts, frameIndex); - if (frame) { - offset += frame.length; - frameIndex++; - } else { - // logger.log('Unable to parse Mpeg audio frame'); - break; - } - } else { - // nothing found, keep looking - offset++; - } - } - } - - _parseID3PES (pes) { - this._id3Track.samples.push(pes); - } -} - -export default TSDemuxer; +} \ No newline at end of file