diff --git a/cmd/mjpeg-player/hlsjs/config.js b/cmd/mjpeg-player/hlsjs/config.js index efc8e27c..94797f6b 100644 --- a/cmd/mjpeg-player/hlsjs/config.js +++ b/cmd/mjpeg-player/hlsjs/config.js @@ -1,195 +1,33 @@ -/** - * HLS config - */ +/* +AUTHOR + Trek Hopton -import AbrController from './controller/abr-controller'; -import BufferController from './controller/buffer-controller'; -import CapLevelController from './controller/cap-level-controller'; -import FPSController from './controller/fps-controller'; -import XhrLoader from './utils/xhr-loader'; -// import FetchLoader from './utils/fetch-loader'; +LICENSE + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) -import AudioTrackController from './controller/audio-track-controller'; -import AudioStreamController from './controller/audio-stream-controller'; + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. -import * as Cues from './utils/cues'; -import TimelineController from './controller/timeline-controller'; -import SubtitleTrackController from './controller/subtitle-track-controller'; -import { SubtitleStreamController } from './controller/subtitle-stream-controller'; -import EMEController from './controller/eme-controller'; -import { requestMediaKeySystemAccess, MediaKeyFunc } from './utils/mediakeys-helper'; + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. -type ABRControllerConfig = { - abrEwmaFastLive: number, - abrEwmaSlowLive: number, - abrEwmaFastVoD: number, - abrEwmaSlowVoD: number, - abrEwmaDefaultEstimate: number, - abrBandWidthFactor: number, - abrBandWidthUpFactor: number, - abrMaxWithRealBitrate: boolean, - maxStarvationDelay: number, - maxLoadingDelay: number, -}; + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see http://www.gnu.org/licenses. -export type BufferControllerConfig = { - appendErrorMaxRetry: number, - liveDurationInfinity: boolean, - liveBackBufferLength: number, -}; + For hls.js Copyright notice and license, see LICENSE file. +*/ -type CapLevelControllerConfig = { - capLevelToPlayerSize: boolean -}; - -export type EMEControllerConfig = { - licenseXhrSetup?: (xhr: XMLHttpRequest, url: string) => void, - emeEnabled: boolean, - widevineLicenseUrl?: string, - requestMediaKeySystemAccessFunc: MediaKeyFunc | null, -}; - -type FragmentLoaderConfig = { - fLoader: any, // TODO(typescript-loader): Once Loader is typed fill this in - - fragLoadingTimeOut: number, - fragLoadingMaxRetry: number, - fragLoadingRetryDelay: number, - fragLoadingMaxRetryTimeout: number, -}; - -type FPSControllerConfig = { - capLevelOnFPSDrop: boolean, - fpsDroppedMonitoringPeriod: number, - fpsDroppedMonitoringThreshold: number, -}; - -type LevelControllerConfig = { - startLevel?: number -}; - -type MP4RemuxerConfig = { - stretchShortVideoTrack: boolean, - maxAudioFramesDrift: number, -}; - -type PlaylistLoaderConfig = { - pLoader: any, // TODO(typescript-loader): Once Loader is typed fill this in - - manifestLoadingTimeOut: number, - manifestLoadingMaxRetry: number, - manifestLoadingRetryDelay: number, - manifestLoadingMaxRetryTimeout: number, - - levelLoadingTimeOut: number, - levelLoadingMaxRetry: number, - levelLoadingRetryDelay: number, - levelLoadingMaxRetryTimeout: number -}; - -type StreamControllerConfig = { - autoStartLoad: boolean, - startPosition: number, - defaultAudioCodec?: string, - initialLiveManifestSize: number, - maxBufferLength: number, - maxBufferSize: number, - maxBufferHole: number, - - lowBufferWatchdogPeriod: number, - highBufferWatchdogPeriod: number, - nudgeOffset: number, - nudgeMaxRetry: number, - maxFragLookUpTolerance: number, - liveSyncDurationCount: number, - liveMaxLatencyDurationCount: number, - liveSyncDuration?: number, - liveMaxLatencyDuration?: number, - maxMaxBufferLength: number, - - startFragPrefetch: boolean, -}; - -type TimelineControllerConfig = { - cueHandler: any, // TODO(typescript-cues): Type once file is done - enableCEA708Captions: boolean, - enableWebVTT: boolean, - captionsTextTrack1Label: string, - captionsTextTrack1LanguageCode: string, - captionsTextTrack2Label: string, - captionsTextTrack2LanguageCode: string, -}; - -type TSDemuxerConfig = { - forceKeyFrameOnDiscontinuity: boolean, -}; - -export type HlsConfig = - { - debug: boolean, - enableWorker: boolean, - enableSoftwareAES: boolean, - minAutoBitrate: number, - loader: any, // TODO(typescript-xhrloader): Type once XHR is done - xhrSetup?: (xhr: XMLHttpRequest, url: string) => void, - - // Alt Audio - audioStreamController?: any, // TODO(typescript-audiostreamcontroller): Type once file is done - audioTrackController?: any, // TODO(typescript-audiotrackcontroller): Type once file is done - // Subtitle - subtitleStreamController?: any, // TODO(typescript-subtitlestreamcontroller): Type once file is done - subtitleTrackController?: any, // TODO(typescript-subtitletrackcontroller): Type once file is done - timelineController?: any, // TODO(typescript-timelinecontroller): Type once file is done - // EME - emeController?: typeof EMEController, - - abrController: any, // TODO(typescript-abrcontroller): Type once file is done - bufferController: typeof BufferController, - capLevelController: any, // TODO(typescript-caplevelcontroller): Type once file is done - fpsController: any, // TODO(typescript-fpscontroller): Type once file is done - } & - ABRControllerConfig & - BufferControllerConfig & - CapLevelControllerConfig & - EMEControllerConfig & - FPSControllerConfig & - FragmentLoaderConfig & - LevelControllerConfig & - MP4RemuxerConfig & - PlaylistLoaderConfig & - StreamControllerConfig & - Partial & - TSDemuxerConfig; +import XhrLoader from './utils/xhr-loader.js'; // If possible, keep hlsDefaultConfig shallow // It is cloned whenever a new Hls instance is created, by keeping the config // shallow the properties are cloned, and we don't end up manipulating the default -export const hlsDefaultConfig: HlsConfig = { - autoStartLoad: true, // used by stream-controller +export const hlsDefaultConfig = { startPosition: -1, // used by stream-controller - defaultAudioCodec: void 0, // used by stream-controller - debug: false, // used by logger - capLevelOnFPSDrop: false, // used by fps-controller - capLevelToPlayerSize: false, // used by cap-level-controller - initialLiveManifestSize: 1, // used by stream-controller - maxBufferLength: 30, // used by stream-controller - maxBufferSize: 60 * 1000 * 1000, // used by stream-controller - maxBufferHole: 0.5, // used by stream-controller - - lowBufferWatchdogPeriod: 0.5, // used by stream-controller - highBufferWatchdogPeriod: 3, // used by stream-controller - nudgeOffset: 0.1, // used by stream-controller - nudgeMaxRetry: 3, // used by stream-controller - maxFragLookUpTolerance: 0.25, // used by stream-controller - liveSyncDurationCount: 3, // used by stream-controller - liveMaxLatencyDurationCount: Infinity, // used by stream-controller - liveSyncDuration: void 0, // used by stream-controller - liveMaxLatencyDuration: void 0, // used by stream-controller - liveDurationInfinity: false, // used by buffer-controller - liveBackBufferLength: Infinity, // used by buffer-controller - maxMaxBufferLength: 600, // used by stream-controller - enableWorker: true, // used by demuxer - enableSoftwareAES: true, // used by decrypter manifestLoadingTimeOut: 10000, // used by playlist-loader manifestLoadingMaxRetry: 1, // used by playlist-loader manifestLoadingRetryDelay: 1000, // used by playlist-loader @@ -203,62 +41,8 @@ export const hlsDefaultConfig: HlsConfig = { fragLoadingMaxRetry: 6, // used by fragment-loader fragLoadingRetryDelay: 1000, // used by fragment-loader fragLoadingMaxRetryTimeout: 64000, // used by fragment-loader - startFragPrefetch: false, // used by stream-controller - fpsDroppedMonitoringPeriod: 5000, // used by fps-controller - fpsDroppedMonitoringThreshold: 0.2, // used by fps-controller - appendErrorMaxRetry: 3, // used by buffer-controller loader: XhrLoader, - // loader: FetchLoader, fLoader: void 0, // used by fragment-loader pLoader: void 0, // used by playlist-loader xhrSetup: void 0, // used by xhr-loader - licenseXhrSetup: void 0, // used by eme-controller - // fetchSetup: void 0, - abrController: AbrController, - bufferController: BufferController, - capLevelController: CapLevelController, - fpsController: FPSController, - stretchShortVideoTrack: false, // used by mp4-remuxer - maxAudioFramesDrift: 1, // used by mp4-remuxer - forceKeyFrameOnDiscontinuity: true, // used by ts-demuxer - abrEwmaFastLive: 3, // used by abr-controller - abrEwmaSlowLive: 9, // used by abr-controller - abrEwmaFastVoD: 3, // used by abr-controller - abrEwmaSlowVoD: 9, // used by abr-controller - abrEwmaDefaultEstimate: 5e5, // 500 kbps // used by abr-controller - abrBandWidthFactor: 0.95, // used by abr-controller - abrBandWidthUpFactor: 0.7, // used by abr-controller - abrMaxWithRealBitrate: false, // used by abr-controller - maxStarvationDelay: 4, // used by abr-controller - maxLoadingDelay: 4, // used by abr-controller - minAutoBitrate: 0, // used by hls - emeEnabled: false, // used by eme-controller - widevineLicenseUrl: void 0, // used by eme-controller - requestMediaKeySystemAccessFunc: requestMediaKeySystemAccess, // used by eme-controller - - // Dynamic Modules - ...timelineConfig(), - subtitleStreamController: (__USE_SUBTITLES__) ? SubtitleStreamController : void 0, - subtitleTrackController: (__USE_SUBTITLES__) ? SubtitleTrackController : void 0, - timelineController: (__USE_SUBTITLES__) ? TimelineController : void 0, - audioStreamController: (__USE_ALT_AUDIO__) ? AudioStreamController : void 0, - audioTrackController: (__USE_ALT_AUDIO__) ? AudioTrackController : void 0, - emeController: (__USE_EME_DRM__) ? EMEController : void 0 -}; - -function timelineConfig (): TimelineControllerConfig { - if (!__USE_SUBTITLES__) { - // intentionally doing this over returning Partial above - // this has the added nice property of still requiring the object below to completely define all props. - return {} as any; - } - return { - cueHandler: Cues, // used by timeline-controller - enableCEA708Captions: true, // used by timeline-controller - enableWebVTT: true, // used by timeline-controller - captionsTextTrack1Label: 'English', // used by timeline-controller - captionsTextTrack1LanguageCode: 'en', // used by timeline-controller - captionsTextTrack2Label: 'Spanish', // used by timeline-controller - captionsTextTrack2LanguageCode: 'es' // used by timeline-controller - }; -} +}; \ No newline at end of file diff --git a/cmd/mjpeg-player/hlsjs/controller/stream-controller.js b/cmd/mjpeg-player/hlsjs/controller/stream-controller.js index 092a55bc..31fd0828 100644 --- a/cmd/mjpeg-player/hlsjs/controller/stream-controller.js +++ b/cmd/mjpeg-player/hlsjs/controller/stream-controller.js @@ -1,801 +1,93 @@ +/* +AUTHOR + Trek Hopton + +LICENSE + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see http://www.gnu.org/licenses. + + For hls.js Copyright notice and license, see LICENSE file. +*/ + /* * Stream Controller */ -import BinarySearch from '../utils/binary-search'; -import { BufferHelper } from '../utils/buffer-helper'; -import Demuxer from '../demux/demuxer'; -import Event from '../events'; -import { FragmentState } from './fragment-tracker'; -import { ElementaryStreamTypes } from '../loader/fragment'; -import { PlaylistLevelType } from '../types/loader'; -import * as LevelHelper from './level-helper'; -import TimeRanges from '../utils/time-ranges'; -import { ErrorDetails } from '../errors'; -import { logger } from '../utils/logger'; -import { alignStream } from '../utils/discontinuities'; -import { findFragmentByPDT, findFragmentByPTS } from './fragment-finders'; -import GapController from './gap-controller'; -import BaseStreamController, { State } from './base-stream-controller'; +import Event from '../events.js'; +import EventHandler from '../event-handler.js'; -const TICK_INTERVAL = 100; // how often to tick in ms - -class StreamController extends BaseStreamController { - constructor (hls, fragmentTracker) { +class StreamController extends EventHandler { + constructor(hls) { super(hls, - Event.MEDIA_ATTACHED, - Event.MEDIA_DETACHING, - Event.MANIFEST_LOADING, - Event.MANIFEST_PARSED, Event.LEVEL_LOADED, - Event.KEY_LOADED, - Event.FRAG_LOADED, - Event.FRAG_LOAD_EMERGENCY_ABORTED, - Event.FRAG_PARSING_INIT_SEGMENT, - Event.FRAG_PARSING_DATA, - Event.FRAG_PARSED, - Event.ERROR, - Event.AUDIO_TRACK_SWITCHING, - Event.AUDIO_TRACK_SWITCHED, - Event.BUFFER_CREATED, - Event.BUFFER_APPENDED, - Event.BUFFER_FLUSHED); - - this.fragmentTracker = fragmentTracker; + Event.FRAG_LOADED); + this.hls = hls; this.config = hls.config; this.audioCodecSwap = false; - this._state = State.STOPPED; this.stallReported = false; this.gapController = null; + this.currentFrag = 0; } - startLoad (startPosition) { - if (this.levels) { - let lastCurrentTime = this.lastCurrentTime, hls = this.hls; - this.stopLoad(); - this.setInterval(TICK_INTERVAL); - this.level = -1; - this.fragLoadError = 0; - if (!this.startFragRequested) { - // determine load level - let startLevel = hls.startLevel; - if (startLevel === -1) { - // -1 : guess start Level by doing a bitrate test by loading first fragment of lowest quality level - startLevel = 0; - this.bitrateTest = true; - } - // set new level to playlist loader : this will trigger start level load - // hls.nextLoadLevel remains until it is set to a new value or until a new frag is successfully loaded - this.level = hls.nextLoadLevel = startLevel; - this.loadedmetadata = false; - } - // if startPosition undefined but lastCurrentTime set, set startPosition to last currentTime - if (lastCurrentTime > 0 && startPosition === -1) { - logger.log(`override startPosition with lastCurrentTime @${lastCurrentTime.toFixed(3)}`); - startPosition = lastCurrentTime; - } - this.state = State.IDLE; - this.nextLoadPosition = this.startPosition = this.lastCurrentTime = startPosition; - this.tick(); - } else { - this.forceStartLoad = true; - this.state = State.STOPPED; - } + _fetchPayloadOrEos(levelDetails) { + this.fragments = levelDetails.fragments; + this._loadFragment(); } - stopLoad () { - this.forceStartLoad = false; - super.stopLoad(); - } - - doTick () { - switch (this.state) { - case State.BUFFER_FLUSHING: - // in buffer flushing state, reset fragLoadError counter - this.fragLoadError = 0; - break; - case State.IDLE: - this._doTickIdle(); - break; - case State.WAITING_LEVEL: - var level = this.levels[this.level]; - // check if playlist is already loaded - if (level && level.details) { - this.state = State.IDLE; - } - - break; - case State.FRAG_LOADING_WAITING_RETRY: - var now = window.performance.now(); - var retryDate = this.retryDate; - // if current time is gt than retryDate, or if media seeking let's switch to IDLE state to retry loading - if (!retryDate || (now >= retryDate) || (this.media && this.media.seeking)) { - logger.log('mediaController: retryDate reached, switch back to IDLE state'); - this.state = State.IDLE; - } - break; - case State.ERROR: - case State.STOPPED: - case State.FRAG_LOADING: - case State.PARSING: - case State.PARSED: - case State.ENDED: - break; - default: - break; - } - // check buffer - this._checkBuffer(); - // check/update current fragment - this._checkFragmentChanged(); - } - - // Ironically the "idle" state is the on we do the most logic in it seems .... - // NOTE: Maybe we could rather schedule a check for buffer length after half of the currently - // played segment, or on pause/play/seek instead of naively checking every 100ms? - _doTickIdle () { - const hls = this.hls, - config = hls.config, - media = this.media; - - // if start level not parsed yet OR - // if video not attached AND start fragment already requested OR start frag prefetch disable - // exit loop, as we either need more info (level not parsed) or we need media to be attached to load new fragment - if (this.levelLastLoaded === undefined || ( - !media && (this.startFragRequested || !config.startFragPrefetch))) { + _loadFragment() { + let fragLen = this.fragments.length; + if (this.currentFrag >= fragLen) { return; } - - // if we have not yet loaded any fragment, start loading from start position - let pos; - if (this.loadedmetadata) { - pos = media.currentTime; - } else { - pos = this.nextLoadPosition; - } - - // determine next load level - let level = hls.nextLoadLevel, - levelInfo = this.levels[level]; - - if (!levelInfo) { - return; - } - - let levelBitrate = levelInfo.bitrate, - maxBufLen; - - // compute max Buffer Length that we could get from this load level, based on level bitrate. - if (levelBitrate) { - maxBufLen = Math.max(8 * config.maxBufferSize / levelBitrate, config.maxBufferLength); - } else { - maxBufLen = config.maxBufferLength; - } - - maxBufLen = Math.min(maxBufLen, config.maxMaxBufferLength); - - // determine next candidate fragment to be loaded, based on current position and end of buffer position - // ensure up to `config.maxMaxBufferLength` of buffer upfront - - const bufferInfo = BufferHelper.bufferInfo(this.mediaBuffer ? this.mediaBuffer : media, pos, config.maxBufferHole), - bufferLen = bufferInfo.len; - // Stay idle if we are still with buffer margins - if (bufferLen >= maxBufLen) { - return; - } - - // if buffer length is less than maxBufLen try to load a new fragment ... - logger.trace(`buffer length of ${bufferLen.toFixed(3)} is below max of ${maxBufLen.toFixed(3)}. checking for more payload ...`); - - // set next load level : this will trigger a playlist load if needed - this.level = hls.nextLoadLevel = level; - - const levelDetails = levelInfo.details; - // if level info not retrieved yet, switch state and wait for level retrieval - // if live playlist, ensure that new playlist has been refreshed to avoid loading/try to load - // a useless and outdated fragment (that might even introduce load error if it is already out of the live playlist) - if (!levelDetails || (levelDetails.live && this.levelLastLoaded !== level)) { - this.state = State.WAITING_LEVEL; - return; - } - - if (this._streamEnded(bufferInfo, levelDetails)) { - const data = {}; - if (this.altAudio) { - data.type = 'video'; - } - - this.hls.trigger(Event.BUFFER_EOS, data); - this.state = State.ENDED; - return; - } - // if we have the levelDetails for the selected variant, lets continue enrichen our stream (load keys/fragments or trigger EOS, etc..) - this._fetchPayloadOrEos(pos, bufferInfo, levelDetails); + this.hls.trigger(Event.FRAG_LOADING, { frag: this.fragments[this.currentFrag++] }); } - _fetchPayloadOrEos (pos, bufferInfo, levelDetails) { - const fragPrevious = this.fragPrevious, - level = this.level, - fragments = levelDetails.fragments, - fragLen = fragments.length; - - // empty playlist - if (fragLen === 0) { - return; - } - - // find fragment index, contiguous with end of buffer position - let start = fragments[0].start, - end = fragments[fragLen - 1].start + fragments[fragLen - 1].duration, - bufferEnd = bufferInfo.end, - frag; - - if (levelDetails.initSegment && !levelDetails.initSegment.data) { - frag = levelDetails.initSegment; - } else { - // in case of live playlist we need to ensure that requested position is not located before playlist start - if (levelDetails.live) { - let initialLiveManifestSize = this.config.initialLiveManifestSize; - if (fragLen < initialLiveManifestSize) { - logger.warn(`Can not start playback of a level, reason: not enough fragments ${fragLen} < ${initialLiveManifestSize}`); - return; - } - - frag = this._ensureFragmentAtLivePoint(levelDetails, bufferEnd, start, end, fragPrevious, fragments, fragLen); - // if it explicitely returns null don't load any fragment and exit function now - if (frag === null) { - return; - } - } else { - // VoD playlist: if bufferEnd before start of playlist, load first fragment - if (bufferEnd < start) { - frag = fragments[0]; - } - } - } - if (!frag) { - frag = this._findFragment(start, fragPrevious, fragLen, fragments, bufferEnd, end, levelDetails); - } - - if (frag) { - if (frag.encrypted) { - logger.log(`Loading key for ${frag.sn} of [${levelDetails.startSN} ,${levelDetails.endSN}],level ${level}`); - this._loadKey(frag); - } else { - logger.log(`Loading ${frag.sn} of [${levelDetails.startSN} ,${levelDetails.endSN}],level ${level}, currentTime:${pos.toFixed(3)},bufferEnd:${bufferEnd.toFixed(3)}`); - this._loadFragment(frag); - } - } - } - - _ensureFragmentAtLivePoint (levelDetails, bufferEnd, start, end, fragPrevious, fragments, fragLen) { - const config = this.hls.config, media = this.media; - - let frag; - - // check if requested position is within seekable boundaries : - // logger.log(`start/pos/bufEnd/seeking:${start.toFixed(3)}/${pos.toFixed(3)}/${bufferEnd.toFixed(3)}/${this.media.seeking}`); - let maxLatency = config.liveMaxLatencyDuration !== undefined ? config.liveMaxLatencyDuration : config.liveMaxLatencyDurationCount * levelDetails.targetduration; - - if (bufferEnd < Math.max(start - config.maxFragLookUpTolerance, end - maxLatency)) { - let liveSyncPosition = this.liveSyncPosition = this.computeLivePosition(start, levelDetails); - logger.log(`buffer end: ${bufferEnd.toFixed(3)} is located too far from the end of live sliding playlist, reset currentTime to : ${liveSyncPosition.toFixed(3)}`); - bufferEnd = liveSyncPosition; - if (media && media.readyState && media.duration > liveSyncPosition) { - media.currentTime = liveSyncPosition; - } - - this.nextLoadPosition = liveSyncPosition; - } - - // if end of buffer greater than live edge, don't load any fragment - // this could happen if live playlist intermittently slides in the past. - // level 1 loaded [182580161,182580167] - // level 1 loaded [182580162,182580169] - // Loading 182580168 of [182580162 ,182580169],level 1 .. - // Loading 182580169 of [182580162 ,182580169],level 1 .. - // level 1 loaded [182580162,182580168] <============= here we should have bufferEnd > end. in that case break to avoid reloading 182580168 - // level 1 loaded [182580164,182580171] - // - // don't return null in case media not loaded yet (readystate === 0) - if (levelDetails.PTSKnown && bufferEnd > end && media && media.readyState) { - return null; - } - - if (this.startFragRequested && !levelDetails.PTSKnown) { - /* we are switching level on live playlist, but we don't have any PTS info for that quality level ... - try to load frag matching with next SN. - even if SN are not synchronized between playlists, loading this frag will help us - compute playlist sliding and find the right one after in case it was not the right consecutive one */ - if (fragPrevious) { - if (levelDetails.hasProgramDateTime) { - // Relies on PDT in order to switch bitrates (Support EXT-X-DISCONTINUITY without EXT-X-DISCONTINUITY-SEQUENCE) - logger.log(`live playlist, switching playlist, load frag with same PDT: ${fragPrevious.programDateTime}`); - frag = findFragmentByPDT(fragments, fragPrevious.endProgramDateTime, config.maxFragLookUpTolerance); - } else { - // Uses buffer and sequence number to calculate switch segment (required if using EXT-X-DISCONTINUITY-SEQUENCE) - const targetSN = fragPrevious.sn + 1; - if (targetSN >= levelDetails.startSN && targetSN <= levelDetails.endSN) { - const fragNext = fragments[targetSN - levelDetails.startSN]; - if (fragPrevious.cc === fragNext.cc) { - frag = fragNext; - logger.log(`live playlist, switching playlist, load frag with next SN: ${frag.sn}`); - } - } - // next frag SN not available (or not with same continuity counter) - // look for a frag sharing the same CC - if (!frag) { - frag = BinarySearch.search(fragments, function (frag) { - return fragPrevious.cc - frag.cc; - }); - if (frag) { - logger.log(`live playlist, switching playlist, load frag with same CC: ${frag.sn}`); - } - } - } - } - if (!frag) { - /* we have no idea about which fragment should be loaded. - so let's load mid fragment. it will help computing playlist sliding and find the right one - */ - frag = fragments[Math.min(fragLen - 1, Math.round(fragLen / 2))]; - logger.log(`live playlist, switching playlist, unknown, load middle frag : ${frag.sn}`); - } - } - - return frag; - } - - _findFragment (start, fragPrevious, fragLen, fragments, bufferEnd, end, levelDetails) { - const config = this.hls.config; - let frag; - - if (bufferEnd < end) { - const lookupTolerance = (bufferEnd > end - config.maxFragLookUpTolerance) ? 0 : config.maxFragLookUpTolerance; - // Remove the tolerance if it would put the bufferEnd past the actual end of stream - // Uses buffer and sequence number to calculate switch segment (required if using EXT-X-DISCONTINUITY-SEQUENCE) - frag = findFragmentByPTS(fragPrevious, fragments, bufferEnd, lookupTolerance); - } else { - // reach end of playlist - frag = fragments[fragLen - 1]; - } - if (frag) { - const curSNIdx = frag.sn - levelDetails.startSN; - const sameLevel = fragPrevious && frag.level === fragPrevious.level; - const prevFrag = fragments[curSNIdx - 1]; - const nextFrag = fragments[curSNIdx + 1]; - // logger.log('find SN matching with pos:' + bufferEnd + ':' + frag.sn); - if (fragPrevious && frag.sn === fragPrevious.sn) { - if (sameLevel && !frag.backtracked) { - if (frag.sn < levelDetails.endSN) { - let deltaPTS = fragPrevious.deltaPTS; - // if there is a significant delta between audio and video, larger than max allowed hole, - // and if previous remuxed fragment did not start with a keyframe. (fragPrevious.dropped) - // let's try to load previous fragment again to get last keyframe - // then we will reload again current fragment (that way we should be able to fill the buffer hole ...) - if (deltaPTS && deltaPTS > config.maxBufferHole && fragPrevious.dropped && curSNIdx) { - frag = prevFrag; - logger.warn('SN just loaded, with large PTS gap between audio and video, maybe frag is not starting with a keyframe ? load previous one to try to overcome this'); - } else { - frag = nextFrag; - logger.log(`SN just loaded, load next one: ${frag.sn}`, frag); - } - } else { - frag = null; - } - } else if (frag.backtracked) { - // Only backtrack a max of 1 consecutive fragment to prevent sliding back too far when little or no frags start with keyframes - if (nextFrag && nextFrag.backtracked) { - logger.warn(`Already backtracked from fragment ${nextFrag.sn}, will not backtrack to fragment ${frag.sn}. Loading fragment ${nextFrag.sn}`); - frag = nextFrag; - } else { - // If a fragment has dropped frames and it's in a same level/sequence, load the previous fragment to try and find the keyframe - // Reset the dropped count now since it won't be reset until we parse the fragment again, which prevents infinite backtracking on the same segment - logger.warn('Loaded fragment with dropped frames, backtracking 1 segment to find a keyframe'); - frag.dropped = 0; - if (prevFrag) { - frag = prevFrag; - frag.backtracked = true; - } else if (curSNIdx) { - // can't backtrack on very first fragment - frag = null; - } - } - } - } - } - return frag; - } - - _loadKey (frag) { - this.state = State.KEY_LOADING; - this.hls.trigger(Event.KEY_LOADING, { frag }); - } - - _loadFragment (frag) { - // Check if fragment is not loaded - let fragState = this.fragmentTracker.getState(frag); - - this.fragCurrent = frag; - if (frag.sn !== 'initSegment') { - this.startFragRequested = true; - } - // Don't update nextLoadPosition for fragments which are not buffered - if (Number.isFinite(frag.sn) && !frag.bitrateTest) { - this.nextLoadPosition = frag.start + frag.duration; - } - - // Allow backtracked fragments to load - if (frag.backtracked || fragState === FragmentState.NOT_LOADED || fragState === FragmentState.PARTIAL) { - frag.autoLevel = this.hls.autoLevelEnabled; - frag.bitrateTest = this.bitrateTest; - - this.hls.trigger(Event.FRAG_LOADING, { frag }); - // lazy demuxer init, as this could take some time ... do it during frag loading - if (!this.demuxer) { - this.demuxer = new Demuxer(this.hls, 'main'); - } - - this.state = State.FRAG_LOADING; - } else if (fragState === FragmentState.APPENDING) { - // Lower the buffer size and try again - if (this._reduceMaxBufferLength(frag.duration)) { - this.fragmentTracker.removeFragment(frag); - } - } - } - - set state (nextState) { - if (this.state !== nextState) { - const previousState = this.state; - this._state = nextState; - logger.log(`main stream:${previousState}->${nextState}`); - this.hls.trigger(Event.STREAM_STATE_TRANSITION, { previousState, nextState }); - } - } - - get state () { - return this._state; - } - - getBufferedFrag (position) { - return this.fragmentTracker.getBufferedFrag(position, PlaylistLevelType.MAIN); - } - - get currentLevel () { - let media = this.media; - if (media) { - const frag = this.getBufferedFrag(media.currentTime); - if (frag) { - return frag.level; - } - } - return -1; - } - - get nextBufferedFrag () { - let media = this.media; - if (media) { - // first get end range of current fragment - return this.followingBufferedFrag(this.getBufferedFrag(media.currentTime)); - } else { - return null; - } - } - - followingBufferedFrag (frag) { - if (frag) { - // try to get range of next fragment (500ms after this range) - return this.getBufferedFrag(frag.endPTS + 0.5); - } - return null; - } - - get nextLevel () { - const frag = this.nextBufferedFrag; - if (frag) { - return frag.level; - } else { - return -1; - } - } - - _checkFragmentChanged () { - let fragPlayingCurrent, currentTime, video = this.media; - if (video && video.readyState && video.seeking === false) { - currentTime = video.currentTime; - /* if video element is in seeked state, currentTime can only increase. - (assuming that playback rate is positive ...) - As sometimes currentTime jumps back to zero after a - media decode error, check this, to avoid seeking back to - wrong position after a media decode error - */ - if (currentTime > this.lastCurrentTime) { - this.lastCurrentTime = currentTime; - } - - if (BufferHelper.isBuffered(video, currentTime)) { - fragPlayingCurrent = this.getBufferedFrag(currentTime); - } else if (BufferHelper.isBuffered(video, currentTime + 0.1)) { - /* ensure that FRAG_CHANGED event is triggered at startup, - when first video frame is displayed and playback is paused. - add a tolerance of 100ms, in case current position is not buffered, - check if current pos+100ms is buffered and use that buffer range - for FRAG_CHANGED event reporting */ - fragPlayingCurrent = this.getBufferedFrag(currentTime + 0.1); - } - if (fragPlayingCurrent) { - let fragPlaying = fragPlayingCurrent; - if (fragPlaying !== this.fragPlaying) { - this.hls.trigger(Event.FRAG_CHANGED, { frag: fragPlaying }); - const fragPlayingLevel = fragPlaying.level; - if (!this.fragPlaying || this.fragPlaying.level !== fragPlayingLevel) { - this.hls.trigger(Event.LEVEL_SWITCHED, { level: fragPlayingLevel }); - } - - this.fragPlaying = fragPlaying; - } - } - } - } - - /* - on immediate level switch : - - pause playback if playing - - cancel any pending load request - - and trigger a buffer flush - */ - immediateLevelSwitch () { - logger.log('immediateLevelSwitch'); - if (!this.immediateSwitch) { - this.immediateSwitch = true; - let media = this.media, previouslyPaused; - if (media) { - previouslyPaused = media.paused; - media.pause(); - } else { - // don't restart playback after instant level switch in case media not attached - previouslyPaused = true; - } - this.previouslyPaused = previouslyPaused; - } - let fragCurrent = this.fragCurrent; - if (fragCurrent && fragCurrent.loader) { - fragCurrent.loader.abort(); - } - - this.fragCurrent = null; - // flush everything - this.flushMainBuffer(0, Number.POSITIVE_INFINITY); - } - - /** - * on immediate level switch end, after new fragment has been buffered: - * - nudge video decoder by slightly adjusting video currentTime (if currentTime buffered) - * - resume the playback if needed - */ - immediateLevelSwitchEnd () { - const media = this.media; - if (media && media.buffered.length) { - this.immediateSwitch = false; - if (BufferHelper.isBuffered(media, media.currentTime)) { - // only nudge if currentTime is buffered - media.currentTime -= 0.0001; - } - if (!this.previouslyPaused) { - media.play(); - } - } - } - - /** - * try to switch ASAP without breaking video playback: - * in order to ensure smooth but quick level switching, - * we need to find the next flushable buffer range - * we should take into account new segment fetch time - */ - nextLevelSwitch () { - const media = this.media; - // ensure that media is defined and that metadata are available (to retrieve currentTime) - if (media && media.readyState) { - let fetchdelay, fragPlayingCurrent, nextBufferedFrag; - fragPlayingCurrent = this.getBufferedFrag(media.currentTime); - if (fragPlayingCurrent && fragPlayingCurrent.startPTS > 1) { - // flush buffer preceding current fragment (flush until current fragment start offset) - // minus 1s to avoid video freezing, that could happen if we flush keyframe of current video ... - this.flushMainBuffer(0, fragPlayingCurrent.startPTS - 1); - } - if (!media.paused) { - // add a safety delay of 1s - let nextLevelId = this.hls.nextLoadLevel, nextLevel = this.levels[nextLevelId], fragLastKbps = this.fragLastKbps; - if (fragLastKbps && this.fragCurrent) { - fetchdelay = this.fragCurrent.duration * nextLevel.bitrate / (1000 * fragLastKbps) + 1; - } else { - fetchdelay = 0; - } - } else { - fetchdelay = 0; - } - // logger.log('fetchdelay:'+fetchdelay); - // find buffer range that will be reached once new fragment will be fetched - nextBufferedFrag = this.getBufferedFrag(media.currentTime + fetchdelay); - if (nextBufferedFrag) { - // we can flush buffer range following this one without stalling playback - nextBufferedFrag = this.followingBufferedFrag(nextBufferedFrag); - if (nextBufferedFrag) { - // if we are here, we can also cancel any loading/demuxing in progress, as they are useless - let fragCurrent = this.fragCurrent; - if (fragCurrent && fragCurrent.loader) { - fragCurrent.loader.abort(); - } - - this.fragCurrent = null; - // start flush position is the start PTS of next buffered frag. - // we use frag.naxStartPTS which is max(audio startPTS, video startPTS). - // in case there is a small PTS Delta between audio and video, using maxStartPTS avoids flushing last samples from current fragment - this.flushMainBuffer(nextBufferedFrag.maxStartPTS, Number.POSITIVE_INFINITY); - } - } - } - } - - flushMainBuffer (startOffset, endOffset) { - this.state = State.BUFFER_FLUSHING; - let flushScope = { startOffset: startOffset, endOffset: endOffset }; - // if alternate audio tracks are used, only flush video, otherwise flush everything - if (this.altAudio) { - flushScope.type = 'video'; - } - - this.hls.trigger(Event.BUFFER_FLUSHING, flushScope); - } - - onMediaAttached (data) { - let media = this.media = this.mediaBuffer = data.media; - this.onvseeking = this.onMediaSeeking.bind(this); - this.onvseeked = this.onMediaSeeked.bind(this); - this.onvended = this.onMediaEnded.bind(this); - media.addEventListener('seeking', this.onvseeking); - media.addEventListener('seeked', this.onvseeked); - media.addEventListener('ended', this.onvended); - let config = this.config; - if (this.levels && config.autoStartLoad) { - this.hls.startLoad(config.startPosition); - } - - this.gapController = new GapController(config, media, this.fragmentTracker, this.hls); - } - - onMediaDetaching () { - let media = this.media; - if (media && media.ended) { - logger.log('MSE detaching and video ended, reset startPosition'); - this.startPosition = this.lastCurrentTime = 0; - } - - // reset fragment backtracked flag - let levels = this.levels; - if (levels) { - levels.forEach(level => { - if (level.details) { - level.details.fragments.forEach(fragment => { - fragment.backtracked = undefined; - }); - } - }); - } - // remove video listeners - if (media) { - media.removeEventListener('seeking', this.onvseeking); - media.removeEventListener('seeked', this.onvseeked); - media.removeEventListener('ended', this.onvended); - this.onvseeking = this.onvseeked = this.onvended = null; - } - this.media = this.mediaBuffer = null; - this.loadedmetadata = false; - this.stopLoad(); - } - - onMediaSeeked () { - const media = this.media, currentTime = media ? media.currentTime : undefined; - if (Number.isFinite(currentTime)) { - logger.log(`media seeked to ${currentTime.toFixed(3)}`); - } - - // tick to speed up FRAGMENT_PLAYING triggering - this.tick(); - } - - onManifestLoading () { - // reset buffer on manifest loading - logger.log('trigger BUFFER_RESET'); - this.hls.trigger(Event.BUFFER_RESET); - this.fragmentTracker.removeAllFragments(); - this.stalled = false; - this.startPosition = this.lastCurrentTime = 0; - } - - onManifestParsed (data) { - let aac = false, heaac = false, codec; - data.levels.forEach(level => { - // detect if we have different kind of audio codecs used amongst playlists - codec = level.audioCodec; - if (codec) { - if (codec.indexOf('mp4a.40.2') !== -1) { - aac = true; - } - - if (codec.indexOf('mp4a.40.5') !== -1) { - heaac = true; - } - } - }); - this.audioCodecSwitch = (aac && heaac); - if (this.audioCodecSwitch) { - logger.log('both AAC/HE-AAC audio found in levels; declaring level codec as HE-AAC'); - } - - this.levels = data.levels; - this.startFragRequested = false; - let config = this.config; - if (config.autoStartLoad || this.forceStartLoad) { - this.hls.startLoad(config.startPosition); - } - } - - onLevelLoaded (data) { + onLevelLoaded(data) { const newDetails = data.details; const newLevelId = data.level; - const lastLevel = this.levels[this.levelLastLoaded]; - const curLevel = this.levels[newLevelId]; + const levelDetails = data.details; const duration = newDetails.totalduration; let sliding = 0; - logger.log(`level ${newLevelId} loaded [${newDetails.startSN},${newDetails.endSN}],duration:${duration}`); + console.log(`level ${newLevelId} loaded [${newDetails.startSN},${newDetails.endSN}],duration:${duration}`); if (newDetails.live) { - let curDetails = curLevel.details; - if (curDetails && newDetails.fragments.length > 0) { - // we already have details for that level, merge them - LevelHelper.mergeDetails(curDetails, newDetails); - sliding = newDetails.fragments[0].start; - this.liveSyncPosition = this.computeLivePosition(sliding, curDetails); - if (newDetails.PTSKnown && Number.isFinite(sliding)) { - logger.log(`live playlist sliding:${sliding.toFixed(3)}`); - } else { - logger.log('live playlist - outdated PTS, unknown sliding'); - alignStream(this.fragPrevious, lastLevel, newDetails); - } - } else { - logger.log('live playlist - first load, unknown sliding'); - newDetails.PTSKnown = false; - alignStream(this.fragPrevious, lastLevel, newDetails); - } + console.log("handling of this case is not implemented"); } else { newDetails.PTSKnown = false; } // override level info - curLevel.details = newDetails; this.levelLastLoaded = newLevelId; this.hls.trigger(Event.LEVEL_UPDATED, { details: newDetails, level: newLevelId }); if (this.startFragRequested === false) { - // compute start position if set to -1. use it straight away if value is defined + // compute start position if set to -1. use it straight away if value is defined if (this.startPosition === -1 || this.lastCurrentTime === -1) { // first, check if start time offset has been set in playlist, if yes, use this value let startTimeOffset = newDetails.startTimeOffset; if (Number.isFinite(startTimeOffset)) { if (startTimeOffset < 0) { - logger.log(`negative start time offset ${startTimeOffset}, count from end of last fragment`); + console.log(`negative start time offset ${startTimeOffset}, count from end of last fragment`); startTimeOffset = sliding + duration + startTimeOffset; } - logger.log(`start time offset found in playlist, adjust startPosition to ${startTimeOffset}`); + console.log(`start time offset found in playlist, adjust startPosition to ${startTimeOffset}`); this.startPosition = startTimeOffset; } else { // if live playlist, set start position to be fragment N-this.config.liveSyncDurationCount (usually 3) if (newDetails.live) { - this.startPosition = this.computeLivePosition(sliding, newDetails); - logger.log(`configure startPosition to ${this.startPosition}`); + console.log("handling of this case is not implemented"); } else { this.startPosition = 0; } @@ -804,534 +96,14 @@ class StreamController extends BaseStreamController { } this.nextLoadPosition = this.startPosition; } - // only switch batck to IDLE state if we were waiting for level to start downloading a new fragment - if (this.state === State.WAITING_LEVEL) { - this.state = State.IDLE; - } - // trigger handler right now - this.tick(); + this._fetchPayloadOrEos(levelDetails); } - onKeyLoaded () { - if (this.state === State.KEY_LOADING) { - this.state = State.IDLE; - this.tick(); - } - } + onFragLoaded(data) { + this.hls.loadSuccess(data.payload); + this._loadFragment(); - onFragLoaded (data) { - const { fragCurrent, hls, levels, media } = this; - const fragLoaded = data.frag; - if (this.state === State.FRAG_LOADING && - fragCurrent && - fragLoaded.type === 'main' && - fragLoaded.level === fragCurrent.level && - fragLoaded.sn === fragCurrent.sn) { - const stats = data.stats; - const currentLevel = levels[fragCurrent.level]; - const details = currentLevel.details; - // reset frag bitrate test in any case after frag loaded event - // if this frag was loaded to perform a bitrate test AND if hls.nextLoadLevel is greater than 0 - // then this means that we should be able to load a fragment at a higher quality level - this.bitrateTest = false; - this.stats = stats; - - logger.log(`Loaded ${fragCurrent.sn} of [${details.startSN} ,${details.endSN}],level ${fragCurrent.level}`); - if (fragLoaded.bitrateTest && hls.nextLoadLevel) { - // switch back to IDLE state ... we just loaded a fragment to determine adequate start bitrate and initialize autoswitch algo - this.state = State.IDLE; - this.startFragRequested = false; - stats.tparsed = stats.tbuffered = window.performance.now(); - hls.trigger(Event.FRAG_BUFFERED, { stats: stats, frag: fragCurrent, id: 'main' }); - this.tick(); - } else if (fragLoaded.sn === 'initSegment') { - this.state = State.IDLE; - stats.tparsed = stats.tbuffered = window.performance.now(); - details.initSegment.data = data.payload; - hls.trigger(Event.FRAG_BUFFERED, { stats: stats, frag: fragCurrent, id: 'main' }); - this.tick(); - } else { - logger.log(`Parsing ${fragCurrent.sn} of [${details.startSN} ,${details.endSN}],level ${fragCurrent.level}, cc ${fragCurrent.cc}`); - this.state = State.PARSING; - this.pendingBuffering = true; - this.appended = false; - - // Bitrate test frags are not usually buffered so the fragment tracker ignores them. If Hls.js decides to buffer - // it (and therefore ends up at this line), then the fragment tracker needs to be manually informed. - if (fragLoaded.bitrateTest) { - fragLoaded.bitrateTest = false; - this.fragmentTracker.onFragLoaded({ - frag: fragLoaded - }); - } - - // time Offset is accurate if level PTS is known, or if playlist is not sliding (not live) and if media is not seeking (this is to overcome potential timestamp drifts between playlists and fragments) - const accurateTimeOffset = !(media && media.seeking) && (details.PTSKnown || !details.live); - const initSegmentData = details.initSegment ? details.initSegment.data : []; - const audioCodec = this._getAudioCodec(currentLevel); - - // transmux the MPEG-TS data to ISO-BMFF segments - const demuxer = this.demuxer = this.demuxer || new Demuxer(this.hls, 'main'); - demuxer.push( - data.payload, - initSegmentData, - audioCodec, - currentLevel.videoCodec, - fragCurrent, - details.totalduration, - accurateTimeOffset - ); - } - } - this.fragLoadError = 0; - } - - onFragParsingInitSegment (data) { - const fragCurrent = this.fragCurrent; - const fragNew = data.frag; - - if (fragCurrent && - data.id === 'main' && - fragNew.sn === fragCurrent.sn && - fragNew.level === fragCurrent.level && - this.state === State.PARSING) { - let tracks = data.tracks, trackName, track; - - // if audio track is expected to come from audio stream controller, discard any coming from main - if (tracks.audio && this.altAudio) { - delete tracks.audio; - } - - // include levelCodec in audio and video tracks - track = tracks.audio; - if (track) { - let audioCodec = this.levels[this.level].audioCodec, - ua = navigator.userAgent.toLowerCase(); - if (audioCodec && this.audioCodecSwap) { - logger.log('swapping playlist audio codec'); - if (audioCodec.indexOf('mp4a.40.5') !== -1) { - audioCodec = 'mp4a.40.2'; - } else { - audioCodec = 'mp4a.40.5'; - } - } - // in case AAC and HE-AAC audio codecs are signalled in manifest - // force HE-AAC , as it seems that most browsers prefers that way, - // except for mono streams OR on FF - // these conditions might need to be reviewed ... - if (this.audioCodecSwitch) { - // don't force HE-AAC if mono stream - if (track.metadata.channelCount !== 1 && - // don't force HE-AAC if firefox - ua.indexOf('firefox') === -1) { - audioCodec = 'mp4a.40.5'; - } - } - // HE-AAC is broken on Android, always signal audio codec as AAC even if variant manifest states otherwise - if (ua.indexOf('android') !== -1 && track.container !== 'audio/mpeg') { // Exclude mpeg audio - audioCodec = 'mp4a.40.2'; - logger.log(`Android: force audio codec to ${audioCodec}`); - } - track.levelCodec = audioCodec; - track.id = data.id; - } - track = tracks.video; - if (track) { - track.levelCodec = this.levels[this.level].videoCodec; - track.id = data.id; - } - this.hls.trigger(Event.BUFFER_CODECS, tracks); - // loop through tracks that are going to be provided to bufferController - for (trackName in tracks) { - track = tracks[trackName]; - logger.log(`main track:${trackName},container:${track.container},codecs[level/parsed]=[${track.levelCodec}/${track.codec}]`); - let initSegment = track.initSegment; - if (initSegment) { - this.appended = true; - // arm pending Buffering flag before appending a segment - this.pendingBuffering = true; - this.hls.trigger(Event.BUFFER_APPENDING, { type: trackName, data: initSegment, parent: 'main', content: 'initSegment' }); - } - } - // trigger handler right now - this.tick(); - } - } - - onFragParsingData (data) { - const fragCurrent = this.fragCurrent; - const fragNew = data.frag; - if (fragCurrent && - data.id === 'main' && - fragNew.sn === fragCurrent.sn && - fragNew.level === fragCurrent.level && - !(data.type === 'audio' && this.altAudio) && // filter out main audio if audio track is loaded through audio stream controller - this.state === State.PARSING) { - let level = this.levels[this.level], - frag = fragCurrent; - if (!Number.isFinite(data.endPTS)) { - data.endPTS = data.startPTS + fragCurrent.duration; - data.endDTS = data.startDTS + fragCurrent.duration; - } - - if (data.hasAudio === true) { - frag.addElementaryStream(ElementaryStreamTypes.AUDIO); - } - - if (data.hasVideo === true) { - frag.addElementaryStream(ElementaryStreamTypes.VIDEO); - } - - logger.log(`Parsed ${data.type},PTS:[${data.startPTS.toFixed(3)},${data.endPTS.toFixed(3)}],DTS:[${data.startDTS.toFixed(3)}/${data.endDTS.toFixed(3)}],nb:${data.nb},dropped:${data.dropped || 0}`); - - // Detect gaps in a fragment and try to fix it by finding a keyframe in the previous fragment (see _findFragments) - if (data.type === 'video') { - frag.dropped = data.dropped; - if (frag.dropped) { - if (!frag.backtracked) { - const levelDetails = level.details; - if (levelDetails && frag.sn === levelDetails.startSN) { - logger.warn('missing video frame(s) on first frag, appending with gap', frag.sn); - } else { - logger.warn('missing video frame(s), backtracking fragment', frag.sn); - // Return back to the IDLE state without appending to buffer - // Causes findFragments to backtrack a segment and find the keyframe - // Audio fragments arriving before video sets the nextLoadPosition, causing _findFragments to skip the backtracked fragment - this.fragmentTracker.removeFragment(frag); - frag.backtracked = true; - this.nextLoadPosition = data.startPTS; - this.state = State.IDLE; - this.fragPrevious = frag; - this.tick(); - return; - } - } else { - logger.warn('Already backtracked on this fragment, appending with the gap', frag.sn); - } - } else { - // Only reset the backtracked flag if we've loaded the frag without any dropped frames - frag.backtracked = false; - } - } - - let drift = LevelHelper.updateFragPTSDTS(level.details, frag, data.startPTS, data.endPTS, data.startDTS, data.endDTS), - hls = this.hls; - hls.trigger(Event.LEVEL_PTS_UPDATED, { details: level.details, level: this.level, drift: drift, type: data.type, start: data.startPTS, end: data.endPTS }); - // has remuxer dropped video frames located before first keyframe ? - [data.data1, data.data2].forEach(buffer => { - // only append in PARSING state (rationale is that an appending error could happen synchronously on first segment appending) - // in that case it is useless to append following segments - if (buffer && buffer.length && this.state === State.PARSING) { - this.appended = true; - // arm pending Buffering flag before appending a segment - this.pendingBuffering = true; - hls.trigger(Event.BUFFER_APPENDING, { type: data.type, data: buffer, parent: 'main', content: 'data' }); - } - }); - // trigger handler right now - this.tick(); - } - } - - onFragParsed (data) { - const fragCurrent = this.fragCurrent; - const fragNew = data.frag; - if (fragCurrent && - data.id === 'main' && - fragNew.sn === fragCurrent.sn && - fragNew.level === fragCurrent.level && - this.state === State.PARSING) { - this.stats.tparsed = window.performance.now(); - this.state = State.PARSED; - this._checkAppendedParsed(); - } - } - - onAudioTrackSwitching (data) { - // if any URL found on new audio track, it is an alternate audio track - let altAudio = !!data.url, - trackId = data.id; - // if we switch on main audio, ensure that main fragment scheduling is synced with media.buffered - // don't do anything if we switch to alt audio: audio stream controller is handling it. - // we will just have to change buffer scheduling on audioTrackSwitched - if (!altAudio) { - if (this.mediaBuffer !== this.media) { - logger.log('switching on main audio, use media.buffered to schedule main fragment loading'); - this.mediaBuffer = this.media; - let fragCurrent = this.fragCurrent; - // we need to refill audio buffer from main: cancel any frag loading to speed up audio switch - if (fragCurrent.loader) { - logger.log('switching to main audio track, cancel main fragment load'); - fragCurrent.loader.abort(); - } - this.fragCurrent = null; - this.fragPrevious = null; - // destroy demuxer to force init segment generation (following audio switch) - if (this.demuxer) { - this.demuxer.destroy(); - this.demuxer = null; - } - // switch to IDLE state to load new fragment - this.state = State.IDLE; - } - let hls = this.hls; - // switching to main audio, flush all audio and trigger track switched - hls.trigger(Event.BUFFER_FLUSHING, { startOffset: 0, endOffset: Number.POSITIVE_INFINITY, type: 'audio' }); - hls.trigger(Event.AUDIO_TRACK_SWITCHED, { id: trackId }); - this.altAudio = false; - } - } - - onAudioTrackSwitched (data) { - let trackId = data.id, - altAudio = !!this.hls.audioTracks[trackId].url; - if (altAudio) { - let videoBuffer = this.videoBuffer; - // if we switched on alternate audio, ensure that main fragment scheduling is synced with video sourcebuffer buffered - if (videoBuffer && this.mediaBuffer !== videoBuffer) { - logger.log('switching on alternate audio, use video.buffered to schedule main fragment loading'); - this.mediaBuffer = videoBuffer; - } - } - this.altAudio = altAudio; - this.tick(); - } - - onBufferCreated (data) { - let tracks = data.tracks, mediaTrack, name, alternate = false; - for (let type in tracks) { - let track = tracks[type]; - if (track.id === 'main') { - name = type; - mediaTrack = track; - // keep video source buffer reference - if (type === 'video') { - this.videoBuffer = tracks[type].buffer; - } - } else { - alternate = true; - } - } - if (alternate && mediaTrack) { - logger.log(`alternate track found, use ${name}.buffered to schedule main fragment loading`); - this.mediaBuffer = mediaTrack.buffer; - } else { - this.mediaBuffer = this.media; - } - } - - onBufferAppended (data) { - if (data.parent === 'main') { - const state = this.state; - if (state === State.PARSING || state === State.PARSED) { - // check if all buffers have been appended - this.pendingBuffering = (data.pending > 0); - this._checkAppendedParsed(); - } - } - } - - _checkAppendedParsed () { - // trigger handler right now - if (this.state === State.PARSED && (!this.appended || !this.pendingBuffering)) { - const frag = this.fragCurrent; - if (frag) { - const media = this.mediaBuffer ? this.mediaBuffer : this.media; - logger.log(`main buffered : ${TimeRanges.toString(media.buffered)}`); - this.fragPrevious = frag; - const stats = this.stats; - stats.tbuffered = window.performance.now(); - // we should get rid of this.fragLastKbps - this.fragLastKbps = Math.round(8 * stats.total / (stats.tbuffered - stats.tfirst)); - this.hls.trigger(Event.FRAG_BUFFERED, { stats: stats, frag: frag, id: 'main' }); - this.state = State.IDLE; - } - this.tick(); - } - } - - onError (data) { - let frag = data.frag || this.fragCurrent; - // don't handle frag error not related to main fragment - if (frag && frag.type !== 'main') { - return; - } - - // 0.5 : tolerance needed as some browsers stalls playback before reaching buffered end - let mediaBuffered = !!this.media && BufferHelper.isBuffered(this.media, this.media.currentTime) && BufferHelper.isBuffered(this.media, this.media.currentTime + 0.5); - - switch (data.details) { - case ErrorDetails.FRAG_LOAD_ERROR: - case ErrorDetails.FRAG_LOAD_TIMEOUT: - case ErrorDetails.KEY_LOAD_ERROR: - case ErrorDetails.KEY_LOAD_TIMEOUT: - if (!data.fatal) { - // keep retrying until the limit will be reached - if ((this.fragLoadError + 1) <= this.config.fragLoadingMaxRetry) { - // exponential backoff capped to config.fragLoadingMaxRetryTimeout - let delay = Math.min(Math.pow(2, this.fragLoadError) * this.config.fragLoadingRetryDelay, this.config.fragLoadingMaxRetryTimeout); - logger.warn(`mediaController: frag loading failed, retry in ${delay} ms`); - this.retryDate = window.performance.now() + delay; - // retry loading state - // if loadedmetadata is not set, it means that we are emergency switch down on first frag - // in that case, reset startFragRequested flag - if (!this.loadedmetadata) { - this.startFragRequested = false; - this.nextLoadPosition = this.startPosition; - } - this.fragLoadError++; - this.state = State.FRAG_LOADING_WAITING_RETRY; - } else { - logger.error(`mediaController: ${data.details} reaches max retry, redispatch as fatal ...`); - // switch error to fatal - data.fatal = true; - this.state = State.ERROR; - } - } - break; - case ErrorDetails.LEVEL_LOAD_ERROR: - case ErrorDetails.LEVEL_LOAD_TIMEOUT: - if (this.state !== State.ERROR) { - if (data.fatal) { - // if fatal error, stop processing - this.state = State.ERROR; - logger.warn(`streamController: ${data.details},switch to ${this.state} state ...`); - } else { - // in case of non fatal error while loading level, if level controller is not retrying to load level , switch back to IDLE - if (!data.levelRetry && this.state === State.WAITING_LEVEL) { - this.state = State.IDLE; - } - } - } - break; - case ErrorDetails.BUFFER_FULL_ERROR: - // if in appending state - if (data.parent === 'main' && (this.state === State.PARSING || this.state === State.PARSED)) { - // reduce max buf len if current position is buffered - if (mediaBuffered) { - this._reduceMaxBufferLength(this.config.maxBufferLength); - this.state = State.IDLE; - } else { - // current position is not buffered, but browser is still complaining about buffer full error - // this happens on IE/Edge, refer to https://github.com/video-dev/hls.js/pull/708 - // in that case flush the whole buffer to recover - logger.warn('buffer full error also media.currentTime is not buffered, flush everything'); - this.fragCurrent = null; - // flush everything - this.flushMainBuffer(0, Number.POSITIVE_INFINITY); - } - } - break; - default: - break; - } - } - - _reduceMaxBufferLength (minLength) { - let config = this.config; - if (config.maxMaxBufferLength >= minLength) { - // reduce max buffer length as it might be too high. we do this to avoid loop flushing ... - config.maxMaxBufferLength /= 2; - logger.warn(`main:reduce max buffer length to ${config.maxMaxBufferLength}s`); - return true; - } - return false; - } - - /** - * Checks the health of the buffer and attempts to resolve playback stalls. - * @private - */ - _checkBuffer () { - const { media } = this; - if (!media || media.readyState === 0) { - // Exit early if we don't have media or if the media hasn't bufferd anything yet (readyState 0) - return; - } - - const mediaBuffer = this.mediaBuffer ? this.mediaBuffer : media; - const buffered = mediaBuffer.buffered; - - if (!this.loadedmetadata && buffered.length) { - this.loadedmetadata = true; - this._seekToStartPos(); - } else if (this.immediateSwitch) { - this.immediateLevelSwitchEnd(); - } else { - this.gapController.poll(this.lastCurrentTime, buffered); - } - } - - onFragLoadEmergencyAborted () { - this.state = State.IDLE; - // if loadedmetadata is not set, it means that we are emergency switch down on first frag - // in that case, reset startFragRequested flag - if (!this.loadedmetadata) { - this.startFragRequested = false; - this.nextLoadPosition = this.startPosition; - } - this.tick(); - } - - onBufferFlushed () { - /* after successful buffer flushing, filter flushed fragments from bufferedFrags - use mediaBuffered instead of media (so that we will check against video.buffered ranges in case of alt audio track) - */ - const media = this.mediaBuffer ? this.mediaBuffer : this.media; - if (media) { - // filter fragments potentially evicted from buffer. this is to avoid memleak on live streams - this.fragmentTracker.detectEvictedFragments(ElementaryStreamTypes.VIDEO, media.buffered); - } - // move to IDLE once flush complete. this should trigger new fragment loading - this.state = State.IDLE; - // reset reference to frag - this.fragPrevious = null; - } - - swapAudioCodec () { - this.audioCodecSwap = !this.audioCodecSwap; - } - /** - * Seeks to the set startPosition if not equal to the mediaElement's current time. - * @private - */ - _seekToStartPos () { - const { media } = this; - const currentTime = media.currentTime; - // only adjust currentTime if different from startPosition or if startPosition not buffered - // at that stage, there should be only one buffered range, as we reach that code after first fragment has been buffered - const startPosition = media.seeking ? currentTime : this.startPosition; - // if currentTime not matching with expected startPosition or startPosition not buffered but close to first buffered - if (currentTime !== startPosition) { - // if startPosition not buffered, let's seek to buffered.start(0) - logger.log(`target start position not buffered, seek to buffered.start(0) ${startPosition} from current time ${currentTime} `); - media.currentTime = startPosition; - } - } - - _getAudioCodec (currentLevel) { - let audioCodec = this.config.defaultAudioCodec || currentLevel.audioCodec; - if (this.audioCodecSwap) { - logger.log('swapping playlist audio codec'); - if (audioCodec) { - if (audioCodec.indexOf('mp4a.40.5') !== -1) { - audioCodec = 'mp4a.40.2'; - } else { - audioCodec = 'mp4a.40.5'; - } - } - } - - return audioCodec; - } - - get liveSyncPosition () { - return this._liveSyncPosition; - } - - set liveSyncPosition (value) { - this._liveSyncPosition = value; } } export default StreamController; diff --git a/cmd/mjpeg-player/hlsjs/events.js b/cmd/mjpeg-player/hlsjs/events.js index 82314117..e71d71c5 100644 --- a/cmd/mjpeg-player/hlsjs/events.js +++ b/cmd/mjpeg-player/hlsjs/events.js @@ -1,110 +1,55 @@ +/* +AUTHOR + Trek Hopton + +LICENSE + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see http://www.gnu.org/licenses. + + For hls.js Copyright notice and license, see LICENSE file. +*/ + /** * @readonly * @enum {string} */ const HlsEvents = { - // fired before MediaSource is attaching to media element - data: { media } - MEDIA_ATTACHING: 'hlsMediaAttaching', - // fired when MediaSource has been succesfully attached to media element - data: { } - MEDIA_ATTACHED: 'hlsMediaAttached', - // fired before detaching MediaSource from media element - data: { } - MEDIA_DETACHING: 'hlsMediaDetaching', - // fired when MediaSource has been detached from media element - data: { } - MEDIA_DETACHED: 'hlsMediaDetached', - // fired when we buffer is going to be reset - data: { } - BUFFER_RESET: 'hlsBufferReset', - // fired when we know about the codecs that we need buffers for to push into - data: {tracks : { container, codec, levelCodec, initSegment, metadata }} - BUFFER_CODECS: 'hlsBufferCodecs', - // fired when sourcebuffers have been created - data: { tracks : tracks } - BUFFER_CREATED: 'hlsBufferCreated', - // fired when we append a segment to the buffer - data: { segment: segment object } - BUFFER_APPENDING: 'hlsBufferAppending', - // fired when we are done with appending a media segment to the buffer - data : { parent : segment parent that triggered BUFFER_APPENDING, pending : nb of segments waiting for appending for this segment parent} - BUFFER_APPENDED: 'hlsBufferAppended', - // fired when the stream is finished and we want to notify the media buffer that there will be no more data - data: { } - BUFFER_EOS: 'hlsBufferEos', - // fired when the media buffer should be flushed - data { startOffset, endOffset } - BUFFER_FLUSHING: 'hlsBufferFlushing', - // fired when the media buffer has been flushed - data: { } - BUFFER_FLUSHED: 'hlsBufferFlushed', // fired to signal that a manifest loading starts - data: { url : manifestURL} MANIFEST_LOADING: 'hlsManifestLoading', // fired after manifest has been loaded - data: { levels : [available quality levels], audioTracks : [ available audio tracks], url : manifestURL, stats : { trequest, tfirst, tload, mtime}} MANIFEST_LOADED: 'hlsManifestLoaded', - // fired after manifest has been parsed - data: { levels : [available quality levels], firstLevel : index of first quality level appearing in Manifest} - MANIFEST_PARSED: 'hlsManifestParsed', - // fired when a level switch is requested - data: { level : id of new level } - LEVEL_SWITCHING: 'hlsLevelSwitching', - // fired when a level switch is effective - data: { level : id of new level } - LEVEL_SWITCHED: 'hlsLevelSwitched', // fired when a level playlist loading starts - data: { url : level URL, level : id of level being loaded} LEVEL_LOADING: 'hlsLevelLoading', // fired when a level playlist loading finishes - data: { details : levelDetails object, level : id of loaded level, stats : { trequest, tfirst, tload, mtime} } LEVEL_LOADED: 'hlsLevelLoaded', // fired when a level's details have been updated based on previous details, after it has been loaded - data: { details : levelDetails object, level : id of updated level } LEVEL_UPDATED: 'hlsLevelUpdated', - // fired when a level's PTS information has been updated after parsing a fragment - data: { details : levelDetails object, level : id of updated level, drift: PTS drift observed when parsing last fragment } - LEVEL_PTS_UPDATED: 'hlsLevelPtsUpdated', - // fired to notify that audio track lists has been updated - data: { audioTracks : audioTracks } - AUDIO_TRACKS_UPDATED: 'hlsAudioTracksUpdated', - // fired when an audio track switching is requested - data: { id : audio track id } - AUDIO_TRACK_SWITCHING: 'hlsAudioTrackSwitching', - // fired when an audio track switch actually occurs - data: { id : audio track id } - AUDIO_TRACK_SWITCHED: 'hlsAudioTrackSwitched', // fired when an audio track loading starts - data: { url : audio track URL, id : audio track id } AUDIO_TRACK_LOADING: 'hlsAudioTrackLoading', // fired when an audio track loading finishes - data: { details : levelDetails object, id : audio track id, stats : { trequest, tfirst, tload, mtime } } AUDIO_TRACK_LOADED: 'hlsAudioTrackLoaded', - // fired to notify that subtitle track lists has been updated - data: { subtitleTracks : subtitleTracks } - SUBTITLE_TRACKS_UPDATED: 'hlsSubtitleTracksUpdated', - // fired when an subtitle track switch occurs - data: { id : subtitle track id } - SUBTITLE_TRACK_SWITCH: 'hlsSubtitleTrackSwitch', // fired when a subtitle track loading starts - data: { url : subtitle track URL, id : subtitle track id } SUBTITLE_TRACK_LOADING: 'hlsSubtitleTrackLoading', // fired when a subtitle track loading finishes - data: { details : levelDetails object, id : subtitle track id, stats : { trequest, tfirst, tload, mtime } } SUBTITLE_TRACK_LOADED: 'hlsSubtitleTrackLoaded', - // fired when a subtitle fragment has been processed - data: { success : boolean, frag : the processed frag } - SUBTITLE_FRAG_PROCESSED: 'hlsSubtitleFragProcessed', - // fired when the first timestamp is found - data: { id : demuxer id, initPTS: initPTS, frag : fragment object } - INIT_PTS_FOUND: 'hlsInitPtsFound', // fired when a fragment loading starts - data: { frag : fragment object } FRAG_LOADING: 'hlsFragLoading', // fired when a fragment loading is progressing - data: { frag : fragment object, { trequest, tfirst, loaded } } FRAG_LOAD_PROGRESS: 'hlsFragLoadProgress', - // Identifier for fragment load aborting for emergency switch down - data: { frag : fragment object } - FRAG_LOAD_EMERGENCY_ABORTED: 'hlsFragLoadEmergencyAborted', // fired when a fragment loading is completed - data: { frag : fragment object, payload : fragment payload, stats : { trequest, tfirst, tload, length } } - FRAG_LOADED: 'hlsFragLoaded', - // fired when a fragment has finished decrypting - data: { id : demuxer id, frag: fragment object, payload : fragment payload, stats : { tstart, tdecrypt } } - FRAG_DECRYPTED: 'hlsFragDecrypted', - // fired when Init Segment has been extracted from fragment - data: { id : demuxer id, frag: fragment object, moov : moov MP4 box, codecs : codecs found while parsing fragment } - FRAG_PARSING_INIT_SEGMENT: 'hlsFragParsingInitSegment', - // fired when parsing sei text is completed - data: { id : demuxer id, frag: fragment object, samples : [ sei samples pes ] } - FRAG_PARSING_USERDATA: 'hlsFragParsingUserdata', - // fired when parsing id3 is completed - data: { id : demuxer id, frag: fragment object, samples : [ id3 samples pes ] } - FRAG_PARSING_METADATA: 'hlsFragParsingMetadata', - // fired when data have been extracted from fragment - data: { id : demuxer id, frag: fragment object, data1 : moof MP4 box or TS fragments, data2 : mdat MP4 box or null} - FRAG_PARSING_DATA: 'hlsFragParsingData', - // fired when fragment parsing is completed - data: { id : demuxer id, frag: fragment object } - FRAG_PARSED: 'hlsFragParsed', - // fired when fragment remuxed MP4 boxes have all been appended into SourceBuffer - data: { id : demuxer id, frag : fragment object, stats : { trequest, tfirst, tload, tparsed, tbuffered, length, bwEstimate } } - FRAG_BUFFERED: 'hlsFragBuffered', - // fired when fragment matching with current media position is changing - data : { id : demuxer id, frag : fragment object } - FRAG_CHANGED: 'hlsFragChanged', - // Identifier for a FPS drop event - data: { curentDropped, currentDecoded, totalDroppedFrames } - FPS_DROP: 'hlsFpsDrop', - // triggered when FPS drop triggers auto level capping - data: { level, droppedlevel } - FPS_DROP_LEVEL_CAPPING: 'hlsFpsDropLevelCapping', - // Identifier for an error event - data: { type : error type, details : error details, fatal : if true, hls.js cannot/will not try to recover, if false, hls.js will try to recover,other error specific data } - ERROR: 'hlsError', - // fired when hls.js instance starts destroying. Different from MEDIA_DETACHED as one could want to detach and reattach a media to the instance of hls.js to handle mid-rolls for example - data: { } - DESTROYING: 'hlsDestroying', - // fired when a decrypt key loading starts - data: { frag : fragment object } - KEY_LOADING: 'hlsKeyLoading', - // fired when a decrypt key loading is completed - data: { frag : fragment object, payload : key payload, stats : { trequest, tfirst, tload, length } } - KEY_LOADED: 'hlsKeyLoaded', - // fired upon stream controller state transitions - data: { previousState, nextState } - STREAM_STATE_TRANSITION: 'hlsStreamStateTransition' + FRAG_LOADED: 'hlsFragLoaded' }; export default HlsEvents; diff --git a/cmd/mjpeg-player/hlsjs/hls.js b/cmd/mjpeg-player/hlsjs/hls.js index 88107af2..a344d330 100644 --- a/cmd/mjpeg-player/hlsjs/hls.js +++ b/cmd/mjpeg-player/hlsjs/hls.js @@ -1,674 +1,50 @@ -import * as URLToolkit from 'url-toolkit'; +/* +AUTHOR + Trek Hopton -import { - ErrorTypes, - ErrorDetails -} from './errors'; +LICENSE + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) -import PlaylistLoader from './loader/playlist-loader'; -import FragmentLoader from './loader/fragment-loader'; -import KeyLoader from './loader/key-loader'; + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. -import { FragmentTracker } from './controller/fragment-tracker'; -import StreamController from './controller/stream-controller'; -import LevelController from './controller/level-controller'; -import ID3TrackController from './controller/id3-track-controller'; + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. -import { isSupported } from './is-supported'; -import { logger, enableLogs } from './utils/logger'; -import { hlsDefaultConfig, HlsConfig } from './config'; + You should have received a copy of the GNU General Public License in gpl.txt. + If not, see http://www.gnu.org/licenses. -import HlsEvents from './events'; + For hls.js Copyright notice and license, see LICENSE file. +*/ -import { Observer } from './observer'; +import URLToolkit from '../url-toolkit/url-toolkit.js'; +import HlsEvents from './events.js'; +import PlaylistLoader from './loader/playlist-loader.js'; +import FragmentLoader from './loader/fragment-loader.js'; +import StreamController from './controller/stream-controller.js'; +import { hlsDefaultConfig } from './config.js'; +import { Observer } from './observer.js'; -/** - * @module Hls - * @class - * @constructor - */ -export default class Hls extends Observer { - public static defaultConfig?: HlsConfig; - public config: HlsConfig; - - private _autoLevelCapping: number; - private abrController: any; - private capLevelController: any; - private levelController: any; - private streamController: any; - private networkControllers: any[]; - private audioTrackController: any; - private subtitleTrackController: any; - private emeController: any; - private coreComponents: any[]; - private media: HTMLMediaElement | null = null; - private url: string | null = null; - - /** - * @type {string} - */ - static get version (): string { - return __VERSION__; - } - - /** - * @type {boolean} - */ - static isSupported (): boolean { - return isSupported(); - } - - /** - * @type {HlsEvents} - */ - static get Events () { - return HlsEvents; - } - - /** - * @type {HlsErrorTypes} - */ - static get ErrorTypes () { - return ErrorTypes; - } - - /** - * @type {HlsErrorDetails} - */ - static get ErrorDetails () { - return ErrorDetails; - } - - /** - * @type {HlsConfig} - */ - static get DefaultConfig (): HlsConfig { - if (!Hls.defaultConfig) { - return hlsDefaultConfig; - } - - return Hls.defaultConfig; - } - - /** - * @type {HlsConfig} - */ - static set DefaultConfig (defaultConfig: HlsConfig) { - Hls.defaultConfig = defaultConfig; - } - - /** - * Creates an instance of an HLS client that can attach to exactly one `HTMLMediaElement`. - * - * @constructs Hls - * @param {HlsConfig} config - */ - constructor (userConfig: Partial = {}) { +class Hls extends Observer { + constructor() { super(); + this.pLoader = new PlaylistLoader(this); + this.streamController = new StreamController(this); + this.fragmentLoader = new FragmentLoader(this); - const defaultConfig = Hls.DefaultConfig; - - if ((userConfig.liveSyncDurationCount || userConfig.liveMaxLatencyDurationCount) && (userConfig.liveSyncDuration || userConfig.liveMaxLatencyDuration)) { - throw new Error('Illegal hls.js config: don\'t mix up liveSyncDurationCount/liveMaxLatencyDurationCount and liveSyncDuration/liveMaxLatencyDuration'); - } - - // Shallow clone - this.config = { - ...defaultConfig, - ...userConfig - }; - - const { config } = this; - - if (config.liveMaxLatencyDurationCount !== void 0 && config.liveMaxLatencyDurationCount <= config.liveSyncDurationCount) { - throw new Error('Illegal hls.js config: "liveMaxLatencyDurationCount" must be gt "liveSyncDurationCount"'); - } - - if (config.liveMaxLatencyDuration !== void 0 && (config.liveSyncDuration === void 0 || config.liveMaxLatencyDuration <= config.liveSyncDuration)) { - throw new Error('Illegal hls.js config: "liveMaxLatencyDuration" must be gt "liveSyncDuration"'); - } - - enableLogs(config.debug); - - this._autoLevelCapping = -1; - - // core controllers and network loaders - - /** - * @member {AbrController} abrController - */ - const abrController = this.abrController = new config.abrController(this); // eslint-disable-line new-cap - const bufferController = new config.bufferController(this); // eslint-disable-line new-cap - const capLevelController = this.capLevelController = new config.capLevelController(this); // eslint-disable-line new-cap - const fpsController = new config.fpsController(this); // eslint-disable-line new-cap - const playListLoader = new PlaylistLoader(this); - const fragmentLoader = new FragmentLoader(this); - const keyLoader = new KeyLoader(this); - const id3TrackController = new ID3TrackController(this); - - // network controllers - - /** - * @member {LevelController} levelController - */ - const levelController = this.levelController = new LevelController(this); - - // FIXME: FragmentTracker must be defined before StreamController because the order of event handling is important - const fragmentTracker = new FragmentTracker(this); - - /** - * @member {StreamController} streamController - */ - const streamController = this.streamController = new StreamController(this, fragmentTracker); - - let networkControllers = [levelController, streamController]; - - // optional audio stream controller - /** - * @var {ICoreComponent | Controller} - */ - let Controller = config.audioStreamController; - if (Controller) { - networkControllers.push(new Controller(this, fragmentTracker)); - } - - /** - * @member {INetworkController[]} networkControllers - */ - this.networkControllers = networkControllers; - - /** - * @var {ICoreComponent[]} - */ - const coreComponents = [ - playListLoader, - fragmentLoader, - keyLoader, - abrController, - bufferController, - capLevelController, - fpsController, - id3TrackController, - fragmentTracker - ]; - - // optional audio track and subtitle controller - Controller = config.audioTrackController; - if (Controller) { - const audioTrackController = new Controller(this); - - /** - * @member {AudioTrackController} audioTrackController - */ - this.audioTrackController = audioTrackController; - coreComponents.push(audioTrackController); - } - - Controller = config.subtitleTrackController; - if (Controller) { - const subtitleTrackController = new Controller(this); - - /** - * @member {SubtitleTrackController} subtitleTrackController - */ - this.subtitleTrackController = subtitleTrackController; - networkControllers.push(subtitleTrackController); - } - - Controller = config.emeController; - if (Controller) { - const emeController = new Controller(this); - - /** - * @member {EMEController} emeController - */ - this.emeController = emeController; - coreComponents.push(emeController); - } - - // optional subtitle controllers - Controller = config.subtitleStreamController; - if (Controller) { - networkControllers.push(new Controller(this, fragmentTracker)); - } - Controller = config.timelineController; - if (Controller) { - coreComponents.push(new Controller(this)); - } - - /** - * @member {ICoreComponent[]} - */ - this.coreComponents = coreComponents; + this.config = hlsDefaultConfig; } - /** - * Dispose of the instance - */ - destroy () { - logger.log('destroy'); - this.trigger(HlsEvents.DESTROYING); - this.detachMedia(); - this.coreComponents.concat(this.networkControllers).forEach(component => { - component.destroy(); - }); - this.url = null; - this.removeAllListeners(); - this._autoLevelCapping = -1; - } - - /** - * Attach a media element - * @param {HTMLMediaElement} media - */ - attachMedia (media: HTMLMediaElement) { - logger.log('attachMedia'); - this.media = media; - this.trigger(HlsEvents.MEDIA_ATTACHING, { media: media }); - } - - /** - * Detach from the media - */ - detachMedia () { - logger.log('detachMedia'); - this.trigger(HlsEvents.MEDIA_DETACHING); - this.media = null; - } - - /** - * Set the source URL. Can be relative or absolute. - * @param {string} url - */ - loadSource (url: string) { + // url is the source URL. Can be relative or absolute. + loadSource(url, callback) { + this.loadSuccess = callback; url = URLToolkit.buildAbsoluteURL(window.location.href, url, { alwaysNormalize: true }); - logger.log(`loadSource:${url}`); - this.url = url; - // when attaching to a source URL, trigger a playlist load this.trigger(HlsEvents.MANIFEST_LOADING, { url: url }); } - - /** - * Start loading data from the stream source. - * Depending on default config, client starts loading automatically when a source is set. - * - * @param {number} startPosition Set the start position to stream from - * @default -1 None (from earliest point) - */ - startLoad (startPosition: number = -1) { - logger.log(`startLoad(${startPosition})`); - this.networkControllers.forEach(controller => { - controller.startLoad(startPosition); - }); - } - - /** - * Stop loading of any stream data. - */ - stopLoad () { - logger.log('stopLoad'); - this.networkControllers.forEach(controller => { - controller.stopLoad(); - }); - } - - /** - * Swap through possible audio codecs in the stream (for example to switch from stereo to 5.1) - */ - swapAudioCodec () { - logger.log('swapAudioCodec'); - this.streamController.swapAudioCodec(); - } - - /** - * When the media-element fails, this allows to detach and then re-attach it - * as one call (convenience method). - * - * Automatic recovery of media-errors by this process is configurable. - */ - recoverMediaError () { - logger.log('recoverMediaError'); - let media = this.media; - this.detachMedia(); - if (media) { - this.attachMedia(media); - } - } - - /** - * @type {QualityLevel[]} - */ - // todo(typescript-levelController) - get levels (): any[] { - return this.levelController.levels; - } - - /** - * Index of quality level currently played - * @type {number} - */ - get currentLevel (): number { - return this.streamController.currentLevel; - } - - /** - * Set quality level index immediately . - * This will flush the current buffer to replace the quality asap. - * That means playback will interrupt at least shortly to re-buffer and re-sync eventually. - * @type {number} -1 for automatic level selection - */ - set currentLevel (newLevel: number) { - logger.log(`set currentLevel:${newLevel}`); - this.loadLevel = newLevel; - this.streamController.immediateLevelSwitch(); - } - - /** - * Index of next quality level loaded as scheduled by stream controller. - * @type {number} - */ - get nextLevel (): number { - return this.streamController.nextLevel; - } - - /** - * Set quality level index for next loaded data. - * This will switch the video quality asap, without interrupting playback. - * May abort current loading of data, and flush parts of buffer (outside currently played fragment region). - * @type {number} -1 for automatic level selection - */ - set nextLevel (newLevel: number) { - logger.log(`set nextLevel:${newLevel}`); - this.levelController.manualLevel = newLevel; - this.streamController.nextLevelSwitch(); - } - - /** - * Return the quality level of the currently or last (of none is loaded currently) segment - * @type {number} - */ - get loadLevel (): number { - return this.levelController.level; - } - - /** - * Set quality level index for next loaded data in a conservative way. - * This will switch the quality without flushing, but interrupt current loading. - * Thus the moment when the quality switch will appear in effect will only be after the already existing buffer. - * @type {number} newLevel -1 for automatic level selection - */ - set loadLevel (newLevel: number) { - logger.log(`set loadLevel:${newLevel}`); - this.levelController.manualLevel = newLevel; - } - - /** - * get next quality level loaded - * @type {number} - */ - get nextLoadLevel (): number { - return this.levelController.nextLoadLevel; - } - - /** - * Set quality level of next loaded segment in a fully "non-destructive" way. - * Same as `loadLevel` but will wait for next switch (until current loading is done). - * @type {number} level - */ - set nextLoadLevel (level: number) { - this.levelController.nextLoadLevel = level; - } - - /** - * Return "first level": like a default level, if not set, - * falls back to index of first level referenced in manifest - * @type {number} - */ - get firstLevel (): number { - return Math.max(this.levelController.firstLevel, this.minAutoLevel); - } - - /** - * Sets "first-level", see getter. - * @type {number} - */ - set firstLevel (newLevel: number) { - logger.log(`set firstLevel:${newLevel}`); - this.levelController.firstLevel = newLevel; - } - - /** - * Return start level (level of first fragment that will be played back) - * if not overrided by user, first level appearing in manifest will be used as start level - * if -1 : automatic start level selection, playback will start from level matching download bandwidth - * (determined from download of first segment) - * @type {number} - */ - get startLevel (): number { - return this.levelController.startLevel; - } - - /** - * set start level (level of first fragment that will be played back) - * if not overrided by user, first level appearing in manifest will be used as start level - * if -1 : automatic start level selection, playback will start from level matching download bandwidth - * (determined from download of first segment) - * @type {number} newLevel - */ - set startLevel (newLevel: number) { - logger.log(`set startLevel:${newLevel}`); - // if not in automatic start level detection, ensure startLevel is greater than minAutoLevel - if (newLevel !== -1) { - newLevel = Math.max(newLevel, this.minAutoLevel); - } - - this.levelController.startLevel = newLevel; - } - - /** - * set dynamically set capLevelToPlayerSize against (`CapLevelController`) - * - * @type {boolean} - */ - set capLevelToPlayerSize (shouldStartCapping: boolean) { - const newCapLevelToPlayerSize = !!shouldStartCapping; - - if (newCapLevelToPlayerSize !== this.config.capLevelToPlayerSize) { - if (newCapLevelToPlayerSize) { - this.capLevelController.startCapping(); // If capping occurs, nextLevelSwitch will happen based on size. - } else { - this.capLevelController.stopCapping(); - this.autoLevelCapping = -1; - this.streamController.nextLevelSwitch(); // Now we're uncapped, get the next level asap. - } - - this.config.capLevelToPlayerSize = newCapLevelToPlayerSize; - } - } - - /** - * Capping/max level value that should be used by automatic level selection algorithm (`ABRController`) - * @type {number} - */ - get autoLevelCapping (): number { - return this._autoLevelCapping; - } - - /** - * get bandwidth estimate - * @type {number} - */ - get bandwidthEstimate (): number { - const bwEstimator = this.abrController._bwEstimator; - return bwEstimator ? bwEstimator.getEstimate() : NaN; - } - - /** - * Capping/max level value that should be used by automatic level selection algorithm (`ABRController`) - * @type {number} - */ - set autoLevelCapping (newLevel: number) { - logger.log(`set autoLevelCapping:${newLevel}`); - this._autoLevelCapping = newLevel; - } - - /** - * True when automatic level selection enabled - * @type {boolean} - */ - get autoLevelEnabled (): boolean { - return (this.levelController.manualLevel === -1); - } - - /** - * Level set manually (if any) - * @type {number} - */ - get manualLevel (): number { - return this.levelController.manualLevel; - } - - /** - * min level selectable in auto mode according to config.minAutoBitrate - * @type {number} - */ - get minAutoLevel (): number { - const { levels, config: { minAutoBitrate } } = this; - const len = levels ? levels.length : 0; - - for (let i = 0; i < len; i++) { - const levelNextBitrate = levels[i].realBitrate - ? Math.max(levels[i].realBitrate, levels[i].bitrate) - : levels[i].bitrate; - - if (levelNextBitrate > minAutoBitrate) { - return i; - } - } - - return 0; - } - - /** - * max level selectable in auto mode according to autoLevelCapping - * @type {number} - */ - get maxAutoLevel (): number { - const { levels, autoLevelCapping } = this; - - let maxAutoLevel; - if (autoLevelCapping === -1 && levels && levels.length) { - maxAutoLevel = levels.length - 1; - } else { - maxAutoLevel = autoLevelCapping; - } - - return maxAutoLevel; - } - - /** - * next automatically selected quality level - * @type {number} - */ - get nextAutoLevel (): number { - // ensure next auto level is between min and max auto level - return Math.min(Math.max(this.abrController.nextAutoLevel, this.minAutoLevel), this.maxAutoLevel); - } - - /** - * this setter is used to force next auto level. - * this is useful to force a switch down in auto mode: - * in case of load error on level N, hls.js can set nextAutoLevel to N-1 for example) - * forced value is valid for one fragment. upon succesful frag loading at forced level, - * this value will be resetted to -1 by ABR controller. - * @type {number} - */ - set nextAutoLevel (nextLevel: number) { - this.abrController.nextAutoLevel = Math.max(this.minAutoLevel, nextLevel); - } - - /** - * @type {AudioTrack[]} - */ - // todo(typescript-audioTrackController) - get audioTracks (): any[] { - const audioTrackController = this.audioTrackController; - return audioTrackController ? audioTrackController.audioTracks : []; - } - - /** - * index of the selected audio track (index in audio track lists) - * @type {number} - */ - get audioTrack (): number { - const audioTrackController = this.audioTrackController; - return audioTrackController ? audioTrackController.audioTrack : -1; - } - - /** - * selects an audio track, based on its index in audio track lists - * @type {number} - */ - set audioTrack (audioTrackId: number) { - const audioTrackController = this.audioTrackController; - if (audioTrackController) { - audioTrackController.audioTrack = audioTrackId; - } - } - - /** - * @type {Seconds} - */ - get liveSyncPosition (): number { - return this.streamController.liveSyncPosition; - } - - /** - * get alternate subtitle tracks list from playlist - * @type {SubtitleTrack[]} - */ - // todo(typescript-subtitleTrackController) - get subtitleTracks (): any[] { - const subtitleTrackController = this.subtitleTrackController; - return subtitleTrackController ? subtitleTrackController.subtitleTracks : []; - } - - /** - * index of the selected subtitle track (index in subtitle track lists) - * @type {number} - */ - get subtitleTrack (): number { - const subtitleTrackController = this.subtitleTrackController; - return subtitleTrackController ? subtitleTrackController.subtitleTrack : -1; - } - - /** - * select an subtitle track, based on its index in subtitle track lists - * @type {number} - */ - set subtitleTrack (subtitleTrackId: number) { - const subtitleTrackController = this.subtitleTrackController; - if (subtitleTrackController) { - subtitleTrackController.subtitleTrack = subtitleTrackId; - } - } - - /** - * @type {boolean} - */ - get subtitleDisplay (): boolean { - const subtitleTrackController = this.subtitleTrackController; - return subtitleTrackController ? subtitleTrackController.subtitleDisplay : false; - } - - /** - * Enable/disable subtitle display rendering - * @type {boolean} - */ - set subtitleDisplay (value: boolean) { - const subtitleTrackController = this.subtitleTrackController; - if (subtitleTrackController) { - subtitleTrackController.subtitleDisplay = value; - } - } } + +export default Hls \ No newline at end of file diff --git a/cmd/mjpeg-player/index.html b/cmd/mjpeg-player/index.html index 064f436c..d42069fb 100644 --- a/cmd/mjpeg-player/index.html +++ b/cmd/mjpeg-player/index.html @@ -1,34 +1,59 @@ + + - - Mjpeg Player - - + + Mjpeg Player + -
-
-
-
- -
-
-
- Frame Rate: fps -
-
- -
+
+
+
+
+
+
+
+ + + +
+
+ Frame Rate: fps +
+
+ +
-
-
- ©2019 Australian Ocean Laboratory Limited (AusOcean) (License) -
-
+
+
+
+ ©2020 Australian Ocean Laboratory Limited (AusOcean) (License) +
+
\ No newline at end of file diff --git a/cmd/mjpeg-player/main.js b/cmd/mjpeg-player/main.js index 037b31ed..c51ee771 100644 --- a/cmd/mjpeg-player/main.js +++ b/cmd/mjpeg-player/main.js @@ -1,12 +1,9 @@ /* -NAME - main.js - AUTHOR Trek Hopton LICENSE - This file is Copyright (C) 2019 the Australian Ocean Lab (AusOcean) + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) It is free software: you can redistribute it and/or modify them under the terms of the GNU General Public License as published by the @@ -22,54 +19,114 @@ LICENSE If not, see http://www.gnu.org/licenses. */ -// play will process and play the chosen target file. -function play() { - const viewer = document.getElementById('viewer'); - const input = event.target.files[0]; - const reader = new FileReader(); +import Hls from "./hlsjs/hls.js"; - reader.onload = event => { - const player = new Worker("player.js"); +let started = false; +let player, viewer; - let rate = document.getElementById('rate'); - if (rate.value && rate.value > 0) { - player.postMessage({ msg: "setFrameRate", data: rate.value }); - } +// init gets DOM elements once the document has been loaded and adds listeners where necessary. +function init() { + document.addEventListener('DOMContentLoaded', load); + document.addEventListener('DOMContentLoaded', function () { + document.getElementById('urlBtn').addEventListener('click', load); + document.getElementById('fileInput').addEventListener('change', play); + viewer = document.getElementById('viewer'); + } + ); +} - player.onmessage = e => { - switch (e.data.msg) { - case "frame": - const blob = new Blob([new Uint8Array(e.data.data)], { - type: 'video/x-motion-jpeg' - }); - const url = URL.createObjectURL(blob); - viewer.src = url; - break; - case "log": - console.log(e.data.data); - break; - case "stop": - break; - default: - console.error("unknown message from player"); - break; - } - }; +init(); - switch (input.name.split('.')[1]) { - case "mjpeg": - case "mjpg": - player.postMessage({ msg: "loadMjpeg", data: event.target.result }, [event.target.result]); - break; - case "ts": - player.postMessage({ msg: "loadMtsMjpeg", data: event.target.result }, [event.target.result]); - break; - default: - console.error("unknown file format"); - break; - } - }; - reader.onerror = error => reject(error); - reader.readAsArrayBuffer(input); +// load gets the URL from the URL input element or the browser's URL bar +// and creates an Hls instance to load the content from the URL. +function load() { + let url = document.getElementById('url').value; + if (url == "") { + // Get everything following the ? from the browser's URL bar. + url = window.location.search.slice(1); + document.getElementById('url').value = url; + } else if (url[0] == '/') { + url = window.location.protocol + '//' + window.location.host + url; + } + if (url == "") { + return; + } + + let hls = new Hls(); + hls.loadSource(url, append); +} + +// append, on the first call, starts a player worker and passes it a frame rate and the video data, +// on subsequent calls it passes the video data to the player worker. +function append(data) { + if (!started) { + player = new Worker("player.js"); + + let rate = document.getElementById('rate'); + if (rate.value && rate.value > 0) { + player.postMessage({ msg: "setFrameRate", data: rate.value }); + } + + player.onmessage = handleMessage; + + player.postMessage({ msg: "loadMtsMjpeg", data: data }, [data]); + started = true; + } else { + player.postMessage({ msg: "appendMtsMjpeg", data: data }, [data]); + } } + +// play will process and play the target file chosen with the file input element. +function play() { + const input = event.target.files[0]; + const reader = new FileReader(); + + reader.onload = event => { + const player = new Worker("player.js"); + + let rate = document.getElementById('rate'); + if (rate.value > 0) { + player.postMessage({ msg: "setFrameRate", data: rate.value }); + } + + player.onmessage = handleMessage; + + switch (input.name.split('.')[1]) { + case "mjpeg": + case "mjpg": + player.postMessage({ msg: "loadMjpeg", data: event.target.result }, [event.target.result]); + break; + case "ts": + player.postMessage({ msg: "loadMtsMjpeg", data: event.target.result }, [event.target.result]); + break; + default: + console.error("unknown file format"); + break; + } + }; + reader.onerror = error => reject(error); + reader.readAsArrayBuffer(input); +} + +// handleMessage handles messgaes from the player workers, its main job is to update the display when a frame is received. +function handleMessage(e) { + switch (e.data.msg) { + case "frame": + const blob = new Blob([new Uint8Array(e.data.data)], { + type: 'video/x-motion-jpeg' + }); + const url = URL.createObjectURL(blob); + viewer.src = url; + break; + case "log": + console.log(e.data.data); + break; + case "stop": + console.log("stopped"); + break; + default: + console.error("unknown message from player"); + break; + } +} \ No newline at end of file diff --git a/cmd/mjpeg-player/player.js b/cmd/mjpeg-player/player.js index 34ff75ab..ea96e031 100644 --- a/cmd/mjpeg-player/player.js +++ b/cmd/mjpeg-player/player.js @@ -1,12 +1,9 @@ /* -NAME - player.js - AUTHOR Trek Hopton LICENSE - This file is Copyright (C) 2019 the Australian Ocean Lab (AusOcean) + This file is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) It is free software: you can redistribute it and/or modify them under the terms of the GNU General Public License as published by the @@ -22,68 +19,116 @@ LICENSE If not, see http://www.gnu.org/licenses. */ -let frameRate = 30; +let frameRate = 25; //Keeps track of the frame rate, default is 25fps. +self.importScripts('./lex-mjpeg.js'); +self.importScripts('./hlsjs/mts-demuxer.js'); +const codecs = { + MJPEG: 1, + MTS_MJPEG: 2, +} + +// onmessage is called whenever the main thread sends this worker a message. onmessage = e => { - switch (e.data.msg) { - case "setFrameRate": - frameRate = e.data.data; - break; - case "loadMjpeg": - self.importScripts('./lex-mjpeg.js'); - let mjpeg = new Uint8Array(e.data.data); - let lex = new MJPEGLexer(mjpeg); - player = new Player(lex); - player.setFrameRate(frameRate); - player.start(); - break; - case "loadMtsMjpeg": - self.importScripts('./hlsjs/mts-demuxer.js'); - let mtsMjpeg = new Uint8Array(e.data.data); - let demux = new MTSDemuxer(); - let tracks = demux._getTracks(); - demux.append(mtsMjpeg); - buf = new FrameBuffer(tracks.video.data); - player = new Player(buf); - player.setFrameRate(frameRate); //TODO: read frame rate from metadata. - player.start(); - break; - default: - console.error("unknown message from main thread"); - break; - } + switch (e.data.msg) { + case "setFrameRate": + frameRate = e.data.data; + break; + case "loadMjpeg": + player = new PlayerWorker(); + player.init(codecs.MJPEG); + player.append(e.data.data); + player.setFrameRate(frameRate); + player.start(); + break; + case "loadMtsMjpeg": + player = new PlayerWorker(); + player.init(codecs.MTS_MJPEG); + player.append(e.data.data); + player.start(); + break; + case "appendMtsMjpeg": + player.append(e.data.data); + break; + default: + console.error("unknown message from main thread"); + break; + } }; -class Player { - constructor(buffer) { - this.buffer = buffer; - this.frameRate = frameRate; +// PlayerWorker has a FrameBuffer to hold frames and once started, passes them one at a time to the main thread. +class PlayerWorker { + init(codec) { + this.frameRate = frameRate; + this.codec = codec; + switch (codec) { + case codecs.MJPEG: + this.frameSrc = new MJPEGLexer(); + break; + case codecs.MTS_MJPEG: + this.frameSrc = new FrameBuffer(); + break; } + } - setFrameRate(rate) { - this.frameRate = rate; - } + setFrameRate(rate) { + this.frameRate = rate; + } - start() { - let frame = this.buffer.read(); - if (frame != null) { - postMessage({ msg: "frame", data: frame.buffer }, [frame.buffer]); - setTimeout(() => { this.start(); }, 1000 / this.frameRate); - } else { - postMessage({ msg: "stop" }); - } + start() { + let frame = this.frameSrc.read(); + if (frame != null) { + postMessage({ msg: "frame", data: frame.buffer }, [frame.buffer]); + setTimeout(() => { this.start(); }, 1000 / this.frameRate); + } else { + postMessage({ msg: "stop" }); } + } + + append(data) { + this.frameSrc.append(data); + } } // FrameBuffer allows an array of subarrays (MJPEG frames) to be read one at a time. class FrameBuffer { - constructor(src) { - this.src = src; - this.off = 0; - } + constructor() { + this.segments = []; + this.off = { segment: 0, frame: 0 }; + this.demuxer = new MTSDemuxer(); + } - // read returns the next single frame. - read() { - return this.src[this.off++]; + // read returns the next single frame. + read() { + let off = this.off; + let prevOff = off; + if (this.incrementOff()) { + return this.segments[prevOff.segment][prevOff.frame]; + } else { + return null; } + } + + append(data) { + let demuxed = this.demuxer.demux(new Uint8Array(data)); + this.segments.push(demuxed.data); + } + + incrementOff() { + if (!this.segments || !this.segments[this.off.segment]) { + return false; + } + if (this.off.frame + 1 >= this.segments[this.off.segment].length) { + if (this.off.segment + 1 >= this.segments.length) { + return false; + } else { + this.off.segment++; + this.off.frame = 0; + return true; + } + } else { + this.off.frame++; + return true; + } + } } \ No newline at end of file diff --git a/cmd/revid-cli/main.go b/cmd/revid-cli/main.go index db6e17ec..454bde10 100644 --- a/cmd/revid-cli/main.go +++ b/cmd/revid-cli/main.go @@ -128,6 +128,7 @@ func handleFlags() config.Config { httpAddressPtr = flag.String("HttpAddress", "", "Destination address of http posts") verticalFlipPtr = flag.Bool("VerticalFlip", false, "Flip video vertically: Yes, No") horizontalFlipPtr = flag.Bool("HorizontalFlip", false, "Flip video horizontally: Yes, No") + loopPtr = flag.Bool("Loop", false, "Loop input source on completion (true/false)") bitratePtr = flag.Uint("Bitrate", 0, "Bitrate of recorded video") heightPtr = flag.Uint("Height", 0, "Height in pixels") widthPtr = flag.Uint("Width", 0, "Width in pixels") @@ -138,6 +139,7 @@ func handleFlags() config.Config { saturationPtr = flag.Int("Saturation", 0, "Set Saturation. (100-100)") exposurePtr = flag.String("Exposure", "auto", "Set exposure mode. ("+strings.Join(raspivid.ExposureModes[:], ",")+")") autoWhiteBalancePtr = flag.String("Awb", "auto", "Set automatic white balance mode. ("+strings.Join(raspivid.AutoWhiteBalanceModes[:], ",")+")") + fileFPSPtr = flag.Int("FileFPS", 0, "File source frame processing FPS") // Audio specific flags. sampleRatePtr = flag.Int("SampleRate", 48000, "Sample rate of recorded audio") @@ -254,6 +256,8 @@ func handleFlags() config.Config { netsender.ConfigFile = *configFilePtr } + cfg.FileFPS = *fileFPSPtr + cfg.Loop = *loopPtr cfg.CameraIP = *cameraIPPtr cfg.Rotation = *rotationPtr cfg.HorizontalFlip = *horizontalFlipPtr diff --git a/filter/difference.go b/filter/difference.go new file mode 100644 index 00000000..d01cf0a7 --- /dev/null +++ b/filter/difference.go @@ -0,0 +1,129 @@ +// +build !circleci + +/* +DESCRIPTION + A filter that detects motion and discards frames without motion. The + algorithm calculates the absolute difference for each pixel between + two frames, then finds the mean. If the mean is above a given threshold, + then it is considered motion. + +AUTHORS + Scott Barnard + +LICENSE + difference.go is Copyright (C) 2020 the Australian Ocean Lab (AusOcean) + + It is free software: you can redistribute it and/or modify them + under the terms of the GNU General Public License as published by the + Free Software Foundation, either version 3 of the License, or (at your + option) any later version. + + It is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + in gpl.txt. If not, see http://www.gnu.org/licenses. +*/ + +package filter + +import ( + "fmt" + "image" + "image/color" + "io" + + "gocv.io/x/gocv" +) + +// Difference is a filter that provides basic motion detection. Difference calculates +// the absolute difference for each pixel between two frames, then finds the mean. If +// the mean is above a given threshold, then it is considered motion. +type Difference struct { + dst io.WriteCloser + thresh float64 + prev gocv.Mat + debug bool + windows []*gocv.Window +} + +// NewDifference returns a pointer to a new Difference struct. +func NewDifference(dst io.WriteCloser, debug bool, threshold float64) *Difference { + var windows []*gocv.Window + if debug { + windows = []*gocv.Window{gocv.NewWindow("Diff: Bounding boxes"), gocv.NewWindow("Diff: Motion")} + } + return &Difference{dst, threshold, gocv.NewMat(), debug, windows} +} + +// Implements io.Closer. +// Close frees resources used by gocv, because it has to be done manually, due to +// it using c-go. +func (d *Difference) Close() error { + d.prev.Close() + for _, window := range d.windows { + window.Close() + } + return nil +} + +// Implements io.Writer. +// Write applies the motion filter to the video stream. Only frames with motion +// are written to the destination encoder, frames without are discarded. +func (d *Difference) Write(f []byte) (int, error) { + if d.prev.Empty() { + var err error + d.prev, err = gocv.IMDecode(f, gocv.IMReadColor) + if err != nil { + return 0, err + } + return len(f), nil + } + + img, err := gocv.IMDecode(f, gocv.IMReadColor) + defer img.Close() + if err != nil { + return 0, err + } + + imgDelta := gocv.NewMat() + defer imgDelta.Close() + + // Seperate foreground and background. + gocv.AbsDiff(img, d.prev, &imgDelta) + gocv.CvtColor(imgDelta, &imgDelta, gocv.ColorBGRToGray) + + mean := imgDelta.Mean().Val1 + + // Update History. + d.prev = img.Clone() + + // Draw debug information. + if d.debug { + if mean >= d.thresh { + gocv.PutText( + &img, + fmt.Sprintf("motion - mean:%f", mean), + image.Pt(32, 32), + gocv.FontHersheyPlain, + 2.0, + color.RGBA{255, 0, 0, 0}, + 2, + ) + } + + d.windows[0].IMShow(img) + d.windows[1].IMShow(imgDelta) + d.windows[0].WaitKey(1) + } + + // Don't write to destination if there is no motion. + if mean < d.thresh { + return len(f), nil + } + + // Write to destination. + return d.dst.Write(f) +} diff --git a/filter/filters_circleci.go b/filter/filters_circleci.go index 374e8b0a..aaca83f6 100644 --- a/filter/filters_circleci.go +++ b/filter/filters_circleci.go @@ -36,6 +36,12 @@ func NewMOGFilter(dst io.WriteCloser, area, threshold float64, history int, debu return &NoOp{dst: dst} } +// NewKNNFilter returns a pointer to a new NoOp struct for testing purposes only. func NewKNNFilter(dst io.WriteCloser, area, threshold float64, history, kernelSize int, debug bool, hf int) *NoOp { return &NoOp{dst: dst} } + +// NewDiffference returns a pointer to a new NoOp struct for testing purposes only. +func NewDifference(dst io.WriteCloser, debug bool, threshold float64) *NoOp { + return &NoOp{dst: dst} +} diff --git a/go.mod b/go.mod index 2ed4c734..dfa6f093 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.13 require ( bitbucket.org/ausocean/iot v1.2.13 - bitbucket.org/ausocean/utils v1.2.12 + bitbucket.org/ausocean/utils v1.2.13 github.com/Comcast/gots v0.0.0-20190305015453-8d56e473f0f7 github.com/go-audio/audio v0.0.0-20181013203223-7b2a6ca21480 github.com/go-audio/wav v0.0.0-20181013172942-de841e69b884 diff --git a/go.sum b/go.sum index ef1fb9ff..d74fe617 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ bitbucket.org/ausocean/utils v1.2.11 h1:zA0FOaPjN960ryp8PKCkV5y50uWBYrIxCVnXjwbv bitbucket.org/ausocean/utils v1.2.11/go.mod h1:uXzX9z3PLemyURTMWRhVI8uLhPX4uuvaaO85v2hcob8= bitbucket.org/ausocean/utils v1.2.12 h1:VnskjWTDM475TnQRhBQE0cNp9D6Y6OELrd4UkD2VVIQ= bitbucket.org/ausocean/utils v1.2.12/go.mod h1:uXzX9z3PLemyURTMWRhVI8uLhPX4uuvaaO85v2hcob8= +bitbucket.org/ausocean/utils v1.2.13 h1:tUaIywtoMc1+zl1GCVQokX4mL5X7LNHX5O51AgAPrWA= +bitbucket.org/ausocean/utils v1.2.13/go.mod h1:uXzX9z3PLemyURTMWRhVI8uLhPX4uuvaaO85v2hcob8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Comcast/gots v0.0.0-20190305015453-8d56e473f0f7 h1:LdOc9B9Bj6LEsKiXShkLA3/kpxXb6LJpH+ekU2krbzw= diff --git a/revid/config/config.go b/revid/config/config.go index 194faf33..0192b1bd 100644 --- a/revid/config/config.go +++ b/revid/config/config.go @@ -86,6 +86,7 @@ const ( defaultAudioInputCodec = codecutil.ADPCM defaultPSITime = 2 defaultMotionInterval = 5 + defaultFileFPS = 0 // Ring buffer defaults. defaultRBMaxElements = 10000 @@ -126,6 +127,7 @@ const ( FilterMOG FilterVariableFPS FilterKNN + FilterDifference ) // OS names @@ -297,6 +299,14 @@ type Config struct { MOGMinArea float64 // Used to ignore small areas of motion detection. MOGThreshold float64 // Intensity value from the KNN motion detection algorithm that is considered motion. MOGHistory uint // Length of MOG filter's history + + // If true will restart reading of input after an io.EOF. + Loop bool + + // Defines the rate at which frames from a file source are processed. + FileFPS int + // Difference filter parameters. + DiffThreshold float64 // Intensity value from the Difference motion detection algorithm that is considered motion. } // TypeData contains information about all of the variables that @@ -310,8 +320,10 @@ var TypeData = map[string]string{ "CameraIP": "string", "CBR": "bool", "ClipDuration": "uint", + "DiffThreshold": "float", "Exposure": "enum:auto,night,nightpreview,backlight,spotlight,sports,snow,beach,verylong,fixedfps,antishake,fireworks", - "Filters": "enums:NoOp,MOG,VariableFPS,KNN", + "FileFPS": "int", + "Filters": "enums:NoOp,MOG,VariableFPS,KNN,Difference", "FrameRate": "uint", "Height": "uint", "HorizontalFlip": "bool", @@ -324,20 +336,21 @@ var TypeData = map[string]string{ "KNNMinArea": "float", "KNNThreshold": "float", "logging": "enum:Debug,Info,Warning,Error,Fatal", + "Loop": "bool", "MinFPS": "float", "MinFrames": "uint", - "mode": "enum:Normal,Paused,Burst", + "mode": "enum:Normal,Paused,Burst,Loop", "MOGHistory": "uint", "MOGMinArea": "float", "MOGThreshold": "float", "MotionInterval": "int", - "RBCapacity": "uint", - "RBMaxElements": "uint", - "RBWriteTimeout": "uint", "Output": "enum:File,Http,Rtmp,Rtp", "OutputPath": "string", "Outputs": "enums:File,Http,Rtmp,Rtp", "Quantization": "uint", + "RBCapacity": "uint", + "RBMaxElements": "uint", + "RBWriteTimeout": "uint", "Rotation": "uint", "RTMPURL": "string", "RTPAddress": "string", @@ -521,6 +534,11 @@ func (c *Config) Validate() error { } } + if c.FileFPS <= 0 || (c.FileFPS > 0 && c.Input != InputFile) { + c.Logger.Log(logger.Info, pkg+"FileFPS bad or unset, defaulting", "FileFPS", defaultFileFPS) + c.FileFPS = defaultFileFPS + } + return nil } diff --git a/revid/revid.go b/revid/revid.go index 8b1b42da..5eb5b7bf 100644 --- a/revid/revid.go +++ b/revid/revid.go @@ -7,6 +7,7 @@ AUTHORS Alan Noble Dan Kortschak Trek Hopton + Scott Barnard LICENSE revid is Copyright (C) 2017-2020 the Australian Ocean Lab (AusOcean) @@ -52,6 +53,7 @@ import ( "bitbucket.org/ausocean/av/filter" "bitbucket.org/ausocean/av/revid/config" "bitbucket.org/ausocean/iot/pi/netsender" + "bitbucket.org/ausocean/utils/bitrate" "bitbucket.org/ausocean/utils/ioext" "bitbucket.org/ausocean/utils/logger" "bitbucket.org/ausocean/utils/vring" @@ -115,6 +117,9 @@ type Revid struct { // err will channel errors from revid routines to the handle errors routine. err chan error + + // bitrate is used for bitrate calculations. + bitrate bitrate.Calculator } // New returns a pointer to a new Revid with the desired configuration, and/or @@ -148,10 +153,8 @@ func (r *Revid) handleErrors() { } // Bitrate returns the result of the most recent bitrate check. -// -// TODO: get this working again. func (r *Revid) Bitrate() int { - return -1 + return r.bitrate.Bitrate() } // reset swaps the current config of a Revid with the passed @@ -266,14 +269,14 @@ func (r *Revid) setupPipeline(mtsEnc func(dst io.WriteCloser, rate float64) (io. return fmt.Errorf("could not initialise MTS ring buffer: %w", err) } w = newMtsSender( - newHttpSender(r.ns, r.cfg.Logger.Log), + newHttpSender(r.ns, r.cfg.Logger.Log, r.bitrate.Report), r.cfg.Logger.Log, rb, r.cfg.ClipDuration, ) mtsSenders = append(mtsSenders, w) case config.OutputRTP: - w, err := newRtpSender(r.cfg.RTPAddress, r.cfg.Logger.Log, r.cfg.FrameRate) + w, err := newRtpSender(r.cfg.RTPAddress, r.cfg.Logger.Log, r.cfg.FrameRate, r.bitrate.Report) if err != nil { r.cfg.Logger.Log(logger.Warning, pkg+"rtp connect error", "error", err.Error()) } @@ -295,6 +298,7 @@ func (r *Revid) setupPipeline(mtsEnc func(dst io.WriteCloser, rate float64) (io. rtmpConnectionMaxTries, rb, r.cfg.Logger.Log, + r.bitrate.Report, ) if err != nil { r.cfg.Logger.Log(logger.Warning, pkg+"rtmp connect error", "error", err.Error()) @@ -342,6 +346,8 @@ func (r *Revid) setupPipeline(mtsEnc func(dst io.WriteCloser, rate float64) (io. r.filters[i] = filter.NewVariableFPSFilter(dst, r.cfg.MinFPS, filter.NewMOGFilter(dst, r.cfg.MOGMinArea, r.cfg.MOGThreshold, int(r.cfg.MOGHistory), r.cfg.ShowWindows, r.cfg.MotionInterval)) case config.FilterKNN: r.filters[i] = filter.NewKNNFilter(dst, r.cfg.KNNMinArea, r.cfg.KNNThreshold, int(r.cfg.KNNHistory), int(r.cfg.KNNKernel), r.cfg.ShowWindows, r.cfg.MotionInterval) + case config.FilterDifference: + r.filters[i] = filter.NewDifference(dst, r.cfg.ShowWindows, r.cfg.DiffThreshold) default: panic("Undefined Filter") } @@ -428,13 +434,14 @@ func (r *Revid) Start() error { return err } - err = r.input.Start() - if err != nil { - return fmt.Errorf("could not start input device: %w", err) + // Calculate delay between frames based on FileFPS. + d := time.Duration(0) + if r.cfg.FileFPS != 0 { + d = time.Duration(1000/r.cfg.FileFPS) * time.Millisecond } r.wg.Add(1) - go r.processFrom(r.input, 0) + go r.processFrom(r.input, d) r.running = true return nil @@ -662,7 +669,7 @@ func (r *Revid) Update(vars map[string]string) error { } case "Filters": filters := strings.Split(value, ",") - m := map[string]int{"NoOp": config.FilterNoOp, "MOG": config.FilterMOG, "VariableFPS": config.FilterVariableFPS, "KNN": config.FilterKNN} + m := map[string]int{"NoOp": config.FilterNoOp, "MOG": config.FilterMOG, "VariableFPS": config.FilterVariableFPS, "KNN": config.FilterKNN, "Difference": config.FilterDifference} r.cfg.Filters = make([]int, len(filters)) for i, filter := range filters { v, ok := m[filter] @@ -810,6 +817,13 @@ func (r *Revid) Update(vars map[string]string) error { break } r.cfg.KNNThreshold = v + case "DiffThreshold": + v, err := strconv.ParseFloat(value, 64) + if err != nil { + r.cfg.Logger.Log(logger.Warning, pkg+"invalid DiffThreshold var", "value", value) + break + } + r.cfg.DiffThreshold = v case "KNNKernel": v, err := strconv.Atoi(value) if err != nil { @@ -845,6 +859,18 @@ func (r *Revid) Update(vars map[string]string) error { break } r.cfg.MOGHistory = uint(v) + case "FileFPS": + v, err := strconv.Atoi(value) + if err != nil { + r.cfg.Logger.Log(logger.Warning, pkg+"invalid FileFPS var", "value", value) + break + } + r.cfg.FileFPS = v + case "mode": + r.cfg.Loop = false + if value == "Loop" { + r.cfg.Loop = true + } } } r.cfg.Logger.Log(logger.Info, pkg+"revid config changed", "config", fmt.Sprintf("%+v", r.cfg)) @@ -853,14 +879,34 @@ func (r *Revid) Update(vars map[string]string) error { // processFrom is run as a routine to read from a input data source, lex and // then send individual access units to revid's encoders. -func (r *Revid) processFrom(read io.Reader, delay time.Duration) { - err := r.lexTo(r.filters[0], read, delay) - r.cfg.Logger.Log(logger.Debug, pkg+"finished lexing") - switch err { - case nil: // Do nothing. - case io.EOF: // TODO: handle this depending on loop mode. - default: - r.err <- err +func (r *Revid) processFrom(in device.AVDevice, delay time.Duration) { + defer r.wg.Done() + + for l := true; l; l = r.cfg.Loop { + err := in.Start() + if err != nil { + r.err <- fmt.Errorf("could not start input device: %w", err) + return + } + + // Lex data from input device, in, until finished or an error is encountered. + // For a continuous source e.g. a camera or microphone, we should remain + // in this call indefinitely unless in.Stop() is called and an io.EOF is forced. + r.cfg.Logger.Log(logger.Info, pkg+"lexing") + err = r.lexTo(r.filters[0], in, delay) + switch err { + case nil, io.EOF: + case io.ErrUnexpectedEOF: + r.cfg.Logger.Log(logger.Info, pkg+"unexpected EOF from input") + default: + r.err <- err + } + + err = in.Stop() + if err != nil { + r.err <- fmt.Errorf("could not stop input source: %w", err) + } } - r.wg.Done() + + r.cfg.Logger.Log(logger.Info, pkg+"finished lexing") } diff --git a/revid/senders.go b/revid/senders.go index 4871ddb5..4929bd40 100644 --- a/revid/senders.go +++ b/revid/senders.go @@ -61,19 +61,25 @@ const ( type httpSender struct { client *netsender.Sender log func(lvl int8, msg string, args ...interface{}) + report func(sent int) } // newHttpSender returns a pointer to a new httpSender. -func newHttpSender(ns *netsender.Sender, log func(lvl int8, msg string, args ...interface{})) *httpSender { +func newHttpSender(ns *netsender.Sender, log func(lvl int8, msg string, args ...interface{}), report func(sent int)) *httpSender { return &httpSender{ client: ns, log: log, + report: report, } } // Write implements io.Writer. func (s *httpSender) Write(d []byte) (int, error) { - return len(d), httpSend(d, s.client, s.log) + err := httpSend(d, s.client, s.log) + if err == nil { + s.report(len(d)) + } + return len(d), err } func (s *httpSender) Close() error { return nil } @@ -276,9 +282,10 @@ type rtmpSender struct { ring *vring.Buffer done chan struct{} wg sync.WaitGroup + report func(sent int) } -func newRtmpSender(url string, timeout uint, retries int, rb *vring.Buffer, log func(lvl int8, msg string, args ...interface{})) (*rtmpSender, error) { +func newRtmpSender(url string, timeout uint, retries int, rb *vring.Buffer, log func(lvl int8, msg string, args ...interface{}), report func(sent int)) (*rtmpSender, error) { var conn *rtmp.Conn var err error for n := 0; n < retries; n++ { @@ -299,6 +306,7 @@ func newRtmpSender(url string, timeout uint, retries int, rb *vring.Buffer, log log: log, ring: rb, done: make(chan struct{}), + report: report, } s.wg.Add(1) go s.output() @@ -364,6 +372,7 @@ func (s *rtmpSender) Write(d []byte) (int, error) { if err != nil { s.log(logger.Warning, pkg+"rtmpSender: ring buffer write error", "error", err.Error()) } + s.report(len(d)) return len(d), nil } @@ -404,9 +413,10 @@ type rtpSender struct { log func(lvl int8, msg string, args ...interface{}) encoder *rtp.Encoder data []byte + report func(sent int) } -func newRtpSender(addr string, log func(lvl int8, msg string, args ...interface{}), fps uint) (*rtpSender, error) { +func newRtpSender(addr string, log func(lvl int8, msg string, args ...interface{}), fps uint, report func(sent int)) (*rtpSender, error) { conn, err := net.Dial("udp", addr) if err != nil { return nil, err @@ -414,6 +424,7 @@ func newRtpSender(addr string, log func(lvl int8, msg string, args ...interface{ s := &rtpSender{ log: log, encoder: rtp.NewEncoder(conn, int(fps)), + report: report, } return s, nil } @@ -426,6 +437,7 @@ func (s *rtpSender) Write(d []byte) (int, error) { if err != nil { s.log(logger.Warning, pkg+"rtpSender: write error", err.Error()) } + s.report(len(d)) return len(d), nil }