Favivon - Correção

This commit is contained in:
2026-05-30 19:59:39 -03:00
parent 76ddaa815d
commit d7dfd221f0
32859 changed files with 5459654 additions and 404 deletions
+96
View File
@@ -0,0 +1,96 @@
/**
* AAC demuxer
*/
import { getId3Data } from '@svta/common-media-library/id3/getId3Data';
import * as ADTS from './adts';
import BaseAudioDemuxer from './base-audio-demuxer';
import * as MpegAudio from './mpegaudio';
import type { HlsConfig } from '../../config';
import type { HlsEventEmitter } from '../../events';
import type { DemuxedAudioTrack } from '../../types/demuxer';
import type { ILogger } from '../../utils/logger';
class AACDemuxer extends BaseAudioDemuxer {
private readonly observer: HlsEventEmitter;
private readonly config: HlsConfig;
constructor(observer: HlsEventEmitter, config) {
super();
this.observer = observer;
this.config = config;
}
resetInitSegment(
initSegment: Uint8Array | undefined,
audioCodec: string | undefined,
videoCodec: string | undefined,
trackDuration: number,
) {
super.resetInitSegment(initSegment, audioCodec, videoCodec, trackDuration);
this._audioTrack = {
container: 'audio/adts',
type: 'audio',
id: 2,
pid: -1,
sequenceNumber: 0,
segmentCodec: 'aac',
samples: [],
manifestCodec: audioCodec,
duration: trackDuration,
inputTimeScale: 90000,
dropped: 0,
};
}
// Source for probe info - https://wiki.multimedia.cx/index.php?title=ADTS
static probe(data: Uint8Array | undefined, logger: ILogger): boolean {
if (!data) {
return false;
}
// Check for the ADTS sync word
// Look for ADTS header | 1111 1111 | 1111 X00X | where X can be either 0 or 1
// Layer bits (position 14 and 15) in header should be always 0 for ADTS
// More info https://wiki.multimedia.cx/index.php?title=ADTS
const id3Data = getId3Data(data, 0);
let offset = id3Data?.length || 0;
if (MpegAudio.probe(data, offset)) {
return false;
}
for (let length = data.length; offset < length; offset++) {
if (ADTS.probe(data, offset)) {
logger.log('ADTS sync word found !');
return true;
}
}
return false;
}
canParse(data, offset) {
return ADTS.canParse(data, offset);
}
appendFrame(track: DemuxedAudioTrack, data: Uint8Array, offset: number) {
ADTS.initTrackConfig(
track,
this.observer,
data,
offset,
track.manifestCodec,
);
const frame = ADTS.appendFrame(
track,
data,
offset,
this.basePTS as number,
this.frameIndex,
);
if (frame && frame.missing === 0) {
return frame;
}
}
}
export default AACDemuxer;
+170
View File
@@ -0,0 +1,170 @@
import { getId3Data } from '@svta/common-media-library/id3/getId3Data';
import { getId3Timestamp } from '@svta/common-media-library/id3/getId3Timestamp';
import BaseAudioDemuxer from './base-audio-demuxer';
import { getAudioBSID } from './dolby';
import type { HlsEventEmitter } from '../../events';
import type { AudioFrame, DemuxedAudioTrack } from '../../types/demuxer';
export class AC3Demuxer extends BaseAudioDemuxer {
private readonly observer: HlsEventEmitter;
constructor(observer: HlsEventEmitter) {
super();
this.observer = observer;
}
resetInitSegment(
initSegment: Uint8Array | undefined,
audioCodec: string | undefined,
videoCodec: string | undefined,
trackDuration: number,
) {
super.resetInitSegment(initSegment, audioCodec, videoCodec, trackDuration);
this._audioTrack = {
container: 'audio/ac-3',
type: 'audio',
id: 2,
pid: -1,
sequenceNumber: 0,
segmentCodec: 'ac3',
samples: [],
manifestCodec: audioCodec,
duration: trackDuration,
inputTimeScale: 90000,
dropped: 0,
};
}
canParse(data: Uint8Array, offset: number): boolean {
return offset + 64 < data.length;
}
appendFrame(
track: DemuxedAudioTrack,
data: Uint8Array,
offset: number,
): AudioFrame | void {
const frameLength = appendFrame(
track,
data,
offset,
this.basePTS as number,
this.frameIndex,
);
if (frameLength !== -1) {
const sample = track.samples[track.samples.length - 1];
return { sample, length: frameLength, missing: 0 };
}
}
static probe(data: Uint8Array | undefined): boolean {
if (!data) {
return false;
}
const id3Data = getId3Data(data, 0);
if (!id3Data) {
return false;
}
// look for the ac-3 sync bytes
const offset = id3Data.length;
if (
data[offset] === 0x0b &&
data[offset + 1] === 0x77 &&
getId3Timestamp(id3Data) !== undefined &&
// check the bsid to confirm ac-3
getAudioBSID(data, offset) < 16
) {
return true;
}
return false;
}
}
export function appendFrame(
track: DemuxedAudioTrack,
data: Uint8Array,
start: number,
pts: number,
frameIndex: number,
): number {
if (start + 8 > data.length) {
return -1; // not enough bytes left
}
if (data[start] !== 0x0b || data[start + 1] !== 0x77) {
return -1; // invalid magic
}
// get sample rate
const samplingRateCode = data[start + 4] >> 6;
if (samplingRateCode >= 3) {
return -1; // invalid sampling rate
}
const samplingRateMap = [48000, 44100, 32000];
const sampleRate = samplingRateMap[samplingRateCode];
// get frame size
const frameSizeCode = data[start + 4] & 0x3f;
const frameSizeMap = [
64, 69, 96, 64, 70, 96, 80, 87, 120, 80, 88, 120, 96, 104, 144, 96, 105,
144, 112, 121, 168, 112, 122, 168, 128, 139, 192, 128, 140, 192, 160, 174,
240, 160, 175, 240, 192, 208, 288, 192, 209, 288, 224, 243, 336, 224, 244,
336, 256, 278, 384, 256, 279, 384, 320, 348, 480, 320, 349, 480, 384, 417,
576, 384, 418, 576, 448, 487, 672, 448, 488, 672, 512, 557, 768, 512, 558,
768, 640, 696, 960, 640, 697, 960, 768, 835, 1152, 768, 836, 1152, 896, 975,
1344, 896, 976, 1344, 1024, 1114, 1536, 1024, 1115, 1536, 1152, 1253, 1728,
1152, 1254, 1728, 1280, 1393, 1920, 1280, 1394, 1920,
];
const frameLength = frameSizeMap[frameSizeCode * 3 + samplingRateCode] * 2;
if (start + frameLength > data.length) {
return -1;
}
// get channel count
const channelMode = data[start + 6] >> 5;
let skipCount = 0;
if (channelMode === 2) {
skipCount += 2;
} else {
if (channelMode & 1 && channelMode !== 1) {
skipCount += 2;
}
if (channelMode & 4) {
skipCount += 2;
}
}
const lfeon =
(((data[start + 6] << 8) | data[start + 7]) >> (12 - skipCount)) & 1;
const channelsMap = [2, 1, 2, 3, 3, 4, 4, 5];
const channelCount = channelsMap[channelMode] + lfeon;
// build dac3 box
const bsid = data[start + 5] >> 3;
const bsmod = data[start + 5] & 7;
const config = new Uint8Array([
(samplingRateCode << 6) | (bsid << 1) | (bsmod >> 2),
((bsmod & 3) << 6) |
(channelMode << 3) |
(lfeon << 2) |
(frameSizeCode >> 4),
(frameSizeCode << 4) & 0xe0,
]);
const frameDuration = (1536 / sampleRate) * 90000;
const stamp = pts + frameIndex * frameDuration;
const unit = data.subarray(start, start + frameLength);
track.config = config;
track.channelCount = channelCount;
track.samplerate = sampleRate;
track.samples.push({ unit, pts: stamp });
return frameLength;
}
+249
View File
@@ -0,0 +1,249 @@
/**
* ADTS parser helper
* @link https://wiki.multimedia.cx/index.php?title=ADTS
*/
import { ErrorDetails, ErrorTypes } from '../../errors';
import { Events } from '../../events';
import { logger } from '../../utils/logger';
import type { HlsEventEmitter } from '../../events';
import type {
AudioFrame,
AudioSample,
DemuxedAudioTrack,
} from '../../types/demuxer';
type AudioConfig = {
config: [number, number];
samplerate: number;
channelCount: number;
codec: string;
parsedCodec: string;
manifestCodec: string | undefined;
};
type FrameHeader = {
headerLength: number;
frameLength: number;
};
export function getAudioConfig(
observer: HlsEventEmitter,
data: Uint8Array,
offset: number,
manifestCodec: string | undefined,
): AudioConfig | void {
const adtsSamplingRates = [
96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025,
8000, 7350,
];
const byte2 = data[offset + 2];
const adtsSamplingIndex = (byte2 >> 2) & 0xf;
if (adtsSamplingIndex > 12) {
const error = new Error(`invalid ADTS sampling index:${adtsSamplingIndex}`);
observer.emit(Events.ERROR, Events.ERROR, {
type: ErrorTypes.MEDIA_ERROR,
details: ErrorDetails.FRAG_PARSING_ERROR,
fatal: true,
error,
reason: error.message,
});
return;
}
// MPEG-4 Audio Object Type (profile_ObjectType+1)
const adtsObjectType = ((byte2 >> 6) & 0x3) + 1;
const channelCount = ((data[offset + 3] >> 6) & 0x3) | ((byte2 & 1) << 2);
const codec = 'mp4a.40.' + adtsObjectType;
/* refer to http://wiki.multimedia.cx/index.php?title=MPEG-4_Audio#Audio_Specific_Config
ISO/IEC 14496-3 - Table 1.13 — Syntax of AudioSpecificConfig()
Audio Profile / Audio Object Type
0: Null
1: AAC Main
2: AAC LC (Low Complexity)
3: AAC SSR (Scalable Sample Rate)
4: AAC LTP (Long Term Prediction)
5: SBR (Spectral Band Replication)
6: AAC Scalable
sampling freq
0: 96000 Hz
1: 88200 Hz
2: 64000 Hz
3: 48000 Hz
4: 44100 Hz
5: 32000 Hz
6: 24000 Hz
7: 22050 Hz
8: 16000 Hz
9: 12000 Hz
10: 11025 Hz
11: 8000 Hz
12: 7350 Hz
13: Reserved
14: Reserved
15: frequency is written explictly
Channel Configurations
These are the channel configurations:
0: Defined in AOT Specifc Config
1: 1 channel: front-center
2: 2 channels: front-left, front-right
*/
// audioObjectType = profile => profile, the MPEG-4 Audio Object Type minus 1
const samplerate = adtsSamplingRates[adtsSamplingIndex];
let aacSampleIndex = adtsSamplingIndex;
if (adtsObjectType === 5 || adtsObjectType === 29) {
// HE-AAC uses SBR (Spectral Band Replication) , high frequencies are constructed from low frequencies
// there is a factor 2 between frame sample rate and output sample rate
// multiply frequency by 2 (see table above, equivalent to substract 3)
aacSampleIndex -= 3;
}
const config: [number, number] = [
(adtsObjectType << 3) | ((aacSampleIndex & 0x0e) >> 1),
((aacSampleIndex & 0x01) << 7) | (channelCount << 3),
];
logger.log(
`manifest codec:${manifestCodec}, parsed codec:${codec}, channels:${channelCount}, rate:${samplerate} (ADTS object type:${adtsObjectType} sampling index:${adtsSamplingIndex})`,
);
return {
config,
samplerate,
channelCount,
codec,
parsedCodec: codec,
manifestCodec,
};
}
export function isHeaderPattern(data: Uint8Array, offset: number): boolean {
return data[offset] === 0xff && (data[offset + 1] & 0xf6) === 0xf0;
}
export function getHeaderLength(data: Uint8Array, offset: number): number {
return data[offset + 1] & 0x01 ? 7 : 9;
}
export function getFullFrameLength(data: Uint8Array, offset: number): number {
return (
((data[offset + 3] & 0x03) << 11) |
(data[offset + 4] << 3) |
((data[offset + 5] & 0xe0) >>> 5)
);
}
export function canGetFrameLength(data: Uint8Array, offset: number): boolean {
return offset + 5 < data.length;
}
export function isHeader(data: Uint8Array, offset: number): boolean {
// Look for ADTS header | 1111 1111 | 1111 X00X | where X can be either 0 or 1
// Layer bits (position 14 and 15) in header should be always 0 for ADTS
// More info https://wiki.multimedia.cx/index.php?title=ADTS
return offset + 1 < data.length && isHeaderPattern(data, offset);
}
export function canParse(data: Uint8Array, offset: number): boolean {
return (
canGetFrameLength(data, offset) &&
isHeaderPattern(data, offset) &&
getFullFrameLength(data, offset) <= data.length - offset
);
}
export function probe(data: Uint8Array, offset: number): boolean {
// same as isHeader but we also check that ADTS frame follows last ADTS frame
// or end of data is reached
if (isHeader(data, offset)) {
// ADTS header Length
const headerLength = getHeaderLength(data, offset);
if (offset + headerLength >= data.length) {
return false;
}
// ADTS frame Length
const frameLength = getFullFrameLength(data, offset);
if (frameLength <= headerLength) {
return false;
}
const newOffset = offset + frameLength;
return newOffset === data.length || isHeader(data, newOffset);
}
return false;
}
export function initTrackConfig(
track: DemuxedAudioTrack,
observer: HlsEventEmitter,
data: Uint8Array,
offset: number,
audioCodec: string | undefined,
) {
if (!track.samplerate) {
const config = getAudioConfig(observer, data, offset, audioCodec);
if (!config) {
return;
}
Object.assign(track, config);
}
}
export function getFrameDuration(samplerate: number): number {
return (1024 * 90000) / samplerate;
}
export function parseFrameHeader(
data: Uint8Array,
offset: number,
): FrameHeader | void {
// The protection skip bit tells us if we have 2 bytes of CRC data at the end of the ADTS header
const headerLength = getHeaderLength(data, offset);
if (offset + headerLength <= data.length) {
// retrieve frame size
const frameLength = getFullFrameLength(data, offset) - headerLength;
if (frameLength > 0) {
// logger.log(`AAC frame, offset/length/total/pts:${offset+headerLength}/${frameLength}/${data.byteLength}`);
return { headerLength, frameLength };
}
}
}
export function appendFrame(
track: DemuxedAudioTrack,
data: Uint8Array,
offset: number,
pts: number,
frameIndex: number,
): AudioFrame {
const frameDuration = getFrameDuration(track.samplerate as number);
const stamp = pts + frameIndex * frameDuration;
const header = parseFrameHeader(data, offset);
let unit: Uint8Array;
if (header) {
const { frameLength, headerLength } = header;
const length = headerLength + frameLength;
const missing = Math.max(0, offset + length - data.length);
// logger.log(`AAC frame ${frameIndex}, pts:${stamp} length@offset/total: ${frameLength}@${offset+headerLength}/${data.byteLength} missing: ${missing}`);
if (missing) {
unit = new Uint8Array(length - headerLength);
unit.set(data.subarray(offset + headerLength, data.length), 0);
} else {
unit = data.subarray(offset + headerLength, offset + length);
}
const sample: AudioSample = {
unit,
pts: stamp,
};
if (!missing) {
track.samples.push(sample as AudioSample);
}
return { sample, length, missing };
}
// overflow incomplete header
const length = data.length - offset;
unit = new Uint8Array(length);
unit.set(data.subarray(offset, data.length), 0);
const sample: AudioSample = {
unit,
pts: stamp,
};
return { sample, length, missing: -1 };
}
+205
View File
@@ -0,0 +1,205 @@
import { canParseId3 } from '@svta/common-media-library/id3/canParseId3';
import { getId3Data } from '@svta/common-media-library/id3/getId3Data';
import { getId3Timestamp } from '@svta/common-media-library/id3/getId3Timestamp';
import {
type AudioFrame,
type DemuxedAudioTrack,
type DemuxedMetadataTrack,
type DemuxedUserdataTrack,
type DemuxedVideoTrackBase,
type Demuxer,
type DemuxerResult,
type KeyData,
MetadataSchema,
} from '../../types/demuxer';
import { appendUint8Array } from '../../utils/mp4-tools';
import { dummyTrack } from '../dummy-demuxed-track';
import type {
RationalTimestamp,
TimestampOffset,
} from '../../utils/timescale-conversion';
class BaseAudioDemuxer implements Demuxer {
protected _audioTrack?: DemuxedAudioTrack;
protected _id3Track?: DemuxedMetadataTrack;
protected frameIndex: number = 0;
protected cachedData: Uint8Array | null = null;
protected basePTS: number | null = null;
protected initPTS: TimestampOffset | null = null;
protected lastPTS: number | null = null;
resetInitSegment(
initSegment: Uint8Array | undefined,
audioCodec: string | undefined,
videoCodec: string | undefined,
trackDuration: number,
) {
this._id3Track = {
type: 'id3',
id: 3,
pid: -1,
inputTimeScale: 90000,
sequenceNumber: 0,
samples: [],
dropped: 0,
};
}
resetTimeStamp(deaultTimestamp: TimestampOffset | null) {
this.initPTS = deaultTimestamp;
this.resetContiguity();
}
resetContiguity(): void {
this.basePTS = null;
this.lastPTS = null;
this.frameIndex = 0;
}
canParse(data: Uint8Array, offset: number): boolean {
return false;
}
appendFrame(
track: DemuxedAudioTrack,
data: Uint8Array,
offset: number,
): AudioFrame | void {}
// feed incoming data to the front of the parsing pipeline
demux(data: Uint8Array, timeOffset: number): DemuxerResult {
if (this.cachedData) {
data = appendUint8Array(this.cachedData, data);
this.cachedData = null;
}
let id3Data: Uint8Array | undefined = getId3Data(data, 0);
let offset = id3Data ? id3Data.length : 0;
let lastDataIndex;
const track = this._audioTrack as DemuxedAudioTrack;
const id3Track = this._id3Track as DemuxedMetadataTrack;
const timestamp = id3Data ? getId3Timestamp(id3Data) : undefined;
const length = data.length;
if (
this.basePTS === null ||
(this.frameIndex === 0 && Number.isFinite(timestamp))
) {
this.basePTS = initPTSFn(timestamp, timeOffset, this.initPTS);
this.lastPTS = this.basePTS;
}
if (this.lastPTS === null) {
this.lastPTS = this.basePTS;
}
// more expressive than alternative: id3Data?.length
if (id3Data && id3Data.length > 0) {
id3Track.samples.push({
pts: this.lastPTS,
dts: this.lastPTS,
data: id3Data,
type: MetadataSchema.audioId3,
duration: Number.POSITIVE_INFINITY,
});
}
while (offset < length) {
if (this.canParse(data, offset)) {
const frame = this.appendFrame(track, data, offset);
if (frame) {
this.frameIndex++;
this.lastPTS = frame.sample.pts;
offset += frame.length;
lastDataIndex = offset;
} else {
offset = length;
}
} else if (canParseId3(data, offset)) {
// after a canParse, a call to getId3Data *should* always returns some data
id3Data = getId3Data(data, offset)!;
id3Track.samples.push({
pts: this.lastPTS,
dts: this.lastPTS,
data: id3Data,
type: MetadataSchema.audioId3,
duration: Number.POSITIVE_INFINITY,
});
offset += id3Data.length;
lastDataIndex = offset;
} else {
offset++;
}
if (offset === length && lastDataIndex !== length) {
const partialData = data.slice(lastDataIndex);
if (this.cachedData) {
this.cachedData = appendUint8Array(this.cachedData, partialData);
} else {
this.cachedData = partialData;
}
}
}
return {
audioTrack: track,
videoTrack: dummyTrack() as DemuxedVideoTrackBase,
id3Track,
textTrack: dummyTrack() as DemuxedUserdataTrack,
};
}
demuxSampleAes(
data: Uint8Array,
keyData: KeyData,
timeOffset: number,
): Promise<DemuxerResult> {
return Promise.reject(
new Error(
`[${this}] This demuxer does not support Sample-AES decryption`,
),
);
}
flush(timeOffset: number): DemuxerResult {
// Parse cache in case of remaining frames.
const cachedData = this.cachedData;
if (cachedData) {
this.cachedData = null;
this.demux(cachedData, 0);
}
return {
audioTrack: this._audioTrack as DemuxedAudioTrack,
videoTrack: dummyTrack() as DemuxedVideoTrackBase,
id3Track: this._id3Track as DemuxedMetadataTrack,
textTrack: dummyTrack() as DemuxedUserdataTrack,
};
}
destroy() {
this.cachedData = null;
// @ts-ignore
this._audioTrack = this._id3Track = undefined;
}
}
/**
* Initialize PTS
* <p>
* use timestamp unless it is undefined, NaN or Infinity
* </p>
*/
export const initPTSFn = (
timestamp: number | undefined,
timeOffset: number,
initPTS: RationalTimestamp | null,
): number => {
if (Number.isFinite(timestamp as number)) {
return timestamp! * 90;
}
const init90kHz = initPTS
? (initPTS.baseTime * 90000) / initPTS.timescale
: 0;
return timeOffset * 90000 + init90kHz;
};
export default BaseAudioDemuxer;
+21
View File
@@ -0,0 +1,21 @@
export const getAudioBSID = (data: Uint8Array, offset: number): number => {
// check the bsid to confirm ac-3 | ec-3
let bsid = 0;
let numBits = 5;
offset += numBits;
const temp = new Uint32Array(1); // unsigned 32 bit for temporary storage
const mask = new Uint32Array(1); // unsigned 32 bit mask value
const byte = new Uint8Array(1); // unsigned 8 bit for temporary storage
while (numBits > 0) {
byte[0] = data[offset];
// read remaining bits, upto 8 bits at a time
const bits = Math.min(numBits, 8);
const shift = 8 - bits;
mask[0] = (0xff000000 >>> (24 + shift)) << shift;
temp[0] = (byte[0] & mask[0]) >> shift;
bsid = !bsid ? temp[0] : (bsid << bits) | temp[0];
offset += 1;
numBits -= bits;
}
return bsid;
};
+85
View File
@@ -0,0 +1,85 @@
/**
* MP3 demuxer
*/
import { getId3Data } from '@svta/common-media-library/id3/getId3Data';
import { getId3Timestamp } from '@svta/common-media-library/id3/getId3Timestamp';
import BaseAudioDemuxer from './base-audio-demuxer';
import { getAudioBSID } from './dolby';
import * as MpegAudio from './mpegaudio';
import { logger } from '../../utils/logger';
class MP3Demuxer extends BaseAudioDemuxer {
resetInitSegment(
initSegment: Uint8Array | undefined,
audioCodec: string | undefined,
videoCodec: string | undefined,
trackDuration: number,
) {
super.resetInitSegment(initSegment, audioCodec, videoCodec, trackDuration);
this._audioTrack = {
container: 'audio/mpeg',
type: 'audio',
id: 2,
pid: -1,
sequenceNumber: 0,
segmentCodec: 'mp3',
samples: [],
manifestCodec: audioCodec,
duration: trackDuration,
inputTimeScale: 90000,
dropped: 0,
};
}
static probe(data: Uint8Array | undefined): boolean {
if (!data) {
return false;
}
// check if data contains ID3 timestamp and MPEG sync word
// Look for MPEG header | 1111 1111 | 111X XYZX | where X can be either 0 or 1 and Y or Z should be 1
// Layer bits (position 14 and 15) in header should be always different from 0 (Layer I or Layer II or Layer III)
// More info http://www.mp3-tech.org/programmer/frame_header.html
const id3Data = getId3Data(data, 0);
let offset = id3Data?.length || 0;
// Check for ac-3|ec-3 sync bytes and return false if present
if (
id3Data &&
data[offset] === 0x0b &&
data[offset + 1] === 0x77 &&
getId3Timestamp(id3Data) !== undefined &&
// check the bsid to confirm ac-3 or ec-3 (not mp3)
getAudioBSID(data, offset) <= 16
) {
return false;
}
for (let length = data.length; offset < length; offset++) {
if (MpegAudio.probe(data, offset)) {
logger.log('MPEG Audio sync word found !');
return true;
}
}
return false;
}
canParse(data, offset) {
return MpegAudio.canParse(data, offset);
}
appendFrame(track, data, offset) {
if (this.basePTS === null) {
return;
}
return MpegAudio.appendFrame(
track,
data,
offset,
this.basePTS,
this.frameIndex,
);
}
}
export default MP3Demuxer;
+177
View File
@@ -0,0 +1,177 @@
/**
* MPEG parser helper
*/
import type { DemuxedAudioTrack } from '../../types/demuxer';
let chromeVersion: number | null = null;
const BitratesMap = [
32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 352, 384, 416, 448, 32, 48, 56,
64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384, 32, 40, 48, 56, 64, 80,
96, 112, 128, 160, 192, 224, 256, 320, 32, 48, 56, 64, 80, 96, 112, 128, 144,
160, 176, 192, 224, 256, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144,
160,
];
const SamplingRateMap = [
44100, 48000, 32000, 22050, 24000, 16000, 11025, 12000, 8000,
];
const SamplesCoefficients = [
// MPEG 2.5
[
0, // Reserved
72, // Layer3
144, // Layer2
12, // Layer1
],
// Reserved
[
0, // Reserved
0, // Layer3
0, // Layer2
0, // Layer1
],
// MPEG 2
[
0, // Reserved
72, // Layer3
144, // Layer2
12, // Layer1
],
// MPEG 1
[
0, // Reserved
144, // Layer3
144, // Layer2
12, // Layer1
],
];
const BytesInSlot = [
0, // Reserved
1, // Layer3
1, // Layer2
4, // Layer1
];
export function appendFrame(
track: DemuxedAudioTrack,
data: Uint8Array,
offset: number,
pts: number,
frameIndex: number,
) {
// Using http://www.datavoyage.com/mpgscript/mpeghdr.htm as a reference
if (offset + 24 > data.length) {
return;
}
const header = parseHeader(data, offset);
if (header && offset + header.frameLength <= data.length) {
const frameDuration = (header.samplesPerFrame * 90000) / header.sampleRate;
const stamp = pts + frameIndex * frameDuration;
const sample = {
unit: data.subarray(offset, offset + header.frameLength),
pts: stamp,
dts: stamp,
};
track.config = [];
track.channelCount = header.channelCount;
track.samplerate = header.sampleRate;
track.samples.push(sample);
return { sample, length: header.frameLength, missing: 0 };
}
}
export function parseHeader(data: Uint8Array, offset: number) {
const mpegVersion = (data[offset + 1] >> 3) & 3;
const mpegLayer = (data[offset + 1] >> 1) & 3;
const bitRateIndex = (data[offset + 2] >> 4) & 15;
const sampleRateIndex = (data[offset + 2] >> 2) & 3;
if (
mpegVersion !== 1 &&
bitRateIndex !== 0 &&
bitRateIndex !== 15 &&
sampleRateIndex !== 3
) {
const paddingBit = (data[offset + 2] >> 1) & 1;
const channelMode = data[offset + 3] >> 6;
const columnInBitrates =
mpegVersion === 3 ? 3 - mpegLayer : mpegLayer === 3 ? 3 : 4;
const bitRate =
BitratesMap[columnInBitrates * 14 + bitRateIndex - 1] * 1000;
const columnInSampleRates =
mpegVersion === 3 ? 0 : mpegVersion === 2 ? 1 : 2;
const sampleRate =
SamplingRateMap[columnInSampleRates * 3 + sampleRateIndex];
const channelCount = channelMode === 3 ? 1 : 2; // If bits of channel mode are `11` then it is a single channel (Mono)
const sampleCoefficient = SamplesCoefficients[mpegVersion][mpegLayer];
const bytesInSlot = BytesInSlot[mpegLayer];
const samplesPerFrame = sampleCoefficient * 8 * bytesInSlot;
const frameLength =
Math.floor((sampleCoefficient * bitRate) / sampleRate + paddingBit) *
bytesInSlot;
if (chromeVersion === null) {
const userAgent = navigator.userAgent || '';
const result = userAgent.match(/Chrome\/(\d+)/i);
chromeVersion = result ? parseInt(result[1]) : 0;
}
const needChromeFix = !!chromeVersion && chromeVersion <= 87;
if (
needChromeFix &&
mpegLayer === 2 &&
bitRate >= 224000 &&
channelMode === 0
) {
// Work around bug in Chromium by setting channelMode to dual-channel (01) instead of stereo (00)
data[offset + 3] = data[offset + 3] | 0x80;
}
return { sampleRate, channelCount, frameLength, samplesPerFrame };
}
}
export function isHeaderPattern(data: Uint8Array, offset: number): boolean {
return (
data[offset] === 0xff &&
(data[offset + 1] & 0xe0) === 0xe0 &&
(data[offset + 1] & 0x06) !== 0x00
);
}
export function isHeader(data: Uint8Array, offset: number): boolean {
// Look for MPEG header | 1111 1111 | 111X XYZX | where X can be either 0 or 1 and Y or Z should be 1
// Layer bits (position 14 and 15) in header should be always different from 0 (Layer I or Layer II or Layer III)
// More info http://www.mp3-tech.org/programmer/frame_header.html
return offset + 1 < data.length && isHeaderPattern(data, offset);
}
export function canParse(data: Uint8Array, offset: number): boolean {
const headerSize = 4;
return isHeaderPattern(data, offset) && headerSize <= data.length - offset;
}
export function probe(data: Uint8Array, offset: number): boolean {
// same as isHeader but we also check that MPEG frame follows last MPEG frame
// or end of data is reached
if (offset + 1 < data.length && isHeaderPattern(data, offset)) {
// MPEG header Length
const headerLength = 4;
// MPEG frame Length
const header = parseHeader(data, offset);
let frameLength = headerLength;
if (header?.frameLength) {
frameLength = header.frameLength;
}
const newOffset = offset + frameLength;
return newOffset === data.length || isHeader(data, newOffset);
}
return false;
}
+42
View File
@@ -0,0 +1,42 @@
export default class ChunkCache {
private chunks: Array<Uint8Array<ArrayBuffer>> = [];
public dataLength: number = 0;
push(chunk: Uint8Array<ArrayBuffer>) {
this.chunks.push(chunk);
this.dataLength += chunk.length;
}
flush(): Uint8Array<ArrayBuffer> {
const { chunks, dataLength } = this;
let result: Uint8Array<ArrayBuffer>;
if (!chunks.length) {
return new Uint8Array(0);
} else if (chunks.length === 1) {
result = chunks[0];
} else {
result = concatUint8Arrays(chunks, dataLength);
}
this.reset();
return result;
}
reset() {
this.chunks.length = 0;
this.dataLength = 0;
}
}
function concatUint8Arrays(
chunks: Array<Uint8Array<ArrayBuffer>>,
dataLength: number,
): Uint8Array<ArrayBuffer> {
const result = new Uint8Array(dataLength);
let offset = 0;
for (let i = 0; i < chunks.length; i++) {
const chunk = chunks[i];
result.set(chunk, offset);
offset += chunk.length;
}
return result;
}
+13
View File
@@ -0,0 +1,13 @@
import type { DemuxedTrack } from '../types/demuxer';
export function dummyTrack(type = '', inputTimeScale = 90000): DemuxedTrack {
return {
type,
id: -1,
pid: -1,
inputTimeScale,
sequenceNumber: -1,
samples: [],
dropped: 0,
};
}
+75
View File
@@ -0,0 +1,75 @@
// ensure the worker ends up in the bundle
// If the worker should not be included this gets aliased to empty.js
import './transmuxer-worker';
import { version } from '../version';
const workerStore: Record<string, WorkerContext> = {};
export function hasUMDWorker(): boolean {
return typeof __HLS_WORKER_BUNDLE__ === 'function';
}
export type WorkerContext = {
worker: Worker;
objectURL?: string;
scriptURL?: string;
clientCount: number;
};
export function injectWorker(): WorkerContext {
const workerContext = workerStore[version];
if (workerContext) {
workerContext.clientCount++;
return workerContext;
}
const blob = new self.Blob(
[
`var exports={};var module={exports:exports};function define(f){f()};define.amd=true;(${__HLS_WORKER_BUNDLE__.toString()})(true);`,
],
{
type: 'text/javascript',
},
);
const objectURL = self.URL.createObjectURL(blob);
const worker = new self.Worker(objectURL);
const result = {
worker,
objectURL,
clientCount: 1,
};
workerStore[version] = result;
return result;
}
export function loadWorker(path: string): WorkerContext {
const workerContext = workerStore[path];
if (workerContext) {
workerContext.clientCount++;
return workerContext;
}
const scriptURL = new self.URL(path, self.location.href).href;
const worker = new self.Worker(scriptURL);
const result = {
worker,
scriptURL,
clientCount: 1,
};
workerStore[path] = result;
return result;
}
export function removeWorkerFromStore(path?: string | null) {
const workerContext = workerStore[path || version];
if (workerContext) {
const clientCount = workerContext.clientCount--;
if (clientCount === 1) {
const { worker, objectURL } = workerContext;
delete workerStore[path || version];
if (objectURL) {
// revoke the Object URL that was used to create transmuxer worker, so as not to leak it
self.URL.revokeObjectURL(objectURL);
}
worker.terminate();
}
}
}
+230
View File
@@ -0,0 +1,230 @@
/**
* MP4 demuxer
*/
import { dummyTrack } from './dummy-demuxed-track';
import {
type DemuxedAudioTrack,
type DemuxedMetadataTrack,
type DemuxedUserdataTrack,
type Demuxer,
type DemuxerResult,
type KeyData,
MetadataSchema,
type PassthroughTrack,
} from '../types/demuxer';
import {
appendUint8Array,
findBox,
hasMoofData,
parseEmsg,
parseInitSegment,
parseSamples,
RemuxerTrackIdConfig,
segmentValidRange,
} from '../utils/mp4-tools';
import type { HlsConfig } from '../config';
import type { HlsEventEmitter } from '../events';
import type { IEmsgParsingData } from '../utils/mp4-tools';
const emsgSchemePattern = /\/emsg[-/]ID3/i;
class MP4Demuxer implements Demuxer {
private remainderData: Uint8Array | null = null;
private timeOffset: number = 0;
private config: HlsConfig;
private videoTrack?: PassthroughTrack;
private audioTrack?: DemuxedAudioTrack;
private id3Track?: DemuxedMetadataTrack;
private txtTrack?: DemuxedUserdataTrack;
constructor(observer: HlsEventEmitter, config: HlsConfig) {
this.config = config;
}
public resetTimeStamp() {}
public resetInitSegment(
initSegment: Uint8Array | undefined,
audioCodec: string | undefined,
videoCodec: string | undefined,
trackDuration: number,
) {
const videoTrack = (this.videoTrack = dummyTrack(
'video',
1,
) as PassthroughTrack);
const audioTrack = (this.audioTrack = dummyTrack(
'audio',
1,
) as DemuxedAudioTrack);
const captionTrack = (this.txtTrack = dummyTrack(
'text',
1,
) as DemuxedUserdataTrack);
this.id3Track = dummyTrack('id3', 1) as DemuxedMetadataTrack;
this.timeOffset = 0;
if (!initSegment?.byteLength) {
return;
}
const initData = parseInitSegment(initSegment);
if (initData.video) {
const { id, timescale, codec, supplemental } = initData.video;
videoTrack.id = id;
videoTrack.timescale = captionTrack.timescale = timescale;
videoTrack.codec = codec;
videoTrack.supplemental = supplemental;
}
if (initData.audio) {
const { id, timescale, codec } = initData.audio;
audioTrack.id = id;
audioTrack.timescale = timescale;
audioTrack.codec = codec;
}
captionTrack.id = RemuxerTrackIdConfig.text;
videoTrack.sampleDuration = 0;
videoTrack.duration = audioTrack.duration = trackDuration;
}
public resetContiguity(): void {
this.remainderData = null;
}
static probe(data: Uint8Array) {
return hasMoofData(data);
}
public demux(data: Uint8Array, timeOffset: number): DemuxerResult {
this.timeOffset = timeOffset;
// Load all data into the avc track. The CMAF remuxer will look for the data in the samples object; the rest of the fields do not matter
let videoSamples = data;
const videoTrack = this.videoTrack as PassthroughTrack;
const textTrack = this.txtTrack as DemuxedUserdataTrack;
if (this.config.progressive) {
// Split the bytestream into two ranges: one encompassing all data up until the start of the last moof, and everything else.
// This is done to guarantee that we're sending valid data to MSE - when demuxing progressively, we have no guarantee
// that the fetch loader gives us flush moof+mdat pairs. If we push jagged data to MSE, it will throw an exception.
if (this.remainderData) {
videoSamples = appendUint8Array(this.remainderData, data);
}
const segmentedData = segmentValidRange(videoSamples);
this.remainderData = segmentedData.remainder;
videoTrack.samples = segmentedData.valid || new Uint8Array();
} else {
videoTrack.samples = videoSamples;
}
const id3Track = this.extractID3Track(videoTrack, timeOffset);
textTrack.samples = parseSamples(timeOffset, videoTrack);
return {
videoTrack,
audioTrack: this.audioTrack as DemuxedAudioTrack,
id3Track,
textTrack: this.txtTrack as DemuxedUserdataTrack,
};
}
public flush() {
const timeOffset = this.timeOffset;
const videoTrack = this.videoTrack as PassthroughTrack;
const textTrack = this.txtTrack as DemuxedUserdataTrack;
videoTrack.samples = this.remainderData || new Uint8Array();
this.remainderData = null;
const id3Track = this.extractID3Track(videoTrack, this.timeOffset);
textTrack.samples = parseSamples(timeOffset, videoTrack);
return {
videoTrack,
audioTrack: dummyTrack() as DemuxedAudioTrack,
id3Track,
textTrack: dummyTrack() as DemuxedUserdataTrack,
};
}
private extractID3Track(
videoTrack: PassthroughTrack,
timeOffset: number,
): DemuxedMetadataTrack {
const id3Track = this.id3Track as DemuxedMetadataTrack;
if (videoTrack.samples.length) {
const emsgs = findBox(videoTrack.samples, ['emsg']);
if (emsgs) {
emsgs.forEach((data: Uint8Array) => {
const emsgInfo = parseEmsg(data);
if (emsgSchemePattern.test(emsgInfo.schemeIdUri)) {
const pts = getEmsgStartTime(emsgInfo, timeOffset);
let duration =
emsgInfo.eventDuration === 0xffffffff
? Number.POSITIVE_INFINITY
: emsgInfo.eventDuration / emsgInfo.timeScale;
// Safari takes anything <= 0.001 seconds and maps it to Infinity
if (duration <= 0.001) {
duration = Number.POSITIVE_INFINITY;
}
const payload = emsgInfo.payload;
id3Track.samples.push({
data: payload,
len: payload.byteLength,
dts: pts,
pts: pts,
type: MetadataSchema.emsg,
duration: duration,
});
} else if (
this.config.enableEmsgKLVMetadata &&
emsgInfo.schemeIdUri.startsWith('urn:misb:KLV:bin:1910.1')
) {
const pts = getEmsgStartTime(emsgInfo, timeOffset);
id3Track.samples.push({
data: emsgInfo.payload,
len: emsgInfo.payload.byteLength,
dts: pts,
pts: pts,
type: MetadataSchema.misbklv,
duration: Number.POSITIVE_INFINITY,
});
}
});
}
}
return id3Track;
}
demuxSampleAes(
data: Uint8Array,
keyData: KeyData,
timeOffset: number,
): Promise<DemuxerResult> {
return Promise.reject(
new Error('The MP4 demuxer does not support SAMPLE-AES decryption'),
);
}
destroy() {
// @ts-ignore
this.config = null;
this.remainderData = null;
this.videoTrack =
this.audioTrack =
this.id3Track =
this.txtTrack =
undefined;
}
}
function getEmsgStartTime(
emsgInfo: IEmsgParsingData,
timeOffset: number,
): number {
return Number.isFinite(emsgInfo.presentationTime)
? (emsgInfo.presentationTime as number) / emsgInfo.timeScale
: timeOffset +
(emsgInfo.presentationTimeDelta as number) / emsgInfo.timeScale;
}
export default MP4Demuxer;
+198
View File
@@ -0,0 +1,198 @@
/**
* SAMPLE-AES decrypter
*/
import Decrypter from '../crypt/decrypter';
import { DecrypterAesMode } from '../crypt/decrypter-aes-mode';
import { discardEPB } from '../utils/mp4-tools';
import type { HlsConfig } from '../config';
import type { HlsEventEmitter } from '../events';
import type {
AACAudioSample,
DemuxedVideoTrackBase,
KeyData,
VideoSample,
VideoSampleUnit,
} from '../types/demuxer';
class SampleAesDecrypter {
private keyData: KeyData;
private decrypter: Decrypter;
constructor(observer: HlsEventEmitter, config: HlsConfig, keyData: KeyData) {
this.keyData = keyData;
this.decrypter = new Decrypter(config, {
removePKCS7Padding: false,
});
}
decryptBuffer(encryptedData: Uint8Array | ArrayBuffer): Promise<ArrayBuffer> {
return this.decrypter.decrypt(
encryptedData,
this.keyData.key.buffer,
this.keyData.iv.buffer,
DecrypterAesMode.cbc,
);
}
// AAC - encrypt all full 16 bytes blocks starting from offset 16
private decryptAacSample(
samples: AACAudioSample[],
sampleIndex: number,
callback: () => void,
) {
const curUnit = samples[sampleIndex].unit;
if (curUnit.length <= 16) {
// No encrypted portion in this sample (first 16 bytes is not
// encrypted, see https://developer.apple.com/library/archive/documentation/AudioVideo/Conceptual/HLS_Sample_Encryption/Encryption/Encryption.html),
return;
}
const encryptedData = curUnit.subarray(
16,
curUnit.length - (curUnit.length % 16),
);
const encryptedBuffer = encryptedData.buffer.slice(
encryptedData.byteOffset,
encryptedData.byteOffset + encryptedData.length,
);
this.decryptBuffer(encryptedBuffer)
.then((decryptedBuffer: ArrayBuffer) => {
const decryptedData = new Uint8Array(decryptedBuffer);
curUnit.set(decryptedData, 16);
if (!this.decrypter.isSync()) {
this.decryptAacSamples(samples, sampleIndex + 1, callback);
}
})
.catch(callback);
}
decryptAacSamples(
samples: AACAudioSample[],
sampleIndex: number,
callback: () => void,
) {
for (; ; sampleIndex++) {
if (sampleIndex >= samples.length) {
callback();
return;
}
if (samples[sampleIndex].unit.length < 32) {
continue;
}
this.decryptAacSample(samples, sampleIndex, callback);
if (!this.decrypter.isSync()) {
return;
}
}
}
// AVC - encrypt one 16 bytes block out of ten, starting from offset 32
getAvcEncryptedData(decodedData: Uint8Array) {
const encryptedDataLen =
Math.floor((decodedData.length - 48) / 160) * 16 + 16;
const encryptedData = new Int8Array(encryptedDataLen);
let outputPos = 0;
for (
let inputPos = 32;
inputPos < decodedData.length - 16;
inputPos += 160, outputPos += 16
) {
encryptedData.set(
decodedData.subarray(inputPos, inputPos + 16),
outputPos,
);
}
return encryptedData;
}
getAvcDecryptedUnit(decodedData: Uint8Array, decryptedData: ArrayBufferLike) {
const uint8DecryptedData = new Uint8Array(decryptedData);
let inputPos = 0;
for (
let outputPos = 32;
outputPos < decodedData.length - 16;
outputPos += 160, inputPos += 16
) {
decodedData.set(
uint8DecryptedData.subarray(inputPos, inputPos + 16),
outputPos,
);
}
return decodedData;
}
decryptAvcSample(
samples: VideoSample[],
sampleIndex: number,
unitIndex: number,
callback: () => void,
curUnit: VideoSampleUnit,
) {
const decodedData = discardEPB(curUnit.data);
const encryptedData = this.getAvcEncryptedData(decodedData);
this.decryptBuffer(encryptedData.buffer)
.then((decryptedBuffer) => {
curUnit.data = this.getAvcDecryptedUnit(decodedData, decryptedBuffer);
if (!this.decrypter.isSync()) {
this.decryptAvcSamples(samples, sampleIndex, unitIndex + 1, callback);
}
})
.catch(callback);
}
decryptAvcSamples(
samples: DemuxedVideoTrackBase['samples'],
sampleIndex: number,
unitIndex: number,
callback: () => void,
) {
if (samples instanceof Uint8Array) {
throw new Error('Cannot decrypt samples of type Uint8Array');
}
for (; ; sampleIndex++, unitIndex = 0) {
if (sampleIndex >= samples.length) {
callback();
return;
}
const curUnits = samples[sampleIndex].units;
for (; ; unitIndex++) {
if (unitIndex >= curUnits.length) {
break;
}
const curUnit = curUnits[unitIndex];
if (
curUnit.data.length <= 48 ||
(curUnit.type !== 1 && curUnit.type !== 5)
) {
continue;
}
this.decryptAvcSample(
samples,
sampleIndex,
unitIndex,
callback,
curUnit,
);
if (!this.decrypter.isSync()) {
return;
}
}
}
}
}
export default SampleAesDecrypter;
+449
View File
@@ -0,0 +1,449 @@
import { EventEmitter } from 'eventemitter3';
import {
hasUMDWorker,
injectWorker,
loadWorker,
removeWorkerFromStore as removeWorkerClient,
} from './inject-worker';
import Transmuxer, {
isPromise,
TransmuxConfig,
TransmuxState,
} from '../demux/transmuxer';
import { ErrorDetails, ErrorTypes } from '../errors';
import { Events } from '../events';
import { PlaylistLevelType } from '../types/loader';
import { getM2TSSupportedAudioTypes } from '../utils/codecs';
import { stringify } from '../utils/safe-json-stringify';
import type { WorkerContext } from './inject-worker';
import type { HlsEventEmitter, HlsListeners } from '../events';
import type Hls from '../hls';
import type { MediaFragment, Part } from '../loader/fragment';
import type { ErrorData, FragDecryptedData } from '../types/events';
import type { ChunkMetadata, TransmuxerResult } from '../types/transmuxer';
import type { TimestampOffset } from '../utils/timescale-conversion';
let transmuxerInstanceCount: number = 0;
export default class TransmuxerInterface {
public error: Error | null = null;
private hls: Hls;
private id: PlaylistLevelType;
private instanceNo: number = transmuxerInstanceCount++;
private observer: HlsEventEmitter;
private frag: MediaFragment | null = null;
private part: Part | null = null;
private useWorker: boolean;
private workerContext: WorkerContext | null = null;
private transmuxer: Transmuxer | null = null;
private onTransmuxComplete: (transmuxResult: TransmuxerResult) => void;
private onFlush: (chunkMeta: ChunkMetadata) => void;
constructor(
hls: Hls,
id: PlaylistLevelType,
onTransmuxComplete: (transmuxResult: TransmuxerResult) => void,
onFlush: (chunkMeta: ChunkMetadata) => void,
) {
const config = hls.config;
this.hls = hls;
this.id = id;
this.useWorker = !!config.enableWorker;
this.onTransmuxComplete = onTransmuxComplete;
this.onFlush = onFlush;
const forwardMessage = (
ev: Events.ERROR | Events.FRAG_DECRYPTED,
data: ErrorData | FragDecryptedData,
) => {
data = data || {};
data.frag = this.frag || undefined;
if (ev === Events.ERROR) {
data = data as ErrorData;
data.parent = this.id;
data.part = this.part;
this.error = data.error;
}
this.hls.trigger(ev, data);
};
// forward events to main thread
this.observer = new EventEmitter() as HlsEventEmitter;
this.observer.on(Events.FRAG_DECRYPTED, forwardMessage);
this.observer.on(Events.ERROR, forwardMessage);
const m2tsTypeSupported = getM2TSSupportedAudioTypes(
config.preferManagedMediaSource,
);
if (this.useWorker && typeof Worker !== 'undefined') {
const logger = this.hls.logger;
const canCreateWorker = config.workerPath || hasUMDWorker();
if (canCreateWorker) {
try {
if (config.workerPath) {
logger.log(`loading Web Worker ${config.workerPath} for "${id}"`);
this.workerContext = loadWorker(config.workerPath);
} else {
logger.log(`injecting Web Worker for "${id}"`);
this.workerContext = injectWorker();
}
const { worker } = this.workerContext;
worker.addEventListener('message', this.onWorkerMessage);
worker.addEventListener('error', this.onWorkerError);
worker.postMessage({
instanceNo: this.instanceNo,
cmd: 'init',
typeSupported: m2tsTypeSupported,
id,
config: stringify(config),
});
} catch (err) {
logger.warn(
`Error setting up "${id}" Web Worker, fallback to inline`,
err,
);
this.terminateWorker();
this.error = null;
this.transmuxer = new Transmuxer(
this.observer,
m2tsTypeSupported,
config,
'',
id,
hls.logger,
);
}
return;
}
}
this.transmuxer = new Transmuxer(
this.observer,
m2tsTypeSupported,
config,
'',
id,
hls.logger,
);
}
reset() {
this.frag = null;
this.part = null;
if (this.workerContext) {
const instanceNo = this.instanceNo;
this.instanceNo = transmuxerInstanceCount++;
const config = this.hls.config;
const m2tsTypeSupported = getM2TSSupportedAudioTypes(
config.preferManagedMediaSource,
);
this.workerContext.worker.postMessage({
instanceNo: this.instanceNo,
cmd: 'reset',
resetNo: instanceNo,
typeSupported: m2tsTypeSupported,
id: this.id,
config: stringify(config),
});
}
}
private terminateWorker() {
if (this.workerContext) {
const { worker } = this.workerContext;
this.workerContext = null;
worker.removeEventListener('message', this.onWorkerMessage);
worker.removeEventListener('error', this.onWorkerError);
removeWorkerClient(this.hls.config.workerPath);
}
}
destroy() {
if (this.workerContext) {
this.terminateWorker();
// @ts-ignore
this.onWorkerMessage = this.onWorkerError = null;
} else {
const transmuxer = this.transmuxer;
if (transmuxer) {
transmuxer.destroy();
this.transmuxer = null;
}
}
const observer = this.observer;
if (observer) {
observer.removeAllListeners();
}
this.frag = null;
this.part = null;
// @ts-ignore
this.observer = null;
// @ts-ignore
this.hls = null;
}
push(
data: ArrayBuffer,
initSegmentData: Uint8Array | undefined,
audioCodec: string | undefined,
videoCodec: string | undefined,
frag: MediaFragment,
part: Part | null,
duration: number,
accurateTimeOffset: boolean,
chunkMeta: ChunkMetadata,
defaultInitPTS?: TimestampOffset,
) {
chunkMeta.transmuxing.start = self.performance.now();
const { instanceNo, transmuxer } = this;
const timeOffset = part ? part.start : frag.start;
// TODO: push "clear-lead" decrypt data for unencrypted fragments in streams with encrypted ones
const decryptdata = frag.decryptdata;
const lastFrag = this.frag;
const discontinuity = !(lastFrag && frag.cc === lastFrag.cc);
const trackSwitch = !(lastFrag && chunkMeta.level === lastFrag.level);
const snDiff = lastFrag ? chunkMeta.sn - lastFrag.sn : -1;
const partDiff = this.part ? chunkMeta.part - this.part.index : -1;
const progressive =
snDiff === 0 &&
chunkMeta.id > 1 &&
chunkMeta.id === lastFrag?.stats.chunkCount;
const contiguous =
!trackSwitch &&
(snDiff === 1 ||
(snDiff === 0 && (partDiff === 1 || (progressive && partDiff <= 0))));
const now = self.performance.now();
if (trackSwitch || snDiff || frag.stats.parsing.start === 0) {
frag.stats.parsing.start = now;
}
if (part && (partDiff || !contiguous)) {
part.stats.parsing.start = now;
}
const initSegmentChange = !(
lastFrag && frag.initSegment?.url === lastFrag.initSegment?.url
);
const state = new TransmuxState(
discontinuity,
contiguous,
accurateTimeOffset,
trackSwitch,
timeOffset,
initSegmentChange,
);
if (!contiguous || discontinuity || initSegmentChange) {
this.hls.logger
.log(`[transmuxer-interface]: Starting new transmux session for ${frag.type} sn: ${chunkMeta.sn}${
chunkMeta.part > -1 ? ' part: ' + chunkMeta.part : ''
} ${this.id === PlaylistLevelType.MAIN ? 'level' : 'track'}: ${chunkMeta.level} id: ${chunkMeta.id}
discontinuity: ${discontinuity}
trackSwitch: ${trackSwitch}
contiguous: ${contiguous}
accurateTimeOffset: ${accurateTimeOffset}
timeOffset: ${timeOffset}
initSegmentChange: ${initSegmentChange}`);
const config = new TransmuxConfig(
audioCodec,
videoCodec,
initSegmentData,
duration,
defaultInitPTS,
);
this.configureTransmuxer(config);
}
this.frag = frag;
this.part = part;
// Frags with sn of 'initSegment' are not transmuxed
if (this.workerContext) {
// post fragment payload as transferable objects for ArrayBuffer (no copy)
this.workerContext.worker.postMessage(
{
instanceNo,
cmd: 'demux',
data,
decryptdata,
chunkMeta,
state,
},
data instanceof ArrayBuffer ? [data] : [],
);
} else if (transmuxer) {
const transmuxResult = transmuxer.push(
data,
decryptdata,
chunkMeta,
state,
);
if (isPromise(transmuxResult)) {
transmuxResult
.then((data) => {
this.handleTransmuxComplete(data);
})
.catch((error) => {
this.transmuxerError(
error,
chunkMeta,
'transmuxer-interface push error',
);
});
} else {
this.handleTransmuxComplete(transmuxResult as TransmuxerResult);
}
}
}
flush(chunkMeta: ChunkMetadata) {
chunkMeta.transmuxing.start = self.performance.now();
const { instanceNo, transmuxer } = this;
if (this.workerContext) {
1;
this.workerContext.worker.postMessage({
instanceNo,
cmd: 'flush',
chunkMeta,
});
} else if (transmuxer) {
const transmuxResult = transmuxer.flush(chunkMeta);
if (isPromise(transmuxResult)) {
transmuxResult
.then((data) => {
this.handleFlushResult(data, chunkMeta);
})
.catch((error) => {
this.transmuxerError(
error,
chunkMeta,
'transmuxer-interface flush error',
);
});
} else {
this.handleFlushResult(transmuxResult, chunkMeta);
}
}
}
private transmuxerError(
error: Error,
chunkMeta: ChunkMetadata,
reason: string,
) {
if (!this.hls) {
return;
}
this.error = error;
this.hls.trigger(Events.ERROR, {
type: ErrorTypes.MEDIA_ERROR,
details: ErrorDetails.FRAG_PARSING_ERROR,
chunkMeta,
frag: this.frag || undefined,
part: this.part || undefined,
fatal: false,
error,
err: error,
reason,
});
}
private handleFlushResult(
results: Array<TransmuxerResult>,
chunkMeta: ChunkMetadata,
) {
results.forEach((result) => {
this.handleTransmuxComplete(result);
});
this.onFlush(chunkMeta);
}
private onWorkerMessage = (
event: MessageEvent<{
event: string;
data?: any;
instanceNo?: number;
} | null>,
) => {
const data = event.data;
const hls = this.hls;
if (!hls || !data?.event || data.instanceNo !== this.instanceNo) {
return;
}
switch (data.event) {
case 'init': {
const objectURL = this.workerContext?.objectURL;
if (objectURL) {
// revoke the Object URL that was used to create transmuxer worker, so as not to leak it
self.URL.revokeObjectURL(objectURL);
}
break;
}
case 'transmuxComplete': {
this.handleTransmuxComplete(data.data);
break;
}
case 'flush': {
this.onFlush(data.data);
break;
}
// pass logs from the worker thread to the main logger
case 'workerLog': {
if (hls.logger[data.data.logType]) {
hls.logger[data.data.logType](data.data.message);
}
break;
}
default: {
data.data = data.data || {};
data.data.frag = this.frag;
data.data.part = this.part;
data.data.id = this.id;
hls.trigger(data.event as keyof HlsListeners, data.data);
break;
}
}
};
private onWorkerError = (event) => {
if (!this.hls) {
return;
}
const error = new Error(
`${event.message} (${event.filename}:${event.lineno})`,
);
this.hls.config.enableWorker = false;
this.hls.logger.warn(
`Error in "${this.id}" Web Worker, fallback to inline`,
);
this.hls.trigger(Events.ERROR, {
type: ErrorTypes.OTHER_ERROR,
details: ErrorDetails.INTERNAL_EXCEPTION,
fatal: false,
event: 'demuxerWorker',
error,
});
};
private configureTransmuxer(config: TransmuxConfig) {
const { instanceNo, transmuxer } = this;
if (this.workerContext) {
this.workerContext.worker.postMessage({
instanceNo,
cmd: 'configure',
config,
});
} else if (transmuxer) {
transmuxer.configure(config);
}
}
private handleTransmuxComplete(result: TransmuxerResult) {
result.chunkMeta.transmuxing.end = self.performance.now();
this.onTransmuxComplete(result);
}
}
+221
View File
@@ -0,0 +1,221 @@
import { EventEmitter } from 'eventemitter3';
import Transmuxer, { isPromise } from '../demux/transmuxer';
import { ErrorDetails, ErrorTypes } from '../errors';
import { Events } from '../events';
import { enableLogs, type ILogger } from '../utils/logger';
import type { RemuxedTrack, RemuxerResult } from '../types/remuxer';
import type { ChunkMetadata, TransmuxerResult } from '../types/transmuxer';
const transmuxers: (Transmuxer | undefined)[] = [];
if (typeof __IN_WORKER__ !== 'undefined' && __IN_WORKER__) {
startWorker();
}
function startWorker() {
self.addEventListener('message', (ev) => {
const data = ev.data;
const instanceNo = data.instanceNo;
if (instanceNo === undefined) {
return;
}
const transmuxer = transmuxers[instanceNo];
if (data.cmd === 'reset') {
delete transmuxers[data.resetNo];
if (transmuxer) {
transmuxer.destroy();
}
data.cmd = 'init';
}
if (data.cmd === 'init') {
const config = JSON.parse(data.config);
const observer = new EventEmitter();
observer.on(Events.FRAG_DECRYPTED, forwardMessage);
observer.on(Events.ERROR, forwardMessage);
const logger = enableLogs(config.debug, data.id);
forwardWorkerLogs(logger, instanceNo);
transmuxers[instanceNo] = new Transmuxer(
observer,
data.typeSupported,
config,
'',
data.id,
logger,
);
forwardMessage('init', null, instanceNo);
return;
}
if (!transmuxer) {
return;
}
switch (data.cmd) {
case 'configure': {
transmuxer.configure(data.config);
break;
}
case 'demux': {
const transmuxResult: TransmuxerResult | Promise<TransmuxerResult> =
transmuxer.push(
data.data,
data.decryptdata,
data.chunkMeta,
data.state,
);
if (isPromise(transmuxResult)) {
transmuxResult
.then((data) => {
emitTransmuxComplete(self, data, instanceNo);
})
.catch((error) => {
forwardMessage(
Events.ERROR,
{
instanceNo,
type: ErrorTypes.MEDIA_ERROR,
details: ErrorDetails.FRAG_PARSING_ERROR,
chunkMeta: data.chunkMeta,
fatal: false,
error,
err: error,
reason: `transmuxer-worker push error`,
},
instanceNo,
);
});
} else {
emitTransmuxComplete(self, transmuxResult, instanceNo);
}
break;
}
case 'flush': {
const chunkMeta = data.chunkMeta as ChunkMetadata;
const transmuxResult = transmuxer.flush(chunkMeta);
if (isPromise(transmuxResult)) {
transmuxResult
.then((results: Array<TransmuxerResult>) => {
handleFlushResult(
self,
results as Array<TransmuxerResult>,
chunkMeta,
instanceNo,
);
})
.catch((error) => {
forwardMessage(
Events.ERROR,
{
type: ErrorTypes.MEDIA_ERROR,
details: ErrorDetails.FRAG_PARSING_ERROR,
chunkMeta: data.chunkMeta,
fatal: false,
error,
err: error,
reason: `transmuxer-worker flush error`,
},
instanceNo,
);
});
} else {
handleFlushResult(
self,
transmuxResult as Array<TransmuxerResult>,
chunkMeta,
instanceNo,
);
}
break;
}
default:
break;
}
});
}
function emitTransmuxComplete(
self: any,
transmuxResult: TransmuxerResult,
instanceNo: number,
): boolean {
if (isEmptyResult(transmuxResult.remuxResult)) {
return false;
}
const transferable: Array<ArrayBuffer> = [];
const { audio, video } = transmuxResult.remuxResult;
if (audio) {
addToTransferable(transferable, audio);
}
if (video) {
addToTransferable(transferable, video);
}
self.postMessage(
{ event: 'transmuxComplete', data: transmuxResult, instanceNo },
transferable,
);
return true;
}
// Converts data to a transferable object https://developers.google.com/web/updates/2011/12/Transferable-Objects-Lightning-Fast)
// in order to minimize message passing overhead
function addToTransferable(
transferable: Array<ArrayBuffer>,
track: RemuxedTrack,
) {
if (track.data1) {
transferable.push(track.data1.buffer);
}
if (track.data2) {
transferable.push(track.data2.buffer);
}
}
function handleFlushResult(
self: any,
results: Array<TransmuxerResult>,
chunkMeta: ChunkMetadata,
instanceNo: number,
) {
const parsed = results.reduce(
(parsed, result) =>
emitTransmuxComplete(self, result, instanceNo) || parsed,
false,
);
if (!parsed) {
// Emit at least one "transmuxComplete" message even if media is not found to update stream-controller state to PARSING
self.postMessage({
event: 'transmuxComplete',
data: results[0],
instanceNo,
});
}
self.postMessage({ event: 'flush', data: chunkMeta, instanceNo });
}
function forwardMessage(event, data, instanceNo) {
self.postMessage({ event, data, instanceNo });
}
function forwardWorkerLogs(logger: ILogger, instanceNo: number) {
for (const logFn in logger) {
logger[logFn] = function () {
const message = Array.prototype.join.call(arguments, ' ');
forwardMessage(
'workerLog',
{
logType: logFn,
message,
},
instanceNo,
);
};
}
}
function isEmptyResult(remuxResult: RemuxerResult) {
return (
!remuxResult.audio &&
!remuxResult.video &&
!remuxResult.text &&
!remuxResult.id3 &&
!remuxResult.initSegment
);
}
+560
View File
@@ -0,0 +1,560 @@
import AACDemuxer from './audio/aacdemuxer';
import { AC3Demuxer } from './audio/ac3-demuxer';
import MP3Demuxer from './audio/mp3demuxer';
import Decrypter from '../crypt/decrypter';
import MP4Demuxer from '../demux/mp4demuxer';
import TSDemuxer from '../demux/tsdemuxer';
import { ErrorDetails, ErrorTypes } from '../errors';
import { Events } from '../events';
import MP4Remuxer from '../remux/mp4-remuxer';
import PassThroughRemuxer from '../remux/passthrough-remuxer';
import { PlaylistLevelType } from '../types/loader';
import {
getAesModeFromFullSegmentMethod,
isFullSegmentEncryption,
} from '../utils/encryption-methods-util';
import type { HlsConfig } from '../config';
import type { HlsEventEmitter } from '../events';
import type { DecryptData } from '../loader/level-key';
import type { Demuxer, DemuxerResult, KeyData } from '../types/demuxer';
import type { Remuxer } from '../types/remuxer';
import type { ChunkMetadata, TransmuxerResult } from '../types/transmuxer';
import type { TypeSupported } from '../utils/codecs';
import type { ILogger } from '../utils/logger';
import type { TimestampOffset } from '../utils/timescale-conversion';
let now: () => number;
// performance.now() not available on WebWorker, at least on Safari Desktop
try {
now = self.performance.now.bind(self.performance);
} catch (err) {
now = Date.now;
}
type MuxConfig =
| { demux: typeof MP4Demuxer; remux: typeof PassThroughRemuxer }
| { demux: typeof TSDemuxer; remux: typeof MP4Remuxer }
| { demux: typeof AC3Demuxer; remux: typeof MP4Remuxer }
| { demux: typeof AACDemuxer; remux: typeof MP4Remuxer }
| { demux: typeof MP3Demuxer; remux: typeof MP4Remuxer };
const muxConfig: MuxConfig[] = [
{ demux: MP4Demuxer, remux: PassThroughRemuxer },
{ demux: TSDemuxer, remux: MP4Remuxer },
{ demux: AACDemuxer, remux: MP4Remuxer },
{ demux: MP3Demuxer, remux: MP4Remuxer },
];
if (__USE_M2TS_ADVANCED_CODECS__) {
muxConfig.splice(2, 0, { demux: AC3Demuxer, remux: MP4Remuxer });
}
export default class Transmuxer {
private asyncResult: boolean = false;
private logger: ILogger;
private observer: HlsEventEmitter;
private typeSupported: TypeSupported;
private config: HlsConfig;
private id: PlaylistLevelType;
private demuxer?: Demuxer;
private remuxer?: Remuxer;
private decrypter?: Decrypter;
private probe!: Function;
private decryptionPromise: Promise<TransmuxerResult> | null = null;
private transmuxConfig!: TransmuxConfig;
private currentTransmuxState!: TransmuxState;
constructor(
observer: HlsEventEmitter,
typeSupported: TypeSupported,
config: HlsConfig,
vendor: string,
id: PlaylistLevelType,
logger: ILogger,
) {
this.observer = observer;
this.typeSupported = typeSupported;
this.config = config;
this.id = id;
this.logger = logger;
}
configure(transmuxConfig: TransmuxConfig) {
this.transmuxConfig = transmuxConfig;
if (this.decrypter) {
this.decrypter.reset();
}
}
push(
data: ArrayBuffer,
decryptdata: DecryptData | null,
chunkMeta: ChunkMetadata,
state?: TransmuxState,
): TransmuxerResult | Promise<TransmuxerResult> {
const stats = chunkMeta.transmuxing;
stats.executeStart = now();
let uintData: Uint8Array<ArrayBuffer> = new Uint8Array(data);
const { currentTransmuxState, transmuxConfig } = this;
if (state) {
this.currentTransmuxState = state;
}
const {
contiguous,
discontinuity,
trackSwitch,
accurateTimeOffset,
timeOffset,
initSegmentChange,
} = state || currentTransmuxState;
const {
audioCodec,
videoCodec,
defaultInitPts,
duration,
initSegmentData,
} = transmuxConfig;
const keyData = getEncryptionType(uintData, decryptdata);
if (keyData && isFullSegmentEncryption(keyData.method)) {
const decrypter = this.getDecrypter();
const aesMode = getAesModeFromFullSegmentMethod(keyData.method);
// Software decryption is synchronous; webCrypto is not
if (decrypter.isSync()) {
// Software decryption is progressive. Progressive decryption may not return a result on each call. Any cached
// data is handled in the flush() call
let decryptedData = decrypter.softwareDecrypt(
uintData,
keyData.key.buffer,
keyData.iv.buffer,
aesMode,
);
// For Low-Latency HLS Parts, decrypt in place, since part parsing is expected on push progress
const loadingParts = chunkMeta.part > -1;
if (loadingParts) {
const data = decrypter.flush();
decryptedData = data ? data.buffer : data;
}
if (!decryptedData) {
stats.executeEnd = now();
return emptyResult(chunkMeta);
}
uintData = new Uint8Array(decryptedData);
} else {
this.asyncResult = true;
this.decryptionPromise = decrypter
.webCryptoDecrypt(
uintData,
keyData.key.buffer,
keyData.iv.buffer,
aesMode,
)
.then((decryptedData): TransmuxerResult => {
// Calling push here is important; if flush() is called while this is still resolving, this ensures that
// the decrypted data has been transmuxed
const result = this.push(
decryptedData,
null,
chunkMeta,
) as TransmuxerResult;
this.decryptionPromise = null;
return result;
});
return this.decryptionPromise;
}
}
const resetMuxers = this.needsProbing(discontinuity, trackSwitch);
if (resetMuxers) {
const error = this.configureTransmuxer(uintData);
if (error) {
this.logger.warn(`[transmuxer] ${error.message}`);
this.observer.emit(Events.ERROR, Events.ERROR, {
type: ErrorTypes.MEDIA_ERROR,
details: ErrorDetails.FRAG_PARSING_ERROR,
fatal: false,
error,
reason: error.message,
});
stats.executeEnd = now();
return emptyResult(chunkMeta);
}
}
if (discontinuity || trackSwitch || initSegmentChange || resetMuxers) {
this.resetInitSegment(
initSegmentData,
audioCodec,
videoCodec,
duration,
decryptdata,
);
}
if (discontinuity || initSegmentChange || resetMuxers) {
this.resetInitialTimestamp(defaultInitPts);
}
if (!contiguous) {
this.resetContiguity();
}
const result = this.transmux(
uintData,
keyData,
timeOffset,
accurateTimeOffset,
chunkMeta,
);
this.asyncResult = isPromise(result);
const currentState = this.currentTransmuxState;
currentState.contiguous = true;
currentState.discontinuity = false;
currentState.trackSwitch = false;
stats.executeEnd = now();
return result;
}
// Due to data caching, flush calls can produce more than one TransmuxerResult (hence the Array type)
flush(
chunkMeta: ChunkMetadata,
): TransmuxerResult[] | Promise<TransmuxerResult[]> {
const stats = chunkMeta.transmuxing;
stats.executeStart = now();
const { decrypter, currentTransmuxState, decryptionPromise } = this;
if (decryptionPromise) {
this.asyncResult = true;
// Upon resolution, the decryption promise calls push() and returns its TransmuxerResult up the stack. Therefore
// only flushing is required for async decryption
return decryptionPromise.then(() => {
return this.flush(chunkMeta);
});
}
const transmuxResults: TransmuxerResult[] = [];
const { timeOffset } = currentTransmuxState;
if (decrypter) {
// The decrypter may have data cached, which needs to be demuxed. In this case we'll have two TransmuxResults
// This happens in the case that we receive only 1 push call for a segment (either for non-progressive downloads,
// or for progressive downloads with small segments)
const decryptedData = decrypter.flush();
if (decryptedData) {
// Push always returns a TransmuxerResult if decryptdata is null
transmuxResults.push(
this.push(decryptedData.buffer, null, chunkMeta) as TransmuxerResult,
);
}
}
const { demuxer, remuxer } = this;
if (!demuxer || !remuxer) {
// If probing failed, then Hls.js has been given content its not able to handle
stats.executeEnd = now();
const emptyResults = [emptyResult(chunkMeta)];
if (this.asyncResult) {
return Promise.resolve(emptyResults);
}
return emptyResults;
}
const demuxResultOrPromise = demuxer.flush(timeOffset);
if (isPromise(demuxResultOrPromise)) {
this.asyncResult = true;
// Decrypt final SAMPLE-AES samples
return demuxResultOrPromise.then((demuxResult) => {
this.flushRemux(transmuxResults, demuxResult, chunkMeta);
return transmuxResults;
});
}
this.flushRemux(transmuxResults, demuxResultOrPromise, chunkMeta);
if (this.asyncResult) {
return Promise.resolve(transmuxResults);
}
return transmuxResults;
}
private flushRemux(
transmuxResults: TransmuxerResult[],
demuxResult: DemuxerResult,
chunkMeta: ChunkMetadata,
) {
const { audioTrack, videoTrack, id3Track, textTrack } = demuxResult;
const { accurateTimeOffset, timeOffset } = this.currentTransmuxState;
this.logger.log(
`[transmuxer.ts]: Flushed ${this.id} sn: ${chunkMeta.sn}${
chunkMeta.part > -1 ? ' part: ' + chunkMeta.part : ''
} of ${this.id === PlaylistLevelType.MAIN ? 'level' : 'track'} ${chunkMeta.level}`,
);
const remuxResult = this.remuxer!.remux(
audioTrack,
videoTrack,
id3Track,
textTrack,
timeOffset,
accurateTimeOffset,
true,
this.id,
);
transmuxResults.push({
remuxResult,
chunkMeta,
});
chunkMeta.transmuxing.executeEnd = now();
}
resetInitialTimestamp(defaultInitPts: TimestampOffset | null) {
const { demuxer, remuxer } = this;
if (!demuxer || !remuxer) {
return;
}
demuxer.resetTimeStamp(defaultInitPts);
remuxer.resetTimeStamp(defaultInitPts);
}
resetContiguity() {
const { demuxer, remuxer } = this;
if (!demuxer || !remuxer) {
return;
}
demuxer.resetContiguity();
remuxer.resetNextTimestamp();
}
resetInitSegment(
initSegmentData: Uint8Array | undefined,
audioCodec: string | undefined,
videoCodec: string | undefined,
trackDuration: number,
decryptdata: DecryptData | null,
) {
const { demuxer, remuxer } = this;
if (!demuxer || !remuxer) {
return;
}
demuxer.resetInitSegment(
initSegmentData,
audioCodec,
videoCodec,
trackDuration,
);
remuxer.resetInitSegment(
initSegmentData,
audioCodec,
videoCodec,
decryptdata,
);
}
destroy(): void {
if (this.demuxer) {
this.demuxer.destroy();
this.demuxer = undefined;
}
if (this.remuxer) {
this.remuxer.destroy();
this.remuxer = undefined;
}
}
private transmux(
data: Uint8Array,
keyData: KeyData | null,
timeOffset: number,
accurateTimeOffset: boolean,
chunkMeta: ChunkMetadata,
): TransmuxerResult | Promise<TransmuxerResult> {
let result: TransmuxerResult | Promise<TransmuxerResult>;
if (keyData && keyData.method === 'SAMPLE-AES') {
result = this.transmuxSampleAes(
data,
keyData,
timeOffset,
accurateTimeOffset,
chunkMeta,
);
} else {
result = this.transmuxUnencrypted(
data,
timeOffset,
accurateTimeOffset,
chunkMeta,
);
}
return result;
}
private transmuxUnencrypted(
data: Uint8Array,
timeOffset: number,
accurateTimeOffset: boolean,
chunkMeta: ChunkMetadata,
): TransmuxerResult {
const { audioTrack, videoTrack, id3Track, textTrack } = (
this.demuxer as Demuxer
).demux(data, timeOffset, false, !this.config.progressive);
const remuxResult = this.remuxer!.remux(
audioTrack,
videoTrack,
id3Track,
textTrack,
timeOffset,
accurateTimeOffset,
false,
this.id,
);
return {
remuxResult,
chunkMeta,
};
}
private transmuxSampleAes(
data: Uint8Array,
decryptData: KeyData,
timeOffset: number,
accurateTimeOffset: boolean,
chunkMeta: ChunkMetadata,
): Promise<TransmuxerResult> {
return (this.demuxer as Demuxer)
.demuxSampleAes(data, decryptData, timeOffset)
.then((demuxResult) => {
const remuxResult = this.remuxer!.remux(
demuxResult.audioTrack,
demuxResult.videoTrack,
demuxResult.id3Track,
demuxResult.textTrack,
timeOffset,
accurateTimeOffset,
false,
this.id,
);
return {
remuxResult,
chunkMeta,
};
});
}
private configureTransmuxer(data: Uint8Array): void | Error {
const { config, observer, typeSupported } = this;
// probe for content type
let mux;
for (let i = 0, len = muxConfig.length; i < len; i++) {
if (muxConfig[i].demux?.probe(data, this.logger)) {
mux = muxConfig[i];
break;
}
}
if (!mux) {
return new Error('Failed to find demuxer by probing fragment data');
}
// so let's check that current remuxer and demuxer are still valid
const demuxer = this.demuxer;
const remuxer = this.remuxer;
const Remuxer: MuxConfig['remux'] = mux.remux;
const Demuxer: MuxConfig['demux'] = mux.demux;
if (!remuxer || !(remuxer instanceof Remuxer)) {
this.remuxer = new Remuxer(observer, config, typeSupported, this.logger);
}
if (!demuxer || !(demuxer instanceof Demuxer)) {
this.demuxer = new Demuxer(observer, config, typeSupported, this.logger);
this.probe = Demuxer.probe;
}
}
private needsProbing(discontinuity: boolean, trackSwitch: boolean): boolean {
// in case of continuity change, or track switch
// we might switch from content type (AAC container to TS container, or TS to fmp4 for example)
return !this.demuxer || !this.remuxer || discontinuity || trackSwitch;
}
private getDecrypter(): Decrypter {
let decrypter = this.decrypter;
if (!decrypter) {
decrypter = this.decrypter = new Decrypter(this.config);
}
return decrypter;
}
}
function getEncryptionType(
data: Uint8Array,
decryptData: DecryptData | null,
): KeyData | null {
let encryptionType: KeyData | null = null;
if (
data.byteLength > 0 &&
decryptData?.key != null &&
decryptData.iv !== null &&
decryptData.method != null
) {
encryptionType = decryptData as KeyData;
}
return encryptionType;
}
const emptyResult = (chunkMeta): TransmuxerResult => ({
remuxResult: {},
chunkMeta,
});
export function isPromise<T>(p: Promise<T> | any): p is Promise<T> {
return 'then' in p && p.then instanceof Function;
}
export class TransmuxConfig {
public audioCodec?: string;
public videoCodec?: string;
public initSegmentData?: Uint8Array;
public duration: number;
public defaultInitPts: TimestampOffset | null;
constructor(
audioCodec: string | undefined,
videoCodec: string | undefined,
initSegmentData: Uint8Array | undefined,
duration: number,
defaultInitPts?: TimestampOffset,
) {
this.audioCodec = audioCodec;
this.videoCodec = videoCodec;
this.initSegmentData = initSegmentData;
this.duration = duration;
this.defaultInitPts = defaultInitPts || null;
}
}
export class TransmuxState {
public discontinuity: boolean;
public contiguous: boolean;
public accurateTimeOffset: boolean;
public trackSwitch: boolean;
public timeOffset: number;
public initSegmentChange: boolean;
constructor(
discontinuity: boolean,
contiguous: boolean,
accurateTimeOffset: boolean,
trackSwitch: boolean,
timeOffset: number,
initSegmentChange: boolean,
) {
this.discontinuity = discontinuity;
this.contiguous = contiguous;
this.accurateTimeOffset = accurateTimeOffset;
this.trackSwitch = trackSwitch;
this.timeOffset = timeOffset;
this.initSegmentChange = initSegmentChange;
}
}
+1053
View File
File diff suppressed because it is too large Load Diff
+401
View File
@@ -0,0 +1,401 @@
import BaseVideoParser from './base-video-parser';
import ExpGolomb from './exp-golomb';
import { parseSEIMessageFromNALu } from '../../utils/mp4-tools';
import type {
DemuxedUserdataTrack,
DemuxedVideoTrack,
} from '../../types/demuxer';
import type { PES } from '../tsdemuxer';
class AvcVideoParser extends BaseVideoParser {
public parsePES(
track: DemuxedVideoTrack,
textTrack: DemuxedUserdataTrack,
pes: PES,
endOfSegment: boolean,
) {
const units = this.parseNALu(track, pes.data, endOfSegment);
let VideoSample = this.VideoSample;
let push: boolean;
let spsfound = false;
// free pes.data to save up some memory
(pes as any).data = null;
// if new NAL units found and last sample still there, let's push ...
// this helps parsing streams with missing AUD (only do this if AUD never found)
if (VideoSample && units.length && !track.audFound) {
this.pushAccessUnit(VideoSample, track);
VideoSample = this.VideoSample = this.createVideoSample(
false,
pes.pts,
pes.dts,
);
}
units.forEach((unit) => {
switch (unit.type) {
// NDR
case 1: {
let iskey = false;
push = true;
const data = unit.data;
// only check slice type to detect KF in case SPS found in same packet (any keyframe is preceded by SPS ...)
if (spsfound && data.length > 4) {
// retrieve slice type by parsing beginning of NAL unit (follow H264 spec, slice_header definition) to detect keyframe embedded in NDR
const sliceType = this.readSliceType(data);
// 2 : I slice, 4 : SI slice, 7 : I slice, 9: SI slice
// SI slice : A slice that is coded using intra prediction only and using quantisation of the prediction samples.
// An SI slice can be coded such that its decoded samples can be constructed identically to an SP slice.
// I slice: A slice that is not an SI slice that is decoded using intra prediction only.
// if (sliceType === 2 || sliceType === 7) {
if (
sliceType === 2 ||
sliceType === 4 ||
sliceType === 7 ||
sliceType === 9
) {
iskey = true;
}
}
if (iskey) {
// if we have non-keyframe data already, that cannot belong to the same frame as a keyframe, so force a push
if (VideoSample?.frame && !VideoSample.key) {
this.pushAccessUnit(VideoSample, track);
VideoSample = this.VideoSample = null;
}
}
if (!VideoSample) {
VideoSample = this.VideoSample = this.createVideoSample(
true,
pes.pts,
pes.dts,
);
}
VideoSample.frame = true;
VideoSample.key = iskey;
break;
// IDR
}
case 5:
push = true;
// handle PES not starting with AUD
// if we have frame data already, that cannot belong to the same frame, so force a push
if (VideoSample?.frame && !VideoSample.key) {
this.pushAccessUnit(VideoSample, track);
VideoSample = this.VideoSample = null;
}
if (!VideoSample) {
VideoSample = this.VideoSample = this.createVideoSample(
true,
pes.pts,
pes.dts,
);
}
VideoSample.key = true;
VideoSample.frame = true;
break;
// SEI
case 6: {
push = true;
parseSEIMessageFromNALu(
unit.data,
1,
pes.pts as number,
textTrack.samples,
);
break;
// SPS
}
case 7: {
push = true;
spsfound = true;
const sps = unit.data;
const config = this.readSPS(sps);
if (
!track.sps ||
track.width !== config.width ||
track.height !== config.height ||
track.pixelRatio?.[0] !== config.pixelRatio[0] ||
track.pixelRatio?.[1] !== config.pixelRatio[1]
) {
track.width = config.width;
track.height = config.height;
track.pixelRatio = config.pixelRatio;
track.sps = [sps];
const codecarray = sps.subarray(1, 4);
let codecstring = 'avc1.';
for (let i = 0; i < 3; i++) {
let h = codecarray[i].toString(16);
if (h.length < 2) {
h = '0' + h;
}
codecstring += h;
}
track.codec = codecstring;
}
break;
}
// PPS
case 8:
push = true;
track.pps = [unit.data];
break;
// AUD
case 9:
push = true;
track.audFound = true;
if (VideoSample?.frame) {
this.pushAccessUnit(VideoSample, track);
VideoSample = null;
}
if (!VideoSample) {
VideoSample = this.VideoSample = this.createVideoSample(
false,
pes.pts,
pes.dts,
);
}
break;
// Filler Data
case 12:
push = true;
break;
default:
push = false;
break;
}
if (VideoSample && push) {
const units = VideoSample.units;
units.push(unit);
}
});
// if last PES packet, push samples
if (endOfSegment && VideoSample) {
this.pushAccessUnit(VideoSample, track);
this.VideoSample = null;
}
}
protected getNALuType(data: Uint8Array, offset: number): number {
return data[offset] & 0x1f;
}
readSliceType(data: Uint8Array) {
const eg = new ExpGolomb(data);
// skip NALu type
eg.readUByte();
// discard first_mb_in_slice
eg.readUEG();
// return slice_type
return eg.readUEG();
}
/**
* The scaling list is optionally transmitted as part of a sequence parameter
* set and is not relevant to transmuxing.
* @param count the number of entries in this scaling list
* @see Recommendation ITU-T H.264, Section 7.3.2.1.1.1
*/
skipScalingList(count: number, reader: ExpGolomb): void {
let lastScale = 8;
let nextScale = 8;
let deltaScale;
for (let j = 0; j < count; j++) {
if (nextScale !== 0) {
deltaScale = reader.readEG();
nextScale = (lastScale + deltaScale + 256) % 256;
}
lastScale = nextScale === 0 ? lastScale : nextScale;
}
}
/**
* Read a sequence parameter set and return some interesting video
* properties. A sequence parameter set is the H264 metadata that
* describes the properties of upcoming video frames.
* @returns an object with configuration parsed from the
* sequence parameter set, including the dimensions of the
* associated video frames.
*/
readSPS(sps: Uint8Array): {
width: number;
height: number;
pixelRatio: [number, number];
} {
const eg = new ExpGolomb(sps);
let frameCropLeftOffset = 0;
let frameCropRightOffset = 0;
let frameCropTopOffset = 0;
let frameCropBottomOffset = 0;
let numRefFramesInPicOrderCntCycle;
let scalingListCount;
let i;
const readUByte = eg.readUByte.bind(eg);
const readBits = eg.readBits.bind(eg);
const readUEG = eg.readUEG.bind(eg);
const readBoolean = eg.readBoolean.bind(eg);
const skipBits = eg.skipBits.bind(eg);
const skipEG = eg.skipEG.bind(eg);
const skipUEG = eg.skipUEG.bind(eg);
const skipScalingList = this.skipScalingList.bind(this);
readUByte();
const profileIdc = readUByte(); // profile_idc
readBits(5); // profileCompat constraint_set[0-4]_flag, u(5)
skipBits(3); // reserved_zero_3bits u(3),
readUByte(); // level_idc u(8)
skipUEG(); // seq_parameter_set_id
// some profiles have more optional data we don't need
if (
profileIdc === 100 ||
profileIdc === 110 ||
profileIdc === 122 ||
profileIdc === 244 ||
profileIdc === 44 ||
profileIdc === 83 ||
profileIdc === 86 ||
profileIdc === 118 ||
profileIdc === 128
) {
const chromaFormatIdc = readUEG();
if (chromaFormatIdc === 3) {
skipBits(1);
} // separate_colour_plane_flag
skipUEG(); // bit_depth_luma_minus8
skipUEG(); // bit_depth_chroma_minus8
skipBits(1); // qpprime_y_zero_transform_bypass_flag
if (readBoolean()) {
// seq_scaling_matrix_present_flag
scalingListCount = chromaFormatIdc !== 3 ? 8 : 12;
for (i = 0; i < scalingListCount; i++) {
if (readBoolean()) {
// seq_scaling_list_present_flag[ i ]
if (i < 6) {
skipScalingList(16, eg);
} else {
skipScalingList(64, eg);
}
}
}
}
}
skipUEG(); // log2_max_frame_num_minus4
const picOrderCntType = readUEG();
if (picOrderCntType === 0) {
readUEG(); // log2_max_pic_order_cnt_lsb_minus4
} else if (picOrderCntType === 1) {
skipBits(1); // delta_pic_order_always_zero_flag
skipEG(); // offset_for_non_ref_pic
skipEG(); // offset_for_top_to_bottom_field
numRefFramesInPicOrderCntCycle = readUEG();
for (i = 0; i < numRefFramesInPicOrderCntCycle; i++) {
skipEG();
} // offset_for_ref_frame[ i ]
}
skipUEG(); // max_num_ref_frames
skipBits(1); // gaps_in_frame_num_value_allowed_flag
const picWidthInMbsMinus1 = readUEG();
const picHeightInMapUnitsMinus1 = readUEG();
const frameMbsOnlyFlag = readBits(1);
if (frameMbsOnlyFlag === 0) {
skipBits(1);
} // mb_adaptive_frame_field_flag
skipBits(1); // direct_8x8_inference_flag
if (readBoolean()) {
// frame_cropping_flag
frameCropLeftOffset = readUEG();
frameCropRightOffset = readUEG();
frameCropTopOffset = readUEG();
frameCropBottomOffset = readUEG();
}
let pixelRatio: [number, number] = [1, 1];
if (readBoolean()) {
// vui_parameters_present_flag
if (readBoolean()) {
// aspect_ratio_info_present_flag
const aspectRatioIdc = readUByte();
switch (aspectRatioIdc) {
case 1:
pixelRatio = [1, 1];
break;
case 2:
pixelRatio = [12, 11];
break;
case 3:
pixelRatio = [10, 11];
break;
case 4:
pixelRatio = [16, 11];
break;
case 5:
pixelRatio = [40, 33];
break;
case 6:
pixelRatio = [24, 11];
break;
case 7:
pixelRatio = [20, 11];
break;
case 8:
pixelRatio = [32, 11];
break;
case 9:
pixelRatio = [80, 33];
break;
case 10:
pixelRatio = [18, 11];
break;
case 11:
pixelRatio = [15, 11];
break;
case 12:
pixelRatio = [64, 33];
break;
case 13:
pixelRatio = [160, 99];
break;
case 14:
pixelRatio = [4, 3];
break;
case 15:
pixelRatio = [3, 2];
break;
case 16:
pixelRatio = [2, 1];
break;
case 255: {
pixelRatio = [
(readUByte() << 8) | readUByte(),
(readUByte() << 8) | readUByte(),
];
break;
}
}
}
}
return {
width: Math.ceil(
(picWidthInMbsMinus1 + 1) * 16 -
frameCropLeftOffset * 2 -
frameCropRightOffset * 2,
),
height:
(2 - frameMbsOnlyFlag) * (picHeightInMapUnitsMinus1 + 1) * 16 -
(frameMbsOnlyFlag ? 2 : 4) *
(frameCropTopOffset + frameCropBottomOffset),
pixelRatio: pixelRatio,
};
}
}
export default AvcVideoParser;
+198
View File
@@ -0,0 +1,198 @@
import { appendUint8Array } from '../../utils/mp4-tools';
import type {
DemuxedUserdataTrack,
DemuxedVideoTrack,
VideoSample,
VideoSampleUnit,
} from '../../types/demuxer';
import type { ParsedVideoSample } from '../tsdemuxer';
import type { PES } from '../tsdemuxer';
abstract class BaseVideoParser {
protected VideoSample: ParsedVideoSample | null = null;
protected createVideoSample(
key: boolean,
pts: number | undefined,
dts: number | undefined,
): ParsedVideoSample {
return {
key,
frame: false,
pts,
dts,
units: [],
length: 0,
};
}
protected getLastNalUnit(
samples: VideoSample[],
): VideoSampleUnit | undefined {
let VideoSample = this.VideoSample;
let lastUnit: VideoSampleUnit | undefined;
// try to fallback to previous sample if current one is empty
if (!VideoSample || VideoSample.units.length === 0) {
VideoSample = samples[samples.length - 1];
}
if (VideoSample?.units) {
const units = VideoSample.units;
lastUnit = units[units.length - 1];
}
return lastUnit;
}
protected pushAccessUnit(
VideoSample: ParsedVideoSample,
videoTrack: DemuxedVideoTrack,
) {
if (VideoSample.units.length && VideoSample.frame) {
// if sample does not have PTS/DTS, patch with last sample PTS/DTS
if (VideoSample.pts === undefined) {
const samples = videoTrack.samples;
const nbSamples = samples.length;
if (nbSamples) {
const lastSample = samples[nbSamples - 1];
VideoSample.pts = lastSample.pts;
VideoSample.dts = lastSample.dts;
} else {
// dropping samples, no timestamp found
videoTrack.dropped++;
return;
}
}
videoTrack.samples.push(VideoSample as VideoSample);
}
}
abstract parsePES(
track: DemuxedVideoTrack,
textTrack: DemuxedUserdataTrack,
pes: PES,
last: boolean,
);
protected abstract getNALuType(data: Uint8Array, offset: number): number;
protected parseNALu(
track: DemuxedVideoTrack,
array: Uint8Array,
endOfSegment: boolean,
): Array<{
data: Uint8Array;
type: number;
state?: number;
}> {
const len = array.byteLength;
let state = track.naluState || 0;
const lastState = state;
const units: VideoSampleUnit[] = [];
let i = 0;
let value: number;
let overflow: number;
let unitType: number;
let lastUnitStart = -1;
let lastUnitType: number = 0;
// logger.log('PES:' + Hex.hexDump(array));
if (state === -1) {
// special use case where we found 3 or 4-byte start codes exactly at the end of previous PES packet
lastUnitStart = 0;
// NALu type is value read from offset 0
lastUnitType = this.getNALuType(array, 0);
state = 0;
i = 1;
}
while (i < len) {
value = array[i++];
// optimization. state 0 and 1 are the predominant case. let's handle them outside of the switch/case
if (!state) {
state = value ? 0 : 1;
continue;
}
if (state === 1) {
state = value ? 0 : 2;
continue;
}
// here we have state either equal to 2 or 3
if (!value) {
state = 3;
} else if (value === 1) {
overflow = i - state - 1;
if (lastUnitStart >= 0) {
const unit: VideoSampleUnit = {
data: array.subarray(lastUnitStart, overflow),
type: lastUnitType,
};
// logger.log('pushing NALU, type/size:' + unit.type + '/' + unit.data.byteLength);
units.push(unit);
} else {
// lastUnitStart is undefined => this is the first start code found in this PES packet
// first check if start code delimiter is overlapping between 2 PES packets,
// ie it started in last packet (lastState not zero)
// and ended at the beginning of this PES packet (i <= 4 - lastState)
const lastUnit = this.getLastNalUnit(track.samples);
if (lastUnit) {
if (lastState && i <= 4 - lastState) {
// start delimiter overlapping between PES packets
// strip start delimiter bytes from the end of last NAL unit
// check if lastUnit had a state different from zero
if (lastUnit.state) {
// strip last bytes
lastUnit.data = lastUnit.data.subarray(
0,
lastUnit.data.byteLength - lastState,
);
}
}
// If NAL units are not starting right at the beginning of the PES packet, push preceding data into previous NAL unit.
if (overflow > 0) {
// logger.log('first NALU found with overflow:' + overflow);
lastUnit.data = appendUint8Array(
lastUnit.data,
array.subarray(0, overflow),
);
lastUnit.state = 0;
}
}
}
// check if we can read unit type
if (i < len) {
unitType = this.getNALuType(array, i);
// logger.log('find NALU @ offset:' + i + ',type:' + unitType);
lastUnitStart = i;
lastUnitType = unitType;
state = 0;
} else {
// not enough byte to read unit type. let's read it on next PES parsing
state = -1;
}
} else {
state = 0;
}
}
if (lastUnitStart >= 0 && state >= 0) {
const unit: VideoSampleUnit = {
data: array.subarray(lastUnitStart, len),
type: lastUnitType,
state: state,
};
units.push(unit);
// logger.log('pushing NALU, type/size/state:' + unit.type + '/' + unit.data.byteLength + '/' + state);
}
// no NALu found
if (units.length === 0) {
// append pes.data to previous NAL unit
const lastUnit = this.getLastNalUnit(track.samples);
if (lastUnit) {
lastUnit.data = appendUint8Array(lastUnit.data, array);
}
}
track.naluState = state;
return units;
}
}
export default BaseVideoParser;
+153
View File
@@ -0,0 +1,153 @@
/**
* Parser for exponential Golomb codes, a variable-bitwidth number encoding scheme used by h264.
*/
import { logger } from '../../utils/logger';
class ExpGolomb {
private data: Uint8Array;
public bytesAvailable: number;
private word: number;
private bitsAvailable: number;
constructor(data: Uint8Array) {
this.data = data;
// the number of bytes left to examine in this.data
this.bytesAvailable = data.byteLength;
// the current word being examined
this.word = 0; // :uint
// the number of bits left to examine in the current word
this.bitsAvailable = 0; // :uint
}
// ():void
loadWord(): void {
const data = this.data;
const bytesAvailable = this.bytesAvailable;
const position = data.byteLength - bytesAvailable;
const workingBytes = new Uint8Array(4);
const availableBytes = Math.min(4, bytesAvailable);
if (availableBytes === 0) {
throw new Error('no bytes available');
}
workingBytes.set(data.subarray(position, position + availableBytes));
this.word = new DataView(workingBytes.buffer).getUint32(0);
// track the amount of this.data that has been processed
this.bitsAvailable = availableBytes * 8;
this.bytesAvailable -= availableBytes;
}
// (count:int):void
skipBits(count: number): void {
let skipBytes; // :int
count = Math.min(count, this.bytesAvailable * 8 + this.bitsAvailable);
if (this.bitsAvailable > count) {
this.word <<= count;
this.bitsAvailable -= count;
} else {
count -= this.bitsAvailable;
skipBytes = count >> 3;
count -= skipBytes << 3;
this.bytesAvailable -= skipBytes;
this.loadWord();
this.word <<= count;
this.bitsAvailable -= count;
}
}
// (size:int):uint
readBits(size: number): number {
let bits = Math.min(this.bitsAvailable, size); // :uint
const valu = this.word >>> (32 - bits); // :uint
if (size > 32) {
logger.error('Cannot read more than 32 bits at a time');
}
this.bitsAvailable -= bits;
if (this.bitsAvailable > 0) {
this.word <<= bits;
} else if (this.bytesAvailable > 0) {
this.loadWord();
} else {
throw new Error('no bits available');
}
bits = size - bits;
if (bits > 0 && this.bitsAvailable) {
return (valu << bits) | this.readBits(bits);
} else {
return valu;
}
}
// ():uint
skipLZ(): number {
let leadingZeroCount; // :uint
for (
leadingZeroCount = 0;
leadingZeroCount < this.bitsAvailable;
++leadingZeroCount
) {
if ((this.word & (0x80000000 >>> leadingZeroCount)) !== 0) {
// the first bit of working word is 1
this.word <<= leadingZeroCount;
this.bitsAvailable -= leadingZeroCount;
return leadingZeroCount;
}
}
// we exhausted word and still have not found a 1
this.loadWord();
return leadingZeroCount + this.skipLZ();
}
// ():void
skipUEG(): void {
this.skipBits(1 + this.skipLZ());
}
// ():void
skipEG(): void {
this.skipBits(1 + this.skipLZ());
}
// ():uint
readUEG(): number {
const clz = this.skipLZ(); // :uint
return this.readBits(clz + 1) - 1;
}
// ():int
readEG(): number {
const valu = this.readUEG(); // :int
if (0x01 & valu) {
// the number is odd if the low order bit is set
return (1 + valu) >>> 1; // add 1 to make it even, and divide by 2
} else {
return -1 * (valu >>> 1); // divide by two then make it negative
}
}
// Some convenience functions
// :Boolean
readBoolean(): boolean {
return this.readBits(1) === 1;
}
// ():int
readUByte(): number {
return this.readBits(8);
}
// ():int
readUShort(): number {
return this.readBits(16);
}
// ():int
readUInt(): number {
return this.readBits(32);
}
}
export default ExpGolomb;
+736
View File
@@ -0,0 +1,736 @@
import BaseVideoParser from './base-video-parser';
import ExpGolomb from './exp-golomb';
import { parseSEIMessageFromNALu } from '../../utils/mp4-tools';
import type {
DemuxedUserdataTrack,
DemuxedVideoTrack,
} from '../../types/demuxer';
import type { ParsedVideoSample } from '../tsdemuxer';
import type { PES } from '../tsdemuxer';
class HevcVideoParser extends BaseVideoParser {
protected initVPS: Uint8Array | null = null;
public parsePES(
track: DemuxedVideoTrack,
textTrack: DemuxedUserdataTrack,
pes: PES,
endOfSegment: boolean,
) {
const units = this.parseNALu(track, pes.data, endOfSegment);
let VideoSample = this.VideoSample;
let push: boolean;
let spsfound = false;
// free pes.data to save up some memory
(pes as any).data = null;
// if new NAL units found and last sample still there, let's push ...
// this helps parsing streams with missing AUD (only do this if AUD never found)
if (VideoSample && units.length && !track.audFound) {
this.pushAccessUnit(VideoSample, track);
VideoSample = this.VideoSample = this.createVideoSample(
false,
pes.pts,
pes.dts,
);
}
units.forEach((unit) => {
switch (unit.type) {
// NON-IDR, NON RANDOM ACCESS SLICE
case 0:
case 1:
case 2:
case 3:
case 4:
case 5:
case 6:
case 7:
case 8:
case 9:
if (!VideoSample) {
VideoSample = this.VideoSample = this.createVideoSample(
false,
pes.pts,
pes.dts,
);
}
VideoSample.frame = true;
push = true;
break;
// CRA, BLA (random access picture)
case 16:
case 17:
case 18:
case 21:
push = true;
if (spsfound) {
// handle PES not starting with AUD
// if we have frame data already, that cannot belong to the same frame, so force a push
if (VideoSample?.frame && !VideoSample.key) {
this.pushAccessUnit(VideoSample, track);
VideoSample = this.VideoSample = null;
}
}
if (!VideoSample) {
VideoSample = this.VideoSample = this.createVideoSample(
true,
pes.pts,
pes.dts,
);
}
VideoSample.key = true;
VideoSample.frame = true;
break;
// IDR
case 19:
case 20:
push = true;
// handle PES not starting with AUD
// if we have frame data already, that cannot belong to the same frame, so force a push
if (VideoSample?.frame && !VideoSample.key) {
this.pushAccessUnit(VideoSample, track);
VideoSample = this.VideoSample = null;
}
if (!VideoSample) {
VideoSample = this.VideoSample = this.createVideoSample(
true,
pes.pts,
pes.dts,
);
}
VideoSample.key = true;
VideoSample.frame = true;
break;
// SEI
case 39:
push = true;
parseSEIMessageFromNALu(
unit.data,
2, // NALu header size
pes.pts as number,
textTrack.samples,
);
break;
// VPS
case 32:
push = true;
if (!track.vps) {
if (typeof track.params !== 'object') {
track.params = {};
}
track.params = Object.assign(track.params, this.readVPS(unit.data));
this.initVPS = unit.data;
}
track.vps = [unit.data];
break;
// SPS
case 33:
push = true;
spsfound = true;
if (
track.vps !== undefined &&
track.vps[0] !== this.initVPS &&
track.sps !== undefined &&
!this.matchSPS(track.sps[0], unit.data)
) {
this.initVPS = track.vps[0];
track.sps = track.pps = undefined;
}
if (!track.sps) {
const config = this.readSPS(unit.data);
track.width = config.width;
track.height = config.height;
track.pixelRatio = config.pixelRatio;
track.codec = config.codecString;
track.sps = [];
if (typeof track.params !== 'object') {
track.params = {};
}
for (const prop in config.params) {
track.params[prop] = config.params[prop];
}
}
this.pushParameterSet(track.sps, unit.data, track.vps);
if (!VideoSample) {
VideoSample = this.VideoSample = this.createVideoSample(
true,
pes.pts,
pes.dts,
);
}
VideoSample.key = true;
break;
// PPS
case 34:
push = true;
if (typeof track.params === 'object') {
if (!track.pps) {
track.pps = [];
const config = this.readPPS(unit.data);
for (const prop in config) {
track.params[prop] = config[prop];
}
}
this.pushParameterSet(track.pps, unit.data, track.vps);
}
break;
// ACCESS UNIT DELIMITER
case 35:
push = true;
track.audFound = true;
if (VideoSample?.frame) {
this.pushAccessUnit(VideoSample, track);
VideoSample = null;
}
if (!VideoSample) {
VideoSample = this.VideoSample = this.createVideoSample(
false,
pes.pts,
pes.dts,
);
}
break;
default:
push = false;
break;
}
if (VideoSample && push) {
const units = VideoSample.units;
units.push(unit);
}
});
// if last PES packet, push samples
if (endOfSegment && VideoSample) {
this.pushAccessUnit(VideoSample, track);
this.VideoSample = null;
}
}
private pushParameterSet(
parameterSets: Uint8Array[],
data: Uint8Array,
vps: Uint8Array[] | undefined,
) {
if ((vps && vps[0] === this.initVPS) || (!vps && !parameterSets.length)) {
parameterSets.push(data);
}
}
protected getNALuType(data: Uint8Array, offset: number): number {
return (data[offset] & 0x7e) >>> 1;
}
protected ebsp2rbsp(arr: Uint8Array): Uint8Array {
const dst = new Uint8Array(arr.byteLength);
let dstIdx = 0;
for (let i = 0; i < arr.byteLength; i++) {
if (i >= 2) {
// Unescape: Skip 0x03 after 00 00
if (arr[i] === 0x03 && arr[i - 1] === 0x00 && arr[i - 2] === 0x00) {
continue;
}
}
dst[dstIdx] = arr[i];
dstIdx++;
}
return new Uint8Array(dst.buffer, 0, dstIdx);
}
protected pushAccessUnit(
VideoSample: ParsedVideoSample,
videoTrack: DemuxedVideoTrack,
) {
super.pushAccessUnit(VideoSample, videoTrack);
if (this.initVPS) {
this.initVPS = null; // null initVPS to prevent possible track's sps/pps growth until next VPS
}
}
readVPS(vps: Uint8Array): {
numTemporalLayers: number;
temporalIdNested: boolean;
} {
const eg = new ExpGolomb(vps);
// remove header
eg.readUByte();
eg.readUByte();
eg.readBits(4); // video_parameter_set_id
eg.skipBits(2);
eg.readBits(6); // max_layers_minus1
const max_sub_layers_minus1 = eg.readBits(3);
const temporal_id_nesting_flag = eg.readBoolean();
// ...vui fps can be here, but empty fps value is not critical for metadata
return {
numTemporalLayers: max_sub_layers_minus1 + 1,
temporalIdNested: temporal_id_nesting_flag,
};
}
readSPS(sps: Uint8Array): {
codecString: string;
params: object;
width: number;
height: number;
pixelRatio: [number, number];
} {
const eg = new ExpGolomb(this.ebsp2rbsp(sps));
eg.readUByte();
eg.readUByte();
eg.readBits(4); //video_parameter_set_id
const max_sub_layers_minus1 = eg.readBits(3);
eg.readBoolean(); // temporal_id_nesting_flag
// profile_tier_level
const general_profile_space = eg.readBits(2);
const general_tier_flag = eg.readBoolean();
const general_profile_idc = eg.readBits(5);
const general_profile_compatibility_flags_1 = eg.readUByte();
const general_profile_compatibility_flags_2 = eg.readUByte();
const general_profile_compatibility_flags_3 = eg.readUByte();
const general_profile_compatibility_flags_4 = eg.readUByte();
const general_constraint_indicator_flags_1 = eg.readUByte();
const general_constraint_indicator_flags_2 = eg.readUByte();
const general_constraint_indicator_flags_3 = eg.readUByte();
const general_constraint_indicator_flags_4 = eg.readUByte();
const general_constraint_indicator_flags_5 = eg.readUByte();
const general_constraint_indicator_flags_6 = eg.readUByte();
const general_level_idc = eg.readUByte();
const sub_layer_profile_present_flags: boolean[] = [];
const sub_layer_level_present_flags: boolean[] = [];
for (let i = 0; i < max_sub_layers_minus1; i++) {
sub_layer_profile_present_flags.push(eg.readBoolean());
sub_layer_level_present_flags.push(eg.readBoolean());
}
if (max_sub_layers_minus1 > 0) {
for (let i = max_sub_layers_minus1; i < 8; i++) {
eg.readBits(2);
}
}
for (let i = 0; i < max_sub_layers_minus1; i++) {
if (sub_layer_profile_present_flags[i]) {
eg.readUByte(); // sub_layer_profile_space, sub_layer_tier_flag, sub_layer_profile_idc
eg.readUByte();
eg.readUByte();
eg.readUByte();
eg.readUByte(); // sub_layer_profile_compatibility_flag
eg.readUByte();
eg.readUByte();
eg.readUByte();
eg.readUByte();
eg.readUByte();
eg.readUByte();
}
if (sub_layer_level_present_flags[i]) {
eg.readUByte();
}
}
eg.readUEG(); // seq_parameter_set_id
const chroma_format_idc = eg.readUEG();
if (chroma_format_idc == 3) {
eg.skipBits(1); //separate_colour_plane_flag
}
const pic_width_in_luma_samples = eg.readUEG();
const pic_height_in_luma_samples = eg.readUEG();
const conformance_window_flag = eg.readBoolean();
let pic_left_offset = 0,
pic_right_offset = 0,
pic_top_offset = 0,
pic_bottom_offset = 0;
if (conformance_window_flag) {
pic_left_offset += eg.readUEG();
pic_right_offset += eg.readUEG();
pic_top_offset += eg.readUEG();
pic_bottom_offset += eg.readUEG();
}
const bit_depth_luma_minus8 = eg.readUEG();
const bit_depth_chroma_minus8 = eg.readUEG();
const log2_max_pic_order_cnt_lsb_minus4 = eg.readUEG();
const sub_layer_ordering_info_present_flag = eg.readBoolean();
for (
let i = sub_layer_ordering_info_present_flag ? 0 : max_sub_layers_minus1;
i <= max_sub_layers_minus1;
i++
) {
eg.skipUEG(); // max_dec_pic_buffering_minus1[i]
eg.skipUEG(); // max_num_reorder_pics[i]
eg.skipUEG(); // max_latency_increase_plus1[i]
}
eg.skipUEG(); // log2_min_luma_coding_block_size_minus3
eg.skipUEG(); // log2_diff_max_min_luma_coding_block_size
eg.skipUEG(); // log2_min_transform_block_size_minus2
eg.skipUEG(); // log2_diff_max_min_transform_block_size
eg.skipUEG(); // max_transform_hierarchy_depth_inter
eg.skipUEG(); // max_transform_hierarchy_depth_intra
const scaling_list_enabled_flag = eg.readBoolean();
if (scaling_list_enabled_flag) {
const sps_scaling_list_data_present_flag = eg.readBoolean();
if (sps_scaling_list_data_present_flag) {
for (let sizeId = 0; sizeId < 4; sizeId++) {
for (
let matrixId = 0;
matrixId < (sizeId === 3 ? 2 : 6);
matrixId++
) {
const scaling_list_pred_mode_flag = eg.readBoolean();
if (!scaling_list_pred_mode_flag) {
eg.readUEG(); // scaling_list_pred_matrix_id_delta
} else {
const coefNum = Math.min(64, 1 << (4 + (sizeId << 1)));
if (sizeId > 1) {
eg.readEG();
}
for (let i = 0; i < coefNum; i++) {
eg.readEG();
}
}
}
}
}
}
eg.readBoolean(); // amp_enabled_flag
eg.readBoolean(); // sample_adaptive_offset_enabled_flag
const pcm_enabled_flag = eg.readBoolean();
if (pcm_enabled_flag) {
eg.readUByte();
eg.skipUEG();
eg.skipUEG();
eg.readBoolean();
}
const num_short_term_ref_pic_sets = eg.readUEG();
let num_delta_pocs = 0;
for (let i = 0; i < num_short_term_ref_pic_sets; i++) {
let inter_ref_pic_set_prediction_flag = false;
if (i !== 0) {
inter_ref_pic_set_prediction_flag = eg.readBoolean();
}
if (inter_ref_pic_set_prediction_flag) {
if (i === num_short_term_ref_pic_sets) {
eg.readUEG();
}
eg.readBoolean();
eg.readUEG();
let next_num_delta_pocs = 0;
for (let j = 0; j <= num_delta_pocs; j++) {
const used_by_curr_pic_flag = eg.readBoolean();
let use_delta_flag = false;
if (!used_by_curr_pic_flag) {
use_delta_flag = eg.readBoolean();
}
if (used_by_curr_pic_flag || use_delta_flag) {
next_num_delta_pocs++;
}
}
num_delta_pocs = next_num_delta_pocs;
} else {
const num_negative_pics = eg.readUEG();
const num_positive_pics = eg.readUEG();
num_delta_pocs = num_negative_pics + num_positive_pics;
for (let j = 0; j < num_negative_pics; j++) {
eg.readUEG();
eg.readBoolean();
}
for (let j = 0; j < num_positive_pics; j++) {
eg.readUEG();
eg.readBoolean();
}
}
}
const long_term_ref_pics_present_flag = eg.readBoolean();
if (long_term_ref_pics_present_flag) {
const num_long_term_ref_pics_sps = eg.readUEG();
for (let i = 0; i < num_long_term_ref_pics_sps; i++) {
for (let j = 0; j < log2_max_pic_order_cnt_lsb_minus4 + 4; j++) {
eg.readBits(1);
}
eg.readBits(1);
}
}
let min_spatial_segmentation_idc = 0;
let sar_width = 1,
sar_height = 1;
let fps_fixed = true,
fps_den = 1,
fps_num = 0;
eg.readBoolean(); // sps_temporal_mvp_enabled_flag
eg.readBoolean(); // strong_intra_smoothing_enabled_flag
let default_display_window_flag = false;
const vui_parameters_present_flag = eg.readBoolean();
if (vui_parameters_present_flag) {
const aspect_ratio_info_present_flag = eg.readBoolean();
if (aspect_ratio_info_present_flag) {
const aspect_ratio_idc = eg.readUByte();
const sar_width_table = [
1, 12, 10, 16, 40, 24, 20, 32, 80, 18, 15, 64, 160, 4, 3, 2,
];
const sar_height_table = [
1, 11, 11, 11, 33, 11, 11, 11, 33, 11, 11, 33, 99, 3, 2, 1,
];
if (aspect_ratio_idc > 0 && aspect_ratio_idc < 16) {
sar_width = sar_width_table[aspect_ratio_idc - 1];
sar_height = sar_height_table[aspect_ratio_idc - 1];
} else if (aspect_ratio_idc === 255) {
sar_width = eg.readBits(16);
sar_height = eg.readBits(16);
}
}
const overscan_info_present_flag = eg.readBoolean();
if (overscan_info_present_flag) {
eg.readBoolean();
}
const video_signal_type_present_flag = eg.readBoolean();
if (video_signal_type_present_flag) {
eg.readBits(3);
eg.readBoolean();
const colour_description_present_flag = eg.readBoolean();
if (colour_description_present_flag) {
eg.readUByte();
eg.readUByte();
eg.readUByte();
}
}
const chroma_loc_info_present_flag = eg.readBoolean();
if (chroma_loc_info_present_flag) {
eg.readUEG();
eg.readUEG();
}
eg.readBoolean(); // neutral_chroma_indication_flag
eg.readBoolean(); // field_seq_flag
eg.readBoolean(); // frame_field_info_present_flag
default_display_window_flag = eg.readBoolean();
if (default_display_window_flag) {
eg.skipUEG();
eg.skipUEG();
eg.skipUEG();
eg.skipUEG();
}
const vui_timing_info_present_flag = eg.readBoolean();
if (vui_timing_info_present_flag) {
fps_den = eg.readBits(32);
fps_num = eg.readBits(32);
const vui_poc_proportional_to_timing_flag = eg.readBoolean();
if (vui_poc_proportional_to_timing_flag) {
eg.readUEG();
}
const vui_hrd_parameters_present_flag = eg.readBoolean();
if (vui_hrd_parameters_present_flag) {
//const commonInfPresentFlag = true;
//if (commonInfPresentFlag) {
const nal_hrd_parameters_present_flag = eg.readBoolean();
const vcl_hrd_parameters_present_flag = eg.readBoolean();
let sub_pic_hrd_params_present_flag = false;
if (
nal_hrd_parameters_present_flag ||
vcl_hrd_parameters_present_flag
) {
sub_pic_hrd_params_present_flag = eg.readBoolean();
if (sub_pic_hrd_params_present_flag) {
eg.readUByte();
eg.readBits(5);
eg.readBoolean();
eg.readBits(5);
}
eg.readBits(4); // bit_rate_scale
eg.readBits(4); // cpb_size_scale
if (sub_pic_hrd_params_present_flag) {
eg.readBits(4);
}
eg.readBits(5);
eg.readBits(5);
eg.readBits(5);
}
//}
for (let i = 0; i <= max_sub_layers_minus1; i++) {
fps_fixed = eg.readBoolean(); // fixed_pic_rate_general_flag
const fixed_pic_rate_within_cvs_flag =
fps_fixed || eg.readBoolean();
let low_delay_hrd_flag = false;
if (fixed_pic_rate_within_cvs_flag) {
eg.readEG();
} else {
low_delay_hrd_flag = eg.readBoolean();
}
const cpb_cnt = low_delay_hrd_flag ? 1 : eg.readUEG() + 1;
if (nal_hrd_parameters_present_flag) {
for (let j = 0; j < cpb_cnt; j++) {
eg.readUEG();
eg.readUEG();
if (sub_pic_hrd_params_present_flag) {
eg.readUEG();
eg.readUEG();
}
eg.skipBits(1);
}
}
if (vcl_hrd_parameters_present_flag) {
for (let j = 0; j < cpb_cnt; j++) {
eg.readUEG();
eg.readUEG();
if (sub_pic_hrd_params_present_flag) {
eg.readUEG();
eg.readUEG();
}
eg.skipBits(1);
}
}
}
}
}
const bitstream_restriction_flag = eg.readBoolean();
if (bitstream_restriction_flag) {
eg.readBoolean(); // tiles_fixed_structure_flag
eg.readBoolean(); // motion_vectors_over_pic_boundaries_flag
eg.readBoolean(); // restricted_ref_pic_lists_flag
min_spatial_segmentation_idc = eg.readUEG();
}
}
let width = pic_width_in_luma_samples,
height = pic_height_in_luma_samples;
if (conformance_window_flag) {
let chroma_scale_w = 1,
chroma_scale_h = 1;
if (chroma_format_idc === 1) {
// YUV 420
chroma_scale_w = chroma_scale_h = 2;
} else if (chroma_format_idc == 2) {
// YUV 422
chroma_scale_w = 2;
}
width =
pic_width_in_luma_samples -
chroma_scale_w * pic_right_offset -
chroma_scale_w * pic_left_offset;
height =
pic_height_in_luma_samples -
chroma_scale_h * pic_bottom_offset -
chroma_scale_h * pic_top_offset;
}
const profile_space_string = general_profile_space
? ['A', 'B', 'C'][general_profile_space]
: '';
const profile_compatibility_buf =
(general_profile_compatibility_flags_1 << 24) |
(general_profile_compatibility_flags_2 << 16) |
(general_profile_compatibility_flags_3 << 8) |
general_profile_compatibility_flags_4;
let profile_compatibility_rev = 0;
for (let i = 0; i < 32; i++) {
profile_compatibility_rev =
(profile_compatibility_rev |
(((profile_compatibility_buf >> i) & 1) << (31 - i))) >>>
0; // reverse bit position (and cast as UInt32)
}
let profile_compatibility_flags_string =
profile_compatibility_rev.toString(16);
if (
general_profile_idc === 1 &&
profile_compatibility_flags_string === '2'
) {
profile_compatibility_flags_string = '6';
}
const tier_flag_string = general_tier_flag ? 'H' : 'L';
return {
codecString: `hvc1.${profile_space_string}${general_profile_idc}.${profile_compatibility_flags_string}.${tier_flag_string}${general_level_idc}.B0`,
params: {
general_tier_flag,
general_profile_idc,
general_profile_space,
general_profile_compatibility_flags: [
general_profile_compatibility_flags_1,
general_profile_compatibility_flags_2,
general_profile_compatibility_flags_3,
general_profile_compatibility_flags_4,
],
general_constraint_indicator_flags: [
general_constraint_indicator_flags_1,
general_constraint_indicator_flags_2,
general_constraint_indicator_flags_3,
general_constraint_indicator_flags_4,
general_constraint_indicator_flags_5,
general_constraint_indicator_flags_6,
],
general_level_idc,
bit_depth: bit_depth_luma_minus8 + 8,
bit_depth_luma_minus8,
bit_depth_chroma_minus8,
min_spatial_segmentation_idc,
chroma_format_idc: chroma_format_idc,
frame_rate: {
fixed: fps_fixed,
fps: fps_num / fps_den,
},
},
width,
height,
pixelRatio: [sar_width, sar_height],
};
}
readPPS(pps: Uint8Array): {
parallelismType: number;
} {
const eg = new ExpGolomb(this.ebsp2rbsp(pps));
eg.readUByte();
eg.readUByte();
eg.skipUEG(); // pic_parameter_set_id
eg.skipUEG(); // seq_parameter_set_id
eg.skipBits(2); // dependent_slice_segments_enabled_flag, output_flag_present_flag
eg.skipBits(3); // num_extra_slice_header_bits
eg.skipBits(2); // sign_data_hiding_enabled_flag, cabac_init_present_flag
eg.skipUEG();
eg.skipUEG();
eg.skipEG(); // init_qp_minus26
eg.skipBits(2); // constrained_intra_pred_flag, transform_skip_enabled_flag
const cu_qp_delta_enabled_flag = eg.readBoolean();
if (cu_qp_delta_enabled_flag) {
eg.skipUEG();
}
eg.skipEG(); // cb_qp_offset
eg.skipEG(); // cr_qp_offset
eg.skipBits(4); // pps_slice_chroma_qp_offsets_present_flag, weighted_pred_flag, weighted_bipred_flag, transquant_bypass_enabled_flag
const tiles_enabled_flag = eg.readBoolean();
const entropy_coding_sync_enabled_flag = eg.readBoolean();
let parallelismType = 1; // slice-based parallel decoding
if (entropy_coding_sync_enabled_flag && tiles_enabled_flag) {
parallelismType = 0; // mixed-type parallel decoding
} else if (entropy_coding_sync_enabled_flag) {
parallelismType = 3; // wavefront-based parallel decoding
} else if (tiles_enabled_flag) {
parallelismType = 2; // tile-based parallel decoding
}
return {
parallelismType,
};
}
matchSPS(sps1: Uint8Array, sps2: Uint8Array): boolean {
// compare without headers and VPS related params
return (
String.fromCharCode.apply(null, sps1).substr(3) ===
String.fromCharCode.apply(null, sps2).substr(3)
);
}
}
export default HevcVideoParser;