Home Reference Source

src/demux/transmuxer.ts

  1. import type { HlsEventEmitter } from '../events';
  2. import { Events } from '../events';
  3. import { ErrorTypes, ErrorDetails } from '../errors';
  4. import Decrypter from '../crypt/decrypter';
  5. import AACDemuxer from '../demux/aacdemuxer';
  6. import MP4Demuxer from '../demux/mp4demuxer';
  7. import TSDemuxer, { TypeSupported } from '../demux/tsdemuxer';
  8. import MP3Demuxer from '../demux/mp3demuxer';
  9. import MP4Remuxer from '../remux/mp4-remuxer';
  10. import PassThroughRemuxer from '../remux/passthrough-remuxer';
  11. import { logger } from '../utils/logger';
  12. import type { Demuxer, DemuxerResult, KeyData } from '../types/demuxer';
  13. import type { Remuxer } from '../types/remuxer';
  14. import type { TransmuxerResult, ChunkMetadata } from '../types/transmuxer';
  15. import type { HlsConfig } from '../config';
  16. import type { LevelKey } from '../loader/level-key';
  17. import type { PlaylistLevelType } from '../types/loader';
  18.  
  19. let now;
  20. // performance.now() not available on WebWorker, at least on Safari Desktop
  21. try {
  22. now = self.performance.now.bind(self.performance);
  23. } catch (err) {
  24. logger.debug('Unable to use Performance API on this environment');
  25. now = self.Date.now;
  26. }
  27.  
  28. type MuxConfig =
  29. | { demux: typeof TSDemuxer; remux: typeof MP4Remuxer }
  30. | { demux: typeof MP4Demuxer; remux: typeof PassThroughRemuxer }
  31. | { demux: typeof AACDemuxer; remux: typeof MP4Remuxer }
  32. | { demux: typeof MP3Demuxer; remux: typeof MP4Remuxer };
  33.  
  34. const muxConfig: MuxConfig[] = [
  35. { demux: TSDemuxer, remux: MP4Remuxer },
  36. { demux: MP4Demuxer, remux: PassThroughRemuxer },
  37. { demux: AACDemuxer, remux: MP4Remuxer },
  38. { demux: MP3Demuxer, remux: MP4Remuxer },
  39. ];
  40.  
  41. export default class Transmuxer {
  42. private observer: HlsEventEmitter;
  43. private typeSupported: TypeSupported;
  44. private config: HlsConfig;
  45. private vendor: string;
  46. private id: PlaylistLevelType;
  47. private demuxer?: Demuxer;
  48. private remuxer?: Remuxer;
  49. private decrypter?: Decrypter;
  50. private probe!: Function;
  51. private decryptionPromise: Promise<TransmuxerResult> | null = null;
  52. private transmuxConfig!: TransmuxConfig;
  53. private currentTransmuxState!: TransmuxState;
  54.  
  55. constructor(
  56. observer: HlsEventEmitter,
  57. typeSupported: TypeSupported,
  58. config: HlsConfig,
  59. vendor: string,
  60. id: PlaylistLevelType
  61. ) {
  62. this.observer = observer;
  63. this.typeSupported = typeSupported;
  64. this.config = config;
  65. this.vendor = vendor;
  66. this.id = id;
  67. }
  68.  
  69. configure(transmuxConfig: TransmuxConfig) {
  70. this.transmuxConfig = transmuxConfig;
  71. if (this.decrypter) {
  72. this.decrypter.reset();
  73. }
  74. }
  75.  
  76. push(
  77. data: ArrayBuffer,
  78. decryptdata: LevelKey | null,
  79. chunkMeta: ChunkMetadata,
  80. state?: TransmuxState
  81. ): TransmuxerResult | Promise<TransmuxerResult> {
  82. const stats = chunkMeta.transmuxing;
  83. stats.executeStart = now();
  84.  
  85. let uintData: Uint8Array = new Uint8Array(data);
  86. const { config, currentTransmuxState, transmuxConfig } = this;
  87. if (state) {
  88. this.currentTransmuxState = state;
  89. }
  90.  
  91. const {
  92. contiguous,
  93. discontinuity,
  94. trackSwitch,
  95. accurateTimeOffset,
  96. timeOffset,
  97. initSegmentChange,
  98. } = state || currentTransmuxState;
  99. const {
  100. audioCodec,
  101. videoCodec,
  102. defaultInitPts,
  103. duration,
  104. initSegmentData,
  105. } = transmuxConfig;
  106.  
  107. // Reset muxers before probing to ensure that their state is clean, even if flushing occurs before a successful probe
  108. if (discontinuity || trackSwitch || initSegmentChange) {
  109. this.resetInitSegment(initSegmentData, audioCodec, videoCodec, duration);
  110. }
  111.  
  112. if (discontinuity || initSegmentChange) {
  113. this.resetInitialTimestamp(defaultInitPts);
  114. }
  115.  
  116. if (!contiguous) {
  117. this.resetContiguity();
  118. }
  119.  
  120. const keyData = getEncryptionType(uintData, decryptdata);
  121. if (keyData && keyData.method === 'AES-128') {
  122. const decrypter = this.getDecrypter();
  123. // Software decryption is synchronous; webCrypto is not
  124. if (config.enableSoftwareAES) {
  125. // Software decryption is progressive. Progressive decryption may not return a result on each call. Any cached
  126. // data is handled in the flush() call
  127. const decryptedData = decrypter.softwareDecrypt(
  128. uintData,
  129. keyData.key.buffer,
  130. keyData.iv.buffer
  131. );
  132. if (!decryptedData) {
  133. stats.executeEnd = now();
  134. return emptyResult(chunkMeta);
  135. }
  136. uintData = new Uint8Array(decryptedData);
  137. } else {
  138. this.decryptionPromise = decrypter
  139. .webCryptoDecrypt(uintData, keyData.key.buffer, keyData.iv.buffer)
  140. .then((decryptedData): TransmuxerResult => {
  141. // Calling push here is important; if flush() is called while this is still resolving, this ensures that
  142. // the decrypted data has been transmuxed
  143. const result = this.push(
  144. decryptedData,
  145. null,
  146. chunkMeta
  147. ) as TransmuxerResult;
  148. this.decryptionPromise = null;
  149. return result;
  150. });
  151. return this.decryptionPromise!;
  152. }
  153. }
  154.  
  155. if (this.needsProbing(uintData, discontinuity, trackSwitch)) {
  156. this.configureTransmuxer(uintData, transmuxConfig);
  157. }
  158.  
  159. const result = this.transmux(
  160. uintData,
  161. keyData,
  162. timeOffset,
  163. accurateTimeOffset,
  164. chunkMeta
  165. );
  166. const currentState = this.currentTransmuxState;
  167.  
  168. currentState.contiguous = true;
  169. currentState.discontinuity = false;
  170. currentState.trackSwitch = false;
  171.  
  172. stats.executeEnd = now();
  173. return result;
  174. }
  175.  
  176. // Due to data caching, flush calls can produce more than one TransmuxerResult (hence the Array type)
  177. flush(
  178. chunkMeta: ChunkMetadata
  179. ): TransmuxerResult[] | Promise<TransmuxerResult[]> {
  180. const stats = chunkMeta.transmuxing;
  181. stats.executeStart = now();
  182.  
  183. const { decrypter, currentTransmuxState, decryptionPromise } = this;
  184.  
  185. if (decryptionPromise) {
  186. // Upon resolution, the decryption promise calls push() and returns its TransmuxerResult up the stack. Therefore
  187. // only flushing is required for async decryption
  188. return decryptionPromise.then(() => {
  189. return this.flush(chunkMeta);
  190. });
  191. }
  192.  
  193. const transmuxResults: TransmuxerResult[] = [];
  194. const { timeOffset } = currentTransmuxState;
  195. if (decrypter) {
  196. // The decrypter may have data cached, which needs to be demuxed. In this case we'll have two TransmuxResults
  197. // This happens in the case that we receive only 1 push call for a segment (either for non-progressive downloads,
  198. // or for progressive downloads with small segments)
  199. const decryptedData = decrypter.flush();
  200. if (decryptedData) {
  201. // Push always returns a TransmuxerResult if decryptdata is null
  202. transmuxResults.push(
  203. this.push(decryptedData, null, chunkMeta) as TransmuxerResult
  204. );
  205. }
  206. }
  207.  
  208. const { demuxer, remuxer } = this;
  209. if (!demuxer || !remuxer) {
  210. // If probing failed, then Hls.js has been given content its not able to handle
  211. this.observer.emit(Events.ERROR, Events.ERROR, {
  212. type: ErrorTypes.MEDIA_ERROR,
  213. details: ErrorDetails.FRAG_PARSING_ERROR,
  214. fatal: true,
  215. reason: 'no demux matching with content found',
  216. });
  217. stats.executeEnd = now();
  218. return [emptyResult(chunkMeta)];
  219. }
  220.  
  221. const demuxResultOrPromise = demuxer.flush(timeOffset);
  222. if (isPromise(demuxResultOrPromise)) {
  223. // Decrypt final SAMPLE-AES samples
  224. return demuxResultOrPromise.then((demuxResult) => {
  225. this.flushRemux(transmuxResults, demuxResult, chunkMeta);
  226. return transmuxResults;
  227. });
  228. }
  229.  
  230. this.flushRemux(transmuxResults, demuxResultOrPromise, chunkMeta);
  231. return transmuxResults;
  232. }
  233.  
  234. private flushRemux(
  235. transmuxResults: TransmuxerResult[],
  236. demuxResult: DemuxerResult,
  237. chunkMeta: ChunkMetadata
  238. ) {
  239. const { audioTrack, videoTrack, id3Track, textTrack } = demuxResult;
  240. const { accurateTimeOffset, timeOffset } = this.currentTransmuxState;
  241. logger.log(
  242. `[transmuxer.ts]: Flushed fragment ${chunkMeta.sn}${
  243. chunkMeta.part > -1 ? ' p: ' + chunkMeta.part : ''
  244. } of level ${chunkMeta.level}`
  245. );
  246. const remuxResult = this.remuxer!.remux(
  247. audioTrack,
  248. videoTrack,
  249. id3Track,
  250. textTrack,
  251. timeOffset,
  252. accurateTimeOffset,
  253. true,
  254. this.id
  255. );
  256. transmuxResults.push({
  257. remuxResult,
  258. chunkMeta,
  259. });
  260.  
  261. chunkMeta.transmuxing.executeEnd = now();
  262. }
  263.  
  264. resetInitialTimestamp(defaultInitPts: number | undefined) {
  265. const { demuxer, remuxer } = this;
  266. if (!demuxer || !remuxer) {
  267. return;
  268. }
  269. demuxer.resetTimeStamp(defaultInitPts);
  270. remuxer.resetTimeStamp(defaultInitPts);
  271. }
  272.  
  273. resetContiguity() {
  274. const { demuxer, remuxer } = this;
  275. if (!demuxer || !remuxer) {
  276. return;
  277. }
  278. demuxer.resetContiguity();
  279. remuxer.resetNextTimestamp();
  280. }
  281.  
  282. resetInitSegment(
  283. initSegmentData: Uint8Array | undefined,
  284. audioCodec: string | undefined,
  285. videoCodec: string | undefined,
  286. trackDuration: number
  287. ) {
  288. const { demuxer, remuxer } = this;
  289. if (!demuxer || !remuxer) {
  290. return;
  291. }
  292. demuxer.resetInitSegment(
  293. initSegmentData,
  294. audioCodec,
  295. videoCodec,
  296. trackDuration
  297. );
  298. remuxer.resetInitSegment(initSegmentData, audioCodec, videoCodec);
  299. }
  300.  
  301. destroy(): void {
  302. if (this.demuxer) {
  303. this.demuxer.destroy();
  304. this.demuxer = undefined;
  305. }
  306. if (this.remuxer) {
  307. this.remuxer.destroy();
  308. this.remuxer = undefined;
  309. }
  310. }
  311.  
  312. private transmux(
  313. data: Uint8Array,
  314. keyData: KeyData | null,
  315. timeOffset: number,
  316. accurateTimeOffset: boolean,
  317. chunkMeta: ChunkMetadata
  318. ): TransmuxerResult | Promise<TransmuxerResult> {
  319. let result: TransmuxerResult | Promise<TransmuxerResult>;
  320. if (keyData && keyData.method === 'SAMPLE-AES') {
  321. result = this.transmuxSampleAes(
  322. data,
  323. keyData,
  324. timeOffset,
  325. accurateTimeOffset,
  326. chunkMeta
  327. );
  328. } else {
  329. result = this.transmuxUnencrypted(
  330. data,
  331. timeOffset,
  332. accurateTimeOffset,
  333. chunkMeta
  334. );
  335. }
  336. return result;
  337. }
  338.  
  339. private transmuxUnencrypted(
  340. data: Uint8Array,
  341. timeOffset: number,
  342. accurateTimeOffset: boolean,
  343. chunkMeta: ChunkMetadata
  344. ): TransmuxerResult {
  345. const { audioTrack, videoTrack, id3Track, textTrack } = (
  346. this.demuxer as Demuxer
  347. ).demux(data, timeOffset, false, !this.config.progressive);
  348. const remuxResult = this.remuxer!.remux(
  349. audioTrack,
  350. videoTrack,
  351. id3Track,
  352. textTrack,
  353. timeOffset,
  354. accurateTimeOffset,
  355. false,
  356. this.id
  357. );
  358. return {
  359. remuxResult,
  360. chunkMeta,
  361. };
  362. }
  363.  
  364. private transmuxSampleAes(
  365. data: Uint8Array,
  366. decryptData: KeyData,
  367. timeOffset: number,
  368. accurateTimeOffset: boolean,
  369. chunkMeta: ChunkMetadata
  370. ): Promise<TransmuxerResult> {
  371. return (this.demuxer as Demuxer)
  372. .demuxSampleAes(data, decryptData, timeOffset)
  373. .then((demuxResult) => {
  374. const remuxResult = this.remuxer!.remux(
  375. demuxResult.audioTrack,
  376. demuxResult.videoTrack,
  377. demuxResult.id3Track,
  378. demuxResult.textTrack,
  379. timeOffset,
  380. accurateTimeOffset,
  381. false,
  382. this.id
  383. );
  384. return {
  385. remuxResult,
  386. chunkMeta,
  387. };
  388. });
  389. }
  390.  
  391. private configureTransmuxer(
  392. data: Uint8Array,
  393. transmuxConfig: TransmuxConfig
  394. ) {
  395. const { config, observer, typeSupported, vendor } = this;
  396. const {
  397. audioCodec,
  398. defaultInitPts,
  399. duration,
  400. initSegmentData,
  401. videoCodec,
  402. } = transmuxConfig;
  403. // probe for content type
  404. let mux;
  405. for (let i = 0, len = muxConfig.length; i < len; i++) {
  406. if (muxConfig[i].demux.probe(data)) {
  407. mux = muxConfig[i];
  408. break;
  409. }
  410. }
  411. if (!mux) {
  412. // If probing previous configs fail, use mp4 passthrough
  413. logger.warn(
  414. 'Failed to find demuxer by probing frag, treating as mp4 passthrough'
  415. );
  416. mux = { demux: MP4Demuxer, remux: PassThroughRemuxer };
  417. }
  418. // so let's check that current remuxer and demuxer are still valid
  419. const demuxer = this.demuxer;
  420. const remuxer = this.remuxer;
  421. const Remuxer: MuxConfig['remux'] = mux.remux;
  422. const Demuxer: MuxConfig['demux'] = mux.demux;
  423. if (!remuxer || !(remuxer instanceof Remuxer)) {
  424. this.remuxer = new Remuxer(observer, config, typeSupported, vendor);
  425. }
  426. if (!demuxer || !(demuxer instanceof Demuxer)) {
  427. this.demuxer = new Demuxer(observer, config, typeSupported);
  428. this.probe = Demuxer.probe;
  429. }
  430. // Ensure that muxers are always initialized with an initSegment
  431. this.resetInitSegment(initSegmentData, audioCodec, videoCodec, duration);
  432. this.resetInitialTimestamp(defaultInitPts);
  433. }
  434.  
  435. private needsProbing(
  436. data: Uint8Array,
  437. discontinuity: boolean,
  438. trackSwitch: boolean
  439. ): boolean {
  440. // in case of continuity change, or track switch
  441. // we might switch from content type (AAC container to TS container, or TS to fmp4 for example)
  442. return !this.demuxer || !this.remuxer || discontinuity || trackSwitch;
  443. }
  444.  
  445. private getDecrypter(): Decrypter {
  446. let decrypter = this.decrypter;
  447. if (!decrypter) {
  448. decrypter = this.decrypter = new Decrypter(this.observer, this.config);
  449. }
  450. return decrypter;
  451. }
  452. }
  453.  
  454. function getEncryptionType(
  455. data: Uint8Array,
  456. decryptData: LevelKey | null
  457. ): KeyData | null {
  458. let encryptionType: KeyData | null = null;
  459. if (
  460. data.byteLength > 0 &&
  461. decryptData != null &&
  462. decryptData.key != null &&
  463. decryptData.iv !== null &&
  464. decryptData.method != null
  465. ) {
  466. encryptionType = decryptData as KeyData;
  467. }
  468. return encryptionType;
  469. }
  470.  
  471. const emptyResult = (chunkMeta): TransmuxerResult => ({
  472. remuxResult: {},
  473. chunkMeta,
  474. });
  475.  
  476. export function isPromise<T>(p: Promise<T> | any): p is Promise<T> {
  477. return 'then' in p && p.then instanceof Function;
  478. }
  479.  
  480. export class TransmuxConfig {
  481. public audioCodec?: string;
  482. public videoCodec?: string;
  483. public initSegmentData?: Uint8Array;
  484. public duration: number;
  485. public defaultInitPts?: number;
  486.  
  487. constructor(
  488. audioCodec: string | undefined,
  489. videoCodec: string | undefined,
  490. initSegmentData: Uint8Array | undefined,
  491. duration: number,
  492. defaultInitPts?: number
  493. ) {
  494. this.audioCodec = audioCodec;
  495. this.videoCodec = videoCodec;
  496. this.initSegmentData = initSegmentData;
  497. this.duration = duration;
  498. this.defaultInitPts = defaultInitPts;
  499. }
  500. }
  501.  
  502. export class TransmuxState {
  503. public discontinuity: boolean;
  504. public contiguous: boolean;
  505. public accurateTimeOffset: boolean;
  506. public trackSwitch: boolean;
  507. public timeOffset: number;
  508. public initSegmentChange: boolean;
  509.  
  510. constructor(
  511. discontinuity: boolean,
  512. contiguous: boolean,
  513. accurateTimeOffset: boolean,
  514. trackSwitch: boolean,
  515. timeOffset: number,
  516. initSegmentChange: boolean
  517. ) {
  518. this.discontinuity = discontinuity;
  519. this.contiguous = contiguous;
  520. this.accurateTimeOffset = accurateTimeOffset;
  521. this.trackSwitch = trackSwitch;
  522. this.timeOffset = timeOffset;
  523. this.initSegmentChange = initSegmentChange;
  524. }
  525. }