ffmpeg_demuxer.h 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. // Implements the Demuxer interface using FFmpeg's libavformat. At this time
  5. // will support demuxing any audio/video format thrown at it. The streams
  6. // output mime types audio/x-ffmpeg and video/x-ffmpeg and include an integer
  7. // key FFmpegCodecID which contains the CodecID enumeration value. The CodecIDs
  8. // can be used to create and initialize the corresponding FFmpeg decoder.
  9. //
  10. // FFmpegDemuxer sets the duration of pipeline during initialization by using
  11. // the duration of the longest audio/video stream.
  12. //
  13. // NOTE: since FFmpegDemuxer reads packets sequentially without seeking, media
  14. // files with very large drift between audio/video streams may result in
  15. // excessive memory consumption.
  16. //
  17. // When stopped, FFmpegDemuxer and FFmpegDemuxerStream release all callbacks
  18. // and buffered packets. Reads from a stopped FFmpegDemuxerStream will not be
  19. // replied to.
  20. #ifndef MEDIA_FILTERS_FFMPEG_DEMUXER_H_
  21. #define MEDIA_FILTERS_FFMPEG_DEMUXER_H_
  22. #include <stddef.h>
  23. #include <stdint.h>
  24. #include <memory>
  25. #include <string>
  26. #include <utility>
  27. #include <vector>
  28. #include "base/callback.h"
  29. #include "base/memory/raw_ptr.h"
  30. #include "base/memory/weak_ptr.h"
  31. #include "base/task/sequenced_task_runner.h"
  32. #include "base/time/time.h"
  33. #include "media/base/audio_decoder_config.h"
  34. #include "media/base/decoder_buffer.h"
  35. #include "media/base/decoder_buffer_queue.h"
  36. #include "media/base/demuxer.h"
  37. #include "media/base/media_log.h"
  38. #include "media/base/pipeline_status.h"
  39. #include "media/base/text_track_config.h"
  40. #include "media/base/timestamp_constants.h"
  41. #include "media/base/video_decoder_config.h"
  42. #include "media/ffmpeg/scoped_av_packet.h"
  43. #include "media/filters/blocking_url_protocol.h"
  44. #include "media/media_buildflags.h"
  45. // FFmpeg forward declarations.
  46. struct AVFormatContext;
  47. struct AVRational;
  48. struct AVStream;
  49. namespace media {
  50. class MediaLog;
  51. class FFmpegBitstreamConverter;
  52. class FFmpegDemuxer;
  53. class FFmpegGlue;
  54. class MEDIA_EXPORT FFmpegDemuxerStream : public DemuxerStream {
  55. public:
  56. // Attempts to create FFmpegDemuxerStream form the given AVStream. Will return
  57. // null if the AVStream cannot be translated into a valid decoder config.
  58. //
  59. // FFmpegDemuxerStream keeps a copy of |demuxer| and initializes itself using
  60. // information inside |stream|. Both parameters must outlive |this|.
  61. static std::unique_ptr<FFmpegDemuxerStream> Create(FFmpegDemuxer* demuxer,
  62. AVStream* stream,
  63. MediaLog* media_log);
  64. FFmpegDemuxerStream(const FFmpegDemuxerStream&) = delete;
  65. FFmpegDemuxerStream& operator=(const FFmpegDemuxerStream&) = delete;
  66. ~FFmpegDemuxerStream() override;
  67. // Enqueues the given AVPacket. It is invalid to queue a |packet| after
  68. // SetEndOfStream() has been called.
  69. void EnqueuePacket(ScopedAVPacket packet);
  70. // Enters the end of stream state. After delivering remaining queued buffers
  71. // only end of stream buffers will be delivered.
  72. void SetEndOfStream();
  73. // Drops queued buffers and clears end of stream state.
  74. // Passing |preserve_packet_position| will prevent replay of already seen
  75. // packets.
  76. void FlushBuffers(bool preserve_packet_position);
  77. // Empties the queues and ignores any additional calls to Read().
  78. void Stop();
  79. // Aborts any pending reads.
  80. void Abort();
  81. base::TimeDelta duration() const { return duration_; }
  82. // Enables fixes for files with negative timestamps. Normally all timestamps
  83. // are rebased against FFmpegDemuxer::start_time() whenever that value is
  84. // negative. When this fix is enabled, only AUDIO stream packets will be
  85. // rebased to time zero, all other stream types will use the muxed timestamp.
  86. //
  87. // Further, when no codec delay is present, all AUDIO packets which originally
  88. // had negative timestamps will be marked for post-decode discard. When codec
  89. // delay is present, it is assumed the decoder will handle discard and does
  90. // not need the AUDIO packets to be marked for discard; just rebased to zero.
  91. void enable_negative_timestamp_fixups() {
  92. fixup_negative_timestamps_ = true;
  93. }
  94. void enable_chained_ogg_fixups() { fixup_chained_ogg_ = true; }
  95. // DemuxerStream implementation.
  96. Type type() const override;
  97. StreamLiveness liveness() const override;
  98. void Read(ReadCB read_cb) override;
  99. void EnableBitstreamConverter() override;
  100. bool SupportsConfigChanges() override;
  101. AudioDecoderConfig audio_decoder_config() override;
  102. VideoDecoderConfig video_decoder_config() override;
  103. bool IsEnabled() const;
  104. void SetEnabled(bool enabled, base::TimeDelta timestamp);
  105. void SetLiveness(StreamLiveness liveness);
  106. // Returns the range of buffered data in this stream.
  107. Ranges<base::TimeDelta> GetBufferedRanges() const;
  108. // Returns true if this stream has capacity for additional data.
  109. bool HasAvailableCapacity();
  110. // Returns the total buffer size FFMpegDemuxerStream is holding onto.
  111. size_t MemoryUsage() const;
  112. TextKind GetTextKind() const;
  113. // Returns the value associated with |key| in the metadata for the avstream.
  114. // Returns an empty string if the key is not present.
  115. std::string GetMetadata(const char* key) const;
  116. AVStream* av_stream() const { return stream_; }
  117. base::TimeDelta start_time() const { return start_time_; }
  118. void set_start_time(base::TimeDelta time) { start_time_ = time; }
  119. private:
  120. friend class FFmpegDemuxerTest;
  121. // Use FFmpegDemuxerStream::Create to construct.
  122. // Audio/Video streams must include their respective DecoderConfig. At most
  123. // one DecoderConfig should be provided (leaving the other nullptr). Both
  124. // configs should be null for text streams.
  125. FFmpegDemuxerStream(FFmpegDemuxer* demuxer,
  126. AVStream* stream,
  127. std::unique_ptr<AudioDecoderConfig> audio_config,
  128. std::unique_ptr<VideoDecoderConfig> video_config,
  129. MediaLog* media_log);
  130. // Runs |read_cb_| if present with the front of |buffer_queue_|, calling
  131. // NotifyCapacityAvailable() if capacity is still available.
  132. void SatisfyPendingRead();
  133. // Converts an FFmpeg stream timestamp into a base::TimeDelta.
  134. static base::TimeDelta ConvertStreamTimestamp(const AVRational& time_base,
  135. int64_t timestamp);
  136. // Resets any currently active bitstream converter.
  137. void ResetBitstreamConverter();
  138. // Create new bitstream converter, destroying active converter if present.
  139. void InitBitstreamConverter();
  140. raw_ptr<FFmpegDemuxer> demuxer_;
  141. scoped_refptr<base::SequencedTaskRunner> task_runner_;
  142. raw_ptr<AVStream> stream_;
  143. base::TimeDelta start_time_;
  144. std::unique_ptr<AudioDecoderConfig> audio_config_;
  145. std::unique_ptr<VideoDecoderConfig> video_config_;
  146. raw_ptr<MediaLog> media_log_;
  147. Type type_ = UNKNOWN;
  148. StreamLiveness liveness_ = StreamLiveness::kUnknown;
  149. base::TimeDelta duration_;
  150. bool end_of_stream_;
  151. base::TimeDelta last_packet_timestamp_;
  152. base::TimeDelta last_packet_duration_;
  153. Ranges<base::TimeDelta> buffered_ranges_;
  154. bool is_enabled_;
  155. bool waiting_for_keyframe_;
  156. bool aborted_;
  157. DecoderBufferQueue buffer_queue_;
  158. ReadCB read_cb_;
  159. #if BUILDFLAG(USE_PROPRIETARY_CODECS)
  160. std::unique_ptr<FFmpegBitstreamConverter> bitstream_converter_;
  161. #endif
  162. std::string encryption_key_id_;
  163. bool fixup_negative_timestamps_;
  164. bool fixup_chained_ogg_;
  165. int num_discarded_packet_warnings_;
  166. int64_t last_packet_pos_;
  167. int64_t last_packet_dts_;
  168. };
  169. class MEDIA_EXPORT FFmpegDemuxer : public Demuxer {
  170. public:
  171. FFmpegDemuxer(const scoped_refptr<base::SequencedTaskRunner>& task_runner,
  172. DataSource* data_source,
  173. const EncryptedMediaInitDataCB& encrypted_media_init_data_cb,
  174. MediaTracksUpdatedCB media_tracks_updated_cb,
  175. MediaLog* media_log,
  176. bool is_local_file);
  177. FFmpegDemuxer(const FFmpegDemuxer&) = delete;
  178. FFmpegDemuxer& operator=(const FFmpegDemuxer&) = delete;
  179. ~FFmpegDemuxer() override;
  180. // Demuxer implementation.
  181. std::string GetDisplayName() const override;
  182. void Initialize(DemuxerHost* host, PipelineStatusCallback init_cb) override;
  183. void AbortPendingReads() override;
  184. void Stop() override;
  185. void StartWaitingForSeek(base::TimeDelta seek_time) override;
  186. void CancelPendingSeek(base::TimeDelta seek_time) override;
  187. void Seek(base::TimeDelta time, PipelineStatusCallback cb) override;
  188. base::Time GetTimelineOffset() const override;
  189. std::vector<DemuxerStream*> GetAllStreams() override;
  190. base::TimeDelta GetStartTime() const override;
  191. int64_t GetMemoryUsage() const override;
  192. absl::optional<container_names::MediaContainerName> GetContainerForMetrics()
  193. const override;
  194. // Calls |encrypted_media_init_data_cb_| with the initialization data
  195. // encountered in the file.
  196. void OnEncryptedMediaInitData(EmeInitDataType init_data_type,
  197. const std::string& encryption_key_id);
  198. // Allow FFmpegDemuxerStream to notify us when there is updated information
  199. // about capacity and what buffered data is available.
  200. void NotifyCapacityAvailable();
  201. void NotifyBufferingChanged();
  202. // Allow FFmpegDemxuerStream to notify us about an error.
  203. void NotifyDemuxerError(PipelineStatus error);
  204. void OnEnabledAudioTracksChanged(const std::vector<MediaTrack::Id>& track_ids,
  205. base::TimeDelta curr_time,
  206. TrackChangeCB change_completed_cb) override;
  207. void OnSelectedVideoTrackChanged(const std::vector<MediaTrack::Id>& track_ids,
  208. base::TimeDelta curr_time,
  209. TrackChangeCB change_completed_cb) override;
  210. // The lowest demuxed timestamp. If negative, DemuxerStreams must use this to
  211. // adjust packet timestamps such that external clients see a zero-based
  212. // timeline.
  213. base::TimeDelta start_time() const { return start_time_; }
  214. // Task runner used to execute blocking FFmpeg operations.
  215. scoped_refptr<base::SequencedTaskRunner> ffmpeg_task_runner() {
  216. return blocking_task_runner_;
  217. }
  218. container_names::MediaContainerName container() const {
  219. return glue_ ? glue_->container() : container_names::CONTAINER_UNKNOWN;
  220. }
  221. private:
  222. // To allow tests access to privates.
  223. friend class FFmpegDemuxerTest;
  224. // Helper for vide and audio track changing.
  225. void FindAndEnableProperTracks(const std::vector<MediaTrack::Id>& track_ids,
  226. base::TimeDelta curr_time,
  227. DemuxerStream::Type track_type,
  228. TrackChangeCB change_completed_cb);
  229. // FFmpeg callbacks during initialization.
  230. void OnOpenContextDone(bool result);
  231. void OnFindStreamInfoDone(int result);
  232. void LogMetadata(AVFormatContext* avctx, base::TimeDelta max_duration);
  233. // Finds the stream with the lowest known start time (i.e. not kNoTimestamp
  234. // start time) with enabled status matching |enabled|.
  235. FFmpegDemuxerStream* FindStreamWithLowestStartTimestamp(bool enabled);
  236. // Finds a preferred stream for seeking to |seek_time|. Preference is
  237. // typically given to video streams, unless the |seek_time| is earlier than
  238. // the start time of the video stream. In that case a stream with the earliest
  239. // start time is preferred. Disabled streams are considered only as the last
  240. // fallback option.
  241. FFmpegDemuxerStream* FindPreferredStreamForSeeking(base::TimeDelta seek_time);
  242. // FFmpeg callbacks during seeking.
  243. void OnSeekFrameSuccess();
  244. // FFmpeg callbacks during reading + helper method to initiate reads.
  245. void ReadFrameIfNeeded();
  246. void OnReadFrameDone(ScopedAVPacket packet, int result);
  247. // Returns true iff any stream has additional capacity. Note that streams can
  248. // go over capacity depending on how the file is muxed.
  249. bool StreamsHaveAvailableCapacity();
  250. // Returns true if the maximum allowed memory usage has been reached.
  251. bool IsMaxMemoryUsageReached() const;
  252. // Signal all FFmpegDemuxerStreams that the stream has ended.
  253. void StreamHasEnded();
  254. // Called by |url_protocol_| whenever |data_source_| returns a read error.
  255. void OnDataSourceError();
  256. // Returns the first stream from |streams_| that matches |type| as an
  257. // FFmpegDemuxerStream and is enabled.
  258. FFmpegDemuxerStream* GetFirstEnabledFFmpegStream(
  259. DemuxerStream::Type type) const;
  260. // Called after the streams have been collected from the media, to allow
  261. // the text renderer to bind each text stream to the cue rendering engine.
  262. void AddTextStreams();
  263. void SetLiveness(StreamLiveness liveness);
  264. void SeekInternal(base::TimeDelta time, base::OnceClosure seek_cb);
  265. void OnVideoSeekedForTrackChange(DemuxerStream* video_stream,
  266. base::OnceClosure seek_completed_cb);
  267. void SeekOnVideoTrackChange(base::TimeDelta seek_to_time,
  268. TrackChangeCB seek_completed_cb,
  269. DemuxerStream::Type stream_type,
  270. const std::vector<DemuxerStream*>& streams);
  271. // Executes |init_cb_| with |status| and closes out the async trace.
  272. void RunInitCB(PipelineStatus status);
  273. // Executes |pending_seek_cb_| with |status| and closes out the async trace.
  274. void RunPendingSeekCB(PipelineStatus status);
  275. raw_ptr<DemuxerHost> host_ = nullptr;
  276. scoped_refptr<base::SequencedTaskRunner> task_runner_;
  277. // Task runner on which all blocking FFmpeg operations are executed; retrieved
  278. // from base::ThreadPoolInstance.
  279. scoped_refptr<base::SequencedTaskRunner> blocking_task_runner_;
  280. PipelineStatusCallback init_cb_;
  281. // Indicates if Stop() has been called.
  282. bool stopped_ = false;
  283. // Tracks if there's an outstanding av_read_frame() operation.
  284. //
  285. // TODO(scherkus): Allow more than one read in flight for higher read
  286. // throughput using demuxer_bench to verify improvements.
  287. bool pending_read_ = false;
  288. // Tracks if there's an outstanding av_seek_frame() operation. Used to discard
  289. // results of pre-seek av_read_frame() operations.
  290. PipelineStatusCallback pending_seek_cb_;
  291. // |streams_| mirrors the AVStream array in AVFormatContext. It contains
  292. // FFmpegDemuxerStreams encapsluating AVStream objects at the same index.
  293. //
  294. // Since we only support a single audio and video stream, |streams_| will
  295. // contain NULL entries for additional audio/video streams as well as for
  296. // stream types that we do not currently support.
  297. //
  298. // Once initialized, operations on FFmpegDemuxerStreams should be carried out
  299. // on the demuxer thread.
  300. using StreamVector = std::vector<std::unique_ptr<FFmpegDemuxerStream>>;
  301. StreamVector streams_;
  302. // Provides asynchronous IO to this demuxer. Consumed by |url_protocol_| to
  303. // integrate with libavformat.
  304. raw_ptr<DataSource> data_source_;
  305. raw_ptr<MediaLog> media_log_;
  306. // Derived bitrate after initialization has completed.
  307. int bitrate_ = 0;
  308. // The first timestamp of the audio or video stream, whichever is lower. This
  309. // is used to adjust timestamps so that external consumers always see a zero
  310. // based timeline.
  311. base::TimeDelta start_time_ = kNoTimestamp;
  312. // The Time associated with timestamp 0. Set to a null
  313. // time if the file doesn't have an association to Time.
  314. base::Time timeline_offset_;
  315. // Set if we know duration of the audio stream. Used when processing end of
  316. // stream -- at this moment we definitely know duration.
  317. bool duration_known_ = false;
  318. base::TimeDelta duration_;
  319. // FFmpegURLProtocol implementation and corresponding glue bits.
  320. std::unique_ptr<BlockingUrlProtocol> url_protocol_;
  321. std::unique_ptr<FFmpegGlue> glue_;
  322. const EncryptedMediaInitDataCB encrypted_media_init_data_cb_;
  323. const MediaTracksUpdatedCB media_tracks_updated_cb_;
  324. std::map<MediaTrack::Id, FFmpegDemuxerStream*> track_id_to_demux_stream_map_;
  325. const bool is_local_file_;
  326. // NOTE: Weak pointers must be invalidated before all other member variables.
  327. base::WeakPtr<FFmpegDemuxer> weak_this_;
  328. base::WeakPtrFactory<FFmpegDemuxer> cancel_pending_seek_factory_{this};
  329. base::WeakPtrFactory<FFmpegDemuxer> weak_factory_{this};
  330. };
  331. } // namespace media
  332. #endif // MEDIA_FILTERS_FFMPEG_DEMUXER_H_