source_buffer_state.h 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277
  1. // Copyright (c) 2016 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef MEDIA_FILTERS_SOURCE_BUFFER_STATE_H_
  5. #define MEDIA_FILTERS_SOURCE_BUFFER_STATE_H_
  6. #include "base/bind.h"
  7. #include "base/memory/memory_pressure_listener.h"
  8. #include "base/memory/raw_ptr.h"
  9. #include "base/time/time.h"
  10. #include "media/base/audio_codecs.h"
  11. #include "media/base/demuxer.h"
  12. #include "media/base/demuxer_stream.h"
  13. #include "media/base/media_export.h"
  14. #include "media/base/media_log.h"
  15. #include "media/base/stream_parser.h"
  16. #include "media/base/stream_parser_buffer.h"
  17. #include "media/base/video_codecs.h"
  18. #include "media/filters/source_buffer_parse_warnings.h"
  19. namespace media {
  20. class ChunkDemuxerStream;
  21. class FrameProcessor;
  22. // Contains state belonging to a source id.
  23. class MEDIA_EXPORT SourceBufferState {
  24. public:
  25. // Callback signature used to create ChunkDemuxerStreams.
  26. using CreateDemuxerStreamCB =
  27. base::RepeatingCallback<ChunkDemuxerStream*(DemuxerStream::Type)>;
  28. using NewTextTrackCB = base::RepeatingCallback<void(ChunkDemuxerStream*,
  29. const TextTrackConfig&)>;
  30. SourceBufferState(std::unique_ptr<StreamParser> stream_parser,
  31. std::unique_ptr<FrameProcessor> frame_processor,
  32. CreateDemuxerStreamCB create_demuxer_stream_cb,
  33. MediaLog* media_log);
  34. SourceBufferState(const SourceBufferState&) = delete;
  35. SourceBufferState& operator=(const SourceBufferState&) = delete;
  36. ~SourceBufferState();
  37. void Init(StreamParser::InitCB init_cb,
  38. const std::string& expected_codecs,
  39. const StreamParser::EncryptedMediaInitDataCB&
  40. encrypted_media_init_data_cb,
  41. NewTextTrackCB new_text_track_cb);
  42. // Reconfigures this source buffer to use |new_stream_parser|. Caller must
  43. // first ensure that ResetParserState() was done to flush any pending frames
  44. // from the old stream parser.
  45. void ChangeType(std::unique_ptr<StreamParser> new_stream_parser,
  46. const std::string& new_expected_codecs);
  47. // Appends new data to the StreamParser.
  48. // Returns true if the data was successfully appended. Returns false if an
  49. // error occurred. |*timestamp_offset| is used and possibly updated by the
  50. // append. |append_window_start| and |append_window_end| correspond to the MSE
  51. // spec's similarly named source buffer attributes that are used in coded
  52. // frame processing.
  53. // AppendChunks appends the provided BufferQueue.
  54. bool Append(const uint8_t* data,
  55. size_t length,
  56. base::TimeDelta append_window_start,
  57. base::TimeDelta append_window_end,
  58. base::TimeDelta* timestamp_offset);
  59. bool AppendChunks(std::unique_ptr<StreamParser::BufferQueue> buffer_queue,
  60. base::TimeDelta append_window_start,
  61. base::TimeDelta append_window_end,
  62. base::TimeDelta* timestamp_offset);
  63. // Aborts the current append sequence and resets the parser.
  64. void ResetParserState(base::TimeDelta append_window_start,
  65. base::TimeDelta append_window_end,
  66. base::TimeDelta* timestamp_offset);
  67. // Calls Remove(|start|, |end|, |duration|) on all
  68. // ChunkDemuxerStreams managed by this object.
  69. void Remove(base::TimeDelta start,
  70. base::TimeDelta end,
  71. base::TimeDelta duration);
  72. // If the buffer is full, attempts to try to free up space, as specified in
  73. // the "Coded Frame Eviction Algorithm" in the Media Source Extensions Spec.
  74. // Returns false iff buffer is still full after running eviction.
  75. // https://w3c.github.io/media-source/#sourcebuffer-coded-frame-eviction
  76. bool EvictCodedFrames(base::TimeDelta media_time, size_t newDataSize);
  77. // Gets invoked when the system is experiencing memory pressure, i.e. there's
  78. // not enough free memory. The |media_time| is the media playback position at
  79. // the time of memory pressure notification (needed for accurate GC). The
  80. // |memory_pressure_level| indicates memory pressure severity. The
  81. // |force_instant_gc| is used to force the MSE garbage collection algorithm to
  82. // be run right away, without waiting for the next append.
  83. void OnMemoryPressure(
  84. base::TimeDelta media_time,
  85. base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level,
  86. bool force_instant_gc);
  87. // Returns true if currently parsing a media segment, or false otherwise.
  88. bool parsing_media_segment() const { return parsing_media_segment_; }
  89. // Returns the 'Generate Timestamps Flag' for this SourceBuffer's byte stream
  90. // format parser as described in the MSE Byte Stream Format Registry.
  91. bool generate_timestamps_flag() const {
  92. return stream_parser_->GetGenerateTimestampsFlag();
  93. }
  94. // Sets |frame_processor_|'s sequence mode to |sequence_mode|.
  95. void SetSequenceMode(bool sequence_mode);
  96. // Signals the coded frame processor to update its group start timestamp to be
  97. // |timestamp_offset| if it is in sequence append mode.
  98. void SetGroupStartTimestampIfInSequenceMode(base::TimeDelta timestamp_offset);
  99. // Returns the range of buffered data in this source, capped at |duration|.
  100. // |ended| - Set to true if end of stream has been signaled and the special
  101. // end of stream range logic needs to be executed.
  102. Ranges<base::TimeDelta> GetBufferedRanges(base::TimeDelta duration,
  103. bool ended) const;
  104. // Returns the highest PTS of currently buffered frames in this source, or
  105. // base::TimeDelta() if none of the streams contain buffered data.
  106. base::TimeDelta GetHighestPresentationTimestamp() const;
  107. // Returns the highest buffered duration across all streams managed
  108. // by this object.
  109. // Returns base::TimeDelta() if none of the streams contain buffered data.
  110. base::TimeDelta GetMaxBufferedDuration() const;
  111. // Helper methods that call methods with similar names on all the
  112. // ChunkDemuxerStreams managed by this object.
  113. void StartReturningData();
  114. void AbortReads();
  115. void Seek(base::TimeDelta seek_time);
  116. void CompletePendingReadIfPossible();
  117. void OnSetDuration(base::TimeDelta duration);
  118. void MarkEndOfStream();
  119. void UnmarkEndOfStream();
  120. void Shutdown();
  121. // Sets the memory limit on each stream of a specific type.
  122. // |memory_limit| is the maximum number of bytes each stream of type |type|
  123. // is allowed to hold in its buffer.
  124. void SetMemoryLimits(DemuxerStream::Type type, size_t memory_limit);
  125. bool IsSeekWaitingForData() const;
  126. using RangesList = std::vector<Ranges<base::TimeDelta>>;
  127. static Ranges<base::TimeDelta> ComputeRangesIntersection(
  128. const RangesList& active_ranges,
  129. bool ended);
  130. void SetTracksWatcher(Demuxer::MediaTracksUpdatedCB tracks_updated_cb);
  131. void SetParseWarningCallback(SourceBufferParseWarningCB parse_warning_cb);
  132. private:
  133. // State advances through this list to PARSER_INITIALIZED.
  134. // The intent is to ensure at least one config is received prior to parser
  135. // calling initialization callback, and that such initialization callback
  136. // occurs at most once per parser.
  137. // PENDING_PARSER_RECONFIG occurs if State had reached PARSER_INITIALIZED
  138. // before changing to a new StreamParser in ChangeType(). In such case, State
  139. // would then advance to PENDING_PARSER_REINIT, then PARSER_INITIALIZED upon
  140. // the next initialization segment parsed, but would not run the
  141. // initialization callback in this case (since such would already have
  142. // occurred on the initial transition from PENDING_PARSER_INIT to
  143. // PARSER_INITIALIZED.)
  144. enum State {
  145. UNINITIALIZED = 0,
  146. PENDING_PARSER_CONFIG,
  147. PENDING_PARSER_INIT,
  148. PARSER_INITIALIZED,
  149. PENDING_PARSER_RECONFIG,
  150. PENDING_PARSER_REINIT
  151. };
  152. // Initializes |stream_parser_|. Also, updates |expected_audio_codecs| and
  153. // |expected_video_codecs|.
  154. void InitializeParser(const std::string& expected_codecs);
  155. // Called by the |stream_parser_| when a new initialization segment is
  156. // encountered.
  157. // Returns true on a successful call. Returns false if an error occurred while
  158. // processing decoder configurations.
  159. bool OnNewConfigs(std::string expected_codecs,
  160. std::unique_ptr<MediaTracks> tracks,
  161. const StreamParser::TextTrackConfigMap& text_configs);
  162. // Called by the |stream_parser_| at the beginning of a new media segment.
  163. void OnNewMediaSegment();
  164. // Called by the |stream_parser_| at the end of a media segment.
  165. void OnEndOfMediaSegment();
  166. // Called by the |stream_parser_| when new buffers have been parsed.
  167. // It processes the new buffers using |frame_processor_|, which includes
  168. // appending the processed frames to associated demuxer streams for each
  169. // frame's track.
  170. // Returns true on a successful call. Returns false if an error occurred while
  171. // processing the buffers.
  172. bool OnNewBuffers(const StreamParser::BufferQueueMap& buffer_queue_map);
  173. // Called when StreamParser encounters encrypted media init data.
  174. void OnEncryptedMediaInitData(EmeInitDataType type,
  175. const std::vector<uint8_t>& init_data);
  176. void OnSourceInitDone(const StreamParser::InitParameters& params);
  177. // Sets memory limits for all demuxer streams.
  178. void SetStreamMemoryLimits();
  179. // Tracks the number of MEDIA_LOGs emitted for segments missing expected audio
  180. // or video blocks. Useful to prevent log spam.
  181. int num_missing_track_logs_ = 0;
  182. // During Append(), if OnNewBuffers() coded frame processing updates the
  183. // timestamp offset then |*timestamp_offset_during_append_| is also updated
  184. // so Append()'s caller can know the new offset. This pointer is only non-NULL
  185. // during the lifetime of an Append() call.
  186. raw_ptr<base::TimeDelta> timestamp_offset_during_append_;
  187. // During Append(), coded frame processing triggered by OnNewBuffers()
  188. // requires these two attributes. These are only valid during the lifetime of
  189. // an Append() call.
  190. base::TimeDelta append_window_start_during_append_;
  191. base::TimeDelta append_window_end_during_append_;
  192. // Keeps track of whether a media segment is being parsed.
  193. bool parsing_media_segment_;
  194. // Valid only while |parsing_media_segment_| is true. These flags enable
  195. // warning when the parsed media segment doesn't have frames for some track.
  196. std::map<StreamParser::TrackId, bool> media_segment_has_data_for_track_;
  197. // The object used to parse appended data.
  198. std::unique_ptr<StreamParser> stream_parser_;
  199. // Note that ChunkDemuxerStreams are created and owned by the parent
  200. // ChunkDemuxer. They are not owned by |this|.
  201. using DemuxerStreamMap = std::map<StreamParser::TrackId, ChunkDemuxerStream*>;
  202. DemuxerStreamMap audio_streams_;
  203. DemuxerStreamMap video_streams_;
  204. DemuxerStreamMap text_streams_;
  205. std::unique_ptr<FrameProcessor> frame_processor_;
  206. const CreateDemuxerStreamCB create_demuxer_stream_cb_;
  207. raw_ptr<MediaLog> media_log_;
  208. StreamParser::InitCB init_cb_;
  209. StreamParser::EncryptedMediaInitDataCB encrypted_media_init_data_cb_;
  210. NewTextTrackCB new_text_track_cb_;
  211. State state_;
  212. // During Append(), OnNewConfigs() will trigger the initialization segment
  213. // received algorithm. Note, the MSE spec explicitly disallows this algorithm
  214. // during an Abort(), since Abort() is allowed only to emit coded frames, and
  215. // only if the parser is PARSING_MEDIA_SEGMENT (not an INIT segment). So we
  216. // also have a flag here that indicates if Append is in progress and we can
  217. // invoke this callback.
  218. Demuxer::MediaTracksUpdatedCB init_segment_received_cb_;
  219. bool append_in_progress_ = false;
  220. bool first_init_segment_received_ = false;
  221. bool encrypted_media_init_data_reported_ = false;
  222. std::vector<AudioCodec> expected_audio_codecs_;
  223. std::vector<VideoCodec> expected_video_codecs_;
  224. };
  225. } // namespace media
  226. #endif // MEDIA_FILTERS_SOURCE_BUFFER_STATE_H_