source_buffer_state.cc 36 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026
  1. // Copyright 2016 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/filters/source_buffer_state.h"
  5. #include <set>
  6. #include "base/callback_helpers.h"
  7. #include "base/command_line.h"
  8. #include "base/strings/string_number_conversions.h"
  9. #include "build/build_config.h"
  10. #include "build/chromeos_buildflags.h"
  11. #include "media/base/media_switches.h"
  12. #include "media/base/media_track.h"
  13. #include "media/base/media_tracks.h"
  14. #include "media/base/mime_util.h"
  15. #include "media/filters/chunk_demuxer.h"
  16. #include "media/filters/frame_processor.h"
  17. #include "media/filters/source_buffer_stream.h"
  18. #include "media/media_buildflags.h"
  19. namespace media {
  20. enum {
  21. // Limits the number of MEDIA_LOG() calls warning the user that a muxed stream
  22. // media segment is missing a block from at least one of the audio or video
  23. // tracks.
  24. kMaxMissingTrackInSegmentLogs = 10,
  25. };
  26. namespace {
  27. base::TimeDelta EndTimestamp(const StreamParser::BufferQueue& queue) {
  28. return queue.back()->timestamp() + queue.back()->duration();
  29. }
  30. // Check the input |text_configs| and |bytestream_ids| and return false if
  31. // duplicate track ids are detected.
  32. bool CheckBytestreamTrackIds(
  33. const MediaTracks& tracks,
  34. const StreamParser::TextTrackConfigMap& text_configs) {
  35. std::set<StreamParser::TrackId> bytestream_ids;
  36. for (const auto& track : tracks.tracks()) {
  37. const StreamParser::TrackId& track_id = track->bytestream_track_id();
  38. if (bytestream_ids.find(track_id) != bytestream_ids.end()) {
  39. return false;
  40. }
  41. bytestream_ids.insert(track_id);
  42. }
  43. for (const auto& text_track : text_configs) {
  44. const StreamParser::TrackId& track_id = text_track.first;
  45. if (bytestream_ids.find(track_id) != bytestream_ids.end()) {
  46. return false;
  47. }
  48. bytestream_ids.insert(track_id);
  49. }
  50. return true;
  51. }
  52. unsigned GetMSEBufferSizeLimitIfExists(base::StringPiece switch_string) {
  53. auto* command_line = base::CommandLine::ForCurrentProcess();
  54. unsigned memory_limit;
  55. if (command_line->HasSwitch(switch_string) &&
  56. base::StringToUint(command_line->GetSwitchValueASCII(switch_string),
  57. &memory_limit)) {
  58. return memory_limit * 1024 * 1024;
  59. }
  60. return 0;
  61. }
  62. } // namespace
  63. // List of time ranges for each SourceBuffer.
  64. // static
  65. Ranges<base::TimeDelta> SourceBufferState::ComputeRangesIntersection(
  66. const RangesList& active_ranges,
  67. bool ended) {
  68. // TODO(servolk): Perhaps this can be removed in favor of blink implementation
  69. // (MediaSource::buffered)? Currently this is only used on Android and for
  70. // updating DemuxerHost's buffered ranges during AppendData() as well as
  71. // SourceBuffer.buffered property implementation.
  72. // Implementation of HTMLMediaElement.buffered algorithm in MSE spec.
  73. // https://dvcs.w3.org/hg/html-media/raw-file/default/media-source/media-source.html#dom-htmlmediaelement.buffered
  74. // Step 1: If activeSourceBuffers.length equals 0 then return an empty
  75. // TimeRanges object and abort these steps.
  76. if (active_ranges.empty())
  77. return Ranges<base::TimeDelta>();
  78. // Step 2: Let active ranges be the ranges returned by buffered for each
  79. // SourceBuffer object in activeSourceBuffers.
  80. // Step 3: Let highest end time be the largest range end time in the active
  81. // ranges.
  82. base::TimeDelta highest_end_time;
  83. for (const auto& range : active_ranges) {
  84. if (!range.size())
  85. continue;
  86. highest_end_time = std::max(highest_end_time, range.end(range.size() - 1));
  87. }
  88. // Step 4: Let intersection ranges equal a TimeRange object containing a
  89. // single range from 0 to highest end time.
  90. Ranges<base::TimeDelta> intersection_ranges;
  91. intersection_ranges.Add(base::TimeDelta(), highest_end_time);
  92. // Step 5: For each SourceBuffer object in activeSourceBuffers run the
  93. // following steps:
  94. for (const auto& range : active_ranges) {
  95. // Step 5.1: Let source ranges equal the ranges returned by the buffered
  96. // attribute on the current SourceBuffer.
  97. Ranges<base::TimeDelta> source_ranges = range;
  98. // Step 5.2: If readyState is "ended", then set the end time on the last
  99. // range in source ranges to highest end time.
  100. if (ended && source_ranges.size()) {
  101. source_ranges.Add(source_ranges.start(source_ranges.size() - 1),
  102. highest_end_time);
  103. }
  104. // Step 5.3: Let new intersection ranges equal the intersection between
  105. // the intersection ranges and the source ranges.
  106. // Step 5.4: Replace the ranges in intersection ranges with the new
  107. // intersection ranges.
  108. intersection_ranges = intersection_ranges.IntersectionWith(source_ranges);
  109. }
  110. return intersection_ranges;
  111. }
  112. SourceBufferState::SourceBufferState(
  113. std::unique_ptr<StreamParser> stream_parser,
  114. std::unique_ptr<FrameProcessor> frame_processor,
  115. CreateDemuxerStreamCB create_demuxer_stream_cb,
  116. MediaLog* media_log)
  117. : timestamp_offset_during_append_(nullptr),
  118. parsing_media_segment_(false),
  119. stream_parser_(stream_parser.release()),
  120. frame_processor_(frame_processor.release()),
  121. create_demuxer_stream_cb_(std::move(create_demuxer_stream_cb)),
  122. media_log_(media_log),
  123. state_(UNINITIALIZED) {
  124. DCHECK(create_demuxer_stream_cb_);
  125. DCHECK(frame_processor_);
  126. }
  127. SourceBufferState::~SourceBufferState() {
  128. Shutdown();
  129. }
  130. void SourceBufferState::Init(
  131. StreamParser::InitCB init_cb,
  132. const std::string& expected_codecs,
  133. const StreamParser::EncryptedMediaInitDataCB& encrypted_media_init_data_cb,
  134. NewTextTrackCB new_text_track_cb) {
  135. DCHECK_EQ(state_, UNINITIALIZED);
  136. init_cb_ = std::move(init_cb);
  137. encrypted_media_init_data_cb_ = encrypted_media_init_data_cb;
  138. new_text_track_cb_ = std::move(new_text_track_cb);
  139. state_ = PENDING_PARSER_CONFIG;
  140. InitializeParser(expected_codecs);
  141. }
  142. void SourceBufferState::ChangeType(
  143. std::unique_ptr<StreamParser> new_stream_parser,
  144. const std::string& new_expected_codecs) {
  145. DCHECK_GE(state_, PENDING_PARSER_CONFIG);
  146. DCHECK_NE(state_, PENDING_PARSER_INIT);
  147. DCHECK(!parsing_media_segment_);
  148. // If this source buffer has already handled an initialization segment, avoid
  149. // running |init_cb_| again later.
  150. if (state_ == PARSER_INITIALIZED)
  151. state_ = PENDING_PARSER_RECONFIG;
  152. stream_parser_ = std::move(new_stream_parser);
  153. InitializeParser(new_expected_codecs);
  154. }
  155. void SourceBufferState::SetSequenceMode(bool sequence_mode) {
  156. DCHECK(!parsing_media_segment_);
  157. frame_processor_->SetSequenceMode(sequence_mode);
  158. }
  159. void SourceBufferState::SetGroupStartTimestampIfInSequenceMode(
  160. base::TimeDelta timestamp_offset) {
  161. DCHECK(!parsing_media_segment_);
  162. frame_processor_->SetGroupStartTimestampIfInSequenceMode(timestamp_offset);
  163. }
  164. void SourceBufferState::SetTracksWatcher(
  165. Demuxer::MediaTracksUpdatedCB tracks_updated_cb) {
  166. DCHECK(!init_segment_received_cb_);
  167. DCHECK(tracks_updated_cb);
  168. init_segment_received_cb_ = std::move(tracks_updated_cb);
  169. }
  170. void SourceBufferState::SetParseWarningCallback(
  171. SourceBufferParseWarningCB parse_warning_cb) {
  172. // Give the callback to |frame_processor_|; none of these warnings are
  173. // currently emitted elsewhere.
  174. frame_processor_->SetParseWarningCallback(std::move(parse_warning_cb));
  175. }
  176. bool SourceBufferState::Append(const uint8_t* data,
  177. size_t length,
  178. base::TimeDelta append_window_start,
  179. base::TimeDelta append_window_end,
  180. base::TimeDelta* timestamp_offset) {
  181. append_in_progress_ = true;
  182. DCHECK(timestamp_offset);
  183. DCHECK(!timestamp_offset_during_append_);
  184. append_window_start_during_append_ = append_window_start;
  185. append_window_end_during_append_ = append_window_end;
  186. timestamp_offset_during_append_ = timestamp_offset;
  187. // TODO(wolenetz): Curry and pass a NewBuffersCB here bound with append window
  188. // and timestamp offset pointer. See http://crbug.com/351454.
  189. bool result = stream_parser_->Parse(data, length);
  190. if (!result) {
  191. MEDIA_LOG(ERROR, media_log_)
  192. << __func__ << ": stream parsing failed. Data size=" << length
  193. << " append_window_start=" << append_window_start.InSecondsF()
  194. << " append_window_end=" << append_window_end.InSecondsF();
  195. }
  196. timestamp_offset_during_append_ = nullptr;
  197. append_in_progress_ = false;
  198. return result;
  199. }
  200. bool SourceBufferState::AppendChunks(
  201. std::unique_ptr<StreamParser::BufferQueue> buffer_queue,
  202. base::TimeDelta append_window_start,
  203. base::TimeDelta append_window_end,
  204. base::TimeDelta* timestamp_offset) {
  205. append_in_progress_ = true;
  206. DCHECK(timestamp_offset);
  207. DCHECK(!timestamp_offset_during_append_);
  208. append_window_start_during_append_ = append_window_start;
  209. append_window_end_during_append_ = append_window_end;
  210. timestamp_offset_during_append_ = timestamp_offset;
  211. // TODO(wolenetz): Curry and pass a NewBuffersCB here bound with append window
  212. // and timestamp offset pointer. See http://crbug.com/351454.
  213. bool result = stream_parser_->ProcessChunks(std::move(buffer_queue));
  214. if (!result) {
  215. MEDIA_LOG(ERROR, media_log_)
  216. << __func__ << ": Processing encoded chunks for buffering failed.";
  217. }
  218. timestamp_offset_during_append_ = nullptr;
  219. append_in_progress_ = false;
  220. return result;
  221. }
  222. void SourceBufferState::ResetParserState(base::TimeDelta append_window_start,
  223. base::TimeDelta append_window_end,
  224. base::TimeDelta* timestamp_offset) {
  225. DCHECK(timestamp_offset);
  226. DCHECK(!timestamp_offset_during_append_);
  227. timestamp_offset_during_append_ = timestamp_offset;
  228. append_window_start_during_append_ = append_window_start;
  229. append_window_end_during_append_ = append_window_end;
  230. stream_parser_->Flush();
  231. timestamp_offset_during_append_ = nullptr;
  232. frame_processor_->Reset();
  233. parsing_media_segment_ = false;
  234. media_segment_has_data_for_track_.clear();
  235. }
  236. void SourceBufferState::Remove(base::TimeDelta start,
  237. base::TimeDelta end,
  238. base::TimeDelta duration) {
  239. for (const auto& it : audio_streams_) {
  240. it.second->Remove(start, end, duration);
  241. }
  242. for (const auto& it : video_streams_) {
  243. it.second->Remove(start, end, duration);
  244. }
  245. for (const auto& it : text_streams_) {
  246. it.second->Remove(start, end, duration);
  247. }
  248. }
  249. bool SourceBufferState::EvictCodedFrames(base::TimeDelta media_time,
  250. size_t newDataSize) {
  251. size_t total_buffered_size = 0;
  252. for (const auto& it : audio_streams_)
  253. total_buffered_size += it.second->GetBufferedSize();
  254. for (const auto& it : video_streams_)
  255. total_buffered_size += it.second->GetBufferedSize();
  256. for (const auto& it : text_streams_)
  257. total_buffered_size += it.second->GetBufferedSize();
  258. DVLOG(3) << __func__ << " media_time=" << media_time.InSecondsF()
  259. << " newDataSize=" << newDataSize
  260. << " total_buffered_size=" << total_buffered_size;
  261. if (total_buffered_size == 0)
  262. return true;
  263. bool success = true;
  264. for (const auto& it : audio_streams_) {
  265. uint64_t curr_size = it.second->GetBufferedSize();
  266. if (curr_size == 0)
  267. continue;
  268. uint64_t estimated_new_size = newDataSize * curr_size / total_buffered_size;
  269. DCHECK_LE(estimated_new_size, SIZE_MAX);
  270. success &= it.second->EvictCodedFrames(
  271. media_time, static_cast<size_t>(estimated_new_size));
  272. }
  273. for (const auto& it : video_streams_) {
  274. uint64_t curr_size = it.second->GetBufferedSize();
  275. if (curr_size == 0)
  276. continue;
  277. uint64_t estimated_new_size = newDataSize * curr_size / total_buffered_size;
  278. DCHECK_LE(estimated_new_size, SIZE_MAX);
  279. success &= it.second->EvictCodedFrames(
  280. media_time, static_cast<size_t>(estimated_new_size));
  281. }
  282. for (const auto& it : text_streams_) {
  283. uint64_t curr_size = it.second->GetBufferedSize();
  284. if (curr_size == 0)
  285. continue;
  286. uint64_t estimated_new_size = newDataSize * curr_size / total_buffered_size;
  287. DCHECK_LE(estimated_new_size, SIZE_MAX);
  288. success &= it.second->EvictCodedFrames(
  289. media_time, static_cast<size_t>(estimated_new_size));
  290. }
  291. DVLOG(3) << __func__ << " success=" << success;
  292. return success;
  293. }
  294. void SourceBufferState::OnMemoryPressure(
  295. base::TimeDelta media_time,
  296. base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level,
  297. bool force_instant_gc) {
  298. // TODO(sebmarchand): Check if MEMORY_PRESSURE_LEVEL_MODERATE should also be
  299. // ignored.
  300. if (memory_pressure_level ==
  301. base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE) {
  302. return;
  303. }
  304. // Notify video streams about memory pressure first, since video typically
  305. // takes up the most memory and that's where we can expect most savings.
  306. for (const auto& it : video_streams_) {
  307. it.second->OnMemoryPressure(media_time, memory_pressure_level,
  308. force_instant_gc);
  309. }
  310. for (const auto& it : audio_streams_) {
  311. it.second->OnMemoryPressure(media_time, memory_pressure_level,
  312. force_instant_gc);
  313. }
  314. for (const auto& it : text_streams_) {
  315. it.second->OnMemoryPressure(media_time, memory_pressure_level,
  316. force_instant_gc);
  317. }
  318. }
  319. Ranges<base::TimeDelta> SourceBufferState::GetBufferedRanges(
  320. base::TimeDelta duration,
  321. bool ended) const {
  322. RangesList ranges_list;
  323. for (const auto& it : audio_streams_)
  324. ranges_list.push_back(it.second->GetBufferedRanges(duration));
  325. for (const auto& it : video_streams_)
  326. ranges_list.push_back(it.second->GetBufferedRanges(duration));
  327. for (const auto& it : text_streams_)
  328. ranges_list.push_back(it.second->GetBufferedRanges(duration));
  329. return ComputeRangesIntersection(ranges_list, ended);
  330. }
  331. base::TimeDelta SourceBufferState::GetHighestPresentationTimestamp() const {
  332. base::TimeDelta max_pts;
  333. for (const auto& it : audio_streams_) {
  334. max_pts = std::max(max_pts, it.second->GetHighestPresentationTimestamp());
  335. }
  336. for (const auto& it : video_streams_) {
  337. max_pts = std::max(max_pts, it.second->GetHighestPresentationTimestamp());
  338. }
  339. for (const auto& it : text_streams_) {
  340. max_pts = std::max(max_pts, it.second->GetHighestPresentationTimestamp());
  341. }
  342. return max_pts;
  343. }
  344. base::TimeDelta SourceBufferState::GetMaxBufferedDuration() const {
  345. base::TimeDelta max_duration;
  346. for (const auto& it : audio_streams_) {
  347. max_duration = std::max(max_duration, it.second->GetBufferedDuration());
  348. }
  349. for (const auto& it : video_streams_) {
  350. max_duration = std::max(max_duration, it.second->GetBufferedDuration());
  351. }
  352. for (const auto& it : text_streams_) {
  353. max_duration = std::max(max_duration, it.second->GetBufferedDuration());
  354. }
  355. return max_duration;
  356. }
  357. void SourceBufferState::StartReturningData() {
  358. for (const auto& it : audio_streams_) {
  359. it.second->StartReturningData();
  360. }
  361. for (const auto& it : video_streams_) {
  362. it.second->StartReturningData();
  363. }
  364. for (const auto& it : text_streams_) {
  365. it.second->StartReturningData();
  366. }
  367. }
  368. void SourceBufferState::AbortReads() {
  369. for (const auto& it : audio_streams_) {
  370. it.second->AbortReads();
  371. }
  372. for (const auto& it : video_streams_) {
  373. it.second->AbortReads();
  374. }
  375. for (const auto& it : text_streams_) {
  376. it.second->AbortReads();
  377. }
  378. }
  379. void SourceBufferState::Seek(base::TimeDelta seek_time) {
  380. for (const auto& it : audio_streams_) {
  381. it.second->Seek(seek_time);
  382. }
  383. for (const auto& it : video_streams_) {
  384. it.second->Seek(seek_time);
  385. }
  386. for (const auto& it : text_streams_) {
  387. it.second->Seek(seek_time);
  388. }
  389. }
  390. void SourceBufferState::CompletePendingReadIfPossible() {
  391. for (const auto& it : audio_streams_) {
  392. it.second->CompletePendingReadIfPossible();
  393. }
  394. for (const auto& it : video_streams_) {
  395. it.second->CompletePendingReadIfPossible();
  396. }
  397. for (const auto& it : text_streams_) {
  398. it.second->CompletePendingReadIfPossible();
  399. }
  400. }
  401. void SourceBufferState::OnSetDuration(base::TimeDelta duration) {
  402. for (const auto& it : audio_streams_) {
  403. it.second->OnSetDuration(duration);
  404. }
  405. for (const auto& it : video_streams_) {
  406. it.second->OnSetDuration(duration);
  407. }
  408. for (const auto& it : text_streams_) {
  409. it.second->OnSetDuration(duration);
  410. }
  411. }
  412. void SourceBufferState::MarkEndOfStream() {
  413. for (const auto& it : audio_streams_) {
  414. it.second->MarkEndOfStream();
  415. }
  416. for (const auto& it : video_streams_) {
  417. it.second->MarkEndOfStream();
  418. }
  419. for (const auto& it : text_streams_) {
  420. it.second->MarkEndOfStream();
  421. }
  422. }
  423. void SourceBufferState::UnmarkEndOfStream() {
  424. for (const auto& it : audio_streams_) {
  425. it.second->UnmarkEndOfStream();
  426. }
  427. for (const auto& it : video_streams_) {
  428. it.second->UnmarkEndOfStream();
  429. }
  430. for (const auto& it : text_streams_) {
  431. it.second->UnmarkEndOfStream();
  432. }
  433. }
  434. void SourceBufferState::Shutdown() {
  435. for (const auto& it : audio_streams_) {
  436. it.second->Shutdown();
  437. }
  438. for (const auto& it : video_streams_) {
  439. it.second->Shutdown();
  440. }
  441. for (const auto& it : text_streams_) {
  442. it.second->Shutdown();
  443. }
  444. }
  445. void SourceBufferState::SetMemoryLimits(DemuxerStream::Type type,
  446. size_t memory_limit) {
  447. switch (type) {
  448. case DemuxerStream::AUDIO:
  449. for (const auto& it : audio_streams_) {
  450. it.second->SetStreamMemoryLimit(memory_limit);
  451. }
  452. break;
  453. case DemuxerStream::VIDEO:
  454. for (const auto& it : video_streams_) {
  455. it.second->SetStreamMemoryLimit(memory_limit);
  456. }
  457. break;
  458. case DemuxerStream::TEXT:
  459. for (const auto& it : text_streams_) {
  460. it.second->SetStreamMemoryLimit(memory_limit);
  461. }
  462. break;
  463. case DemuxerStream::UNKNOWN:
  464. NOTREACHED();
  465. break;
  466. }
  467. }
  468. bool SourceBufferState::IsSeekWaitingForData() const {
  469. for (const auto& it : audio_streams_) {
  470. if (it.second->IsSeekWaitingForData())
  471. return true;
  472. }
  473. for (const auto& it : video_streams_) {
  474. if (it.second->IsSeekWaitingForData())
  475. return true;
  476. }
  477. // NOTE: We are intentionally not checking the text tracks
  478. // because text tracks are discontinuous and may not have data
  479. // for the seek position. This is ok and playback should not be
  480. // stalled because we don't have cues. If cues, with timestamps after
  481. // the seek time, eventually arrive they will be delivered properly
  482. // in response to ChunkDemuxerStream::Read() calls.
  483. return false;
  484. }
  485. void SourceBufferState::InitializeParser(const std::string& expected_codecs) {
  486. expected_audio_codecs_.clear();
  487. expected_video_codecs_.clear();
  488. std::vector<std::string> expected_codecs_parsed;
  489. SplitCodecs(expected_codecs, &expected_codecs_parsed);
  490. std::vector<AudioCodec> expected_acodecs;
  491. std::vector<VideoCodec> expected_vcodecs;
  492. for (const auto& codec_id : expected_codecs_parsed) {
  493. AudioCodec acodec = StringToAudioCodec(codec_id);
  494. if (acodec != AudioCodec::kUnknown) {
  495. expected_audio_codecs_.push_back(acodec);
  496. continue;
  497. }
  498. VideoCodec vcodec = StringToVideoCodec(codec_id);
  499. if (vcodec != VideoCodec::kUnknown) {
  500. expected_video_codecs_.push_back(vcodec);
  501. continue;
  502. }
  503. MEDIA_LOG(INFO, media_log_) << "Unrecognized media codec: " << codec_id;
  504. }
  505. stream_parser_->Init(
  506. base::BindOnce(&SourceBufferState::OnSourceInitDone,
  507. base::Unretained(this)),
  508. base::BindRepeating(&SourceBufferState::OnNewConfigs,
  509. base::Unretained(this), expected_codecs),
  510. base::BindRepeating(&SourceBufferState::OnNewBuffers,
  511. base::Unretained(this)),
  512. !new_text_track_cb_,
  513. base::BindRepeating(&SourceBufferState::OnEncryptedMediaInitData,
  514. base::Unretained(this)),
  515. base::BindRepeating(&SourceBufferState::OnNewMediaSegment,
  516. base::Unretained(this)),
  517. base::BindRepeating(&SourceBufferState::OnEndOfMediaSegment,
  518. base::Unretained(this)),
  519. media_log_);
  520. }
  521. bool SourceBufferState::OnNewConfigs(
  522. std::string expected_codecs,
  523. std::unique_ptr<MediaTracks> tracks,
  524. const StreamParser::TextTrackConfigMap& text_configs) {
  525. DCHECK(tracks.get());
  526. DVLOG(1) << __func__ << " expected_codecs=" << expected_codecs
  527. << " tracks=" << tracks->tracks().size();
  528. DCHECK_GE(state_, PENDING_PARSER_CONFIG);
  529. // Check that there is no clashing bytestream track ids.
  530. if (!CheckBytestreamTrackIds(*tracks, text_configs)) {
  531. MEDIA_LOG(ERROR, media_log_) << "Duplicate bytestream track ids detected";
  532. for (const auto& track : tracks->tracks()) {
  533. const StreamParser::TrackId& track_id = track->bytestream_track_id();
  534. MEDIA_LOG(DEBUG, media_log_) << TrackTypeToStr(track->type()) << " track "
  535. << " bytestream track id=" << track_id;
  536. }
  537. return false;
  538. }
  539. // MSE spec allows new configs to be emitted only during Append, but not
  540. // during Flush or parser reset operations.
  541. CHECK(append_in_progress_);
  542. bool success = true;
  543. // TODO(wolenetz): Update codec string strictness, if necessary, once spec
  544. // issue https://github.com/w3c/media-source/issues/161 is resolved.
  545. std::vector<AudioCodec> expected_acodecs = expected_audio_codecs_;
  546. std::vector<VideoCodec> expected_vcodecs = expected_video_codecs_;
  547. // TODO(wolenetz): Once codec strictness is relaxed, we can change
  548. // |allow_codec_changes| to always be true. Until then, we only allow codec
  549. // changes on explicit ChangeType().
  550. const bool allow_codec_changes = state_ == PENDING_PARSER_RECONFIG;
  551. FrameProcessor::TrackIdChanges track_id_changes;
  552. for (const auto& track : tracks->tracks()) {
  553. const auto& track_id = track->bytestream_track_id();
  554. if (track->type() == MediaTrack::Audio) {
  555. AudioDecoderConfig audio_config = tracks->getAudioConfig(track_id);
  556. DVLOG(1) << "Audio track_id=" << track_id
  557. << " config: " << audio_config.AsHumanReadableString();
  558. DCHECK(audio_config.IsValidConfig());
  559. const auto& it = std::find(expected_acodecs.begin(),
  560. expected_acodecs.end(), audio_config.codec());
  561. if (it == expected_acodecs.end()) {
  562. MEDIA_LOG(ERROR, media_log_) << "Audio stream codec "
  563. << GetCodecName(audio_config.codec())
  564. << " doesn't match SourceBuffer codecs.";
  565. return false;
  566. }
  567. expected_acodecs.erase(it);
  568. ChunkDemuxerStream* stream = nullptr;
  569. if (!first_init_segment_received_) {
  570. DCHECK(audio_streams_.find(track_id) == audio_streams_.end());
  571. stream = create_demuxer_stream_cb_.Run(DemuxerStream::AUDIO);
  572. if (!stream || !frame_processor_->AddTrack(track_id, stream)) {
  573. MEDIA_LOG(ERROR, media_log_) << "Failed to create audio stream.";
  574. return false;
  575. }
  576. audio_streams_[track_id] = stream;
  577. media_log_->SetProperty<MediaLogProperty::kAudioTracks>(
  578. std::vector<AudioDecoderConfig>{audio_config});
  579. } else {
  580. if (audio_streams_.size() > 1) {
  581. auto stream_it = audio_streams_.find(track_id);
  582. if (stream_it != audio_streams_.end())
  583. stream = stream_it->second;
  584. } else {
  585. // If there is only one audio track then bytestream id might change in
  586. // a new init segment. So update our state and notify frame processor.
  587. const auto& stream_it = audio_streams_.begin();
  588. if (stream_it != audio_streams_.end()) {
  589. stream = stream_it->second;
  590. if (stream_it->first != track_id) {
  591. track_id_changes[stream_it->first] = track_id;
  592. audio_streams_[track_id] = stream;
  593. audio_streams_.erase(stream_it->first);
  594. }
  595. }
  596. }
  597. if (!stream) {
  598. MEDIA_LOG(ERROR, media_log_) << "Got unexpected audio track"
  599. << " track_id=" << track_id;
  600. return false;
  601. }
  602. }
  603. track->set_id(stream->media_track_id());
  604. frame_processor_->OnPossibleAudioConfigUpdate(audio_config);
  605. success &= stream->UpdateAudioConfig(audio_config, allow_codec_changes,
  606. media_log_);
  607. } else if (track->type() == MediaTrack::Video) {
  608. VideoDecoderConfig video_config = tracks->getVideoConfig(track_id);
  609. DVLOG(1) << "Video track_id=" << track_id
  610. << " config: " << video_config.AsHumanReadableString();
  611. DCHECK(video_config.IsValidConfig());
  612. #if BUILDFLAG(ENABLE_PLATFORM_ENCRYPTED_DOLBY_VISION)
  613. // Only encrypted Dolby Vision (DV) is supported, so require the config
  614. // to be for an encrypted track.
  615. if (video_config.codec() == VideoCodec::kDolbyVision &&
  616. !video_config.is_encrypted()) {
  617. MEDIA_LOG(ERROR, media_log_)
  618. << "MSE playback of DolbyVision is only supported via platform "
  619. "decryptor, but the provided DV track is not encrypted.";
  620. return false;
  621. }
  622. #endif // BUILDFLAG(ENABLE_PLATFORM_ENCRYPTED_DOLBY_VISION)
  623. const auto& it = std::find(expected_vcodecs.begin(),
  624. expected_vcodecs.end(), video_config.codec());
  625. if (it == expected_vcodecs.end()) {
  626. MEDIA_LOG(ERROR, media_log_) << "Video stream codec "
  627. << GetCodecName(video_config.codec())
  628. << " doesn't match SourceBuffer codecs.";
  629. return false;
  630. }
  631. expected_vcodecs.erase(it);
  632. ChunkDemuxerStream* stream = nullptr;
  633. if (!first_init_segment_received_) {
  634. DCHECK(video_streams_.find(track_id) == video_streams_.end());
  635. stream = create_demuxer_stream_cb_.Run(DemuxerStream::VIDEO);
  636. if (!stream || !frame_processor_->AddTrack(track_id, stream)) {
  637. MEDIA_LOG(ERROR, media_log_) << "Failed to create video stream.";
  638. return false;
  639. }
  640. video_streams_[track_id] = stream;
  641. media_log_->SetProperty<MediaLogProperty::kVideoTracks>(
  642. std::vector<VideoDecoderConfig>{video_config});
  643. } else {
  644. if (video_streams_.size() > 1) {
  645. auto stream_it = video_streams_.find(track_id);
  646. if (stream_it != video_streams_.end())
  647. stream = stream_it->second;
  648. } else {
  649. // If there is only one video track then bytestream id might change in
  650. // a new init segment. So update our state and notify frame processor.
  651. const auto& stream_it = video_streams_.begin();
  652. if (stream_it != video_streams_.end()) {
  653. stream = stream_it->second;
  654. if (stream_it->first != track_id) {
  655. track_id_changes[stream_it->first] = track_id;
  656. video_streams_[track_id] = stream;
  657. video_streams_.erase(stream_it->first);
  658. }
  659. }
  660. }
  661. if (!stream) {
  662. MEDIA_LOG(ERROR, media_log_) << "Got unexpected video track"
  663. << " track_id=" << track_id;
  664. return false;
  665. }
  666. }
  667. track->set_id(stream->media_track_id());
  668. success &= stream->UpdateVideoConfig(video_config, allow_codec_changes,
  669. media_log_);
  670. } else {
  671. MEDIA_LOG(ERROR, media_log_) << "Error: unsupported media track type "
  672. << track->type();
  673. return false;
  674. }
  675. }
  676. if (!expected_acodecs.empty() || !expected_vcodecs.empty()) {
  677. for (const auto& acodec : expected_acodecs) {
  678. MEDIA_LOG(ERROR, media_log_) << "Initialization segment misses expected "
  679. << GetCodecName(acodec) << " track.";
  680. }
  681. for (const auto& vcodec : expected_vcodecs) {
  682. MEDIA_LOG(ERROR, media_log_) << "Initialization segment misses expected "
  683. << GetCodecName(vcodec) << " track.";
  684. }
  685. return false;
  686. }
  687. if (text_streams_.empty()) {
  688. for (auto itr = text_configs.begin(); itr != text_configs.end(); ++itr) {
  689. ChunkDemuxerStream* const text_stream =
  690. create_demuxer_stream_cb_.Run(DemuxerStream::TEXT);
  691. if (!frame_processor_->AddTrack(itr->first, text_stream)) {
  692. success &= false;
  693. MEDIA_LOG(ERROR, media_log_) << "Failed to add text track ID "
  694. << itr->first << " to frame processor.";
  695. break;
  696. }
  697. text_stream->UpdateTextConfig(itr->second, media_log_);
  698. text_streams_[itr->first] = text_stream;
  699. new_text_track_cb_.Run(text_stream, itr->second);
  700. }
  701. } else {
  702. const size_t text_count = text_streams_.size();
  703. if (text_configs.size() != text_count) {
  704. success &= false;
  705. MEDIA_LOG(ERROR, media_log_)
  706. << "The number of text track configs changed.";
  707. } else if (text_count == 1) {
  708. auto config_itr = text_configs.begin();
  709. auto stream_itr = text_streams_.begin();
  710. ChunkDemuxerStream* text_stream = stream_itr->second;
  711. TextTrackConfig old_config = text_stream->text_track_config();
  712. TextTrackConfig new_config(
  713. config_itr->second.kind(), config_itr->second.label(),
  714. config_itr->second.language(), old_config.id());
  715. if (!new_config.Matches(old_config)) {
  716. success &= false;
  717. MEDIA_LOG(ERROR, media_log_)
  718. << "New text track config does not match old one.";
  719. } else {
  720. StreamParser::TrackId old_id = stream_itr->first;
  721. StreamParser::TrackId new_id = config_itr->first;
  722. if (new_id != old_id) {
  723. track_id_changes[old_id] = new_id;
  724. text_streams_.erase(old_id);
  725. text_streams_[new_id] = text_stream;
  726. }
  727. }
  728. } else {
  729. for (auto config_itr = text_configs.begin();
  730. config_itr != text_configs.end(); ++config_itr) {
  731. auto stream_itr = text_streams_.find(config_itr->first);
  732. if (stream_itr == text_streams_.end()) {
  733. success &= false;
  734. MEDIA_LOG(ERROR, media_log_)
  735. << "Unexpected text track configuration for track ID "
  736. << config_itr->first;
  737. break;
  738. }
  739. const TextTrackConfig& new_config = config_itr->second;
  740. ChunkDemuxerStream* stream = stream_itr->second;
  741. TextTrackConfig old_config = stream->text_track_config();
  742. if (!new_config.Matches(old_config)) {
  743. success &= false;
  744. MEDIA_LOG(ERROR, media_log_) << "New text track config for track ID "
  745. << config_itr->first
  746. << " does not match old one.";
  747. break;
  748. }
  749. }
  750. }
  751. }
  752. if (audio_streams_.empty() && video_streams_.empty()) {
  753. DVLOG(1) << __func__ << ": couldn't find a valid audio or video stream";
  754. return false;
  755. }
  756. if (!frame_processor_->UpdateTrackIds(track_id_changes)) {
  757. DVLOG(1) << __func__ << ": failed to remap track ids in frame processor";
  758. return false;
  759. }
  760. frame_processor_->SetAllTrackBuffersNeedRandomAccessPoint();
  761. if (!first_init_segment_received_) {
  762. first_init_segment_received_ = true;
  763. SetStreamMemoryLimits();
  764. }
  765. DVLOG(1) << "OnNewConfigs() : " << (success ? "success" : "failed");
  766. if (success) {
  767. if (state_ == PENDING_PARSER_CONFIG)
  768. state_ = PENDING_PARSER_INIT;
  769. if (state_ == PENDING_PARSER_RECONFIG)
  770. state_ = PENDING_PARSER_REINIT;
  771. DCHECK(init_segment_received_cb_);
  772. init_segment_received_cb_.Run(std::move(tracks));
  773. }
  774. return success;
  775. }
  776. void SourceBufferState::SetStreamMemoryLimits() {
  777. size_t audio_buf_size_limit =
  778. GetMSEBufferSizeLimitIfExists(switches::kMSEAudioBufferSizeLimitMb);
  779. if (audio_buf_size_limit) {
  780. MEDIA_LOG(INFO, media_log_)
  781. << "Custom audio per-track SourceBuffer size limit="
  782. << audio_buf_size_limit;
  783. for (const auto& it : audio_streams_)
  784. it.second->SetStreamMemoryLimit(audio_buf_size_limit);
  785. }
  786. size_t video_buf_size_limit =
  787. GetMSEBufferSizeLimitIfExists(switches::kMSEVideoBufferSizeLimitMb);
  788. if (video_buf_size_limit) {
  789. MEDIA_LOG(INFO, media_log_)
  790. << "Custom video per-track SourceBuffer size limit="
  791. << video_buf_size_limit;
  792. for (const auto& it : video_streams_)
  793. it.second->SetStreamMemoryLimit(video_buf_size_limit);
  794. }
  795. }
  796. void SourceBufferState::OnNewMediaSegment() {
  797. DVLOG(2) << "OnNewMediaSegment()";
  798. DCHECK_EQ(state_, PARSER_INITIALIZED);
  799. parsing_media_segment_ = true;
  800. media_segment_has_data_for_track_.clear();
  801. }
  802. void SourceBufferState::OnEndOfMediaSegment() {
  803. DVLOG(2) << "OnEndOfMediaSegment()";
  804. DCHECK_EQ(state_, PARSER_INITIALIZED);
  805. parsing_media_segment_ = false;
  806. for (const auto& it : audio_streams_) {
  807. if (!media_segment_has_data_for_track_[it.first]) {
  808. LIMITED_MEDIA_LOG(DEBUG, media_log_, num_missing_track_logs_,
  809. kMaxMissingTrackInSegmentLogs)
  810. << "Media segment did not contain any coded frames for track "
  811. << it.first << ", mismatching initialization segment. Therefore, MSE"
  812. " coded frame processing may not interoperably detect"
  813. " discontinuities in appended media.";
  814. }
  815. }
  816. for (const auto& it : video_streams_) {
  817. if (!media_segment_has_data_for_track_[it.first]) {
  818. LIMITED_MEDIA_LOG(DEBUG, media_log_, num_missing_track_logs_,
  819. kMaxMissingTrackInSegmentLogs)
  820. << "Media segment did not contain any coded frames for track "
  821. << it.first << ", mismatching initialization segment. Therefore, MSE"
  822. " coded frame processing may not interoperably detect"
  823. " discontinuities in appended media.";
  824. }
  825. }
  826. }
  827. bool SourceBufferState::OnNewBuffers(
  828. const StreamParser::BufferQueueMap& buffer_queue_map) {
  829. DVLOG(2) << __func__ << " buffer_queues=" << buffer_queue_map.size();
  830. DCHECK_EQ(state_, PARSER_INITIALIZED);
  831. DCHECK(timestamp_offset_during_append_);
  832. DCHECK(parsing_media_segment_);
  833. for (const auto& [track_id, buffer_queue] : buffer_queue_map) {
  834. DCHECK(!buffer_queue.empty());
  835. media_segment_has_data_for_track_[track_id] = true;
  836. }
  837. const base::TimeDelta timestamp_offset_before_processing =
  838. *timestamp_offset_during_append_;
  839. // Calculate the new timestamp offset for audio/video tracks if the stream
  840. // parser corresponds to MSE MIME type with 'Generate Timestamps Flag' set
  841. // true.
  842. base::TimeDelta predicted_timestamp_offset =
  843. timestamp_offset_before_processing;
  844. if (generate_timestamps_flag()) {
  845. base::TimeDelta min_end_timestamp = kNoTimestamp;
  846. for (const auto& [track_id, buffer_queue] : buffer_queue_map) {
  847. DCHECK(!buffer_queue.empty());
  848. if (min_end_timestamp == kNoTimestamp ||
  849. EndTimestamp(buffer_queue) < min_end_timestamp) {
  850. min_end_timestamp = EndTimestamp(buffer_queue);
  851. DCHECK_NE(kNoTimestamp, min_end_timestamp);
  852. }
  853. }
  854. if (min_end_timestamp != kNoTimestamp)
  855. predicted_timestamp_offset += min_end_timestamp;
  856. }
  857. if (!frame_processor_->ProcessFrames(
  858. buffer_queue_map, append_window_start_during_append_,
  859. append_window_end_during_append_, timestamp_offset_during_append_)) {
  860. return false;
  861. }
  862. // Only update the timestamp offset if the frame processor hasn't already.
  863. if (generate_timestamps_flag() &&
  864. timestamp_offset_before_processing == *timestamp_offset_during_append_) {
  865. // TODO(wolenetz): This prediction assumes the last frame in each track
  866. // isn't dropped by append window trimming. See https://crbug.com/850316.
  867. *timestamp_offset_during_append_ = predicted_timestamp_offset;
  868. }
  869. return true;
  870. }
  871. void SourceBufferState::OnEncryptedMediaInitData(
  872. EmeInitDataType type,
  873. const std::vector<uint8_t>& init_data) {
  874. encrypted_media_init_data_reported_ = true;
  875. encrypted_media_init_data_cb_.Run(type, init_data);
  876. }
  877. void SourceBufferState::OnSourceInitDone(
  878. const StreamParser::InitParameters& params) {
  879. // We've either yet-to-run |init_cb_| if pending init, or we've previously
  880. // run it if pending reinit.
  881. DCHECK((init_cb_ && state_ == PENDING_PARSER_INIT) ||
  882. (!init_cb_ && state_ == PENDING_PARSER_REINIT));
  883. State old_state = state_;
  884. state_ = PARSER_INITIALIZED;
  885. if (old_state == PENDING_PARSER_INIT)
  886. std::move(init_cb_).Run(params);
  887. }
  888. } // namespace media