source_buffer_stream.cc 79 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/filters/source_buffer_stream.h"
  5. #include <algorithm>
  6. #include <map>
  7. #include <memory>
  8. #include <sstream>
  9. #include <string>
  10. #include "base/bind.h"
  11. #include "base/logging.h"
  12. #include "base/metrics/histogram_macros.h"
  13. #include "base/trace_event/trace_event.h"
  14. #include "media/base/demuxer_memory_limit.h"
  15. #include "media/base/media_switches.h"
  16. #include "media/base/stream_parser_buffer.h"
  17. #include "media/base/timestamp_constants.h"
  18. namespace media {
  19. namespace {
  20. // The minimum interbuffer decode timestamp delta (or buffer duration) for use
  21. // in fudge room for range membership, adjacency and coalescing.
  22. const int kMinimumInterbufferDistanceInMs = 1;
  23. // Limit the number of MEDIA_LOG() logs for track buffer time gaps.
  24. const int kMaxTrackBufferGapWarningLogs = 20;
  25. // Limit the number of MEDIA_LOG() logs for MSE GC algorithm warnings.
  26. const int kMaxGarbageCollectAlgorithmWarningLogs = 20;
  27. // Limit the number of MEDIA_LOG() logs for splice overlap trimming.
  28. const int kMaxAudioSpliceLogs = 20;
  29. // Helper method that returns true if |ranges| is sorted in increasing order,
  30. // false otherwise.
  31. bool IsRangeListSorted(const SourceBufferStream::RangeList& ranges) {
  32. base::TimeDelta prev = kNoTimestamp;
  33. for (const auto& range_ptr : ranges) {
  34. if (prev != kNoTimestamp && prev >= range_ptr->GetStartTimestamp())
  35. return false;
  36. prev = range_ptr->GetBufferedEndTimestamp();
  37. }
  38. return true;
  39. }
  40. // Returns an estimate of how far from the beginning or end of a range a buffer
  41. // can be to still be considered in the range, given the |approximate_duration|
  42. // of a buffer in the stream.
  43. // TODO(wolenetz): Once all stream parsers emit accurate frame durations, use
  44. // logic like FrameProcessor (2*last_frame_duration + last_decode_timestamp)
  45. // instead of an overall maximum interbuffer delta for range discontinuity
  46. // detection.
  47. // See http://crbug.com/351489 and http://crbug.com/351166.
  48. base::TimeDelta ComputeFudgeRoom(base::TimeDelta approximate_duration) {
  49. // Because we do not know exactly when is the next timestamp, any buffer
  50. // that starts within 2x the approximate duration of a buffer is considered
  51. // within this range.
  52. return 2 * approximate_duration;
  53. }
  54. // The amount of time the beginning of the buffered data can differ from the
  55. // start time in order to still be considered the start of stream.
  56. base::TimeDelta kSeekToStartFudgeRoom() {
  57. return base::Milliseconds(1000);
  58. }
  59. // Helper method for logging.
  60. std::string StatusToString(const SourceBufferStreamStatus& status) {
  61. switch (status) {
  62. case SourceBufferStreamStatus::kSuccess:
  63. return "kSuccess";
  64. case SourceBufferStreamStatus::kNeedBuffer:
  65. return "kNeedBuffer";
  66. case SourceBufferStreamStatus::kConfigChange:
  67. return "kConfigChange";
  68. case SourceBufferStreamStatus::kEndOfStream:
  69. return "kEndOfStream";
  70. }
  71. NOTREACHED();
  72. return "";
  73. }
  74. // Helper method for logging, converts a range into a readable string.
  75. std::string RangeToString(const SourceBufferRange& range) {
  76. if (range.size_in_bytes() == 0) {
  77. return "[]";
  78. }
  79. std::stringstream ss;
  80. ss << "[" << range.GetStartTimestamp().InMicroseconds() << "us;"
  81. << range.GetEndTimestamp().InMicroseconds() << "us("
  82. << range.GetBufferedEndTimestamp().InMicroseconds() << "us)]";
  83. return ss.str();
  84. }
  85. // Helper method for logging, converts a set of ranges into a readable string.
  86. std::string RangesToString(const SourceBufferStream::RangeList& ranges) {
  87. if (ranges.empty())
  88. return "<EMPTY>";
  89. std::stringstream ss;
  90. for (const auto& range_ptr : ranges) {
  91. if (range_ptr != ranges.front())
  92. ss << " ";
  93. ss << RangeToString(*range_ptr);
  94. }
  95. return ss.str();
  96. }
  97. std::string BufferQueueBuffersToLogString(
  98. const SourceBufferStream::BufferQueue& buffers) {
  99. std::stringstream result;
  100. result << "Buffers:\n";
  101. for (const auto& buf : buffers) {
  102. result << "\tdts=" << buf->GetDecodeTimestamp().InMicroseconds() << " "
  103. << buf->AsHumanReadableString()
  104. << ", is_duration_estimated=" << buf->is_duration_estimated()
  105. << "\n";
  106. }
  107. return result.str();
  108. }
  109. std::string BufferQueueMetadataToLogString(
  110. const SourceBufferStream::BufferQueue& buffers) {
  111. std::stringstream result;
  112. base::TimeDelta pts_interval_start;
  113. base::TimeDelta pts_interval_end;
  114. SourceBufferStream::GetTimestampInterval(buffers, &pts_interval_start,
  115. &pts_interval_end);
  116. result << "dts=[" << buffers.front()->GetDecodeTimestamp().InMicroseconds()
  117. << "us;" << buffers.back()->GetDecodeTimestamp().InMicroseconds()
  118. << "us(last frame dur=" << buffers.back()->duration().InMicroseconds()
  119. << "us)], pts interval=[" << pts_interval_start.InMicroseconds()
  120. << "us," << pts_interval_end.InMicroseconds() << "us)";
  121. return result.str();
  122. }
  123. SourceBufferRange::GapPolicy TypeToGapPolicy(SourceBufferStreamType type) {
  124. switch (type) {
  125. case SourceBufferStreamType::kAudio:
  126. case SourceBufferStreamType::kVideo:
  127. return SourceBufferRange::NO_GAPS_ALLOWED;
  128. case SourceBufferStreamType::kText:
  129. return SourceBufferRange::ALLOW_GAPS;
  130. }
  131. NOTREACHED();
  132. return SourceBufferRange::NO_GAPS_ALLOWED;
  133. }
  134. } // namespace
  135. SourceBufferStream::SourceBufferStream(const AudioDecoderConfig& audio_config,
  136. MediaLog* media_log)
  137. : media_log_(media_log),
  138. seek_buffer_timestamp_(kNoTimestamp),
  139. coded_frame_group_start_pts_(kNoTimestamp),
  140. range_for_next_append_(ranges_.end()),
  141. highest_output_buffer_timestamp_(kNoTimestamp),
  142. max_interbuffer_distance_(
  143. base::Milliseconds(kMinimumInterbufferDistanceInMs)),
  144. memory_limit_(GetDemuxerStreamAudioMemoryLimit(&audio_config)) {
  145. DCHECK(audio_config.IsValidConfig());
  146. audio_configs_.push_back(audio_config);
  147. DVLOG(2) << __func__ << ": audio_buffer_size= " << memory_limit_;
  148. }
  149. SourceBufferStream::SourceBufferStream(const VideoDecoderConfig& video_config,
  150. MediaLog* media_log)
  151. : media_log_(media_log),
  152. seek_buffer_timestamp_(kNoTimestamp),
  153. coded_frame_group_start_pts_(kNoTimestamp),
  154. range_for_next_append_(ranges_.end()),
  155. highest_output_buffer_timestamp_(kNoTimestamp),
  156. max_interbuffer_distance_(
  157. base::Milliseconds(kMinimumInterbufferDistanceInMs)),
  158. memory_limit_(
  159. GetDemuxerStreamVideoMemoryLimit(Demuxer::DemuxerTypes::kChunkDemuxer,
  160. &video_config)) {
  161. DCHECK(video_config.IsValidConfig());
  162. video_configs_.push_back(video_config);
  163. DVLOG(2) << __func__ << ": video_buffer_size= " << memory_limit_;
  164. }
  165. SourceBufferStream::SourceBufferStream(const TextTrackConfig& text_config,
  166. MediaLog* media_log)
  167. : media_log_(media_log),
  168. text_track_config_(text_config),
  169. seek_buffer_timestamp_(kNoTimestamp),
  170. coded_frame_group_start_pts_(kNoTimestamp),
  171. range_for_next_append_(ranges_.end()),
  172. highest_output_buffer_timestamp_(kNoTimestamp),
  173. max_interbuffer_distance_(
  174. base::Milliseconds(kMinimumInterbufferDistanceInMs)),
  175. memory_limit_(
  176. GetDemuxerStreamAudioMemoryLimit(nullptr /*audio_config*/)) {}
  177. SourceBufferStream::~SourceBufferStream() = default;
  178. void SourceBufferStream::OnStartOfCodedFrameGroup(
  179. base::TimeDelta coded_frame_group_start_pts) {
  180. DVLOG(1) << __func__ << " " << GetStreamTypeName() << " (pts "
  181. << coded_frame_group_start_pts.InMicroseconds() << "us)";
  182. DCHECK(!end_of_stream_);
  183. coded_frame_group_start_pts_ = coded_frame_group_start_pts;
  184. new_coded_frame_group_ = true;
  185. auto last_range = range_for_next_append_;
  186. range_for_next_append_ = FindExistingRangeFor(coded_frame_group_start_pts_);
  187. // Only reset |last_appended_buffer_timestamp_| if this new coded frame group
  188. // is not adjacent to the previous coded frame group appended to the stream.
  189. if (range_for_next_append_ == ranges_.end() ||
  190. !IsNextGopAdjacentToEndOfCurrentAppendSequence(
  191. coded_frame_group_start_pts_)) {
  192. ResetLastAppendedState();
  193. DVLOG(3) << __func__ << " next appended buffers will "
  194. << (range_for_next_append_ == ranges_.end()
  195. ? "be in a new range"
  196. : "overlap an existing range");
  197. if (range_for_next_append_ != ranges_.end()) {
  198. // If this new coded frame group overlaps an existing range, preserve
  199. // continuity from that range to the new group by moving the start time
  200. // earlier (but not at or beyond the most recent buffered frame's time
  201. // before |coded_frame_group_start_pts_| in the range, and not beyond the
  202. // range's start time. This update helps prevent discontinuity from being
  203. // introduced by the ::RemoveInternal processing during the next ::Append
  204. // call.
  205. base::TimeDelta adjusted_start_time =
  206. (*range_for_next_append_)
  207. ->FindHighestBufferedTimestampAtOrBefore(
  208. coded_frame_group_start_pts_);
  209. if (adjusted_start_time < coded_frame_group_start_pts_) {
  210. // Exclude removal of that earlier frame during later Append
  211. // processing by adjusting the removal range slightly forward.
  212. coded_frame_group_start_pts_ =
  213. adjusted_start_time + base::Microseconds(1);
  214. }
  215. }
  216. } else if (last_range != ranges_.end()) {
  217. DCHECK(last_range == range_for_next_append_);
  218. DVLOG(3) << __func__ << " next appended buffers will continue range unless "
  219. << "intervening remove makes discontinuity";
  220. }
  221. }
  222. void SourceBufferStream::Append(const BufferQueue& buffers) {
  223. TRACE_EVENT2("media", "SourceBufferStream::Append",
  224. "stream type", GetStreamTypeName(),
  225. "buffers to append", buffers.size());
  226. DCHECK(!buffers.empty());
  227. DCHECK(coded_frame_group_start_pts_ != kNoTimestamp);
  228. DCHECK(!end_of_stream_);
  229. DVLOG(1) << __func__ << " " << GetStreamTypeName() << ": buffers "
  230. << BufferQueueMetadataToLogString(buffers);
  231. DVLOG(4) << BufferQueueBuffersToLogString(buffers);
  232. DCHECK(!buffers.front()->is_key_frame() ||
  233. coded_frame_group_start_pts_ <= buffers.front()->timestamp());
  234. DVLOG_IF(2, coded_frame_group_start_pts_ > buffers.front()->timestamp())
  235. << __func__
  236. << " Suspected SAP-Type-2 occurrence: coded_frame_group_start_pts_="
  237. << coded_frame_group_start_pts_.InMicroseconds()
  238. << "us, first new buffer has timestamp="
  239. << buffers.front()->timestamp().InMicroseconds() << "us";
  240. // New coded frame groups emitted by the coded frame processor must begin with
  241. // a keyframe. Avoid propagating with escalating impact if this assumption is
  242. // broken.
  243. CHECK(!new_coded_frame_group_ || buffers.front()->is_key_frame());
  244. // Buffers within each GOP in a coded frame group must be monotonically
  245. // increasing in DTS order.
  246. DCHECK(IsDtsMonotonicallyIncreasing(buffers));
  247. // Both of these checks enforce what should be guaranteed by how
  248. // FrameProcessor signals OnStartOfCodedFrameGroup and the buffers it tells us
  249. // to Append.
  250. DCHECK(coded_frame_group_start_pts_ >= base::TimeDelta());
  251. DCHECK(buffers.front()->timestamp() >= base::TimeDelta());
  252. if (UpdateMaxInterbufferDtsDistance(buffers)) {
  253. // Coalesce |ranges_| using the new fudge room. This helps keep |ranges_|
  254. // sorted in complex scenarios. See https://crbug.com/793247.
  255. MergeAllAdjacentRanges();
  256. }
  257. SetConfigIds(buffers);
  258. // Save a snapshot of stream state before range modifications are made.
  259. base::TimeDelta next_buffer_timestamp = GetNextBufferTimestamp();
  260. BufferQueue deleted_buffers;
  261. PrepareRangesForNextAppend(buffers, &deleted_buffers);
  262. // If there's a range for |buffers|, insert |buffers| accordingly. Otherwise,
  263. // create a new range with |buffers|.
  264. if (range_for_next_append_ != ranges_.end()) {
  265. if (new_coded_frame_group_) {
  266. // If the first append to this stream in a new coded frame group continues
  267. // a previous range, use the new group's start time instead of the first
  268. // new buffer's timestamp as the proof of adjacency to the existing range.
  269. // A large gap (larger than our normal buffer adjacency test) can occur in
  270. // a muxed set of streams (which share a common coded frame group start
  271. // time) with a significantly jagged start across the streams.
  272. (*range_for_next_append_)
  273. ->AppendBuffersToEnd(buffers, coded_frame_group_start_pts_);
  274. } else {
  275. // Otherwise, use the first new buffer as proof of adjacency.
  276. (*range_for_next_append_)->AppendBuffersToEnd(buffers, kNoTimestamp);
  277. }
  278. last_appended_buffer_timestamp_ = buffers.back()->timestamp();
  279. last_appended_buffer_duration_ = buffers.back()->duration();
  280. last_appended_buffer_is_keyframe_ = buffers.back()->is_key_frame();
  281. last_appended_buffer_decode_timestamp_ =
  282. buffers.back()->GetDecodeTimestamp();
  283. highest_timestamp_in_append_sequence_ =
  284. (*range_for_next_append_)->GetEndTimestamp();
  285. highest_buffered_end_time_in_append_sequence_ =
  286. (*range_for_next_append_)->GetBufferedEndTimestamp();
  287. } else {
  288. base::TimeDelta new_range_start_time =
  289. std::min(coded_frame_group_start_pts_, buffers.front()->timestamp());
  290. const BufferQueue* buffers_for_new_range = &buffers;
  291. BufferQueue trimmed_buffers;
  292. // If the new range is not being created because of a new coded frame group,
  293. // then we must make sure that we start with a key frame. This can happen
  294. // if the GOP in the previous append gets destroyed by a Remove() call.
  295. if (!new_coded_frame_group_) {
  296. BufferQueue::const_iterator itr = buffers.begin();
  297. // Scan past all the non-key-frames.
  298. while (itr != buffers.end() && !(*itr)->is_key_frame()) {
  299. ++itr;
  300. }
  301. // If we didn't find a key frame, then update the last appended
  302. // buffer state and return.
  303. if (itr == buffers.end()) {
  304. last_appended_buffer_timestamp_ = buffers.back()->timestamp();
  305. last_appended_buffer_duration_ = buffers.back()->duration();
  306. last_appended_buffer_is_keyframe_ = buffers.back()->is_key_frame();
  307. last_appended_buffer_decode_timestamp_ =
  308. buffers.back()->GetDecodeTimestamp();
  309. // Since we didn't buffer anything, don't update
  310. // |highest_timestamp_in_append_sequence_|.
  311. DVLOG(1) << __func__ << " " << GetStreamTypeName()
  312. << ": new buffers in the middle of coded frame group depend on"
  313. " keyframe that has been removed, and contain no keyframes."
  314. " Skipping further processing.";
  315. DVLOG(1) << __func__ << " " << GetStreamTypeName()
  316. << ": done. ranges_=" << RangesToString(ranges_);
  317. return;
  318. } else if (itr != buffers.begin()) {
  319. // Copy the first key frame and everything after it into
  320. // |trimmed_buffers|.
  321. trimmed_buffers.assign(itr, buffers.end());
  322. buffers_for_new_range = &trimmed_buffers;
  323. }
  324. new_range_start_time = buffers_for_new_range->front()->timestamp();
  325. }
  326. range_for_next_append_ = AddToRanges(std::make_unique<SourceBufferRange>(
  327. TypeToGapPolicy(GetType()), *buffers_for_new_range,
  328. new_range_start_time,
  329. base::BindRepeating(&SourceBufferStream::GetMaxInterbufferDistance,
  330. base::Unretained(this))));
  331. last_appended_buffer_timestamp_ =
  332. buffers_for_new_range->back()->timestamp();
  333. last_appended_buffer_duration_ = buffers_for_new_range->back()->duration();
  334. last_appended_buffer_is_keyframe_ =
  335. buffers_for_new_range->back()->is_key_frame();
  336. last_appended_buffer_decode_timestamp_ =
  337. buffers_for_new_range->back()->GetDecodeTimestamp();
  338. highest_timestamp_in_append_sequence_ =
  339. (*range_for_next_append_)->GetEndTimestamp();
  340. highest_buffered_end_time_in_append_sequence_ =
  341. (*range_for_next_append_)->GetBufferedEndTimestamp();
  342. }
  343. new_coded_frame_group_ = false;
  344. MergeWithNextRangeIfNecessary(range_for_next_append_);
  345. // Some SAP-Type-2 append sequences require that we coalesce
  346. // |range_for_next_append_| with the range that is *before* it.
  347. if (range_for_next_append_ != ranges_.begin()) {
  348. auto prior_range = range_for_next_append_;
  349. prior_range--;
  350. MergeWithNextRangeIfNecessary(prior_range);
  351. }
  352. // Seek to try to fulfill a previous call to Seek().
  353. if (seek_pending_) {
  354. DCHECK(!selected_range_);
  355. DCHECK(deleted_buffers.empty());
  356. Seek(seek_buffer_timestamp_);
  357. }
  358. if (!deleted_buffers.empty()) {
  359. track_buffer_.insert(track_buffer_.end(), deleted_buffers.begin(),
  360. deleted_buffers.end());
  361. DVLOG(3) << __func__ << " " << GetStreamTypeName() << " Added "
  362. << deleted_buffers.size()
  363. << " buffers to track buffer. TB size is now "
  364. << track_buffer_.size();
  365. } else {
  366. DVLOG(3) << __func__ << " " << GetStreamTypeName()
  367. << " No deleted buffers for track buffer";
  368. }
  369. // Prune any extra buffers in |track_buffer_| if new keyframes
  370. // are appended to the range covered by |track_buffer_|.
  371. if (!track_buffer_.empty()) {
  372. base::TimeDelta keyframe_timestamp =
  373. FindKeyframeAfterTimestamp(track_buffer_.front()->timestamp());
  374. if (keyframe_timestamp != kNoTimestamp)
  375. PruneTrackBuffer(keyframe_timestamp);
  376. }
  377. SetSelectedRangeIfNeeded(next_buffer_timestamp);
  378. DVLOG(1) << __func__ << " " << GetStreamTypeName()
  379. << ": done. ranges_=" << RangesToString(ranges_);
  380. DCHECK(IsRangeListSorted(ranges_));
  381. DCHECK(OnlySelectedRangeIsSeeked());
  382. }
  383. void SourceBufferStream::Remove(base::TimeDelta start,
  384. base::TimeDelta end,
  385. base::TimeDelta duration) {
  386. DVLOG(1) << __func__ << " " << GetStreamTypeName() << " ("
  387. << start.InMicroseconds() << "us, " << end.InMicroseconds() << "us, "
  388. << duration.InMicroseconds() << "us)";
  389. DCHECK(start >= base::TimeDelta()) << start.InMicroseconds() << "us";
  390. DCHECK(start < end) << "start " << start.InMicroseconds() << "us, end "
  391. << end.InMicroseconds() << "us";
  392. DCHECK(duration != kNoTimestamp);
  393. base::TimeDelta remove_end_timestamp = duration;
  394. base::TimeDelta keyframe_timestamp = FindKeyframeAfterTimestamp(end);
  395. if (keyframe_timestamp != kNoTimestamp) {
  396. remove_end_timestamp = keyframe_timestamp;
  397. } else if (end < remove_end_timestamp) {
  398. remove_end_timestamp = end;
  399. }
  400. BufferQueue deleted_buffers;
  401. RemoveInternal(start, remove_end_timestamp, false, &deleted_buffers);
  402. if (!deleted_buffers.empty()) {
  403. // Buffers for the current position have been removed.
  404. SetSelectedRangeIfNeeded(deleted_buffers.front()->timestamp());
  405. if (highest_output_buffer_timestamp_ == kNoTimestamp) {
  406. // We just removed buffers for the current playback position for this
  407. // stream, yet we also had output no buffer since the last Seek.
  408. // Re-seek to prevent stall.
  409. DVLOG(1) << __func__ << " " << GetStreamTypeName() << ": re-seeking to "
  410. << seek_buffer_timestamp_
  411. << " to prevent stall if this time becomes buffered again";
  412. Seek(seek_buffer_timestamp_);
  413. }
  414. }
  415. DCHECK(OnlySelectedRangeIsSeeked());
  416. DCHECK(IsRangeListSorted(ranges_));
  417. }
  418. base::TimeDelta SourceBufferStream::PotentialNextAppendTimestamp() const {
  419. // The next potential append will either be in a GOP adjacent to
  420. // |highest_timestamp_in_append_sequence_| (if known), or if unknown and we
  421. // are still at the beginning of a new coded frame group, then will be into
  422. // the range (if any) to which |coded_frame_group_start_pts_| belongs.
  423. if (highest_timestamp_in_append_sequence_ != kNoTimestamp)
  424. return highest_timestamp_in_append_sequence_;
  425. if (new_coded_frame_group_)
  426. return coded_frame_group_start_pts_;
  427. // If we still don't know a potential next append timestamp, then we have
  428. // removed the range to which it previously belonged and have not completed a
  429. // subsequent append or received a subsequent OnStartOfCodedFrameGroup()
  430. // signal.
  431. return kNoTimestamp;
  432. }
  433. void SourceBufferStream::UpdateLastAppendStateForRemove(
  434. base::TimeDelta remove_start,
  435. base::TimeDelta remove_end,
  436. bool exclude_start) {
  437. // TODO(chcunningham): change exclude_start to include_start in this class and
  438. // SourceBufferRange. Negatives are hard to reason about.
  439. bool include_start = !exclude_start;
  440. // No need to check previous append's GOP if starting a new CFG. New CFG is
  441. // already required to begin with a key frame.
  442. if (new_coded_frame_group_)
  443. return;
  444. if (range_for_next_append_ != ranges_.end()) {
  445. if (last_appended_buffer_timestamp_ != kNoTimestamp) {
  446. // Note start and end of last appended GOP.
  447. base::TimeDelta gop_end = highest_timestamp_in_append_sequence_;
  448. base::TimeDelta gop_start =
  449. (*range_for_next_append_)->KeyframeBeforeTimestamp(gop_end);
  450. // If last append is about to be disrupted, reset associated state so we
  451. // know to create a new range for future appends and require an initial
  452. // key frame.
  453. if (((include_start && remove_start == gop_end) ||
  454. remove_start < gop_end) &&
  455. remove_end > gop_start) {
  456. DVLOG(2) << __func__ << " " << GetStreamTypeName()
  457. << " Resetting next append state for remove ("
  458. << remove_start.InMicroseconds() << "us, "
  459. << remove_end.InMicroseconds() << "us, " << exclude_start
  460. << ")";
  461. range_for_next_append_ = ranges_.end();
  462. ResetLastAppendedState();
  463. }
  464. } else {
  465. NOTREACHED() << __func__ << " " << GetStreamTypeName()
  466. << " range_for_next_append_ set, but not tracking last"
  467. << " append nor new coded frame group.";
  468. }
  469. }
  470. }
  471. void SourceBufferStream::RemoveInternal(base::TimeDelta start,
  472. base::TimeDelta end,
  473. bool exclude_start,
  474. BufferQueue* deleted_buffers) {
  475. DVLOG(2) << __func__ << " " << GetStreamTypeName() << " ("
  476. << start.InMicroseconds() << "us, " << end.InMicroseconds() << "us, "
  477. << exclude_start << ")";
  478. DVLOG(3) << __func__ << " " << GetStreamTypeName()
  479. << ": before remove ranges_=" << RangesToString(ranges_);
  480. DCHECK(start >= base::TimeDelta());
  481. DCHECK(start < end) << "start " << start.InMicroseconds() << "us, end "
  482. << end.InMicroseconds() << "us";
  483. DCHECK(deleted_buffers);
  484. // Doing this up-front simplifies decisions about |range_for_next_append_|
  485. // below.
  486. UpdateLastAppendStateForRemove(start, end, exclude_start);
  487. auto itr = ranges_.begin();
  488. while (itr != ranges_.end()) {
  489. SourceBufferRange* range = itr->get();
  490. if (range->GetStartTimestamp() >= end)
  491. break;
  492. // Split off any remaining GOPs starting at or after |end| and add it to
  493. // |ranges_|.
  494. std::unique_ptr<SourceBufferRange> new_range = range->SplitRange(end);
  495. if (new_range) {
  496. itr = ranges_.insert(++itr, std::move(new_range));
  497. // Update |range_for_next_append_| if it was previously |range| and should
  498. // be the new range (that |itr| is at) now.
  499. if (range_for_next_append_ != ranges_.end() &&
  500. range_for_next_append_->get() == range) {
  501. base::TimeDelta potential_next_append_timestamp =
  502. PotentialNextAppendTimestamp();
  503. if (potential_next_append_timestamp != kNoTimestamp &&
  504. (*itr)->BelongsToRange(potential_next_append_timestamp)) {
  505. range_for_next_append_ = itr;
  506. }
  507. }
  508. // Update the selected range if the next buffer position was transferred
  509. // to the newly inserted range (that |itr| is at now).
  510. if ((*itr)->HasNextBufferPosition())
  511. SetSelectedRange(itr->get());
  512. --itr;
  513. }
  514. // Truncate the current range so that it only contains data before
  515. // the removal range.
  516. BufferQueue saved_buffers;
  517. bool delete_range = range->TruncateAt(start, &saved_buffers, exclude_start);
  518. // Check to see if the current playback position was removed and update the
  519. // selected range appropriately.
  520. if (!saved_buffers.empty()) {
  521. DCHECK(!range->HasNextBufferPosition());
  522. DCHECK(deleted_buffers->empty());
  523. *deleted_buffers = saved_buffers;
  524. }
  525. if (range == selected_range_ && !range->HasNextBufferPosition())
  526. SetSelectedRange(NULL);
  527. // If the current range now is completely covered by the removal
  528. // range then delete it and move on.
  529. if (delete_range) {
  530. DeleteAndRemoveRange(&itr);
  531. continue;
  532. }
  533. // Clear |range_for_next_append_| if we determine that the removal
  534. // operation makes it impossible for the next append to be added
  535. // to the current range.
  536. if (range_for_next_append_ != ranges_.end() &&
  537. range_for_next_append_->get() == range) {
  538. base::TimeDelta potential_next_append_timestamp =
  539. PotentialNextAppendTimestamp();
  540. if (!range->BelongsToRange(potential_next_append_timestamp)) {
  541. DVLOG(1) << "Resetting range_for_next_append_ since the next append"
  542. << " can't add to the current range.";
  543. range_for_next_append_ =
  544. FindExistingRangeFor(potential_next_append_timestamp);
  545. }
  546. }
  547. // Move on to the next range.
  548. ++itr;
  549. }
  550. DVLOG(3) << __func__ << " " << GetStreamTypeName()
  551. << ": after remove ranges_=" << RangesToString(ranges_);
  552. DCHECK(OnlySelectedRangeIsSeeked());
  553. }
  554. void SourceBufferStream::ResetSeekState() {
  555. SetSelectedRange(NULL);
  556. track_buffer_.clear();
  557. config_change_pending_ = false;
  558. highest_output_buffer_timestamp_ = kNoTimestamp;
  559. just_exhausted_track_buffer_ = false;
  560. pending_buffer_.reset();
  561. pending_buffers_complete_ = false;
  562. }
  563. void SourceBufferStream::ResetLastAppendedState() {
  564. last_appended_buffer_timestamp_ = kNoTimestamp;
  565. last_appended_buffer_duration_ = kNoTimestamp;
  566. last_appended_buffer_is_keyframe_ = false;
  567. last_appended_buffer_decode_timestamp_ = kNoDecodeTimestamp;
  568. highest_timestamp_in_append_sequence_ = kNoTimestamp;
  569. highest_buffered_end_time_in_append_sequence_ = kNoTimestamp;
  570. }
  571. bool SourceBufferStream::ShouldSeekToStartOfBuffered(
  572. base::TimeDelta seek_timestamp) const {
  573. if (ranges_.empty())
  574. return false;
  575. base::TimeDelta beginning_of_buffered = ranges_.front()->GetStartTimestamp();
  576. return (seek_timestamp <= beginning_of_buffered &&
  577. beginning_of_buffered < kSeekToStartFudgeRoom());
  578. }
  579. bool SourceBufferStream::IsDtsMonotonicallyIncreasing(
  580. const BufferQueue& buffers) {
  581. DCHECK(!buffers.empty());
  582. DecodeTimestamp prev_dts = last_appended_buffer_decode_timestamp_;
  583. for (BufferQueue::const_iterator itr = buffers.begin();
  584. itr != buffers.end(); ++itr) {
  585. DecodeTimestamp current_dts = (*itr)->GetDecodeTimestamp();
  586. bool current_is_keyframe = (*itr)->is_key_frame();
  587. DCHECK(current_dts != kNoDecodeTimestamp);
  588. DCHECK((*itr)->duration() >= base::TimeDelta())
  589. << "Packet with invalid duration."
  590. << " pts " << (*itr)->timestamp().InMicroseconds() << "us dts "
  591. << (*itr)->GetDecodeTimestamp().InMicroseconds() << "us dur "
  592. << (*itr)->duration().InMicroseconds() << "us";
  593. // Only verify DTS monotonicity within the current GOP (since the last
  594. // keyframe). FrameProcessor should have enforced that all audio frames are
  595. // keyframes already, or are nonkeyframes with monotonically increasing PTS
  596. // since the last keyframe for those types of audio for which nonkeyframes
  597. // may be involved, e.g. xHE-AAC. Video nonkeyframes are not restricted to
  598. // being in-order by PTS, but both audio and video nonkeyframes must be in
  599. // decode sequence since the last keyframe.
  600. if (current_is_keyframe) {
  601. // Reset prev_dts tracking since a new GOP is starting.
  602. prev_dts = kNoDecodeTimestamp;
  603. }
  604. if (prev_dts != kNoDecodeTimestamp) {
  605. if (current_dts < prev_dts) {
  606. MEDIA_LOG(ERROR, media_log_)
  607. << "Buffers did not monotonically increase.";
  608. return false;
  609. }
  610. }
  611. prev_dts = current_dts;
  612. }
  613. return true;
  614. }
  615. bool SourceBufferStream::OnlySelectedRangeIsSeeked() const {
  616. for (auto itr = ranges_.begin(); itr != ranges_.end(); ++itr) {
  617. if ((*itr)->HasNextBufferPosition() && itr->get() != selected_range_)
  618. return false;
  619. }
  620. return !selected_range_ || selected_range_->HasNextBufferPosition();
  621. }
  622. bool SourceBufferStream::UpdateMaxInterbufferDtsDistance(
  623. const BufferQueue& buffers) {
  624. DCHECK(!buffers.empty());
  625. base::TimeDelta old_distance = max_interbuffer_distance_;
  626. DecodeTimestamp prev_dts = last_appended_buffer_decode_timestamp_;
  627. for (BufferQueue::const_iterator itr = buffers.begin();
  628. itr != buffers.end(); ++itr) {
  629. DecodeTimestamp current_dts = (*itr)->GetDecodeTimestamp();
  630. DCHECK(current_dts != kNoDecodeTimestamp);
  631. base::TimeDelta interbuffer_distance = (*itr)->duration();
  632. DCHECK(interbuffer_distance >= base::TimeDelta());
  633. if (prev_dts != kNoDecodeTimestamp) {
  634. interbuffer_distance =
  635. std::max(current_dts - prev_dts, interbuffer_distance);
  636. }
  637. DCHECK(max_interbuffer_distance_ >=
  638. base::Milliseconds(kMinimumInterbufferDistanceInMs));
  639. max_interbuffer_distance_ =
  640. std::max(max_interbuffer_distance_, interbuffer_distance);
  641. prev_dts = current_dts;
  642. }
  643. bool changed_max = max_interbuffer_distance_ != old_distance;
  644. DVLOG_IF(2, changed_max) << __func__ << " " << GetStreamTypeName()
  645. << " Changed max interbuffer DTS distance from "
  646. << old_distance.InMicroseconds() << "us to "
  647. << max_interbuffer_distance_.InMicroseconds()
  648. << "us";
  649. return changed_max;
  650. }
  651. void SourceBufferStream::SetConfigIds(const BufferQueue& buffers) {
  652. for (BufferQueue::const_iterator itr = buffers.begin();
  653. itr != buffers.end(); ++itr) {
  654. (*itr)->SetConfigId(append_config_index_);
  655. }
  656. }
  657. void SourceBufferStream::OnMemoryPressure(
  658. base::TimeDelta media_time,
  659. base::MemoryPressureListener::MemoryPressureLevel memory_pressure_level,
  660. bool force_instant_gc) {
  661. DVLOG(4) << __func__ << " level=" << memory_pressure_level;
  662. // TODO(sebmarchand): Check if MEMORY_PRESSURE_LEVEL_MODERATE should also be
  663. // ignored.
  664. if (memory_pressure_level ==
  665. base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE) {
  666. return;
  667. }
  668. memory_pressure_level_ = memory_pressure_level;
  669. if (force_instant_gc)
  670. GarbageCollectIfNeeded(media_time, 0);
  671. }
  672. bool SourceBufferStream::GarbageCollectIfNeeded(base::TimeDelta media_time,
  673. size_t newDataSize) {
  674. DCHECK(media_time != kNoTimestamp);
  675. // Garbage collection should only happen before/during appending new data,
  676. // which should not happen in end-of-stream state. Unless we also allow GC to
  677. // happen on memory pressure notifications, which might happen even in EOS
  678. // state.
  679. if (!base::FeatureList::IsEnabled(kMemoryPressureBasedSourceBufferGC))
  680. DCHECK(!end_of_stream_);
  681. // Compute size of |ranges_|.
  682. size_t ranges_size = GetBufferedSize();
  683. // Sanity and overflow checks
  684. if ((newDataSize > memory_limit_) ||
  685. (ranges_size + newDataSize < ranges_size)) {
  686. LIMITED_MEDIA_LOG(DEBUG, media_log_, num_garbage_collect_algorithm_logs_,
  687. kMaxGarbageCollectAlgorithmWarningLogs)
  688. << GetStreamTypeName() << " stream: "
  689. << "new append of newDataSize=" << newDataSize
  690. << " bytes exceeds memory_limit_=" << memory_limit_
  691. << ", currently buffered ranges_size=" << ranges_size;
  692. return false;
  693. }
  694. size_t effective_memory_limit = memory_limit_;
  695. if (base::FeatureList::IsEnabled(kMemoryPressureBasedSourceBufferGC)) {
  696. switch (memory_pressure_level_) {
  697. case base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_MODERATE:
  698. effective_memory_limit = memory_limit_ / 2;
  699. break;
  700. case base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_CRITICAL:
  701. effective_memory_limit = 0;
  702. break;
  703. case base::MemoryPressureListener::MEMORY_PRESSURE_LEVEL_NONE:
  704. break;
  705. }
  706. }
  707. // Return if we're under or at the memory limit.
  708. if (ranges_size + newDataSize <= effective_memory_limit)
  709. return true;
  710. size_t bytes_over_hard_memory_limit = 0;
  711. if (ranges_size + newDataSize > memory_limit_)
  712. bytes_over_hard_memory_limit = ranges_size + newDataSize - memory_limit_;
  713. size_t bytes_to_free = ranges_size + newDataSize - effective_memory_limit;
  714. DVLOG(2) << __func__ << " " << GetStreamTypeName()
  715. << ": Before GC media_time=" << media_time.InMicroseconds()
  716. << "us ranges_=" << RangesToString(ranges_)
  717. << " seek_pending_=" << seek_pending_
  718. << " ranges_size=" << ranges_size << " newDataSize=" << newDataSize
  719. << " memory_limit_=" << memory_limit_
  720. << " effective_memory_limit=" << effective_memory_limit
  721. << " last_appended_buffer_timestamp_="
  722. << last_appended_buffer_timestamp_.InMicroseconds()
  723. << "us highest_timestamp_in_append_sequence_="
  724. << highest_timestamp_in_append_sequence_.InMicroseconds()
  725. << "us highest_buffered_end_time_in_append_sequence_="
  726. << highest_buffered_end_time_in_append_sequence_.InMicroseconds()
  727. << "us";
  728. if (selected_range_ && !seek_pending_ &&
  729. media_time > selected_range_->GetBufferedEndTimestamp()) {
  730. // Strictly speaking |media_time| (taken from HTMLMediaElement::currentTime)
  731. // should always be in the buffered ranges, but media::Pipeline uses audio
  732. // stream as the main time source, when audio is present.
  733. // In cases when audio and video streams have different buffered ranges, the
  734. // |media_time| value might be slightly outside of the video stream buffered
  735. // range. In those cases we need to clamp |media_time| value to the current
  736. // stream buffered ranges, to ensure the MSE garbage collection algorithm
  737. // works correctly (see crbug.com/563292 for details).
  738. base::TimeDelta selected_buffered_end =
  739. selected_range_->GetBufferedEndTimestamp();
  740. DVLOG(2) << __func__ << " media_time " << media_time.InMicroseconds()
  741. << "us is outside of selected_range_=["
  742. << selected_range_->GetStartTimestamp().InMicroseconds() << "us;"
  743. << selected_buffered_end.InMicroseconds()
  744. << "us] clamping media_time to be "
  745. << selected_buffered_end.InMicroseconds() << "us";
  746. media_time = selected_buffered_end;
  747. }
  748. size_t bytes_freed = 0;
  749. // If last appended buffer position was earlier than the current playback time
  750. // then try deleting data between last append and current media_time.
  751. if (last_appended_buffer_timestamp_ != kNoTimestamp &&
  752. last_appended_buffer_duration_ != kNoTimestamp &&
  753. highest_buffered_end_time_in_append_sequence_ != kNoTimestamp &&
  754. media_time > highest_buffered_end_time_in_append_sequence_) {
  755. size_t between = FreeBuffersAfterLastAppended(bytes_to_free, media_time);
  756. DVLOG(3) << __func__ << " FreeBuffersAfterLastAppended "
  757. << " released " << between << " bytes"
  758. << " ranges_=" << RangesToString(ranges_);
  759. bytes_freed += between;
  760. // Some players start appending data at the new seek target position before
  761. // actually initiating the seek operation (i.e. they try to improve seek
  762. // performance by prebuffering some data at the seek target position and
  763. // initiating seek once enough data is pre-buffered. In those cases we'll
  764. // see that data is being appended at some new position, but there is no
  765. // pending seek reported yet. In this situation we need to try preserving
  766. // the most recently appended data, i.e. data belonging to the same buffered
  767. // range as the most recent append.
  768. if (range_for_next_append_ != ranges_.end()) {
  769. DCHECK((*range_for_next_append_)->GetStartTimestamp() <= media_time);
  770. media_time = (*range_for_next_append_)->GetStartTimestamp();
  771. DVLOG(3) << __func__ << " media_time adjusted to "
  772. << media_time.InMicroseconds() << "us";
  773. }
  774. }
  775. // If there is an unsatisfied pending seek, we can safely remove all data that
  776. // is earlier than seek target, then remove from the back until we reach the
  777. // most recently appended GOP and then remove from the front if we still don't
  778. // have enough space for the upcoming append.
  779. if (bytes_freed < bytes_to_free && seek_pending_) {
  780. DCHECK(!ranges_.empty());
  781. // All data earlier than the seek target |media_time| can be removed safely
  782. size_t front = FreeBuffers(bytes_to_free - bytes_freed, media_time, false);
  783. DVLOG(3) << __func__ << " Removed " << front
  784. << " bytes from the front. ranges_=" << RangesToString(ranges_);
  785. bytes_freed += front;
  786. // If removing data earlier than |media_time| didn't free up enough space,
  787. // then try deleting from the back until we reach most recently appended GOP
  788. if (bytes_freed < bytes_to_free) {
  789. size_t back = FreeBuffers(bytes_to_free - bytes_freed, media_time, true);
  790. DVLOG(3) << __func__ << " Removed " << back
  791. << " bytes from the back. ranges_=" << RangesToString(ranges_);
  792. bytes_freed += back;
  793. }
  794. // If even that wasn't enough, then try greedily deleting from the front,
  795. // that should allow us to remove as much data as necessary to succeed.
  796. if (bytes_freed < bytes_to_free) {
  797. size_t front2 = FreeBuffers(bytes_to_free - bytes_freed,
  798. ranges_.back()->GetEndTimestamp(), false);
  799. DVLOG(3) << __func__ << " Removed " << front2
  800. << " bytes from the front. ranges_=" << RangesToString(ranges_);
  801. bytes_freed += front2;
  802. }
  803. DCHECK(bytes_freed >= bytes_to_free);
  804. }
  805. // Try removing data from the front of the SourceBuffer up to |media_time|
  806. // position.
  807. if (bytes_freed < bytes_to_free) {
  808. size_t front = FreeBuffers(bytes_to_free - bytes_freed, media_time, false);
  809. DVLOG(3) << __func__ << " Removed " << front
  810. << " bytes from the front. ranges_=" << RangesToString(ranges_);
  811. bytes_freed += front;
  812. }
  813. // Try removing data from the back of the SourceBuffer, until we reach the
  814. // most recent append position.
  815. if (bytes_freed < bytes_to_free) {
  816. size_t back = FreeBuffers(bytes_to_free - bytes_freed, media_time, true);
  817. DVLOG(3) << __func__ << " Removed " << back
  818. << " bytes from the back. ranges_=" << RangesToString(ranges_);
  819. bytes_freed += back;
  820. }
  821. DVLOG(2) << __func__ << " " << GetStreamTypeName()
  822. << ": After GC bytes_to_free=" << bytes_to_free
  823. << " bytes_freed=" << bytes_freed
  824. << " bytes_over_hard_memory_limit=" << bytes_over_hard_memory_limit
  825. << " ranges_=" << RangesToString(ranges_);
  826. return bytes_freed >= bytes_over_hard_memory_limit;
  827. }
  828. size_t SourceBufferStream::FreeBuffersAfterLastAppended(
  829. size_t total_bytes_to_free,
  830. base::TimeDelta media_time) {
  831. DVLOG(4) << __func__ << " highest_buffered_end_time_in_append_sequence_="
  832. << highest_buffered_end_time_in_append_sequence_.InMicroseconds()
  833. << "us media_time=" << media_time.InMicroseconds() << "us";
  834. base::TimeDelta remove_range_start =
  835. highest_buffered_end_time_in_append_sequence_;
  836. if (last_appended_buffer_is_keyframe_)
  837. remove_range_start += GetMaxInterbufferDistance();
  838. base::TimeDelta remove_range_start_keyframe =
  839. FindKeyframeAfterTimestamp(remove_range_start);
  840. if (remove_range_start_keyframe != kNoTimestamp)
  841. remove_range_start = remove_range_start_keyframe;
  842. if (remove_range_start >= media_time)
  843. return 0;
  844. base::TimeDelta remove_range_end;
  845. size_t bytes_freed = GetRemovalRange(remove_range_start,
  846. media_time,
  847. total_bytes_to_free,
  848. &remove_range_end);
  849. if (bytes_freed > 0) {
  850. DVLOG(4) << __func__ << " removing [" << remove_range_start.InMicroseconds()
  851. << "us;" << remove_range_end.InMicroseconds() << "us]";
  852. Remove(remove_range_start, remove_range_end, media_time);
  853. }
  854. return bytes_freed;
  855. }
  856. size_t SourceBufferStream::GetRemovalRange(
  857. base::TimeDelta start_timestamp,
  858. base::TimeDelta end_timestamp,
  859. size_t total_bytes_to_free,
  860. base::TimeDelta* removal_end_timestamp) {
  861. DCHECK(start_timestamp >= base::TimeDelta())
  862. << start_timestamp.InMicroseconds() << "us";
  863. DCHECK(start_timestamp < end_timestamp)
  864. << "start " << start_timestamp.InMicroseconds() << "us, end "
  865. << end_timestamp.InMicroseconds() << "us";
  866. size_t bytes_freed = 0;
  867. for (auto itr = ranges_.begin();
  868. itr != ranges_.end() && bytes_freed < total_bytes_to_free; ++itr) {
  869. SourceBufferRange* range = itr->get();
  870. if (range->GetStartTimestamp() >= end_timestamp)
  871. break;
  872. if (range->GetEndTimestamp() < start_timestamp)
  873. continue;
  874. size_t bytes_to_free = total_bytes_to_free - bytes_freed;
  875. size_t bytes_removed = range->GetRemovalGOP(
  876. start_timestamp, end_timestamp, bytes_to_free, removal_end_timestamp);
  877. bytes_freed += bytes_removed;
  878. }
  879. return bytes_freed;
  880. }
  881. size_t SourceBufferStream::FreeBuffers(size_t total_bytes_to_free,
  882. base::TimeDelta media_time,
  883. bool reverse_direction) {
  884. TRACE_EVENT2("media", "SourceBufferStream::FreeBuffers",
  885. "total bytes to free", total_bytes_to_free,
  886. "reverse direction", reverse_direction);
  887. DCHECK_GT(total_bytes_to_free, 0u);
  888. size_t bytes_freed = 0;
  889. // This range will save the last GOP appended to |range_for_next_append_|
  890. // if the buffers surrounding it get deleted during garbage collection.
  891. std::unique_ptr<SourceBufferRange> new_range_for_append;
  892. while (!ranges_.empty() && bytes_freed < total_bytes_to_free) {
  893. SourceBufferRange* current_range = NULL;
  894. BufferQueue buffers;
  895. size_t bytes_deleted = 0;
  896. if (reverse_direction) {
  897. current_range = ranges_.back().get();
  898. DVLOG(5) << "current_range=" << RangeToString(*current_range);
  899. if (current_range->LastGOPContainsNextBufferPosition()) {
  900. DCHECK_EQ(current_range, selected_range_);
  901. DVLOG(5) << "current_range contains next read position, stopping GC";
  902. break;
  903. }
  904. DVLOG(5) << "Deleting GOP from back: " << RangeToString(*current_range);
  905. bytes_deleted = current_range->DeleteGOPFromBack(&buffers);
  906. } else {
  907. current_range = ranges_.front().get();
  908. DVLOG(5) << "current_range=" << RangeToString(*current_range);
  909. // FirstGOPEarlierThanMediaTime() is useful here especially if
  910. // |seek_pending_| (such that no range contains next buffer
  911. // position).
  912. // FirstGOPContainsNextBufferPosition() is useful here especially if
  913. // |!seek_pending_| to protect against DeleteGOPFromFront() if
  914. // FirstGOPEarlierThanMediaTime() was insufficient alone.
  915. if (!current_range->FirstGOPEarlierThanMediaTime(media_time) ||
  916. current_range->FirstGOPContainsNextBufferPosition()) {
  917. // We have removed all data up to the GOP that contains current playback
  918. // position, we can't delete any further.
  919. DVLOG(5) << "current_range contains playback position, stopping GC";
  920. break;
  921. }
  922. DVLOG(4) << "Deleting GOP from front: " << RangeToString(*current_range)
  923. << ", media_time: " << media_time.InMicroseconds()
  924. << ", current_range->HasNextBufferPosition(): "
  925. << current_range->HasNextBufferPosition();
  926. bytes_deleted = current_range->DeleteGOPFromFront(&buffers);
  927. }
  928. // Check to see if we've just deleted the GOP that was last appended.
  929. base::TimeDelta end_timestamp = buffers.back()->timestamp();
  930. if (end_timestamp == last_appended_buffer_timestamp_) {
  931. DCHECK(last_appended_buffer_timestamp_ != kNoTimestamp);
  932. DCHECK(!new_range_for_append);
  933. // Create a new range containing these buffers.
  934. new_range_for_append = std::make_unique<SourceBufferRange>(
  935. TypeToGapPolicy(GetType()), buffers, kNoTimestamp,
  936. base::BindRepeating(&SourceBufferStream::GetMaxInterbufferDistance,
  937. base::Unretained(this)));
  938. range_for_next_append_ = ranges_.end();
  939. } else {
  940. bytes_freed += bytes_deleted;
  941. }
  942. if (current_range->size_in_bytes() == 0) {
  943. DCHECK_NE(current_range, selected_range_);
  944. DCHECK(range_for_next_append_ == ranges_.end() ||
  945. range_for_next_append_->get() != current_range);
  946. // Delete |current_range| by popping it out of |ranges_|.
  947. reverse_direction ? ranges_.pop_back() : ranges_.pop_front();
  948. }
  949. if (reverse_direction && new_range_for_append) {
  950. // We don't want to delete any further, or we'll be creating gaps
  951. break;
  952. }
  953. }
  954. // Insert |new_range_for_append| into |ranges_|, if applicable.
  955. if (new_range_for_append) {
  956. range_for_next_append_ = AddToRanges(std::move(new_range_for_append));
  957. DCHECK(range_for_next_append_ != ranges_.end());
  958. // Check to see if we need to merge the just added range that was in
  959. // |new_range_for_append| with the range before or after it. That added
  960. // range is created whenever the last GOP appended is encountered,
  961. // regardless of whether any buffers after it are ultimately deleted.
  962. // Merging is necessary if there were no buffers (or very few buffers)
  963. // deleted after creating that added range.
  964. if (range_for_next_append_ != ranges_.begin()) {
  965. auto range_before_next = range_for_next_append_;
  966. --range_before_next;
  967. MergeWithNextRangeIfNecessary(range_before_next);
  968. }
  969. MergeWithNextRangeIfNecessary(range_for_next_append_);
  970. }
  971. return bytes_freed;
  972. }
  973. void SourceBufferStream::TrimSpliceOverlap(const BufferQueue& new_buffers) {
  974. DCHECK(!new_buffers.empty());
  975. DCHECK_EQ(SourceBufferStreamType::kAudio, GetType());
  976. const base::TimeDelta splice_timestamp = new_buffers.front()->timestamp();
  977. // Since some audio formats may have nonkeyframes (in PTS order since last
  978. // keyframe), if the front of the new buffers is one of those, it cannot be
  979. // used to begin a decode following an overlap. Here, we bail in this case,
  980. // since such a splice could not be coherently decoded.
  981. if (!new_buffers.front()->is_key_frame()) {
  982. DVLOG(3) << __func__
  983. << " No splice trimming. Front of |new_buffers| is not a "
  984. "keyframe, at time "
  985. << splice_timestamp.InMicroseconds();
  986. return;
  987. }
  988. // Find the overlapped range (if any).
  989. auto range_itr = FindExistingRangeFor(splice_timestamp);
  990. if (range_itr == ranges_.end()) {
  991. DVLOG(3) << __func__ << " No splice trimming. No range overlap at time "
  992. << splice_timestamp.InMicroseconds();
  993. return;
  994. }
  995. // Search for overlapped buffer needs exclusive end value. Choosing smallest
  996. // possible value.
  997. const base::TimeDelta end_pts = splice_timestamp + base::Microseconds(1);
  998. // Find if new buffer's start would overlap an existing buffer. Note that
  999. // overlapped audio buffers might be nonkeyframes, but if so, FrameProcessor
  1000. // ensures they are in PTS order since the previous keyframe.
  1001. BufferQueue overlapped_buffers;
  1002. if (!(*range_itr)
  1003. ->GetBuffersInRange(splice_timestamp, end_pts,
  1004. &overlapped_buffers)) {
  1005. // Bail if no overlapped buffers found.
  1006. DVLOG(3) << __func__ << " No splice trimming. No buffer overlap at time "
  1007. << splice_timestamp.InMicroseconds();
  1008. return;
  1009. }
  1010. // At most one buffer should exist containing the time of the newly appended
  1011. // buffer's start. It may happen that bad content appends buffers with
  1012. // durations that cause nonsensical overlap. Trimming should not be performed
  1013. // in these cases, as the content is already in a bad state.
  1014. if (overlapped_buffers.size() != 1U) {
  1015. DVLOG(3) << __func__
  1016. << " No splice trimming. Found more than one overlapped buffer"
  1017. " (bad content) at time "
  1018. << splice_timestamp.InMicroseconds();
  1019. MEDIA_LOG(WARNING, media_log_)
  1020. << "Media is badly muxed. Detected " << overlapped_buffers.size()
  1021. << " overlapping audio buffers at time "
  1022. << splice_timestamp.InMicroseconds();
  1023. return;
  1024. }
  1025. StreamParserBuffer* overlapped_buffer = overlapped_buffers.front().get();
  1026. if (overlapped_buffer->timestamp() == splice_timestamp) {
  1027. // Ignore buffers with the same start time. They will be completely removed
  1028. // in PrepareRangesForNextAppend().
  1029. DVLOG(3) << __func__ << " No splice trimming at time "
  1030. << splice_timestamp.InMicroseconds()
  1031. << ". Overlapped buffer will be completely removed.";
  1032. return;
  1033. }
  1034. // Trimming a buffer with estimated duration is too risky. Estimates are rough
  1035. // and what appears to be overlap may really just be a bad estimate. Imprecise
  1036. // trimming may lead to loss of AV sync.
  1037. if (overlapped_buffer->is_duration_estimated()) {
  1038. DVLOG(3) << __func__ << " Skipping audio splice trimming at PTS="
  1039. << splice_timestamp.InMicroseconds() << ". Overlapped buffer has "
  1040. << "estimated duration.";
  1041. return;
  1042. }
  1043. // Determine the duration of overlap.
  1044. base::TimeDelta overlapped_end_time =
  1045. overlapped_buffer->timestamp() + overlapped_buffer->duration();
  1046. base::TimeDelta overlap_duration = overlapped_end_time - splice_timestamp;
  1047. // At this point overlap should be non-empty (ruled out same-timestamp above).
  1048. DCHECK_GT(overlap_duration, base::TimeDelta());
  1049. // Don't trim for overlaps of less than one millisecond (which is frequently
  1050. // the extent of timestamp resolution for poorly encoded media).
  1051. if (overlap_duration < base::Milliseconds(1)) {
  1052. std::stringstream log_string;
  1053. log_string << "Skipping audio splice trimming at PTS="
  1054. << splice_timestamp.InMicroseconds() << "us. Found only "
  1055. << overlap_duration.InMicroseconds()
  1056. << "us of overlap, need at least 1000us. Multiple occurrences "
  1057. << "may result in loss of A/V sync.";
  1058. LIMITED_MEDIA_LOG(DEBUG, media_log_, num_splice_logs_, kMaxAudioSpliceLogs)
  1059. << log_string.str();
  1060. DVLOG(1) << __func__ << log_string.str();
  1061. return;
  1062. }
  1063. // Trim overlap from the existing buffer.
  1064. DecoderBuffer::DiscardPadding discard_padding =
  1065. overlapped_buffer->discard_padding();
  1066. discard_padding.second += overlap_duration;
  1067. overlapped_buffer->set_discard_padding(discard_padding);
  1068. overlapped_buffer->set_duration(overlapped_buffer->duration() -
  1069. overlap_duration);
  1070. // Note that the range's end time tracking shouldn't need explicit updating
  1071. // here due to the overlapped buffer's truncation because the range tracks
  1072. // that end time using a pointer to the buffer (which should be
  1073. // |overlapped_buffer| if the overlap occurred at the end of the range).
  1074. // Every audio frame is either a keyframe, or if a nonkeyframe is in PTS order
  1075. // since the last keyframe, so there is no out-of-order PTS vs DTS sequencing
  1076. // to overcome. If the overlap occurs in the middle of the range, the caller
  1077. // invokes methods on the range which internally update the end time(s) of the
  1078. // resulting range(s) involved in the append.
  1079. std::stringstream log_string;
  1080. log_string << "Audio buffer splice at PTS="
  1081. << splice_timestamp.InMicroseconds()
  1082. << "us. Trimmed tail of overlapped buffer (PTS="
  1083. << overlapped_buffer->timestamp().InMicroseconds() << "us) by "
  1084. << overlap_duration.InMicroseconds() << "us.";
  1085. LIMITED_MEDIA_LOG(DEBUG, media_log_, num_splice_logs_, kMaxAudioSpliceLogs)
  1086. << log_string.str();
  1087. DVLOG(1) << __func__ << log_string.str();
  1088. }
  1089. void SourceBufferStream::PrepareRangesForNextAppend(
  1090. const BufferQueue& new_buffers,
  1091. BufferQueue* deleted_buffers) {
  1092. DCHECK(deleted_buffers);
  1093. if (GetType() == SourceBufferStreamType::kAudio)
  1094. TrimSpliceOverlap(new_buffers);
  1095. base::TimeDelta buffers_start_timestamp = kNoTimestamp;
  1096. base::TimeDelta buffers_end_timestamp = kNoTimestamp;
  1097. GetTimestampInterval(new_buffers, &buffers_start_timestamp,
  1098. &buffers_end_timestamp);
  1099. DCHECK(buffers_start_timestamp != kNoTimestamp);
  1100. DCHECK(buffers_end_timestamp != kNoTimestamp);
  1101. // 1. Clean up the old buffers between the last appended buffers and the
  1102. // beginning of |new_buffers|.
  1103. if (highest_timestamp_in_append_sequence_ != kNoTimestamp &&
  1104. highest_timestamp_in_append_sequence_ < buffers_start_timestamp) {
  1105. RemoveInternal(highest_timestamp_in_append_sequence_,
  1106. buffers_start_timestamp, true, deleted_buffers);
  1107. }
  1108. // 2. Delete the buffers that |new_buffers| overlaps.
  1109. // There may be buffers in |new_buffers| with timestamp before
  1110. // |highest_timestamp_in_append_sequence_| that shouldn't trigger removal of
  1111. // stuff before |highest_timestamp_in_append_sequence_|.
  1112. if (highest_timestamp_in_append_sequence_ != kNoTimestamp &&
  1113. buffers_start_timestamp < highest_timestamp_in_append_sequence_) {
  1114. DCHECK(highest_timestamp_in_append_sequence_ <=
  1115. highest_buffered_end_time_in_append_sequence_);
  1116. buffers_start_timestamp = highest_buffered_end_time_in_append_sequence_;
  1117. }
  1118. if (new_coded_frame_group_) {
  1119. // Extend the deletion range earlier to the coded frame group start time if
  1120. // this is the first append in a new coded frame group.
  1121. DCHECK(coded_frame_group_start_pts_ != kNoTimestamp);
  1122. buffers_start_timestamp =
  1123. std::min(coded_frame_group_start_pts_, buffers_start_timestamp);
  1124. }
  1125. // Return early if no further overlap removal is needed. First check if
  1126. // |buffers_start_timestamp| is in the middle of the range; we could be
  1127. // overlap-appending the middle of a previous coded frame sequence's range
  1128. // with non-keyframes prior to |highest_timestamp_in_append_sequence_|, so we
  1129. // need to split that range appropriately here and then return early. If we
  1130. // don't return early here, overlap removal (including any necessary range
  1131. // splitting) will occur.
  1132. if (buffers_start_timestamp >= buffers_end_timestamp) {
  1133. DCHECK(highest_timestamp_in_append_sequence_ != kNoTimestamp);
  1134. DCHECK(range_for_next_append_ != ranges_.end());
  1135. DCHECK((*range_for_next_append_)->BelongsToRange(buffers_start_timestamp));
  1136. // Split the range at |buffers_start_timestamp|, if necessary, then return
  1137. // early.
  1138. std::unique_ptr<SourceBufferRange> new_range =
  1139. (*range_for_next_append_)->SplitRange(buffers_start_timestamp);
  1140. if (!new_range)
  1141. return;
  1142. range_for_next_append_ =
  1143. ranges_.insert(++range_for_next_append_, std::move(new_range));
  1144. // Update the selected range if the next buffer position was transferred
  1145. // to the newly inserted range.
  1146. if ((*range_for_next_append_)->HasNextBufferPosition())
  1147. SetSelectedRange(range_for_next_append_->get());
  1148. --range_for_next_append_;
  1149. return;
  1150. }
  1151. // Exclude the start from removal to avoid deleting the highest appended
  1152. // buffer in cases where the first buffer in |new_buffers| has same timestamp
  1153. // as the highest appended buffer (even in out-of-order DTS vs PTS sequence).
  1154. // Only do this when :
  1155. // A. Type is video. This may occur in cases of VP9 alt-ref frames or frames
  1156. // with incorrect timestamps. Removing a frame may break decode
  1157. // dependencies and there are no downsides to just keeping it (other than
  1158. // some throw-away decoder work).
  1159. // B. Type is text. TODO(chcunningham): Implement text splicing. See
  1160. // http://crbug.com/661408
  1161. // C. Type is audio and overlapped duration is 0. We've encountered Vorbis
  1162. // streams containing zero-duration buffers (i.e. no real overlap). For
  1163. // non-zero duration removing overlapped frames is important to preserve
  1164. // A/V sync (see AudioClock).
  1165. const bool exclude_start =
  1166. highest_timestamp_in_append_sequence_ ==
  1167. new_buffers.front()->timestamp() &&
  1168. (GetType() == SourceBufferStreamType::kVideo ||
  1169. GetType() == SourceBufferStreamType::kText ||
  1170. last_appended_buffer_duration_ == base::TimeDelta());
  1171. // Finally do the deletion of overlap.
  1172. RemoveInternal(buffers_start_timestamp, buffers_end_timestamp, exclude_start,
  1173. deleted_buffers);
  1174. }
  1175. // static
  1176. void SourceBufferStream::GetTimestampInterval(const BufferQueue& buffers,
  1177. base::TimeDelta* start,
  1178. base::TimeDelta* end) {
  1179. base::TimeDelta start_pts = buffers.front()->timestamp();
  1180. base::TimeDelta end_pts = start_pts;
  1181. for (const auto& buffer : buffers) {
  1182. base::TimeDelta timestamp = buffer->timestamp();
  1183. start_pts = std::min(timestamp, start_pts);
  1184. base::TimeDelta duration = buffer->duration();
  1185. // FrameProcessor should protect against unknown buffer durations.
  1186. DCHECK_NE(duration, kNoTimestamp);
  1187. if (duration.is_positive() && !buffer->is_duration_estimated()) {
  1188. timestamp += duration;
  1189. } else {
  1190. // TODO(chcunningham): Emit warning when 0ms durations are not expected.
  1191. // http://crbug.com/312836
  1192. timestamp += base::Microseconds(1);
  1193. }
  1194. end_pts = std::max(timestamp, end_pts);
  1195. }
  1196. *start = start_pts;
  1197. *end = end_pts;
  1198. }
  1199. bool SourceBufferStream::IsNextGopAdjacentToEndOfCurrentAppendSequence(
  1200. base::TimeDelta next_gop_timestamp) const {
  1201. base::TimeDelta upper_bound = highest_timestamp_in_append_sequence_ +
  1202. ComputeFudgeRoom(GetMaxInterbufferDistance());
  1203. DVLOG(4) << __func__ << " " << GetStreamTypeName()
  1204. << " next_gop_timestamp=" << next_gop_timestamp.InMicroseconds()
  1205. << "us, highest_timestamp_in_append_sequence_="
  1206. << highest_timestamp_in_append_sequence_.InMicroseconds()
  1207. << "us, upper_bound=" << upper_bound.InMicroseconds() << "us";
  1208. return highest_timestamp_in_append_sequence_ < next_gop_timestamp &&
  1209. next_gop_timestamp <= upper_bound;
  1210. }
  1211. void SourceBufferStream::PruneTrackBuffer(const base::TimeDelta timestamp) {
  1212. DCHECK(timestamp != kNoTimestamp);
  1213. // Scan forward until we find a buffer with timestamp at or beyond the limit.
  1214. // Then remove all those at and beyond that point.
  1215. size_t goal_size = 0; // The number of buffers we will keep in the result.
  1216. for (const auto& buf : track_buffer_) {
  1217. if (buf->timestamp() >= timestamp)
  1218. break;
  1219. goal_size++;
  1220. }
  1221. while (track_buffer_.size() > goal_size) {
  1222. track_buffer_.pop_back();
  1223. }
  1224. DVLOG(3) << __func__ << " " << GetStreamTypeName()
  1225. << " Removed all buffers in track buffer sequence starting with the "
  1226. "first at timestamp >= "
  1227. << timestamp.InMicroseconds()
  1228. << "us. New track buffer size:" << track_buffer_.size();
  1229. }
  1230. void SourceBufferStream::MergeWithNextRangeIfNecessary(
  1231. const RangeList::iterator& range_with_new_buffers_itr) {
  1232. DCHECK(range_with_new_buffers_itr != ranges_.end());
  1233. SourceBufferRange* range_with_new_buffers = range_with_new_buffers_itr->get();
  1234. RangeList::iterator next_range_itr = range_with_new_buffers_itr;
  1235. ++next_range_itr;
  1236. if (next_range_itr == ranges_.end() ||
  1237. !range_with_new_buffers->CanAppendRangeToEnd(**next_range_itr)) {
  1238. return;
  1239. }
  1240. bool transfer_current_position = selected_range_ == next_range_itr->get();
  1241. DVLOG(3) << __func__ << " " << GetStreamTypeName() << " merging "
  1242. << RangeToString(*range_with_new_buffers) << " into "
  1243. << RangeToString(**next_range_itr);
  1244. range_with_new_buffers->AppendRangeToEnd(**next_range_itr,
  1245. transfer_current_position);
  1246. // Update |selected_range_| pointer if |range| has become selected after
  1247. // merges.
  1248. if (transfer_current_position)
  1249. SetSelectedRange(range_with_new_buffers);
  1250. if (next_range_itr == range_for_next_append_)
  1251. range_for_next_append_ = range_with_new_buffers_itr;
  1252. DeleteAndRemoveRange(&next_range_itr);
  1253. }
  1254. void SourceBufferStream::MergeAllAdjacentRanges() {
  1255. DVLOG(1) << __func__ << " " << GetStreamTypeName()
  1256. << ": Before: ranges_=" << RangesToString(ranges_);
  1257. auto range_itr = ranges_.begin();
  1258. while (range_itr != ranges_.end()) {
  1259. const size_t old_ranges_size = ranges_.size();
  1260. MergeWithNextRangeIfNecessary(range_itr);
  1261. // Only proceed to the next range if the current range didn't merge with it.
  1262. if (old_ranges_size == ranges_.size())
  1263. range_itr++;
  1264. }
  1265. DVLOG(1) << __func__ << " " << GetStreamTypeName()
  1266. << ": After: ranges_=" << RangesToString(ranges_);
  1267. }
  1268. void SourceBufferStream::Seek(base::TimeDelta timestamp) {
  1269. DCHECK(timestamp >= base::TimeDelta());
  1270. DVLOG(1) << __func__ << " " << GetStreamTypeName() << " ("
  1271. << timestamp.InMicroseconds() << "us)";
  1272. ResetSeekState();
  1273. seek_buffer_timestamp_ = timestamp;
  1274. seek_pending_ = true;
  1275. if (ShouldSeekToStartOfBuffered(timestamp)) {
  1276. ranges_.front()->SeekToStart();
  1277. SetSelectedRange(ranges_.front().get());
  1278. seek_pending_ = false;
  1279. return;
  1280. }
  1281. auto itr = ranges_.end();
  1282. for (itr = ranges_.begin(); itr != ranges_.end(); ++itr) {
  1283. if ((*itr)->CanSeekTo(timestamp))
  1284. break;
  1285. }
  1286. if (itr == ranges_.end())
  1287. return;
  1288. if (!audio_configs_.empty()) {
  1289. // Adjust |timestamp| for an Opus stream backward up to the config's seek
  1290. // preroll, but not further than the range start time, and not at all if
  1291. // there is a config change in the middle of that preroll interval. If
  1292. // |timestamp| is already before the range start time, as can happen due to
  1293. // fudge room, do not adjust it.
  1294. const auto& config = audio_configs_[(*itr)->GetConfigIdAtTime(timestamp)];
  1295. if (config.codec() == AudioCodec::kOpus &&
  1296. timestamp > (*itr)->GetStartTimestamp()) {
  1297. base::TimeDelta preroll_timestamp = std::max(
  1298. timestamp - config.seek_preroll(), (*itr)->GetStartTimestamp());
  1299. if ((*itr)->CanSeekTo(preroll_timestamp) &&
  1300. (*itr)->SameConfigThruRange(preroll_timestamp, timestamp)) {
  1301. timestamp = preroll_timestamp;
  1302. }
  1303. }
  1304. }
  1305. SeekAndSetSelectedRange(itr->get(), timestamp);
  1306. seek_pending_ = false;
  1307. }
  1308. bool SourceBufferStream::IsSeekPending() const {
  1309. return seek_pending_ && !IsEndOfStreamReached();
  1310. }
  1311. // TODO(wolenetz): Disallow duration changes that truncate buffered media. See
  1312. // https://crbug.com/623729.
  1313. void SourceBufferStream::OnSetDuration(base::TimeDelta duration) {
  1314. DVLOG(1) << __func__ << " " << GetStreamTypeName() << " ("
  1315. << duration.InMicroseconds() << "us)";
  1316. DCHECK(!end_of_stream_);
  1317. if (ranges_.empty())
  1318. return;
  1319. base::TimeDelta start = duration;
  1320. base::TimeDelta end = ranges_.back()->GetBufferedEndTimestamp();
  1321. // Trim the end if it exceeds the new duration.
  1322. if (start < end) {
  1323. BufferQueue deleted_buffers;
  1324. RemoveInternal(start, end, false, &deleted_buffers);
  1325. if (!deleted_buffers.empty()) {
  1326. // Truncation removed current position. Clear selected range.
  1327. SetSelectedRange(NULL);
  1328. }
  1329. }
  1330. }
  1331. SourceBufferStreamStatus SourceBufferStream::GetNextBuffer(
  1332. scoped_refptr<StreamParserBuffer>* out_buffer) {
  1333. DVLOG(2) << __func__ << " " << GetStreamTypeName();
  1334. if (!pending_buffer_.get()) {
  1335. const SourceBufferStreamStatus status = GetNextBufferInternal(out_buffer);
  1336. if (status != SourceBufferStreamStatus::kSuccess ||
  1337. !SetPendingBuffer(out_buffer)) {
  1338. DVLOG(2) << __func__ << " " << GetStreamTypeName()
  1339. << ": no pending buffer, returning status "
  1340. << StatusToString(status);
  1341. return status;
  1342. }
  1343. }
  1344. DCHECK(pending_buffer_->preroll_buffer().get());
  1345. const SourceBufferStreamStatus status =
  1346. HandleNextBufferWithPreroll(out_buffer);
  1347. DVLOG(2) << __func__ << " " << GetStreamTypeName()
  1348. << ": handled next buffer with preroll, returning status "
  1349. << StatusToString(status);
  1350. return status;
  1351. }
  1352. SourceBufferStreamStatus SourceBufferStream::HandleNextBufferWithPreroll(
  1353. scoped_refptr<StreamParserBuffer>* out_buffer) {
  1354. // Any config change should have already been handled.
  1355. DCHECK_EQ(current_config_index_, pending_buffer_->GetConfigId());
  1356. // Check if the preroll buffer has already been handed out.
  1357. if (!pending_buffers_complete_) {
  1358. pending_buffers_complete_ = true;
  1359. *out_buffer = pending_buffer_->preroll_buffer();
  1360. return SourceBufferStreamStatus::kSuccess;
  1361. }
  1362. // Preroll complete, hand out the final buffer.
  1363. *out_buffer = std::move(pending_buffer_);
  1364. return SourceBufferStreamStatus::kSuccess;
  1365. }
  1366. SourceBufferStreamStatus SourceBufferStream::GetNextBufferInternal(
  1367. scoped_refptr<StreamParserBuffer>* out_buffer) {
  1368. CHECK(!config_change_pending_);
  1369. if (!track_buffer_.empty()) {
  1370. DCHECK(!selected_range_);
  1371. if (track_buffer_.front()->GetConfigId() != current_config_index_) {
  1372. config_change_pending_ = true;
  1373. DVLOG(1) << "Config change (track buffer config ID does not match).";
  1374. return SourceBufferStreamStatus::kConfigChange;
  1375. }
  1376. DVLOG(3) << __func__ << " Next buffer coming from track_buffer_";
  1377. *out_buffer = std::move(track_buffer_.front());
  1378. track_buffer_.pop_front();
  1379. WarnIfTrackBufferExhaustionSkipsForward(*out_buffer);
  1380. highest_output_buffer_timestamp_ =
  1381. std::max(highest_output_buffer_timestamp_, (*out_buffer)->timestamp());
  1382. // If the track buffer becomes empty, then try to set the selected range
  1383. // based on the timestamp of this buffer being returned.
  1384. if (track_buffer_.empty()) {
  1385. just_exhausted_track_buffer_ = true;
  1386. SetSelectedRangeIfNeeded(highest_output_buffer_timestamp_);
  1387. }
  1388. return SourceBufferStreamStatus::kSuccess;
  1389. }
  1390. DCHECK(track_buffer_.empty());
  1391. if (!selected_range_ || !selected_range_->HasNextBuffer()) {
  1392. if (IsEndOfStreamReached()) {
  1393. return SourceBufferStreamStatus::kEndOfStream;
  1394. }
  1395. DVLOG(3) << __func__ << " " << GetStreamTypeName()
  1396. << ": returning kNeedBuffer "
  1397. << (selected_range_ ? "(selected range has no next buffer)"
  1398. : "(no selected range)");
  1399. return SourceBufferStreamStatus::kNeedBuffer;
  1400. }
  1401. if (selected_range_->GetNextConfigId() != current_config_index_) {
  1402. config_change_pending_ = true;
  1403. DVLOG(1) << "Config change (selected range config ID does not match).";
  1404. return SourceBufferStreamStatus::kConfigChange;
  1405. }
  1406. CHECK(selected_range_->GetNextBuffer(out_buffer));
  1407. WarnIfTrackBufferExhaustionSkipsForward(*out_buffer);
  1408. highest_output_buffer_timestamp_ =
  1409. std::max(highest_output_buffer_timestamp_, (*out_buffer)->timestamp());
  1410. return SourceBufferStreamStatus::kSuccess;
  1411. }
  1412. void SourceBufferStream::WarnIfTrackBufferExhaustionSkipsForward(
  1413. scoped_refptr<StreamParserBuffer> next_buffer) {
  1414. if (!just_exhausted_track_buffer_)
  1415. return;
  1416. just_exhausted_track_buffer_ = false;
  1417. DCHECK(next_buffer->is_key_frame());
  1418. base::TimeDelta next_output_buffer_timestamp = next_buffer->timestamp();
  1419. base::TimeDelta delta =
  1420. next_output_buffer_timestamp - highest_output_buffer_timestamp_;
  1421. if (delta > GetMaxInterbufferDistance()) {
  1422. LIMITED_MEDIA_LOG(DEBUG, media_log_, num_track_buffer_gap_warning_logs_,
  1423. kMaxTrackBufferGapWarningLogs)
  1424. << "Media append that overlapped current playback position may cause "
  1425. "time gap in playing "
  1426. << GetStreamTypeName() << " stream because the next keyframe is "
  1427. << delta.InMilliseconds()
  1428. << "ms beyond last overlapped frame. Media may appear temporarily "
  1429. "frozen.";
  1430. }
  1431. }
  1432. base::TimeDelta SourceBufferStream::GetNextBufferTimestamp() {
  1433. if (!track_buffer_.empty())
  1434. return track_buffer_.front()->timestamp();
  1435. if (!selected_range_)
  1436. return kNoTimestamp;
  1437. DCHECK(selected_range_->HasNextBufferPosition());
  1438. return selected_range_->GetNextTimestamp();
  1439. }
  1440. SourceBufferStream::RangeList::iterator
  1441. SourceBufferStream::FindExistingRangeFor(base::TimeDelta start_timestamp) {
  1442. for (auto itr = ranges_.begin(); itr != ranges_.end(); ++itr) {
  1443. if ((*itr)->BelongsToRange(start_timestamp))
  1444. return itr;
  1445. }
  1446. return ranges_.end();
  1447. }
  1448. SourceBufferStream::RangeList::iterator SourceBufferStream::AddToRanges(
  1449. std::unique_ptr<SourceBufferRange> new_range) {
  1450. base::TimeDelta start_timestamp = new_range->GetStartTimestamp();
  1451. auto itr = ranges_.end();
  1452. for (itr = ranges_.begin(); itr != ranges_.end(); ++itr) {
  1453. if ((*itr)->GetStartTimestamp() > start_timestamp)
  1454. break;
  1455. }
  1456. return ranges_.insert(itr, std::move(new_range));
  1457. }
  1458. void SourceBufferStream::SeekAndSetSelectedRange(
  1459. SourceBufferRange* range,
  1460. base::TimeDelta seek_timestamp) {
  1461. if (range)
  1462. range->Seek(seek_timestamp);
  1463. SetSelectedRange(range);
  1464. }
  1465. void SourceBufferStream::SetSelectedRange(SourceBufferRange* range) {
  1466. DVLOG(1) << __func__ << " " << GetStreamTypeName() << ": " << selected_range_
  1467. << " " << (selected_range_ ? RangeToString(*selected_range_) : "")
  1468. << " -> " << range << " " << (range ? RangeToString(*range) : "");
  1469. if (selected_range_)
  1470. selected_range_->ResetNextBufferPosition();
  1471. DCHECK(!range || range->HasNextBufferPosition());
  1472. selected_range_ = range;
  1473. }
  1474. Ranges<base::TimeDelta> SourceBufferStream::GetBufferedTime() const {
  1475. Ranges<base::TimeDelta> ranges;
  1476. for (auto itr = ranges_.begin(); itr != ranges_.end(); ++itr) {
  1477. ranges.Add((*itr)->GetStartTimestamp(), (*itr)->GetBufferedEndTimestamp());
  1478. }
  1479. return ranges;
  1480. }
  1481. base::TimeDelta SourceBufferStream::GetHighestPresentationTimestamp() const {
  1482. if (ranges_.empty())
  1483. return base::TimeDelta();
  1484. return ranges_.back()->GetEndTimestamp();
  1485. }
  1486. base::TimeDelta SourceBufferStream::GetBufferedDuration() const {
  1487. if (ranges_.empty())
  1488. return base::TimeDelta();
  1489. return ranges_.back()->GetBufferedEndTimestamp();
  1490. }
  1491. size_t SourceBufferStream::GetBufferedSize() const {
  1492. size_t ranges_size = 0;
  1493. for (const auto& range_ptr : ranges_)
  1494. ranges_size += range_ptr->size_in_bytes();
  1495. return ranges_size;
  1496. }
  1497. void SourceBufferStream::MarkEndOfStream() {
  1498. DCHECK(!end_of_stream_);
  1499. end_of_stream_ = true;
  1500. }
  1501. void SourceBufferStream::UnmarkEndOfStream() {
  1502. DCHECK(end_of_stream_);
  1503. end_of_stream_ = false;
  1504. }
  1505. bool SourceBufferStream::IsEndOfStreamReached() const {
  1506. if (!end_of_stream_ || !track_buffer_.empty())
  1507. return false;
  1508. if (ranges_.empty())
  1509. return true;
  1510. if (seek_pending_) {
  1511. base::TimeDelta last_range_end_time =
  1512. ranges_.back()->GetBufferedEndTimestamp();
  1513. return seek_buffer_timestamp_ >= last_range_end_time;
  1514. }
  1515. if (!selected_range_)
  1516. return true;
  1517. return selected_range_ == ranges_.back().get();
  1518. }
  1519. const AudioDecoderConfig& SourceBufferStream::GetCurrentAudioDecoderConfig() {
  1520. if (config_change_pending_)
  1521. CompleteConfigChange();
  1522. // Trying to track down crash. http://crbug.com/715761
  1523. CHECK(current_config_index_ >= 0 &&
  1524. static_cast<size_t>(current_config_index_) < audio_configs_.size());
  1525. return audio_configs_[current_config_index_];
  1526. }
  1527. const VideoDecoderConfig& SourceBufferStream::GetCurrentVideoDecoderConfig() {
  1528. if (config_change_pending_)
  1529. CompleteConfigChange();
  1530. // Trying to track down crash. http://crbug.com/715761
  1531. CHECK(current_config_index_ >= 0 &&
  1532. static_cast<size_t>(current_config_index_) < video_configs_.size());
  1533. return video_configs_[current_config_index_];
  1534. }
  1535. const TextTrackConfig& SourceBufferStream::GetCurrentTextTrackConfig() {
  1536. return text_track_config_;
  1537. }
  1538. base::TimeDelta SourceBufferStream::GetMaxInterbufferDistance() const {
  1539. return max_interbuffer_distance_;
  1540. }
  1541. bool SourceBufferStream::UpdateAudioConfig(const AudioDecoderConfig& config,
  1542. bool allow_codec_change) {
  1543. DCHECK(!audio_configs_.empty());
  1544. DCHECK(video_configs_.empty());
  1545. DVLOG(3) << "UpdateAudioConfig.";
  1546. if (!allow_codec_change &&
  1547. audio_configs_[append_config_index_].codec() != config.codec()) {
  1548. // TODO(wolenetz): When we relax addSourceBuffer() and changeType() codec
  1549. // strictness, codec changes should be allowed even without changing the
  1550. // bytestream.
  1551. // TODO(wolenetz): Remove "experimental" from this error message when
  1552. // changeType() ships without needing experimental blink flag.
  1553. MEDIA_LOG(ERROR, media_log_) << "Audio codec changes not allowed unless "
  1554. "using experimental changeType().";
  1555. return false;
  1556. }
  1557. // Check to see if the new config matches an existing one.
  1558. for (size_t i = 0; i < audio_configs_.size(); ++i) {
  1559. if (config.Matches(audio_configs_[i])) {
  1560. append_config_index_ = i;
  1561. return true;
  1562. }
  1563. }
  1564. // No matches found so let's add this one to the list.
  1565. append_config_index_ = audio_configs_.size();
  1566. DVLOG(2) << "New audio config - index: " << append_config_index_;
  1567. audio_configs_.resize(audio_configs_.size() + 1);
  1568. audio_configs_[append_config_index_] = config;
  1569. return true;
  1570. }
  1571. bool SourceBufferStream::UpdateVideoConfig(const VideoDecoderConfig& config,
  1572. bool allow_codec_change) {
  1573. DCHECK(!video_configs_.empty());
  1574. DCHECK(audio_configs_.empty());
  1575. DVLOG(3) << "UpdateVideoConfig.";
  1576. if (!allow_codec_change &&
  1577. video_configs_[append_config_index_].codec() != config.codec()) {
  1578. // TODO(wolenetz): When we relax addSourceBuffer() and changeType() codec
  1579. // strictness, codec changes should be allowed even without changing the
  1580. // bytestream.
  1581. // TODO(wolenetz): Remove "experimental" from this error message when
  1582. // changeType() ships without needing experimental blink flag.
  1583. MEDIA_LOG(ERROR, media_log_) << "Video codec changes not allowed unless "
  1584. "using experimental changeType()";
  1585. return false;
  1586. }
  1587. // Check to see if the new config matches an existing one.
  1588. for (size_t i = 0; i < video_configs_.size(); ++i) {
  1589. if (config.Matches(video_configs_[i])) {
  1590. append_config_index_ = i;
  1591. return true;
  1592. }
  1593. }
  1594. // No matches found so let's add this one to the list.
  1595. append_config_index_ = video_configs_.size();
  1596. DVLOG(2) << "New video config - index: " << append_config_index_;
  1597. video_configs_.resize(video_configs_.size() + 1);
  1598. video_configs_[append_config_index_] = config;
  1599. return true;
  1600. }
  1601. void SourceBufferStream::CompleteConfigChange() {
  1602. config_change_pending_ = false;
  1603. if (!track_buffer_.empty()) {
  1604. current_config_index_ = track_buffer_.front()->GetConfigId();
  1605. return;
  1606. }
  1607. if (selected_range_ && selected_range_->HasNextBuffer())
  1608. current_config_index_ = selected_range_->GetNextConfigId();
  1609. }
  1610. void SourceBufferStream::SetSelectedRangeIfNeeded(
  1611. const base::TimeDelta timestamp) {
  1612. DVLOG(2) << __func__ << " " << GetStreamTypeName() << "("
  1613. << timestamp.InMicroseconds() << "us)";
  1614. if (selected_range_) {
  1615. DCHECK(track_buffer_.empty());
  1616. return;
  1617. }
  1618. if (!track_buffer_.empty()) {
  1619. DCHECK(!selected_range_);
  1620. return;
  1621. }
  1622. base::TimeDelta start_timestamp = timestamp;
  1623. // If the next buffer timestamp is not known then use a timestamp just after
  1624. // the timestamp on the last buffer returned by GetNextBuffer().
  1625. if (start_timestamp == kNoTimestamp) {
  1626. if (highest_output_buffer_timestamp_ == kNoTimestamp) {
  1627. DVLOG(2) << __func__ << " " << GetStreamTypeName()
  1628. << " no previous output timestamp";
  1629. return;
  1630. }
  1631. start_timestamp = highest_output_buffer_timestamp_ + base::Microseconds(1);
  1632. }
  1633. base::TimeDelta seek_timestamp =
  1634. FindNewSelectedRangeSeekTimestamp(start_timestamp);
  1635. // If we don't have buffered data to seek to, then return.
  1636. if (seek_timestamp == kNoTimestamp) {
  1637. DVLOG(2) << __func__ << " " << GetStreamTypeName()
  1638. << " couldn't find new selected range seek timestamp";
  1639. return;
  1640. }
  1641. DCHECK(track_buffer_.empty());
  1642. SeekAndSetSelectedRange(FindExistingRangeFor(seek_timestamp)->get(),
  1643. seek_timestamp);
  1644. }
  1645. base::TimeDelta SourceBufferStream::FindNewSelectedRangeSeekTimestamp(
  1646. const base::TimeDelta start_timestamp) {
  1647. DCHECK(start_timestamp != kNoTimestamp);
  1648. DCHECK(start_timestamp >= base::TimeDelta());
  1649. auto itr = ranges_.begin();
  1650. // When checking a range to see if it has or begins soon enough after
  1651. // |start_timestamp|, use the fudge room to determine "soon enough".
  1652. base::TimeDelta start_timestamp_plus_fudge =
  1653. start_timestamp + ComputeFudgeRoom(GetMaxInterbufferDistance());
  1654. // Multiple ranges could be within the fudge room, because the fudge room is
  1655. // dynamic based on max inter-buffer distance seen so far. Optimistically
  1656. // check the earliest ones first.
  1657. for (; itr != ranges_.end(); ++itr) {
  1658. base::TimeDelta range_start = (*itr)->GetStartTimestamp();
  1659. if (range_start >= start_timestamp_plus_fudge)
  1660. break;
  1661. if ((*itr)->GetEndTimestamp() < start_timestamp)
  1662. continue;
  1663. base::TimeDelta search_timestamp = start_timestamp;
  1664. if (start_timestamp < range_start &&
  1665. start_timestamp_plus_fudge >= range_start) {
  1666. search_timestamp = range_start;
  1667. }
  1668. base::TimeDelta keyframe_timestamp =
  1669. (*itr)->NextKeyframeTimestamp(search_timestamp);
  1670. if (keyframe_timestamp != kNoTimestamp)
  1671. return keyframe_timestamp;
  1672. }
  1673. DVLOG(2) << __func__ << " " << GetStreamTypeName()
  1674. << " no buffered data for pts=" << start_timestamp.InMicroseconds()
  1675. << "us";
  1676. return kNoTimestamp;
  1677. }
  1678. base::TimeDelta SourceBufferStream::FindKeyframeAfterTimestamp(
  1679. const base::TimeDelta timestamp) {
  1680. DCHECK(timestamp != kNoTimestamp);
  1681. auto itr = FindExistingRangeFor(timestamp);
  1682. if (itr == ranges_.end())
  1683. return kNoTimestamp;
  1684. // First check for a keyframe timestamp >= |timestamp|
  1685. // in the current range.
  1686. return (*itr)->NextKeyframeTimestamp(timestamp);
  1687. }
  1688. std::string SourceBufferStream::GetStreamTypeName() const {
  1689. switch (GetType()) {
  1690. case SourceBufferStreamType::kAudio:
  1691. return "AUDIO";
  1692. case SourceBufferStreamType::kVideo:
  1693. return "VIDEO";
  1694. case SourceBufferStreamType::kText:
  1695. return "TEXT";
  1696. }
  1697. NOTREACHED();
  1698. return "";
  1699. }
  1700. SourceBufferStreamType SourceBufferStream::GetType() const {
  1701. if (!audio_configs_.empty())
  1702. return SourceBufferStreamType::kAudio;
  1703. if (!video_configs_.empty())
  1704. return SourceBufferStreamType::kVideo;
  1705. DCHECK_NE(text_track_config_.kind(), kTextNone);
  1706. return SourceBufferStreamType::kText;
  1707. }
  1708. void SourceBufferStream::DeleteAndRemoveRange(RangeList::iterator* itr) {
  1709. DVLOG(1) << __func__;
  1710. DCHECK(*itr != ranges_.end());
  1711. if ((*itr)->get() == selected_range_) {
  1712. DVLOG(1) << __func__ << " deleting selected range.";
  1713. SetSelectedRange(NULL);
  1714. }
  1715. if (*itr == range_for_next_append_) {
  1716. DVLOG(1) << __func__ << " deleting range_for_next_append_.";
  1717. range_for_next_append_ = ranges_.end();
  1718. ResetLastAppendedState();
  1719. }
  1720. *itr = ranges_.erase(*itr);
  1721. }
  1722. bool SourceBufferStream::SetPendingBuffer(
  1723. scoped_refptr<StreamParserBuffer>* out_buffer) {
  1724. DCHECK(out_buffer->get());
  1725. DCHECK(!pending_buffer_.get());
  1726. const bool have_preroll_buffer = !!(*out_buffer)->preroll_buffer().get();
  1727. if (!have_preroll_buffer)
  1728. return false;
  1729. pending_buffer_.swap(*out_buffer);
  1730. pending_buffers_complete_ = false;
  1731. return true;
  1732. }
  1733. } // namespace media