vp9_svc_layers.cc 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392
  1. // Copyright 2020 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/gpu/vp9_svc_layers.h"
  5. #include <bitset>
  6. #include "base/logging.h"
  7. #include "media/gpu/macros.h"
  8. #include "media/gpu/vp9_picture.h"
  9. namespace media {
  10. namespace {
  11. static_assert(VideoBitrateAllocation::kMaxTemporalLayers >=
  12. VP9SVCLayers::kMaxSupportedTemporalLayers,
  13. "VP9SVCLayers and VideoBitrateAllocation are dimensionally "
  14. "inconsistent.");
  15. static_assert(VideoBitrateAllocation::kMaxSpatialLayers >=
  16. VP9SVCLayers::kMaxSpatialLayers,
  17. "VP9SVCLayers and VideoBitrateAllocation are dimensionally "
  18. "inconsistent.");
  19. enum FrameFlags : uint8_t {
  20. kNone = 0,
  21. kReference = 1,
  22. kUpdate = 2,
  23. kReferenceAndUpdate = kReference | kUpdate,
  24. };
  25. } // namespace
  26. struct VP9SVCLayers::FrameConfig {
  27. FrameConfig(size_t layer_index,
  28. FrameFlags first,
  29. FrameFlags second,
  30. bool temporal_up_switch)
  31. : layer_index_(layer_index),
  32. buffer_flags_{first, second},
  33. temporal_up_switch_(temporal_up_switch) {}
  34. FrameConfig() = delete;
  35. // VP9SVCLayers uses 2 reference frame slots for each spatial layer, and
  36. // totally uses up to 6 reference frame slots. SL0 uses the first two (0, 1)
  37. // slots, SL1 uses middle two (2, 3) slots, and SL2 uses last two (4, 5)
  38. // slots.
  39. std::vector<uint8_t> GetRefFrameIndices(size_t spatial_idx,
  40. size_t frame_num) const {
  41. std::vector<uint8_t> indices;
  42. if (frame_num != 0) {
  43. for (size_t i = 0; i < kMaxNumUsedRefFramesEachSpatialLayer; ++i) {
  44. if (buffer_flags_[i] & FrameFlags::kReference) {
  45. indices.push_back(i +
  46. kMaxNumUsedRefFramesEachSpatialLayer * spatial_idx);
  47. }
  48. }
  49. } else {
  50. // For the key picture (|frame_num| equals 0), the higher spatial layer
  51. // reference the lower spatial layers. e.g. for frame_num 0, SL1 will
  52. // reference SL0, and SL2 will reference SL1.
  53. DCHECK_GT(spatial_idx, 0u);
  54. indices.push_back((spatial_idx - 1) *
  55. kMaxNumUsedRefFramesEachSpatialLayer);
  56. }
  57. return indices;
  58. }
  59. std::vector<uint8_t> GetUpdateIndices(size_t spatial_idx) const {
  60. std::vector<uint8_t> indices;
  61. for (size_t i = 0; i < kMaxNumUsedRefFramesEachSpatialLayer; ++i) {
  62. if (buffer_flags_[i] & FrameFlags::kUpdate) {
  63. indices.push_back(i +
  64. kMaxNumUsedRefFramesEachSpatialLayer * spatial_idx);
  65. }
  66. }
  67. return indices;
  68. }
  69. size_t layer_index() const { return layer_index_; }
  70. bool temporal_up_switch() const { return temporal_up_switch_; }
  71. private:
  72. const size_t layer_index_;
  73. const FrameFlags buffer_flags_[kMaxNumUsedRefFramesEachSpatialLayer];
  74. const bool temporal_up_switch_;
  75. };
  76. namespace {
  77. // GetTemporalLayersReferencePattern() constructs the
  78. // following temporal layers.
  79. std::vector<VP9SVCLayers::FrameConfig> GetTemporalLayersReferencePattern(
  80. size_t num_temporal_layers) {
  81. using FrameConfig = VP9SVCLayers::FrameConfig;
  82. switch (num_temporal_layers) {
  83. case 1:
  84. // In this case, the number of spatial layers must great than 1.
  85. // TL0 references and updates the 'first' buffer.
  86. // [TL0]---[TL0]
  87. return {FrameConfig(0, kReferenceAndUpdate, kNone, true)};
  88. case 2:
  89. // TL0 references and updates the 'first' buffer.
  90. // TL1 references 'first' buffer.
  91. // [TL1]
  92. // /
  93. // [TL0]-----[TL0]
  94. return {FrameConfig(0, kReferenceAndUpdate, kNone, true),
  95. FrameConfig(1, kReference, kNone, true)};
  96. case 3:
  97. // TL0 references and updates the 'first' buffer.
  98. // TL1 references 'first' and updates 'second'.
  99. // TL2 references either 'first' or 'second' buffer.
  100. // [TL2] [TL2]
  101. // _/ [TL1]--/
  102. // /_______/
  103. // [TL0]--------------[TL0]
  104. return {FrameConfig(0, kReferenceAndUpdate, kNone, true),
  105. FrameConfig(2, kReference, kNone, true),
  106. FrameConfig(1, kReference, kUpdate, true),
  107. FrameConfig(2, kNone, kReference, true)};
  108. default:
  109. NOTREACHED();
  110. return {};
  111. }
  112. }
  113. } // namespace
  114. VP9SVCLayers::VP9SVCLayers(const std::vector<SpatialLayer>& spatial_layers)
  115. : num_temporal_layers_(spatial_layers[0].num_of_temporal_layers),
  116. temporal_layers_reference_pattern_(
  117. GetTemporalLayersReferencePattern(num_temporal_layers_)),
  118. pattern_index_(0u),
  119. temporal_pattern_size_(temporal_layers_reference_pattern_.size()) {
  120. for (const auto spatial_layer : spatial_layers) {
  121. spatial_layer_resolutions_.emplace_back(
  122. gfx::Size(spatial_layer.width, spatial_layer.height));
  123. }
  124. active_spatial_layer_resolutions_ = spatial_layer_resolutions_;
  125. begin_active_layer_ = 0;
  126. end_active_layer_ = active_spatial_layer_resolutions_.size();
  127. DCHECK_LE(num_temporal_layers_, kMaxSupportedTemporalLayers);
  128. DCHECK(!spatial_layer_resolutions_.empty());
  129. DCHECK_LE(spatial_layer_resolutions_.size(), kMaxSpatialLayers);
  130. }
  131. VP9SVCLayers::~VP9SVCLayers() = default;
  132. bool VP9SVCLayers::UpdateEncodeJob(bool is_key_frame_requested,
  133. size_t kf_period_frames) {
  134. if (force_key_frame_ || is_key_frame_requested) {
  135. frame_num_ = 0;
  136. spatial_idx_ = 0;
  137. force_key_frame_ = false;
  138. }
  139. if (spatial_idx_ == active_spatial_layer_resolutions_.size()) {
  140. frame_num_++;
  141. frame_num_ %= kf_period_frames;
  142. spatial_idx_ = 0;
  143. }
  144. return frame_num_ == 0 && spatial_idx_ == 0;
  145. }
  146. bool VP9SVCLayers::MaybeUpdateActiveLayer(
  147. VideoBitrateAllocation* bitrate_allocation) {
  148. // Don't update active layer if current picture haven't completed SVC
  149. // encoding. Since the |spatial_idx_| is updated in the beginning of next
  150. // encoding, so the |spatial_idx_| equals 0 (only for the first frame) or the
  151. // number of active spatial layers indicates the complement of SVC picture
  152. // encoding.
  153. if (spatial_idx_ != 0 &&
  154. spatial_idx_ != active_spatial_layer_resolutions_.size()) {
  155. return false;
  156. }
  157. size_t begin_active_layer = kMaxSpatialLayers;
  158. size_t end_active_layer = spatial_layer_resolutions_.size();
  159. for (size_t sid = 0; sid < spatial_layer_resolutions_.size(); ++sid) {
  160. size_t sum = 0;
  161. for (size_t tid = 0; tid < num_temporal_layers_; ++tid) {
  162. const uint32_t tl_bitrate = bitrate_allocation->GetBitrateBps(sid, tid);
  163. // A bitrate of a temporal layer must be zero if the bitrates of lower
  164. // temporal layers are zero, e.g. {0, 0, 100}.
  165. if (tid > 0 && tl_bitrate > 0 && sum == 0)
  166. return false;
  167. // A bitrate of a temporal layer must not be zero if the bitrates of lower
  168. // temporal layers are not zero, e.g. {100, 0, 0}.
  169. if (tid > 0 && tl_bitrate == 0 && sum != 0)
  170. return false;
  171. sum += static_cast<size_t>(tl_bitrate);
  172. }
  173. // Check if the temporal layers larger than |num_temporal_layers_| are zero.
  174. for (size_t tid = num_temporal_layers_;
  175. tid < VideoBitrateAllocation::kMaxTemporalLayers; ++tid) {
  176. if (bitrate_allocation->GetBitrateBps(sid, tid) != 0u)
  177. return false;
  178. }
  179. if (sum == 0) {
  180. // This is the first non-active spatial layer in the end side.
  181. if (begin_active_layer != kMaxSpatialLayers) {
  182. end_active_layer = sid;
  183. break;
  184. }
  185. // No active spatial layer is found yet. Try the upper spatial layer.
  186. continue;
  187. }
  188. // This is the lowest active layer.
  189. if (begin_active_layer == kMaxSpatialLayers)
  190. begin_active_layer = sid;
  191. }
  192. // Check if all the bitrates of unsupported temporal and spatial layers are
  193. // zero.
  194. for (size_t sid = end_active_layer;
  195. sid < VideoBitrateAllocation::kMaxSpatialLayers; ++sid) {
  196. for (size_t tid = 0; tid < VideoBitrateAllocation::kMaxTemporalLayers;
  197. ++tid) {
  198. if (bitrate_allocation->GetBitrateBps(sid, tid) != 0u)
  199. return false;
  200. }
  201. }
  202. // No active layer is found.
  203. if (begin_active_layer == kMaxSpatialLayers)
  204. return false;
  205. DCHECK_LT(begin_active_layer_, end_active_layer_);
  206. DCHECK_LE(end_active_layer_ - begin_active_layer_,
  207. spatial_layer_resolutions_.size());
  208. // Remove non active spatial layer bitrate if |begin_active_layer| > 0.
  209. if (begin_active_layer > 0) {
  210. for (size_t sid = begin_active_layer; sid < end_active_layer; ++sid) {
  211. for (size_t tid = 0; tid < num_temporal_layers_; ++tid) {
  212. uint32_t bitrate = bitrate_allocation->GetBitrateBps(sid, tid);
  213. bitrate_allocation->SetBitrate(sid - begin_active_layer, tid, bitrate);
  214. bitrate_allocation->SetBitrate(sid, tid, 0u);
  215. }
  216. }
  217. }
  218. // Reset SVC parameters and force to produce key frame if active layer
  219. // changed.
  220. if (begin_active_layer != begin_active_layer_ ||
  221. end_active_layer != end_active_layer_) {
  222. // Update the stored active layer range.
  223. begin_active_layer_ = begin_active_layer;
  224. end_active_layer_ = end_active_layer;
  225. active_spatial_layer_resolutions_ = {
  226. spatial_layer_resolutions_.begin() + begin_active_layer,
  227. spatial_layer_resolutions_.begin() + end_active_layer};
  228. force_key_frame_ = true;
  229. }
  230. return true;
  231. }
  232. void VP9SVCLayers::FillUsedRefFramesAndMetadata(
  233. VP9Picture* picture,
  234. std::array<bool, kVp9NumRefsPerFrame>* ref_frames_used) {
  235. DCHECK(picture->frame_hdr);
  236. // Update the spatial layer size for VP9FrameHeader.
  237. gfx::Size updated_size = active_spatial_layer_resolutions_[spatial_idx_];
  238. picture->frame_hdr->render_width = updated_size.width();
  239. picture->frame_hdr->render_height = updated_size.height();
  240. picture->frame_hdr->frame_width = updated_size.width();
  241. picture->frame_hdr->frame_height = updated_size.height();
  242. // Initialize |metadata_for_encoding| with default values.
  243. picture->metadata_for_encoding.emplace();
  244. ref_frames_used->fill(false);
  245. if (picture->frame_hdr->IsKeyframe()) {
  246. DCHECK_EQ(spatial_idx_, 0u);
  247. DCHECK_EQ(frame_num_, 0u);
  248. picture->frame_hdr->refresh_frame_flags = 0xff;
  249. // Start the pattern over from 0 and reset the buffer refresh states.
  250. pattern_index_ = 0;
  251. // For key frame, its temporal_layers_config is (0, kUpdate, kUpdate), so
  252. // its reference_frame_indices is empty, and refresh_frame_indices is {0, 1}
  253. FillVp9MetadataForEncoding(&(*picture->metadata_for_encoding),
  254. /*reference_frame_indices=*/{});
  255. UpdateRefFramesPatternIndex(/*refresh_frame_indices=*/{0, 1});
  256. picture->metadata_for_encoding->temporal_up_switch = true;
  257. DVLOGF(4)
  258. << "Frame num: " << frame_num_
  259. << ", key frame: " << picture->frame_hdr->IsKeyframe()
  260. << ", spatial_idx: " << spatial_idx_ << ", temporal_idx: "
  261. << temporal_layers_reference_pattern_[pattern_index_].layer_index()
  262. << ", pattern index: " << static_cast<int>(pattern_index_)
  263. << ", refresh_frame_flags: "
  264. << std::bitset<8>(picture->frame_hdr->refresh_frame_flags);
  265. spatial_idx_++;
  266. return;
  267. }
  268. if (spatial_idx_ == 0)
  269. pattern_index_ = (pattern_index_ + 1) % temporal_pattern_size_;
  270. const VP9SVCLayers::FrameConfig& temporal_layers_config =
  271. temporal_layers_reference_pattern_[pattern_index_];
  272. // Set the slots in reference frame pool that will be updated.
  273. const std::vector<uint8_t> refresh_frame_indices =
  274. temporal_layers_config.GetUpdateIndices(spatial_idx_);
  275. for (const uint8_t i : refresh_frame_indices)
  276. picture->frame_hdr->refresh_frame_flags |= 1u << i;
  277. // Set the slots of reference frames used for the current frame.
  278. const std::vector<uint8_t> reference_frame_indices =
  279. temporal_layers_config.GetRefFrameIndices(spatial_idx_, frame_num_);
  280. uint8_t ref_flags = 0;
  281. for (size_t i = 0; i < reference_frame_indices.size(); i++) {
  282. (*ref_frames_used)[i] = true;
  283. picture->frame_hdr->ref_frame_idx[i] = reference_frame_indices[i];
  284. ref_flags |= 1 << reference_frame_indices[i];
  285. }
  286. DVLOGF(4) << "Frame num: " << frame_num_
  287. << ", key frame: " << picture->frame_hdr->IsKeyframe()
  288. << ", spatial_idx: " << spatial_idx_ << ", temporal_idx: "
  289. << temporal_layers_reference_pattern_[pattern_index_].layer_index()
  290. << ", pattern index: " << static_cast<int>(pattern_index_)
  291. << ", refresh_frame_flags: "
  292. << std::bitset<8>(picture->frame_hdr->refresh_frame_flags)
  293. << " reference buffers: " << std::bitset<8>(ref_flags);
  294. FillVp9MetadataForEncoding(&(*picture->metadata_for_encoding),
  295. reference_frame_indices);
  296. UpdateRefFramesPatternIndex(refresh_frame_indices);
  297. picture->metadata_for_encoding->temporal_up_switch =
  298. temporal_layers_config.temporal_up_switch();
  299. spatial_idx_++;
  300. }
  301. void VP9SVCLayers::FillVp9MetadataForEncoding(
  302. Vp9Metadata* metadata,
  303. const std::vector<uint8_t>& reference_frame_indices) const {
  304. metadata->end_of_picture =
  305. spatial_idx_ == active_spatial_layer_resolutions_.size() - 1;
  306. metadata->referenced_by_upper_spatial_layers =
  307. frame_num_ == 0 &&
  308. spatial_idx_ < active_spatial_layer_resolutions_.size() - 1;
  309. // |spatial_layer_resolutions| has to be filled if and only if keyframe or the
  310. // number of active spatial layers is changed. However, we fill in the case of
  311. // keyframe, this works because if the number of active spatial layers is
  312. // changed, keyframe is requested.
  313. if (frame_num_ == 0 && spatial_idx_ == 0) {
  314. metadata->spatial_layer_resolutions = active_spatial_layer_resolutions_;
  315. return;
  316. }
  317. // Below parameters only needed to filled for non key frame.
  318. uint8_t temp_temporal_layers_id =
  319. temporal_layers_reference_pattern_[pattern_index_ %
  320. temporal_pattern_size_]
  321. .layer_index();
  322. // If |frame_num_| is zero, it refers only lower spatial layer.
  323. // |inter_pic_predicted| is true if a frame in the same spatial layer is
  324. // referred.
  325. if (frame_num_ != 0)
  326. metadata->inter_pic_predicted = !reference_frame_indices.empty();
  327. metadata->reference_lower_spatial_layers =
  328. frame_num_ == 0 && (spatial_idx_ != 0);
  329. metadata->temporal_idx = temp_temporal_layers_id;
  330. metadata->spatial_idx = spatial_idx_;
  331. for (const uint8_t i : reference_frame_indices) {
  332. // If |frame_num_| is zero, it refers only lower spatial layer, there is no
  333. // need to fill |p_diff|.
  334. if (frame_num_ != 0) {
  335. uint8_t p_diff = (pattern_index_ - pattern_index_of_ref_frames_slots_[i] +
  336. temporal_pattern_size_) %
  337. temporal_pattern_size_;
  338. // For non-key picture, its |p_diff| must large than 0.
  339. if (p_diff == 0)
  340. p_diff = temporal_pattern_size_;
  341. metadata->p_diffs.push_back(p_diff);
  342. }
  343. }
  344. }
  345. // Use current pattern index to update the reference frame's pattern index,
  346. // this is used to calculate |p_diffs|.
  347. void VP9SVCLayers::UpdateRefFramesPatternIndex(
  348. const std::vector<uint8_t>& refresh_frame_indices) {
  349. for (const uint8_t i : refresh_frame_indices)
  350. pattern_index_of_ref_frames_slots_[i] = pattern_index_;
  351. }
  352. } // namespace media