vp9_svc_layers_unittest.cc 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318
  1. // Copyright 2020 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/gpu/vp9_svc_layers.h"
  5. #include <algorithm>
  6. #include <array>
  7. #include <map>
  8. #include <vector>
  9. #include "base/containers/contains.h"
  10. #include "media/filters/vp9_parser.h"
  11. #include "media/gpu/vp9_picture.h"
  12. #include "media/gpu/vp9_reference_frame_vector.h"
  13. #include "media/video/video_encode_accelerator.h"
  14. #include "testing/gtest/include/gtest/gtest.h"
  15. #include "third_party/abseil-cpp/absl/types/optional.h"
  16. namespace media {
  17. namespace {
  18. constexpr gfx::Size kDefaultEncodeSize(1280, 720);
  19. constexpr int kSpatialLayerResolutionDenom[] = {4, 2, 1};
  20. std::vector<VP9SVCLayers::SpatialLayer> GetDefaultSVCLayers(
  21. size_t num_spatial_layers,
  22. size_t num_temporal_layers) {
  23. std::vector<VP9SVCLayers::SpatialLayer> spatial_layers;
  24. for (uint8_t i = 0; i < num_spatial_layers; ++i) {
  25. VP9SVCLayers::SpatialLayer spatial_layer;
  26. const int denom = kSpatialLayerResolutionDenom[i];
  27. spatial_layer.width = kDefaultEncodeSize.width() / denom;
  28. spatial_layer.height = kDefaultEncodeSize.height() / denom;
  29. spatial_layer.num_of_temporal_layers = num_temporal_layers;
  30. spatial_layers.push_back(spatial_layer);
  31. }
  32. return spatial_layers;
  33. }
  34. std::vector<gfx::Size> GetDefaultSVCResolutions(size_t num_spatial_layers) {
  35. std::vector<gfx::Size> spatial_layer_resolutions;
  36. for (size_t i = 0; i < num_spatial_layers; ++i) {
  37. const int denom = kSpatialLayerResolutionDenom[i];
  38. spatial_layer_resolutions.emplace_back(
  39. gfx::Size(kDefaultEncodeSize.width() / denom,
  40. kDefaultEncodeSize.height() / denom));
  41. }
  42. return spatial_layer_resolutions;
  43. }
  44. } // namespace
  45. class VP9SVCLayersTest
  46. : public ::testing::TestWithParam<::testing::tuple<size_t, size_t>> {
  47. public:
  48. VP9SVCLayersTest() = default;
  49. ~VP9SVCLayersTest() = default;
  50. protected:
  51. void VerifyRefFrames(
  52. const Vp9FrameHeader& frame_hdr,
  53. const Vp9Metadata& metadata,
  54. const std::array<bool, kVp9NumRefsPerFrame>& ref_frames_used,
  55. const Vp9ReferenceFrameVector& ref_frames,
  56. const size_t num_spatial_layers,
  57. const bool key_pic);
  58. void VerifySVCStructure(bool keyframe,
  59. size_t num_temporal_layers,
  60. size_t num_spatial_layers,
  61. const Vp9Metadata& metadata);
  62. private:
  63. std::vector<uint8_t> temporal_indices_[VP9SVCLayers::kMaxSpatialLayers];
  64. uint8_t spatial_index_;
  65. };
  66. void VP9SVCLayersTest::VerifySVCStructure(bool key_pic,
  67. size_t num_temporal_layers,
  68. size_t num_spatial_layers,
  69. const Vp9Metadata& metadata) {
  70. const uint8_t temporal_index = metadata.temporal_idx;
  71. const uint8_t spatial_index = metadata.spatial_idx;
  72. // Spatial index monotonically increases modulo |num_spatial_layers|.
  73. if (!key_pic || spatial_index != 0u)
  74. EXPECT_EQ((spatial_index_ + 1) % num_spatial_layers, spatial_index);
  75. spatial_index_ = spatial_index;
  76. auto& temporal_indices = temporal_indices_[spatial_index];
  77. if (key_pic) {
  78. temporal_indices.clear();
  79. temporal_indices.push_back(temporal_index);
  80. return;
  81. }
  82. EXPECT_FALSE(temporal_indices.empty());
  83. if (num_temporal_layers > 1)
  84. EXPECT_NE(temporal_indices.back(), temporal_index);
  85. else
  86. EXPECT_EQ(temporal_indices.back(), temporal_index);
  87. temporal_indices.push_back(temporal_index);
  88. if (spatial_index != num_spatial_layers - 1)
  89. return;
  90. // Check if the temporal layer structures in all spatial layers are identical.
  91. // Spatial index monotonically increases module |num_spatial_layers|.
  92. for (size_t i = 0; i < num_spatial_layers - 1; ++i)
  93. EXPECT_EQ(temporal_indices_[i], temporal_indices_[i + 1]);
  94. constexpr size_t kTemporalLayerCycle = 4u;
  95. constexpr size_t kSVCLayerCycle = kTemporalLayerCycle;
  96. if (temporal_indices.size() % kSVCLayerCycle == 0) {
  97. std::vector<size_t> count(num_temporal_layers, 0u);
  98. for (const uint8_t index : temporal_indices) {
  99. ASSERT_LE(index, num_temporal_layers);
  100. count[index]++;
  101. }
  102. // The number of frames in a higher temporal layer is not less than one in a
  103. // lower temporal layer.
  104. EXPECT_TRUE(std::is_sorted(count.begin(), count.end()));
  105. }
  106. }
  107. void VP9SVCLayersTest::VerifyRefFrames(
  108. const Vp9FrameHeader& frame_hdr,
  109. const Vp9Metadata& metadata,
  110. const std::array<bool, kVp9NumRefsPerFrame>& ref_frames_used,
  111. const Vp9ReferenceFrameVector& ref_frames,
  112. const size_t num_spatial_layers,
  113. const bool key_pic) {
  114. const uint8_t temporal_index = metadata.temporal_idx;
  115. const uint8_t spatial_index = metadata.spatial_idx;
  116. if (frame_hdr.IsKeyframe()) {
  117. EXPECT_EQ(frame_hdr.refresh_frame_flags, 0xff);
  118. EXPECT_FALSE(base::Contains(ref_frames_used, true));
  119. EXPECT_EQ(temporal_index, 0u);
  120. EXPECT_EQ(spatial_index, 0u);
  121. EXPECT_EQ(metadata.referenced_by_upper_spatial_layers,
  122. num_spatial_layers > 1);
  123. EXPECT_FALSE(metadata.reference_lower_spatial_layers);
  124. EXPECT_TRUE(metadata.p_diffs.empty());
  125. EXPECT_EQ(metadata.spatial_layer_resolutions,
  126. GetDefaultSVCResolutions(num_spatial_layers));
  127. return;
  128. }
  129. // Six slots at most in the reference pool are used in spatial/temporal layer
  130. // encoding. Additionally, non-keyframe must reference some frames.
  131. // |ref_frames_used| must be {true, false, false} because here is,
  132. // 1. if the frame is in key picture, it references one lower spatial layer,
  133. // 2. otherwise the frame doesn't reference other spatial layers and thus
  134. // references only one frame in the same spatial layer based on the current
  135. // reference pattern.
  136. constexpr std::array<bool, kVp9NumRefsPerFrame> kExpectedRefFramesUsed = {
  137. true, false, false};
  138. EXPECT_EQ(frame_hdr.refresh_frame_flags & ~(0b111111u), 0u);
  139. EXPECT_EQ(ref_frames_used, kExpectedRefFramesUsed);
  140. EXPECT_EQ(metadata.inter_pic_predicted, !metadata.p_diffs.empty());
  141. EXPECT_EQ(metadata.inter_pic_predicted, !key_pic);
  142. if (key_pic) {
  143. EXPECT_TRUE(metadata.reference_lower_spatial_layers);
  144. EXPECT_EQ(metadata.referenced_by_upper_spatial_layers,
  145. spatial_index + 1 != num_spatial_layers);
  146. } else {
  147. EXPECT_FALSE(metadata.referenced_by_upper_spatial_layers);
  148. EXPECT_FALSE(metadata.reference_lower_spatial_layers);
  149. }
  150. // Check that the current frame doesn't reference upper layer frames.
  151. const uint8_t index = frame_hdr.ref_frame_idx[0];
  152. scoped_refptr<VP9Picture> ref_frame = ref_frames.GetFrame(index);
  153. ASSERT_TRUE(!!ref_frame);
  154. const auto& ref_metadata = ref_frame->metadata_for_encoding;
  155. ASSERT_TRUE(ref_metadata.has_value());
  156. const size_t ref_temporal_index = ref_metadata->temporal_idx;
  157. EXPECT_LE(ref_temporal_index, temporal_index);
  158. const uint8_t ref_spatial_index = ref_metadata->spatial_idx;
  159. EXPECT_LE(ref_spatial_index, spatial_index);
  160. // In key picture, upper spatial layers must refer the lower spatial layer.
  161. // Or referenced frames must be in the same spatial layer.
  162. if (key_pic)
  163. EXPECT_EQ(ref_spatial_index, spatial_index - 1);
  164. else
  165. EXPECT_EQ(ref_spatial_index, spatial_index);
  166. }
  167. // This test verifies the bitrate check in MaybeUpdateActiveLayer().
  168. TEST_F(VP9SVCLayersTest, MaybeUpdateActiveLayer) {
  169. constexpr size_t kNumSpatialLayers = 3;
  170. constexpr size_t kNumTemporalLayers = 3;
  171. const std::vector<VP9SVCLayers::SpatialLayer> spatial_layers =
  172. GetDefaultSVCLayers(kNumSpatialLayers, kNumTemporalLayers);
  173. VP9SVCLayers svc_layers(spatial_layers);
  174. // Set Default bitrate allocation.
  175. uint32_t layer_rate = 1u;
  176. VideoBitrateAllocation allocation;
  177. for (size_t sid = 0; sid < VideoBitrateAllocation::kMaxSpatialLayers; ++sid) {
  178. for (size_t tid = 0; tid < VideoBitrateAllocation::kMaxTemporalLayers;
  179. ++tid) {
  180. allocation.SetBitrate(sid, tid, layer_rate++);
  181. }
  182. }
  183. DCHECK_LT(kNumSpatialLayers, VideoBitrateAllocation::kMaxSpatialLayers);
  184. DCHECK_LT(kNumTemporalLayers, VideoBitrateAllocation::kMaxTemporalLayers);
  185. EXPECT_FALSE(svc_layers.MaybeUpdateActiveLayer(&allocation));
  186. // Set unsupported temporal layer bitrate to 0.
  187. for (size_t sid = 0; sid < VideoBitrateAllocation::kMaxSpatialLayers; ++sid) {
  188. for (size_t tid = kNumTemporalLayers;
  189. tid < VideoBitrateAllocation::kMaxTemporalLayers; ++tid) {
  190. allocation.SetBitrate(sid, tid, 0u);
  191. }
  192. }
  193. EXPECT_FALSE(svc_layers.MaybeUpdateActiveLayer(&allocation));
  194. // Set unsupported spatial layer bitrate to 0.
  195. for (size_t sid = kNumSpatialLayers;
  196. sid < VideoBitrateAllocation::kMaxSpatialLayers; ++sid) {
  197. for (size_t tid = 0; tid < VideoBitrateAllocation::kMaxTemporalLayers;
  198. ++tid) {
  199. allocation.SetBitrate(sid, tid, 0u);
  200. }
  201. }
  202. EXPECT_TRUE(svc_layers.MaybeUpdateActiveLayer(&allocation));
  203. // Set lower temporal layer bitrate to zero, e.g. {0, 2, 3}.
  204. allocation.SetBitrate(/*spatial_index=*/0, /*temporal_index=*/0, 0u);
  205. EXPECT_FALSE(svc_layers.MaybeUpdateActiveLayer(&allocation));
  206. allocation.SetBitrate(/*spatial_index=*/0, /*temporal_index=*/0, 1u);
  207. // Set upper temporal layer bitrate to 0, e.g. {1, 2, 0}.
  208. allocation.SetBitrate(/*spatial_index=*/0, /*temporal_index=*/2, 0u);
  209. EXPECT_FALSE(svc_layers.MaybeUpdateActiveLayer(&allocation));
  210. allocation.SetBitrate(/*spatial_index=*/0, /*temporal_index=*/2, 3u);
  211. // Deactivate SL0 and SL1 and verify the new bitrate allocation.
  212. constexpr int kNumDeactivatedLowerSpatialLayer = 2;
  213. VideoBitrateAllocation new_allocation = allocation;
  214. for (size_t sid = 0; sid < kNumDeactivatedLowerSpatialLayer; ++sid) {
  215. for (size_t tid = 0; tid < kNumTemporalLayers; ++tid)
  216. new_allocation.SetBitrate(sid, tid, 0u);
  217. }
  218. EXPECT_TRUE(svc_layers.MaybeUpdateActiveLayer(&new_allocation));
  219. for (size_t sid = 0; sid < kNumSpatialLayers; ++sid) {
  220. for (size_t tid = 0; tid < kNumTemporalLayers; ++tid) {
  221. if (sid + kNumDeactivatedLowerSpatialLayer <
  222. VideoBitrateAllocation::kMaxSpatialLayers)
  223. EXPECT_EQ(new_allocation.GetBitrateBps(sid, tid),
  224. allocation.GetBitrateBps(
  225. sid + kNumDeactivatedLowerSpatialLayer, tid));
  226. else
  227. EXPECT_EQ(new_allocation.GetBitrateBps(sid, tid), 0u);
  228. }
  229. }
  230. // Deactivate SL2 and verify the new bitrate allocation.
  231. new_allocation = allocation;
  232. constexpr int kNumActiveSpatialLayer = 2;
  233. for (size_t tid = 0; tid < kNumTemporalLayers; ++tid)
  234. new_allocation.SetBitrate(/*spatial_index=*/2, tid, 0u);
  235. EXPECT_TRUE(svc_layers.MaybeUpdateActiveLayer(&new_allocation));
  236. for (size_t sid = 0; sid < kNumSpatialLayers; ++sid) {
  237. for (size_t tid = 0; tid < kNumTemporalLayers; ++tid) {
  238. if (sid < kNumActiveSpatialLayer)
  239. EXPECT_EQ(new_allocation.GetBitrateBps(sid, tid),
  240. allocation.GetBitrateBps(sid, tid));
  241. else
  242. EXPECT_EQ(new_allocation.GetBitrateBps(sid, tid), 0u);
  243. }
  244. }
  245. }
  246. TEST_P(VP9SVCLayersTest, ) {
  247. const size_t num_spatial_layers = ::testing::get<0>(GetParam());
  248. const size_t num_temporal_layers = ::testing::get<1>(GetParam());
  249. const std::vector<VP9SVCLayers::SpatialLayer> spatial_layers =
  250. GetDefaultSVCLayers(num_spatial_layers, num_temporal_layers);
  251. VP9SVCLayers svc_layers(spatial_layers);
  252. constexpr size_t kNumFramesToEncode = 32;
  253. Vp9ReferenceFrameVector ref_frames;
  254. constexpr size_t kKeyFrameInterval = 10;
  255. for (size_t frame_num = 0; frame_num < kNumFramesToEncode; ++frame_num) {
  256. // True iff the picture in the bottom spatial layer is key frame.
  257. bool key_pic = false;
  258. for (size_t sid = 0; sid < num_spatial_layers; ++sid) {
  259. scoped_refptr<VP9Picture> picture(new VP9Picture);
  260. picture->frame_hdr = std::make_unique<Vp9FrameHeader>();
  261. const bool keyframe = svc_layers.UpdateEncodeJob(
  262. /*is_key_frame_requested=*/false, kKeyFrameInterval);
  263. picture->frame_hdr->frame_type =
  264. keyframe ? Vp9FrameHeader::KEYFRAME : Vp9FrameHeader::INTERFRAME;
  265. if (sid == 0)
  266. key_pic = keyframe;
  267. std::array<bool, kVp9NumRefsPerFrame> ref_frames_used;
  268. svc_layers.FillUsedRefFramesAndMetadata(picture.get(), &ref_frames_used);
  269. ASSERT_TRUE(picture->metadata_for_encoding.has_value());
  270. VerifyRefFrames(*picture->frame_hdr, *picture->metadata_for_encoding,
  271. ref_frames_used, ref_frames, num_spatial_layers, key_pic);
  272. VerifySVCStructure(key_pic, num_temporal_layers, num_spatial_layers,
  273. *picture->metadata_for_encoding);
  274. ref_frames.Refresh(picture);
  275. }
  276. }
  277. }
  278. // std::make_tuple(num_spatial_layers, num_temporal_layers)
  279. INSTANTIATE_TEST_SUITE_P(,
  280. VP9SVCLayersTest,
  281. ::testing::Values(std::make_tuple(1, 2),
  282. std::make_tuple(1, 3),
  283. std::make_tuple(2, 1),
  284. std::make_tuple(2, 2),
  285. std::make_tuple(2, 3),
  286. std::make_tuple(3, 1),
  287. std::make_tuple(3, 2),
  288. std::make_tuple(3, 3)));
  289. } // namespace media