video_encoder_fallback_test.cc 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425
  1. // Copyright 2021 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/video/video_encoder_fallback.h"
  5. #include "base/bind.h"
  6. #include "base/logging.h"
  7. #include "base/memory/raw_ptr.h"
  8. #include "base/memory/ref_counted.h"
  9. #include "base/sequence_checker.h"
  10. #include "base/test/bind.h"
  11. #include "base/test/gmock_callback_support.h"
  12. #include "base/test/task_environment.h"
  13. #include "media/base/bind_to_current_loop.h"
  14. #include "media/base/mock_filters.h"
  15. #include "media/base/video_frame.h"
  16. #include "testing/gtest/include/gtest/gtest.h"
  17. #include "ui/gfx/geometry/size.h"
  18. using ::testing::_;
  19. using ::testing::Invoke;
  20. namespace media {
  21. const gfx::Size kFrameSize(320, 200);
  22. const int kFrameCount = 10;
  23. class VideoEncoderFallbackTest : public testing::Test {
  24. protected:
  25. void SetUp() override {
  26. auto main_video_encoder = std::make_unique<MockVideoEncoder>();
  27. main_video_encoder_ = main_video_encoder.get();
  28. secondary_video_encoder_holder_ = std::make_unique<MockVideoEncoder>();
  29. secondary_video_encoder_ = secondary_video_encoder_holder_.get();
  30. callback_runner_ = base::SequencedTaskRunnerHandle::Get();
  31. fallback_encoder_ = std::make_unique<VideoEncoderFallback>(
  32. std::move(main_video_encoder),
  33. base::BindOnce(&VideoEncoderFallbackTest::CreateSecondaryEncoder,
  34. base::Unretained(this)));
  35. EXPECT_CALL(*main_video_encoder_, Dtor());
  36. EXPECT_CALL(*secondary_video_encoder_, Dtor());
  37. }
  38. void RunLoop() { task_environment_.RunUntilIdle(); }
  39. std::unique_ptr<VideoEncoder> CreateSecondaryEncoder() {
  40. return std::unique_ptr<VideoEncoder>(
  41. secondary_video_encoder_holder_.release());
  42. }
  43. bool FallbackHappened() { return !secondary_video_encoder_holder_; }
  44. void RunStatusCallbackAync(
  45. VideoEncoder::EncoderStatusCB callback,
  46. EncoderStatus::Codes code = EncoderStatus::Codes::kOk) {
  47. base::BindPostTask(callback_runner_, std::move(callback)).Run(code);
  48. }
  49. VideoEncoder::EncoderStatusCB ValidatingStatusCB(
  50. EncoderStatus::Codes code = EncoderStatus::Codes::kOk,
  51. base::Location loc = FROM_HERE) {
  52. struct CallEnforcer {
  53. bool called = false;
  54. std::string location;
  55. ~CallEnforcer() {
  56. EXPECT_TRUE(called) << "Callback created: " << location;
  57. }
  58. };
  59. auto enforcer = std::make_unique<CallEnforcer>();
  60. enforcer->location = loc.ToString();
  61. return BindToCurrentLoop(base::BindLambdaForTesting(
  62. [code, this, enforcer{std::move(enforcer)}](EncoderStatus s) {
  63. EXPECT_TRUE(callback_runner_->RunsTasksInCurrentSequence());
  64. EXPECT_EQ(s.code(), code)
  65. << " Callback created: " << enforcer->location;
  66. enforcer->called = true;
  67. }));
  68. }
  69. base::test::TaskEnvironment task_environment_;
  70. scoped_refptr<base::SequencedTaskRunner> callback_runner_;
  71. raw_ptr<MockVideoEncoder> main_video_encoder_;
  72. raw_ptr<MockVideoEncoder> secondary_video_encoder_;
  73. std::unique_ptr<MockVideoEncoder> secondary_video_encoder_holder_;
  74. std::unique_ptr<VideoEncoderFallback> fallback_encoder_;
  75. };
  76. TEST_F(VideoEncoderFallbackTest, NoFallbackEncoding) {
  77. int outputs = 0;
  78. VideoEncoder::Options options;
  79. VideoCodecProfile profile = VIDEO_CODEC_PROFILE_UNKNOWN;
  80. VideoEncoder::OutputCB output_cb =
  81. BindToCurrentLoop(base::BindLambdaForTesting(
  82. [&](VideoEncoderOutput,
  83. absl::optional<VideoEncoder::CodecDescription>) { outputs++; }));
  84. VideoEncoder::OutputCB saved_output_cb;
  85. EXPECT_CALL(*main_video_encoder_, Initialize(_, _, _, _))
  86. .WillOnce(Invoke([&, this](VideoCodecProfile profile,
  87. const VideoEncoder::Options& options,
  88. VideoEncoder::OutputCB output_cb,
  89. VideoEncoder::EncoderStatusCB done_cb) {
  90. saved_output_cb = std::move(output_cb);
  91. RunStatusCallbackAync(std::move(done_cb));
  92. }));
  93. EXPECT_CALL(*main_video_encoder_, Encode(_, _, _))
  94. .WillRepeatedly(
  95. Invoke([&, this](scoped_refptr<VideoFrame> frame, bool key_frame,
  96. VideoEncoder::EncoderStatusCB done_cb) {
  97. VideoEncoderOutput output;
  98. output.timestamp = frame->timestamp();
  99. saved_output_cb.Run(std::move(output), {});
  100. RunStatusCallbackAync(std::move(done_cb));
  101. }));
  102. fallback_encoder_->Initialize(profile, options, std::move(output_cb),
  103. ValidatingStatusCB());
  104. RunLoop();
  105. for (int i = 0; i < kFrameCount; i++) {
  106. auto frame = VideoFrame::CreateFrame(PIXEL_FORMAT_I420, kFrameSize,
  107. gfx::Rect(kFrameSize), kFrameSize,
  108. base::Seconds(i));
  109. fallback_encoder_->Encode(frame, true, ValidatingStatusCB());
  110. }
  111. RunLoop();
  112. EXPECT_FALSE(FallbackHappened());
  113. EXPECT_EQ(outputs, kFrameCount);
  114. }
  115. // Test that VideoEncoderFallback can switch to a secondary encoder if
  116. // the main encoder fails to initialize.
  117. TEST_F(VideoEncoderFallbackTest, FallbackOnInitialize) {
  118. int outputs = 0;
  119. VideoEncoder::Options options;
  120. VideoCodecProfile profile = VIDEO_CODEC_PROFILE_UNKNOWN;
  121. VideoEncoder::OutputCB output_cb =
  122. BindToCurrentLoop(base::BindLambdaForTesting(
  123. [&](VideoEncoderOutput,
  124. absl::optional<VideoEncoder::CodecDescription>) { outputs++; }));
  125. VideoEncoder::OutputCB saved_output_cb;
  126. // Initialize() on the main encoder should fail
  127. EXPECT_CALL(*main_video_encoder_, Initialize(_, _, _, _))
  128. .WillOnce(Invoke([&, this](VideoCodecProfile profile,
  129. const VideoEncoder::Options& options,
  130. VideoEncoder::OutputCB output_cb,
  131. VideoEncoder::EncoderStatusCB done_cb) {
  132. RunStatusCallbackAync(
  133. std::move(done_cb),
  134. EncoderStatus::Codes::kEncoderInitializationError);
  135. }));
  136. // Initialize() on the second encoder should succeed
  137. EXPECT_CALL(*secondary_video_encoder_, Initialize(_, _, _, _))
  138. .WillOnce(Invoke([&, this](VideoCodecProfile profile,
  139. const VideoEncoder::Options& options,
  140. VideoEncoder::OutputCB output_cb,
  141. VideoEncoder::EncoderStatusCB done_cb) {
  142. saved_output_cb = std::move(output_cb);
  143. RunStatusCallbackAync(std::move(done_cb));
  144. }));
  145. // All encodes should come to the secondary encoder.
  146. EXPECT_CALL(*secondary_video_encoder_, Encode(_, _, _))
  147. .WillRepeatedly(
  148. Invoke([&, this](scoped_refptr<VideoFrame> frame, bool key_frame,
  149. VideoEncoder::EncoderStatusCB done_cb) {
  150. VideoEncoderOutput output;
  151. output.timestamp = frame->timestamp();
  152. saved_output_cb.Run(std::move(output), {});
  153. RunStatusCallbackAync(std::move(done_cb));
  154. }));
  155. fallback_encoder_->Initialize(profile, options, std::move(output_cb),
  156. ValidatingStatusCB());
  157. RunLoop();
  158. for (int i = 0; i < kFrameCount; i++) {
  159. auto frame = VideoFrame::CreateFrame(PIXEL_FORMAT_I420, kFrameSize,
  160. gfx::Rect(kFrameSize), kFrameSize,
  161. base::Seconds(i));
  162. fallback_encoder_->Encode(frame, true, ValidatingStatusCB());
  163. }
  164. RunLoop();
  165. EXPECT_TRUE(FallbackHappened());
  166. EXPECT_EQ(outputs, kFrameCount);
  167. }
  168. // Test that VideoEncoderFallback can switch to a secondary encoder if
  169. // the main encoder fails on encode.
  170. TEST_F(VideoEncoderFallbackTest, FallbackOnEncode) {
  171. int outputs = 0;
  172. VideoEncoder::Options options;
  173. VideoCodecProfile profile = VIDEO_CODEC_PROFILE_UNKNOWN;
  174. VideoEncoder::OutputCB output_cb =
  175. BindToCurrentLoop(base::BindLambdaForTesting(
  176. [&](VideoEncoderOutput,
  177. absl::optional<VideoEncoder::CodecDescription>) { outputs++; }));
  178. VideoEncoder::OutputCB primary_output_cb;
  179. VideoEncoder::OutputCB secondary_output_cb;
  180. // Initialize() on the main encoder should succeed
  181. EXPECT_CALL(*main_video_encoder_, Initialize(_, _, _, _))
  182. .WillOnce(Invoke([&, this](VideoCodecProfile profile,
  183. const VideoEncoder::Options& options,
  184. VideoEncoder::OutputCB output_cb,
  185. VideoEncoder::EncoderStatusCB done_cb) {
  186. primary_output_cb = std::move(output_cb);
  187. RunStatusCallbackAync(std::move(done_cb));
  188. }));
  189. // Initialize() on the second encoder should succeed as well
  190. EXPECT_CALL(*secondary_video_encoder_, Initialize(_, _, _, _))
  191. .WillOnce(Invoke([&, this](VideoCodecProfile profile,
  192. const VideoEncoder::Options& options,
  193. VideoEncoder::OutputCB output_cb,
  194. VideoEncoder::EncoderStatusCB done_cb) {
  195. secondary_output_cb = std::move(output_cb);
  196. RunStatusCallbackAync(std::move(done_cb));
  197. }));
  198. auto encoder_switch_time = base::Seconds(kFrameCount / 2);
  199. // Start failing encodes after half of the frames.
  200. EXPECT_CALL(*main_video_encoder_, Encode(_, _, _))
  201. .WillRepeatedly(Invoke([&, this](scoped_refptr<VideoFrame> frame,
  202. bool key_frame,
  203. VideoEncoder::EncoderStatusCB done_cb) {
  204. EXPECT_TRUE(frame);
  205. EXPECT_TRUE(done_cb);
  206. if (frame->timestamp() > encoder_switch_time) {
  207. std::move(done_cb).Run(EncoderStatus::Codes::kEncoderFailedEncode);
  208. return;
  209. }
  210. VideoEncoderOutput output;
  211. output.timestamp = frame->timestamp();
  212. primary_output_cb.Run(std::move(output), {});
  213. RunStatusCallbackAync(std::move(done_cb));
  214. }));
  215. // All encodes should come to the secondary encoder.
  216. EXPECT_CALL(*secondary_video_encoder_, Encode(_, _, _))
  217. .WillRepeatedly(
  218. Invoke([&, this](scoped_refptr<VideoFrame> frame, bool key_frame,
  219. VideoEncoder::EncoderStatusCB done_cb) {
  220. EXPECT_TRUE(frame);
  221. EXPECT_TRUE(done_cb);
  222. EXPECT_GT(frame->timestamp(), encoder_switch_time);
  223. VideoEncoderOutput output;
  224. output.timestamp = frame->timestamp();
  225. secondary_output_cb.Run(std::move(output), {});
  226. RunStatusCallbackAync(std::move(done_cb));
  227. }));
  228. fallback_encoder_->Initialize(profile, options, std::move(output_cb),
  229. ValidatingStatusCB());
  230. RunLoop();
  231. for (int i = 0; i < kFrameCount; i++) {
  232. auto frame = VideoFrame::CreateFrame(PIXEL_FORMAT_I420, kFrameSize,
  233. gfx::Rect(kFrameSize), kFrameSize,
  234. base::Seconds(i));
  235. fallback_encoder_->Encode(frame, true, ValidatingStatusCB());
  236. }
  237. RunLoop();
  238. EXPECT_TRUE(FallbackHappened());
  239. EXPECT_EQ(outputs, kFrameCount);
  240. }
  241. // Test how VideoEncoderFallback reports errors in initialization of the
  242. // secondary encoder.
  243. TEST_F(VideoEncoderFallbackTest, SecondaryFailureOnInitialize) {
  244. VideoEncoder::Options options;
  245. VideoCodecProfile profile = VIDEO_CODEC_PROFILE_UNKNOWN;
  246. auto output_cb = BindToCurrentLoop(base::BindLambdaForTesting(
  247. [&](VideoEncoderOutput, absl::optional<VideoEncoder::CodecDescription>) {
  248. }));
  249. // Initialize() on the main encoder should fail
  250. EXPECT_CALL(*main_video_encoder_, Initialize(_, _, _, _))
  251. .WillOnce(Invoke([&, this](VideoCodecProfile profile,
  252. const VideoEncoder::Options& options,
  253. VideoEncoder::OutputCB output_cb,
  254. VideoEncoder::EncoderStatusCB done_cb) {
  255. RunStatusCallbackAync(std::move(done_cb),
  256. EncoderStatus::Codes::kEncoderUnsupportedProfile);
  257. }));
  258. // Initialize() on the second encoder should also fail
  259. EXPECT_CALL(*secondary_video_encoder_, Initialize(_, _, _, _))
  260. .WillOnce(Invoke([&, this](VideoCodecProfile profile,
  261. const VideoEncoder::Options& options,
  262. VideoEncoder::OutputCB output_cb,
  263. VideoEncoder::EncoderStatusCB done_cb) {
  264. RunStatusCallbackAync(std::move(done_cb),
  265. EncoderStatus::Codes::kEncoderUnsupportedCodec);
  266. }));
  267. fallback_encoder_->Initialize(
  268. profile, options, std::move(output_cb),
  269. ValidatingStatusCB(EncoderStatus::Codes::kEncoderUnsupportedCodec));
  270. RunLoop();
  271. EXPECT_CALL(*secondary_video_encoder_, Encode(_, _, _))
  272. .Times(kFrameCount)
  273. .WillRepeatedly(Invoke([&, this](scoped_refptr<VideoFrame> frame,
  274. bool key_frame,
  275. VideoEncoder::EncoderStatusCB done_cb) {
  276. RunStatusCallbackAync(std::move(done_cb),
  277. EncoderStatus::Codes::kEncoderUnsupportedCodec);
  278. }));
  279. for (int i = 0; i < kFrameCount; i++) {
  280. auto frame = VideoFrame::CreateFrame(PIXEL_FORMAT_I420, kFrameSize,
  281. gfx::Rect(kFrameSize), kFrameSize,
  282. base::Seconds(i));
  283. auto done_callback = base::BindLambdaForTesting([this](EncoderStatus s) {
  284. EXPECT_TRUE(callback_runner_->RunsTasksInCurrentSequence());
  285. EXPECT_EQ(s.code(), EncoderStatus::Codes::kEncoderUnsupportedCodec);
  286. callback_runner_->DeleteSoon(FROM_HERE, std::move(fallback_encoder_));
  287. });
  288. fallback_encoder_->Encode(frame, true, std::move(done_callback));
  289. }
  290. RunLoop();
  291. EXPECT_TRUE(FallbackHappened());
  292. }
  293. // Test how VideoEncoderFallback reports errors when encoding with the secondary
  294. // encoder.
  295. TEST_F(VideoEncoderFallbackTest, SecondaryFailureOnEncode) {
  296. int outputs = 0;
  297. VideoEncoder::Options options;
  298. VideoCodecProfile profile = VIDEO_CODEC_PROFILE_UNKNOWN;
  299. VideoEncoder::OutputCB output_cb =
  300. BindToCurrentLoop(base::BindLambdaForTesting(
  301. [&](VideoEncoderOutput,
  302. absl::optional<VideoEncoder::CodecDescription>) { outputs++; }));
  303. VideoEncoder::OutputCB primary_output_cb;
  304. VideoEncoder::OutputCB secondary_output_cb;
  305. // Initialize() on the main encoder should succeed
  306. EXPECT_CALL(*main_video_encoder_, Initialize(_, _, _, _))
  307. .WillOnce(Invoke([&, this](VideoCodecProfile profile,
  308. const VideoEncoder::Options& options,
  309. VideoEncoder::OutputCB output_cb,
  310. VideoEncoder::EncoderStatusCB done_cb) {
  311. primary_output_cb = std::move(output_cb);
  312. RunStatusCallbackAync(std::move(done_cb));
  313. }));
  314. // Initialize() on the second encoder should succeed as well
  315. EXPECT_CALL(*secondary_video_encoder_, Initialize(_, _, _, _))
  316. .WillOnce(Invoke([&, this](VideoCodecProfile profile,
  317. const VideoEncoder::Options& options,
  318. VideoEncoder::OutputCB output_cb,
  319. VideoEncoder::EncoderStatusCB done_cb) {
  320. secondary_output_cb = std::move(output_cb);
  321. RunStatusCallbackAync(std::move(done_cb));
  322. }));
  323. // Start failing encodes after half of the frames.
  324. auto encoder_switch_time = base::Seconds(kFrameCount / 2);
  325. EXPECT_CALL(*main_video_encoder_, Encode(_, _, _))
  326. .WillRepeatedly(Invoke([&, this](scoped_refptr<VideoFrame> frame,
  327. bool key_frame,
  328. VideoEncoder::EncoderStatusCB done_cb) {
  329. EXPECT_TRUE(frame);
  330. EXPECT_TRUE(done_cb);
  331. if (frame->timestamp() > encoder_switch_time) {
  332. std::move(done_cb).Run(EncoderStatus::Codes::kEncoderFailedEncode);
  333. return;
  334. }
  335. VideoEncoderOutput output;
  336. output.timestamp = frame->timestamp();
  337. primary_output_cb.Run(std::move(output), {});
  338. RunStatusCallbackAync(std::move(done_cb));
  339. }));
  340. // All encodes should come to the secondary encoder. Again fail encoding
  341. // once we reach 3/4 the total frame count.
  342. auto second_encoder_fail_time = base::Seconds(3 * kFrameCount / 4);
  343. LOG(ERROR) << second_encoder_fail_time << "!!!!";
  344. EXPECT_CALL(*secondary_video_encoder_, Encode(_, _, _))
  345. .WillRepeatedly(Invoke([&, this](scoped_refptr<VideoFrame> frame,
  346. bool key_frame,
  347. VideoEncoder::EncoderStatusCB done_cb) {
  348. EXPECT_TRUE(frame);
  349. EXPECT_TRUE(done_cb);
  350. EXPECT_GT(frame->timestamp(), encoder_switch_time);
  351. if (frame->timestamp() > second_encoder_fail_time) {
  352. std::move(done_cb).Run(EncoderStatus::Codes::kEncoderFailedEncode);
  353. return;
  354. }
  355. VideoEncoderOutput output;
  356. output.timestamp = frame->timestamp();
  357. secondary_output_cb.Run(std::move(output), {});
  358. RunStatusCallbackAync(std::move(done_cb));
  359. }));
  360. fallback_encoder_->Initialize(profile, options, std::move(output_cb),
  361. ValidatingStatusCB());
  362. RunLoop();
  363. for (int i = 1; i <= kFrameCount; i++) {
  364. auto frame = VideoFrame::CreateFrame(PIXEL_FORMAT_I420, kFrameSize,
  365. gfx::Rect(kFrameSize), kFrameSize,
  366. base::Seconds(i));
  367. auto done_cb =
  368. ValidatingStatusCB((frame->timestamp() <= second_encoder_fail_time)
  369. ? EncoderStatus::Codes::kOk
  370. : EncoderStatus::Codes::kEncoderFailedEncode);
  371. fallback_encoder_->Encode(frame, true, std::move(done_cb));
  372. }
  373. RunLoop();
  374. EXPECT_TRUE(FallbackHappened());
  375. EXPECT_EQ(outputs, 3 * kFrameCount / 4);
  376. }
  377. } // namespace media