vpx_video_encoder.cc 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727
  1. // Copyright 2020 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "media/video/vpx_video_encoder.h"
  5. #include "base/cxx17_backports.h"
  6. #include "base/logging.h"
  7. #include "base/numerics/checked_math.h"
  8. #include "base/strings/stringprintf.h"
  9. #include "base/system/sys_info.h"
  10. #include "base/time/time.h"
  11. #include "base/trace_event/trace_event.h"
  12. #include "media/base/bind_to_current_loop.h"
  13. #include "media/base/svc_scalability_mode.h"
  14. #include "media/base/timestamp_constants.h"
  15. #include "media/base/video_frame.h"
  16. #include "media/base/video_util.h"
  17. #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h"
  18. #include "third_party/libyuv/include/libyuv/convert.h"
  19. namespace media {
  20. namespace {
  21. constexpr vpx_enc_frame_flags_t VP8_UPDATE_NOTHING =
  22. VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_LAST;
  23. // Frame Pattern:
  24. // Layer Index 0: |0| |2| |4| |6| |8|
  25. // Layer Index 1: | |1| |3| |5| |7| |
  26. vpx_enc_frame_flags_t vp8_2layers_temporal_flags[] = {
  27. // Layer 0 : update and reference only last frame
  28. VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_GF |
  29. VP8_EFLAG_NO_UPD_ARF,
  30. // Layer 1: only reference last frame, no updates
  31. VP8_UPDATE_NOTHING | VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF};
  32. // Frame Pattern:
  33. // Layer Index 0: |0| | | |4| | | |8| | | |12|
  34. // Layer Index 1: | | |2| | | |6| | | |10| | |
  35. // Layer Index 2: | |1| |3| |5| |7| |9| |11| |
  36. vpx_enc_frame_flags_t vp8_3layers_temporal_flags[] = {
  37. // Layer 0 : update and reference only last frame
  38. // It only depends on layer 0
  39. VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_GF |
  40. VP8_EFLAG_NO_UPD_ARF,
  41. // Layer 2: only reference last frame, no updates
  42. // It only depends on layer 0
  43. VP8_UPDATE_NOTHING | VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_REF_GF,
  44. // Layer 1: only reference last frame, update gold frame
  45. // It only depends on layer 0
  46. VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF | VP8_EFLAG_NO_UPD_ARF |
  47. VP8_EFLAG_NO_UPD_LAST,
  48. // Layer 2: reference last frame and gold frame, no updates
  49. // It depends on layer 0 and layer 1
  50. VP8_UPDATE_NOTHING | VP8_EFLAG_NO_REF_ARF,
  51. };
  52. // Returns the number of threads.
  53. int GetNumberOfThreads(int width) {
  54. // Default to 1 thread for less than VGA.
  55. int desired_threads = 1;
  56. if (width >= 3840)
  57. desired_threads = 16;
  58. else if (width >= 2560)
  59. desired_threads = 8;
  60. else if (width >= 1280)
  61. desired_threads = 4;
  62. else if (width >= 640)
  63. desired_threads = 2;
  64. // Clamp to the number of available logical processors/cores.
  65. desired_threads =
  66. std::min(desired_threads, base::SysInfo::NumberOfProcessors());
  67. return desired_threads;
  68. }
  69. EncoderStatus SetUpVpxConfig(const VideoEncoder::Options& opts,
  70. vpx_codec_enc_cfg_t* config) {
  71. if (opts.frame_size.width() <= 0 || opts.frame_size.height() <= 0)
  72. return EncoderStatus(EncoderStatus::Codes::kEncoderUnsupportedConfig,
  73. "Negative width or height values.");
  74. if (!opts.frame_size.GetCheckedArea().IsValid())
  75. return EncoderStatus(EncoderStatus::Codes::kEncoderUnsupportedConfig,
  76. "Frame is too large.");
  77. config->g_pass = VPX_RC_ONE_PASS;
  78. config->g_lag_in_frames = 0;
  79. config->rc_max_quantizer = 58;
  80. config->rc_min_quantizer = 2;
  81. config->rc_resize_allowed = 0;
  82. config->rc_dropframe_thresh = 0; // Don't drop frames
  83. config->g_timebase.num = 1;
  84. config->g_timebase.den = base::Time::kMicrosecondsPerSecond;
  85. // Set the number of threads based on the image width and num of cores.
  86. config->g_threads = GetNumberOfThreads(opts.frame_size.width());
  87. // Insert keyframes at will with a given max interval
  88. if (opts.keyframe_interval.has_value()) {
  89. config->kf_mode = VPX_KF_AUTO;
  90. config->kf_min_dist = 0;
  91. config->kf_max_dist = opts.keyframe_interval.value();
  92. }
  93. if (opts.bitrate.has_value()) {
  94. auto& bitrate = opts.bitrate.value();
  95. config->rc_target_bitrate = bitrate.target_bps() / 1000;
  96. switch (bitrate.mode()) {
  97. case Bitrate::Mode::kVariable:
  98. config->rc_end_usage = VPX_VBR;
  99. break;
  100. case Bitrate::Mode::kConstant:
  101. config->rc_end_usage = VPX_CBR;
  102. break;
  103. }
  104. } else {
  105. config->rc_target_bitrate = GetDefaultVideoEncodeBitrate(
  106. opts.frame_size, opts.framerate.value_or(30));
  107. }
  108. config->g_w = opts.frame_size.width();
  109. config->g_h = opts.frame_size.height();
  110. if (!opts.scalability_mode)
  111. return EncoderStatus::Codes::kOk;
  112. switch (opts.scalability_mode.value()) {
  113. case SVCScalabilityMode::kL1T2:
  114. // Frame Pattern:
  115. // Layer Index 0: |0| |2| |4| |6| |8|
  116. // Layer Index 1: | |1| |3| |5| |7| |
  117. config->ts_number_layers = 2;
  118. config->ts_periodicity = 2;
  119. DCHECK_EQ(config->ts_periodicity,
  120. sizeof(vp8_2layers_temporal_flags) /
  121. sizeof(vp8_2layers_temporal_flags[0]));
  122. config->ts_layer_id[0] = 0;
  123. config->ts_layer_id[1] = 1;
  124. config->ts_rate_decimator[0] = 2;
  125. config->ts_rate_decimator[1] = 1;
  126. // Bitrate allocation L0: 60% L1: 40%
  127. config->layer_target_bitrate[0] = config->ts_target_bitrate[0] =
  128. 60 * config->rc_target_bitrate / 100;
  129. config->layer_target_bitrate[1] = config->ts_target_bitrate[1] =
  130. config->rc_target_bitrate;
  131. config->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_0101;
  132. config->g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT;
  133. break;
  134. case SVCScalabilityMode::kL1T3:
  135. // Frame Pattern:
  136. // Layer Index 0: |0| | | |4| | | |8| | | |12|
  137. // Layer Index 1: | | |2| | | |6| | | |10| | |
  138. // Layer Index 2: | |1| |3| |5| |7| |9| |11| |
  139. config->ts_number_layers = 3;
  140. config->ts_periodicity = 4;
  141. DCHECK_EQ(config->ts_periodicity,
  142. sizeof(vp8_3layers_temporal_flags) /
  143. sizeof(vp8_3layers_temporal_flags[0]));
  144. config->ts_layer_id[0] = 0;
  145. config->ts_layer_id[1] = 2;
  146. config->ts_layer_id[2] = 1;
  147. config->ts_layer_id[3] = 2;
  148. config->ts_rate_decimator[0] = 4;
  149. config->ts_rate_decimator[1] = 2;
  150. config->ts_rate_decimator[2] = 1;
  151. // Bitrate allocation L0: 50% L1: 20% L2: 30%
  152. config->layer_target_bitrate[0] = config->ts_target_bitrate[0] =
  153. 50 * config->rc_target_bitrate / 100;
  154. config->layer_target_bitrate[1] = config->ts_target_bitrate[1] =
  155. 70 * config->rc_target_bitrate / 100;
  156. config->layer_target_bitrate[2] = config->ts_target_bitrate[2] =
  157. config->rc_target_bitrate;
  158. config->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_0212;
  159. config->g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT;
  160. break;
  161. default: {
  162. return EncoderStatus(EncoderStatus::Codes::kEncoderUnsupportedConfig,
  163. "Unsupported number of temporal layers.");
  164. }
  165. }
  166. return EncoderStatus::Codes::kOk;
  167. }
  168. vpx_svc_extra_cfg_t MakeSvcExtraConfig(const vpx_codec_enc_cfg_t& config) {
  169. vpx_svc_extra_cfg_t result = {};
  170. result.temporal_layering_mode = config.temporal_layering_mode;
  171. for (size_t i = 0; i < config.ts_number_layers; ++i) {
  172. result.scaling_factor_num[i] = 1;
  173. result.scaling_factor_den[i] = 1;
  174. result.max_quantizers[i] = config.rc_max_quantizer;
  175. result.min_quantizers[i] = config.rc_min_quantizer;
  176. }
  177. return result;
  178. }
  179. EncoderStatus ReallocateVpxImageIfNeeded(vpx_image_t* vpx_image,
  180. const vpx_img_fmt fmt,
  181. int width,
  182. int height) {
  183. if (vpx_image->fmt != fmt || static_cast<int>(vpx_image->w) != width ||
  184. static_cast<int>(vpx_image->h) != height) {
  185. vpx_img_free(vpx_image);
  186. if (vpx_image != vpx_img_alloc(vpx_image, fmt, width, height, 1)) {
  187. return EncoderStatus(EncoderStatus::Codes::kEncoderFailedEncode,
  188. "Invalid format or frame size.");
  189. }
  190. vpx_image->bit_depth = (fmt & VPX_IMG_FMT_HIGHBITDEPTH) ? 16 : 8;
  191. }
  192. // else no-op since the image don't need to change format.
  193. return EncoderStatus::Codes::kOk;
  194. }
  195. void FreeCodecCtx(vpx_codec_ctx_t* codec_ctx) {
  196. if (codec_ctx->name) {
  197. // Codec has been initialized, we need to destroy it.
  198. auto error = vpx_codec_destroy(codec_ctx);
  199. DCHECK_EQ(error, VPX_CODEC_OK);
  200. }
  201. delete codec_ctx;
  202. }
  203. } // namespace
  204. VpxVideoEncoder::VpxVideoEncoder() : codec_(nullptr, FreeCodecCtx) {}
  205. void VpxVideoEncoder::Initialize(VideoCodecProfile profile,
  206. const Options& options,
  207. OutputCB output_cb,
  208. EncoderStatusCB done_cb) {
  209. done_cb = BindToCurrentLoop(std::move(done_cb));
  210. if (codec_) {
  211. std::move(done_cb).Run(EncoderStatus::Codes::kEncoderInitializeTwice);
  212. return;
  213. }
  214. profile_ = profile;
  215. bool is_vp9 = false;
  216. vpx_codec_iface_t* iface = nullptr;
  217. if (profile == VP8PROFILE_ANY) {
  218. iface = vpx_codec_vp8_cx();
  219. } else if (profile == VP9PROFILE_PROFILE0 || profile == VP9PROFILE_PROFILE2) {
  220. // TODO(https://crbug.com/1116617): Consider support for profiles 1 and 3.
  221. is_vp9 = true;
  222. iface = vpx_codec_vp9_cx();
  223. } else {
  224. auto status =
  225. EncoderStatus(EncoderStatus::Codes::kEncoderUnsupportedProfile)
  226. .WithData("profile", profile);
  227. std::move(done_cb).Run(status);
  228. return;
  229. }
  230. auto vpx_error = vpx_codec_enc_config_default(iface, &codec_config_, 0);
  231. if (vpx_error != VPX_CODEC_OK) {
  232. auto status =
  233. EncoderStatus(EncoderStatus::Codes::kEncoderInitializationError,
  234. "Failed to get default VPX config.")
  235. .WithData("vpx_error", vpx_error);
  236. std::move(done_cb).Run(status);
  237. return;
  238. }
  239. vpx_img_fmt img_fmt = VPX_IMG_FMT_NONE;
  240. unsigned int bits_for_storage = 8;
  241. switch (profile) {
  242. case VP9PROFILE_PROFILE1:
  243. codec_config_.g_profile = 1;
  244. break;
  245. case VP9PROFILE_PROFILE2:
  246. codec_config_.g_profile = 2;
  247. img_fmt = VPX_IMG_FMT_I42016;
  248. bits_for_storage = 16;
  249. codec_config_.g_bit_depth = VPX_BITS_10;
  250. codec_config_.g_input_bit_depth = 10;
  251. break;
  252. case VP9PROFILE_PROFILE3:
  253. codec_config_.g_profile = 3;
  254. break;
  255. default:
  256. codec_config_.g_profile = 0;
  257. img_fmt = VPX_IMG_FMT_I420;
  258. bits_for_storage = 8;
  259. codec_config_.g_bit_depth = VPX_BITS_8;
  260. codec_config_.g_input_bit_depth = 8;
  261. break;
  262. }
  263. auto status = SetUpVpxConfig(options, &codec_config_);
  264. if (!status.is_ok()) {
  265. std::move(done_cb).Run(status);
  266. return;
  267. }
  268. vpx_codec_unique_ptr codec(new vpx_codec_ctx_t, FreeCodecCtx);
  269. codec->name = nullptr; // We are allowed to use vpx_codec_ctx_t.name
  270. vpx_error = vpx_codec_enc_init(
  271. codec.get(), iface, &codec_config_,
  272. codec_config_.g_bit_depth == VPX_BITS_8 ? 0 : VPX_CODEC_USE_HIGHBITDEPTH);
  273. if (vpx_error != VPX_CODEC_OK) {
  274. std::string msg = base::StringPrintf(
  275. "VPX encoder initialization error: %s %s",
  276. vpx_codec_err_to_string(vpx_error), codec->err_detail);
  277. DLOG(ERROR) << msg;
  278. std::move(done_cb).Run(
  279. EncoderStatus(EncoderStatus::Codes::kEncoderInitializationError, msg));
  280. return;
  281. }
  282. // For VP9 the values used for real-time encoding mode are 5, 6, 7,
  283. // 8, 9. Higher means faster encoding, but lower quality.
  284. // For VP8 typical values used for real-time encoding are -4, -6,
  285. // -8, -10. Again larger magnitude means faster encoding but lower
  286. // quality.
  287. int cpu_used = is_vp9 ? 7 : -6;
  288. vpx_error = vpx_codec_control(codec.get(), VP8E_SET_CPUUSED, cpu_used);
  289. if (vpx_error != VPX_CODEC_OK) {
  290. std::string msg =
  291. base::StringPrintf("VPX encoder VP8E_SET_CPUUSED error: %s",
  292. vpx_codec_err_to_string(vpx_error));
  293. DLOG(ERROR) << msg;
  294. std::move(done_cb).Run(
  295. EncoderStatus(EncoderStatus::Codes::kEncoderInitializationError, msg));
  296. return;
  297. }
  298. if (&vpx_image_ != vpx_img_alloc(&vpx_image_, img_fmt,
  299. options.frame_size.width(),
  300. options.frame_size.height(), 1)) {
  301. std::move(done_cb).Run(
  302. EncoderStatus(EncoderStatus::Codes::kEncoderInitializationError,
  303. "Invalid format or frame size."));
  304. return;
  305. }
  306. vpx_image_.bit_depth = bits_for_storage;
  307. if (is_vp9) {
  308. // Set the number of column tiles in encoding an input frame, with number of
  309. // tile columns (in Log2 unit) as the parameter.
  310. // The minimum width of a tile column is 256 pixels, the maximum is 4096.
  311. int log2_tile_columns =
  312. static_cast<int>(std::log2(codec_config_.g_w / 256));
  313. vpx_codec_control(codec.get(), VP9E_SET_TILE_COLUMNS, log2_tile_columns);
  314. // Turn on row level multi-threading.
  315. vpx_codec_control(codec.get(), VP9E_SET_ROW_MT, 1);
  316. if (codec_config_.ts_number_layers > 1) {
  317. vpx_svc_extra_cfg_t svc_conf = MakeSvcExtraConfig(codec_config_);
  318. // VP9 needs SVC to be turned on explicitly
  319. vpx_codec_control(codec.get(), VP9E_SET_SVC_PARAMETERS, &svc_conf);
  320. vpx_error = vpx_codec_control(codec.get(), VP9E_SET_SVC, 1);
  321. if (vpx_error != VPX_CODEC_OK) {
  322. std::string msg =
  323. base::StringPrintf("Can't activate SVC encoding: %s",
  324. vpx_codec_err_to_string(vpx_error));
  325. DLOG(ERROR) << msg;
  326. status = EncoderStatus(
  327. EncoderStatus::Codes::kEncoderInitializationError, msg);
  328. std::move(done_cb).Run(status);
  329. return;
  330. }
  331. }
  332. // In CBR mode use aq-mode=3 is enabled for quality improvement
  333. if (codec_config_.rc_end_usage == VPX_CBR)
  334. vpx_codec_control(codec.get(), VP9E_SET_AQ_MODE, 3);
  335. }
  336. options_ = options;
  337. originally_configured_size_ = options.frame_size;
  338. output_cb_ = BindToCurrentLoop(std::move(output_cb));
  339. codec_ = std::move(codec);
  340. std::move(done_cb).Run(EncoderStatus::Codes::kOk);
  341. }
  342. void VpxVideoEncoder::Encode(scoped_refptr<VideoFrame> frame,
  343. bool key_frame,
  344. EncoderStatusCB done_cb) {
  345. done_cb = BindToCurrentLoop(std::move(done_cb));
  346. if (!codec_) {
  347. std::move(done_cb).Run(
  348. EncoderStatus::Codes::kEncoderInitializeNeverCompleted);
  349. return;
  350. }
  351. if (!frame) {
  352. std::move(done_cb).Run(
  353. EncoderStatus(EncoderStatus::Codes::kEncoderFailedEncode,
  354. "No frame provided for encoding."));
  355. return;
  356. }
  357. bool supported_format = frame->format() == PIXEL_FORMAT_NV12 ||
  358. frame->format() == PIXEL_FORMAT_I420 ||
  359. frame->format() == PIXEL_FORMAT_XBGR ||
  360. frame->format() == PIXEL_FORMAT_XRGB ||
  361. frame->format() == PIXEL_FORMAT_ABGR ||
  362. frame->format() == PIXEL_FORMAT_ARGB;
  363. if ((!frame->IsMappable() && !frame->HasGpuMemoryBuffer()) ||
  364. !supported_format) {
  365. std::move(done_cb).Run(
  366. EncoderStatus(EncoderStatus::Codes::kEncoderFailedEncode,
  367. "Unexpected frame format.")
  368. .WithData("IsMappable", frame->IsMappable())
  369. .WithData("format", frame->format()));
  370. return;
  371. }
  372. if (frame->format() == PIXEL_FORMAT_NV12 && frame->HasGpuMemoryBuffer()) {
  373. frame = ConvertToMemoryMappedFrame(frame);
  374. if (!frame) {
  375. std::move(done_cb).Run(
  376. EncoderStatus(EncoderStatus::Codes::kEncoderFailedEncode,
  377. "Convert GMB frame to MemoryMappedFrame failed."));
  378. return;
  379. }
  380. }
  381. const bool is_yuv = IsYuvPlanar(frame->format());
  382. if (frame->visible_rect().size() != options_.frame_size || !is_yuv) {
  383. auto resized_frame = frame_pool_.CreateFrame(
  384. is_yuv ? frame->format() : PIXEL_FORMAT_I420, options_.frame_size,
  385. gfx::Rect(options_.frame_size), options_.frame_size,
  386. frame->timestamp());
  387. if (!resized_frame) {
  388. std::move(done_cb).Run(
  389. EncoderStatus(EncoderStatus::Codes::kEncoderFailedEncode,
  390. "Can't allocate a resized frame"));
  391. return;
  392. }
  393. auto convert_status =
  394. ConvertAndScaleFrame(*frame, *resized_frame, resize_buf_);
  395. if (!convert_status.is_ok()) {
  396. std::move(done_cb).Run(
  397. EncoderStatus(EncoderStatus::Codes::kEncoderFailedEncode)
  398. .AddCause(std::move(convert_status)));
  399. return;
  400. }
  401. frame = std::move(resized_frame);
  402. }
  403. switch (profile_) {
  404. case VP9PROFILE_PROFILE2:
  405. // Profile 2 uses 10bit color,
  406. libyuv::I420ToI010(
  407. frame->visible_data(VideoFrame::kYPlane),
  408. frame->stride(VideoFrame::kYPlane),
  409. frame->visible_data(VideoFrame::kUPlane),
  410. frame->stride(VideoFrame::kUPlane),
  411. frame->visible_data(VideoFrame::kVPlane),
  412. frame->stride(VideoFrame::kVPlane),
  413. reinterpret_cast<uint16_t*>(vpx_image_.planes[VPX_PLANE_Y]),
  414. vpx_image_.stride[VPX_PLANE_Y] / 2,
  415. reinterpret_cast<uint16_t*>(vpx_image_.planes[VPX_PLANE_U]),
  416. vpx_image_.stride[VPX_PLANE_U] / 2,
  417. reinterpret_cast<uint16_t*>(vpx_image_.planes[VPX_PLANE_V]),
  418. vpx_image_.stride[VPX_PLANE_V] / 2, frame->visible_rect().width(),
  419. frame->visible_rect().height());
  420. break;
  421. case VP9PROFILE_PROFILE1:
  422. case VP9PROFILE_PROFILE3:
  423. NOTREACHED();
  424. break;
  425. default:
  426. vpx_img_fmt_t fmt = frame->format() == PIXEL_FORMAT_NV12
  427. ? VPX_IMG_FMT_NV12
  428. : VPX_IMG_FMT_I420;
  429. EncoderStatus status = ReallocateVpxImageIfNeeded(
  430. &vpx_image_, fmt, codec_config_.g_w, codec_config_.g_h);
  431. if (!status.is_ok()) {
  432. std::move(done_cb).Run(status);
  433. return;
  434. }
  435. if (fmt == VPX_IMG_FMT_NV12) {
  436. vpx_image_.planes[VPX_PLANE_Y] =
  437. const_cast<uint8_t*>(frame->visible_data(VideoFrame::kYPlane));
  438. vpx_image_.planes[VPX_PLANE_U] =
  439. const_cast<uint8_t*>(frame->visible_data(VideoFrame::kUVPlane));
  440. // In NV12 Y and U samples are combined in one plane (bytes go YUYUYU),
  441. // but libvpx treats them as two planes with the same stride but shifted
  442. // by one byte.
  443. vpx_image_.planes[VPX_PLANE_V] = vpx_image_.planes[VPX_PLANE_U] + 1;
  444. vpx_image_.stride[VPX_PLANE_Y] = frame->stride(VideoFrame::kYPlane);
  445. vpx_image_.stride[VPX_PLANE_U] = frame->stride(VideoFrame::kUVPlane);
  446. vpx_image_.stride[VPX_PLANE_V] = frame->stride(VideoFrame::kUVPlane);
  447. } else {
  448. vpx_image_.planes[VPX_PLANE_Y] =
  449. const_cast<uint8_t*>(frame->visible_data(VideoFrame::kYPlane));
  450. vpx_image_.planes[VPX_PLANE_U] =
  451. const_cast<uint8_t*>(frame->visible_data(VideoFrame::kUPlane));
  452. vpx_image_.planes[VPX_PLANE_V] =
  453. const_cast<uint8_t*>(frame->visible_data(VideoFrame::kVPlane));
  454. vpx_image_.stride[VPX_PLANE_Y] = frame->stride(VideoFrame::kYPlane);
  455. vpx_image_.stride[VPX_PLANE_U] = frame->stride(VideoFrame::kUPlane);
  456. vpx_image_.stride[VPX_PLANE_V] = frame->stride(VideoFrame::kVPlane);
  457. }
  458. break;
  459. }
  460. // Use zero as a timestamp, so encoder will not use it for rate control.
  461. // In absence of timestamp libvpx uses duration.
  462. constexpr auto timestamp_us = 0;
  463. auto duration_us = GetFrameDuration(*frame).InMicroseconds();
  464. last_frame_timestamp_ = frame->timestamp();
  465. if (last_frame_color_space_ != frame->ColorSpace()) {
  466. last_frame_color_space_ = frame->ColorSpace();
  467. key_frame = true;
  468. }
  469. auto deadline = VPX_DL_REALTIME;
  470. vpx_codec_flags_t flags = key_frame ? VPX_EFLAG_FORCE_KF : 0;
  471. int temporal_id = 0;
  472. if (codec_config_.ts_number_layers > 1) {
  473. if (key_frame)
  474. temporal_svc_frame_index = 0;
  475. int index_in_temp_cycle =
  476. temporal_svc_frame_index % codec_config_.ts_periodicity;
  477. temporal_id = codec_config_.ts_layer_id[index_in_temp_cycle];
  478. temporal_svc_frame_index++;
  479. if (profile_ == VP8PROFILE_ANY) {
  480. auto* vp8_layers_flags = codec_config_.ts_number_layers == 2
  481. ? vp8_2layers_temporal_flags
  482. : vp8_3layers_temporal_flags;
  483. flags |= vp8_layers_flags[index_in_temp_cycle];
  484. vpx_codec_control(codec_.get(), VP8E_SET_TEMPORAL_LAYER_ID, temporal_id);
  485. }
  486. }
  487. TRACE_EVENT1("media", "vpx_codec_encode", "timestamp", frame->timestamp());
  488. auto vpx_error = vpx_codec_encode(codec_.get(), &vpx_image_, timestamp_us,
  489. duration_us, flags, deadline);
  490. if (vpx_error != VPX_CODEC_OK) {
  491. std::string msg = base::StringPrintf("VPX encoding error: %s (%s)",
  492. vpx_codec_err_to_string(vpx_error),
  493. vpx_codec_error_detail(codec_.get()));
  494. DLOG(ERROR) << msg;
  495. std::move(done_cb).Run(
  496. EncoderStatus(EncoderStatus::Codes::kEncoderFailedEncode, msg)
  497. .WithData("vpx_error", vpx_error));
  498. return;
  499. }
  500. DrainOutputs(temporal_id, frame->timestamp(), frame->ColorSpace());
  501. std::move(done_cb).Run(EncoderStatus::Codes::kOk);
  502. }
  503. void VpxVideoEncoder::ChangeOptions(const Options& options,
  504. OutputCB output_cb,
  505. EncoderStatusCB done_cb) {
  506. done_cb = BindToCurrentLoop(std::move(done_cb));
  507. if (!codec_) {
  508. std::move(done_cb).Run(
  509. EncoderStatus::Codes::kEncoderInitializeNeverCompleted);
  510. return;
  511. }
  512. // Libvpx is very peculiar about encoded frame size changes,
  513. // - VP8: As long as the frame area doesn't increase, internal codec
  514. // structures don't need to be reallocated and codec can be simply
  515. // reconfigured.
  516. // - VP9: The codec cannot increase encoded width or height larger than their
  517. // initial values.
  518. //
  519. // Mind the difference between old frame sizes:
  520. // - |originally_configured_size_| is set only once when the vpx_codec_ctx_t
  521. // is created.
  522. // - |options_.frame_size| changes every time ChangeOptions() is called.
  523. // More info can be found here:
  524. // https://bugs.chromium.org/p/webm/issues/detail?id=1642
  525. // https://bugs.chromium.org/p/webm/issues/detail?id=912
  526. if (profile_ == VP8PROFILE_ANY) {
  527. // VP8 resize restrictions
  528. auto old_area = originally_configured_size_.GetCheckedArea();
  529. auto new_area = options.frame_size.GetCheckedArea();
  530. DCHECK(old_area.IsValid());
  531. if (!new_area.IsValid() || new_area.ValueOrDie() > old_area.ValueOrDie()) {
  532. auto status = EncoderStatus(
  533. EncoderStatus::Codes::kEncoderUnsupportedConfig,
  534. "libvpx/VP8 doesn't support dynamically increasing frame area");
  535. std::move(done_cb).Run(std::move(status));
  536. return;
  537. }
  538. } else {
  539. // VP9 resize restrictions
  540. if (options.frame_size.width() > originally_configured_size_.width() ||
  541. options.frame_size.height() > originally_configured_size_.height()) {
  542. auto status = EncoderStatus(
  543. EncoderStatus::Codes::kEncoderUnsupportedConfig,
  544. "libvpx/VP9 doesn't support dynamically increasing frame dimentions");
  545. std::move(done_cb).Run(std::move(status));
  546. return;
  547. }
  548. }
  549. vpx_codec_enc_cfg_t new_config = codec_config_;
  550. auto status = SetUpVpxConfig(options, &new_config);
  551. if (!status.is_ok()) {
  552. std::move(done_cb).Run(status);
  553. return;
  554. }
  555. status = ReallocateVpxImageIfNeeded(&vpx_image_, vpx_image_.fmt,
  556. options.frame_size.width(),
  557. options.frame_size.height());
  558. if (!status.is_ok()) {
  559. std::move(done_cb).Run(status);
  560. return;
  561. }
  562. auto error = vpx_codec_enc_config_set(codec_.get(), &new_config);
  563. const bool is_vp9 = (profile_ != VP8PROFILE_ANY);
  564. if (is_vp9 && error == VPX_CODEC_OK && new_config.ts_number_layers > 1) {
  565. vpx_svc_extra_cfg_t svc_conf = MakeSvcExtraConfig(new_config);
  566. vpx_codec_control(codec_.get(), VP9E_SET_SVC_PARAMETERS, &svc_conf);
  567. error = vpx_codec_control(codec_.get(), VP9E_SET_SVC, 1);
  568. }
  569. if (error == VPX_CODEC_OK) {
  570. codec_config_ = new_config;
  571. options_ = options;
  572. if (!output_cb.is_null())
  573. output_cb_ = BindToCurrentLoop(std::move(output_cb));
  574. } else {
  575. status = EncoderStatus(EncoderStatus::Codes::kEncoderUnsupportedConfig,
  576. "Failed to set new VPX config")
  577. .WithData("vpx_error", error);
  578. }
  579. std::move(done_cb).Run(std::move(status));
  580. }
  581. base::TimeDelta VpxVideoEncoder::GetFrameDuration(const VideoFrame& frame) {
  582. // Frame has duration in metadata, use it.
  583. if (frame.metadata().frame_duration.has_value())
  584. return frame.metadata().frame_duration.value();
  585. // Options have framerate specified, use it.
  586. if (options_.framerate.has_value())
  587. return base::Seconds(1.0 / options_.framerate.value());
  588. // No real way to figure out duration, use time passed since the last frame
  589. // as an educated guess, but clamp it within a reasonable limits.
  590. constexpr auto min_duration = base::Seconds(1.0 / 60.0);
  591. constexpr auto max_duration = base::Seconds(1.0 / 24.0);
  592. auto duration = frame.timestamp() - last_frame_timestamp_;
  593. return base::clamp(duration, min_duration, max_duration);
  594. }
  595. VpxVideoEncoder::~VpxVideoEncoder() {
  596. if (!codec_)
  597. return;
  598. // It's safe to call vpx_img_free, even if vpx_image_ has never been
  599. // initialized. vpx_img_free is not going to deallocate the vpx_image_
  600. // itself, only internal buffers.
  601. vpx_img_free(&vpx_image_);
  602. }
  603. void VpxVideoEncoder::Flush(EncoderStatusCB done_cb) {
  604. done_cb = BindToCurrentLoop(std::move(done_cb));
  605. if (!codec_) {
  606. std::move(done_cb).Run(
  607. EncoderStatus::Codes::kEncoderInitializeNeverCompleted);
  608. return;
  609. }
  610. auto vpx_error = vpx_codec_encode(codec_.get(), nullptr, -1, 0, 0, 0);
  611. if (vpx_error != VPX_CODEC_OK) {
  612. std::string msg = base::StringPrintf("VPX flushing error: %s (%s)",
  613. vpx_codec_err_to_string(vpx_error),
  614. vpx_codec_error_detail(codec_.get()));
  615. DLOG(ERROR) << msg;
  616. auto status = EncoderStatus(EncoderStatus::Codes::kEncoderFailedEncode, msg)
  617. .WithData("vpx_error", vpx_error);
  618. std::move(done_cb).Run(std::move(status));
  619. return;
  620. }
  621. DrainOutputs(0, base::TimeDelta(), gfx::ColorSpace());
  622. std::move(done_cb).Run(EncoderStatus::Codes::kOk);
  623. }
  624. void VpxVideoEncoder::DrainOutputs(int temporal_id,
  625. base::TimeDelta ts,
  626. gfx::ColorSpace color_space) {
  627. vpx_codec_iter_t iter = nullptr;
  628. const vpx_codec_cx_pkt_t* pkt = nullptr;
  629. while ((pkt = vpx_codec_get_cx_data(codec_.get(), &iter))) {
  630. if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) {
  631. VideoEncoderOutput result;
  632. result.key_frame = (pkt->data.frame.flags & VPX_FRAME_IS_KEY) != 0;
  633. if (result.key_frame) {
  634. // If we got an unexpected key frame, temporal_svc_frame_index needs to
  635. // be adjusted, because the next frame should have index 1.
  636. temporal_svc_frame_index = 1;
  637. result.temporal_id = 0;
  638. } else {
  639. result.temporal_id = temporal_id;
  640. }
  641. // We don't given timestamps to vpx_codec_encode() that's why
  642. // pkt->data.frame.pts can't be used here.
  643. result.timestamp = ts;
  644. result.color_space = color_space;
  645. result.size = pkt->data.frame.sz;
  646. result.data = std::make_unique<uint8_t[]>(result.size);
  647. memcpy(result.data.get(), pkt->data.frame.buf, result.size);
  648. output_cb_.Run(std::move(result), {});
  649. }
  650. }
  651. }
  652. } // namespace media