webrtc_video_encoder_av1.cc 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419
  1. // Copyright 2022 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "remoting/codec/webrtc_video_encoder_av1.h"
  5. #include "base/callback.h"
  6. #include "base/logging.h"
  7. #include "base/notreached.h"
  8. #include "base/system/sys_info.h"
  9. #include "remoting/base/cpu_utils.h"
  10. #include "remoting/base/util.h"
  11. #include "third_party/libaom/source/libaom/aom/aomcx.h"
  12. #include "third_party/libyuv/include/libyuv/convert_from_argb.h"
  13. #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
  14. #include "third_party/webrtc/modules/desktop_capture/desktop_geometry.h"
  15. #include "third_party/webrtc/modules/desktop_capture/desktop_region.h"
  16. namespace remoting {
  17. namespace {
  18. void DestroyAomCodecContext(aom_codec_ctx_t* codec_ctx) {
  19. if (codec_ctx->name) {
  20. // Codec has been initialized so we need to destroy it.
  21. auto error = aom_codec_destroy(codec_ctx);
  22. DCHECK_EQ(error, AOM_CODEC_OK);
  23. }
  24. delete codec_ctx;
  25. }
  26. // Minimum target bitrate per megapixel.
  27. // TODO(joedow): Perform some additional testing to see if this needs tweaking.
  28. constexpr int kAv1MinimumTargetBitrateKbpsPerMegapixel = 2500;
  29. } // namespace
  30. WebrtcVideoEncoderAV1::WebrtcVideoEncoderAV1()
  31. : codec_(nullptr, DestroyAomCodecContext),
  32. image_(nullptr, aom_img_free),
  33. bitrate_filter_(kAv1MinimumTargetBitrateKbpsPerMegapixel) {
  34. ConfigureCodecParams();
  35. }
  36. WebrtcVideoEncoderAV1::~WebrtcVideoEncoderAV1() = default;
  37. void WebrtcVideoEncoderAV1::SetLosslessEncode(bool want_lossless) {
  38. NOTIMPLEMENTED();
  39. }
  40. void WebrtcVideoEncoderAV1::SetLosslessColor(bool want_lossless) {
  41. NOTIMPLEMENTED();
  42. }
  43. bool WebrtcVideoEncoderAV1::InitializeCodec(const webrtc::DesktopSize& size) {
  44. // Set the width, height, and thread count now that the frame size is known.
  45. config_.g_w = size.width();
  46. config_.g_h = size.height();
  47. // Determine the number of threads to use for encoding by choosing the larger
  48. // of the two dimensions. If we only checked the width, the performance for
  49. // displays in portrait mode will be degraded due to a lower thread count.
  50. // Since AV1 supports dividing the image into both rows and cols, we set the
  51. // maximum number of threads here and then set the tile_row and tile_column
  52. // values based on the frame dimensions later on.
  53. config_.g_threads = GetEncoderThreadCount(std::max(config_.g_w, config_.g_h));
  54. // Initialize an encoder instance.
  55. scoped_aom_codec codec(new aom_codec_ctx_t, DestroyAomCodecContext);
  56. codec->name = nullptr;
  57. aom_codec_flags_t flags = 0;
  58. auto error =
  59. aom_codec_enc_init(codec.get(), aom_codec_av1_cx(), &config_, flags);
  60. if (error != AOM_CODEC_OK) {
  61. LOG(ERROR) << "Failed to initialize codec: " << error;
  62. return false;
  63. }
  64. DCHECK_NE(codec->name, nullptr);
  65. if (use_active_map_) {
  66. active_map_.Initialize(size);
  67. }
  68. error = aom_codec_control(codec.get(), AOME_SET_CPUUSED, 10);
  69. DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AOME_SET_CPUUSED";
  70. error = aom_codec_control(codec.get(), AV1E_SET_AQ_MODE, 3);
  71. DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_AQ_MODE";
  72. if (config_.g_threads > 1) {
  73. error = aom_codec_control(codec.get(), AV1E_SET_ROW_MT, 1);
  74. DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ROW_MT";
  75. }
  76. // The param used to set the tile columns and tile rows is in log2 so 0 is ok.
  77. int log2_threads = static_cast<int>(std::log2(config_.g_threads));
  78. // Split the cols/rows as evenly as possible, favoring columns for widescreen.
  79. int log2_cols = (log2_threads + 1) / 2;
  80. int log2_rows = log2_threads - log2_cols;
  81. if (size.height() > size.width()) {
  82. // Swap for portrait mode since we want more rows than columns.
  83. std::swap(log2_cols, log2_rows);
  84. }
  85. error = aom_codec_control(codec.get(), AV1E_SET_TILE_COLUMNS, log2_cols);
  86. DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_TILE_COLUMNS";
  87. error = aom_codec_control(codec.get(), AV1E_SET_TILE_ROWS, log2_rows);
  88. DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_TILE_ROWS";
  89. // These make realtime encoding faster.
  90. error =
  91. aom_codec_control(codec.get(), AV1E_SET_TUNE_CONTENT, AOM_CONTENT_SCREEN);
  92. DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AOM_CONTENT_SCREEN";
  93. error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_PALETTE, 1);
  94. DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_PALETTE";
  95. error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_INTRABC, 0);
  96. DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_INTRABC";
  97. // These default to on but should not be used for real-time encoding.
  98. error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_TPL_MODEL, 0);
  99. DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_TPL_MODEL";
  100. error = aom_codec_control(codec.get(), AV1E_SET_DELTAQ_MODE, 0);
  101. DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_DELTAQ_MODE";
  102. error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_ORDER_HINT, 0);
  103. DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_ORDER_HINT";
  104. error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_OBMC, 0);
  105. DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_OBMC";
  106. error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_WARPED_MOTION, 0);
  107. DCHECK_EQ(error, AOM_CODEC_OK)
  108. << "Failed to set AV1E_SET_ENABLE_WARPED_MOTION";
  109. error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_GLOBAL_MOTION, 0);
  110. DCHECK_EQ(error, AOM_CODEC_OK)
  111. << "Failed to set AV1E_SET_ENABLE_GLOBAL_MOTION";
  112. error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_REF_FRAME_MVS, 0);
  113. DCHECK_EQ(error, AOM_CODEC_OK)
  114. << "Failed to set AV1E_SET_ENABLE_REF_FRAME_MVS";
  115. // These should significantly speed up key frame encoding.
  116. error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_CFL_INTRA, 0);
  117. DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_CFL_INTRA";
  118. error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_SMOOTH_INTRA, 0);
  119. DCHECK_EQ(error, AOM_CODEC_OK)
  120. << "Failed to set AV1E_SET_ENABLE_SMOOTH_INTRA";
  121. error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_ANGLE_DELTA, 0);
  122. DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_ANGLE_DELTA";
  123. error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_FILTER_INTRA, 0);
  124. DCHECK_EQ(error, AOM_CODEC_OK)
  125. << "Failed to set AV1E_SET_ENABLE_FILTER_INTRA";
  126. error = aom_codec_control(codec.get(), AV1E_SET_INTRA_DEFAULT_TX_ONLY, 1);
  127. DCHECK_EQ(error, AOM_CODEC_OK)
  128. << "Failed to set AV1E_SET_INTRA_DEFAULT_TX_ONLY";
  129. // Misc. quality settings.
  130. error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_CDEF, 1);
  131. DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_CDEF";
  132. error = aom_codec_control(codec.get(), AV1E_SET_NOISE_SENSITIVITY, 0);
  133. DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_NOISE_SENSITIVITY";
  134. error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_PAETH_INTRA, 0);
  135. DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_PAETH_INTRA";
  136. error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_QM, 0);
  137. DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_QM";
  138. error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_RECT_PARTITIONS, 0);
  139. DCHECK_EQ(error, AOM_CODEC_OK)
  140. << "Failed to set AV1E_SET_ENABLE_RECT_PARTITIONS";
  141. error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_RESTORATION, 0);
  142. DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_RESTORATION";
  143. error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_SMOOTH_INTERINTRA, 0);
  144. DCHECK_EQ(error, AOM_CODEC_OK)
  145. << "Failed to set AV1E_SET_ENABLE_SMOOTH_INTERINTRA";
  146. error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_TX64, 0);
  147. DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_TX64";
  148. error = aom_codec_control(codec.get(), AV1E_SET_MAX_REFERENCE_FRAMES, 3);
  149. DCHECK_EQ(error, AOM_CODEC_OK)
  150. << "Failed to set AV1E_SET_MAX_REFERENCE_FRAMES";
  151. codec_ = std::move(codec);
  152. return true;
  153. }
  154. void WebrtcVideoEncoderAV1::PrepareImage(
  155. const webrtc::DesktopFrame* frame,
  156. webrtc::DesktopRegion& updated_region) {
  157. updated_region.Clear();
  158. if (!frame) {
  159. return;
  160. }
  161. if (image_) {
  162. DCHECK_EQ(frame->size().width(), static_cast<int>(image_->d_w));
  163. DCHECK_EQ(frame->size().height(), static_cast<int>(image_->d_h));
  164. for (webrtc::DesktopRegion::Iterator r(frame->updated_region());
  165. !r.IsAtEnd(); r.Advance()) {
  166. const webrtc::DesktopRect& rect = r.rect();
  167. updated_region.AddRect(AlignRect(webrtc::DesktopRect::MakeLTRB(
  168. rect.left(), rect.top(), rect.right(), rect.bottom())));
  169. }
  170. // Clip back to the screen dimensions.
  171. updated_region.IntersectWith(
  172. webrtc::DesktopRect::MakeWH(image_->d_w, image_->d_h));
  173. } else {
  174. image_.reset(aom_img_alloc(nullptr, AOM_IMG_FMT_I420, frame->size().width(),
  175. frame->size().height(),
  176. GetSimdMemoryAlignment()));
  177. updated_region.AddRect(
  178. webrtc::DesktopRect::MakeWH(image_->d_w, image_->d_h));
  179. }
  180. // Convert the updated region to YUV ready for encoding.
  181. const uint8_t* rgb_data = frame->data();
  182. const int rgb_stride = frame->stride();
  183. const int y_stride = image_->stride[0];
  184. DCHECK_EQ(image_->stride[1], image_->stride[2]);
  185. const int uv_stride = image_->stride[1];
  186. uint8_t* y_data = image_->planes[0];
  187. uint8_t* u_data = image_->planes[1];
  188. uint8_t* v_data = image_->planes[2];
  189. CHECK_EQ(image_->fmt, AOM_IMG_FMT_I420);
  190. for (webrtc::DesktopRegion::Iterator r(updated_region); !r.IsAtEnd();
  191. r.Advance()) {
  192. webrtc::DesktopRect rect = GetRowAlignedRect(r.rect(), image_->d_w);
  193. int rgb_offset = rgb_stride * rect.top() +
  194. rect.left() * webrtc::DesktopFrame::kBytesPerPixel;
  195. int y_offset = y_stride * rect.top() + rect.left();
  196. int uv_offset = uv_stride * rect.top() / 2 + rect.left() / 2;
  197. libyuv::ARGBToI420(rgb_data + rgb_offset, rgb_stride, y_data + y_offset,
  198. y_stride, u_data + uv_offset, uv_stride,
  199. v_data + uv_offset, uv_stride, rect.width(),
  200. rect.height());
  201. }
  202. }
  203. void WebrtcVideoEncoderAV1::ConfigureCodecParams() {
  204. auto error = aom_codec_enc_config_default(aom_codec_av1_cx(), &config_,
  205. AOM_USAGE_REALTIME);
  206. if (error != AOM_CODEC_OK) {
  207. LOG(ERROR) << "Failed to configure default codec params: " << error;
  208. return;
  209. }
  210. // Encoder config values are defined in:
  211. // //third_party/libaom/source/libaom/aom/aom_encoder.h
  212. config_.g_profile = 0;
  213. // Width, height, and thread count are set once the frame size is known.
  214. config_.g_w = 0;
  215. config_.g_h = 0;
  216. config_.g_threads = 0;
  217. config_.g_input_bit_depth = 8;
  218. config_.g_pass = AOM_RC_ONE_PASS;
  219. config_.g_lag_in_frames = 0;
  220. config_.g_error_resilient = 0;
  221. config_.g_timebase.num = 1;
  222. config_.g_timebase.den = base::Time::kMicrosecondsPerSecond;
  223. config_.kf_mode = AOM_KF_DISABLED;
  224. config_.rc_max_quantizer = 0;
  225. config_.rc_min_quantizer = 0;
  226. config_.rc_dropframe_thresh = 0;
  227. config_.rc_undershoot_pct = 50;
  228. config_.rc_overshoot_pct = 50;
  229. config_.rc_buf_initial_sz = 600;
  230. config_.rc_buf_optimal_sz = 600;
  231. config_.rc_buf_sz = 1000;
  232. config_.rc_end_usage = AOM_CBR;
  233. config_.rc_target_bitrate = 1024; // 1024 Kbps = 1Mbps.
  234. }
  235. void WebrtcVideoEncoderAV1::UpdateConfig(const FrameParams& params) {
  236. bool changed = false;
  237. if (params.bitrate_kbps >= 0) {
  238. bitrate_filter_.SetBandwidthEstimateKbps(params.bitrate_kbps);
  239. if (config_.rc_target_bitrate !=
  240. static_cast<unsigned int>(bitrate_filter_.GetTargetBitrateKbps())) {
  241. config_.rc_target_bitrate = bitrate_filter_.GetTargetBitrateKbps();
  242. changed = true;
  243. }
  244. }
  245. if (params.vpx_min_quantizer >= 0 &&
  246. config_.rc_min_quantizer !=
  247. static_cast<unsigned int>(params.vpx_min_quantizer)) {
  248. config_.rc_min_quantizer = params.vpx_min_quantizer;
  249. changed = true;
  250. }
  251. if (params.vpx_max_quantizer >= 0 &&
  252. config_.rc_max_quantizer !=
  253. static_cast<unsigned int>(params.vpx_max_quantizer)) {
  254. config_.rc_max_quantizer = params.vpx_max_quantizer;
  255. changed = true;
  256. }
  257. if (!changed)
  258. return;
  259. // Update encoder context.
  260. if (aom_codec_enc_config_set(codec_.get(), &config_)) {
  261. NOTREACHED() << "Unable to set encoder config";
  262. }
  263. }
  264. void WebrtcVideoEncoderAV1::Encode(std::unique_ptr<webrtc::DesktopFrame> frame,
  265. const FrameParams& params,
  266. EncodeCallback done) {
  267. if (frame) {
  268. bitrate_filter_.SetFrameSize(frame->size().width(), frame->size().height());
  269. }
  270. webrtc::DesktopSize previous_frame_size =
  271. image_ ? webrtc::DesktopSize(image_->d_w, image_->d_h)
  272. : webrtc::DesktopSize();
  273. webrtc::DesktopSize frame_size = frame ? frame->size() : previous_frame_size;
  274. if (frame_size.is_empty()) {
  275. std::move(done).Run(EncodeResult::SUCCEEDED, nullptr);
  276. return;
  277. }
  278. DCHECK_GE(frame_size.width(), 32);
  279. DCHECK_GE(frame_size.height(), 32);
  280. // Create or reconfigure the codec to match the size of |frame|.
  281. if (!codec_ || !frame_size.equals(previous_frame_size)) {
  282. image_.reset();
  283. if (!InitializeCodec(frame_size)) {
  284. std::move(done).Run(EncodeResult::UNKNOWN_ERROR, nullptr);
  285. return;
  286. }
  287. }
  288. UpdateConfig(params);
  289. // Transfer the frame data to the image buffer for encoding.
  290. // TODO(joedow): Look into using aom_img_wrap instead of our own buffer.
  291. webrtc::DesktopRegion updated_region;
  292. PrepareImage(frame.get(), updated_region);
  293. aom_active_map_t act_map;
  294. if (use_active_map_) {
  295. if (params.clear_active_map)
  296. active_map_.Clear();
  297. if (params.key_frame)
  298. updated_region.SetRect(webrtc::DesktopRect::MakeSize(frame_size));
  299. active_map_.Update(updated_region);
  300. // Apply active map to the encoder.
  301. act_map.rows = active_map_.height();
  302. act_map.cols = active_map_.width();
  303. act_map.active_map = active_map_.data();
  304. if (aom_codec_control(codec_.get(), AOME_SET_ACTIVEMAP, &act_map)) {
  305. LOG(ERROR) << "Unable to apply active map";
  306. }
  307. }
  308. auto duration_us = params.duration.InMicroseconds();
  309. DCHECK_GT(duration_us, 0);
  310. aom_codec_err_t ret = aom_codec_encode(
  311. codec_.get(), image_.get(), artificial_timestamp_us_, duration_us,
  312. (params.key_frame) ? AOM_EFLAG_FORCE_KF : 0);
  313. artificial_timestamp_us_ += duration_us;
  314. if (ret != AOM_CODEC_OK) {
  315. const char* error_detail = aom_codec_error_detail(codec_.get());
  316. LOG(ERROR) << "Encoding error: " << aom_codec_err_to_string(ret) << "\n "
  317. << aom_codec_error(codec_.get()) << "\n "
  318. << (error_detail ? error_detail : "No error details");
  319. std::move(done).Run(EncodeResult::UNKNOWN_ERROR, nullptr);
  320. return;
  321. }
  322. if (use_active_map_) {
  323. // Update our active map based on the internal map in the encoder.
  324. ret = aom_codec_control(codec_.get(), AV1E_GET_ACTIVEMAP, &act_map);
  325. DCHECK_EQ(ret, AOM_CODEC_OK)
  326. << "Failed to fetch active map: " << aom_codec_err_to_string(ret)
  327. << "\n";
  328. }
  329. // Read the encoded data.
  330. aom_codec_iter_t iter = nullptr;
  331. bool got_data = false;
  332. auto encoded_frame = std::make_unique<EncodedFrame>();
  333. encoded_frame->dimensions = frame_size;
  334. encoded_frame->codec = webrtc::kVideoCodecAV1;
  335. while (!got_data) {
  336. const aom_codec_cx_pkt_t* aom_packet =
  337. aom_codec_get_cx_data(codec_.get(), &iter);
  338. if (!aom_packet)
  339. continue;
  340. switch (aom_packet->kind) {
  341. case AOM_CODEC_CX_FRAME_PKT: {
  342. got_data = true;
  343. encoded_frame->data = webrtc::EncodedImageBuffer::Create(
  344. reinterpret_cast<const uint8_t*>(aom_packet->data.frame.buf),
  345. aom_packet->data.frame.sz);
  346. encoded_frame->key_frame =
  347. aom_packet->data.frame.flags & AOM_FRAME_IS_KEY;
  348. CHECK_EQ(aom_codec_control(codec_.get(), AOME_GET_LAST_QUANTIZER_64,
  349. &(encoded_frame->quantizer)),
  350. AOM_CODEC_OK);
  351. break;
  352. }
  353. default:
  354. break;
  355. }
  356. }
  357. std::move(done).Run(EncodeResult::SUCCEEDED, std::move(encoded_frame));
  358. }
  359. } // namespace remoting