123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419 |
- // Copyright 2022 The Chromium Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style license that can be
- // found in the LICENSE file.
- #include "remoting/codec/webrtc_video_encoder_av1.h"
- #include "base/callback.h"
- #include "base/logging.h"
- #include "base/notreached.h"
- #include "base/system/sys_info.h"
- #include "remoting/base/cpu_utils.h"
- #include "remoting/base/util.h"
- #include "third_party/libaom/source/libaom/aom/aomcx.h"
- #include "third_party/libyuv/include/libyuv/convert_from_argb.h"
- #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
- #include "third_party/webrtc/modules/desktop_capture/desktop_geometry.h"
- #include "third_party/webrtc/modules/desktop_capture/desktop_region.h"
- namespace remoting {
- namespace {
- void DestroyAomCodecContext(aom_codec_ctx_t* codec_ctx) {
- if (codec_ctx->name) {
- // Codec has been initialized so we need to destroy it.
- auto error = aom_codec_destroy(codec_ctx);
- DCHECK_EQ(error, AOM_CODEC_OK);
- }
- delete codec_ctx;
- }
- // Minimum target bitrate per megapixel.
- // TODO(joedow): Perform some additional testing to see if this needs tweaking.
- constexpr int kAv1MinimumTargetBitrateKbpsPerMegapixel = 2500;
- } // namespace
- WebrtcVideoEncoderAV1::WebrtcVideoEncoderAV1()
- : codec_(nullptr, DestroyAomCodecContext),
- image_(nullptr, aom_img_free),
- bitrate_filter_(kAv1MinimumTargetBitrateKbpsPerMegapixel) {
- ConfigureCodecParams();
- }
- WebrtcVideoEncoderAV1::~WebrtcVideoEncoderAV1() = default;
- void WebrtcVideoEncoderAV1::SetLosslessEncode(bool want_lossless) {
- NOTIMPLEMENTED();
- }
- void WebrtcVideoEncoderAV1::SetLosslessColor(bool want_lossless) {
- NOTIMPLEMENTED();
- }
- bool WebrtcVideoEncoderAV1::InitializeCodec(const webrtc::DesktopSize& size) {
- // Set the width, height, and thread count now that the frame size is known.
- config_.g_w = size.width();
- config_.g_h = size.height();
- // Determine the number of threads to use for encoding by choosing the larger
- // of the two dimensions. If we only checked the width, the performance for
- // displays in portrait mode will be degraded due to a lower thread count.
- // Since AV1 supports dividing the image into both rows and cols, we set the
- // maximum number of threads here and then set the tile_row and tile_column
- // values based on the frame dimensions later on.
- config_.g_threads = GetEncoderThreadCount(std::max(config_.g_w, config_.g_h));
- // Initialize an encoder instance.
- scoped_aom_codec codec(new aom_codec_ctx_t, DestroyAomCodecContext);
- codec->name = nullptr;
- aom_codec_flags_t flags = 0;
- auto error =
- aom_codec_enc_init(codec.get(), aom_codec_av1_cx(), &config_, flags);
- if (error != AOM_CODEC_OK) {
- LOG(ERROR) << "Failed to initialize codec: " << error;
- return false;
- }
- DCHECK_NE(codec->name, nullptr);
- if (use_active_map_) {
- active_map_.Initialize(size);
- }
- error = aom_codec_control(codec.get(), AOME_SET_CPUUSED, 10);
- DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AOME_SET_CPUUSED";
- error = aom_codec_control(codec.get(), AV1E_SET_AQ_MODE, 3);
- DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_AQ_MODE";
- if (config_.g_threads > 1) {
- error = aom_codec_control(codec.get(), AV1E_SET_ROW_MT, 1);
- DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ROW_MT";
- }
- // The param used to set the tile columns and tile rows is in log2 so 0 is ok.
- int log2_threads = static_cast<int>(std::log2(config_.g_threads));
- // Split the cols/rows as evenly as possible, favoring columns for widescreen.
- int log2_cols = (log2_threads + 1) / 2;
- int log2_rows = log2_threads - log2_cols;
- if (size.height() > size.width()) {
- // Swap for portrait mode since we want more rows than columns.
- std::swap(log2_cols, log2_rows);
- }
- error = aom_codec_control(codec.get(), AV1E_SET_TILE_COLUMNS, log2_cols);
- DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_TILE_COLUMNS";
- error = aom_codec_control(codec.get(), AV1E_SET_TILE_ROWS, log2_rows);
- DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_TILE_ROWS";
- // These make realtime encoding faster.
- error =
- aom_codec_control(codec.get(), AV1E_SET_TUNE_CONTENT, AOM_CONTENT_SCREEN);
- DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AOM_CONTENT_SCREEN";
- error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_PALETTE, 1);
- DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_PALETTE";
- error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_INTRABC, 0);
- DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_INTRABC";
- // These default to on but should not be used for real-time encoding.
- error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_TPL_MODEL, 0);
- DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_TPL_MODEL";
- error = aom_codec_control(codec.get(), AV1E_SET_DELTAQ_MODE, 0);
- DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_DELTAQ_MODE";
- error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_ORDER_HINT, 0);
- DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_ORDER_HINT";
- error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_OBMC, 0);
- DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_OBMC";
- error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_WARPED_MOTION, 0);
- DCHECK_EQ(error, AOM_CODEC_OK)
- << "Failed to set AV1E_SET_ENABLE_WARPED_MOTION";
- error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_GLOBAL_MOTION, 0);
- DCHECK_EQ(error, AOM_CODEC_OK)
- << "Failed to set AV1E_SET_ENABLE_GLOBAL_MOTION";
- error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_REF_FRAME_MVS, 0);
- DCHECK_EQ(error, AOM_CODEC_OK)
- << "Failed to set AV1E_SET_ENABLE_REF_FRAME_MVS";
- // These should significantly speed up key frame encoding.
- error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_CFL_INTRA, 0);
- DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_CFL_INTRA";
- error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_SMOOTH_INTRA, 0);
- DCHECK_EQ(error, AOM_CODEC_OK)
- << "Failed to set AV1E_SET_ENABLE_SMOOTH_INTRA";
- error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_ANGLE_DELTA, 0);
- DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_ANGLE_DELTA";
- error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_FILTER_INTRA, 0);
- DCHECK_EQ(error, AOM_CODEC_OK)
- << "Failed to set AV1E_SET_ENABLE_FILTER_INTRA";
- error = aom_codec_control(codec.get(), AV1E_SET_INTRA_DEFAULT_TX_ONLY, 1);
- DCHECK_EQ(error, AOM_CODEC_OK)
- << "Failed to set AV1E_SET_INTRA_DEFAULT_TX_ONLY";
- // Misc. quality settings.
- error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_CDEF, 1);
- DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_CDEF";
- error = aom_codec_control(codec.get(), AV1E_SET_NOISE_SENSITIVITY, 0);
- DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_NOISE_SENSITIVITY";
- error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_PAETH_INTRA, 0);
- DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_PAETH_INTRA";
- error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_QM, 0);
- DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_QM";
- error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_RECT_PARTITIONS, 0);
- DCHECK_EQ(error, AOM_CODEC_OK)
- << "Failed to set AV1E_SET_ENABLE_RECT_PARTITIONS";
- error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_RESTORATION, 0);
- DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_RESTORATION";
- error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_SMOOTH_INTERINTRA, 0);
- DCHECK_EQ(error, AOM_CODEC_OK)
- << "Failed to set AV1E_SET_ENABLE_SMOOTH_INTERINTRA";
- error = aom_codec_control(codec.get(), AV1E_SET_ENABLE_TX64, 0);
- DCHECK_EQ(error, AOM_CODEC_OK) << "Failed to set AV1E_SET_ENABLE_TX64";
- error = aom_codec_control(codec.get(), AV1E_SET_MAX_REFERENCE_FRAMES, 3);
- DCHECK_EQ(error, AOM_CODEC_OK)
- << "Failed to set AV1E_SET_MAX_REFERENCE_FRAMES";
- codec_ = std::move(codec);
- return true;
- }
- void WebrtcVideoEncoderAV1::PrepareImage(
- const webrtc::DesktopFrame* frame,
- webrtc::DesktopRegion& updated_region) {
- updated_region.Clear();
- if (!frame) {
- return;
- }
- if (image_) {
- DCHECK_EQ(frame->size().width(), static_cast<int>(image_->d_w));
- DCHECK_EQ(frame->size().height(), static_cast<int>(image_->d_h));
- for (webrtc::DesktopRegion::Iterator r(frame->updated_region());
- !r.IsAtEnd(); r.Advance()) {
- const webrtc::DesktopRect& rect = r.rect();
- updated_region.AddRect(AlignRect(webrtc::DesktopRect::MakeLTRB(
- rect.left(), rect.top(), rect.right(), rect.bottom())));
- }
- // Clip back to the screen dimensions.
- updated_region.IntersectWith(
- webrtc::DesktopRect::MakeWH(image_->d_w, image_->d_h));
- } else {
- image_.reset(aom_img_alloc(nullptr, AOM_IMG_FMT_I420, frame->size().width(),
- frame->size().height(),
- GetSimdMemoryAlignment()));
- updated_region.AddRect(
- webrtc::DesktopRect::MakeWH(image_->d_w, image_->d_h));
- }
- // Convert the updated region to YUV ready for encoding.
- const uint8_t* rgb_data = frame->data();
- const int rgb_stride = frame->stride();
- const int y_stride = image_->stride[0];
- DCHECK_EQ(image_->stride[1], image_->stride[2]);
- const int uv_stride = image_->stride[1];
- uint8_t* y_data = image_->planes[0];
- uint8_t* u_data = image_->planes[1];
- uint8_t* v_data = image_->planes[2];
- CHECK_EQ(image_->fmt, AOM_IMG_FMT_I420);
- for (webrtc::DesktopRegion::Iterator r(updated_region); !r.IsAtEnd();
- r.Advance()) {
- webrtc::DesktopRect rect = GetRowAlignedRect(r.rect(), image_->d_w);
- int rgb_offset = rgb_stride * rect.top() +
- rect.left() * webrtc::DesktopFrame::kBytesPerPixel;
- int y_offset = y_stride * rect.top() + rect.left();
- int uv_offset = uv_stride * rect.top() / 2 + rect.left() / 2;
- libyuv::ARGBToI420(rgb_data + rgb_offset, rgb_stride, y_data + y_offset,
- y_stride, u_data + uv_offset, uv_stride,
- v_data + uv_offset, uv_stride, rect.width(),
- rect.height());
- }
- }
- void WebrtcVideoEncoderAV1::ConfigureCodecParams() {
- auto error = aom_codec_enc_config_default(aom_codec_av1_cx(), &config_,
- AOM_USAGE_REALTIME);
- if (error != AOM_CODEC_OK) {
- LOG(ERROR) << "Failed to configure default codec params: " << error;
- return;
- }
- // Encoder config values are defined in:
- // //third_party/libaom/source/libaom/aom/aom_encoder.h
- config_.g_profile = 0;
- // Width, height, and thread count are set once the frame size is known.
- config_.g_w = 0;
- config_.g_h = 0;
- config_.g_threads = 0;
- config_.g_input_bit_depth = 8;
- config_.g_pass = AOM_RC_ONE_PASS;
- config_.g_lag_in_frames = 0;
- config_.g_error_resilient = 0;
- config_.g_timebase.num = 1;
- config_.g_timebase.den = base::Time::kMicrosecondsPerSecond;
- config_.kf_mode = AOM_KF_DISABLED;
- config_.rc_max_quantizer = 0;
- config_.rc_min_quantizer = 0;
- config_.rc_dropframe_thresh = 0;
- config_.rc_undershoot_pct = 50;
- config_.rc_overshoot_pct = 50;
- config_.rc_buf_initial_sz = 600;
- config_.rc_buf_optimal_sz = 600;
- config_.rc_buf_sz = 1000;
- config_.rc_end_usage = AOM_CBR;
- config_.rc_target_bitrate = 1024; // 1024 Kbps = 1Mbps.
- }
- void WebrtcVideoEncoderAV1::UpdateConfig(const FrameParams& params) {
- bool changed = false;
- if (params.bitrate_kbps >= 0) {
- bitrate_filter_.SetBandwidthEstimateKbps(params.bitrate_kbps);
- if (config_.rc_target_bitrate !=
- static_cast<unsigned int>(bitrate_filter_.GetTargetBitrateKbps())) {
- config_.rc_target_bitrate = bitrate_filter_.GetTargetBitrateKbps();
- changed = true;
- }
- }
- if (params.vpx_min_quantizer >= 0 &&
- config_.rc_min_quantizer !=
- static_cast<unsigned int>(params.vpx_min_quantizer)) {
- config_.rc_min_quantizer = params.vpx_min_quantizer;
- changed = true;
- }
- if (params.vpx_max_quantizer >= 0 &&
- config_.rc_max_quantizer !=
- static_cast<unsigned int>(params.vpx_max_quantizer)) {
- config_.rc_max_quantizer = params.vpx_max_quantizer;
- changed = true;
- }
- if (!changed)
- return;
- // Update encoder context.
- if (aom_codec_enc_config_set(codec_.get(), &config_)) {
- NOTREACHED() << "Unable to set encoder config";
- }
- }
- void WebrtcVideoEncoderAV1::Encode(std::unique_ptr<webrtc::DesktopFrame> frame,
- const FrameParams& params,
- EncodeCallback done) {
- if (frame) {
- bitrate_filter_.SetFrameSize(frame->size().width(), frame->size().height());
- }
- webrtc::DesktopSize previous_frame_size =
- image_ ? webrtc::DesktopSize(image_->d_w, image_->d_h)
- : webrtc::DesktopSize();
- webrtc::DesktopSize frame_size = frame ? frame->size() : previous_frame_size;
- if (frame_size.is_empty()) {
- std::move(done).Run(EncodeResult::SUCCEEDED, nullptr);
- return;
- }
- DCHECK_GE(frame_size.width(), 32);
- DCHECK_GE(frame_size.height(), 32);
- // Create or reconfigure the codec to match the size of |frame|.
- if (!codec_ || !frame_size.equals(previous_frame_size)) {
- image_.reset();
- if (!InitializeCodec(frame_size)) {
- std::move(done).Run(EncodeResult::UNKNOWN_ERROR, nullptr);
- return;
- }
- }
- UpdateConfig(params);
- // Transfer the frame data to the image buffer for encoding.
- // TODO(joedow): Look into using aom_img_wrap instead of our own buffer.
- webrtc::DesktopRegion updated_region;
- PrepareImage(frame.get(), updated_region);
- aom_active_map_t act_map;
- if (use_active_map_) {
- if (params.clear_active_map)
- active_map_.Clear();
- if (params.key_frame)
- updated_region.SetRect(webrtc::DesktopRect::MakeSize(frame_size));
- active_map_.Update(updated_region);
- // Apply active map to the encoder.
- act_map.rows = active_map_.height();
- act_map.cols = active_map_.width();
- act_map.active_map = active_map_.data();
- if (aom_codec_control(codec_.get(), AOME_SET_ACTIVEMAP, &act_map)) {
- LOG(ERROR) << "Unable to apply active map";
- }
- }
- auto duration_us = params.duration.InMicroseconds();
- DCHECK_GT(duration_us, 0);
- aom_codec_err_t ret = aom_codec_encode(
- codec_.get(), image_.get(), artificial_timestamp_us_, duration_us,
- (params.key_frame) ? AOM_EFLAG_FORCE_KF : 0);
- artificial_timestamp_us_ += duration_us;
- if (ret != AOM_CODEC_OK) {
- const char* error_detail = aom_codec_error_detail(codec_.get());
- LOG(ERROR) << "Encoding error: " << aom_codec_err_to_string(ret) << "\n "
- << aom_codec_error(codec_.get()) << "\n "
- << (error_detail ? error_detail : "No error details");
- std::move(done).Run(EncodeResult::UNKNOWN_ERROR, nullptr);
- return;
- }
- if (use_active_map_) {
- // Update our active map based on the internal map in the encoder.
- ret = aom_codec_control(codec_.get(), AV1E_GET_ACTIVEMAP, &act_map);
- DCHECK_EQ(ret, AOM_CODEC_OK)
- << "Failed to fetch active map: " << aom_codec_err_to_string(ret)
- << "\n";
- }
- // Read the encoded data.
- aom_codec_iter_t iter = nullptr;
- bool got_data = false;
- auto encoded_frame = std::make_unique<EncodedFrame>();
- encoded_frame->dimensions = frame_size;
- encoded_frame->codec = webrtc::kVideoCodecAV1;
- while (!got_data) {
- const aom_codec_cx_pkt_t* aom_packet =
- aom_codec_get_cx_data(codec_.get(), &iter);
- if (!aom_packet)
- continue;
- switch (aom_packet->kind) {
- case AOM_CODEC_CX_FRAME_PKT: {
- got_data = true;
- encoded_frame->data = webrtc::EncodedImageBuffer::Create(
- reinterpret_cast<const uint8_t*>(aom_packet->data.frame.buf),
- aom_packet->data.frame.sz);
- encoded_frame->key_frame =
- aom_packet->data.frame.flags & AOM_FRAME_IS_KEY;
- CHECK_EQ(aom_codec_control(codec_.get(), AOME_GET_LAST_QUANTIZER_64,
- &(encoded_frame->quantizer)),
- AOM_CODEC_OK);
- break;
- }
- default:
- break;
- }
- }
- std::move(done).Run(EncodeResult::SUCCEEDED, std::move(encoded_frame));
- }
- } // namespace remoting
|