123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547 |
- // Copyright 2016 The Chromium Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style license that can be
- // found in the LICENSE file.
- #include "remoting/codec/webrtc_video_encoder_vpx.h"
- #include <algorithm>
- #include <utility>
- #include "base/bind.h"
- #include "base/callback.h"
- #include "base/cxx17_backports.h"
- #include "base/logging.h"
- #include "base/memory/ptr_util.h"
- #include "base/system/sys_info.h"
- #include "build/build_config.h"
- #include "build/chromeos_buildflags.h"
- #include "remoting/base/cpu_utils.h"
- #include "remoting/base/util.h"
- #include "remoting/proto/video.pb.h"
- #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h"
- #include "third_party/libvpx/source/libvpx/vpx/vpx_encoder.h"
- #include "third_party/libyuv/include/libyuv/convert_from_argb.h"
- #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
- #include "third_party/webrtc/modules/desktop_capture/desktop_geometry.h"
- #include "third_party/webrtc/modules/desktop_capture/desktop_region.h"
- namespace remoting {
- namespace {
- // Number of bytes in an RGBx pixel.
- constexpr int kBytesPerRgbPixel = 4;
- // Magic encoder profile numbers for I420 and I444 input formats.
- constexpr int kVp9I420ProfileNumber = 0;
- constexpr int kVp9I444ProfileNumber = 1;
- // Magic encoder constants for adaptive quantization strategy.
- constexpr int kVp9AqModeNone = 0;
- constexpr int kVp9AqModeCyclicRefresh = 3;
- constexpr int kDefaultTargetBitrateKbps = 1000;
- // Minimum target bitrate per megapixel. The value is chosen experimentally such
- // that when screen is not changing the codec converges to the target quantizer
- // above in less than 10 frames.
- // TODO(zijiehe): This value is for VP8 only; reconsider the value for VP9.
- constexpr int kVp8MinimumTargetBitrateKbpsPerMegapixel = 2500;
- // Default values for the encoder speed of the supported codecs.
- constexpr int kVp9LosslessEncodeSpeed = 5;
- constexpr int kVp9DefaultEncoderSpeed = 6;
- constexpr int kVp9MaxEncoderSpeed = 9;
- void SetCommonCodecParameters(vpx_codec_enc_cfg_t* config,
- const webrtc::DesktopSize& size) {
- // Use millisecond granularity time base.
- config->g_timebase.num = 1;
- config->g_timebase.den = base::Time::kMicrosecondsPerSecond;
- config->g_w = size.width();
- config->g_h = size.height();
- config->g_pass = VPX_RC_ONE_PASS;
- // TODO(joedow): Determine whether it is possible to support portrait mode.
- // VP9 only supports breaking an image up into columns for parallel encoding,
- // this means that a display in portrait mode cannot be broken up into as many
- // columns as a display in landscape mode can so performance will be degraded.
- config->g_threads = WebrtcVideoEncoder::GetEncoderThreadCount(config->g_w);
- // Start emitting packets immediately.
- config->g_lag_in_frames = 0;
- // Since the transport layer is reliable, keyframes should not be necessary.
- // However, due to crbug.com/440223, decoding fails after 30,000 non-key
- // frames, so take the hit of an "unnecessary" key-frame every 10,000 frames.
- config->kf_min_dist = 10000;
- config->kf_max_dist = 10000;
- // Do not drop any frames at encoder.
- config->rc_dropframe_thresh = 0;
- // We do not want variations in bandwidth.
- config->rc_end_usage = VPX_CBR;
- config->rc_undershoot_pct = 100;
- config->rc_overshoot_pct = 15;
- }
- void SetVp8CodecParameters(vpx_codec_enc_cfg_t* config,
- const webrtc::DesktopSize& size) {
- SetCommonCodecParameters(config, size);
- #if BUILDFLAG(IS_LINUX) && !BUILDFLAG(IS_CHROMEOS_LACROS)
- // On Linux, using too many threads for VP8 encoding has been linked to high
- // CPU usage on machines that are under stress. See http://crbug.com/1151148.
- // 5/3/2022 update: Perf testing has shown that doubling the number of threads
- // on machines with a large number of cores will improve performance at higher
- // desktop resolutions. Doubling the number of threads leads to ~30% increase
- // in framerate at 4K. This could be increased further however we don't want
- // to risk reintroducing the problem from the bug above so take the safe gains
- // and leave plenty of cores for the non-remoting workload.
- uint threshold = config->g_threads >= 16 ? 4U : 2U;
- config->g_threads = std::min(config->g_threads, threshold);
- #endif // BUILDFLAG(IS_LINUX) && !BUILDFLAG(IS_CHROMEOS_LACROS)
- // Value of 2 means using the real time profile. This is basically a
- // redundant option since we explicitly select real time mode when doing
- // encoding.
- config->g_profile = 2;
- }
- void SetVp9CodecParameters(vpx_codec_enc_cfg_t* config,
- const webrtc::DesktopSize& size,
- bool lossless_color,
- bool lossless_encode) {
- SetCommonCodecParameters(config, size);
- // Configure VP9 for I420 or I444 source frames.
- config->g_profile =
- lossless_color ? kVp9I444ProfileNumber : kVp9I420ProfileNumber;
- if (lossless_encode) {
- // Disable quantization entirely, putting the encoder in "lossless" mode.
- config->rc_min_quantizer = 0;
- config->rc_max_quantizer = 0;
- config->rc_end_usage = VPX_VBR;
- } else {
- config->rc_end_usage = VPX_CBR;
- // In the absence of a good bandwidth estimator set the target bitrate to a
- // conservative default.
- config->rc_target_bitrate = 500;
- }
- }
- void SetVp8CodecOptions(vpx_codec_ctx_t* codec) {
- // CPUUSED of 16 will have the smallest CPU load. This turns off sub-pixel
- // motion search.
- vpx_codec_err_t ret = vpx_codec_control(codec, VP8E_SET_CPUUSED, 16);
- DCHECK_EQ(VPX_CODEC_OK, ret) << "Failed to set CPUUSED";
- // Use the lowest level of noise sensitivity so as to spend less time
- // on motion estimation and inter-prediction mode.
- ret = vpx_codec_control(codec, VP8E_SET_NOISE_SENSITIVITY, 0);
- DCHECK_EQ(VPX_CODEC_OK, ret) << "Failed to set noise sensitivity";
- }
- void SetVp9CodecOptions(vpx_codec_ctx_t* codec,
- bool lossless_encode,
- int encoder_speed) {
- // Note that this knob uses the same parameter name as VP8.
- vpx_codec_err_t ret =
- vpx_codec_control(codec, VP8E_SET_CPUUSED, encoder_speed);
- DCHECK_EQ(VPX_CODEC_OK, ret) << "Failed to set CPUUSED";
- // Turn on row-based multi-threading if more than one thread is available.
- if (codec->config.enc->g_threads > 1) {
- vpx_codec_control(codec, VP9E_SET_ROW_MT, 1);
- }
- // The param for this knob is a log2 value so 0 is reasonable here.
- vpx_codec_control(codec, VP9E_SET_TILE_COLUMNS,
- static_cast<int>(std::log2(codec->config.enc->g_threads)));
- // Use the lowest level of noise sensitivity so as to spend less time
- // on motion estimation and inter-prediction mode.
- ret = vpx_codec_control(codec, VP9E_SET_NOISE_SENSITIVITY, 0);
- DCHECK_EQ(VPX_CODEC_OK, ret) << "Failed to set noise sensitivity";
- // Configure the codec to tune it for screen media.
- ret = vpx_codec_control(codec, VP9E_SET_TUNE_CONTENT, VP9E_CONTENT_SCREEN);
- DCHECK_EQ(VPX_CODEC_OK, ret) << "Failed to set screen content mode";
- // Set cyclic refresh (aka "top-off") only for lossy encoding.
- int aq_mode = lossless_encode ? kVp9AqModeNone : kVp9AqModeCyclicRefresh;
- ret = vpx_codec_control(codec, VP9E_SET_AQ_MODE, aq_mode);
- DCHECK_EQ(VPX_CODEC_OK, ret) << "Failed to set aq mode";
- }
- } // namespace
- // static
- std::unique_ptr<WebrtcVideoEncoder> WebrtcVideoEncoderVpx::CreateForVP8() {
- LOG(WARNING) << "VP8 video encoder is created.";
- return base::WrapUnique(new WebrtcVideoEncoderVpx(false));
- }
- // static
- std::unique_ptr<WebrtcVideoEncoder> WebrtcVideoEncoderVpx::CreateForVP9() {
- LOG(WARNING) << "VP9 video encoder is created.";
- return base::WrapUnique(new WebrtcVideoEncoderVpx(true));
- }
- WebrtcVideoEncoderVpx::~WebrtcVideoEncoderVpx() = default;
- void WebrtcVideoEncoderVpx::SetTickClockForTests(
- const base::TickClock* tick_clock) {
- clock_ = tick_clock;
- }
- void WebrtcVideoEncoderVpx::SetLosslessEncode(bool want_lossless) {
- if (!use_vp9_)
- return;
- if (want_lossless != lossless_encode_) {
- lossless_encode_ = want_lossless;
- SetEncoderSpeed(lossless_encode_ ? kVp9LosslessEncodeSpeed
- : kVp9DefaultEncoderSpeed);
- if (codec_)
- Configure(webrtc::DesktopSize(codec_->config.enc->g_w,
- codec_->config.enc->g_h));
- }
- }
- void WebrtcVideoEncoderVpx::SetLosslessColor(bool want_lossless) {
- if (!use_vp9_)
- return;
- if (want_lossless != lossless_color_) {
- lossless_color_ = want_lossless;
- // TODO(wez): Switch to ConfigureCodec() path once libvpx supports it.
- // See https://code.google.com/p/webm/issues/detail?id=913.
- // if (codec_)
- // Configure(webrtc::DesktopSize(codec_->config.enc->g_w,
- // codec_->config.enc->g_h));
- codec_.reset();
- }
- }
- void WebrtcVideoEncoderVpx::SetEncoderSpeed(int encoder_speed) {
- if (!use_vp9_)
- return;
- vp9_encoder_speed_ = base::clamp<int>(encoder_speed, kVp9LosslessEncodeSpeed,
- kVp9MaxEncoderSpeed);
- }
- void WebrtcVideoEncoderVpx::Encode(std::unique_ptr<webrtc::DesktopFrame> frame,
- const FrameParams& params,
- EncodeCallback done) {
- // TODO(zijiehe): Replace "if (frame)" with "DCHECK(frame)".
- if (frame) {
- bitrate_filter_.SetFrameSize(frame->size().width(), frame->size().height());
- }
- webrtc::DesktopSize previous_frame_size =
- image_ ? webrtc::DesktopSize(image_->d_w, image_->d_h)
- : webrtc::DesktopSize();
- webrtc::DesktopSize frame_size = frame ? frame->size() : previous_frame_size;
- // Don't need to send anything until we get the first non-null frame.
- if (frame_size.is_empty()) {
- std::move(done).Run(EncodeResult::SUCCEEDED, nullptr);
- return;
- }
- DCHECK_GE(frame_size.width(), 32);
- DCHECK_GE(frame_size.height(), 32);
- // Create or reconfigure the codec to match the size of |frame|.
- if (!codec_ || !frame_size.equals(previous_frame_size)) {
- Configure(frame_size);
- }
- UpdateConfig(params);
- webrtc::DesktopRegion updated_region;
- // Convert the updated capture data ready for encode.
- PrepareImage(frame.get(), &updated_region);
- vpx_active_map_t act_map;
- if (use_active_map_) {
- if (params.clear_active_map)
- active_map_.Clear();
- if (params.key_frame)
- updated_region.SetRect(webrtc::DesktopRect::MakeSize(frame_size));
- active_map_.Update(updated_region);
- act_map.rows = active_map_.height();
- act_map.cols = active_map_.width();
- act_map.active_map = active_map_.data();
- if (vpx_codec_control(codec_.get(), VP8E_SET_ACTIVEMAP, &act_map)) {
- LOG(ERROR) << "Unable to apply active map";
- }
- }
- vpx_codec_err_t ret = vpx_codec_encode(
- codec_.get(), image_.get(), 0, params.duration.InMicroseconds(),
- (params.key_frame) ? VPX_EFLAG_FORCE_KF : 0, VPX_DL_REALTIME);
- if (ret != VPX_CODEC_OK) {
- const char* error_detail = vpx_codec_error_detail(codec_.get());
- LOG(ERROR) << "Encoding error: " << vpx_codec_err_to_string(ret) << "\n "
- << "Details: " << vpx_codec_error(codec_.get()) << "\n "
- << (error_detail ? error_detail : "No error details");
- // TODO(zijiehe): A more exact error type is preferred.
- std::move(done).Run(EncodeResult::UNKNOWN_ERROR, nullptr);
- return;
- }
- if (use_active_map_) {
- // VP8 doesn't return an active map so we assume it hasn't changed.
- if (use_vp9_ && !lossless_encode_) {
- ret = vpx_codec_control(codec_.get(), VP9E_GET_ACTIVEMAP, &act_map);
- DCHECK_EQ(ret, VPX_CODEC_OK)
- << "Failed to fetch active map: " << vpx_codec_err_to_string(ret)
- << "\n";
- }
- }
- // Read the encoded data.
- vpx_codec_iter_t iter = nullptr;
- bool got_data = false;
- auto encoded_frame = std::make_unique<EncodedFrame>();
- encoded_frame->dimensions = frame_size;
- if (use_vp9_) {
- encoded_frame->codec = webrtc::kVideoCodecVP9;
- } else {
- encoded_frame->codec = webrtc::kVideoCodecVP8;
- }
- while (!got_data) {
- const vpx_codec_cx_pkt_t* vpx_packet =
- vpx_codec_get_cx_data(codec_.get(), &iter);
- if (!vpx_packet)
- continue;
- switch (vpx_packet->kind) {
- case VPX_CODEC_CX_FRAME_PKT: {
- got_data = true;
- encoded_frame->data = webrtc::EncodedImageBuffer::Create(
- reinterpret_cast<const uint8_t*>(vpx_packet->data.frame.buf),
- vpx_packet->data.frame.sz);
- encoded_frame->key_frame =
- vpx_packet->data.frame.flags & VPX_FRAME_IS_KEY;
- CHECK_EQ(vpx_codec_control(codec_.get(), VP8E_GET_LAST_QUANTIZER_64,
- &(encoded_frame->quantizer)),
- VPX_CODEC_OK);
- break;
- }
- default:
- break;
- }
- }
- std::move(done).Run(EncodeResult::SUCCEEDED, std::move(encoded_frame));
- }
- WebrtcVideoEncoderVpx::WebrtcVideoEncoderVpx(bool use_vp9)
- : use_vp9_(use_vp9),
- image_(nullptr, vpx_img_free),
- clock_(base::DefaultTickClock::GetInstance()),
- bitrate_filter_(kVp8MinimumTargetBitrateKbpsPerMegapixel) {
- // Indicates config is still uninitialized.
- config_.g_timebase.den = 0;
- if (use_vp9_) {
- SetEncoderSpeed(kVp9DefaultEncoderSpeed);
- }
- }
- void WebrtcVideoEncoderVpx::Configure(const webrtc::DesktopSize& size) {
- DCHECK(use_vp9_ || !lossless_color_);
- DCHECK(use_vp9_ || !lossless_encode_);
- if (use_vp9_) {
- VLOG(0) << "Configuring VP9 encoder with lossless-color="
- << (lossless_color_ ? "true" : "false")
- << ", lossless-encode=" << (lossless_encode_ ? "true" : "false")
- << ".";
- }
- // Tear down |image_| if it doesn't match the new frame size.
- if (image_ && !size.equals(webrtc::DesktopSize(image_->d_w, image_->d_h))) {
- image_.reset();
- }
- // Tear down |codec_| if the new size does not match what this codec instance
- // was configured for.
- if (codec_ && !size.equals(webrtc::DesktopSize(codec_->config.enc->g_w,
- codec_->config.enc->g_h))) {
- codec_.reset();
- }
- if (use_active_map_) {
- active_map_.Initialize(size);
- }
- // Fetch a default configuration for the desired codec.
- const vpx_codec_iface_t* interface =
- use_vp9_ ? vpx_codec_vp9_cx() : vpx_codec_vp8_cx();
- vpx_codec_err_t ret = vpx_codec_enc_config_default(interface, &config_, 0);
- DCHECK_EQ(VPX_CODEC_OK, ret) << "Failed to fetch default configuration";
- // Customize the default configuration to our needs.
- if (use_vp9_) {
- SetVp9CodecParameters(&config_, size, lossless_color_, lossless_encode_);
- } else {
- SetVp8CodecParameters(&config_, size);
- }
- config_.rc_target_bitrate = kDefaultTargetBitrateKbps;
- // Initialize or re-configure the codec with the custom configuration.
- if (!codec_) {
- codec_.reset(new vpx_codec_ctx_t);
- ret = vpx_codec_enc_init(codec_.get(), interface, &config_, 0);
- CHECK_EQ(VPX_CODEC_OK, ret) << "Failed to initialize codec";
- } else {
- ret = vpx_codec_enc_config_set(codec_.get(), &config_);
- CHECK_EQ(VPX_CODEC_OK, ret) << "Failed to reconfigure codec";
- }
- // Apply further customizations to the codec now it's initialized.
- if (use_vp9_) {
- SetVp9CodecOptions(codec_.get(), lossless_encode_, vp9_encoder_speed_);
- } else {
- SetVp8CodecOptions(codec_.get());
- }
- }
- void WebrtcVideoEncoderVpx::UpdateConfig(const FrameParams& params) {
- // Configuration not initialized.
- if (config_.g_timebase.den == 0)
- return;
- bool changed = false;
- if (params.bitrate_kbps >= 0) {
- bitrate_filter_.SetBandwidthEstimateKbps(params.bitrate_kbps);
- if (config_.rc_target_bitrate !=
- static_cast<unsigned int>(bitrate_filter_.GetTargetBitrateKbps())) {
- config_.rc_target_bitrate = bitrate_filter_.GetTargetBitrateKbps();
- changed = true;
- }
- }
- if (params.vpx_min_quantizer >= 0 &&
- config_.rc_min_quantizer !=
- static_cast<unsigned int>(params.vpx_min_quantizer)) {
- config_.rc_min_quantizer = params.vpx_min_quantizer;
- changed = true;
- }
- if (params.vpx_max_quantizer >= 0 &&
- config_.rc_max_quantizer !=
- static_cast<unsigned int>(params.vpx_max_quantizer)) {
- config_.rc_max_quantizer = params.vpx_max_quantizer;
- changed = true;
- }
- if (!changed)
- return;
- // Update encoder context.
- if (vpx_codec_enc_config_set(codec_.get(), &config_))
- NOTREACHED() << "Unable to set encoder config";
- }
- void WebrtcVideoEncoderVpx::PrepareImage(
- const webrtc::DesktopFrame* frame,
- webrtc::DesktopRegion* updated_region) {
- updated_region->Clear();
- if (!frame) {
- return;
- }
- if (image_) {
- // Pad each rectangle to avoid the block-artifact filters in libvpx from
- // introducing artifacts; VP9 includes up to 8px either side, and VP8 up to
- // 3px, so unchanged pixels up to that far out may still be affected by the
- // changes in the updated region, and so must be listed in the active map.
- // After padding we align each rectangle to 16x16 active-map macroblocks.
- // This implicitly ensures all rects have even top-left coords, which is
- // is required by ConvertRGBToYUVWithRect().
- // TODO(wez): Do we still need 16x16 align, or is even alignment sufficient?
- int padding = use_vp9_ ? 8 : 3;
- for (webrtc::DesktopRegion::Iterator r(frame->updated_region());
- !r.IsAtEnd(); r.Advance()) {
- const webrtc::DesktopRect& rect = r.rect();
- updated_region->AddRect(AlignRect(webrtc::DesktopRect::MakeLTRB(
- rect.left() - padding, rect.top() - padding, rect.right() + padding,
- rect.bottom() + padding)));
- }
- // Clip back to the screen dimensions, in case they're not macroblock
- // aligned. The conversion routines don't require even width & height,
- // so this is safe even if the source dimensions are not even.
- updated_region->IntersectWith(
- webrtc::DesktopRect::MakeWH(image_->d_w, image_->d_h));
- } else {
- vpx_img_fmt_t fmt = lossless_color_ ? VPX_IMG_FMT_I444 : VPX_IMG_FMT_YV12;
- image_.reset(vpx_img_alloc(nullptr, fmt, frame->size().width(),
- frame->size().height(),
- GetSimdMemoryAlignment()));
- updated_region->AddRect(
- webrtc::DesktopRect::MakeWH(image_->d_w, image_->d_h));
- }
- // Convert the updated region to YUV ready for encoding.
- const uint8_t* rgb_data = frame->data();
- const int rgb_stride = frame->stride();
- const int y_stride = image_->stride[0];
- DCHECK_EQ(image_->stride[1], image_->stride[2]);
- const int uv_stride = image_->stride[1];
- uint8_t* y_data = image_->planes[0];
- uint8_t* u_data = image_->planes[1];
- uint8_t* v_data = image_->planes[2];
- switch (image_->fmt) {
- case VPX_IMG_FMT_I444:
- for (webrtc::DesktopRegion::Iterator r(*updated_region); !r.IsAtEnd();
- r.Advance()) {
- webrtc::DesktopRect rect = GetRowAlignedRect(r.rect(), image_->d_w);
- int rgb_offset =
- rgb_stride * rect.top() + rect.left() * kBytesPerRgbPixel;
- int yuv_offset = uv_stride * rect.top() + rect.left();
- libyuv::ARGBToI444(rgb_data + rgb_offset, rgb_stride,
- y_data + yuv_offset, y_stride, u_data + yuv_offset,
- uv_stride, v_data + yuv_offset, uv_stride,
- rect.width(), rect.height());
- }
- break;
- case VPX_IMG_FMT_YV12:
- for (webrtc::DesktopRegion::Iterator r(*updated_region); !r.IsAtEnd();
- r.Advance()) {
- webrtc::DesktopRect rect = GetRowAlignedRect(r.rect(), image_->d_w);
- int rgb_offset =
- rgb_stride * rect.top() + rect.left() * kBytesPerRgbPixel;
- int y_offset = y_stride * rect.top() + rect.left();
- int uv_offset = uv_stride * rect.top() / 2 + rect.left() / 2;
- libyuv::ARGBToI420(rgb_data + rgb_offset, rgb_stride, y_data + y_offset,
- y_stride, u_data + uv_offset, uv_stride,
- v_data + uv_offset, uv_stride, rect.width(),
- rect.height());
- }
- break;
- default:
- NOTREACHED();
- break;
- }
- }
- } // namespace remoting
|