webrtc_video_encoder_wrapper.cc 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570
  1. // Copyright 2021 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "remoting/protocol/webrtc_video_encoder_wrapper.h"
  5. #include <stdint.h>
  6. #include <functional>
  7. #include <string>
  8. #include <vector>
  9. #include "base/bind.h"
  10. #include "base/cxx17_backports.h"
  11. #include "base/logging.h"
  12. #include "base/memory/ptr_util.h"
  13. #include "base/task/bind_post_task.h"
  14. #include "base/threading/sequenced_task_runner_handle.h"
  15. #include "base/time/time.h"
  16. #include "build/build_config.h"
  17. #include "remoting/base/constants.h"
  18. #include "remoting/base/session_options.h"
  19. #include "remoting/codec/webrtc_video_encoder_av1.h"
  20. #include "remoting/codec/webrtc_video_encoder_vpx.h"
  21. #include "remoting/protocol/video_channel_state_observer.h"
  22. #include "remoting/protocol/webrtc_video_frame_adapter.h"
  23. #include "third_party/webrtc/api/video_codecs/sdp_video_format.h"
  24. #include "third_party/webrtc/api/video_codecs/vp9_profile.h"
  25. #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
  26. #include "third_party/webrtc/modules/video_coding/include/video_codec_interface.h"
  27. #include "third_party/webrtc/modules/video_coding/include/video_error_codes.h"
  28. #if defined(USE_H264_ENCODER)
  29. #include "remoting/codec/webrtc_video_encoder_gpu.h"
  30. #endif
  31. namespace remoting::protocol {
  32. namespace {
  33. // Maximum quantizer at which to encode frames. Lowering this value will
  34. // improve image quality (in cases of low-bandwidth or large frames) at the
  35. // cost of latency. Increasing the value will improve latency (in these cases)
  36. // at the cost of image quality, resulting in longer top-off times.
  37. const int kMaxQuantizer = 50;
  38. // Minimum quantizer at which to encode frames. The value is chosen such that
  39. // sending higher-quality (lower quantizer) frames would use up bandwidth
  40. // without any appreciable gain in image quality.
  41. const int kMinQuantizer = 10;
  42. const int64_t kPixelsPerMegapixel = 1000000;
  43. // Threshold in number of updated pixels used to detect "big" frames. These
  44. // frames update significant portion of the screen compared to the preceding
  45. // frames. For these frames min quantizer may need to be adjusted in order to
  46. // ensure that they get delivered to the client as soon as possible, in exchange
  47. // for lower-quality image.
  48. const int kBigFrameThresholdPixels = 300000;
  49. // Estimated size (in bytes per megapixel) of encoded frame at target quantizer
  50. // value (see kTargetQuantizerForTopOff). Compression ratio varies depending
  51. // on the image, so this is just a rough estimate. It's used to predict when
  52. // encoded "big" frame may be too large to be delivered to the client quickly.
  53. const int kEstimatedBytesPerMegapixel = 100000;
  54. // Minimum interval between frames needed to keep the connection alive. The
  55. // client will request a key-frame if it does not receive any frames for a
  56. // 3-second period. This is effectively a minimum frame-rate, so the value
  57. // should not be too small, otherwise the client may waste CPU cycles on
  58. // processing and rendering lots of identical frames.
  59. constexpr base::TimeDelta kKeepAliveInterval = base::Seconds(2);
  60. std::string EncodeResultToString(WebrtcVideoEncoder::EncodeResult result) {
  61. using EncodeResult = WebrtcVideoEncoder::EncodeResult;
  62. switch (result) {
  63. case EncodeResult::SUCCEEDED:
  64. return "Succeeded";
  65. case EncodeResult::FRAME_SIZE_EXCEEDS_CAPABILITY:
  66. return "Frame size exceeds capability";
  67. case EncodeResult::UNKNOWN_ERROR:
  68. return "Unknown error";
  69. }
  70. NOTREACHED();
  71. return "";
  72. }
  73. } // namespace
  74. WebrtcVideoEncoderWrapper::WebrtcVideoEncoderWrapper(
  75. const webrtc::SdpVideoFormat& format,
  76. const SessionOptions& session_options,
  77. scoped_refptr<base::SingleThreadTaskRunner> main_task_runner,
  78. scoped_refptr<base::SingleThreadTaskRunner> encode_task_runner,
  79. base::WeakPtr<VideoChannelStateObserver> video_channel_state_observer)
  80. : main_task_runner_(main_task_runner),
  81. encode_task_runner_(encode_task_runner),
  82. video_channel_state_observer_(video_channel_state_observer) {
  83. // Set the target frame rate based on the session options.
  84. absl::optional<int> frame_rate = session_options.GetInt("Video-Frame-Rate");
  85. if (frame_rate) {
  86. // Clamp the range to prevent a bad experience in case of a client bug.
  87. frame_rate = base::clamp<int>(frame_rate.value(), kTargetFrameRate, 1000);
  88. target_frame_rate_ = frame_rate.value();
  89. }
  90. target_frame_interval_ = base::Milliseconds(1000 / target_frame_rate_);
  91. codec_type_ = webrtc::PayloadStringToCodecType(format.name);
  92. switch (codec_type_) {
  93. case webrtc::kVideoCodecVP8:
  94. VLOG(0) << "Creating VP8 encoder.";
  95. encoder_ = WebrtcVideoEncoderVpx::CreateForVP8();
  96. break;
  97. case webrtc::kVideoCodecVP9: {
  98. absl::optional<webrtc::VP9Profile> profile =
  99. webrtc::ParseSdpForVP9Profile(format.parameters);
  100. bool lossless_color = profile.has_value() &&
  101. profile.value() == webrtc::VP9Profile::kProfile1;
  102. VLOG(0) << "Creating VP9 encoder, lossless_color="
  103. << (lossless_color ? "true" : "false");
  104. encoder_ = WebrtcVideoEncoderVpx::CreateForVP9();
  105. encoder_->SetLosslessColor(lossless_color);
  106. absl::optional<int> encoder_speed =
  107. session_options.GetInt("Vp9-Encoder-Speed");
  108. if (encoder_speed) {
  109. VLOG(0) << "Setting VP9 encoder speed to " << encoder_speed.value();
  110. encoder_->SetEncoderSpeed(encoder_speed.value());
  111. }
  112. break;
  113. }
  114. case webrtc::kVideoCodecH264:
  115. #if defined(USE_H264_ENCODER)
  116. VLOG(0) << "Creating H264 encoder.";
  117. encoder_ = WebrtcVideoEncoderGpu::CreateForH264();
  118. #else
  119. NOTIMPLEMENTED();
  120. #endif
  121. break;
  122. case webrtc::kVideoCodecAV1:
  123. VLOG(0) << "Creating AV1 encoder.";
  124. encoder_ = std::make_unique<WebrtcVideoEncoderAV1>();
  125. break;
  126. default:
  127. LOG(FATAL) << "Unknown codec type: " << codec_type_;
  128. }
  129. }
  130. WebrtcVideoEncoderWrapper::~WebrtcVideoEncoderWrapper() {
  131. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  132. if (encode_pending_) {
  133. // If the encoder is still running, then delete it on |encode_task_runner_|
  134. // as it will no longer be called on this sequence and isn't sequence bound.
  135. encode_task_runner_->DeleteSoon(FROM_HERE, encoder_.release());
  136. }
  137. }
  138. void WebrtcVideoEncoderWrapper::SetEncoderForTest(
  139. std::unique_ptr<WebrtcVideoEncoder> encoder) {
  140. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  141. encoder_ = std::move(encoder);
  142. }
  143. int32_t WebrtcVideoEncoderWrapper::InitEncode(
  144. const webrtc::VideoCodec* codec_settings,
  145. const webrtc::VideoEncoder::Settings& settings) {
  146. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  147. DCHECK(codec_settings);
  148. DCHECK_EQ(codec_settings->codecType, codec_type_);
  149. // Validate request is to support a single stream.
  150. DCHECK_EQ(1, codec_settings->numberOfSimulcastStreams);
  151. if (codec_type_ == webrtc::kVideoCodecVP9) {
  152. // SVC is not supported.
  153. DCHECK_EQ(1, codec_settings->VP9().numberOfSpatialLayers);
  154. }
  155. return WEBRTC_VIDEO_CODEC_OK;
  156. }
  157. int32_t WebrtcVideoEncoderWrapper::RegisterEncodeCompleteCallback(
  158. webrtc::EncodedImageCallback* callback) {
  159. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  160. encoded_callback_ = callback;
  161. return WEBRTC_VIDEO_CODEC_OK;
  162. }
  163. int32_t WebrtcVideoEncoderWrapper::Release() {
  164. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  165. encoded_callback_ = nullptr;
  166. return WEBRTC_VIDEO_CODEC_OK;
  167. }
  168. int32_t WebrtcVideoEncoderWrapper::Encode(
  169. const webrtc::VideoFrame& frame,
  170. const std::vector<webrtc::VideoFrameType>* frame_types) {
  171. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  172. auto now = base::TimeTicks::Now();
  173. // Simulcast is unsupported, so only the first vector element is needed.
  174. bool key_frame_requested =
  175. (frame_types && !frame_types->empty() &&
  176. ((*frame_types)[0] == webrtc::VideoFrameType::kVideoFrameKey));
  177. if (key_frame_requested) {
  178. pending_key_frame_request_ = true;
  179. }
  180. bool webrtc_dropped_frame = false;
  181. if (next_frame_id_ != frame.id()) {
  182. webrtc_dropped_frame = true;
  183. next_frame_id_ = frame.id();
  184. }
  185. next_frame_id_++;
  186. // WebRTC calls Encode() after each successful capture. If we drop the frame
  187. // immediately when we are currently encoding instead of storing the frame
  188. // data, then the encoder would need to wait until the next capture request
  189. // has succeeded before it can encode another frame, this period can be
  190. // several milliseconds or more. To reduce this latency, we store the new
  191. // frame when the encoder is busy so it can be encoded immediately after the
  192. // encoder finishes the current frame.
  193. if (encode_pending_) {
  194. if (pending_frame_) {
  195. accumulated_update_rect_.Union(pending_frame_->update_rect());
  196. base::SequencedTaskRunnerHandle::Get()->PostTask(
  197. FROM_HERE,
  198. base::BindOnce(&WebrtcVideoEncoderWrapper::NotifyFrameDropped,
  199. weak_factory_.GetWeakPtr()));
  200. }
  201. pending_frame_ = std::make_unique<webrtc::VideoFrame>(frame);
  202. return WEBRTC_VIDEO_CODEC_OK;
  203. }
  204. // Frames of type kNative are expected to have the adapter that was used to
  205. // wrap the DesktopFrame, so the downcast should be safe.
  206. if (frame.video_frame_buffer()->type() !=
  207. webrtc::VideoFrameBuffer::Type::kNative) {
  208. LOG(ERROR) << "Only kNative frames are supported.";
  209. return WEBRTC_VIDEO_CODEC_ERROR;
  210. }
  211. auto* video_frame_adapter =
  212. static_cast<WebrtcVideoFrameAdapter*>(frame.video_frame_buffer().get());
  213. // Store RTP timestamp and FrameStats so they can be added to the
  214. // EncodedImage and EncodedFrame when encoding is complete.
  215. rtp_timestamp_ = frame.timestamp();
  216. frame_stats_ = video_frame_adapter->TakeFrameStats();
  217. if (!frame_stats_) {
  218. // This could happen if WebRTC tried to encode the same frame twice.
  219. // Taking the frame-stats twice from the same frame-adapter would return
  220. // nullptr the second time.
  221. LOG(ERROR) << "Frame provided with missing frame-stats.";
  222. return WEBRTC_VIDEO_CODEC_ERROR;
  223. }
  224. frame_stats_->encode_started_time = now;
  225. auto desktop_frame = video_frame_adapter->TakeDesktopFrame();
  226. // If any frames were dropped by WebRTC or by this class, the
  227. // original DesktopFrame's updated-region should not be used as-is
  228. // (because that region is the difference between this frame and the
  229. // previous frame, which the encoder has not seen because it was dropped).
  230. // In this case, the DesktopFrame's update-region should be set to the
  231. // union of all the dropped frames' update-rectangles.
  232. bool this_class_dropped_frame = !accumulated_update_rect_.IsEmpty();
  233. if (webrtc_dropped_frame || this_class_dropped_frame) {
  234. // Get the update-rect that WebRTC provides, which will include any
  235. // accumulated updates from frames that WebRTC dropped.
  236. auto update_rect = frame.update_rect();
  237. // Combine it with any updates from frames dropped by this class.
  238. update_rect.Union(accumulated_update_rect_);
  239. // In case the new frame has a different resolution, ensure the update-rect
  240. // is constrained by the frame's bounds. On the first frame with a new
  241. // resolution, WebRTC sets the update-rect to the full area of the frame, so
  242. // this line will give the correct result in that case. If the resolution
  243. // did not change (for this frame or any prior dropped frames), the
  244. // update-region will already be constrained by the resolution, so this line
  245. // will be a no-op.
  246. update_rect.Intersect(
  247. webrtc::VideoFrame::UpdateRect{0, 0, frame.width(), frame.height()});
  248. desktop_frame->mutable_updated_region()->SetRect(
  249. webrtc::DesktopRect::MakeXYWH(update_rect.offset_x,
  250. update_rect.offset_y, update_rect.width,
  251. update_rect.height));
  252. // The update-region has now been applied to the desktop_frame which is
  253. // being sent to the encoder, so empty it here.
  254. accumulated_update_rect_.MakeEmptyUpdate();
  255. }
  256. // Limit the encoding and sending of empty frames to |kKeepAliveInterval|.
  257. // This is done to save on network bandwidth and CPU usage.
  258. if (desktop_frame->updated_region().is_empty() && !top_off_active_ &&
  259. !pending_key_frame_request_ &&
  260. (now - latest_frame_encode_start_time_ < kKeepAliveInterval)) {
  261. // Drop the frame. There is no need to track the update-rect as the
  262. // frame being dropped is empty.
  263. base::SequencedTaskRunnerHandle::Get()->PostTask(
  264. FROM_HERE,
  265. base::BindOnce(&WebrtcVideoEncoderWrapper::NotifyFrameDropped,
  266. weak_factory_.GetWeakPtr()));
  267. return WEBRTC_VIDEO_CODEC_OK;
  268. }
  269. latest_frame_encode_start_time_ = now;
  270. WebrtcVideoEncoder::FrameParams frame_params;
  271. // SetRates() must be called prior to Encode(), with a non-zero bitrate.
  272. DCHECK_NE(0, bitrate_kbps_);
  273. frame_params.bitrate_kbps = bitrate_kbps_;
  274. frame_params.duration = target_frame_interval_;
  275. // TODO(crbug.com/1192865): Copy the FPS estimator from the scheduler,
  276. // instead of hard-coding this value here.
  277. frame_params.fps = target_frame_rate_;
  278. frame_params.vpx_min_quantizer =
  279. ShouldDropQualityForLargeFrame(*desktop_frame) ? kMaxQuantizer
  280. : kMinQuantizer;
  281. frame_params.vpx_max_quantizer = kMaxQuantizer;
  282. frame_params.clear_active_map = !top_off_active_;
  283. frame_params.key_frame = pending_key_frame_request_;
  284. pending_key_frame_request_ = false;
  285. encode_pending_ = true;
  286. auto encode_callback = base::BindPostTask(
  287. base::SequencedTaskRunnerHandle::Get(),
  288. base::BindOnce(&WebrtcVideoEncoderWrapper::OnFrameEncoded,
  289. weak_factory_.GetWeakPtr()));
  290. encode_task_runner_->PostTask(
  291. FROM_HERE,
  292. base::BindOnce(&WebrtcVideoEncoder::Encode,
  293. base::Unretained(encoder_.get()), std::move(desktop_frame),
  294. frame_params, std::move(encode_callback)));
  295. return WEBRTC_VIDEO_CODEC_OK;
  296. }
  297. void WebrtcVideoEncoderWrapper::SetRates(
  298. const RateControlParameters& parameters) {
  299. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  300. int bitrate_kbps = parameters.bitrate.get_sum_kbps();
  301. if (bitrate_kbps_ != bitrate_kbps) {
  302. bitrate_kbps_ = bitrate_kbps;
  303. main_task_runner_->PostTask(
  304. FROM_HERE,
  305. base::BindOnce(&VideoChannelStateObserver::OnTargetBitrateChanged,
  306. video_channel_state_observer_, bitrate_kbps));
  307. }
  308. }
  309. void WebrtcVideoEncoderWrapper::OnRttUpdate(int64_t rtt_ms) {
  310. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  311. rtt_estimate_ = base::Milliseconds(rtt_ms);
  312. }
  313. webrtc::VideoEncoder::EncoderInfo WebrtcVideoEncoderWrapper::GetEncoderInfo()
  314. const {
  315. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  316. return EncoderInfo();
  317. }
  318. webrtc::EncodedImageCallback::Result
  319. WebrtcVideoEncoderWrapper::ReturnEncodedFrame(
  320. const WebrtcVideoEncoder::EncodedFrame& frame) {
  321. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  322. // Non-null, because WebRTC registers a callback before calling Encode().
  323. DCHECK(encoded_callback_);
  324. webrtc::EncodedImage encoded_image;
  325. encoded_image.SetEncodedData(frame.data);
  326. encoded_image._encodedWidth = frame.dimensions.width();
  327. encoded_image._encodedHeight = frame.dimensions.height();
  328. encoded_image._frameType = frame.key_frame
  329. ? webrtc::VideoFrameType::kVideoFrameKey
  330. : webrtc::VideoFrameType::kVideoFrameDelta;
  331. encoded_image.SetTimestamp(frame.rtp_timestamp);
  332. encoded_image.playout_delay_.min_ms = 0;
  333. encoded_image.playout_delay_.max_ms = 0;
  334. encoded_image.content_type_ = webrtc::VideoContentType::SCREENSHARE;
  335. webrtc::CodecSpecificInfo codec_specific_info;
  336. codec_specific_info.codecType = frame.codec;
  337. if (frame.codec == webrtc::kVideoCodecVP8) {
  338. webrtc::CodecSpecificInfoVP8* vp8_info =
  339. &codec_specific_info.codecSpecific.VP8;
  340. vp8_info->temporalIdx = webrtc::kNoTemporalIdx;
  341. } else if (frame.codec == webrtc::kVideoCodecVP9) {
  342. webrtc::CodecSpecificInfoVP9* vp9_info =
  343. &codec_specific_info.codecSpecific.VP9;
  344. vp9_info->inter_pic_predicted = !frame.key_frame;
  345. vp9_info->ss_data_available = frame.key_frame;
  346. vp9_info->spatial_layer_resolution_present = frame.key_frame;
  347. if (frame.key_frame) {
  348. vp9_info->width[0] = frame.dimensions.width();
  349. vp9_info->height[0] = frame.dimensions.height();
  350. }
  351. vp9_info->num_spatial_layers = 1;
  352. vp9_info->gof_idx = webrtc::kNoGofIdx;
  353. vp9_info->temporal_idx = webrtc::kNoTemporalIdx;
  354. vp9_info->flexible_mode = false;
  355. vp9_info->temporal_up_switch = true;
  356. vp9_info->inter_layer_predicted = false;
  357. vp9_info->first_frame_in_picture = true;
  358. vp9_info->end_of_picture = true;
  359. vp9_info->spatial_layer_resolution_present = false;
  360. } else if (frame.codec == webrtc::kVideoCodecH264) {
  361. #if defined(USE_H264_ENCODER)
  362. webrtc::CodecSpecificInfoH264* h264_info =
  363. &codec_specific_info.codecSpecific.H264;
  364. h264_info->packetization_mode =
  365. webrtc::H264PacketizationMode::NonInterleaved;
  366. #else
  367. NOTREACHED();
  368. #endif
  369. } else if (frame.codec == webrtc::kVideoCodecAV1) {
  370. // TODO(joedow): Set codec specific params for AV1 here.
  371. } else {
  372. NOTREACHED();
  373. }
  374. return encoded_callback_->OnEncodedImage(encoded_image, &codec_specific_info);
  375. }
  376. void WebrtcVideoEncoderWrapper::OnFrameEncoded(
  377. WebrtcVideoEncoder::EncodeResult encode_result,
  378. std::unique_ptr<WebrtcVideoEncoder::EncodedFrame> encoded_frame) {
  379. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  380. // Keep |encoded_frame| alive until frame-encoded/frame-sent notifications
  381. // have executed on |main_task_runner_|.
  382. std::unique_ptr<WebrtcVideoEncoder::EncodedFrame, base::OnTaskRunnerDeleter>
  383. frame(encoded_frame.release(),
  384. base::OnTaskRunnerDeleter(main_task_runner_));
  385. DCHECK(encode_pending_);
  386. encode_pending_ = false;
  387. // Transfer the cached frame stats into the encoded frame.
  388. if (frame) {
  389. // This is non-null because the |encode_pending_| flag ensures that
  390. // frame-encodings are serialized. So there cannot be 2 consecutive calls to
  391. // this method without an intervening call to Encode() which sets
  392. // |frame_stats_| to non-null.
  393. DCHECK(frame_stats_);
  394. frame_stats_->encode_ended_time = base::TimeTicks::Now();
  395. frame_stats_->rtt_estimate = rtt_estimate_;
  396. frame_stats_->bandwidth_estimate_kbps = bitrate_kbps_;
  397. frame->stats = std::move(frame_stats_);
  398. frame->rtp_timestamp = rtp_timestamp_;
  399. }
  400. main_task_runner_->PostTask(
  401. FROM_HERE, base::BindOnce(&VideoChannelStateObserver::OnFrameEncoded,
  402. video_channel_state_observer_, encode_result,
  403. frame.get()));
  404. if (encode_result != WebrtcVideoEncoder::EncodeResult::SUCCEEDED) {
  405. // TODO(crbug.com/1192865): Store this error and communicate it to WebRTC
  406. // via the next call to Encode(). The VPX encoders are never expected to
  407. // return any error, but hardware-decoders such as H264 may fail.
  408. LOG(ERROR) << "Video encoder returned error "
  409. << EncodeResultToString(encode_result);
  410. NotifyFrameDropped();
  411. DropPendingFrame();
  412. return;
  413. }
  414. if (!frame || !frame->data || !frame->data->size()) {
  415. top_off_active_ = false;
  416. NotifyFrameDropped();
  417. DropPendingFrame();
  418. return;
  419. }
  420. // Top-off until the best quantizer value is reached.
  421. top_off_active_ = (frame->quantizer > kMinQuantizer);
  422. // If there was a successful capture while the encoder was working then there
  423. // will be a frame waiting to be encoded. Send it to the encoder now that its
  424. // no longer busy and we've copied the frame stats for the current frame.
  425. // Note: This function is called here instead of at the end of the function as
  426. // this saves a few hundred microseconds per frame. It can certainly be moved
  427. // if ever there is a need but be sure to profile the per-frame cost.
  428. SchedulePendingFrame();
  429. // WARNING: No frame-specific class members should be accessed after this
  430. // point as they may be updated in Encode() when the pending frame is sent to
  431. // the encoder.
  432. webrtc::EncodedImageCallback::Result send_result = ReturnEncodedFrame(*frame);
  433. // std::ref() is used here because base::BindOnce() would otherwise try to
  434. // copy the referenced frame object, which is move-only. This is safe because
  435. // base::OnTaskRunnerDeleter posts the frame-deleter task to run after this
  436. // task has executed.
  437. main_task_runner_->PostTask(
  438. FROM_HERE, base::BindOnce(&VideoChannelStateObserver::OnEncodedFrameSent,
  439. video_channel_state_observer_, send_result,
  440. std::ref(*frame)));
  441. }
  442. void WebrtcVideoEncoderWrapper::NotifyFrameDropped() {
  443. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  444. DCHECK(encoded_callback_);
  445. encoded_callback_->OnDroppedFrame(
  446. webrtc::EncodedImageCallback::DropReason::kDroppedByEncoder);
  447. }
  448. bool WebrtcVideoEncoderWrapper::ShouldDropQualityForLargeFrame(
  449. const webrtc::DesktopFrame& frame) {
  450. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  451. if (codec_type_ != webrtc::kVideoCodecVP8) {
  452. return false;
  453. }
  454. int64_t updated_area = 0;
  455. for (webrtc::DesktopRegion::Iterator r(frame.updated_region()); !r.IsAtEnd();
  456. r.Advance()) {
  457. updated_area += r.rect().width() * r.rect().height();
  458. }
  459. bool should_drop_quality = false;
  460. if (updated_area - updated_region_area_.Max() > kBigFrameThresholdPixels) {
  461. int expected_frame_size =
  462. updated_area * kEstimatedBytesPerMegapixel / kPixelsPerMegapixel;
  463. base::TimeDelta expected_send_delay =
  464. base::Seconds(expected_frame_size * 8 / (bitrate_kbps_ * 1000.0));
  465. if (expected_send_delay > target_frame_interval_) {
  466. should_drop_quality = true;
  467. }
  468. }
  469. updated_region_area_.Record(updated_area);
  470. return should_drop_quality;
  471. }
  472. void WebrtcVideoEncoderWrapper::SchedulePendingFrame() {
  473. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  474. if (pending_frame_) {
  475. auto pending_frame = std::move(pending_frame_);
  476. Encode(*pending_frame, nullptr);
  477. }
  478. }
  479. void WebrtcVideoEncoderWrapper::DropPendingFrame() {
  480. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  481. if (pending_frame_) {
  482. pending_frame_.reset();
  483. NotifyFrameDropped();
  484. }
  485. }
  486. } // namespace remoting::protocol