webrtc_video_encoder_wrapper.h 8.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182
  1. // Copyright 2021 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef REMOTING_PROTOCOL_WEBRTC_VIDEO_ENCODER_WRAPPER_H_
  5. #define REMOTING_PROTOCOL_WEBRTC_VIDEO_ENCODER_WRAPPER_H_
  6. #include <memory>
  7. #include "base/memory/raw_ptr.h"
  8. #include "base/memory/weak_ptr.h"
  9. #include "base/sequence_checker.h"
  10. #include "base/task/single_thread_task_runner.h"
  11. #include "base/thread_annotations.h"
  12. #include "remoting/base/constants.h"
  13. #include "remoting/base/running_samples.h"
  14. #include "remoting/base/session_options.h"
  15. #include "remoting/codec/webrtc_video_encoder.h"
  16. #include "third_party/webrtc/api/video/video_codec_type.h"
  17. #include "third_party/webrtc/api/video_codecs/sdp_video_format.h"
  18. #include "third_party/webrtc/api/video_codecs/video_encoder.h"
  19. namespace remoting::protocol {
  20. class VideoChannelStateObserver;
  21. // WebrtcVideoEncoderWrapper is a wrapper around the remoting codecs, which
  22. // implements the webrtc::VideoEncoder interface. This class is instantiated
  23. // by WebRTC via the webrtc::VideoEncoderFactory, and all methods (including
  24. // the ctor) are called on WebRTC's foreground worker thread.
  25. class WebrtcVideoEncoderWrapper : public webrtc::VideoEncoder {
  26. public:
  27. // Called by the VideoEncoderFactory. |video_channel_state_observer| is
  28. // notified of important events on the |main_task_runner| thread.
  29. WebrtcVideoEncoderWrapper(
  30. const webrtc::SdpVideoFormat& format,
  31. const SessionOptions& session_options,
  32. scoped_refptr<base::SingleThreadTaskRunner> main_task_runner,
  33. scoped_refptr<base::SingleThreadTaskRunner> encode_task_runner,
  34. base::WeakPtr<VideoChannelStateObserver> video_channel_state_observer);
  35. ~WebrtcVideoEncoderWrapper() override;
  36. void SetEncoderForTest(std::unique_ptr<WebrtcVideoEncoder> encoder);
  37. // webrtc::VideoEncoder interface.
  38. int32_t InitEncode(const webrtc::VideoCodec* codec_settings,
  39. const webrtc::VideoEncoder::Settings& settings) override;
  40. int32_t RegisterEncodeCompleteCallback(
  41. webrtc::EncodedImageCallback* callback) override;
  42. int32_t Release() override;
  43. int32_t Encode(
  44. const webrtc::VideoFrame& frame,
  45. const std::vector<webrtc::VideoFrameType>* frame_types) override;
  46. void SetRates(const RateControlParameters& parameters) override;
  47. void OnRttUpdate(int64_t rtt_ms) override;
  48. webrtc::VideoEncoder::EncoderInfo GetEncoderInfo() const override;
  49. private:
  50. static constexpr int kStatsWindow = 5;
  51. // Returns an encoded frame to WebRTC's registered callback.
  52. webrtc::EncodedImageCallback::Result ReturnEncodedFrame(
  53. const WebrtcVideoEncoder::EncodedFrame& frame);
  54. // Called when |encoder_| has finished encoding a frame.
  55. void OnFrameEncoded(WebrtcVideoEncoder::EncodeResult encode_result,
  56. std::unique_ptr<WebrtcVideoEncoder::EncodedFrame> frame);
  57. // Notifies WebRTC that this encoder has dropped a frame.
  58. void NotifyFrameDropped();
  59. // Returns whether the frame should be encoded at low quality, to reduce
  60. // latency for large frame updates. This is only done here for VP8, as VP9
  61. // automatically detects target-overshoot and re-encodes the frame at
  62. // lower quality. This calculation is based on |frame|'s update-region
  63. // (compared with recent history) and the current bandwidth-estimation.
  64. bool ShouldDropQualityForLargeFrame(const webrtc::DesktopFrame& frame);
  65. // Begins encoding |pending_frame_| if it contains valid frame data.
  66. void SchedulePendingFrame();
  67. // Clears |pending_frame_| and notifies WebRTC of the dropped frame when
  68. // |pending_frame_| contains valid frame data.
  69. void DropPendingFrame();
  70. std::unique_ptr<WebrtcVideoEncoder> encoder_
  71. GUARDED_BY_CONTEXT(sequence_checker_);
  72. // Callback registered by WebRTC to receive encoded frames.
  73. raw_ptr<webrtc::EncodedImageCallback> encoded_callback_
  74. GUARDED_BY_CONTEXT(sequence_checker_) = nullptr;
  75. // Timestamp to be added to the EncodedImage when sending it to
  76. // |encoded_callback_|. This value comes from the frame that WebRTC
  77. // passes to Encode().
  78. uint32_t rtp_timestamp_ GUARDED_BY_CONTEXT(sequence_checker_);
  79. // FrameStats taken from the input VideoFrameAdapter, then added to the
  80. // EncodedFrame when encoding is complete.
  81. std::unique_ptr<WebrtcVideoEncoder::FrameStats> frame_stats_;
  82. // Bandwidth estimate from SetRates(), which is expected to be called before
  83. // Encode().
  84. int bitrate_kbps_ GUARDED_BY_CONTEXT(sequence_checker_) = 0;
  85. // Latest RTT estimate provided by OnRttUpdate().
  86. base::TimeDelta rtt_estimate_ GUARDED_BY_CONTEXT(sequence_checker_){
  87. base::TimeDelta::Max()};
  88. // True when encoding unchanged frames for top-off.
  89. bool top_off_active_ GUARDED_BY_CONTEXT(sequence_checker_) = false;
  90. webrtc::VideoCodecType codec_type_ GUARDED_BY_CONTEXT(sequence_checker_);
  91. // True when a frame is being encoded. This guards against encoding multiple
  92. // frames in parallel, which the encoders are not prepared to handle.
  93. bool encode_pending_ GUARDED_BY_CONTEXT(sequence_checker_) = false;
  94. std::unique_ptr<webrtc::VideoFrame> pending_frame_
  95. GUARDED_BY_CONTEXT(sequence_checker_);
  96. // Stores the expected id of the next incoming frame to be encoded. If this
  97. // does not match, it means that WebRTC dropped a frame, and the original
  98. // DesktopFrame's updated-region should not be passed to the encoder.
  99. // Consecutive frames have incrementing IDs, wrapping around to 0 (which can
  100. // happen many times during a connection - the unsigned type guarantees that
  101. // the '++' operator will wrap to 0 after overflow).
  102. uint16_t next_frame_id_ GUARDED_BY_CONTEXT(sequence_checker_) = 0;
  103. // Keeps track of any update-rectangles from dropped frames. When WebRTC
  104. // requests to encode a frame, this class will either:
  105. // * Send it to be encoded - if any prior frames were dropped, this
  106. // accumulated update-rect will be added to the incoming frame, then it will
  107. // be reset to empty.
  108. // * Drop the frame - the frame's update-rect will be stored and combined with
  109. // this accumulated update-rect.
  110. // This tracking is similar to what WebRTC does whenever it drops frames
  111. // internally. WebRTC will also detect resolution-changes and set the
  112. // frame's update-rect to the full area, so no special logic is needed here
  113. // for changes in resolution (except to make sure that any frame's update-rect
  114. // always lies within the frame's bounding rect).
  115. webrtc::VideoFrame::UpdateRect accumulated_update_rect_
  116. GUARDED_BY_CONTEXT(sequence_checker_){};
  117. // Used by ShouldDropQualityForLargeFrame(). This stores the most recent
  118. // update-region areas of previously-encoded frames, in order to detect an
  119. // unusually-large update.
  120. RunningSamples updated_region_area_ GUARDED_BY_CONTEXT(sequence_checker_){
  121. kStatsWindow};
  122. // Stores the time when the most recent frame was sent to the encoder. This is
  123. // used to rate-limit the encoding and sending of empty frames.
  124. base::TimeTicks latest_frame_encode_start_time_
  125. GUARDED_BY_CONTEXT(sequence_checker_);
  126. // If a key-frame is requested, but this class needs to drop the frame, this
  127. // flag remembers the request so it can be applied to the next frame.
  128. bool pending_key_frame_request_ GUARDED_BY_CONTEXT(sequence_checker_) = false;
  129. // TaskRunner used for notifying |video_channel_state_observer_|.
  130. scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;
  131. // TaskRunner used for scheduling encoding tasks.
  132. scoped_refptr<base::SingleThreadTaskRunner> encode_task_runner_;
  133. // Stores the taret frame rate used for capture and encode scheduling. May be
  134. // overridden by the client via SessionOptions. This value is applied to all
  135. // codecs and cannot be changed during a session.
  136. int target_frame_rate_ = kTargetFrameRate;
  137. base::TimeDelta target_frame_interval_;
  138. base::WeakPtr<VideoChannelStateObserver> video_channel_state_observer_;
  139. // This class lives on WebRTC's encoding thread. All methods (including ctor
  140. // and dtor) are expected to be called on the same thread.
  141. SEQUENCE_CHECKER(sequence_checker_);
  142. base::WeakPtrFactory<WebrtcVideoEncoderWrapper> weak_factory_{this};
  143. };
  144. } // namespace remoting::protocol
  145. #endif // REMOTING_PROTOCOL_WEBRTC_VIDEO_ENCODER_WRAPPER_H_