123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182 |
- // Copyright 2021 The Chromium Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style license that can be
- // found in the LICENSE file.
- #ifndef REMOTING_PROTOCOL_WEBRTC_VIDEO_ENCODER_WRAPPER_H_
- #define REMOTING_PROTOCOL_WEBRTC_VIDEO_ENCODER_WRAPPER_H_
- #include <memory>
- #include "base/memory/raw_ptr.h"
- #include "base/memory/weak_ptr.h"
- #include "base/sequence_checker.h"
- #include "base/task/single_thread_task_runner.h"
- #include "base/thread_annotations.h"
- #include "remoting/base/constants.h"
- #include "remoting/base/running_samples.h"
- #include "remoting/base/session_options.h"
- #include "remoting/codec/webrtc_video_encoder.h"
- #include "third_party/webrtc/api/video/video_codec_type.h"
- #include "third_party/webrtc/api/video_codecs/sdp_video_format.h"
- #include "third_party/webrtc/api/video_codecs/video_encoder.h"
- namespace remoting::protocol {
- class VideoChannelStateObserver;
- // WebrtcVideoEncoderWrapper is a wrapper around the remoting codecs, which
- // implements the webrtc::VideoEncoder interface. This class is instantiated
- // by WebRTC via the webrtc::VideoEncoderFactory, and all methods (including
- // the ctor) are called on WebRTC's foreground worker thread.
- class WebrtcVideoEncoderWrapper : public webrtc::VideoEncoder {
- public:
- // Called by the VideoEncoderFactory. |video_channel_state_observer| is
- // notified of important events on the |main_task_runner| thread.
- WebrtcVideoEncoderWrapper(
- const webrtc::SdpVideoFormat& format,
- const SessionOptions& session_options,
- scoped_refptr<base::SingleThreadTaskRunner> main_task_runner,
- scoped_refptr<base::SingleThreadTaskRunner> encode_task_runner,
- base::WeakPtr<VideoChannelStateObserver> video_channel_state_observer);
- ~WebrtcVideoEncoderWrapper() override;
- void SetEncoderForTest(std::unique_ptr<WebrtcVideoEncoder> encoder);
- // webrtc::VideoEncoder interface.
- int32_t InitEncode(const webrtc::VideoCodec* codec_settings,
- const webrtc::VideoEncoder::Settings& settings) override;
- int32_t RegisterEncodeCompleteCallback(
- webrtc::EncodedImageCallback* callback) override;
- int32_t Release() override;
- int32_t Encode(
- const webrtc::VideoFrame& frame,
- const std::vector<webrtc::VideoFrameType>* frame_types) override;
- void SetRates(const RateControlParameters& parameters) override;
- void OnRttUpdate(int64_t rtt_ms) override;
- webrtc::VideoEncoder::EncoderInfo GetEncoderInfo() const override;
- private:
- static constexpr int kStatsWindow = 5;
- // Returns an encoded frame to WebRTC's registered callback.
- webrtc::EncodedImageCallback::Result ReturnEncodedFrame(
- const WebrtcVideoEncoder::EncodedFrame& frame);
- // Called when |encoder_| has finished encoding a frame.
- void OnFrameEncoded(WebrtcVideoEncoder::EncodeResult encode_result,
- std::unique_ptr<WebrtcVideoEncoder::EncodedFrame> frame);
- // Notifies WebRTC that this encoder has dropped a frame.
- void NotifyFrameDropped();
- // Returns whether the frame should be encoded at low quality, to reduce
- // latency for large frame updates. This is only done here for VP8, as VP9
- // automatically detects target-overshoot and re-encodes the frame at
- // lower quality. This calculation is based on |frame|'s update-region
- // (compared with recent history) and the current bandwidth-estimation.
- bool ShouldDropQualityForLargeFrame(const webrtc::DesktopFrame& frame);
- // Begins encoding |pending_frame_| if it contains valid frame data.
- void SchedulePendingFrame();
- // Clears |pending_frame_| and notifies WebRTC of the dropped frame when
- // |pending_frame_| contains valid frame data.
- void DropPendingFrame();
- std::unique_ptr<WebrtcVideoEncoder> encoder_
- GUARDED_BY_CONTEXT(sequence_checker_);
- // Callback registered by WebRTC to receive encoded frames.
- raw_ptr<webrtc::EncodedImageCallback> encoded_callback_
- GUARDED_BY_CONTEXT(sequence_checker_) = nullptr;
- // Timestamp to be added to the EncodedImage when sending it to
- // |encoded_callback_|. This value comes from the frame that WebRTC
- // passes to Encode().
- uint32_t rtp_timestamp_ GUARDED_BY_CONTEXT(sequence_checker_);
- // FrameStats taken from the input VideoFrameAdapter, then added to the
- // EncodedFrame when encoding is complete.
- std::unique_ptr<WebrtcVideoEncoder::FrameStats> frame_stats_;
- // Bandwidth estimate from SetRates(), which is expected to be called before
- // Encode().
- int bitrate_kbps_ GUARDED_BY_CONTEXT(sequence_checker_) = 0;
- // Latest RTT estimate provided by OnRttUpdate().
- base::TimeDelta rtt_estimate_ GUARDED_BY_CONTEXT(sequence_checker_){
- base::TimeDelta::Max()};
- // True when encoding unchanged frames for top-off.
- bool top_off_active_ GUARDED_BY_CONTEXT(sequence_checker_) = false;
- webrtc::VideoCodecType codec_type_ GUARDED_BY_CONTEXT(sequence_checker_);
- // True when a frame is being encoded. This guards against encoding multiple
- // frames in parallel, which the encoders are not prepared to handle.
- bool encode_pending_ GUARDED_BY_CONTEXT(sequence_checker_) = false;
- std::unique_ptr<webrtc::VideoFrame> pending_frame_
- GUARDED_BY_CONTEXT(sequence_checker_);
- // Stores the expected id of the next incoming frame to be encoded. If this
- // does not match, it means that WebRTC dropped a frame, and the original
- // DesktopFrame's updated-region should not be passed to the encoder.
- // Consecutive frames have incrementing IDs, wrapping around to 0 (which can
- // happen many times during a connection - the unsigned type guarantees that
- // the '++' operator will wrap to 0 after overflow).
- uint16_t next_frame_id_ GUARDED_BY_CONTEXT(sequence_checker_) = 0;
- // Keeps track of any update-rectangles from dropped frames. When WebRTC
- // requests to encode a frame, this class will either:
- // * Send it to be encoded - if any prior frames were dropped, this
- // accumulated update-rect will be added to the incoming frame, then it will
- // be reset to empty.
- // * Drop the frame - the frame's update-rect will be stored and combined with
- // this accumulated update-rect.
- // This tracking is similar to what WebRTC does whenever it drops frames
- // internally. WebRTC will also detect resolution-changes and set the
- // frame's update-rect to the full area, so no special logic is needed here
- // for changes in resolution (except to make sure that any frame's update-rect
- // always lies within the frame's bounding rect).
- webrtc::VideoFrame::UpdateRect accumulated_update_rect_
- GUARDED_BY_CONTEXT(sequence_checker_){};
- // Used by ShouldDropQualityForLargeFrame(). This stores the most recent
- // update-region areas of previously-encoded frames, in order to detect an
- // unusually-large update.
- RunningSamples updated_region_area_ GUARDED_BY_CONTEXT(sequence_checker_){
- kStatsWindow};
- // Stores the time when the most recent frame was sent to the encoder. This is
- // used to rate-limit the encoding and sending of empty frames.
- base::TimeTicks latest_frame_encode_start_time_
- GUARDED_BY_CONTEXT(sequence_checker_);
- // If a key-frame is requested, but this class needs to drop the frame, this
- // flag remembers the request so it can be applied to the next frame.
- bool pending_key_frame_request_ GUARDED_BY_CONTEXT(sequence_checker_) = false;
- // TaskRunner used for notifying |video_channel_state_observer_|.
- scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;
- // TaskRunner used for scheduling encoding tasks.
- scoped_refptr<base::SingleThreadTaskRunner> encode_task_runner_;
- // Stores the taret frame rate used for capture and encode scheduling. May be
- // overridden by the client via SessionOptions. This value is applied to all
- // codecs and cannot be changed during a session.
- int target_frame_rate_ = kTargetFrameRate;
- base::TimeDelta target_frame_interval_;
- base::WeakPtr<VideoChannelStateObserver> video_channel_state_observer_;
- // This class lives on WebRTC's encoding thread. All methods (including ctor
- // and dtor) are expected to be called on the same thread.
- SEQUENCE_CHECKER(sequence_checker_);
- base::WeakPtrFactory<WebrtcVideoEncoderWrapper> weak_factory_{this};
- };
- } // namespace remoting::protocol
- #endif // REMOTING_PROTOCOL_WEBRTC_VIDEO_ENCODER_WRAPPER_H_
|