// Copyright 2015 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "remoting/protocol/webrtc_video_stream.h" #include #include #include "base/bind.h" #include "base/logging.h" #include "base/time/time.h" #include "remoting/base/constants.h" #include "remoting/protocol/frame_stats.h" #include "remoting/protocol/host_video_stats_dispatcher.h" #include "remoting/protocol/webrtc_frame_scheduler_constant_rate.h" #include "remoting/protocol/webrtc_transport.h" #include "remoting/protocol/webrtc_video_encoder_factory.h" #include "remoting/protocol/webrtc_video_frame_adapter.h" #include "remoting/protocol/webrtc_video_track_source.h" #include "third_party/webrtc/api/media_stream_interface.h" #include "third_party/webrtc/api/notifier.h" #include "third_party/webrtc/api/peer_connection_interface.h" namespace remoting { namespace protocol { struct WebrtcVideoStream::FrameStats : public WebrtcVideoEncoder::FrameStats { FrameStats() = default; FrameStats(const FrameStats&) = default; FrameStats& operator=(const FrameStats&) = default; ~FrameStats() override = default; // The input-event fields are non-null only for one frame after each // incoming input event. InputEventTimestamps input_event_timestamps; base::TimeDelta capture_delay; uint32_t capturer_id = 0; webrtc::ScreenId screen_id = webrtc::kInvalidScreenId; }; WebrtcVideoStream::WebrtcVideoStream(const std::string& stream_name, const SessionOptions& session_options) : stream_name_(stream_name), session_options_(session_options) {} WebrtcVideoStream::~WebrtcVideoStream() { DCHECK(thread_checker_.CalledOnValidThread()); if (peer_connection_ && transceiver_) { // Ignore any error here, as this may return an error if the // peer-connection has been closed. peer_connection_->RemoveTrackOrError(transceiver_->sender()); } } void WebrtcVideoStream::Start( std::unique_ptr desktop_capturer, WebrtcTransport* webrtc_transport, WebrtcVideoEncoderFactory* video_encoder_factory) { DCHECK(thread_checker_.CalledOnValidThread()); DCHECK(desktop_capturer); DCHECK(webrtc_transport); DCHECK(video_encoder_factory); scoped_refptr peer_connection_factory( webrtc_transport->peer_connection_factory()); peer_connection_ = webrtc_transport->peer_connection(); DCHECK(peer_connection_factory); DCHECK(peer_connection_); capturer_ = std::move(desktop_capturer); capturer_->Start(this); video_track_source_ = new rtc::RefCountedObject( base::BindRepeating(&WebrtcVideoStream::OnSinkAddedOrUpdated, weak_factory_.GetWeakPtr())); rtc::scoped_refptr video_track = peer_connection_factory->CreateVideoTrack(stream_name_, video_track_source_.get()); webrtc::RtpTransceiverInit init; init.stream_ids = {stream_name_}; // value() DCHECKs if AddTransceiver() fails, which only happens if a track // was already added with the stream label. transceiver_ = peer_connection_->AddTransceiver(video_track, init).value(); webrtc_transport->OnVideoTransceiverCreated(transceiver_); video_encoder_factory->SetVideoChannelStateObserver( weak_factory_.GetWeakPtr()); scheduler_ = std::make_unique(); scheduler_->Start(base::BindRepeating(&WebrtcVideoStream::CaptureNextFrame, base::Unretained(this))); } void WebrtcVideoStream::SelectSource(webrtc::ScreenId id) { screen_id_ = id; capturer_->SelectSource(id); } void WebrtcVideoStream::SetEventTimestampsSource( scoped_refptr event_timestamps_source) { event_timestamps_source_ = event_timestamps_source; } void WebrtcVideoStream::Pause(bool pause) { DCHECK(thread_checker_.CalledOnValidThread()); scheduler_->Pause(pause); } void WebrtcVideoStream::SetLosslessEncode(bool want_lossless) { NOTIMPLEMENTED(); } void WebrtcVideoStream::SetLosslessColor(bool want_lossless) { NOTIMPLEMENTED() << "Changing lossless-color for VP9 requires SDP " "offer/answer exchange."; } void WebrtcVideoStream::SetObserver(Observer* observer) { DCHECK(thread_checker_.CalledOnValidThread()); observer_ = observer; } void WebrtcVideoStream::OnKeyFrameRequested() { DCHECK(thread_checker_.CalledOnValidThread()); scheduler_->OnKeyFrameRequested(); } void WebrtcVideoStream::OnTargetBitrateChanged(int bitrate_kbps) { DCHECK(thread_checker_.CalledOnValidThread()); scheduler_->OnTargetBitrateChanged(bitrate_kbps); } void WebrtcVideoStream::OnCaptureResult( webrtc::DesktopCapturer::Result result, std::unique_ptr frame) { DCHECK(thread_checker_.CalledOnValidThread()); current_frame_stats_->capture_ended_time = base::TimeTicks::Now(); current_frame_stats_->capture_delay = base::Milliseconds(frame ? frame->capture_time_ms() : 0); if (!frame || frame->size().is_empty()) { scheduler_->OnFrameCaptured(nullptr); return; } // TODO(sergeyu): Handle ERROR_PERMANENT result here. webrtc::DesktopVector dpi = frame->dpi().is_zero() ? webrtc::DesktopVector(kDefaultDpi, kDefaultDpi) : frame->dpi(); if (!frame_size_.equals(frame->size()) || !frame_dpi_.equals(dpi)) { frame_size_ = frame->size(); frame_dpi_ = dpi; if (observer_) observer_->OnVideoSizeChanged(this, frame_size_, frame_dpi_); } current_frame_stats_->capturer_id = frame->capturer_id(); scheduler_->OnFrameCaptured(frame.get()); // Send the captured frame to the registered sink, if any. WebRTC will route // this to the appropriate encoder. video_track_source_->SendCapturedFrame(std::move(frame), std::move(current_frame_stats_)); } void WebrtcVideoStream::CaptureNextFrame() { DCHECK(thread_checker_.CalledOnValidThread()); current_frame_stats_ = std::make_unique(); current_frame_stats_->capture_started_time = base::TimeTicks::Now(); current_frame_stats_->input_event_timestamps = event_timestamps_source_->TakeLastEventTimestamps(); current_frame_stats_->screen_id = screen_id_; capturer_->CaptureFrame(); } void WebrtcVideoStream::OnSinkAddedOrUpdated(const rtc::VideoSinkWants& wants) { DCHECK(thread_checker_.CalledOnValidThread()); VLOG(0) << "WebRTC requested max framerate: " << wants.max_framerate_fps << " FPS"; scheduler_->SetMaxFramerateFps(wants.max_framerate_fps); } void WebrtcVideoStream::OnFrameEncoded( WebrtcVideoEncoder::EncodeResult encode_result, const WebrtcVideoEncoder::EncodedFrame* frame) { DCHECK(thread_checker_.CalledOnValidThread()); scheduler_->OnFrameEncoded(encode_result, frame); } void WebrtcVideoStream::OnEncodedFrameSent( webrtc::EncodedImageCallback::Result result, const WebrtcVideoEncoder::EncodedFrame& frame) { if (result.error != webrtc::EncodedImageCallback::Result::OK) { // TODO(sergeyu): Stop the stream. LOG(ERROR) << "Failed to send video frame."; return; } // Send FrameStats message. if (video_stats_dispatcher_ && video_stats_dispatcher_->is_connected()) { // The down-cast is safe, because the |stats| object was originally created // by this class and attached to the frame. const auto* current_frame_stats = static_cast(frame.stats.get()); DCHECK(current_frame_stats); HostFrameStats stats; stats.bandwidth_estimate_kbps = current_frame_stats->bandwidth_estimate_kbps; stats.rtt_estimate = current_frame_stats->rtt_estimate; stats.send_pending_delay = current_frame_stats->send_pending_delay; stats.frame_size = frame.data->size(); if (!current_frame_stats->input_event_timestamps.is_null()) { stats.capture_pending_delay = current_frame_stats->capture_started_time - current_frame_stats->input_event_timestamps.host_timestamp; stats.latest_event_timestamp = current_frame_stats->input_event_timestamps.client_timestamp; } stats.capture_delay = current_frame_stats->capture_delay; // Total overhead time for IPC and threading when capturing frames. stats.capture_overhead_delay = (current_frame_stats->capture_ended_time - current_frame_stats->capture_started_time) - stats.capture_delay; stats.encode_pending_delay = current_frame_stats->encode_started_time - current_frame_stats->capture_ended_time; stats.encode_delay = current_frame_stats->encode_ended_time - current_frame_stats->encode_started_time; stats.capturer_id = current_frame_stats->capturer_id; // Convert the frame quantizer to a measure of frame quality between 0 and // 100, for a simple visualization of quality over time. The quantizer from // VP8/VP9 encoder lies within 0-63, with 0 representing a lossless // frame. // TODO(crbug.com/891571): Remove |quantizer| from the WebrtcVideoEncoder // interface, and move this logic to the encoders. stats.frame_quality = (63 - frame.quantizer) * 100 / 63; stats.screen_id = current_frame_stats->screen_id; video_stats_dispatcher_->OnVideoFrameStats(result.frame_id, stats); } } } // namespace protocol } // namespace remoting