webrtc_video_stream.cc 9.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265
  1. // Copyright 2015 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "remoting/protocol/webrtc_video_stream.h"
  5. #include <memory>
  6. #include <utility>
  7. #include "base/bind.h"
  8. #include "base/logging.h"
  9. #include "base/time/time.h"
  10. #include "remoting/base/constants.h"
  11. #include "remoting/protocol/frame_stats.h"
  12. #include "remoting/protocol/host_video_stats_dispatcher.h"
  13. #include "remoting/protocol/webrtc_frame_scheduler_constant_rate.h"
  14. #include "remoting/protocol/webrtc_transport.h"
  15. #include "remoting/protocol/webrtc_video_encoder_factory.h"
  16. #include "remoting/protocol/webrtc_video_frame_adapter.h"
  17. #include "remoting/protocol/webrtc_video_track_source.h"
  18. #include "third_party/webrtc/api/media_stream_interface.h"
  19. #include "third_party/webrtc/api/notifier.h"
  20. #include "third_party/webrtc/api/peer_connection_interface.h"
  21. namespace remoting {
  22. namespace protocol {
  23. struct WebrtcVideoStream::FrameStats : public WebrtcVideoEncoder::FrameStats {
  24. FrameStats() = default;
  25. FrameStats(const FrameStats&) = default;
  26. FrameStats& operator=(const FrameStats&) = default;
  27. ~FrameStats() override = default;
  28. // The input-event fields are non-null only for one frame after each
  29. // incoming input event.
  30. InputEventTimestamps input_event_timestamps;
  31. base::TimeDelta capture_delay;
  32. uint32_t capturer_id = 0;
  33. webrtc::ScreenId screen_id = webrtc::kInvalidScreenId;
  34. };
  35. WebrtcVideoStream::WebrtcVideoStream(const std::string& stream_name,
  36. const SessionOptions& session_options)
  37. : stream_name_(stream_name), session_options_(session_options) {}
  38. WebrtcVideoStream::~WebrtcVideoStream() {
  39. DCHECK(thread_checker_.CalledOnValidThread());
  40. if (peer_connection_ && transceiver_) {
  41. // Ignore any error here, as this may return an error if the
  42. // peer-connection has been closed.
  43. peer_connection_->RemoveTrackOrError(transceiver_->sender());
  44. }
  45. }
  46. void WebrtcVideoStream::Start(
  47. std::unique_ptr<webrtc::DesktopCapturer> desktop_capturer,
  48. WebrtcTransport* webrtc_transport,
  49. WebrtcVideoEncoderFactory* video_encoder_factory) {
  50. DCHECK(thread_checker_.CalledOnValidThread());
  51. DCHECK(desktop_capturer);
  52. DCHECK(webrtc_transport);
  53. DCHECK(video_encoder_factory);
  54. scoped_refptr<webrtc::PeerConnectionFactoryInterface> peer_connection_factory(
  55. webrtc_transport->peer_connection_factory());
  56. peer_connection_ = webrtc_transport->peer_connection();
  57. DCHECK(peer_connection_factory);
  58. DCHECK(peer_connection_);
  59. capturer_ = std::move(desktop_capturer);
  60. capturer_->Start(this);
  61. video_track_source_ = new rtc::RefCountedObject<WebrtcVideoTrackSource>(
  62. base::BindRepeating(&WebrtcVideoStream::OnSinkAddedOrUpdated,
  63. weak_factory_.GetWeakPtr()));
  64. rtc::scoped_refptr<webrtc::VideoTrackInterface> video_track =
  65. peer_connection_factory->CreateVideoTrack(stream_name_,
  66. video_track_source_.get());
  67. webrtc::RtpTransceiverInit init;
  68. init.stream_ids = {stream_name_};
  69. // value() DCHECKs if AddTransceiver() fails, which only happens if a track
  70. // was already added with the stream label.
  71. transceiver_ = peer_connection_->AddTransceiver(video_track, init).value();
  72. webrtc_transport->OnVideoTransceiverCreated(transceiver_);
  73. video_encoder_factory->SetVideoChannelStateObserver(
  74. weak_factory_.GetWeakPtr());
  75. scheduler_ = std::make_unique<WebrtcFrameSchedulerConstantRate>();
  76. scheduler_->Start(base::BindRepeating(&WebrtcVideoStream::CaptureNextFrame,
  77. base::Unretained(this)));
  78. }
  79. void WebrtcVideoStream::SelectSource(webrtc::ScreenId id) {
  80. screen_id_ = id;
  81. capturer_->SelectSource(id);
  82. }
  83. void WebrtcVideoStream::SetEventTimestampsSource(
  84. scoped_refptr<InputEventTimestampsSource> event_timestamps_source) {
  85. event_timestamps_source_ = event_timestamps_source;
  86. }
  87. void WebrtcVideoStream::Pause(bool pause) {
  88. DCHECK(thread_checker_.CalledOnValidThread());
  89. scheduler_->Pause(pause);
  90. }
  91. void WebrtcVideoStream::SetLosslessEncode(bool want_lossless) {
  92. NOTIMPLEMENTED();
  93. }
  94. void WebrtcVideoStream::SetLosslessColor(bool want_lossless) {
  95. NOTIMPLEMENTED() << "Changing lossless-color for VP9 requires SDP "
  96. "offer/answer exchange.";
  97. }
  98. void WebrtcVideoStream::SetObserver(Observer* observer) {
  99. DCHECK(thread_checker_.CalledOnValidThread());
  100. observer_ = observer;
  101. }
  102. void WebrtcVideoStream::OnKeyFrameRequested() {
  103. DCHECK(thread_checker_.CalledOnValidThread());
  104. scheduler_->OnKeyFrameRequested();
  105. }
  106. void WebrtcVideoStream::OnTargetBitrateChanged(int bitrate_kbps) {
  107. DCHECK(thread_checker_.CalledOnValidThread());
  108. scheduler_->OnTargetBitrateChanged(bitrate_kbps);
  109. }
  110. void WebrtcVideoStream::OnCaptureResult(
  111. webrtc::DesktopCapturer::Result result,
  112. std::unique_ptr<webrtc::DesktopFrame> frame) {
  113. DCHECK(thread_checker_.CalledOnValidThread());
  114. current_frame_stats_->capture_ended_time = base::TimeTicks::Now();
  115. current_frame_stats_->capture_delay =
  116. base::Milliseconds(frame ? frame->capture_time_ms() : 0);
  117. if (!frame || frame->size().is_empty()) {
  118. scheduler_->OnFrameCaptured(nullptr);
  119. return;
  120. }
  121. // TODO(sergeyu): Handle ERROR_PERMANENT result here.
  122. webrtc::DesktopVector dpi =
  123. frame->dpi().is_zero() ? webrtc::DesktopVector(kDefaultDpi, kDefaultDpi)
  124. : frame->dpi();
  125. if (!frame_size_.equals(frame->size()) || !frame_dpi_.equals(dpi)) {
  126. frame_size_ = frame->size();
  127. frame_dpi_ = dpi;
  128. if (observer_)
  129. observer_->OnVideoSizeChanged(this, frame_size_, frame_dpi_);
  130. }
  131. current_frame_stats_->capturer_id = frame->capturer_id();
  132. scheduler_->OnFrameCaptured(frame.get());
  133. // Send the captured frame to the registered sink, if any. WebRTC will route
  134. // this to the appropriate encoder.
  135. video_track_source_->SendCapturedFrame(std::move(frame),
  136. std::move(current_frame_stats_));
  137. }
  138. void WebrtcVideoStream::CaptureNextFrame() {
  139. DCHECK(thread_checker_.CalledOnValidThread());
  140. current_frame_stats_ = std::make_unique<FrameStats>();
  141. current_frame_stats_->capture_started_time = base::TimeTicks::Now();
  142. current_frame_stats_->input_event_timestamps =
  143. event_timestamps_source_->TakeLastEventTimestamps();
  144. current_frame_stats_->screen_id = screen_id_;
  145. capturer_->CaptureFrame();
  146. }
  147. void WebrtcVideoStream::OnSinkAddedOrUpdated(const rtc::VideoSinkWants& wants) {
  148. DCHECK(thread_checker_.CalledOnValidThread());
  149. VLOG(0) << "WebRTC requested max framerate: " << wants.max_framerate_fps
  150. << " FPS";
  151. scheduler_->SetMaxFramerateFps(wants.max_framerate_fps);
  152. }
  153. void WebrtcVideoStream::OnFrameEncoded(
  154. WebrtcVideoEncoder::EncodeResult encode_result,
  155. const WebrtcVideoEncoder::EncodedFrame* frame) {
  156. DCHECK(thread_checker_.CalledOnValidThread());
  157. scheduler_->OnFrameEncoded(encode_result, frame);
  158. }
  159. void WebrtcVideoStream::OnEncodedFrameSent(
  160. webrtc::EncodedImageCallback::Result result,
  161. const WebrtcVideoEncoder::EncodedFrame& frame) {
  162. if (result.error != webrtc::EncodedImageCallback::Result::OK) {
  163. // TODO(sergeyu): Stop the stream.
  164. LOG(ERROR) << "Failed to send video frame.";
  165. return;
  166. }
  167. // Send FrameStats message.
  168. if (video_stats_dispatcher_ && video_stats_dispatcher_->is_connected()) {
  169. // The down-cast is safe, because the |stats| object was originally created
  170. // by this class and attached to the frame.
  171. const auto* current_frame_stats =
  172. static_cast<const FrameStats*>(frame.stats.get());
  173. DCHECK(current_frame_stats);
  174. HostFrameStats stats;
  175. stats.bandwidth_estimate_kbps =
  176. current_frame_stats->bandwidth_estimate_kbps;
  177. stats.rtt_estimate = current_frame_stats->rtt_estimate;
  178. stats.send_pending_delay = current_frame_stats->send_pending_delay;
  179. stats.frame_size = frame.data->size();
  180. if (!current_frame_stats->input_event_timestamps.is_null()) {
  181. stats.capture_pending_delay =
  182. current_frame_stats->capture_started_time -
  183. current_frame_stats->input_event_timestamps.host_timestamp;
  184. stats.latest_event_timestamp =
  185. current_frame_stats->input_event_timestamps.client_timestamp;
  186. }
  187. stats.capture_delay = current_frame_stats->capture_delay;
  188. // Total overhead time for IPC and threading when capturing frames.
  189. stats.capture_overhead_delay = (current_frame_stats->capture_ended_time -
  190. current_frame_stats->capture_started_time) -
  191. stats.capture_delay;
  192. stats.encode_pending_delay = current_frame_stats->encode_started_time -
  193. current_frame_stats->capture_ended_time;
  194. stats.encode_delay = current_frame_stats->encode_ended_time -
  195. current_frame_stats->encode_started_time;
  196. stats.capturer_id = current_frame_stats->capturer_id;
  197. // Convert the frame quantizer to a measure of frame quality between 0 and
  198. // 100, for a simple visualization of quality over time. The quantizer from
  199. // VP8/VP9 encoder lies within 0-63, with 0 representing a lossless
  200. // frame.
  201. // TODO(crbug.com/891571): Remove |quantizer| from the WebrtcVideoEncoder
  202. // interface, and move this logic to the encoders.
  203. stats.frame_quality = (63 - frame.quantizer) * 100 / 63;
  204. stats.screen_id = current_frame_stats->screen_id;
  205. video_stats_dispatcher_->OnVideoFrameStats(result.frame_id, stats);
  206. }
  207. }
  208. } // namespace protocol
  209. } // namespace remoting