courier_renderer.h 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255
  1. // Copyright 2016 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef MEDIA_REMOTING_COURIER_RENDERER_H_
  5. #define MEDIA_REMOTING_COURIER_RENDERER_H_
  6. #include <memory>
  7. #include <tuple>
  8. #include <utility>
  9. #include "base/callback.h"
  10. #include "base/containers/circular_deque.h"
  11. #include "base/memory/raw_ptr.h"
  12. #include "base/memory/ref_counted.h"
  13. #include "base/memory/weak_ptr.h"
  14. #include "base/synchronization/lock.h"
  15. #include "base/task/single_thread_task_runner.h"
  16. #include "base/time/time.h"
  17. #include "base/timer/timer.h"
  18. #include "media/base/pipeline_status.h"
  19. #include "media/base/renderer.h"
  20. #include "media/mojo/mojom/remoting.mojom.h"
  21. #include "media/remoting/metrics.h"
  22. #include "mojo/public/cpp/bindings/pending_remote.h"
  23. #include "mojo/public/cpp/system/data_pipe.h"
  24. #include "third_party/abseil-cpp/absl/types/optional.h"
  25. #include "third_party/openscreen/src/cast/streaming/remoting.pb.h"
  26. #include "third_party/openscreen/src/cast/streaming/rpc_messenger.h"
  27. #include "third_party/openscreen/src/util/weak_ptr.h"
  28. namespace media {
  29. class RendererClient;
  30. class VideoRendererSink;
  31. namespace remoting {
  32. class DemuxerStreamAdapter;
  33. class RendererController;
  34. // A media::Renderer implementation that proxies all operations to a remote
  35. // renderer via RPCs. The CourierRenderer is instantiated by
  36. // AdaptiveRendererFactory when media remoting is meant to take place.
  37. class CourierRenderer final : public Renderer {
  38. public:
  39. // The whole class except for constructor and GetMediaTime() runs on
  40. // |media_task_runner|. The constructor and GetMediaTime() run on render main
  41. // thread.
  42. CourierRenderer(scoped_refptr<base::SingleThreadTaskRunner> media_task_runner,
  43. const base::WeakPtr<RendererController>& controller,
  44. VideoRendererSink* video_renderer_sink);
  45. CourierRenderer(const CourierRenderer&) = delete;
  46. CourierRenderer& operator=(const CourierRenderer&) = delete;
  47. ~CourierRenderer() final;
  48. private:
  49. // Callback when attempting to establish data pipe. The function is set to
  50. // static in order to post task to media thread in order to avoid threading
  51. // race condition.
  52. static void OnDataPipeCreatedOnMainThread(
  53. scoped_refptr<base::SingleThreadTaskRunner> media_task_runner,
  54. base::WeakPtr<CourierRenderer> self,
  55. openscreen::WeakPtr<openscreen::cast::RpcMessenger> rpc_messenger,
  56. mojo::PendingRemote<mojom::RemotingDataStreamSender> audio,
  57. mojo::PendingRemote<mojom::RemotingDataStreamSender> video,
  58. mojo::ScopedDataPipeProducerHandle audio_handle,
  59. mojo::ScopedDataPipeProducerHandle video_handle);
  60. // Callback function when RPC message is received. The function is set to
  61. // static in order to post task to media thread in order to avoid threading
  62. // race condition.
  63. static void OnMessageReceivedOnMainThread(
  64. scoped_refptr<base::SingleThreadTaskRunner> media_task_runner,
  65. base::WeakPtr<CourierRenderer> self,
  66. std::unique_ptr<openscreen::cast::RpcMessage> message);
  67. public:
  68. // media::Renderer implementation.
  69. void Initialize(MediaResource* media_resource,
  70. RendererClient* client,
  71. PipelineStatusCallback init_cb) final;
  72. void SetLatencyHint(absl::optional<base::TimeDelta> latency_hint) final;
  73. void Flush(base::OnceClosure flush_cb) final;
  74. void StartPlayingFrom(base::TimeDelta time) final;
  75. void SetPlaybackRate(double playback_rate) final;
  76. void SetVolume(float volume) final;
  77. base::TimeDelta GetMediaTime() final;
  78. private:
  79. friend class CourierRendererTest;
  80. enum State {
  81. STATE_UNINITIALIZED,
  82. STATE_CREATE_PIPE,
  83. STATE_ACQUIRING,
  84. STATE_INITIALIZING,
  85. STATE_FLUSHING,
  86. STATE_PLAYING,
  87. STATE_ERROR
  88. };
  89. // Callback when attempting to establish data pipe. Runs on media thread only.
  90. void OnDataPipeCreated(
  91. mojo::PendingRemote<mojom::RemotingDataStreamSender> audio,
  92. mojo::PendingRemote<mojom::RemotingDataStreamSender> video,
  93. mojo::ScopedDataPipeProducerHandle audio_handle,
  94. mojo::ScopedDataPipeProducerHandle video_handle,
  95. int audio_rpc_handle,
  96. int video_rpc_handle);
  97. // Callback function when RPC message is received. Runs on media thread only.
  98. void OnReceivedRpc(std::unique_ptr<openscreen::cast::RpcMessage> message);
  99. // Function to post task to main thread in order to send RPC message.
  100. void SendRpcToRemote(std::unique_ptr<openscreen::cast::RpcMessage> message);
  101. // Functions when RPC message is received.
  102. void AcquireRendererDone(
  103. std::unique_ptr<openscreen::cast::RpcMessage> message);
  104. void InitializeCallback(
  105. std::unique_ptr<openscreen::cast::RpcMessage> message);
  106. void FlushUntilCallback();
  107. void OnTimeUpdate(std::unique_ptr<openscreen::cast::RpcMessage> message);
  108. void OnBufferingStateChange(
  109. std::unique_ptr<openscreen::cast::RpcMessage> message);
  110. void OnAudioConfigChange(
  111. std::unique_ptr<openscreen::cast::RpcMessage> message);
  112. void OnVideoConfigChange(
  113. std::unique_ptr<openscreen::cast::RpcMessage> message);
  114. void OnVideoNaturalSizeChange(
  115. std::unique_ptr<openscreen::cast::RpcMessage> message);
  116. void OnVideoOpacityChange(
  117. std::unique_ptr<openscreen::cast::RpcMessage> message);
  118. void OnStatisticsUpdate(
  119. std::unique_ptr<openscreen::cast::RpcMessage> message);
  120. // Called when |current_media_time_| is updated.
  121. void OnMediaTimeUpdated();
  122. // Called to update the |video_stats_queue_|.
  123. void UpdateVideoStatsQueue(int video_frames_decoded,
  124. int video_frames_dropped);
  125. // Called to clear all recent measurements history and schedule resuming after
  126. // a stabilization period elapses.
  127. void ResetMeasurements();
  128. // Called when a fatal runtime error occurs. |stop_trigger| is the error code
  129. // handed to the RendererController.
  130. void OnFatalError(StopTrigger stop_trigger);
  131. // Called periodically to measure the data flows from the
  132. // DemuxerStreamAdapters and record this information in the metrics.
  133. void MeasureAndRecordDataRates();
  134. // Helper to check whether is waiting for data from the Demuxers while
  135. // receiver is waiting for buffering. If yes, remoting will be continued even
  136. // though the playback might be delayed or paused.
  137. bool IsWaitingForDataFromDemuxers() const;
  138. // Helper to deregister the renderer from the RPC messenger.
  139. void DeregisterFromRpcMessaging();
  140. State state_;
  141. const scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;
  142. const scoped_refptr<base::SingleThreadTaskRunner> media_task_runner_;
  143. // Current renderer playback time information.
  144. base::TimeDelta current_media_time_;
  145. base::TimeDelta current_max_time_;
  146. // Both |current_media_time_| and |current_max_time_| should be protected by
  147. // lock because it can be accessed from both media and render main thread.
  148. base::Lock time_lock_;
  149. raw_ptr<MediaResource> media_resource_;
  150. raw_ptr<RendererClient> client_;
  151. std::unique_ptr<DemuxerStreamAdapter> audio_demuxer_stream_adapter_;
  152. std::unique_ptr<DemuxerStreamAdapter> video_demuxer_stream_adapter_;
  153. // Component to establish mojo remoting service on browser process.
  154. const base::WeakPtr<RendererController> controller_;
  155. // Broker class to process incoming and outgoing RPC messages.
  156. // Only accessed on |main_task_runner_|. NOTE: the messenger is wrapped
  157. // in an |openscreen::WeakPtr| instead of |base|'s implementation due to
  158. // it being defined in the third_party/openscreen repository.
  159. const openscreen::WeakPtr<openscreen::cast::RpcMessenger> rpc_messenger_;
  160. // RPC handle value for CourierRenderer component.
  161. const int rpc_handle_;
  162. // RPC handle value for render on receiver endpoint.
  163. int remote_renderer_handle_;
  164. // Callbacks.
  165. PipelineStatusCallback init_workflow_done_callback_;
  166. base::OnceClosure flush_cb_;
  167. const raw_ptr<VideoRendererSink>
  168. video_renderer_sink_; // Outlives this class.
  169. // Current playback rate.
  170. double playback_rate_ = 0;
  171. // Current volume.
  172. float volume_ = 1.0f;
  173. // Ignores updates until this time.
  174. base::TimeTicks ignore_updates_until_time_;
  175. // Indicates whether stats has been updated.
  176. bool stats_updated_ = false;
  177. // Stores all |current_media_time_| and the local time when updated in the
  178. // moving time window. This is used to check whether the playback duration
  179. // matches the update duration in the window.
  180. base::circular_deque<std::pair<base::TimeTicks, base::TimeDelta>>
  181. media_time_queue_;
  182. // Stores all updates on the number of video frames decoded/dropped, and the
  183. // local time when updated in the moving time window. This is used to check
  184. // whether too many video frames were dropped.
  185. base::circular_deque<std::tuple<base::TimeTicks, int, int>>
  186. video_stats_queue_;
  187. // The total number of frames decoded/dropped in the time window.
  188. int sum_video_frames_decoded_ = 0;
  189. int sum_video_frames_dropped_ = 0;
  190. // Records the number of consecutive times that remoting playback was delayed.
  191. int times_playback_delayed_ = 0;
  192. // Records events and measurements of interest.
  193. RendererMetricsRecorder metrics_recorder_;
  194. raw_ptr<const base::TickClock> clock_;
  195. // A timer that polls the DemuxerStreamAdapters periodically to measure
  196. // the data flow rates for metrics.
  197. base::RepeatingTimer data_flow_poll_timer_;
  198. // Indicates whether is waiting for data from the Demuxers while receiver
  199. // reported buffer underflow.
  200. bool receiver_is_blocked_on_local_demuxers_ = true;
  201. base::WeakPtrFactory<CourierRenderer> weak_factory_{this};
  202. };
  203. } // namespace remoting
  204. } // namespace media
  205. #endif // MEDIA_REMOTING_COURIER_RENDERER_H_