video_sender.h 6.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167
  1. // Copyright 2014 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef MEDIA_CAST_SENDER_VIDEO_SENDER_H_
  5. #define MEDIA_CAST_SENDER_VIDEO_SENDER_H_
  6. #include <memory>
  7. #include "base/callback.h"
  8. #include "base/memory/ref_counted.h"
  9. #include "base/memory/weak_ptr.h"
  10. #include "base/time/tick_clock.h"
  11. #include "base/time/time.h"
  12. #include "media/capture/video/video_capture_feedback.h"
  13. #include "media/cast/cast_config.h"
  14. #include "media/cast/cast_sender.h"
  15. #include "media/cast/common/rtp_time.h"
  16. #include "media/cast/sender/frame_sender.h"
  17. namespace openscreen::cast {
  18. class Sender;
  19. }
  20. namespace media {
  21. class VideoFrame;
  22. }
  23. namespace media::cast {
  24. class CastTransport;
  25. class VideoEncoder;
  26. class VideoFrameFactory;
  27. using PlayoutDelayChangeCB = base::RepeatingCallback<void(base::TimeDelta)>;
  28. // Not thread safe. Only called from the main cast thread.
  29. // This class owns all objects related to sending video, objects that create RTP
  30. // packets, congestion control, video encoder, parsing and sending of
  31. // RTCP packets.
  32. // Additionally it posts a bunch of delayed tasks to the main thread for various
  33. // timeouts.
  34. class VideoSender : public FrameSender::Client {
  35. public:
  36. // Old way to instantiate, using a cast transport.
  37. // TODO(https://crbug.com/1316434): should be removed once libcast sender is
  38. // successfully launched.
  39. VideoSender(scoped_refptr<CastEnvironment> cast_environment,
  40. const FrameSenderConfig& video_config,
  41. StatusChangeCallback status_change_cb,
  42. const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
  43. CastTransport* const transport_sender,
  44. PlayoutDelayChangeCB playout_delay_change_cb,
  45. media::VideoCaptureFeedbackCB feedback_callback);
  46. // New way of instantiating using an openscreen::cast::Sender. Since the
  47. // |Sender| instance is destroyed when renegotiation is complete, |this|
  48. // is also invalid and should be immediately torn down.
  49. VideoSender(scoped_refptr<CastEnvironment> cast_environment,
  50. const FrameSenderConfig& video_config,
  51. StatusChangeCallback status_change_cb,
  52. const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
  53. openscreen::cast::Sender* sender,
  54. PlayoutDelayChangeCB playout_delay_change_cb,
  55. media::VideoCaptureFeedbackCB feedback_callback);
  56. VideoSender(const VideoSender&) = delete;
  57. VideoSender& operator=(const VideoSender&) = delete;
  58. ~VideoSender() override;
  59. // Note: It is not guaranteed that |video_frame| will actually be encoded and
  60. // sent, if VideoSender detects too many frames in flight. Therefore, clients
  61. // should be careful about the rate at which this method is called.
  62. void InsertRawVideoFrame(scoped_refptr<media::VideoFrame> video_frame,
  63. const base::TimeTicks& reference_time);
  64. // Creates a |VideoFrameFactory| object to vend |VideoFrame| object with
  65. // encoder affinity (defined as offering some sort of performance benefit). If
  66. // the encoder does not have any such capability, returns null.
  67. std::unique_ptr<VideoFrameFactory> CreateVideoFrameFactory();
  68. void SetTargetPlayoutDelay(base::TimeDelta new_target_playout_delay);
  69. base::TimeDelta GetTargetPlayoutDelay() const;
  70. base::WeakPtr<VideoSender> AsWeakPtr();
  71. protected:
  72. // FrameSender::Client overrides.
  73. int GetNumberOfFramesInEncoder() const final;
  74. base::TimeDelta GetEncoderBacklogDuration() const final;
  75. // Exposed as protected for testing.
  76. FrameSender* frame_sender_for_testing() { return frame_sender_.get(); }
  77. private:
  78. VideoSender(scoped_refptr<CastEnvironment> cast_environment,
  79. const FrameSenderConfig& video_config,
  80. StatusChangeCallback status_change_cb,
  81. const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
  82. std::unique_ptr<FrameSender> sender,
  83. PlayoutDelayChangeCB playout_delay_change_cb,
  84. media::VideoCaptureFeedbackCB feedback_callback);
  85. // Called by the |video_encoder_| with the next EncodedFrame to send.
  86. void OnEncodedVideoFrame(scoped_refptr<media::VideoFrame> video_frame,
  87. std::unique_ptr<SenderEncodedFrame> encoded_frame);
  88. // The backing frame sender implementation.
  89. std::unique_ptr<FrameSender> frame_sender_;
  90. // Encodes media::VideoFrame images into EncodedFrames. Per configuration,
  91. // this will point to either the internal software-based encoder or a proxy to
  92. // a hardware-based encoder.
  93. std::unique_ptr<VideoEncoder> video_encoder_;
  94. scoped_refptr<CastEnvironment> cast_environment_;
  95. // The number of frames queued for encoding, but not yet sent.
  96. int frames_in_encoder_ = 0;
  97. // The duration of video queued for encoding, but not yet sent.
  98. base::TimeDelta duration_in_encoder_;
  99. // The timestamp of the frame that was last enqueued in |video_encoder_|.
  100. RtpTimeTicks last_enqueued_frame_rtp_timestamp_;
  101. base::TimeTicks last_enqueued_frame_reference_time_;
  102. // Remember what we set the bitrate to before, no need to set it again if
  103. // we get the same value.
  104. int last_bitrate_ = 0;
  105. // The total amount of time between a frame's capture/recording on the sender
  106. // and its playback on the receiver (i.e., shown to a user).
  107. base::TimeDelta min_playout_delay_;
  108. base::TimeDelta max_playout_delay_;
  109. // Starting playout delay when streaming animated content.
  110. base::TimeDelta animated_playout_delay_;
  111. PlayoutDelayChangeCB playout_delay_change_cb_;
  112. media::VideoCaptureFeedbackCB feedback_cb_;
  113. // Indicates we are operating in a mode where the target playout latency is
  114. // low for best user experience. When operating in low latency mode, we
  115. // prefer dropping frames over increasing target playout time.
  116. bool low_latency_mode_ = false;
  117. // The video encoder's performance metrics as of the last call to
  118. // OnEncodedVideoFrame(). See header file comments for SenderEncodedFrame for
  119. // an explanation of these values.
  120. double last_reported_encoder_utilization_ = -1.0;
  121. double last_reported_lossiness_ = -1.0;
  122. // This tracks the time when the request was sent to encoder to encode a key
  123. // frame on receiving a Pli message. It is used to limit the sender not
  124. // to duplicately respond to multiple Pli messages in a short period.
  125. base::TimeTicks last_time_attempted_to_resolve_pli_;
  126. // NOTE: Weak pointers must be invalidated before all other member variables.
  127. base::WeakPtrFactory<VideoSender> weak_factory_{this};
  128. };
  129. } // namespace media::cast
  130. #endif // MEDIA_CAST_SENDER_VIDEO_SENDER_H_