video_capture_oracle.h 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296
  1. // Copyright (c) 2013 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef MEDIA_CAPTURE_CONTENT_VIDEO_CAPTURE_ORACLE_H_
  5. #define MEDIA_CAPTURE_CONTENT_VIDEO_CAPTURE_ORACLE_H_
  6. #include <string>
  7. #include "base/callback.h"
  8. #include "base/time/time.h"
  9. #include "media/base/feedback_signal_accumulator.h"
  10. #include "media/capture/capture_export.h"
  11. #include "media/capture/content/animated_content_sampler.h"
  12. #include "media/capture/content/capture_resolution_chooser.h"
  13. #include "media/capture/content/smooth_event_sampler.h"
  14. #include "media/capture/video/video_capture_feedback.h"
  15. #include "ui/gfx/geometry/rect.h"
  16. namespace media {
  17. // VideoCaptureOracle manages the producer-side throttling of captured frames
  18. // from a video capture device. It is informed of every update by the device;
  19. // this empowers it to look into the future and decide if a particular frame
  20. // ought to be captured in order to achieve its target frame rate.
  21. class CAPTURE_EXPORT VideoCaptureOracle {
  22. public:
  23. enum Event {
  24. kCompositorUpdate,
  25. // A "refresh request" means that we want to update to keep things
  26. // relatively fresh and in sync, and thus should capture a frame as long as
  27. // it's not happening too frequently (in practice this ends up being 1-5
  28. // frames per second).
  29. kRefreshRequest,
  30. // A "refresh demand" means that we know we have new information, such as a
  31. // mouse cursor location change, and thus generating a frame is higher
  32. // priority than a "refresh request."
  33. kRefreshDemand,
  34. kNumEvents,
  35. };
  36. // Constructs a VideoCaptureOracle with a default min capture period and
  37. // capture size constraints. Clients should call SetMinCapturePeriod() and
  38. // SetCaptureSizeConstraints() to provide more-accurate hard limits.
  39. //
  40. // See SetAutoThrottlingEnabled() for |enable_auto_throttling| semantics.
  41. explicit VideoCaptureOracle(bool enable_auto_throttling);
  42. virtual ~VideoCaptureOracle();
  43. // Get/Update the minimum capture period.
  44. base::TimeDelta min_capture_period() const {
  45. return smoothing_sampler_.min_capture_period();
  46. }
  47. void SetMinCapturePeriod(base::TimeDelta period);
  48. // Sets the range of acceptable capture sizes and whether a fixed aspect ratio
  49. // is required. If a fixed aspect ratio is required, the aspect ratio of
  50. // |max_size| is used.
  51. void SetCaptureSizeConstraints(const gfx::Size& min_size,
  52. const gfx::Size& max_size,
  53. bool use_fixed_aspect_ratio);
  54. // Specifies whether the oracle should propose varying capture sizes, in
  55. // response to consumer feedback. If not |enabled|, capture_size() will always
  56. // return the source_size().
  57. //
  58. // See: SetMinSizeChangePeriod().
  59. void SetAutoThrottlingEnabled(bool enabled);
  60. // Get/Update the source content size. Changes may not have an immediate
  61. // effect on the proposed capture size, as the oracle will prevent too-
  62. // frequent changes from occurring.
  63. gfx::Size source_size() const { return resolution_chooser_.source_size(); }
  64. void SetSourceSize(const gfx::Size& source_size);
  65. // Record a event of type |event|, and decide whether the caller should do a
  66. // frame capture. |damage_rect| is the region of a frame about to be drawn,
  67. // and may be an empty Rect, if this is not known. If the caller accepts the
  68. // oracle's proposal, it should call RecordCapture() to indicate this.
  69. bool ObserveEventAndDecideCapture(Event event,
  70. const gfx::Rect& damage_rect,
  71. base::TimeTicks event_time);
  72. // Returns the |frame_number| to be used with CompleteCapture().
  73. int next_frame_number() const { return next_frame_number_; }
  74. // Record and update internal state based on whether the frame capture will be
  75. // started. |pool_utilization| is a value in the range 0.0 to 1.0 to indicate
  76. // the current buffer pool utilization relative to a sustainable maximum (not
  77. // the absolute maximum). This method should only be called if the last call
  78. // to ObserveEventAndDecideCapture() returned true.
  79. void RecordCapture(double pool_utilization);
  80. void RecordWillNotCapture(double pool_utilization);
  81. // Notify of the completion of a capture, and whether it was successful.
  82. // Returns true iff the captured frame should be delivered. |frame_timestamp|
  83. // is set to the timestamp that should be provided to the consumer of the
  84. // frame.
  85. virtual bool CompleteCapture(int frame_number,
  86. bool capture_was_successful,
  87. base::TimeTicks* frame_timestamp);
  88. // Notify that all in-flight captures have been canceled. This has the same
  89. // effect as calling CompleteCapture() with a non-success status for all
  90. // outstanding frames.
  91. void CancelAllCaptures();
  92. // Record the resource utilization feedback for a frame that was processed by
  93. // the consumer. This allows the oracle to reduce/increase future data volume
  94. // if the consumer is overloaded/under-utilized. |resource_utilization| is a
  95. // value in the range 0.0 to 1.0 to indicate the current consumer resource
  96. // utilization relative to a sustainable maximum (not the absolute maximum).
  97. // This method should only be called for frames where CompleteCapture()
  98. // returned true.
  99. void RecordConsumerFeedback(int frame_number,
  100. const media::VideoCaptureFeedback& feedback);
  101. // Sets the minimum amount of time that must pass between changes to the
  102. // capture size due to autothrottling. This throttles the rate of size
  103. // changes, to avoid stressing consumers and to allow the end-to-end system
  104. // sufficient time to stabilize before re-evaluating the capture size.
  105. void SetMinSizeChangePeriod(base::TimeDelta period);
  106. // Returns the oracle's estimate of the duration of the next frame. This
  107. // should be called just after ObserveEventAndDecideCapture(), and will only
  108. // be non-zero if the call returned true.
  109. base::TimeDelta estimated_frame_duration() const {
  110. return duration_of_next_frame_;
  111. }
  112. // Returns the capture frame size the client should use. This is updated by
  113. // calls to ObserveEventAndDecideCapture(). The oracle prevents too-frequent
  114. // changes to the capture size, to avoid stressing the end-to-end pipeline.
  115. virtual gfx::Size capture_size() const;
  116. // Returns the oracle's estimate of the last time animation was detected.
  117. base::TimeTicks last_time_animation_was_detected() const {
  118. return last_time_animation_was_detected_;
  119. }
  120. // Returns a NUL-terminated string containing a short, human-readable form of
  121. // |event|.
  122. static const char* EventAsString(Event event);
  123. // Default minimum capture period. This is a rather low framerate for safety.
  124. // Clients are expected to set a better minimum capture period after
  125. // VideoCaptureOracle is constructed.
  126. static constexpr base::TimeDelta kDefaultMinCapturePeriod =
  127. base::Milliseconds(200); // 5 FPS
  128. // Default minimum size change period if SetMinSizeChangePeriod is not called.
  129. static constexpr base::TimeDelta kDefaultMinSizeChangePeriod =
  130. base::Seconds(3);
  131. void SetLogCallback(
  132. base::RepeatingCallback<void(const std::string&)> emit_log_cb);
  133. void Log(const std::string& message);
  134. private:
  135. // Retrieve/Assign a frame timestamp by capture |frame_number|. Only valid
  136. // when IsFrameInRecentHistory(frame_number) returns true.
  137. base::TimeTicks GetFrameTimestamp(int frame_number) const;
  138. void SetFrameTimestamp(int frame_number, base::TimeTicks timestamp);
  139. // Returns true if the frame timestamp ring-buffer currently includes a
  140. // slot for the given |frame_number|.
  141. bool IsFrameInRecentHistory(int frame_number) const;
  142. // Queries the ResolutionChooser to update |capture_size_|, and resets all the
  143. // FeedbackSignalAccumulator members to stable-state starting values. The
  144. // accumulators are reset such that they can only apply feedback updates for
  145. // future frames (those with a timestamp after |last_frame_time|).
  146. void CommitCaptureSizeAndReset(base::TimeTicks last_frame_time);
  147. // Called after a capture or no-capture decision was recorded. This analyzes
  148. // current state and may result in a future change to the capture size.
  149. void AnalyzeAndAdjust(base::TimeTicks analyze_time);
  150. // Analyzes current feedback signal accumulators for an indication that the
  151. // capture size should be decreased. Returns either a decreased area, or -1
  152. // if no decrease should be made.
  153. int AnalyzeForDecreasedArea(base::TimeTicks analyze_time);
  154. // Analyzes current feedback signal accumulators for an indication that the
  155. // the system has had excellent long-term performance and the capture size
  156. // should be increased to improve quality. Returns either an increased area,
  157. // or -1 if no increase should be made.
  158. int AnalyzeForIncreasedArea(base::TimeTicks analyze_time);
  159. // Returns the amount of time, since the source size last changed, to allow
  160. // frequent increases in capture area. This allows the system a period of
  161. // time to quickly explore up and down to find an ideal point before being
  162. // more careful about capture size increases.
  163. base::TimeDelta GetExplorationPeriodAfterSourceSizeChange();
  164. // Returns true if updates have been accumulated by |accumulator| for a
  165. // sufficient amount of time and the latest update was fairly recent, relative
  166. // to |now|.
  167. bool HasSufficientRecentFeedback(
  168. const FeedbackSignalAccumulator<base::TimeTicks>& accumulator,
  169. base::TimeTicks now);
  170. // Set to disabled/enabled via SetAutoThrottlingEnabled(). Data collection and
  171. // analysis for capture size changes only occurs while in "active" mode, which
  172. // is only engaged when in "enabled" mode and consumer feedback is received
  173. // for the first time.
  174. enum {
  175. kThrottlingDisabled,
  176. kThrottlingEnabled,
  177. kThrottlingActive
  178. } capture_size_throttling_mode_;
  179. // The minimum amount of time that must pass between changes to the capture
  180. // size.
  181. base::TimeDelta min_size_change_period_;
  182. // Incremented every time RecordCapture() is called.
  183. int next_frame_number_;
  184. // Stores the last |event_time| from the last observation/decision. Used to
  185. // sanity-check that event times are monotonically non-decreasing.
  186. base::TimeTicks last_event_time_[kNumEvents];
  187. // Updated by the last call to ObserveEventAndDecideCapture() with the
  188. // estimated duration of the next frame to sample. This is zero if the method
  189. // returned false.
  190. base::TimeDelta duration_of_next_frame_;
  191. // Stores the frame number from the last successfully delivered frame.
  192. int last_successfully_delivered_frame_number_;
  193. // Stores the number of pending frame captures.
  194. int num_frames_pending_;
  195. // These track present/paint history and propose whether to sample each event
  196. // for capture. |smoothing_sampler_| uses a "works for all" heuristic, while
  197. // |content_sampler_| specifically detects animated content (e.g., video
  198. // playback) and decides which events to sample to "lock into" that content.
  199. SmoothEventSampler smoothing_sampler_;
  200. AnimatedContentSampler content_sampler_;
  201. // This is the overall setting, provided in the last call to
  202. // SetMinCapturePeriod(), but the live setting in |smoothing_sampler_| and
  203. // |content_sampler_| could be greater if the consumer feedback requests it.
  204. base::TimeDelta min_capture_period_;
  205. // Determines video capture frame sizes.
  206. CaptureResolutionChooser resolution_chooser_;
  207. // The timestamp of the frame just before the last call to SetSourceSize().
  208. base::TimeTicks source_size_change_time_;
  209. // The current capture size. |resolution_chooser_| may hold an updated value
  210. // because the oracle prevents this size from changing too frequently. This
  211. // avoids over-stressing consumers (e.g., when a window is being activly
  212. // drag-resized) and allowing the end-to-end system time to stabilize.
  213. gfx::Size capture_size_;
  214. // Recent history of frame timestamps proposed by VideoCaptureOracle. This is
  215. // a ring-buffer, and should only be accessed by the Get/SetFrameTimestamp()
  216. // methods.
  217. enum { kMaxFrameTimestamps = 16 };
  218. base::TimeTicks frame_timestamps_[kMaxFrameTimestamps];
  219. // Recent average buffer pool utilization for capture.
  220. FeedbackSignalAccumulator<base::TimeTicks> buffer_pool_utilization_;
  221. // Estimated maximum frame area that currently can be handled by the consumer,
  222. // in number of pixels per frame. This is used to adjust the capture size up
  223. // or down to a data volume the consumer can handle. Note that some consumers
  224. // do not provide feedback, and the analysis logic should account for that.
  225. FeedbackSignalAccumulator<base::TimeTicks> estimated_capable_area_;
  226. // The time of the first analysis which concluded the end-to-end system was
  227. // under-utilized. If null, the system is not currently under-utilized. This
  228. // is used to determine when a proving period has elapsed that justifies an
  229. // increase in capture size.
  230. base::TimeTicks start_time_of_underutilization_;
  231. // The timestamp of the frame where |content_sampler_| last detected
  232. // animation. This determines whether capture size increases will be
  233. // aggressive (because content is not animating).
  234. base::TimeTicks last_time_animation_was_detected_;
  235. // Callback for webrtc native log. It should check for corresponding feature
  236. // inside.
  237. base::RepeatingCallback<void(const std::string&)> emit_log_message_cb_;
  238. };
  239. } // namespace media
  240. #endif // MEDIA_CAPTURE_CONTENT_VIDEO_CAPTURE_ORACLE_H_