ukm_recorder_impl.h 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366
  1. // Copyright 2017 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef COMPONENTS_UKM_UKM_RECORDER_IMPL_H_
  5. #define COMPONENTS_UKM_UKM_RECORDER_IMPL_H_
  6. #include <limits>
  7. #include <map>
  8. #include <memory>
  9. #include <set>
  10. #include <string>
  11. #include <unordered_set>
  12. #include <vector>
  13. #include "base/callback_forward.h"
  14. #include "base/component_export.h"
  15. #include "base/containers/flat_map.h"
  16. #include "base/containers/flat_set.h"
  17. #include "base/gtest_prod_util.h"
  18. #include "base/observer_list_threadsafe.h"
  19. #include "base/sequence_checker.h"
  20. #include "base/strings/string_piece.h"
  21. #include "base/synchronization/lock.h"
  22. #include "components/ukm/ukm_entry_filter.h"
  23. #include "services/metrics/public/cpp/ukm_decode.h"
  24. #include "services/metrics/public/cpp/ukm_recorder.h"
  25. #include "services/metrics/public/cpp/ukm_source_id.h"
  26. #include "services/metrics/public/mojom/ukm_interface.mojom-forward.h"
  27. namespace metrics {
  28. class UkmBrowserTestBase;
  29. }
  30. namespace ukm {
  31. class Aggregate;
  32. class Report;
  33. class UkmRecorderImplTest;
  34. class UkmRecorderObserver;
  35. class UkmSource;
  36. class UkmTestHelper;
  37. class UkmUtilsForTest;
  38. COMPONENT_EXPORT(UKM_RECORDER)
  39. extern const base::Feature kUkmSamplingRateFeature;
  40. namespace debug {
  41. class UkmDebugDataExtractor;
  42. }
  43. class COMPONENT_EXPORT(UKM_RECORDER) UkmRecorderImpl : public UkmRecorder {
  44. using IsWebstoreExtensionCallback =
  45. base::RepeatingCallback<bool(base::StringPiece id)>;
  46. public:
  47. UkmRecorderImpl();
  48. ~UkmRecorderImpl() override;
  49. // Enables/disables recording control if data is allowed to be collected. The
  50. // |extensions| flag separately controls recording of chrome-extension://
  51. // URLs; this flag should reflect the "sync extensions" user setting.
  52. void EnableRecording(bool extensions);
  53. void DisableRecording();
  54. // Controls sampling for testing purposes. Sampling is 1-in-N (N==rate).
  55. void SetSamplingForTesting(int rate) override;
  56. // True if sampling has been configured.
  57. bool IsSamplingConfigured() const;
  58. // Deletes all stored recordings.
  59. void Purge();
  60. // Deletes stored Sources containing URLs of the given scheme and events
  61. // attributed with these Sources.
  62. void PurgeRecordingsWithUrlScheme(const std::string& url_scheme);
  63. // Deletes stored Sources with the given Source id type and events
  64. // attributed with these Sources.
  65. void PurgeRecordingsWithSourceIdType(ukm::SourceIdType source_id_type);
  66. // Marks a source as no longer needed to be kept alive in memory. The source
  67. // with given id will be removed from in-memory recordings at the next
  68. // reporting cycle.
  69. void MarkSourceForDeletion(ukm::SourceId source_id) override;
  70. // Sets a callback for determining if an extension URL can be recorded.
  71. void SetIsWebstoreExtensionCallback(
  72. const IsWebstoreExtensionCallback& callback);
  73. // Sets the UkmEntryFilter that will be applied to all subsequent entries
  74. // reported via AddEntry(). Does not apply the filter to any entries that are
  75. // already recorded.
  76. //
  77. // Currently only accommodates one entry filter.
  78. void SetEntryFilter(std::unique_ptr<UkmEntryFilter> entry_filter);
  79. // Register an observer to be notified when a new UKM entry that comes with
  80. // one of the |event_hashes| is added. This method can be called on any
  81. // thread.
  82. void AddUkmRecorderObserver(const base::flat_set<uint64_t>& event_hashes,
  83. UkmRecorderObserver* observer);
  84. // Clears the given |observer| from |observers_|. This method can be called
  85. // on any thread. If an observer is registered for multiple event sets, it
  86. // will be removed from all the sets. If an event set no longer has any
  87. // observers as a result of this call, it will be removed from |observers_|
  88. // map.
  89. void RemoveUkmRecorderObserver(UkmRecorderObserver* observer);
  90. // Called when UKM allow state changed.
  91. void OnUkmAllowedStateChanged(bool allowed);
  92. // Sets the sampling seed for testing purposes.
  93. void SetSamplingSeedForTesting(uint32_t seed) {
  94. // Normally the seed is set during object construction and remains
  95. // constant in order to provide consistent results when doing an "is
  96. // sampled in" query for a given source and event. A "const cast" is
  97. // necessary to override that.
  98. *const_cast<uint32_t*>(&sampling_seed_) = seed;
  99. }
  100. bool recording_enabled() const { return recording_enabled_; }
  101. protected:
  102. // Calculates sampled in/out for a specific source/event based on internal
  103. // configuration. This function is guaranteed to always return the same
  104. // result over the life of this object for the same config & input parameters.
  105. bool IsSampledIn(int64_t source_id, uint64_t event_id);
  106. // Like above but uses a passed |sampling_rate| instead of internal config.
  107. bool IsSampledIn(int64_t source_id, uint64_t event_id, int sampling_rate);
  108. void InitDecodeMap();
  109. // Writes recordings into a report proto, and clears recordings.
  110. void StoreRecordingsInReport(Report* report);
  111. // Prunes data after storing records in the report. Returns the time elapsed
  112. // in seconds from the moment the newest truncated source was created to the
  113. // moment it was discarded from memory, if pruning happened due to number
  114. // of sources exceeding the max threshold.
  115. int PruneData(std::set<SourceId>& source_ids_seen);
  116. // Deletes Sources and Events with these source_ids.
  117. void PurgeSourcesAndEventsBySourceIds(
  118. const std::unordered_set<SourceId>& source_ids);
  119. const std::map<SourceId, std::unique_ptr<UkmSource>>& sources() const {
  120. return recordings_.sources;
  121. }
  122. const std::vector<mojom::UkmEntryPtr>& entries() const {
  123. return recordings_.entries;
  124. }
  125. // Keep only newest |max_kept_sources| sources when the number of sources
  126. // in recordings_ exceeds this threshold. We only consider the set of ids
  127. // contained in |pruning_set|. Returns the age of newest truncated
  128. // source in seconds.
  129. int PruneOldSources(size_t max_kept_sources,
  130. const std::set<SourceId>& pruning_set);
  131. // UkmRecorder:
  132. void AddEntry(mojom::UkmEntryPtr entry) override;
  133. void UpdateSourceURL(SourceId source_id, const GURL& url) override;
  134. void UpdateAppURL(SourceId source_id,
  135. const GURL& url,
  136. const AppType app_type) override;
  137. void RecordNavigation(
  138. SourceId source_id,
  139. const UkmSource::NavigationData& navigation_data) override;
  140. using UkmRecorder::RecordOtherURL;
  141. virtual bool ShouldRestrictToWhitelistedSourceIds() const;
  142. private:
  143. friend ::metrics::UkmBrowserTestBase;
  144. friend ::ukm::debug::UkmDebugDataExtractor;
  145. friend ::ukm::UkmRecorderImplTest;
  146. friend ::ukm::UkmTestHelper;
  147. friend ::ukm::UkmUtilsForTest;
  148. FRIEND_TEST_ALL_PREFIXES(UkmRecorderImplTest, IsSampledIn);
  149. FRIEND_TEST_ALL_PREFIXES(UkmRecorderImplTest, PurgeExtensionRecordings);
  150. FRIEND_TEST_ALL_PREFIXES(UkmRecorderImplTest, WebApkSourceUrl);
  151. FRIEND_TEST_ALL_PREFIXES(UkmRecorderImplTest, PaymentAppScopeUrl);
  152. FRIEND_TEST_ALL_PREFIXES(UkmRecorderImplTest, WebIdentityScopeUrl);
  153. FRIEND_TEST_ALL_PREFIXES(UkmRecorderImplTest, ObserverNotifiedOnNewEntry);
  154. FRIEND_TEST_ALL_PREFIXES(UkmRecorderImplTest, AddRemoveObserver);
  155. struct MetricAggregate {
  156. uint64_t total_count = 0;
  157. double value_sum = 0;
  158. double value_square_sum = 0.0;
  159. uint64_t dropped_due_to_limits = 0;
  160. uint64_t dropped_due_to_sampling = 0;
  161. uint64_t dropped_due_to_filter = 0;
  162. uint64_t dropped_due_to_unconfigured = 0;
  163. };
  164. struct EventAggregate {
  165. EventAggregate();
  166. ~EventAggregate();
  167. // Fills the proto message from the struct.
  168. void FillProto(Aggregate* proto_aggregate) const;
  169. base::flat_map<uint64_t, MetricAggregate> metrics;
  170. uint64_t total_count = 0;
  171. uint64_t dropped_due_to_limits = 0;
  172. uint64_t dropped_due_to_sampling = 0;
  173. uint64_t dropped_due_to_filter = 0;
  174. uint64_t dropped_due_to_unconfigured = 0;
  175. };
  176. // Result for ShouldRecordUrl() method.
  177. enum class ShouldRecordUrlResult {
  178. kOk = 0, // URL will be recorded and observers will be notified.
  179. kObserverOnly, // The client has opted out from uploading UKM metrics.
  180. // As a result, observers will be notified but URL will not
  181. // be recorded.
  182. kDropped, // The URL is not allowed to be recorded and will be
  183. // dropped. Observers are not nofitied either.
  184. };
  185. using MetricAggregateMap = std::map<uint64_t, MetricAggregate>;
  186. // Marks for deletion if the |source_id| is of a certain type.
  187. void MaybeMarkForDeletion(SourceId source_id);
  188. // Returns the result whether |sanitized_url| should be recorded.
  189. ShouldRecordUrlResult ShouldRecordUrl(SourceId source_id,
  190. const GURL& sanitized_url) const;
  191. void RecordSource(std::unique_ptr<UkmSource> source);
  192. // Applies UkmEntryFilter if there is one registered.
  193. bool ApplyEntryFilter(mojom::UkmEntry* entry);
  194. // Loads sampling configurations from field-trial information.
  195. void LoadExperimentSamplingInfo();
  196. // Loads sampling configuration from the key/value "params" of a field-trial.
  197. // This is separated from the above to ease testing.
  198. void LoadExperimentSamplingParams(
  199. const std::map<std::string, std::string>& params);
  200. // Called to notify interested observers about a newly added UKM entry.
  201. void NotifyObserversWithNewEntry(const mojom::UkmEntry& entry);
  202. // Helper method to notify all observers on UKM events.
  203. template <typename Method, typename... Params>
  204. void NotifyAllObservers(Method m, Params&&... params);
  205. // Whether recording new data is currently allowed.
  206. bool recording_enabled_ = false;
  207. // Indicates whether recording is enabled for extensions.
  208. bool extensions_enabled_ = false;
  209. // Indicates whether recording continuity has been broken since last report.
  210. bool recording_is_continuous_ = true;
  211. // Indicates if sampling has been forced for testing.
  212. bool sampling_forced_for_testing_ = false;
  213. // A pseudo-random number used as the base for sampling choices. This
  214. // allows consistent "is sampled in" results for a given source and event
  215. // type throughout the life of this object.
  216. const uint32_t sampling_seed_;
  217. // Callback for checking extension IDs.
  218. IsWebstoreExtensionCallback is_webstore_extension_callback_;
  219. // Filter applied to AddEntry().
  220. std::unique_ptr<UkmEntryFilter> entry_filter_;
  221. // Map from hashes to entry and metric names.
  222. ukm::builders::DecodeMap decode_map_;
  223. // Sampling configurations, loaded from a field-trial.
  224. int default_sampling_rate_ = -1; // -1 == not yet loaded
  225. base::flat_map<uint64_t, int> event_sampling_rates_;
  226. // If an event's sampling is "slaved" to another, the hashes of the slave
  227. // and the master are recorded here.
  228. base::flat_map<uint64_t, uint64_t> event_sampling_master_;
  229. // Contains data from various recordings which periodically get serialized
  230. // and cleared by StoreRecordingsInReport() and may be Purged().
  231. struct Recordings {
  232. Recordings();
  233. Recordings& operator=(Recordings&&);
  234. ~Recordings();
  235. // Data captured by UpdateSourceUrl().
  236. std::map<SourceId, std::unique_ptr<UkmSource>> sources;
  237. // Data captured by AddEntry().
  238. std::vector<mojom::UkmEntryPtr> entries;
  239. // Source ids that have been marked as no longer needed, to denote the
  240. // subset of |sources| that can be purged after next report.
  241. std::unordered_set<ukm::SourceId> obsolete_source_ids;
  242. // URLs of sources that matched a whitelist url, but were not included in
  243. // the report generated by the last log rotation because we haven't seen any
  244. // events for that source yet.
  245. std::unordered_set<std::string> carryover_urls_whitelist;
  246. // Aggregate information for collected event metrics.
  247. std::map<uint64_t, EventAggregate> event_aggregations;
  248. // Aggregated counters about Sources recorded in the current log.
  249. struct SourceCounts {
  250. // Count of URLs recorded for all sources.
  251. size_t observed = 0;
  252. // Count of URLs recorded for all SourceIdType::NAVIGATION_ID Sources.
  253. size_t navigation_sources = 0;
  254. // Sources carried over (not recorded) from a previous logging rotation.
  255. size_t carryover_sources = 0;
  256. // Resets all of the data.
  257. void Reset();
  258. };
  259. SourceCounts source_counts;
  260. // Resets all of the data.
  261. void Reset();
  262. };
  263. Recordings recordings_;
  264. // The maximum number of Sources we'll keep in memory before discarding any
  265. // new ones being added.
  266. size_t max_sources_ = 500;
  267. // The maximum number of Sources we can keep in memory at the end of the
  268. // current reporting cycle that will stay accessible in the next reporting
  269. // interval.
  270. size_t max_kept_sources_ = 100;
  271. // The maximum number of Entries we'll keep in memory before discarding any
  272. // new ones being added.
  273. size_t max_entries_ = 5000;
  274. using UkmRecorderObserverList =
  275. base::ObserverListThreadSafe<UkmRecorderObserver>;
  276. // Map from event hashes to observers. The key is a set of event hashes that
  277. // their corresponding value pair will be norified when one of those events
  278. // is added. The value is a non-empty observer list whose members are
  279. // observing those events.
  280. using UkmRecorderObserverMap =
  281. base::flat_map<base::flat_set<uint64_t> /*event_hashes*/,
  282. scoped_refptr<UkmRecorderObserverList>>;
  283. // Lock used to ensure mutual exclusive access to |observers_|.
  284. mutable base::Lock lock_;
  285. // Observers that will be notified on UKM events.
  286. UkmRecorderObserverMap observers_ GUARDED_BY(lock_);
  287. SEQUENCE_CHECKER(sequence_checker_);
  288. };
  289. } // namespace ukm
  290. #endif // COMPONENTS_UKM_UKM_RECORDER_IMPL_H_