ukm_recorder_impl.cc 41 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137
  1. // Copyright 2017 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/ukm/ukm_recorder_impl.h"
  5. #include <memory>
  6. #include <string>
  7. #include <unordered_map>
  8. #include <utility>
  9. #include "base/component_export.h"
  10. #include "base/containers/contains.h"
  11. #include "base/feature_list.h"
  12. #include "base/metrics/crc32.h"
  13. #include "base/metrics/field_trial.h"
  14. #include "base/metrics/field_trial_params.h"
  15. #include "base/metrics/histogram_functions.h"
  16. #include "base/metrics/histogram_macros.h"
  17. #include "base/metrics/metrics_hashes.h"
  18. #include "base/rand_util.h"
  19. #include "base/strings/string_number_conversions.h"
  20. #include "base/strings/string_split.h"
  21. #include "base/time/time.h"
  22. #include "components/ukm/scheme_constants.h"
  23. #include "components/ukm/ukm_recorder_observer.h"
  24. #include "components/variations/variations_associated_data.h"
  25. #include "services/metrics/public/cpp/ukm_builders.h"
  26. #include "services/metrics/public/cpp/ukm_decode.h"
  27. #include "services/metrics/public/cpp/ukm_source.h"
  28. #include "services/metrics/public/cpp/ukm_source_id.h"
  29. #include "services/metrics/public/mojom/ukm_interface.mojom.h"
  30. #include "third_party/metrics_proto/ukm/entry.pb.h"
  31. #include "third_party/metrics_proto/ukm/report.pb.h"
  32. #include "third_party/metrics_proto/ukm/source.pb.h"
  33. #include "url/gurl.h"
  34. namespace ukm {
  35. const base::Feature kUkmSamplingRateFeature{"UkmSamplingRate",
  36. base::FEATURE_DISABLED_BY_DEFAULT};
  37. namespace {
  38. bool IsWhitelistedSourceId(SourceId source_id) {
  39. SourceIdType type = GetSourceIdType(source_id);
  40. switch (type) {
  41. case ukm::SourceIdObj::Type::NAVIGATION_ID:
  42. case ukm::SourceIdObj::Type::APP_ID:
  43. case ukm::SourceIdObj::Type::HISTORY_ID:
  44. case ukm::SourceIdObj::Type::WEBAPK_ID:
  45. case ukm::SourceIdObj::Type::PAYMENT_APP_ID:
  46. case ukm::SourceIdObj::Type::NO_URL_ID:
  47. case ukm::SourceIdObj::Type::REDIRECT_ID:
  48. case ukm::SourceIdObj::Type::WEB_IDENTITY_ID: {
  49. return true;
  50. }
  51. case ukm::SourceIdObj::Type::DEFAULT:
  52. case ukm::SourceIdObj::Type::DESKTOP_WEB_APP_ID:
  53. case ukm::SourceIdObj::Type::WORKER_ID:
  54. return false;
  55. }
  56. }
  57. bool IsAppIdType(SourceId source_id) {
  58. SourceIdType type = GetSourceIdType(source_id);
  59. return type == SourceIdType::APP_ID;
  60. }
  61. // Returns whether |url| has one of the schemes supported for logging to UKM.
  62. // URLs with other schemes will not be logged.
  63. bool HasSupportedScheme(const GURL& url) {
  64. return url.SchemeIsHTTPOrHTTPS() || url.SchemeIs(url::kAboutScheme) ||
  65. url.SchemeIs(kChromeUIScheme) || url.SchemeIs(kExtensionScheme) ||
  66. url.SchemeIs(kAppScheme);
  67. }
  68. void LogEventHashAsUmaHistogram(const std::string& histogram_name,
  69. uint64_t event_hash) {
  70. // The enum for this histogram gets populated by the PopulateEnumWithUkmEvents
  71. // function in populate_enums.py when producing the merged XML.
  72. base::UmaHistogramSparse(histogram_name,
  73. // Truncate the unsigned 64-bit hash to 31 bits, to
  74. // make it a suitable histogram sample.
  75. event_hash & 0x7fffffff);
  76. }
  77. enum class DroppedDataReason {
  78. NOT_DROPPED = 0,
  79. RECORDING_DISABLED = 1,
  80. MAX_HIT = 2,
  81. NOT_WHITELISTED = 3,
  82. UNSUPPORTED_URL_SCHEME = 4,
  83. SAMPLED_OUT = 5,
  84. EXTENSION_URLS_DISABLED = 6,
  85. EXTENSION_NOT_SYNCED = 7,
  86. NOT_MATCHED = 8,
  87. EMPTY_URL = 9,
  88. REJECTED_BY_FILTER = 10,
  89. SAMPLING_UNCONFIGURED = 11,
  90. NUM_DROPPED_DATA_REASONS
  91. };
  92. void RecordDroppedSource(DroppedDataReason reason) {
  93. UMA_HISTOGRAM_ENUMERATION(
  94. "UKM.Sources.Dropped", static_cast<int>(reason),
  95. static_cast<int>(DroppedDataReason::NUM_DROPPED_DATA_REASONS));
  96. }
  97. void RecordDroppedSource(bool already_recorded_another_reason,
  98. DroppedDataReason reason) {
  99. if (!already_recorded_another_reason)
  100. RecordDroppedSource(reason);
  101. }
  102. void RecordDroppedEntry(uint64_t event_hash, DroppedDataReason reason) {
  103. LogEventHashAsUmaHistogram("UKM.Entries.Dropped.ByEntryHash", event_hash);
  104. // Because the "UKM.Entries.Dropped.ByEntryHash" histogram will be emitted to
  105. // every single time an entry is dropped, it will be dominated by the
  106. // RECORDING_DISABLED reason (which is not very insightful). More interesting
  107. // dropped reasons are MAX_HIT and SAMPLED_OUT, so we also emit histograms
  108. // split by those reasons.
  109. switch (reason) {
  110. case DroppedDataReason::MAX_HIT:
  111. LogEventHashAsUmaHistogram("UKM.Entries.Dropped.MaxHit.ByEntryHash",
  112. event_hash);
  113. break;
  114. case DroppedDataReason::SAMPLED_OUT:
  115. LogEventHashAsUmaHistogram("UKM.Entries.Dropped.SampledOut.ByEntryHash",
  116. event_hash);
  117. break;
  118. default:
  119. break;
  120. }
  121. UMA_HISTOGRAM_ENUMERATION(
  122. "UKM.Entries.Dropped", static_cast<int>(reason),
  123. static_cast<int>(DroppedDataReason::NUM_DROPPED_DATA_REASONS));
  124. }
  125. void StoreEntryProto(const mojom::UkmEntry& in, Entry* out) {
  126. DCHECK(!out->has_source_id());
  127. DCHECK(!out->has_event_hash());
  128. out->set_source_id(in.source_id);
  129. out->set_event_hash(in.event_hash);
  130. for (const auto& metric : in.metrics) {
  131. Entry::Metric* proto_metric = out->add_metrics();
  132. proto_metric->set_metric_hash(metric.first);
  133. proto_metric->set_value(metric.second);
  134. }
  135. }
  136. GURL SanitizeURL(const GURL& url) {
  137. GURL::Replacements remove_params;
  138. remove_params.ClearUsername();
  139. remove_params.ClearPassword();
  140. // chrome:// and about: URLs params are never used for navigation, only to
  141. // prepopulate data on the page, so don't include their params.
  142. if (url.SchemeIs(url::kAboutScheme) || url.SchemeIs("chrome")) {
  143. remove_params.ClearQuery();
  144. }
  145. if (url.SchemeIs(kExtensionScheme)) {
  146. remove_params.ClearPath();
  147. remove_params.ClearQuery();
  148. remove_params.ClearRef();
  149. }
  150. return url.ReplaceComponents(remove_params);
  151. }
  152. void AppendWhitelistedUrls(
  153. const std::map<SourceId, std::unique_ptr<UkmSource>>& sources,
  154. std::unordered_set<std::string>* urls) {
  155. for (const auto& kv : sources) {
  156. if (IsWhitelistedSourceId(kv.first)) {
  157. urls->insert(kv.second->url().spec());
  158. // Some non-navigation sources only record origin as a URL.
  159. // Add the origin from the navigation source to match those too.
  160. urls->insert(kv.second->url().DeprecatedGetOriginAsURL().spec());
  161. }
  162. }
  163. }
  164. // Returns true if the event corresponding to |event_hash| has a comprehensive
  165. // decode map that includes all valid metrics.
  166. bool HasComprehensiveDecodeMap(int64_t event_hash) {
  167. // All events other than "Identifiability" conforms to its decode map.
  168. // TODO(asanka): It is technically an abstraction violation for
  169. // //components/ukm to know this fact.
  170. return event_hash != builders::Identifiability::kEntryNameHash;
  171. }
  172. bool HasUnknownMetrics(const builders::DecodeMap& decode_map,
  173. const mojom::UkmEntry& entry) {
  174. const auto it = decode_map.find(entry.event_hash);
  175. if (it == decode_map.end())
  176. return true;
  177. if (!HasComprehensiveDecodeMap(entry.event_hash))
  178. return false;
  179. const auto& metric_map = it->second.metric_map;
  180. for (const auto& metric : entry.metrics) {
  181. if (metric_map.count(metric.first) == 0)
  182. return true;
  183. }
  184. return false;
  185. }
  186. } // namespace
  187. UkmRecorderImpl::UkmRecorderImpl()
  188. : sampling_seed_(static_cast<uint32_t>(base::RandUint64())) {
  189. max_kept_sources_ =
  190. static_cast<size_t>(base::GetFieldTrialParamByFeatureAsInt(
  191. kUkmFeature, "MaxKeptSources", max_kept_sources_));
  192. }
  193. UkmRecorderImpl::~UkmRecorderImpl() = default;
  194. UkmRecorderImpl::Recordings::Recordings() = default;
  195. UkmRecorderImpl::Recordings& UkmRecorderImpl::Recordings::operator=(
  196. Recordings&&) = default;
  197. UkmRecorderImpl::Recordings::~Recordings() = default;
  198. void UkmRecorderImpl::Recordings::Reset() {
  199. *this = Recordings();
  200. }
  201. void UkmRecorderImpl::Recordings::SourceCounts::Reset() {
  202. *this = SourceCounts();
  203. }
  204. void UkmRecorderImpl::EnableRecording(bool extensions) {
  205. DVLOG(1) << "UkmRecorderImpl::EnableRecording, extensions=" << extensions;
  206. recording_enabled_ = true;
  207. extensions_enabled_ = extensions;
  208. }
  209. void UkmRecorderImpl::DisableRecording() {
  210. DVLOG(1) << "UkmRecorderImpl::DisableRecording";
  211. if (recording_enabled_)
  212. recording_is_continuous_ = false;
  213. recording_enabled_ = false;
  214. extensions_enabled_ = false;
  215. }
  216. void UkmRecorderImpl::SetSamplingForTesting(int rate) {
  217. sampling_forced_for_testing_ = true;
  218. default_sampling_rate_ = rate;
  219. event_sampling_rates_.clear();
  220. }
  221. bool UkmRecorderImpl::IsSamplingConfigured() const {
  222. return sampling_forced_for_testing_ ||
  223. base::FeatureList::IsEnabled(kUkmSamplingRateFeature);
  224. }
  225. void UkmRecorderImpl::Purge() {
  226. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  227. recordings_.Reset();
  228. recording_is_continuous_ = false;
  229. NotifyAllObservers(&UkmRecorderObserver::OnPurge);
  230. }
  231. void UkmRecorderImpl::PurgeRecordingsWithUrlScheme(
  232. const std::string& url_scheme) {
  233. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  234. // Discard all sources that have a URL with the given URL scheme as well as
  235. // all the entries associated with these sources.
  236. std::unordered_set<SourceId> relevant_source_ids;
  237. for (const auto& kv : recordings_.sources) {
  238. if (kv.second->url().SchemeIs(url_scheme)) {
  239. relevant_source_ids.insert(kv.first);
  240. }
  241. }
  242. PurgeSourcesAndEventsBySourceIds(relevant_source_ids);
  243. recording_is_continuous_ = false;
  244. NotifyAllObservers(&UkmRecorderObserver::OnPurgeRecordingsWithUrlScheme,
  245. url_scheme);
  246. }
  247. void UkmRecorderImpl::PurgeRecordingsWithSourceIdType(
  248. ukm::SourceIdType source_id_type) {
  249. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  250. std::unordered_set<SourceId> relevant_source_ids;
  251. for (const auto& kv : recordings_.sources) {
  252. if (GetSourceIdType(kv.first) == source_id_type) {
  253. relevant_source_ids.insert(kv.first);
  254. }
  255. }
  256. PurgeSourcesAndEventsBySourceIds(relevant_source_ids);
  257. recording_is_continuous_ = false;
  258. }
  259. void UkmRecorderImpl::PurgeSourcesAndEventsBySourceIds(
  260. const std::unordered_set<SourceId>& source_ids) {
  261. for (const auto source_id : source_ids) {
  262. recordings_.sources.erase(source_id);
  263. }
  264. std::vector<mojom::UkmEntryPtr>& events = recordings_.entries;
  265. events.erase(std::remove_if(events.begin(), events.end(),
  266. [&](const auto& event) {
  267. return source_ids.count(event->source_id);
  268. }),
  269. events.end());
  270. }
  271. void UkmRecorderImpl::MarkSourceForDeletion(SourceId source_id) {
  272. if (source_id == kInvalidSourceId)
  273. return;
  274. recordings_.obsolete_source_ids.insert(source_id);
  275. }
  276. void UkmRecorderImpl::SetIsWebstoreExtensionCallback(
  277. const IsWebstoreExtensionCallback& callback) {
  278. is_webstore_extension_callback_ = callback;
  279. }
  280. void UkmRecorderImpl::SetEntryFilter(
  281. std::unique_ptr<UkmEntryFilter> entry_filter) {
  282. DCHECK(!entry_filter_ || !entry_filter);
  283. entry_filter_ = std::move(entry_filter);
  284. }
  285. void UkmRecorderImpl::AddUkmRecorderObserver(
  286. const base::flat_set<uint64_t>& event_hashes,
  287. UkmRecorderObserver* observer) {
  288. DCHECK(observer);
  289. base::AutoLock auto_lock(lock_);
  290. scoped_refptr<UkmRecorderObserverList> observers;
  291. if (observers_.find(event_hashes) == observers_.end()) {
  292. observers_.insert(
  293. {event_hashes, base::MakeRefCounted<UkmRecorderObserverList>()});
  294. }
  295. observers_[event_hashes]->AddObserver(observer);
  296. }
  297. void UkmRecorderImpl::RemoveUkmRecorderObserver(UkmRecorderObserver* observer) {
  298. base::AutoLock auto_lock(lock_);
  299. for (auto it = observers_.begin(); it != observers_.end();) {
  300. if (it->second->RemoveObserver(observer) ==
  301. UkmRecorderObserverList::RemoveObserverResult::kWasOrBecameEmpty) {
  302. it = observers_.erase(it);
  303. } else {
  304. ++it;
  305. }
  306. }
  307. }
  308. void UkmRecorderImpl::OnUkmAllowedStateChanged(bool allowed) {
  309. NotifyAllObservers(&UkmRecorderObserver::OnUkmAllowedStateChanged, allowed);
  310. }
  311. void UkmRecorderImpl::StoreRecordingsInReport(Report* report) {
  312. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  313. // Set of source ids seen by entries in recordings_.
  314. std::set<SourceId> source_ids_seen;
  315. for (const auto& entry : recordings_.entries) {
  316. Entry* proto_entry = report->add_entries();
  317. StoreEntryProto(*entry, proto_entry);
  318. source_ids_seen.insert(entry->source_id);
  319. }
  320. // Number of sources excluded from this report because no entries referred to
  321. // them.
  322. const int num_sources_unsent =
  323. recordings_.sources.size() - source_ids_seen.size();
  324. // Construct set of whitelisted URLs by merging those carried over from the
  325. // previous report cycle and those from sources recorded in this cycle.
  326. std::unordered_set<std::string> url_whitelist;
  327. recordings_.carryover_urls_whitelist.swap(url_whitelist);
  328. AppendWhitelistedUrls(recordings_.sources, &url_whitelist);
  329. // Number of sources discarded due to not matching a navigation URL.
  330. int num_sources_unmatched = 0;
  331. std::unordered_map<SourceIdType, int> serialized_source_type_counts;
  332. for (const auto& kv : recordings_.sources) {
  333. MaybeMarkForDeletion(kv.first);
  334. // If the source id is not whitelisted, don't send it unless it has
  335. // associated entries and the URL matches that of a whitelisted source.
  336. // Note: If ShouldRestrictToWhitelistedSourceIds() is true, this logic will
  337. // not be hit as the source would have already been filtered in
  338. // UpdateSourceURL().
  339. if (!IsWhitelistedSourceId(kv.first)) {
  340. // UkmSource should not keep initial_url for non-navigation source IDs.
  341. DCHECK_EQ(1u, kv.second->urls().size());
  342. if (!url_whitelist.count(kv.second->url().spec())) {
  343. RecordDroppedSource(DroppedDataReason::NOT_MATCHED);
  344. MarkSourceForDeletion(kv.first);
  345. num_sources_unmatched++;
  346. continue;
  347. }
  348. // Omit entryless sources from the report.
  349. if (!base::Contains(source_ids_seen, kv.first)) {
  350. continue;
  351. }
  352. // Non-whitelisted Source types will not be kept after entries are logged.
  353. MarkSourceForDeletion(kv.first);
  354. }
  355. // Minimal validations before serializing into a proto message.
  356. // See crbug/1274876.
  357. DCHECK_NE(kv.second->id(), ukm::kInvalidSourceId);
  358. DCHECK_NE(kv.second->urls().size(), 0u);
  359. Source* proto_source = report->add_sources();
  360. kv.second->PopulateProto(proto_source);
  361. serialized_source_type_counts[GetSourceIdType(kv.first)]++;
  362. }
  363. for (const auto& event_and_aggregate : recordings_.event_aggregations) {
  364. Aggregate* proto_aggregate = report->add_aggregates();
  365. proto_aggregate->set_event_hash(event_and_aggregate.first);
  366. const EventAggregate& event_aggregate = event_and_aggregate.second;
  367. event_aggregate.FillProto(proto_aggregate);
  368. }
  369. int num_serialized_sources = 0;
  370. for (const auto& source_type_and_count : serialized_source_type_counts) {
  371. num_serialized_sources += source_type_and_count.second;
  372. }
  373. UMA_HISTOGRAM_COUNTS_1000("UKM.Sources.SerializedCount2",
  374. num_serialized_sources);
  375. UMA_HISTOGRAM_COUNTS_100000("UKM.Entries.SerializedCount2",
  376. recordings_.entries.size());
  377. UMA_HISTOGRAM_COUNTS_1000("UKM.Sources.UnsentSourcesCount",
  378. num_sources_unsent);
  379. UMA_HISTOGRAM_COUNTS_1000("UKM.Sources.UnmatchedSourcesCount",
  380. num_sources_unmatched);
  381. UMA_HISTOGRAM_COUNTS_1000(
  382. "UKM.Sources.SerializedCount2.Default",
  383. serialized_source_type_counts[SourceIdType::DEFAULT]);
  384. UMA_HISTOGRAM_COUNTS_1000(
  385. "UKM.Sources.SerializedCount2.Navigation",
  386. serialized_source_type_counts[SourceIdType::NAVIGATION_ID]);
  387. UMA_HISTOGRAM_COUNTS_1000(
  388. "UKM.Sources.SerializedCount2.App",
  389. serialized_source_type_counts[SourceIdType::APP_ID]);
  390. // We record a UMA metric specifically for the number of serialized events
  391. // with the FCP metric. This is for data quality verification.
  392. const uint64_t pageload_hash =
  393. base::HashMetricName(ukm::builders::PageLoad::kEntryName);
  394. const uint64_t fcp_hash = base::HashMetricName(
  395. ukm::builders::PageLoad::
  396. kPaintTiming_NavigationToFirstContentfulPaintName);
  397. int num_recorded_fcp = 0;
  398. for (const auto& entry : recordings_.entries) {
  399. if (entry->event_hash == pageload_hash) {
  400. if (entry->metrics.find(fcp_hash) != entry->metrics.end()) {
  401. num_recorded_fcp++;
  402. }
  403. }
  404. }
  405. UMA_HISTOGRAM_COUNTS_100000("UKM.Entries.SerializedCountFCP",
  406. num_recorded_fcp);
  407. // For each matching id in obsolete_source_ids, remove the Source from
  408. // recordings_.sources. The remaining sources form the deferred sources for
  409. // the next report.
  410. for (const SourceId& source_id : recordings_.obsolete_source_ids) {
  411. recordings_.sources.erase(source_id);
  412. }
  413. recordings_.obsolete_source_ids.clear();
  414. // Populate SourceCounts field on the report then clear the recordings.
  415. Report::SourceCounts* source_counts_proto = report->mutable_source_counts();
  416. source_counts_proto->set_observed(recordings_.source_counts.observed);
  417. source_counts_proto->set_navigation_sources(
  418. recordings_.source_counts.navigation_sources);
  419. source_counts_proto->set_unmatched_sources(num_sources_unmatched);
  420. source_counts_proto->set_carryover_sources(
  421. recordings_.source_counts.carryover_sources);
  422. recordings_.source_counts.Reset();
  423. recordings_.entries.clear();
  424. recordings_.event_aggregations.clear();
  425. report->set_is_continuous(recording_is_continuous_);
  426. recording_is_continuous_ = true;
  427. int pruned_sources_age_sec = PruneData(source_ids_seen);
  428. // Record how old the newest truncated source is.
  429. source_counts_proto->set_pruned_sources_age_seconds(pruned_sources_age_sec);
  430. // Set deferred sources count after pruning.
  431. source_counts_proto->set_deferred_sources(recordings_.sources.size());
  432. // Same value as the deferred source count, for setting the carryover count
  433. // in the next reporting cycle.
  434. recordings_.source_counts.carryover_sources = recordings_.sources.size();
  435. // We already matched these deferred sources against the URL whitelist.
  436. // Re-whitelist them for the next report.
  437. for (const auto& kv : recordings_.sources) {
  438. recordings_.carryover_urls_whitelist.insert(kv.second->url().spec());
  439. }
  440. UMA_HISTOGRAM_COUNTS_1000("UKM.Sources.KeptSourcesCount",
  441. recordings_.sources.size());
  442. // Record number of sources after pruning that were carried over due to not
  443. // having any events in this reporting cycle.
  444. int num_sources_entryless = 0;
  445. for (const auto& kv : recordings_.sources) {
  446. if (!base::Contains(source_ids_seen, kv.first)) {
  447. num_sources_entryless++;
  448. }
  449. }
  450. source_counts_proto->set_entryless_sources(num_sources_entryless);
  451. // Notify observers that a report was generated.
  452. if (entry_filter_) {
  453. entry_filter_->OnStoreRecordingsInReport();
  454. }
  455. }
  456. int UkmRecorderImpl::PruneData(std::set<SourceId>& source_ids_seen) {
  457. // Modify the set source_ids_seen by removing sources that aren't in
  458. // recordings_. We do this here as there is a few places for
  459. // recordings_.sources to be modified. The resulting set will be currently
  460. // existing sources that were seen in this report.
  461. auto it = source_ids_seen.begin();
  462. while (it != source_ids_seen.end()) {
  463. if (!base::Contains(recordings_.sources, *it)) {
  464. it = source_ids_seen.erase(it);
  465. } else {
  466. it++;
  467. }
  468. }
  469. // Build the set of sources that exist in recordings_.sources that were not
  470. // seen in this report.
  471. std::set<SourceId> source_ids_unseen;
  472. for (const auto& kv : recordings_.sources) {
  473. if (!base::Contains(source_ids_seen, kv.first)) {
  474. source_ids_unseen.insert(kv.first);
  475. }
  476. }
  477. // Special case APP_IDs. Ideally this is not going to exist for too long, as
  478. // it would be preferable to have a more general purpose solution.
  479. std::set<SourceId> source_ids_app_id;
  480. // Only done if we are in the experiment that will leave APP_ID metrics for
  481. // last when pruning. This block extracts out all source_ids from the
  482. // seen/unseen lists and stores them in |source_ids_app_id|.
  483. if (base::GetFieldTrialParamByFeatureAsBool(kUkmFeature, "PruneAppIdLast",
  484. false)) {
  485. it = source_ids_seen.begin();
  486. while (it != source_ids_seen.end()) {
  487. if (IsAppIdType(*it)) {
  488. source_ids_app_id.insert(*it);
  489. it = source_ids_seen.erase(it);
  490. } else {
  491. it++;
  492. }
  493. }
  494. it = source_ids_unseen.begin();
  495. while (it != source_ids_unseen.end()) {
  496. if (IsAppIdType(*it)) {
  497. source_ids_app_id.insert(*it);
  498. it = source_ids_unseen.erase(it);
  499. } else {
  500. it++;
  501. }
  502. }
  503. }
  504. int pruned_sources_age_sec = 0;
  505. int num_sources = recordings_.sources.size();
  506. // Setup an experiment to test what will occur if we prune unseen sources
  507. // first.
  508. if (base::GetFieldTrialParamByFeatureAsBool(
  509. kUkmFeature, "PruneUnseenSourcesFirst", false)) {
  510. int pruned_sources_age_from_unseen_sec =
  511. PruneOldSources(max_kept_sources_, source_ids_unseen);
  512. UMA_HISTOGRAM_COUNTS_10000("UKM.PrunedSources.NumUnseen",
  513. num_sources - recordings_.sources.size());
  514. num_sources = recordings_.sources.size();
  515. // Prune again from seen sources. Note that if we've already pruned enough
  516. // from the unseen sources, this will be a noop.
  517. int pruned_sources_age_from_seen_sec =
  518. PruneOldSources(max_kept_sources_, source_ids_seen);
  519. UMA_HISTOGRAM_COUNTS_10000("UKM.PrunedSources.NumSeen",
  520. num_sources - recordings_.sources.size());
  521. num_sources = recordings_.sources.size();
  522. int pruned_sources_age_from_app_id_sec = 0;
  523. // Technically this should be fine without the feature, since the group
  524. // will be empty, but might as well add the feature check.
  525. // Still prune the APP_ID entries. We don't want it to be unbounded, but
  526. // providing a higher default here in case.
  527. if (base::GetFieldTrialParamByFeatureAsBool(kUkmFeature, "PruneAppIdLast",
  528. false)) {
  529. pruned_sources_age_from_app_id_sec =
  530. PruneOldSources(500, source_ids_app_id);
  531. UMA_HISTOGRAM_COUNTS_10000("UKM.PrunedSources.NumAppId",
  532. num_sources - recordings_.sources.size());
  533. }
  534. // We're looking for the newest age, which will be the largest between the
  535. // two sets we pruned from.
  536. pruned_sources_age_sec = std::max({pruned_sources_age_from_unseen_sec,
  537. pruned_sources_age_from_seen_sec,
  538. pruned_sources_age_from_app_id_sec});
  539. } else {
  540. // In this case, we prune all sources without caring if they were seen or
  541. // not. Make a set of all existing sources so we can use the same
  542. // PruneOldSources method.
  543. std::set<SourceId> all_sources;
  544. for (const auto& kv : recordings_.sources) {
  545. all_sources.insert(kv.first);
  546. }
  547. if (base::GetFieldTrialParamByFeatureAsBool(kUkmFeature, "PruneAppIdLast",
  548. false)) {
  549. std::set<SourceId> all_sources_without_app_id;
  550. // This will put into |all_sources_without_app_id| the set of
  551. // |all_sources| - |source_ids_app_id|.
  552. std::set_difference(all_sources.begin(), all_sources.end(),
  553. source_ids_app_id.begin(), source_ids_app_id.end(),
  554. std::inserter(all_sources_without_app_id,
  555. all_sources_without_app_id.end()));
  556. // Now, prune the non-APP_ID, then the APP_ID.
  557. int pruned_sources_age_sec_non_app_id =
  558. PruneOldSources(max_kept_sources_, all_sources_without_app_id);
  559. UMA_HISTOGRAM_COUNTS_10000("UKM.PrunedSources.AppExpNumNonAppId",
  560. num_sources - recordings_.sources.size());
  561. num_sources = recordings_.sources.size();
  562. int pruned_sources_age_sec_app_id =
  563. PruneOldSources(500, source_ids_app_id);
  564. UMA_HISTOGRAM_COUNTS_10000("UKM.PrunedSources.AppExpNumAppId",
  565. num_sources - recordings_.sources.size());
  566. pruned_sources_age_sec = std::max(pruned_sources_age_sec_non_app_id,
  567. pruned_sources_age_sec_app_id);
  568. } else {
  569. pruned_sources_age_sec = PruneOldSources(max_kept_sources_, all_sources);
  570. UMA_HISTOGRAM_COUNTS_10000("UKM.PrunedSources.NoExp",
  571. num_sources - recordings_.sources.size());
  572. }
  573. }
  574. return pruned_sources_age_sec;
  575. }
  576. bool UkmRecorderImpl::ShouldRestrictToWhitelistedSourceIds() const {
  577. return base::GetFieldTrialParamByFeatureAsBool(
  578. kUkmFeature, "RestrictToWhitelistedSourceIds", false);
  579. }
  580. bool UkmRecorderImpl::ApplyEntryFilter(mojom::UkmEntry* entry) {
  581. base::flat_set<uint64_t> dropped_metric_hashes;
  582. if (!entry_filter_)
  583. return true;
  584. bool keep_entry = entry_filter_->FilterEntry(entry, &dropped_metric_hashes);
  585. for (auto metric : dropped_metric_hashes) {
  586. recordings_.event_aggregations[entry->event_hash]
  587. .metrics[metric]
  588. .dropped_due_to_filter++;
  589. }
  590. if (!keep_entry) {
  591. recordings_.event_aggregations[entry->event_hash].dropped_due_to_filter++;
  592. return false;
  593. }
  594. return true;
  595. }
  596. int UkmRecorderImpl::PruneOldSources(size_t max_kept_sources,
  597. const std::set<SourceId>& pruning_set) {
  598. long num_prune_required = recordings_.sources.size() - max_kept_sources;
  599. // In either case here, nothing to be done.
  600. if (num_prune_required <= 0 || pruning_set.size() == 0)
  601. return 0;
  602. // We can prune everything, so let's do that directly.
  603. if (static_cast<unsigned long>(num_prune_required) >= pruning_set.size()) {
  604. base::TimeTicks pruned_sources_age = base::TimeTicks();
  605. for (const auto& source_id : pruning_set) {
  606. auto creation_time = recordings_.sources[source_id]->creation_time();
  607. if (creation_time > pruned_sources_age)
  608. pruned_sources_age = creation_time;
  609. recordings_.sources.erase(source_id);
  610. }
  611. base::TimeDelta age_delta = base::TimeTicks::Now() - pruned_sources_age;
  612. // Technically the age we return here isn't quite right, this is the age of
  613. // the newest element of the pruned set, while we actually want the age of
  614. // the last one kept. However it's very unlikely to make a difference in
  615. // practice as if all are pruned here, it is very likely we'll need to prune
  616. // from the seen set next. Since it would be logically quite a bit more
  617. // complex to get this exactly right, it's ok for this to be very slightly
  618. // off in an edge case just to keep complexity down.
  619. return age_delta.InSeconds();
  620. }
  621. // In this case we cannot prune everything, so we will select only the oldest
  622. // sources to prune.
  623. // Build a list of timestamp->source pairs for all source we consider for
  624. // pruning.
  625. std::vector<std::pair<base::TimeTicks, SourceId>> timestamp_source_id_pairs;
  626. for (const auto& source_id : pruning_set) {
  627. auto creation_time = recordings_.sources[source_id]->creation_time();
  628. timestamp_source_id_pairs.emplace_back(
  629. std::make_pair(creation_time, source_id));
  630. }
  631. // Partially sort so that the last |num_prune_required| elements are the
  632. // newest.
  633. std::nth_element(timestamp_source_id_pairs.begin(),
  634. timestamp_source_id_pairs.end() - num_prune_required,
  635. timestamp_source_id_pairs.end());
  636. // Actually prune |num_prune_required| sources.
  637. for (int i = 0; i < num_prune_required; i++) {
  638. auto source_id = timestamp_source_id_pairs[i].second;
  639. recordings_.sources.erase(source_id);
  640. }
  641. base::TimeDelta pruned_sources_age =
  642. base::TimeTicks::Now() -
  643. (timestamp_source_id_pairs.end() - (num_prune_required + 1))->first;
  644. return pruned_sources_age.InSeconds();
  645. }
  646. void UkmRecorderImpl::UpdateSourceURL(SourceId source_id,
  647. const GURL& unsanitized_url) {
  648. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  649. DCHECK(GetSourceIdType(source_id) != SourceIdType::NO_URL_ID);
  650. if (base::Contains(recordings_.sources, source_id))
  651. return;
  652. const GURL sanitized_url = SanitizeURL(unsanitized_url);
  653. if (ShouldRecordUrl(source_id, sanitized_url) ==
  654. ShouldRecordUrlResult::kDropped) {
  655. return;
  656. }
  657. RecordSource(std::make_unique<UkmSource>(source_id, sanitized_url));
  658. }
  659. void UkmRecorderImpl::UpdateAppURL(SourceId source_id,
  660. const GURL& url,
  661. const AppType app_type) {
  662. if (app_type != AppType::kPWA && !extensions_enabled_) {
  663. RecordDroppedSource(DroppedDataReason::EXTENSION_URLS_DISABLED);
  664. return;
  665. }
  666. UpdateSourceURL(source_id, url);
  667. }
  668. void UkmRecorderImpl::RecordNavigation(
  669. SourceId source_id,
  670. const UkmSource::NavigationData& unsanitized_navigation_data) {
  671. DCHECK(GetSourceIdType(source_id) == SourceIdType::NAVIGATION_ID);
  672. DCHECK(!base::Contains(recordings_.sources, source_id));
  673. // TODO(csharrison): Consider changing this behavior so the Source isn't even
  674. // recorded at all if the final URL in |unsanitized_navigation_data| should
  675. // not be recorded.
  676. std::vector<GURL> urls;
  677. for (const GURL& url : unsanitized_navigation_data.urls) {
  678. const GURL sanitized_url = SanitizeURL(url);
  679. if (ShouldRecordUrl(source_id, sanitized_url) !=
  680. ShouldRecordUrlResult::kDropped) {
  681. urls.push_back(std::move(sanitized_url));
  682. }
  683. }
  684. // None of the URLs passed the ShouldRecordUrl check, so do not create a new
  685. // Source for them.
  686. if (urls.empty())
  687. return;
  688. UkmSource::NavigationData sanitized_navigation_data =
  689. unsanitized_navigation_data.CopyWithSanitizedUrls(urls);
  690. RecordSource(
  691. std::make_unique<UkmSource>(source_id, sanitized_navigation_data));
  692. }
  693. UkmRecorderImpl::EventAggregate::EventAggregate() = default;
  694. UkmRecorderImpl::EventAggregate::~EventAggregate() = default;
  695. void UkmRecorderImpl::EventAggregate::FillProto(
  696. Aggregate* proto_aggregate) const {
  697. proto_aggregate->set_source_id(0); // Across all sources.
  698. proto_aggregate->set_total_count(total_count);
  699. proto_aggregate->set_dropped_due_to_limits(dropped_due_to_limits);
  700. proto_aggregate->set_dropped_due_to_sampling(dropped_due_to_sampling);
  701. proto_aggregate->set_dropped_due_to_filter(dropped_due_to_filter);
  702. proto_aggregate->set_dropped_due_to_unconfigured(dropped_due_to_unconfigured);
  703. for (const auto& metric_and_aggregate : metrics) {
  704. const MetricAggregate& aggregate = metric_and_aggregate.second;
  705. Aggregate::Metric* proto_metric = proto_aggregate->add_metrics();
  706. proto_metric->set_metric_hash(metric_and_aggregate.first);
  707. proto_metric->set_value_sum(aggregate.value_sum);
  708. proto_metric->set_value_square_sum(aggregate.value_square_sum);
  709. if (aggregate.total_count != total_count) {
  710. proto_metric->set_total_count(aggregate.total_count);
  711. }
  712. if (aggregate.dropped_due_to_limits != dropped_due_to_limits) {
  713. proto_metric->set_dropped_due_to_limits(aggregate.dropped_due_to_limits);
  714. }
  715. if (aggregate.dropped_due_to_sampling != dropped_due_to_sampling) {
  716. proto_metric->set_dropped_due_to_sampling(
  717. aggregate.dropped_due_to_sampling);
  718. }
  719. if (aggregate.dropped_due_to_filter != dropped_due_to_filter) {
  720. proto_metric->set_dropped_due_to_filter(aggregate.dropped_due_to_filter);
  721. }
  722. if (aggregate.dropped_due_to_unconfigured != dropped_due_to_unconfigured) {
  723. proto_metric->set_dropped_due_to_unconfigured(
  724. aggregate.dropped_due_to_unconfigured);
  725. }
  726. }
  727. }
  728. void UkmRecorderImpl::MaybeMarkForDeletion(SourceId source_id) {
  729. SourceIdType type = GetSourceIdType(source_id);
  730. switch (type) {
  731. case ukm::SourceIdObj::Type::HISTORY_ID:
  732. case ukm::SourceIdObj::Type::WEBAPK_ID:
  733. case ukm::SourceIdObj::Type::PAYMENT_APP_ID:
  734. case ukm::SourceIdObj::Type::NO_URL_ID:
  735. case ukm::SourceIdObj::Type::WEB_IDENTITY_ID: {
  736. // Don't keep sources of these types after current report because their
  737. // entries are logged only at source creation time.
  738. MarkSourceForDeletion(source_id);
  739. break;
  740. }
  741. case ukm::SourceIdObj::Type::DEFAULT:
  742. case ukm::SourceIdObj::Type::APP_ID:
  743. case ukm::SourceIdObj::Type::DESKTOP_WEB_APP_ID:
  744. case ukm::SourceIdObj::Type::NAVIGATION_ID:
  745. case ukm::SourceIdObj::Type::WORKER_ID:
  746. case ukm::SourceIdObj::Type::REDIRECT_ID:
  747. break;
  748. }
  749. }
  750. UkmRecorderImpl::ShouldRecordUrlResult UkmRecorderImpl::ShouldRecordUrl(
  751. SourceId source_id,
  752. const GURL& sanitized_url) const {
  753. ShouldRecordUrlResult result = ShouldRecordUrlResult::kOk;
  754. bool has_recorded_reason = false;
  755. if (!recording_enabled_) {
  756. RecordDroppedSource(DroppedDataReason::RECORDING_DISABLED);
  757. // Don't return the result yet. Check if the we are allowed to notify
  758. // observers, as they may rely on the not uploaded metrics to determine
  759. // how some features should work.
  760. result = ShouldRecordUrlResult::kObserverOnly;
  761. has_recorded_reason = true;
  762. }
  763. if (recordings_.sources.size() >= max_sources_) {
  764. RecordDroppedSource(has_recorded_reason, DroppedDataReason::MAX_HIT);
  765. return ShouldRecordUrlResult::kDropped;
  766. }
  767. if (ShouldRestrictToWhitelistedSourceIds() &&
  768. !IsWhitelistedSourceId(source_id)) {
  769. RecordDroppedSource(has_recorded_reason,
  770. DroppedDataReason::NOT_WHITELISTED);
  771. return ShouldRecordUrlResult::kDropped;
  772. }
  773. if (sanitized_url.is_empty()) {
  774. RecordDroppedSource(has_recorded_reason, DroppedDataReason::EMPTY_URL);
  775. return ShouldRecordUrlResult::kDropped;
  776. }
  777. if (!HasSupportedScheme(sanitized_url)) {
  778. RecordDroppedSource(has_recorded_reason,
  779. DroppedDataReason::UNSUPPORTED_URL_SCHEME);
  780. DVLOG(2) << "Dropped Unsupported UKM URL:" << source_id << ":"
  781. << sanitized_url.spec();
  782. return ShouldRecordUrlResult::kDropped;
  783. }
  784. // Extension URLs need to be specifically enabled and the extension synced.
  785. if (sanitized_url.SchemeIs(kExtensionScheme)) {
  786. DCHECK_EQ(sanitized_url.GetWithEmptyPath(), sanitized_url);
  787. if (!extensions_enabled_) {
  788. RecordDroppedSource(has_recorded_reason,
  789. DroppedDataReason::EXTENSION_URLS_DISABLED);
  790. return ShouldRecordUrlResult::kDropped;
  791. }
  792. if (!is_webstore_extension_callback_ ||
  793. !is_webstore_extension_callback_.Run(sanitized_url.host_piece())) {
  794. RecordDroppedSource(has_recorded_reason,
  795. DroppedDataReason::EXTENSION_NOT_SYNCED);
  796. return ShouldRecordUrlResult::kDropped;
  797. }
  798. }
  799. return result;
  800. }
  801. void UkmRecorderImpl::RecordSource(std::unique_ptr<UkmSource> source) {
  802. SourceId source_id = source->id();
  803. // If UKM recording is disabled due to |recording_enabled_|, still notify
  804. // observers as they might be interested in it.
  805. NotifyAllObservers(&UkmRecorderObserver::OnUpdateSourceURL, source_id,
  806. source->urls());
  807. if (!recording_enabled_) {
  808. return;
  809. }
  810. if (GetSourceIdType(source_id) == SourceIdType::NAVIGATION_ID)
  811. recordings_.source_counts.navigation_sources++;
  812. recordings_.source_counts.observed++;
  813. recordings_.sources.emplace(source_id, std::move(source));
  814. }
  815. void UkmRecorderImpl::AddEntry(mojom::UkmEntryPtr entry) {
  816. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  817. DCHECK(!HasUnknownMetrics(decode_map_, *entry));
  818. NotifyObserversWithNewEntry(*entry);
  819. if (!recording_enabled_) {
  820. RecordDroppedEntry(entry->event_hash,
  821. DroppedDataReason::RECORDING_DISABLED);
  822. return;
  823. }
  824. if (!ApplyEntryFilter(entry.get())) {
  825. RecordDroppedEntry(entry->event_hash,
  826. DroppedDataReason::REJECTED_BY_FILTER);
  827. return;
  828. }
  829. EventAggregate& event_aggregate =
  830. recordings_.event_aggregations[entry->event_hash];
  831. event_aggregate.total_count++;
  832. for (const auto& metric : entry->metrics) {
  833. MetricAggregate& aggregate = event_aggregate.metrics[metric.first];
  834. double value = metric.second;
  835. aggregate.total_count++;
  836. aggregate.value_sum += value;
  837. aggregate.value_square_sum += value * value;
  838. }
  839. if (!IsSamplingConfigured()) {
  840. RecordDroppedEntry(entry->event_hash,
  841. DroppedDataReason::SAMPLING_UNCONFIGURED);
  842. event_aggregate.dropped_due_to_unconfigured++;
  843. for (auto& metric : entry->metrics)
  844. event_aggregate.metrics[metric.first].dropped_due_to_unconfigured++;
  845. return;
  846. }
  847. if (default_sampling_rate_ < 0) {
  848. LoadExperimentSamplingInfo();
  849. }
  850. bool sampled_in = IsSampledIn(entry->source_id, entry->event_hash);
  851. if (!sampled_in) {
  852. RecordDroppedEntry(entry->event_hash, DroppedDataReason::SAMPLED_OUT);
  853. event_aggregate.dropped_due_to_sampling++;
  854. for (auto& metric : entry->metrics)
  855. event_aggregate.metrics[metric.first].dropped_due_to_sampling++;
  856. return;
  857. }
  858. if (recordings_.entries.size() >= max_entries_) {
  859. RecordDroppedEntry(entry->event_hash, DroppedDataReason::MAX_HIT);
  860. event_aggregate.dropped_due_to_limits++;
  861. for (auto& metric : entry->metrics)
  862. event_aggregate.metrics[metric.first].dropped_due_to_limits++;
  863. return;
  864. }
  865. // Log a corresponding entry to UMA so we get a per-metric breakdown of UKM
  866. // entry counts.
  867. LogEventHashAsUmaHistogram("UKM.Entries.Recorded.ByEntryHash",
  868. entry->event_hash);
  869. recordings_.entries.push_back(std::move(entry));
  870. }
  871. void UkmRecorderImpl::LoadExperimentSamplingInfo() {
  872. // This should be called only if a sampling rate hasn't been loaded.
  873. DCHECK_LT(default_sampling_rate_, 0);
  874. // Default rate must be >= 0 to indicate that load is complete.
  875. default_sampling_rate_ = 1;
  876. // If we don't have the feature, no parameters to load.
  877. if (!base::FeatureList::IsEnabled(kUkmSamplingRateFeature)) {
  878. return;
  879. }
  880. // Check the parameters for sampling controls.
  881. std::map<std::string, std::string> params;
  882. if (base::GetFieldTrialParamsByFeature(kUkmSamplingRateFeature, &params)) {
  883. LoadExperimentSamplingParams(params);
  884. }
  885. }
  886. void UkmRecorderImpl::LoadExperimentSamplingParams(
  887. const std::map<std::string, std::string>& params) {
  888. for (const auto& kv : params) {
  889. const std::string& key = kv.first;
  890. if (key.length() == 0)
  891. continue;
  892. // Keys starting with an underscore are global configuration.
  893. if (key.at(0) == '_') {
  894. if (key == "_default_sampling") {
  895. int sampling;
  896. // We only load non-negative global sampling rates.
  897. if (base::StringToInt(kv.second, &sampling) && sampling >= 0)
  898. default_sampling_rate_ = sampling;
  899. }
  900. continue;
  901. }
  902. // Anything else is an event name.
  903. int sampling;
  904. auto hash = base::HashMetricName(key);
  905. if (base::StringToInt(kv.second, &sampling)) {
  906. // If the parameter is a number then that's the sampling rate.
  907. if (sampling >= 0)
  908. event_sampling_rates_[hash] = sampling;
  909. } else {
  910. // If the parameter is a string then it's the name of another metric
  911. // to which it should be slaved. This allows different metrics to be
  912. // sampled in or out together.
  913. event_sampling_master_[hash] = base::HashMetricName(kv.second);
  914. }
  915. }
  916. }
  917. bool UkmRecorderImpl::IsSampledIn(int64_t source_id, uint64_t event_id) {
  918. // Determine the sampling rate. It's one of:
  919. // - the default
  920. // - an explicit sampling rate
  921. // - a group sampling rate
  922. int sampling_rate = default_sampling_rate_;
  923. uint64_t sampling_hash = event_id;
  924. auto master_found = event_sampling_master_.find(sampling_hash);
  925. if (master_found != event_sampling_master_.end()) {
  926. sampling_hash = master_found->second;
  927. }
  928. auto rate_found = event_sampling_rates_.find(sampling_hash);
  929. if (rate_found != event_sampling_rates_.end()) {
  930. sampling_rate = rate_found->second;
  931. }
  932. return IsSampledIn(source_id, sampling_hash, sampling_rate);
  933. }
  934. bool UkmRecorderImpl::IsSampledIn(int64_t source_id,
  935. uint64_t event_id,
  936. int sampling_rate) {
  937. // A sampling rate of 0 is "never"; everything else is 1-in-N but calculated
  938. // deterministically based on a seed, the source-id, and the event-id. Skip
  939. // the calculation, though, if N==1 because it will always be true. A negative
  940. // rate means "unset"; treat it like "never".
  941. if (sampling_rate <= 0)
  942. return false;
  943. if (sampling_rate == 1)
  944. return true;
  945. // Mutate the "sampling seed" number in a predictable manner based on the
  946. // source and event IDs. This makes the result of this function be always
  947. // the same for the same input parameters (since the seed is fixed during
  948. // construction of this object) which is important for proper sampling
  949. // behavior. CRC32 is fast and statistically random enough for these
  950. // purposes.
  951. uint32_t sampled_num = sampling_seed_;
  952. sampled_num = base::Crc32(sampled_num, &source_id, sizeof(source_id));
  953. sampled_num = base::Crc32(sampled_num, &event_id, sizeof(event_id));
  954. return sampled_num % sampling_rate == 0;
  955. }
  956. void UkmRecorderImpl::InitDecodeMap() {
  957. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  958. decode_map_ = builders::CreateDecodeMap();
  959. }
  960. void UkmRecorderImpl::NotifyObserversWithNewEntry(
  961. const mojom::UkmEntry& entry) {
  962. base::AutoLock auto_lock(lock_);
  963. for (const auto& observer : observers_) {
  964. if (observer.first.contains(entry.event_hash)) {
  965. mojom::UkmEntryPtr cloned = entry.Clone();
  966. observer.second->Notify(FROM_HERE, &UkmRecorderObserver::OnEntryAdded,
  967. base::Passed(&cloned));
  968. }
  969. }
  970. }
  971. template <typename Method, typename... Params>
  972. void UkmRecorderImpl::NotifyAllObservers(Method m, Params&&... params) {
  973. base::AutoLock auto_lock(lock_);
  974. for (const auto& observer : observers_) {
  975. observer.second->Notify(FROM_HERE, m, std::forward<Params>(params)...);
  976. }
  977. }
  978. } // namespace ukm