file_metrics_provider.cc 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946
  1. // Copyright 2016 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/metrics/file_metrics_provider.h"
  5. #include <memory>
  6. #include "base/bind.h"
  7. #include "base/command_line.h"
  8. #include "base/containers/flat_map.h"
  9. #include "base/feature_list.h"
  10. #include "base/files/file.h"
  11. #include "base/files/file_enumerator.h"
  12. #include "base/files/file_util.h"
  13. #include "base/files/memory_mapped_file.h"
  14. #include "base/logging.h"
  15. #include "base/metrics/histogram_base.h"
  16. #include "base/metrics/histogram_functions.h"
  17. #include "base/metrics/histogram_macros.h"
  18. #include "base/metrics/persistent_histogram_allocator.h"
  19. #include "base/metrics/persistent_memory_allocator.h"
  20. #include "base/metrics/ranges_manager.h"
  21. #include "base/strings/string_piece.h"
  22. #include "base/task/task_runner.h"
  23. #include "base/task/task_runner_util.h"
  24. #include "base/task/task_traits.h"
  25. #include "base/task/thread_pool.h"
  26. #include "base/threading/thread_task_runner_handle.h"
  27. #include "base/time/time.h"
  28. #include "components/metrics/metrics_pref_names.h"
  29. #include "components/metrics/metrics_service.h"
  30. #include "components/metrics/persistent_histograms.h"
  31. #include "components/metrics/persistent_system_profile.h"
  32. #include "components/prefs/pref_registry_simple.h"
  33. #include "components/prefs/pref_service.h"
  34. #include "components/prefs/scoped_user_pref_update.h"
  35. namespace metrics {
  36. namespace {
  37. // These structures provide values used to define how files are opened and
  38. // accessed. It obviates the need for multiple code-paths within several of
  39. // the methods.
  40. struct SourceOptions {
  41. // The flags to be used to open a file on disk.
  42. int file_open_flags;
  43. // The access mode to be used when mapping a file into memory.
  44. base::MemoryMappedFile::Access memory_mapped_access;
  45. // Indicates if the file is to be accessed read-only.
  46. bool is_read_only;
  47. };
  48. // Opening a file typically requires at least these flags.
  49. constexpr int STD_OPEN = base::File::FLAG_OPEN | base::File::FLAG_READ;
  50. constexpr SourceOptions kSourceOptions[] = {
  51. // SOURCE_HISTOGRAMS_ATOMIC_FILE
  52. {
  53. // Ensure that no other process reads this at the same time.
  54. STD_OPEN | base::File::FLAG_WIN_EXCLUSIVE_READ,
  55. base::MemoryMappedFile::READ_ONLY,
  56. true,
  57. },
  58. // SOURCE_HISTOGRAMS_ATOMIC_DIR
  59. {
  60. // Ensure that no other process reads this at the same time.
  61. STD_OPEN | base::File::FLAG_WIN_EXCLUSIVE_READ,
  62. base::MemoryMappedFile::READ_ONLY,
  63. true,
  64. },
  65. // SOURCE_HISTOGRAMS_ACTIVE_FILE
  66. {
  67. // Allow writing to the file. This is needed so we can keep track of
  68. // deltas that have been uploaded (by modifying the file), while the
  69. // file may still be open by an external process (e.g. Crashpad).
  70. STD_OPEN | base::File::FLAG_WRITE,
  71. base::MemoryMappedFile::READ_WRITE,
  72. false,
  73. },
  74. };
  75. void DeleteFileWhenPossible(const base::FilePath& path) {
  76. // Open (with delete) and then immediately close the file by going out of
  77. // scope. This is the only cross-platform safe way to delete a file that may
  78. // be open elsewhere, a distinct possibility given the asynchronous nature
  79. // of the delete task.
  80. base::File file(path, base::File::FLAG_OPEN | base::File::FLAG_READ |
  81. base::File::FLAG_DELETE_ON_CLOSE);
  82. }
  83. // A task runner to use for testing.
  84. base::TaskRunner* g_task_runner_for_testing = nullptr;
  85. // Returns a task runner appropriate for running background tasks that perform
  86. // file I/O.
  87. scoped_refptr<base::TaskRunner> CreateBackgroundTaskRunner() {
  88. if (g_task_runner_for_testing)
  89. return scoped_refptr<base::TaskRunner>(g_task_runner_for_testing);
  90. return base::ThreadPool::CreateTaskRunner(
  91. {base::MayBlock(), base::TaskPriority::BEST_EFFORT,
  92. base::TaskShutdownBehavior::SKIP_ON_SHUTDOWN});
  93. }
  94. } // namespace
  95. // This structure stores all the information about the sources being monitored
  96. // and their current reporting state.
  97. struct FileMetricsProvider::SourceInfo {
  98. SourceInfo(const Params& params)
  99. : type(params.type),
  100. association(params.association),
  101. prefs_key(params.prefs_key),
  102. filter(params.filter),
  103. max_age(params.max_age),
  104. max_dir_kib(params.max_dir_kib),
  105. max_dir_files(params.max_dir_files) {
  106. switch (type) {
  107. case SOURCE_HISTOGRAMS_ACTIVE_FILE:
  108. DCHECK(prefs_key.empty());
  109. [[fallthrough]];
  110. case SOURCE_HISTOGRAMS_ATOMIC_FILE:
  111. path = params.path;
  112. break;
  113. case SOURCE_HISTOGRAMS_ATOMIC_DIR:
  114. directory = params.path;
  115. break;
  116. }
  117. }
  118. SourceInfo(const SourceInfo&) = delete;
  119. SourceInfo& operator=(const SourceInfo&) = delete;
  120. ~SourceInfo() {}
  121. struct FoundFile {
  122. base::FilePath path;
  123. base::FileEnumerator::FileInfo info;
  124. };
  125. using FoundFiles = base::flat_map<base::Time, FoundFile>;
  126. // How to access this source (file/dir, atomic/active).
  127. const SourceType type;
  128. // With what run this source is associated.
  129. const SourceAssociation association;
  130. // Where on disk the directory is located. This will only be populated when
  131. // a directory is being monitored.
  132. base::FilePath directory;
  133. // The files found in the above directory, ordered by last-modified.
  134. std::unique_ptr<FoundFiles> found_files;
  135. // Where on disk the file is located. If a directory is being monitored,
  136. // this will be updated for whatever file is being read.
  137. base::FilePath path;
  138. // Name used inside prefs to persistent metadata.
  139. std::string prefs_key;
  140. // The filter callback for determining what to do with found files.
  141. FilterCallback filter;
  142. // The maximum allowed age of a file.
  143. base::TimeDelta max_age;
  144. // The maximum allowed bytes in a directory.
  145. size_t max_dir_kib;
  146. // The maximum allowed files in a directory.
  147. size_t max_dir_files;
  148. // The last-seen time of this source to detect change.
  149. base::Time last_seen;
  150. // Indicates if the data has been read out or not.
  151. bool read_complete = false;
  152. // Once a file has been recognized as needing to be read, it is mapped
  153. // into memory and assigned to an |allocator| object.
  154. std::unique_ptr<base::PersistentHistogramAllocator> allocator;
  155. };
  156. FileMetricsProvider::Params::Params(const base::FilePath& path,
  157. SourceType type,
  158. SourceAssociation association,
  159. base::StringPiece prefs_key)
  160. : path(path), type(type), association(association), prefs_key(prefs_key) {}
  161. FileMetricsProvider::Params::~Params() {}
  162. FileMetricsProvider::FileMetricsProvider(PrefService* local_state)
  163. : task_runner_(CreateBackgroundTaskRunner()),
  164. pref_service_(local_state),
  165. main_task_runner_(base::ThreadTaskRunnerHandle::Get()) {
  166. DCHECK(main_task_runner_);
  167. base::StatisticsRecorder::RegisterHistogramProvider(
  168. weak_factory_.GetWeakPtr());
  169. }
  170. FileMetricsProvider::~FileMetricsProvider() {}
  171. void FileMetricsProvider::RegisterSource(const Params& params) {
  172. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  173. // Ensure that kSourceOptions has been filled for this type.
  174. DCHECK_GT(std::size(kSourceOptions), static_cast<size_t>(params.type));
  175. std::unique_ptr<SourceInfo> source(new SourceInfo(params));
  176. // |prefs_key| may be empty if the caller does not wish to persist the
  177. // state across instances of the program.
  178. if (pref_service_ && !params.prefs_key.empty()) {
  179. source->last_seen = pref_service_->GetTime(
  180. metrics::prefs::kMetricsLastSeenPrefix + source->prefs_key);
  181. }
  182. switch (params.association) {
  183. case ASSOCIATE_CURRENT_RUN:
  184. case ASSOCIATE_INTERNAL_PROFILE:
  185. case ASSOCIATE_INTERNAL_PROFILE_SAMPLES_COUNTER:
  186. sources_to_check_.push_back(std::move(source));
  187. break;
  188. case ASSOCIATE_PREVIOUS_RUN:
  189. case ASSOCIATE_INTERNAL_PROFILE_OR_PREVIOUS_RUN:
  190. DCHECK_EQ(SOURCE_HISTOGRAMS_ATOMIC_FILE, source->type);
  191. sources_for_previous_run_.push_back(std::move(source));
  192. break;
  193. }
  194. }
  195. // static
  196. void FileMetricsProvider::RegisterSourcePrefs(
  197. PrefRegistrySimple* prefs,
  198. const base::StringPiece prefs_key) {
  199. prefs->RegisterInt64Pref(
  200. metrics::prefs::kMetricsLastSeenPrefix + std::string(prefs_key), 0);
  201. }
  202. // static
  203. void FileMetricsProvider::RegisterPrefs(PrefRegistrySimple* prefs) {
  204. prefs->RegisterListPref(metrics::prefs::kMetricsFileMetricsMetadata);
  205. }
  206. // static
  207. void FileMetricsProvider::SetTaskRunnerForTesting(
  208. const scoped_refptr<base::TaskRunner>& task_runner) {
  209. DCHECK(!g_task_runner_for_testing || !task_runner);
  210. g_task_runner_for_testing = task_runner.get();
  211. }
  212. // static
  213. void FileMetricsProvider::RecordAccessResult(AccessResult result) {
  214. UMA_HISTOGRAM_ENUMERATION("UMA.FileMetricsProvider.AccessResult", result,
  215. ACCESS_RESULT_MAX);
  216. }
  217. // static
  218. bool FileMetricsProvider::LocateNextFileInDirectory(SourceInfo* source) {
  219. DCHECK_EQ(SOURCE_HISTOGRAMS_ATOMIC_DIR, source->type);
  220. DCHECK(!source->directory.empty());
  221. // Cumulative directory stats. These will remain zero if the directory isn't
  222. // scanned but that's okay since any work they would cause to be done below
  223. // would have been done during the first call where the directory was fully
  224. // scanned.
  225. size_t total_size_kib = 0; // Using KiB allows 4TiB even on 32-bit builds.
  226. size_t file_count = 0;
  227. base::Time now_time = base::Time::Now();
  228. if (!source->found_files) {
  229. source->found_files = std::make_unique<SourceInfo::FoundFiles>();
  230. base::FileEnumerator file_iter(source->directory, /*recursive=*/false,
  231. base::FileEnumerator::FILES);
  232. SourceInfo::FoundFile found_file;
  233. // Open the directory and find all the files, remembering the last-modified
  234. // time of each.
  235. for (found_file.path = file_iter.Next(); !found_file.path.empty();
  236. found_file.path = file_iter.Next()) {
  237. found_file.info = file_iter.GetInfo();
  238. // Ignore directories.
  239. if (found_file.info.IsDirectory())
  240. continue;
  241. // Ignore temporary files.
  242. base::FilePath::CharType first_character =
  243. found_file.path.BaseName().value().front();
  244. if (first_character == FILE_PATH_LITERAL('.') ||
  245. first_character == FILE_PATH_LITERAL('_')) {
  246. continue;
  247. }
  248. // Ignore non-PMA (Persistent Memory Allocator) files.
  249. if (found_file.path.Extension() !=
  250. base::PersistentMemoryAllocator::kFileExtension) {
  251. continue;
  252. }
  253. // Process real files.
  254. total_size_kib += found_file.info.GetSize() >> 10;
  255. base::Time modified = found_file.info.GetLastModifiedTime();
  256. if (modified > source->last_seen) {
  257. // This file hasn't been read. Remember it (unless from the future).
  258. if (modified <= now_time)
  259. source->found_files->emplace(modified, std::move(found_file));
  260. ++file_count;
  261. } else {
  262. // This file has been read. Try to delete it. Ignore any errors because
  263. // the file may be un-removeable by this process. It could, for example,
  264. // have been created by a privileged process like setup.exe. Even if it
  265. // is not removed, it will continue to be ignored bacuse of the older
  266. // modification time.
  267. base::DeleteFile(found_file.path);
  268. }
  269. }
  270. }
  271. // Filter files from the front until one is found for processing.
  272. bool have_file = false;
  273. while (!source->found_files->empty()) {
  274. SourceInfo::FoundFile found =
  275. std::move(source->found_files->begin()->second);
  276. source->found_files->erase(source->found_files->begin());
  277. bool too_many =
  278. source->max_dir_files > 0 && file_count > source->max_dir_files;
  279. bool too_big =
  280. source->max_dir_kib > 0 && total_size_kib > source->max_dir_kib;
  281. bool too_old =
  282. source->max_age != base::TimeDelta() &&
  283. now_time - found.info.GetLastModifiedTime() > source->max_age;
  284. if (too_many || too_big || too_old) {
  285. base::DeleteFile(found.path);
  286. --file_count;
  287. total_size_kib -= found.info.GetSize() >> 10;
  288. RecordAccessResult(too_many ? ACCESS_RESULT_TOO_MANY_FILES
  289. : too_big ? ACCESS_RESULT_TOO_MANY_BYTES
  290. : ACCESS_RESULT_TOO_OLD);
  291. continue;
  292. }
  293. AccessResult result = HandleFilterSource(source, found.path);
  294. if (result == ACCESS_RESULT_SUCCESS) {
  295. source->path = std::move(found.path);
  296. have_file = true;
  297. break;
  298. }
  299. // Record the result. Success will be recorded by the caller.
  300. if (result != ACCESS_RESULT_THIS_PID)
  301. RecordAccessResult(result);
  302. }
  303. return have_file;
  304. }
  305. // static
  306. void FileMetricsProvider::FinishedWithSource(SourceInfo* source,
  307. AccessResult result) {
  308. // Different source types require different post-processing.
  309. switch (source->type) {
  310. case SOURCE_HISTOGRAMS_ATOMIC_FILE:
  311. case SOURCE_HISTOGRAMS_ATOMIC_DIR:
  312. // Done with this file so delete the allocator and its owned file.
  313. source->allocator.reset();
  314. // Remove the file if has been recorded. This prevents them from
  315. // accumulating or also being recorded by different instances of
  316. // the browser.
  317. if (result == ACCESS_RESULT_SUCCESS ||
  318. result == ACCESS_RESULT_NOT_MODIFIED ||
  319. result == ACCESS_RESULT_MEMORY_DELETED ||
  320. result == ACCESS_RESULT_TOO_OLD) {
  321. DeleteFileWhenPossible(source->path);
  322. }
  323. break;
  324. case SOURCE_HISTOGRAMS_ACTIVE_FILE:
  325. // Keep the allocator open so it doesn't have to be re-mapped each
  326. // time. This also allows the contents to be merged on-demand.
  327. break;
  328. }
  329. }
  330. void FileMetricsProvider::CheckAndMergeMetricSourcesOnTaskRunner(
  331. SourceInfoList* sources) {
  332. // This method has all state information passed in |sources| and is intended
  333. // to run on a worker thread rather than the UI thread.
  334. for (std::unique_ptr<SourceInfo>& source : *sources) {
  335. AccessResult result;
  336. do {
  337. result = CheckAndMapMetricSource(source.get());
  338. // Some results are not reported in order to keep the dashboard clean.
  339. if (result != ACCESS_RESULT_DOESNT_EXIST &&
  340. result != ACCESS_RESULT_NOT_MODIFIED &&
  341. result != ACCESS_RESULT_THIS_PID) {
  342. RecordAccessResult(result);
  343. }
  344. // If there are no files (or no more files) in this source, stop now.
  345. if (result == ACCESS_RESULT_DOESNT_EXIST)
  346. break;
  347. // Mapping was successful. Merge it.
  348. if (result == ACCESS_RESULT_SUCCESS) {
  349. // Metrics associated with internal profiles have to be fetched directly
  350. // so just keep the mapping for use by the main thread.
  351. if (source->association == ASSOCIATE_INTERNAL_PROFILE)
  352. break;
  353. if (source->association == ASSOCIATE_INTERNAL_PROFILE_SAMPLES_COUNTER) {
  354. RecordFileMetadataOnTaskRunner(source.get());
  355. } else {
  356. MergeHistogramDeltasFromSource(source.get());
  357. }
  358. DCHECK(source->read_complete);
  359. }
  360. // All done with this source.
  361. FinishedWithSource(source.get(), result);
  362. // If it's a directory, keep trying until a file is successfully opened.
  363. // When there are no more files, ACCESS_RESULT_DOESNT_EXIST will be
  364. // returned and the loop will exit above.
  365. } while (result != ACCESS_RESULT_SUCCESS && !source->directory.empty());
  366. // If the set of known files is empty, clear the object so the next run
  367. // will do a fresh scan of the directory.
  368. if (source->found_files && source->found_files->empty())
  369. source->found_files.reset();
  370. }
  371. }
  372. // This method has all state information passed in |source| and is intended
  373. // to run on a worker thread rather than the UI thread.
  374. // static
  375. FileMetricsProvider::AccessResult FileMetricsProvider::CheckAndMapMetricSource(
  376. SourceInfo* source) {
  377. // If source was read, clean up after it.
  378. if (source->read_complete)
  379. FinishedWithSource(source, ACCESS_RESULT_SUCCESS);
  380. source->read_complete = false;
  381. DCHECK(!source->allocator);
  382. // If the source is a directory, look for files within it.
  383. if (!source->directory.empty() && !LocateNextFileInDirectory(source))
  384. return ACCESS_RESULT_DOESNT_EXIST;
  385. // Do basic validation on the file metadata.
  386. base::File::Info info;
  387. if (!base::GetFileInfo(source->path, &info))
  388. return ACCESS_RESULT_DOESNT_EXIST;
  389. if (info.is_directory || info.size == 0)
  390. return ACCESS_RESULT_INVALID_FILE;
  391. if (source->last_seen >= info.last_modified)
  392. return ACCESS_RESULT_NOT_MODIFIED;
  393. if (source->max_age != base::TimeDelta() &&
  394. base::Time::Now() - info.last_modified > source->max_age) {
  395. return ACCESS_RESULT_TOO_OLD;
  396. }
  397. // Non-directory files still need to be filtered.
  398. if (source->directory.empty()) {
  399. AccessResult result = HandleFilterSource(source, source->path);
  400. if (result != ACCESS_RESULT_SUCCESS)
  401. return result;
  402. }
  403. // A new file of metrics has been found.
  404. base::File file(source->path, kSourceOptions[source->type].file_open_flags);
  405. if (!file.IsValid())
  406. return ACCESS_RESULT_NO_OPEN;
  407. // Check that file is writable if that is expected. If a write is attempted
  408. // on an unwritable memory-mapped file, a SIGBUS will cause a crash.
  409. const bool read_only = kSourceOptions[source->type].is_read_only;
  410. if (!read_only) {
  411. constexpr int kTestSize = 16;
  412. char header[kTestSize];
  413. int amount = file.Read(0, header, kTestSize);
  414. if (amount != kTestSize)
  415. return ACCESS_RESULT_INVALID_CONTENTS;
  416. char zeros[kTestSize] = {0};
  417. file.Write(0, zeros, kTestSize);
  418. file.Flush();
  419. // A crash here would be unfortunate as the file would be left invalid
  420. // and skipped/deleted by later attempts. This is unlikely, however, and
  421. // the benefit of avoiding crashes from mapping as read/write a file that
  422. // can't be written more than justifies the risk.
  423. char check[kTestSize];
  424. amount = file.Read(0, check, kTestSize);
  425. if (amount != kTestSize)
  426. return ACCESS_RESULT_INVALID_CONTENTS;
  427. if (memcmp(check, zeros, kTestSize) != 0)
  428. return ACCESS_RESULT_NOT_WRITABLE;
  429. file.Write(0, header, kTestSize);
  430. file.Flush();
  431. amount = file.Read(0, check, kTestSize);
  432. if (amount != kTestSize)
  433. return ACCESS_RESULT_INVALID_CONTENTS;
  434. if (memcmp(check, header, kTestSize) != 0)
  435. return ACCESS_RESULT_NOT_WRITABLE;
  436. }
  437. std::unique_ptr<base::MemoryMappedFile> mapped(new base::MemoryMappedFile());
  438. if (!mapped->Initialize(std::move(file),
  439. kSourceOptions[source->type].memory_mapped_access)) {
  440. return ACCESS_RESULT_SYSTEM_MAP_FAILURE;
  441. }
  442. // Ensure any problems below don't occur repeatedly.
  443. source->last_seen = info.last_modified;
  444. // Test the validity of the file contents.
  445. if (!base::FilePersistentMemoryAllocator::IsFileAcceptable(*mapped,
  446. read_only)) {
  447. return ACCESS_RESULT_INVALID_CONTENTS;
  448. }
  449. // Map the file and validate it.
  450. std::unique_ptr<base::FilePersistentMemoryAllocator> memory_allocator =
  451. std::make_unique<base::FilePersistentMemoryAllocator>(
  452. std::move(mapped), 0, 0, base::StringPiece(), read_only);
  453. if (memory_allocator->GetMemoryState() ==
  454. base::PersistentMemoryAllocator::MEMORY_DELETED) {
  455. return ACCESS_RESULT_MEMORY_DELETED;
  456. }
  457. if (memory_allocator->IsCorrupt())
  458. return ACCESS_RESULT_DATA_CORRUPTION;
  459. // Cache the file data while running in a background thread so that there
  460. // shouldn't be any I/O when the data is accessed from the main thread.
  461. // Files with an internal profile, those from previous runs that include
  462. // a full system profile and are fetched via ProvideIndependentMetrics(),
  463. // are loaded on a background task and so there's no need to cache the
  464. // data in advance.
  465. if (source->association != ASSOCIATE_INTERNAL_PROFILE)
  466. memory_allocator->Cache();
  467. // Create an allocator for the mapped file. Ownership passes to the allocator.
  468. source->allocator = std::make_unique<base::PersistentHistogramAllocator>(
  469. std::move(memory_allocator));
  470. // Check that an "independent" file has the necessary information present.
  471. if (source->association == ASSOCIATE_INTERNAL_PROFILE &&
  472. !PersistentSystemProfile::GetSystemProfile(
  473. *source->allocator->memory_allocator(), nullptr)) {
  474. return ACCESS_RESULT_NO_PROFILE;
  475. }
  476. return ACCESS_RESULT_SUCCESS;
  477. }
  478. // static
  479. void FileMetricsProvider::MergeHistogramDeltasFromSource(SourceInfo* source) {
  480. DCHECK(source->allocator);
  481. base::PersistentHistogramAllocator::Iterator histogram_iter(
  482. source->allocator.get());
  483. const bool read_only = kSourceOptions[source->type].is_read_only;
  484. int histogram_count = 0;
  485. while (true) {
  486. std::unique_ptr<base::HistogramBase> histogram = histogram_iter.GetNext();
  487. if (!histogram)
  488. break;
  489. if (read_only) {
  490. source->allocator->MergeHistogramFinalDeltaToStatisticsRecorder(
  491. histogram.get());
  492. } else {
  493. source->allocator->MergeHistogramDeltaToStatisticsRecorder(
  494. histogram.get());
  495. }
  496. ++histogram_count;
  497. }
  498. source->read_complete = true;
  499. DVLOG(1) << "Reported " << histogram_count << " histograms from "
  500. << source->path.value();
  501. }
  502. // static
  503. void FileMetricsProvider::RecordHistogramSnapshotsFromSource(
  504. base::HistogramSnapshotManager* snapshot_manager,
  505. SourceInfo* source) {
  506. DCHECK_NE(SOURCE_HISTOGRAMS_ACTIVE_FILE, source->type);
  507. base::PersistentHistogramAllocator::Iterator histogram_iter(
  508. source->allocator.get());
  509. int histogram_count = 0;
  510. while (true) {
  511. std::unique_ptr<base::HistogramBase> histogram = histogram_iter.GetNext();
  512. if (!histogram)
  513. break;
  514. snapshot_manager->PrepareFinalDelta(histogram.get());
  515. ++histogram_count;
  516. }
  517. source->read_complete = true;
  518. DVLOG(1) << "Reported " << histogram_count << " histograms from "
  519. << source->path.value();
  520. }
  521. FileMetricsProvider::AccessResult FileMetricsProvider::HandleFilterSource(
  522. SourceInfo* source,
  523. const base::FilePath& path) {
  524. if (!source->filter)
  525. return ACCESS_RESULT_SUCCESS;
  526. // Alternatively, pass a Params object to the filter like what was originally
  527. // used to configure the source.
  528. // Params params(path, source->type, source->association, source->prefs_key);
  529. FilterAction action = source->filter.Run(path);
  530. switch (action) {
  531. case FILTER_PROCESS_FILE:
  532. // Process the file.
  533. return ACCESS_RESULT_SUCCESS;
  534. case FILTER_ACTIVE_THIS_PID:
  535. // Even the file for the current process has to be touched or its stamp
  536. // will be less than "last processed" and thus skipped on future runs,
  537. // even those done by new instances of the browser if a pref key is
  538. // provided so that the last-uploaded stamp is recorded.
  539. case FILTER_TRY_LATER: {
  540. // Touch the file with the current timestamp making it (presumably) the
  541. // newest file in the directory.
  542. base::Time now = base::Time::Now();
  543. base::TouchFile(path, /*accessed=*/now, /*modified=*/now);
  544. if (action == FILTER_ACTIVE_THIS_PID)
  545. return ACCESS_RESULT_THIS_PID;
  546. return ACCESS_RESULT_FILTER_TRY_LATER;
  547. }
  548. case FILTER_SKIP_FILE:
  549. switch (source->type) {
  550. case SOURCE_HISTOGRAMS_ATOMIC_FILE:
  551. case SOURCE_HISTOGRAMS_ATOMIC_DIR:
  552. // Only "atomic" files are deleted (best-effort).
  553. DeleteFileWhenPossible(path);
  554. break;
  555. case SOURCE_HISTOGRAMS_ACTIVE_FILE:
  556. // File will presumably get modified elsewhere and thus tried again.
  557. break;
  558. }
  559. return ACCESS_RESULT_FILTER_SKIP_FILE;
  560. }
  561. // Code never gets here but some compilers don't realize that and so complain
  562. // that "not all control paths return a value".
  563. NOTREACHED();
  564. return ACCESS_RESULT_SUCCESS;
  565. }
  566. /* static */
  567. bool FileMetricsProvider::ProvideIndependentMetricsOnTaskRunner(
  568. SourceInfo* source,
  569. SystemProfileProto* system_profile_proto,
  570. base::HistogramSnapshotManager* snapshot_manager) {
  571. if (PersistentSystemProfile::GetSystemProfile(
  572. *source->allocator->memory_allocator(), system_profile_proto)) {
  573. // Pass a custom RangesManager so that we do not register the BucketRanges
  574. // with the global statistics recorder. Otherwise, it could add unnecessary
  575. // contention, and a low amount of extra memory that will never be released.
  576. source->allocator->SetRangesManager(new base::RangesManager());
  577. system_profile_proto->mutable_stability()->set_from_previous_run(true);
  578. RecordHistogramSnapshotsFromSource(snapshot_manager, source);
  579. return true;
  580. }
  581. return false;
  582. }
  583. void FileMetricsProvider::AppendToSamplesCountPref(size_t samples_count) {
  584. ListPrefUpdate update(pref_service_,
  585. metrics::prefs::kMetricsFileMetricsMetadata);
  586. update->GetList().Append(static_cast<int>(samples_count));
  587. }
  588. void FileMetricsProvider::RecordFileMetadataOnTaskRunner(SourceInfo* source) {
  589. base::HistogramBase::Count samples_count = 0;
  590. base::PersistentHistogramAllocator::Iterator it{source->allocator.get()};
  591. std::unique_ptr<base::HistogramBase> histogram;
  592. while ((histogram = it.GetNext()) != nullptr) {
  593. samples_count += histogram->SnapshotFinalDelta()->TotalCount();
  594. }
  595. main_task_runner_->PostTask(
  596. FROM_HERE, base::BindOnce(&FileMetricsProvider::AppendToSamplesCountPref,
  597. base::Unretained(this), samples_count));
  598. source->read_complete = true;
  599. }
  600. void FileMetricsProvider::ScheduleSourcesCheck() {
  601. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  602. if (sources_to_check_.empty())
  603. return;
  604. // Create an independent list of sources for checking. This will be Owned()
  605. // by the reply call given to the task-runner, to be deleted when that call
  606. // has returned. It is also passed Unretained() to the task itself, safe
  607. // because that must complete before the reply runs.
  608. SourceInfoList* check_list = new SourceInfoList();
  609. std::swap(sources_to_check_, *check_list);
  610. task_runner_->PostTaskAndReply(
  611. FROM_HERE,
  612. base::BindOnce(
  613. &FileMetricsProvider::CheckAndMergeMetricSourcesOnTaskRunner,
  614. base::Unretained(this), base::Unretained(check_list)),
  615. base::BindOnce(&FileMetricsProvider::RecordSourcesChecked,
  616. weak_factory_.GetWeakPtr(), base::Owned(check_list)));
  617. }
  618. void FileMetricsProvider::RecordSourcesChecked(SourceInfoList* checked) {
  619. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  620. // Sources that still have an allocator at this point are read/write "active"
  621. // files that may need their contents merged on-demand. If there is no
  622. // allocator (not a read/write file) but a read was done on the task-runner,
  623. // try again immediately to see if more is available (in a directory of
  624. // files). Otherwise, remember the source for checking again at a later time.
  625. bool did_read = false;
  626. for (auto iter = checked->begin(); iter != checked->end();) {
  627. auto temp = iter++;
  628. SourceInfo* source = temp->get();
  629. if (source->read_complete) {
  630. RecordSourceAsRead(source);
  631. did_read = true;
  632. }
  633. if (source->allocator) {
  634. if (source->association == ASSOCIATE_INTERNAL_PROFILE) {
  635. sources_with_profile_.splice(sources_with_profile_.end(), *checked,
  636. temp);
  637. } else {
  638. sources_mapped_.splice(sources_mapped_.end(), *checked, temp);
  639. }
  640. } else {
  641. sources_to_check_.splice(sources_to_check_.end(), *checked, temp);
  642. }
  643. }
  644. // If a read was done, schedule another one immediately. In the case of a
  645. // directory of files, this ensures that all entries get processed. It's
  646. // done here instead of as a loop in CheckAndMergeMetricSourcesOnTaskRunner
  647. // so that (a) it gives the disk a rest and (b) testing of individual reads
  648. // is possible.
  649. if (did_read)
  650. ScheduleSourcesCheck();
  651. }
  652. void FileMetricsProvider::DeleteFileAsync(const base::FilePath& path) {
  653. task_runner_->PostTask(FROM_HERE,
  654. base::BindOnce(DeleteFileWhenPossible, path));
  655. }
  656. void FileMetricsProvider::RecordSourceAsRead(SourceInfo* source) {
  657. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  658. // Persistently record the "last seen" timestamp of the source file to
  659. // ensure that the file is never read again unless it is modified again.
  660. if (pref_service_ && !source->prefs_key.empty()) {
  661. pref_service_->SetTime(
  662. metrics::prefs::kMetricsLastSeenPrefix + source->prefs_key,
  663. source->last_seen);
  664. }
  665. }
  666. void FileMetricsProvider::OnDidCreateMetricsLog() {
  667. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  668. // Schedule a check to see if there are new metrics to load. If so, they will
  669. // be reported during the next collection run after this one. The check is run
  670. // off of a MayBlock() TaskRunner so as to not cause delays on the main UI
  671. // thread (which is currently where metric collection is done).
  672. ScheduleSourcesCheck();
  673. // Clear any data for initial metrics since they're always reported
  674. // before the first call to this method. It couldn't be released after
  675. // being reported in RecordInitialHistogramSnapshots because the data
  676. // will continue to be used by the caller after that method returns. Once
  677. // here, though, all actions to be done on the data have been completed.
  678. for (const std::unique_ptr<SourceInfo>& source : sources_for_previous_run_)
  679. DeleteFileAsync(source->path);
  680. sources_for_previous_run_.clear();
  681. }
  682. bool FileMetricsProvider::HasIndependentMetrics() {
  683. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  684. return !sources_with_profile_.empty() || SimulateIndependentMetrics();
  685. }
  686. void FileMetricsProvider::ProvideIndependentMetrics(
  687. base::OnceCallback<void(bool)> done_callback,
  688. ChromeUserMetricsExtension* uma_proto,
  689. base::HistogramSnapshotManager* snapshot_manager) {
  690. SystemProfileProto* system_profile_proto =
  691. uma_proto->mutable_system_profile();
  692. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  693. if (sources_with_profile_.empty()) {
  694. std::move(done_callback).Run(false);
  695. return;
  696. }
  697. std::unique_ptr<SourceInfo> source =
  698. std::move(*sources_with_profile_.begin());
  699. sources_with_profile_.pop_front();
  700. SourceInfo* source_ptr = source.get();
  701. DCHECK(source->allocator);
  702. // Do the actual work as a background task.
  703. base::PostTaskAndReplyWithResult(
  704. task_runner_.get(), FROM_HERE,
  705. base::BindOnce(
  706. &FileMetricsProvider::ProvideIndependentMetricsOnTaskRunner,
  707. source_ptr, system_profile_proto, snapshot_manager),
  708. base::BindOnce(&FileMetricsProvider::ProvideIndependentMetricsCleanup,
  709. weak_factory_.GetWeakPtr(), std::move(done_callback),
  710. std::move(source)));
  711. }
  712. void FileMetricsProvider::ProvideIndependentMetricsCleanup(
  713. base::OnceCallback<void(bool)> done_callback,
  714. std::unique_ptr<SourceInfo> source,
  715. bool success) {
  716. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  717. // Regardless of whether this source was successfully recorded, it is
  718. // never read again.
  719. source->read_complete = true;
  720. RecordSourceAsRead(source.get());
  721. sources_to_check_.push_back(std::move(source));
  722. ScheduleSourcesCheck();
  723. // Execute the chained callback.
  724. std::move(done_callback).Run(success);
  725. }
  726. bool FileMetricsProvider::HasPreviousSessionData() {
  727. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  728. // Check all sources for previous run to see if they need to be read.
  729. for (auto iter = sources_for_previous_run_.begin();
  730. iter != sources_for_previous_run_.end();) {
  731. auto temp = iter++;
  732. SourceInfo* source = temp->get();
  733. // This would normally be done on a background I/O thread but there
  734. // hasn't been a chance to run any at the time this method is called.
  735. // Do the check in-line.
  736. AccessResult result = CheckAndMapMetricSource(source);
  737. UMA_HISTOGRAM_ENUMERATION("UMA.FileMetricsProvider.InitialAccessResult",
  738. result, ACCESS_RESULT_MAX);
  739. // If it couldn't be accessed, remove it from the list. There is only ever
  740. // one chance to record it so no point keeping it around for later. Also
  741. // mark it as having been read since uploading it with a future browser
  742. // run would associate it with the then-previous run which would no longer
  743. // be the run from which it came.
  744. if (result != ACCESS_RESULT_SUCCESS) {
  745. DCHECK(!source->allocator);
  746. RecordSourceAsRead(source);
  747. DeleteFileAsync(source->path);
  748. sources_for_previous_run_.erase(temp);
  749. continue;
  750. }
  751. DCHECK(source->allocator);
  752. // If the source should be associated with an existing internal profile,
  753. // move it to |sources_with_profile_| for later upload.
  754. if (source->association == ASSOCIATE_INTERNAL_PROFILE_OR_PREVIOUS_RUN) {
  755. if (PersistentSystemProfile::HasSystemProfile(
  756. *source->allocator->memory_allocator())) {
  757. sources_with_profile_.splice(sources_with_profile_.end(),
  758. sources_for_previous_run_, temp);
  759. }
  760. }
  761. }
  762. return !sources_for_previous_run_.empty();
  763. }
  764. void FileMetricsProvider::RecordInitialHistogramSnapshots(
  765. base::HistogramSnapshotManager* snapshot_manager) {
  766. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  767. for (const std::unique_ptr<SourceInfo>& source : sources_for_previous_run_) {
  768. // The source needs to have an allocator attached to it in order to read
  769. // histograms out of it.
  770. DCHECK(!source->read_complete);
  771. DCHECK(source->allocator);
  772. // Dump all histograms contained within the source to the snapshot-manager.
  773. RecordHistogramSnapshotsFromSource(snapshot_manager, source.get());
  774. // Update the last-seen time so it isn't read again unless it changes.
  775. RecordSourceAsRead(source.get());
  776. }
  777. }
  778. void FileMetricsProvider::MergeHistogramDeltas() {
  779. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
  780. for (std::unique_ptr<SourceInfo>& source : sources_mapped_) {
  781. MergeHistogramDeltasFromSource(source.get());
  782. }
  783. }
  784. bool FileMetricsProvider::SimulateIndependentMetrics() {
  785. if (!pref_service_->HasPrefPath(
  786. metrics::prefs::kMetricsFileMetricsMetadata)) {
  787. return false;
  788. }
  789. ListPrefUpdate list_pref(pref_service_,
  790. metrics::prefs::kMetricsFileMetricsMetadata);
  791. base::Value::List& list_value = list_pref->GetList();
  792. if (list_value.empty())
  793. return false;
  794. size_t count = pref_service_->GetInteger(
  795. metrics::prefs::kStabilityFileMetricsUnsentSamplesCount);
  796. pref_service_->SetInteger(
  797. metrics::prefs::kStabilityFileMetricsUnsentSamplesCount,
  798. list_value[0].GetInt() + count);
  799. pref_service_->SetInteger(
  800. metrics::prefs::kStabilityFileMetricsUnsentFilesCount,
  801. list_value.size() - 1);
  802. list_value.erase(list_value.begin());
  803. return true;
  804. }
  805. } // namespace metrics