persistent_histogram_allocator.h 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513
  1. // Copyright 2016 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef BASE_METRICS_PERSISTENT_HISTOGRAM_ALLOCATOR_H_
  5. #define BASE_METRICS_PERSISTENT_HISTOGRAM_ALLOCATOR_H_
  6. #include <atomic>
  7. #include <map>
  8. #include <memory>
  9. #include <string>
  10. #include <vector>
  11. #include "base/base_export.h"
  12. #include "base/feature_list.h"
  13. #include "base/memory/raw_ptr.h"
  14. #include "base/metrics/histogram_base.h"
  15. #include "base/metrics/persistent_memory_allocator.h"
  16. #include "base/metrics/ranges_manager.h"
  17. #include "base/process/process_handle.h"
  18. #include "base/strings/string_piece.h"
  19. #include "base/synchronization/lock.h"
  20. #include "build/build_config.h"
  21. namespace base {
  22. class BucketRanges;
  23. class FilePath;
  24. class PersistentSampleMapRecords;
  25. class PersistentSparseHistogramDataManager;
  26. class WritableSharedMemoryRegion;
  27. // Feature definition for enabling histogram persistence.
  28. BASE_EXPORT extern const Feature kPersistentHistogramsFeature;
  29. // A data manager for sparse histograms so each instance of such doesn't have
  30. // to separately iterate over the entire memory segment. Though this class
  31. // will generally be accessed through the PersistentHistogramAllocator above,
  32. // it can be used independently on any PersistentMemoryAllocator (making it
  33. // useable for testing). This object supports only one instance of a sparse
  34. // histogram for a given id. Tests that create multiple identical histograms,
  35. // perhaps to simulate multiple processes, should create a separate manager
  36. // for each.
  37. class BASE_EXPORT PersistentSparseHistogramDataManager {
  38. public:
  39. // Constructs the data manager. The allocator must live longer than any
  40. // managers that reference it.
  41. explicit PersistentSparseHistogramDataManager(
  42. PersistentMemoryAllocator* allocator);
  43. PersistentSparseHistogramDataManager(
  44. const PersistentSparseHistogramDataManager&) = delete;
  45. PersistentSparseHistogramDataManager& operator=(
  46. const PersistentSparseHistogramDataManager&) = delete;
  47. ~PersistentSparseHistogramDataManager();
  48. // Returns the object that manages the persistent-sample-map records for a
  49. // given |id|. Only one |user| of this data is allowed at a time. This does
  50. // an automatic Acquire() on the records. The user must call Release() on
  51. // the returned object when it is finished with it. Ownership of the records
  52. // object stays with this manager.
  53. PersistentSampleMapRecords* UseSampleMapRecords(uint64_t id,
  54. const void* user);
  55. // Convenience method that gets the object for a given reference so callers
  56. // don't have to also keep their own pointer to the appropriate allocator.
  57. template <typename T>
  58. T* GetAsObject(PersistentMemoryAllocator::Reference ref) {
  59. return allocator_->GetAsObject<T>(ref);
  60. }
  61. private:
  62. friend class PersistentSampleMapRecords;
  63. // Gets the object holding records for a given sample-map id.
  64. PersistentSampleMapRecords* GetSampleMapRecordsWhileLocked(uint64_t id)
  65. EXCLUSIVE_LOCKS_REQUIRED(lock_);
  66. // Loads sample-map records looking for those belonging to the specified
  67. // |load_id|. Records found for other sample-maps are held for later use
  68. // without having to iterate again. This should be called only from a
  69. // PersistentSampleMapRecords object because those objects have a contract
  70. // that there are no other threads accessing the internal records_ field
  71. // of the object that is passed in.
  72. bool LoadRecords(PersistentSampleMapRecords* sample_map_records);
  73. // Weak-pointer to the allocator used by the sparse histograms.
  74. raw_ptr<PersistentMemoryAllocator> allocator_;
  75. // Iterator within the allocator for finding sample records.
  76. PersistentMemoryAllocator::Iterator record_iterator_ GUARDED_BY(lock_);
  77. // Mapping of sample-map IDs to their sample records.
  78. std::map<uint64_t, std::unique_ptr<PersistentSampleMapRecords>>
  79. sample_records_ GUARDED_BY(lock_);
  80. base::Lock lock_;
  81. };
  82. // This class manages sample-records used by a PersistentSampleMap container
  83. // that underlies a persistent SparseHistogram object. It is broken out into a
  84. // top-level class so that it can be forward-declared in other header files
  85. // rather than include this entire file as would be necessary if it were
  86. // declared within the PersistentSparseHistogramDataManager class above.
  87. class BASE_EXPORT PersistentSampleMapRecords {
  88. public:
  89. // Constructs an instance of this class. The manager object must live longer
  90. // than all instances of this class that reference it, which is not usually
  91. // a problem since these objects are generally managed from within that
  92. // manager instance.
  93. PersistentSampleMapRecords(PersistentSparseHistogramDataManager* data_manager,
  94. uint64_t sample_map_id);
  95. PersistentSampleMapRecords(const PersistentSampleMapRecords&) = delete;
  96. PersistentSampleMapRecords& operator=(const PersistentSampleMapRecords&) =
  97. delete;
  98. ~PersistentSampleMapRecords();
  99. // Resets the internal state for a new object using this data. The return
  100. // value is "this" as a convenience.
  101. PersistentSampleMapRecords* Acquire(const void* user);
  102. // Indicates that the using object is done with this data.
  103. void Release(const void* user);
  104. // Gets the next reference to a persistent sample-map record. The type and
  105. // layout of the data being referenced is defined entirely within the
  106. // PersistentSampleMap class.
  107. PersistentMemoryAllocator::Reference GetNext();
  108. // Creates a new persistent sample-map record for sample |value| and returns
  109. // a reference to it.
  110. PersistentMemoryAllocator::Reference CreateNew(HistogramBase::Sample value);
  111. // Convenience method that gets the object for a given reference so callers
  112. // don't have to also keep their own pointer to the appropriate allocator.
  113. // This is expected to be used with the SampleRecord structure defined inside
  114. // the persistent_sample_map.cc file but since that isn't exported (for
  115. // cleanliness of the interface), a template is defined that will be
  116. // resolved when used inside that file.
  117. template <typename T>
  118. T* GetAsObject(PersistentMemoryAllocator::Reference ref) {
  119. return data_manager_->GetAsObject<T>(ref);
  120. }
  121. private:
  122. friend PersistentSparseHistogramDataManager;
  123. // Weak-pointer to the parent data-manager object.
  124. raw_ptr<PersistentSparseHistogramDataManager> data_manager_;
  125. // ID of PersistentSampleMap to which these records apply.
  126. const uint64_t sample_map_id_;
  127. // The current user of this set of records. It is used to ensure that no
  128. // more than one object is using these records at a given time.
  129. raw_ptr<const void> user_ = nullptr;
  130. // This is the count of how many "records" have already been read by the
  131. // owning sample-map.
  132. size_t seen_ = 0;
  133. // This is the set of records previously found for a sample map. Because
  134. // there is ever only one object with a given ID (typically a hash of a
  135. // histogram name) and because the parent SparseHistogram has acquired
  136. // its own lock before accessing the PersistentSampleMap it controls, this
  137. // list can be accessed without acquiring any additional lock.
  138. std::vector<PersistentMemoryAllocator::Reference> records_;
  139. // This is the set of records found during iteration through memory. It
  140. // is appended in bulk to "records". Access to this vector can be done
  141. // only while holding the parent manager's lock.
  142. std::vector<PersistentMemoryAllocator::Reference> found_;
  143. };
  144. // This class manages histograms created within a PersistentMemoryAllocator.
  145. class BASE_EXPORT PersistentHistogramAllocator {
  146. public:
  147. // A reference to a histogram. While this is implemented as PMA::Reference,
  148. // it is not conceptually the same thing. Outside callers should always use
  149. // a Reference matching the class it is for and not mix the two.
  150. using Reference = PersistentMemoryAllocator::Reference;
  151. // Iterator used for fetching persistent histograms from an allocator.
  152. // It is lock-free and thread-safe.
  153. // See PersistentMemoryAllocator::Iterator for more information.
  154. class BASE_EXPORT Iterator {
  155. public:
  156. // Constructs an iterator on a given |allocator|, starting at the beginning.
  157. // The allocator must live beyond the lifetime of the iterator.
  158. explicit Iterator(PersistentHistogramAllocator* allocator);
  159. Iterator(const Iterator&) = delete;
  160. Iterator& operator=(const Iterator&) = delete;
  161. // Gets the next histogram from persistent memory; returns null if there
  162. // are no more histograms to be found. This may still be called again
  163. // later to retrieve any new histograms added in the meantime.
  164. std::unique_ptr<HistogramBase> GetNext() { return GetNextWithIgnore(0); }
  165. // Gets the next histogram from persistent memory, ignoring one particular
  166. // reference in the process. Pass |ignore| of zero (0) to ignore nothing.
  167. std::unique_ptr<HistogramBase> GetNextWithIgnore(Reference ignore);
  168. private:
  169. // Weak-pointer to histogram allocator being iterated over.
  170. raw_ptr<PersistentHistogramAllocator> allocator_;
  171. // The iterator used for stepping through objects in persistent memory.
  172. // It is lock-free and thread-safe which is why this class is also such.
  173. PersistentMemoryAllocator::Iterator memory_iter_;
  174. };
  175. // A PersistentHistogramAllocator is constructed from a PersistentMemory-
  176. // Allocator object of which it takes ownership.
  177. explicit PersistentHistogramAllocator(
  178. std::unique_ptr<PersistentMemoryAllocator> memory);
  179. PersistentHistogramAllocator(const PersistentHistogramAllocator&) = delete;
  180. PersistentHistogramAllocator& operator=(const PersistentHistogramAllocator&) =
  181. delete;
  182. virtual ~PersistentHistogramAllocator();
  183. // Direct access to underlying memory allocator. If the segment is shared
  184. // across threads or processes, reading data through these values does
  185. // not guarantee consistency. Use with care. Do not write.
  186. PersistentMemoryAllocator* memory_allocator() {
  187. return memory_allocator_.get();
  188. }
  189. // Implement the "metadata" API of a PersistentMemoryAllocator, forwarding
  190. // those requests to the real one.
  191. uint64_t Id() const { return memory_allocator_->Id(); }
  192. const char* Name() const { return memory_allocator_->Name(); }
  193. const void* data() const { return memory_allocator_->data(); }
  194. size_t length() const { return memory_allocator_->length(); }
  195. size_t size() const { return memory_allocator_->size(); }
  196. size_t used() const { return memory_allocator_->used(); }
  197. // Recreate a Histogram from data held in persistent memory. Though this
  198. // object will be local to the current process, the sample data will be
  199. // shared with all other threads referencing it. This method takes a |ref|
  200. // to where the top-level histogram data may be found in this allocator.
  201. // This method will return null if any problem is detected with the data.
  202. std::unique_ptr<HistogramBase> GetHistogram(Reference ref);
  203. // Allocate a new persistent histogram. The returned histogram will not
  204. // be able to be located by other allocators until it is "finalized".
  205. std::unique_ptr<HistogramBase> AllocateHistogram(
  206. HistogramType histogram_type,
  207. const std::string& name,
  208. int minimum,
  209. int maximum,
  210. const BucketRanges* bucket_ranges,
  211. int32_t flags,
  212. Reference* ref_ptr);
  213. // Finalize the creation of the histogram, making it available to other
  214. // processes if |registered| (as in: added to the StatisticsRecorder) is
  215. // True, forgetting it otherwise.
  216. void FinalizeHistogram(Reference ref, bool registered);
  217. // Merges the data in a persistent histogram with one held globally by the
  218. // StatisticsRecorder, updating the "logged" samples within the passed
  219. // object so that repeated merges are allowed. Don't call this on a "global"
  220. // allocator because histograms created there will already be in the SR.
  221. void MergeHistogramDeltaToStatisticsRecorder(HistogramBase* histogram);
  222. // As above but merge the "final" delta. No update of "logged" samples is
  223. // done which means it can operate on read-only objects. It's essential,
  224. // however, not to call this more than once or those final samples will
  225. // get recorded again.
  226. void MergeHistogramFinalDeltaToStatisticsRecorder(
  227. const HistogramBase* histogram);
  228. // Returns the object that manages the persistent-sample-map records for a
  229. // given |id|. Only one |user| of this data is allowed at a time. This does
  230. // an automatic Acquire() on the records. The user must call Release() on
  231. // the returned object when it is finished with it. Ownership stays with
  232. // this allocator.
  233. PersistentSampleMapRecords* UseSampleMapRecords(uint64_t id,
  234. const void* user);
  235. // Create internal histograms for tracking memory use and allocation sizes
  236. // for allocator of |name| (which can simply be the result of Name()). This
  237. // is done seperately from construction for situations such as when the
  238. // histograms will be backed by memory provided by this very allocator.
  239. //
  240. // IMPORTANT: tools/metrics/histograms/metadata/uma/histograms.xml must
  241. // be updated with the following histograms for each |name| param:
  242. // UMA.PersistentAllocator.name.Errors
  243. // UMA.PersistentAllocator.name.UsedPct
  244. void CreateTrackingHistograms(StringPiece name);
  245. void UpdateTrackingHistograms();
  246. // Sets the internal |ranges_manager_|, which will be used by the allocator to
  247. // register BucketRanges. Takes ownership of the passed |ranges_manager|.
  248. void SetRangesManager(RangesManager* ranges_manager);
  249. // Clears the internal |last_created_| reference so testing can validate
  250. // operation without that optimization.
  251. void ClearLastCreatedReferenceForTesting();
  252. protected:
  253. // The structure used to hold histogram data in persistent memory. It is
  254. // defined and used entirely within the .cc file.
  255. struct PersistentHistogramData;
  256. // Gets the reference of the last histogram created, used to avoid
  257. // trying to import what was just created.
  258. Reference last_created() {
  259. return last_created_.load(std::memory_order_relaxed);
  260. }
  261. // Gets the next histogram in persistent data based on iterator while
  262. // ignoring a particular reference if it is found.
  263. std::unique_ptr<HistogramBase> GetNextHistogramWithIgnore(Iterator* iter,
  264. Reference ignore);
  265. private:
  266. // Create a histogram based on saved (persistent) information about it.
  267. std::unique_ptr<HistogramBase> CreateHistogram(
  268. PersistentHistogramData* histogram_data_ptr);
  269. // Gets or creates an object in the global StatisticsRecorder matching
  270. // the |histogram| passed. Null is returned if one was not found and
  271. // one could not be created.
  272. HistogramBase* GetOrCreateStatisticsRecorderHistogram(
  273. const HistogramBase* histogram);
  274. // The memory allocator that provides the actual histogram storage.
  275. std::unique_ptr<PersistentMemoryAllocator> memory_allocator_;
  276. // The RangesManager that the allocator will register its BucketRanges with.
  277. // If this is null (default), the BucketRanges will be registered with the
  278. // global statistics recorder. Used when loading self-contained metrics coming
  279. // from a previous session. Registering the BucketRanges with the global
  280. // statistics recorder could create unnecessary contention, and a low amount
  281. // of extra memory.
  282. std::unique_ptr<base::RangesManager> ranges_manager_;
  283. // The data-manager used to improve performance of sparse histograms.
  284. PersistentSparseHistogramDataManager sparse_histogram_data_manager_;
  285. // A reference to the last-created histogram in the allocator, used to avoid
  286. // trying to import what was just created.
  287. std::atomic<Reference> last_created_ = 0;
  288. };
  289. // A special case of the PersistentHistogramAllocator that operates on a
  290. // global scale, collecting histograms created through standard macros and
  291. // the FactoryGet() method.
  292. class BASE_EXPORT GlobalHistogramAllocator
  293. : public PersistentHistogramAllocator {
  294. public:
  295. GlobalHistogramAllocator(const GlobalHistogramAllocator&) = delete;
  296. GlobalHistogramAllocator& operator=(const GlobalHistogramAllocator&) = delete;
  297. ~GlobalHistogramAllocator() override;
  298. // Create a global allocator using the passed-in memory |base|, |size|, and
  299. // other parameters. Ownership of the memory segment remains with the caller.
  300. static void CreateWithPersistentMemory(void* base,
  301. size_t size,
  302. size_t page_size,
  303. uint64_t id,
  304. StringPiece name);
  305. // Create a global allocator using an internal block of memory of the
  306. // specified |size| taken from the heap.
  307. static void CreateWithLocalMemory(size_t size, uint64_t id, StringPiece name);
  308. #if !BUILDFLAG(IS_NACL)
  309. // Create a global allocator by memory-mapping a |file|. If the file does
  310. // not exist, it will be created with the specified |size|. If the file does
  311. // exist, the allocator will use and add to its contents, ignoring the passed
  312. // size in favor of the existing size. Returns whether the global allocator
  313. // was set. If |exclusive_write| is true, the file will be opened in a mode
  314. // that disallows multiple concurrent writers (no effect on non-Windows).
  315. static bool CreateWithFile(const FilePath& file_path,
  316. size_t size,
  317. uint64_t id,
  318. StringPiece name,
  319. bool exclusive_write = false);
  320. // Creates a new file at |active_path|. If it already exists, it will first be
  321. // moved to |base_path|. In all cases, any old file at |base_path| will be
  322. // removed. If |spare_path| is non-empty and exists, that will be renamed and
  323. // used as the active file. Otherwise, the file will be created using the
  324. // given size, id, and name. Returns whether the global allocator was set.
  325. static bool CreateWithActiveFile(const FilePath& base_path,
  326. const FilePath& active_path,
  327. const FilePath& spare_path,
  328. size_t size,
  329. uint64_t id,
  330. StringPiece name);
  331. // Uses ConstructBaseActivePairFilePaths() to build a pair of file names which
  332. // are then used for CreateWithActiveFile(). |name| is used for both the
  333. // internal name for the allocator and also for the name of the file inside
  334. // |dir|.
  335. static bool CreateWithActiveFileInDir(const FilePath& dir,
  336. size_t size,
  337. uint64_t id,
  338. StringPiece name);
  339. // Constructs a filename using a name.
  340. static FilePath ConstructFilePath(const FilePath& dir, StringPiece name);
  341. // Constructs a filename using a name for an "active" file.
  342. static FilePath ConstructFilePathForActiveFile(const FilePath& dir,
  343. StringPiece name);
  344. // Like above but with timestamp and pid for use in upload directories.
  345. static FilePath ConstructFilePathForUploadDir(const FilePath& dir,
  346. StringPiece name,
  347. base::Time stamp,
  348. ProcessId pid);
  349. // Override that uses the current time stamp and current process id.
  350. static FilePath ConstructFilePathForUploadDir(const FilePath& dir,
  351. StringPiece name);
  352. // Parses a filename to extract name, timestamp, and pid.
  353. static bool ParseFilePath(const FilePath& path,
  354. std::string* out_name,
  355. Time* out_stamp,
  356. ProcessId* out_pid);
  357. // Create a "spare" file that can later be made the "active" file. This
  358. // should be done on a background thread if possible.
  359. static bool CreateSpareFile(const FilePath& spare_path, size_t size);
  360. #endif
  361. // Create a global allocator using a block of shared memory accessed
  362. // through the given |region|. The allocator maps the shared memory into
  363. // current process's virtual address space and frees it upon destruction.
  364. // The memory will continue to live if other processes have access to it.
  365. static void CreateWithSharedMemoryRegion(
  366. const WritableSharedMemoryRegion& region);
  367. // Sets a GlobalHistogramAllocator for globally storing histograms in
  368. // a space that can be persisted or shared between processes. There is only
  369. // ever one allocator for all such histograms created by a single process.
  370. // This takes ownership of the object and should be called as soon as
  371. // possible during startup to capture as many histograms as possible and
  372. // while operating single-threaded so there are no race-conditions.
  373. static void Set(std::unique_ptr<GlobalHistogramAllocator> allocator);
  374. // Gets a pointer to the global histogram allocator. Returns null if none
  375. // exists.
  376. static GlobalHistogramAllocator* Get();
  377. // This access to the persistent allocator is only for testing; it extracts
  378. // the current allocator completely. This allows easy creation of histograms
  379. // within persistent memory segments which can then be extracted and used in
  380. // other ways.
  381. static std::unique_ptr<GlobalHistogramAllocator> ReleaseForTesting();
  382. // Stores a pathname to which the contents of this allocator should be saved
  383. // in order to persist the data for a later use.
  384. void SetPersistentLocation(const FilePath& location);
  385. // Retrieves a previously set pathname to which the contents of this allocator
  386. // are to be saved.
  387. const FilePath& GetPersistentLocation() const;
  388. // Writes the internal data to a previously set location. This is generally
  389. // called when a process is exiting from a section of code that may not know
  390. // the filesystem. The data is written in an atomic manner. The return value
  391. // indicates success.
  392. bool WriteToPersistentLocation();
  393. // If there is a global metrics file being updated on disk, mark it to be
  394. // deleted when the process exits.
  395. void DeletePersistentLocation();
  396. private:
  397. friend class StatisticsRecorder;
  398. // Creates a new global histogram allocator.
  399. explicit GlobalHistogramAllocator(
  400. std::unique_ptr<PersistentMemoryAllocator> memory);
  401. // Import new histograms from the global histogram allocator. It's possible
  402. // for other processes to create histograms in the active memory segment;
  403. // this adds those to the internal list of known histograms to avoid creating
  404. // duplicates that would have to be merged during reporting. Every call to
  405. // this method resumes from the last entry it saw; it costs nothing if
  406. // nothing new has been added.
  407. void ImportHistogramsToStatisticsRecorder();
  408. // Builds a FilePath for a metrics file.
  409. static FilePath MakeMetricsFilePath(const FilePath& dir, StringPiece name);
  410. // Import always continues from where it left off, making use of a single
  411. // iterator to continue the work.
  412. Iterator import_iterator_;
  413. // The location to which the data should be persisted.
  414. FilePath persistent_location_;
  415. };
  416. } // namespace base
  417. #endif // BASE_METRICS_PERSISTENT_HISTOGRAM_ALLOCATOR_H__