optimization_guide_store.h 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462
  1. // Copyright 2019 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef COMPONENTS_OPTIMIZATION_GUIDE_CORE_OPTIMIZATION_GUIDE_STORE_H_
  5. #define COMPONENTS_OPTIMIZATION_GUIDE_CORE_OPTIMIZATION_GUIDE_STORE_H_
  6. #include <map>
  7. #include <string>
  8. #include "base/callback.h"
  9. #include "base/containers/flat_set.h"
  10. #include "base/memory/scoped_refptr.h"
  11. #include "base/memory/weak_ptr.h"
  12. #include "base/sequence_checker.h"
  13. #include "base/time/time.h"
  14. #include "base/version.h"
  15. #include "components/leveldb_proto/public/proto_database.h"
  16. #include "components/leveldb_proto/public/proto_database_provider.h"
  17. #include "components/optimization_guide/core/memory_hint.h"
  18. #include "components/optimization_guide/core/store_update_data.h"
  19. #include "components/optimization_guide/proto/models.pb.h"
  20. #include "third_party/abseil-cpp/absl/types/optional.h"
  21. class PrefService;
  22. namespace base {
  23. class SequencedTaskRunner;
  24. } // namespace base
  25. namespace optimization_guide {
  26. namespace proto {
  27. class StoreEntry;
  28. } // namespace proto
  29. // The HintCache backing store, which is responsible for storing all hints that
  30. // are locally available. While the HintCache itself may retain some hints in a
  31. // memory cache, all of its hints are initially loaded asynchronously by the
  32. // store. All calls to this store must be made from the same thread.
  33. class OptimizationGuideStore {
  34. public:
  35. using HintLoadedCallback =
  36. base::OnceCallback<void(const std::string&, std::unique_ptr<MemoryHint>)>;
  37. using PredictionModelLoadedCallback =
  38. base::OnceCallback<void(std::unique_ptr<proto::PredictionModel>)>;
  39. using EntryKey = std::string;
  40. using StoreEntryProtoDatabase =
  41. leveldb_proto::ProtoDatabase<proto::StoreEntry>;
  42. // Status of the store. The store begins in kUninitialized, transitions to
  43. // kInitializing after Initialize() is called, and transitions to kAvailable
  44. // if initialization successfully completes. In the case where anything fails,
  45. // the store transitions to kFailed, at which point it is fully purged and
  46. // becomes unusable.
  47. //
  48. // Keep in sync with OptimizationGuideHintCacheLevelDBStoreStatus in
  49. // tools/metrics/histograms/enums.xml.
  50. enum class Status {
  51. kUninitialized = 0,
  52. kInitializing = 1,
  53. kAvailable = 2,
  54. kFailed = 3,
  55. kMaxValue = kFailed,
  56. };
  57. // Store entry types within the store appear at the start of the keys of
  58. // entries. These values are converted into strings within the key: a key
  59. // starting with "1_" signifies a metadata entry and one starting with "2_"
  60. // signifies a component hint entry. Adding this to the start of the key
  61. // allows the store to quickly perform operations on all entries of a specific
  62. // key type. Given that store entry type comparisons may be performed many
  63. // times, the entry type string is kept as small as possible.
  64. // Example metadata entry type key:
  65. // "[StoreEntryType::kMetadata]_[MetadataType::kSchema]" ==> "1_1"
  66. // Example component hint store entry type key:
  67. // "[StoreEntryType::kComponentHint]_[component_version]_[host]"
  68. // ==> "2_55_foo.com"
  69. // NOTE: The order and value of the existing store entry types within the enum
  70. // cannot be changed, but new types can be added to the end.
  71. // StoreEntryType should remain synchronized with the
  72. // HintCacheStoreEntryType in enums.xml.
  73. enum class StoreEntryType {
  74. kEmpty = 0,
  75. kMetadata = 1,
  76. kComponentHint = 2,
  77. kFetchedHint = 3,
  78. kPredictionModel = 4,
  79. kDeprecatedHostModelFeatures = 5, // deprecated.
  80. kMaxValue = kDeprecatedHostModelFeatures,
  81. };
  82. OptimizationGuideStore(
  83. leveldb_proto::ProtoDatabaseProvider* database_provider,
  84. const base::FilePath& database_dir,
  85. scoped_refptr<base::SequencedTaskRunner> store_task_runner,
  86. PrefService* pref_service);
  87. // For tests only.
  88. explicit OptimizationGuideStore(
  89. std::unique_ptr<StoreEntryProtoDatabase> database,
  90. scoped_refptr<base::SequencedTaskRunner> store_task_runner,
  91. PrefService* pref_service);
  92. OptimizationGuideStore(const OptimizationGuideStore&) = delete;
  93. OptimizationGuideStore& operator=(const OptimizationGuideStore&) = delete;
  94. virtual ~OptimizationGuideStore();
  95. // Initializes the store. If |purge_existing_data| is set to true,
  96. // then the cache is purged during initialization and starts in a fresh state.
  97. // When initialization completes, the provided callback is run asynchronously.
  98. // Virtualized for testing.
  99. virtual void Initialize(bool purge_existing_data, base::OnceClosure callback);
  100. // Creates and returns a StoreUpdateData object for component hints. This
  101. // object is used to collect hints within a component in a format usable on a
  102. // background thread and is later returned to the store in
  103. // UpdateComponentHints(). The StoreUpdateData object is only created when the
  104. // provided component version is newer than the store's version, indicating
  105. // fresh hints. If the component's version is not newer than the store's
  106. // version, then no StoreUpdateData is created and nullptr is returned. This
  107. // prevents unnecessary processing of the component's hints by the caller.
  108. std::unique_ptr<StoreUpdateData> MaybeCreateUpdateDataForComponentHints(
  109. const base::Version& version) const;
  110. // Creates and returns a HintsUpdateData object for Fetched Hints.
  111. // This object is used to collect a batch of hints in a format that is usable
  112. // to update the store on a background thread. This is always created when
  113. // hints have been successfully fetched from the remote Optimization Guide
  114. // Service so the store can expire old hints, remove hints specified by the
  115. // server, and store the fresh hints.
  116. std::unique_ptr<StoreUpdateData> CreateUpdateDataForFetchedHints(
  117. base::Time update_time) const;
  118. // Updates the component hints and version contained within the store. When
  119. // this is called, all pre-existing component hints within the store is purged
  120. // and only the new hints are retained. After the store is fully updated with
  121. // the new component hints, the callback is run asynchronously.
  122. void UpdateComponentHints(std::unique_ptr<StoreUpdateData> component_data,
  123. base::OnceClosure callback);
  124. // Updates the fetched hints contained in the store, including the
  125. // metadata entry. The callback is run asynchronously after the database
  126. // stores the hints.
  127. void UpdateFetchedHints(std::unique_ptr<StoreUpdateData> fetched_hints_data,
  128. base::OnceClosure callback);
  129. // Removes fetched hint store entries from |this|. |entry_keys_| is
  130. // updated after the fetched hint entries are removed.
  131. void ClearFetchedHintsFromDatabase();
  132. // Finds the most specific hint entry key for the specified host. Returns
  133. // true if a hint entry key is found, in which case |out_hint_entry_key| is
  134. // populated with the key. All keys for kFetched hints are considered before
  135. // kComponent hints as they are updated more frequently.
  136. bool FindHintEntryKey(const std::string& host,
  137. EntryKey* out_hint_entry_key) const;
  138. // Loads the hint specified by |hint_entry_key|.
  139. // After the load finishes, the hint data is passed to |callback|. In the case
  140. // where the hint cannot be loaded, the callback is run with a nullptr.
  141. // Depending on the load result, the callback may be synchronous or
  142. // asynchronous.
  143. void LoadHint(const EntryKey& hint_entry_key, HintLoadedCallback callback);
  144. // Returns the time that the fetched hints in the store can be updated. If
  145. // |this| is not available, base::Time() is returned.
  146. base::Time GetFetchedHintsUpdateTime() const;
  147. // Removes all fetched hints that have expired from the store.
  148. // |entry_keys_| is updated after the expired fetched hints are
  149. // removed.
  150. void PurgeExpiredFetchedHints();
  151. // Removes all models that have not been loaded in the max inactive duration
  152. // configured. |entry_keys| is updated after the inactive models are removed.
  153. // Respects models' |keep_beyond_valid_duration| setting.
  154. void PurgeInactiveModels();
  155. // Creates and returns a StoreUpdateData object for Prediction Models. This
  156. // object is used to collect a batch of prediction models in a format that is
  157. // usable to update the store on a background thread. This is always created
  158. // when prediction models have been successfully fetched from the remote
  159. // Optimization Guide Service so the store can update old prediction models.
  160. std::unique_ptr<StoreUpdateData> CreateUpdateDataForPredictionModels(
  161. base::Time expiry_time) const;
  162. // Updates the prediction models contained in the store. The callback is run
  163. // asynchronously after the database stores the prediction models. Virtualized
  164. // for testing.
  165. virtual void UpdatePredictionModels(
  166. std::unique_ptr<StoreUpdateData> prediction_models_update_data,
  167. base::OnceClosure callback);
  168. // Finds the entry key for the prediction model if it is still valid in the
  169. // store. Returns true if an entry key is valid and
  170. // |out_prediction_model_entry_key| is populated with any matching key.
  171. // Virtualized for testing.
  172. virtual bool FindPredictionModelEntryKey(
  173. proto::OptimizationTarget optimization_target,
  174. OptimizationGuideStore::EntryKey* out_prediction_model_entry_key);
  175. // Loads the prediction model specified by |prediction_model_entry_key|. After
  176. // the load finishes, the prediction model data is passed to |callback|. In
  177. // the case where the prediction model cannot be loaded, the callback is run
  178. // with a nullptr. Depending on the load result, the callback may be
  179. // synchronous or asynchronous.
  180. // Virtualized for testing.
  181. virtual void LoadPredictionModel(const EntryKey& prediction_model_entry_key,
  182. PredictionModelLoadedCallback callback);
  183. // Removes the prediction model specified by |entry_key| if it exists. Returns
  184. // true if |entry_key| is found and the remove operation is initiated, and
  185. // false otherwise.
  186. bool RemovePredictionModelFromEntryKey(const EntryKey& entry_key);
  187. // Removes fetched hints whose keys are in |hint_keys| and runs |on_success|
  188. // if successful, otherwise the callback is not run.
  189. void RemoveFetchedHintsByKey(base::OnceClosure on_success,
  190. const base::flat_set<std::string>& hint_keys);
  191. // Returns true if the current status is Status::kAvailable.
  192. bool IsAvailable() const;
  193. // Returns the weak ptr of |this|.
  194. base::WeakPtr<OptimizationGuideStore> AsWeakPtr() {
  195. return weak_ptr_factory_.GetWeakPtr();
  196. }
  197. private:
  198. friend class OptimizationGuideStoreTest;
  199. friend class StoreUpdateData;
  200. friend class TestOptimizationGuideStore;
  201. using EntryKeyPrefix = std::string;
  202. using EntryKeySet = base::flat_set<EntryKey>;
  203. using EntryVector =
  204. leveldb_proto::ProtoDatabase<proto::StoreEntry>::KeyEntryVector;
  205. using EntryMap = std::map<EntryKey, proto::StoreEntry>;
  206. // Metadata types within the store. The metadata type appears at the end of
  207. // metadata entry keys. These values are converted into strings within the
  208. // key.
  209. // Example metadata type keys:
  210. // "[StoreEntryType::kMetadata]_[MetadataType::kSchema]" ==> "1_1"
  211. // "[StoreEntryType::kMetadata]_[MetadataType::kComponent]" ==> "1_2"
  212. // NOTE: The order and value of the existing metadata types within the enum
  213. // cannot be changed, but new types can be added to the end.
  214. enum class MetadataType {
  215. kSchema = 1,
  216. kComponent = 2,
  217. kFetched = 3,
  218. kDeprecatedHostModelFeatures = 4, // deprecated.
  219. };
  220. // Current schema version of the hint cache store. When this is changed,
  221. // pre-existing store data from an earlier version is purged.
  222. static const char kStoreSchemaVersion[];
  223. // Returns prefix in the key of every metadata entry type entry: "1_"
  224. static EntryKeyPrefix GetMetadataEntryKeyPrefix();
  225. // Returns entry key for the specified metadata type entry: "1_[MetadataType]"
  226. static EntryKey GetMetadataTypeEntryKey(MetadataType metadata_type);
  227. // Returns prefix in the key of every component hint entry: "2_"
  228. static EntryKeyPrefix GetComponentHintEntryKeyPrefixWithoutVersion();
  229. // Returns prefix in the key of component hint entries for the specified
  230. // component version: "2_[component_version]_"
  231. static EntryKeyPrefix GetComponentHintEntryKeyPrefix(
  232. const base::Version& component_version);
  233. // Returns prefix of the key of every fetched hint entry: "3_".
  234. static EntryKeyPrefix GetFetchedHintEntryKeyPrefix();
  235. // Returns prefix of the key of every prediction model entry: "4_".
  236. static EntryKeyPrefix GetPredictionModelEntryKeyPrefix();
  237. // Returns the OptimizationTarget from |prediction_model_entry_key|.
  238. static proto::OptimizationTarget
  239. GetOptimizationTargetFromPredictionModelEntryKey(
  240. const EntryKey& prediction_model_entry_key);
  241. // Updates the status of the store to the specified value, validates the
  242. // transition, and destroys the database in the case where the status
  243. // transitions to Status::kFailed.
  244. void UpdateStatus(Status new_status);
  245. // Asynchronously purges all existing entries from the database and runs the
  246. // callback after it completes. This should only be run during initialization.
  247. void PurgeDatabase(base::OnceClosure callback);
  248. // Updates |component_version_| and |component_hint_entry_key_prefix_| for
  249. // the new component version. This must be called rather than directly
  250. // modifying |component_version_|, as it ensures that both member variables
  251. // are updated in sync.
  252. void SetComponentVersion(const base::Version& component_version);
  253. // Resets |component_version_| and |component_hint_entry_key_prefix_| back to
  254. // their default state. Called after the database is destroyed.
  255. void ClearComponentVersion();
  256. // Asynchronously loads the entry keys from the store, populates |entry_keys_|
  257. // with them, and runs the provided callback after they finish loading. In the
  258. // case where there is currently an in-flight update, this does nothing, as
  259. // the entry keys will be loaded after the update completes.
  260. void MaybeLoadEntryKeys(base::OnceClosure callback);
  261. // Returns the total entry keys contained within the store.
  262. size_t GetEntryKeyCount() const;
  263. // Finds the most specific host suffix of the host name that the store has an
  264. // entry with the provided prefix, |entry_key_prefix|. |out_entry_key| is
  265. // populated with the entry key for the corresponding hint. Returns true if
  266. // an entry was successfully found.
  267. bool FindEntryKeyForHostWithPrefix(
  268. const std::string& host,
  269. EntryKey* out_entry_key,
  270. const EntryKeyPrefix& entry_key_prefix) const;
  271. // Callback that identifies any expired |entries| and
  272. // asynchronously removes them from the store.
  273. void OnLoadEntriesToPurgeExpired(bool success,
  274. std::unique_ptr<EntryMap> entries);
  275. // Callback that runs after the database finishes being initialized. If
  276. // |purge_existing_data| is true, then unconditionally purges the database;
  277. // otherwise, triggers loading of the metadata.
  278. void OnDatabaseInitialized(bool purge_existing_data,
  279. base::OnceClosure callback,
  280. leveldb_proto::Enums::InitStatus status);
  281. // Callback that is run after the database finishes being destroyed.
  282. void OnDatabaseDestroyed(bool success);
  283. // Callback that runs after the metadata finishes being loaded. This
  284. // validates the schema version, sets the component version, and either purges
  285. // the store (on a schema version mismatch) or loads all hint entry keys (on
  286. // a schema version match).
  287. void OnLoadMetadata(base::OnceClosure callback,
  288. bool success,
  289. std::unique_ptr<EntryMap> metadata_entries);
  290. // Callback that runs after the database is purged during initialization.
  291. void OnPurgeDatabase(base::OnceClosure callback, bool success);
  292. // Callback that runs after the data within the store is fully
  293. // updated. If the update was successful, it attempts to load all of the
  294. // entry keys contained within the database.
  295. void OnUpdateStore(base::OnceClosure callback, bool success);
  296. // Callback that runs after |keys| have been removed from the store. If
  297. // |success|, |on_success| is run. Note that |on_success| is not guaranteed to
  298. // run and that the calling code must be able to handle the callback not
  299. // coming back.
  300. void OnFetchedEntriesRemoved(base::OnceClosure on_success,
  301. const EntryKeySet& keys,
  302. bool success);
  303. // Callback that runs after the hint entry keys are fully loaded. If there's
  304. // currently an in-flight component update, then the hint entry keys will be
  305. // loaded again after the component update completes, so the results are
  306. // tossed; otherwise, |entry_keys| is moved into |entry_keys_|. Regardless of
  307. // the outcome of loading the keys, the callback always runs.
  308. void OnLoadEntryKeys(std::unique_ptr<EntryKeySet> entry_keys,
  309. base::OnceClosure callback,
  310. bool success,
  311. std::unique_ptr<EntryMap> unused);
  312. // Callback that runs after a hint entry is loaded from the database. If
  313. // there's currently an in-flight component update, then the hint is about to
  314. // be invalidated, so results are tossed; otherwise, the hint is released into
  315. // the callback, allowing the caller to own the hint without copying it.
  316. // Regardless of the success or failure of retrieving the key, the callback
  317. // always runs (it simply runs with a nullptr on failure).
  318. void OnLoadHint(const EntryKey& entry_key,
  319. HintLoadedCallback callback,
  320. bool success,
  321. std::unique_ptr<proto::StoreEntry> entry);
  322. // Callback that runs after a prediction model entry is loaded from the
  323. // database. If there's currently an in-flight update, then the data could be
  324. // invalidated, so loaded model is discarded. Otherwise, the prediction model
  325. // is released into the callback, allowing the caller to own the prediction
  326. // model without copying it. Regardless of the success or failure of
  327. // retrieving the key, the callback always runs (it simply runs with a nullptr
  328. // on failure).
  329. void OnLoadPredictionModel(PredictionModelLoadedCallback callback,
  330. bool success,
  331. std::unique_ptr<proto::StoreEntry> entry);
  332. // Callback that runs when the prediction models that need to be updated and
  333. // removed are loaded from the database. This will remove the files associated
  334. // with those models and run the update routine with |update_vector| and
  335. // |remove_vector| after that.
  336. void OnLoadModelsToBeUpdated(
  337. std::unique_ptr<EntryVector> update_vector,
  338. std::unique_ptr<leveldb_proto::KeyVector> remove_vector,
  339. base::OnceClosure callback,
  340. bool success,
  341. std::unique_ptr<EntryMap> entries);
  342. // Callback that runs after the download URL in |loaded_model| has been
  343. // verified. If |success| is false, the associated entry from |database_| will
  344. // be removed and |callback| will run as if the model is not loaded.
  345. void OnModelFilePathVerified(
  346. std::unique_ptr<proto::PredictionModel> loaded_model,
  347. PredictionModelLoadedCallback callback,
  348. bool success);
  349. // Clean up file paths that were slated for deletion in previous sessions.
  350. void CleanUpFilePaths();
  351. // Callback invoked when |deleted_file_path| completed its attempt to be
  352. // deleted. Will clean up the path if |success| is true.
  353. void OnFilePathDeleted(const std::string& deleted_file_path, bool success);
  354. // Proto database used by the store.
  355. std::unique_ptr<StoreEntryProtoDatabase> database_;
  356. // The current status of the store. It should only be updated through
  357. // UpdateStatus(), which validates status transitions and triggers
  358. // accompanying logic.
  359. Status status_ = Status::kUninitialized;
  360. // The current component version of the store. This should only be updated
  361. // via SetComponentVersion(), which ensures that both |component_version_|
  362. // and |component_hint_key_prefix_| are updated at the same time.
  363. absl::optional<base::Version> component_version_;
  364. // The current entry key prefix shared by all component hints containd within
  365. // the store. While this could be generated on the fly using
  366. // |component_version_|, it is retaind separately as an optimization, as it
  367. // is needed often.
  368. EntryKeyPrefix component_hint_entry_key_prefix_;
  369. // The next update time for the fetched hints that are currently in the
  370. // store.
  371. base::Time fetched_update_time_;
  372. // The next update time for the host model features that are currently in the
  373. // store.
  374. base::Time host_model_features_update_time_;
  375. // The keys of the entries available within the store.
  376. std::unique_ptr<EntryKeySet> entry_keys_;
  377. // The background task runner used to perform operations on the store.
  378. scoped_refptr<base::SequencedTaskRunner> store_task_runner_;
  379. // Pref service. Not owned. Guaranteed to outlive |this|.
  380. raw_ptr<PrefService> pref_service_;
  381. SEQUENCE_CHECKER(sequence_checker_);
  382. base::WeakPtrFactory<OptimizationGuideStore> weak_ptr_factory_{this};
  383. };
  384. } // namespace optimization_guide
  385. #endif // COMPONENTS_OPTIMIZATION_GUIDE_CORE_OPTIMIZATION_GUIDE_STORE_H_