remote_suggestions_provider_impl.h 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468
  1. // Copyright 2015 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #ifndef COMPONENTS_NTP_SNIPPETS_REMOTE_REMOTE_SUGGESTIONS_PROVIDER_IMPL_H_
  5. #define COMPONENTS_NTP_SNIPPETS_REMOTE_REMOTE_SUGGESTIONS_PROVIDER_IMPL_H_
  6. #include <cstddef>
  7. #include <map>
  8. #include <memory>
  9. #include <set>
  10. #include <string>
  11. #include <utility>
  12. #include <vector>
  13. #include "base/callback_forward.h"
  14. #include "base/containers/circular_deque.h"
  15. #include "base/gtest_prod_util.h"
  16. #include "base/memory/raw_ptr.h"
  17. #include "base/time/clock.h"
  18. #include "base/time/time.h"
  19. #include "base/timer/timer.h"
  20. #include "components/ntp_snippets/category.h"
  21. #include "components/ntp_snippets/category_status.h"
  22. #include "components/ntp_snippets/content_suggestion.h"
  23. #include "components/ntp_snippets/content_suggestions_provider.h"
  24. #include "components/ntp_snippets/remote/cached_image_fetcher.h"
  25. #include "components/ntp_snippets/remote/json_to_categories.h"
  26. #include "components/ntp_snippets/remote/remote_suggestion.h"
  27. #include "components/ntp_snippets/remote/remote_suggestions_fetcher.h"
  28. #include "components/ntp_snippets/remote/remote_suggestions_provider.h"
  29. #include "components/ntp_snippets/remote/remote_suggestions_status_service.h"
  30. #include "components/ntp_snippets/remote/request_params.h"
  31. #include "components/ntp_snippets/remote/request_throttler.h"
  32. #include "third_party/abseil-cpp/absl/types/optional.h"
  33. class PrefRegistrySimple;
  34. class PrefService;
  35. namespace image_fetcher {
  36. class ImageFetcher;
  37. } // namespace image_fetcher
  38. namespace ntp_snippets {
  39. class CategoryRanker;
  40. class RemoteSuggestionsDatabase;
  41. class RemoteSuggestionsScheduler;
  42. // Retrieves fresh content data (articles) from the server, stores them and
  43. // provides them as content suggestions.
  44. // This class is final because it does things in its constructor which make it
  45. // unsafe to derive from it.
  46. // TODO(treib): Introduce two-phase initialization and make the class not final?
  47. class RemoteSuggestionsProviderImpl final : public RemoteSuggestionsProvider {
  48. public:
  49. // |application_language_code| should be a ISO 639-1 compliant string, e.g.
  50. // 'en' or 'en-US'. Note that this code should only specify the language, not
  51. // the locale, so 'en_US' (English language with US locale) and 'en-GB_US'
  52. // (British English person in the US) are not language codes.
  53. RemoteSuggestionsProviderImpl(
  54. Observer* observer,
  55. PrefService* pref_service,
  56. const std::string& application_language_code,
  57. CategoryRanker* category_ranker,
  58. RemoteSuggestionsScheduler* scheduler,
  59. std::unique_ptr<RemoteSuggestionsFetcher> suggestions_fetcher,
  60. std::unique_ptr<image_fetcher::ImageFetcher> image_fetcher,
  61. std::unique_ptr<RemoteSuggestionsDatabase> database,
  62. std::unique_ptr<RemoteSuggestionsStatusService> status_service,
  63. std::unique_ptr<base::OneShotTimer> fetch_timeout_timer);
  64. RemoteSuggestionsProviderImpl(const RemoteSuggestionsProviderImpl&) = delete;
  65. RemoteSuggestionsProviderImpl& operator=(
  66. const RemoteSuggestionsProviderImpl&) = delete;
  67. ~RemoteSuggestionsProviderImpl() override;
  68. static void RegisterProfilePrefs(PrefRegistrySimple* registry);
  69. // Returns whether the service is successfully initialized. While this is
  70. // false, some calls may trigger DCHECKs.
  71. bool initialized() const { return ready() || state_ == State::DISABLED; }
  72. // RemoteSuggestionsProvider implementation.
  73. void RefetchInTheBackground(FetchStatusCallback callback) override;
  74. void RefetchWhileDisplaying(FetchStatusCallback callback) override;
  75. // TODO(fhorschig): Remove this getter when there is an interface for the
  76. // fetcher that allows better mocks.
  77. const RemoteSuggestionsFetcher* suggestions_fetcher_for_debugging()
  78. const override;
  79. GURL GetUrlWithFavicon(
  80. const ContentSuggestion::ID& suggestion_id) const override;
  81. bool IsDisabled() const override;
  82. bool ready() const override;
  83. // ContentSuggestionsProvider implementation.
  84. CategoryStatus GetCategoryStatus(Category category) override;
  85. CategoryInfo GetCategoryInfo(Category category) override;
  86. void DismissSuggestion(const ContentSuggestion::ID& suggestion_id) override;
  87. void FetchSuggestionImage(const ContentSuggestion::ID& suggestion_id,
  88. ImageFetchedCallback callback) override;
  89. void FetchSuggestionImageData(const ContentSuggestion::ID& suggestion_id,
  90. ImageDataFetchedCallback callback) override;
  91. void Fetch(const Category& category,
  92. const std::set<std::string>& known_suggestion_ids,
  93. FetchDoneCallback callback) override;
  94. void ReloadSuggestions() override;
  95. void ClearHistory(
  96. base::Time begin,
  97. base::Time end,
  98. const base::RepeatingCallback<bool(const GURL& url)>& filter) override;
  99. void ClearCachedSuggestions() override;
  100. void OnSignInStateChanged(bool has_signed_in) override;
  101. void GetDismissedSuggestionsForDebugging(
  102. Category category,
  103. DismissedSuggestionsCallback callback) override;
  104. void ClearDismissedSuggestionsForDebugging(Category category) override;
  105. // Returns the maximum number of suggestions we expect to receive from the
  106. // server during a normal (not fetch-more) fetch..
  107. static int GetMaxNormalFetchSuggestionCountForTesting();
  108. // Available suggestions, only for unit tests.
  109. // TODO(treib): Get rid of this. Tests should use a fake observer instead.
  110. const RemoteSuggestion::PtrVector& GetSuggestionsForTesting(
  111. Category category) const {
  112. return category_contents_.find(category)->second.suggestions;
  113. }
  114. // Dismissed suggestions, only for unit tests.
  115. const RemoteSuggestion::PtrVector& GetDismissedSuggestionsForTesting(
  116. Category category) const {
  117. return category_contents_.find(category)->second.dismissed;
  118. }
  119. // Overrides internal clock for testing purposes.
  120. void SetClockForTesting(base::Clock* clock) { clock_ = clock; }
  121. // TODO(tschumann): remove this method as soon as we inject the fetcher into
  122. // the constructor.
  123. CachedImageFetcher& GetImageFetcherForTesting() { return image_fetcher_; }
  124. private:
  125. friend class RemoteSuggestionsProviderImplTest;
  126. // TODO(jkrcal): Mock the database to trigger the error naturally (or remove
  127. // the error state and get rid of the test).
  128. FRIEND_TEST_ALL_PREFIXES(RemoteSuggestionsProviderImplTest,
  129. CallsSchedulerOnError);
  130. // TODO(jkrcal): Mock the status service and remove these friend declarations.
  131. FRIEND_TEST_ALL_PREFIXES(RemoteSuggestionsProviderImplTest,
  132. CallsSchedulerWhenDisabled);
  133. FRIEND_TEST_ALL_PREFIXES(RemoteSuggestionsProviderImplTest,
  134. DontNotifyIfNotAvailable);
  135. FRIEND_TEST_ALL_PREFIXES(RemoteSuggestionsProviderImplTest,
  136. CallsSchedulerWhenSignedIn);
  137. FRIEND_TEST_ALL_PREFIXES(RemoteSuggestionsProviderImplTest,
  138. CallsSchedulerWhenSignedOut);
  139. FRIEND_TEST_ALL_PREFIXES(RemoteSuggestionsProviderImplTest,
  140. RestartsFetchWhenSignedInWhileFetching);
  141. FRIEND_TEST_ALL_PREFIXES(RemoteSuggestionsProviderImplTest,
  142. ShouldHandleCategoryDisabledBeforeTimeout);
  143. FRIEND_TEST_ALL_PREFIXES(
  144. RemoteSuggestionsProviderImplTest,
  145. ShouldNotSetExclusiveCategoryWhenFetchingSuggestions);
  146. // Possible state transitions:
  147. // NOT_INITED --------+
  148. // / \ |
  149. // v v |
  150. // READY <--> DISABLED |
  151. // \ / |
  152. // v v |
  153. // ERROR_OCCURRED <-----+
  154. // TODO(jkrcal): Do we need to keep the distinction between states DISABLED
  155. // and ERROR_OCCURED?
  156. enum class State {
  157. // The service has just been created. Can change to states:
  158. // - DISABLED: After the database is done loading,
  159. // GetStateForDependenciesStatus can identify the next state to
  160. // be DISABLED.
  161. // - READY: if GetStateForDependenciesStatus returns it, after the database
  162. // is done loading.
  163. // - ERROR_OCCURRED: when an unrecoverable error occurred.
  164. NOT_INITED,
  165. // The service registered observers, timers, etc. and is ready to answer to
  166. // queries, fetch suggestions... Can change to states:
  167. // - DISABLED: when the global Chrome state changes, for example after
  168. // |OnStateChanged| is called and sync is disabled.
  169. // - ERROR_OCCURRED: when an unrecoverable error occurred.
  170. READY,
  171. // The service is disabled and unregistered the related resources.
  172. // Can change to states:
  173. // - READY: when the global Chrome state changes, for example after
  174. // |OnStateChanged| is called and sync is enabled.
  175. // - ERROR_OCCURRED: when an unrecoverable error occurred.
  176. DISABLED,
  177. // The service or one of its dependencies encountered an unrecoverable error
  178. // and the service can't be used anymore.
  179. ERROR_OCCURRED,
  180. COUNT
  181. };
  182. // Documents the status of the ongoing request and what action should be taken
  183. // on completion.
  184. enum class FetchRequestStatus {
  185. // There is no request in progress for remote suggestions.
  186. NONE,
  187. // There is a valid request in progress that should be treated normally on
  188. // completion.
  189. IN_PROGRESS,
  190. // There is a canceled request in progress. The response should be ignored
  191. // when it arrives.
  192. IN_PROGRESS_CANCELED,
  193. // There is an invalidated request in progress. On completion, we should
  194. // ignore the response and initiate a new fetch (with updated parameters).
  195. IN_PROGRESS_NEEDS_REFETCH
  196. };
  197. struct CategoryContent {
  198. // The current status of the category.
  199. CategoryStatus status = CategoryStatus::INITIALIZING;
  200. // The additional information about a category.
  201. CategoryInfo info;
  202. // TODO(vitaliii): Remove this field. It is always true, because we now
  203. // remove categories not included in the last fetch.
  204. // True iff the server returned results in this category in the last fetch.
  205. // We never remove categories that the server still provides.
  206. bool included_in_last_server_response = true;
  207. // All currently active suggestions (excl. the dismissed ones).
  208. RemoteSuggestion::PtrVector suggestions;
  209. // All previous suggestions that we keep around in memory because they can
  210. // be on some open NTP. We do not persist this list so that on a new start
  211. // of Chrome, this is empty.
  212. // |archived| is a FIFO buffer with a maximum length.
  213. base::circular_deque<std::unique_ptr<RemoteSuggestion>> archived;
  214. // Suggestions that the user dismissed. We keep these around until they
  215. // expire so we won't re-add them to |suggestions| on the next fetch.
  216. RemoteSuggestion::PtrVector dismissed;
  217. // Returns a non-dismissed suggestion with the given |id_within_category|,
  218. // or null if none exist.
  219. const RemoteSuggestion* FindSuggestion(
  220. const std::string& id_within_category) const;
  221. explicit CategoryContent(const CategoryInfo& info);
  222. CategoryContent(CategoryContent&&);
  223. ~CategoryContent();
  224. CategoryContent& operator=(CategoryContent&&);
  225. };
  226. // Fetches suggestions from the server and replaces old suggestions by the new
  227. // ones. Requests can be marked more important by setting
  228. // |interactive_request| to true (such request might circumvent the daily
  229. // quota for requests, etc.), useful for requests triggered by the user. After
  230. // the fetch finished, the provided |callback| will be triggered with the
  231. // status of the fetch.
  232. void FetchSuggestions(bool interactive_request, FetchStatusCallback callback);
  233. // Similar To FetchSuggestions, only adds a loading indicator on top of that.
  234. // If |enable_loading_indication_timeout| is true, the indicator is hidden if
  235. // the fetch does not finish within a certain amount of time (the fetch itself
  236. // is not canceled, though).
  237. void FetchSuggestionsWithLoadingIndicator(
  238. bool interactive_request,
  239. FetchStatusCallback callback,
  240. bool enable_loading_indication_timeout);
  241. void OnFetchSuggestionsWithLoadingIndicatorFinished(
  242. FetchStatusCallback callback,
  243. Status status);
  244. // Returns the URL of the image of a suggestion if it is among the current or
  245. // among the archived suggestions in the matching category. Returns an empty
  246. // URL otherwise.
  247. GURL FindSuggestionImageUrl(const ContentSuggestion::ID& suggestion_id) const;
  248. // Callbacks for the RemoteSuggestionsDatabase.
  249. void OnDatabaseLoaded(RemoteSuggestion::PtrVector suggestions);
  250. void OnDatabaseError();
  251. // Callback for fetch-more requests with the RemoteSuggestionsFetcher.
  252. void OnFetchMoreFinished(
  253. FetchDoneCallback fetching_callback,
  254. Status status,
  255. RemoteSuggestionsFetcher::OptionalFetchedCategories fetched_categories);
  256. // Callback for regular fetch requests with the RemoteSuggestionsFetcher.
  257. void OnFetchFinished(
  258. FetchStatusCallback callback,
  259. bool interactive_request,
  260. Status status,
  261. RemoteSuggestionsFetcher::OptionalFetchedCategories fetched_categories);
  262. // Moves all suggestions from |to_archive| into the archive of the |content|.
  263. // Clears |to_archive|. As the archive is a FIFO buffer of limited size, this
  264. // function will also delete images from the database in case the associated
  265. // suggestion gets evicted from the archive.
  266. void ArchiveSuggestions(CategoryContent* content,
  267. RemoteSuggestion::PtrVector* to_archive);
  268. // Sanitizes newly fetched suggestions -- e.g. adding missing dates and
  269. // filtering out incomplete results or dismissed suggestions (indicated by
  270. // |dismissed|).
  271. void SanitizeReceivedSuggestions(const RemoteSuggestion::PtrVector& dismissed,
  272. RemoteSuggestion::PtrVector* suggestions);
  273. // Adds newly available suggestions to |content| corresponding to |category|.
  274. void IntegrateSuggestions(Category category,
  275. CategoryContent* content,
  276. RemoteSuggestion::PtrVector new_suggestions);
  277. // Adds newly available suggestion at the top of Articles category.
  278. void PrependArticleSuggestion(
  279. std::unique_ptr<RemoteSuggestion> remote_suggestion);
  280. // Refreshes the content suggestions upon receiving a push-to-refresh request.
  281. void RefreshSuggestionsUponPushToRefreshRequest();
  282. // Dismisses a suggestion within a given category content.
  283. // Note that this modifies the suggestion datastructures of |content|
  284. // invalidating iterators.
  285. void DismissSuggestionFromCategoryContent(
  286. CategoryContent* content,
  287. const std::string& id_within_category);
  288. // Sets categories status to NOT_PROVIDED and deletes them (including their
  289. // suggestions from the database).
  290. void DeleteCategories(const std::vector<Category>& categories);
  291. // Removes expired dismissed suggestions from the service and the database.
  292. void ClearExpiredDismissedSuggestions();
  293. // Removes images from the DB that are not referenced from any known
  294. // suggestion. Needs to iterate the whole suggestion database -- so do it
  295. // often enough to keep it small but not too often as it still iterates over
  296. // the file system.
  297. void ClearOrphanedImages();
  298. // Clears suggestions because any history item has been removed.
  299. void ClearHistoryDependentState();
  300. // Clears the cached suggestions
  301. void ClearCachedSuggestionsImpl();
  302. // Clears all stored suggestions and updates the observer.
  303. void NukeAllSuggestions();
  304. // Completes the initialization phase of the service, registering the last
  305. // observers. This is done after construction, once the database is loaded.
  306. void FinishInitialization();
  307. // Triggers a state transition depending on the provided status. This method
  308. // is called when a change is detected by |status_service_|.
  309. void OnStatusChanged(RemoteSuggestionsStatus old_status,
  310. RemoteSuggestionsStatus new_status);
  311. // Verifies state transitions (see |State|'s documentation) and applies them.
  312. // Also updates the provider status. Does nothing except updating the provider
  313. // status if called with the current state.
  314. void EnterState(State state);
  315. // Notifies the state change to ProviderStatusCallback specified by
  316. // SetProviderStatusCallback().
  317. void NotifyStateChanged();
  318. // Converts the given |suggestions| to content suggestions and notifies the
  319. // observer with them for category |category|.
  320. void NotifyNewSuggestions(Category category,
  321. const RemoteSuggestion::PtrVector& suggestions);
  322. // Updates the internal status for |category| to |category_status_| and
  323. // notifies the content suggestions observer if it changed.
  324. void UpdateCategoryStatus(Category category, CategoryStatus status);
  325. // Calls UpdateCategoryStatus() for all provided categories.
  326. void UpdateAllCategoryStatus(CategoryStatus status);
  327. // Updates the category info for |category|. If a corresponding
  328. // CategoryContent object does not exist, it will be created.
  329. // Returns the existing or newly created object.
  330. CategoryContent* UpdateCategoryInfo(Category category,
  331. const CategoryInfo& info);
  332. void RestoreCategoriesFromPrefs();
  333. void StoreCategoriesToPrefs();
  334. // If |fetched_category| is nullopt, fetches all categories. Otherwise,
  335. // fetches at most |count_to_fetch| suggestions only from |fetched_category|.
  336. // TODO(vitaliii): Also support |count_to_fetch| when |fetched_category| is
  337. // nullopt.
  338. RequestParams BuildFetchParams(absl::optional<Category> fetched_category,
  339. int count_to_fetch) const;
  340. bool AreArticlesEmpty() const;
  341. bool AreArticlesAvailable() const;
  342. void NotifyFetchWithLoadingIndicatorStarted();
  343. void NotifyFetchWithLoadingIndicatorFailedOrTimeouted();
  344. GURL GetImageURLToFetch(const ContentSuggestion::ID& suggestion_id) const;
  345. State state_;
  346. raw_ptr<PrefService> pref_service_;
  347. const Category articles_category_;
  348. std::map<Category, CategoryContent, Category::CompareByID> category_contents_;
  349. // The ISO 639-1 code of the language used by the application.
  350. const std::string application_language_code_;
  351. // Ranker that orders the categories. Not owned.
  352. raw_ptr<CategoryRanker> category_ranker_;
  353. // Scheduler to inform about scheduling-related events. Not owned.
  354. raw_ptr<RemoteSuggestionsScheduler> remote_suggestions_scheduler_;
  355. // The suggestions fetcher.
  356. std::unique_ptr<RemoteSuggestionsFetcher> suggestions_fetcher_;
  357. // The database for persisting suggestions.
  358. std::unique_ptr<RemoteSuggestionsDatabase> database_;
  359. base::TimeTicks database_load_start_;
  360. // The image fetcher.
  361. CachedImageFetcher image_fetcher_;
  362. // The service that provides events and data about the signin and sync state.
  363. std::unique_ptr<RemoteSuggestionsStatusService> status_service_;
  364. // Set to true if ClearHistoryDependentState is called while the service isn't
  365. // ready. The nuke will be executed once the service finishes initialization
  366. // or enters the READY state.
  367. bool clear_history_dependent_state_when_initialized_;
  368. // Set to true if ClearCachedSuggestions has been called while the service
  369. // isn't ready. The clearing will be executed once the service finishes
  370. // initialization or enters the READY state.
  371. bool clear_cached_suggestions_when_initialized_;
  372. // A clock for getting the time. This allows to inject a clock in tests.
  373. raw_ptr<base::Clock> clock_;
  374. // A Timer for canceling too long fetches.
  375. std::unique_ptr<base::OneShotTimer> fetch_timeout_timer_;
  376. // Keeps track of the status of the ongoing request(s) and what action should
  377. // be taken on completion. Requests via Fetch() (fetching more) are _not_
  378. // tracked by this variable (as they do not need any special actions on
  379. // completion).
  380. FetchRequestStatus request_status_;
  381. };
  382. } // namespace ntp_snippets
  383. #endif // COMPONENTS_NTP_SNIPPETS_REMOTE_REMOTE_SUGGESTIONS_PROVIDER_IMPL_H_