remote_suggestions_fetcher_impl.cc 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430
  1. // Copyright 2017 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/ntp_snippets/remote/remote_suggestions_fetcher_impl.h"
  5. #include "base/bind.h"
  6. #include "base/metrics/histogram_functions.h"
  7. #include "base/metrics/histogram_macros.h"
  8. #include "base/strings/stringprintf.h"
  9. #include "base/strings/utf_string_conversions.h"
  10. #include "base/time/default_clock.h"
  11. #include "base/time/time.h"
  12. #include "base/values.h"
  13. #include "components/language/core/browser/url_language_histogram.h"
  14. #include "components/ntp_snippets/category.h"
  15. #include "components/ntp_snippets/features.h"
  16. #include "components/ntp_snippets/ntp_snippets_constants.h"
  17. #include "components/ntp_snippets/user_classifier.h"
  18. #include "components/signin/public/identity_manager/identity_manager.h"
  19. #include "components/signin/public/identity_manager/primary_account_access_token_fetcher.h"
  20. #include "components/signin/public/identity_manager/scope_set.h"
  21. #include "components/variations/variations_associated_data.h"
  22. #include "net/base/url_util.h"
  23. #include "services/network/public/cpp/shared_url_loader_factory.h"
  24. using language::UrlLanguageHistogram;
  25. namespace ntp_snippets {
  26. using internal::FetchResult;
  27. using internal::JsonRequest;
  28. namespace {
  29. const char kApiKeyQueryParam[] = "key";
  30. const char kPriorityQueryParam[] = "priority";
  31. const char kInteractivePriority[] = "user_action";
  32. const char kNonInteractivePriority[] = "background_prefetch";
  33. const char kAuthorizationRequestHeaderFormat[] = "Bearer %s";
  34. const int kFetchTimeHistogramResolution = 5;
  35. // Enables appending request priority as a query parameter to the fetch url,
  36. // when fetching article suggestions.
  37. const char kAppendRequestPriorityAsQueryParameterParamName[] =
  38. "append_request_priority_as_query_parameter";
  39. const bool kAppendRequestPriorityAsQueryParameterParamDefault = true;
  40. bool IsAppendingRequestPriorityAsQueryParameterEnabled() {
  41. return variations::GetVariationParamByFeatureAsBool(
  42. ntp_snippets::kArticleSuggestionsFeature,
  43. kAppendRequestPriorityAsQueryParameterParamName,
  44. kAppendRequestPriorityAsQueryParameterParamDefault);
  45. }
  46. GURL AppendPriorityQueryParameterIfEnabled(const GURL& url,
  47. bool is_interactive_request) {
  48. if (IsAppendingRequestPriorityAsQueryParameterEnabled()) {
  49. return net::AppendQueryParameter(url, kPriorityQueryParam,
  50. is_interactive_request
  51. ? kInteractivePriority
  52. : kNonInteractivePriority);
  53. }
  54. return url;
  55. }
  56. std::string FetchResultToString(FetchResult result) {
  57. switch (result) {
  58. case FetchResult::SUCCESS:
  59. return "OK";
  60. case FetchResult::URL_REQUEST_STATUS_ERROR:
  61. return "URLRequestStatus error";
  62. case FetchResult::HTTP_ERROR:
  63. return "HTTP error";
  64. case FetchResult::JSON_PARSE_ERROR:
  65. return "Received invalid JSON";
  66. case FetchResult::INVALID_SNIPPET_CONTENT_ERROR:
  67. return "Invalid / empty list.";
  68. case FetchResult::OAUTH_TOKEN_ERROR:
  69. return "Error in obtaining an OAuth2 access token.";
  70. case FetchResult::MISSING_API_KEY:
  71. return "No API key available.";
  72. case FetchResult::HTTP_ERROR_UNAUTHORIZED:
  73. return "Access token invalid";
  74. case FetchResult::RESULT_MAX:
  75. break;
  76. }
  77. NOTREACHED();
  78. return "Unknown error";
  79. }
  80. Status FetchResultToStatus(FetchResult result) {
  81. switch (result) {
  82. case FetchResult::SUCCESS:
  83. return Status::Success();
  84. // Permanent errors occur if it is more likely that the error originated
  85. // from the client.
  86. case FetchResult::OAUTH_TOKEN_ERROR:
  87. case FetchResult::MISSING_API_KEY:
  88. return Status(StatusCode::PERMANENT_ERROR, FetchResultToString(result));
  89. // Temporary errors occur if it's more likely that the client behaved
  90. // correctly but the server failed to respond as expected.
  91. // TODO(fhorschig): Revisit HTTP_ERROR once the rescheduling was reworked.
  92. case FetchResult::HTTP_ERROR:
  93. case FetchResult::HTTP_ERROR_UNAUTHORIZED:
  94. case FetchResult::URL_REQUEST_STATUS_ERROR:
  95. case FetchResult::INVALID_SNIPPET_CONTENT_ERROR:
  96. case FetchResult::JSON_PARSE_ERROR:
  97. return Status(StatusCode::TEMPORARY_ERROR, FetchResultToString(result));
  98. case FetchResult::RESULT_MAX:
  99. break;
  100. }
  101. NOTREACHED();
  102. return Status(StatusCode::PERMANENT_ERROR, std::string());
  103. }
  104. int GetMinuteOfTheDay(bool local_time,
  105. bool reduced_resolution,
  106. const base::Clock* clock) {
  107. base::Time now(clock->Now());
  108. base::Time::Exploded now_exploded{};
  109. local_time ? now.LocalExplode(&now_exploded) : now.UTCExplode(&now_exploded);
  110. int now_minute = reduced_resolution
  111. ? now_exploded.minute / kFetchTimeHistogramResolution *
  112. kFetchTimeHistogramResolution
  113. : now_exploded.minute;
  114. return now_exploded.hour * 60 + now_minute;
  115. }
  116. // The response from the backend might include suggestions from multiple
  117. // categories. If only a single category was requested, this function filters
  118. // all other categories out.
  119. void FilterCategories(FetchedCategoriesVector* categories,
  120. absl::optional<Category> exclusive_category) {
  121. if (!exclusive_category.has_value()) {
  122. return;
  123. }
  124. Category exclusive = exclusive_category.value();
  125. auto category_it =
  126. std::find_if(categories->begin(), categories->end(),
  127. [&exclusive](const FetchedCategory& c) -> bool {
  128. return c.category == exclusive;
  129. });
  130. if (category_it == categories->end()) {
  131. categories->clear();
  132. return;
  133. }
  134. FetchedCategory category = std::move(*category_it);
  135. categories->clear();
  136. categories->push_back(std::move(category));
  137. }
  138. } // namespace
  139. bool RemoteSuggestionsFetcherImpl::skip_api_key_check_for_testing_ = false;
  140. RemoteSuggestionsFetcherImpl::RemoteSuggestionsFetcherImpl(
  141. signin::IdentityManager* identity_manager,
  142. scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory,
  143. PrefService* pref_service,
  144. UrlLanguageHistogram* language_histogram,
  145. const ParseJSONCallback& parse_json_callback,
  146. const GURL& api_endpoint,
  147. const std::string& api_key,
  148. const UserClassifier* user_classifier)
  149. : identity_manager_(identity_manager),
  150. url_loader_factory_(std::move(url_loader_factory)),
  151. language_histogram_(language_histogram),
  152. parse_json_callback_(parse_json_callback),
  153. fetch_url_(api_endpoint),
  154. api_key_(api_key),
  155. clock_(base::DefaultClock::GetInstance()),
  156. user_classifier_(user_classifier),
  157. last_fetch_authenticated_(false) {}
  158. RemoteSuggestionsFetcherImpl::~RemoteSuggestionsFetcherImpl() = default;
  159. const std::string& RemoteSuggestionsFetcherImpl::GetLastStatusForDebugging()
  160. const {
  161. return last_status_;
  162. }
  163. const std::string& RemoteSuggestionsFetcherImpl::GetLastJsonForDebugging()
  164. const {
  165. return last_fetch_json_;
  166. }
  167. bool RemoteSuggestionsFetcherImpl::WasLastFetchAuthenticatedForDebugging()
  168. const {
  169. return last_fetch_authenticated_;
  170. }
  171. const GURL& RemoteSuggestionsFetcherImpl::GetFetchUrlForDebugging() const {
  172. return fetch_url_;
  173. }
  174. void RemoteSuggestionsFetcherImpl::FetchSnippets(
  175. const RequestParams& params,
  176. SnippetsAvailableCallback callback) {
  177. SnippetsAvailableCallback wrapped_callback = base::BindOnce(
  178. &RemoteSuggestionsFetcherImpl::EmitDurationAndInvokeCallback,
  179. base::Unretained(this), base::Time::Now(), std::move(callback));
  180. if (!params.interactive_request) {
  181. base::UmaHistogramSparse(
  182. "NewTabPage.Snippets.FetchTimeLocal",
  183. GetMinuteOfTheDay(/*local_time=*/true,
  184. /*reduced_resolution=*/true, clock_));
  185. base::UmaHistogramSparse(
  186. "NewTabPage.Snippets.FetchTimeUTC",
  187. GetMinuteOfTheDay(/*local_time=*/false,
  188. /*reduced_resolution=*/true, clock_));
  189. }
  190. JsonRequest::Builder builder;
  191. builder.SetLanguageHistogram(language_histogram_)
  192. .SetParams(params)
  193. .SetParseJsonCallback(parse_json_callback_)
  194. .SetClock(clock_)
  195. .SetUrlLoaderFactory(url_loader_factory_)
  196. .SetUserClassifier(*user_classifier_)
  197. .SetOptionalImagesCapability(true);
  198. if (identity_manager_->HasPrimaryAccount(signin::ConsentLevel::kSync)) {
  199. // Signed-in: get OAuth token --> fetch suggestions.
  200. pending_requests_.emplace(std::move(builder), std::move(wrapped_callback));
  201. StartTokenRequest();
  202. } else {
  203. // Not signed in: fetch suggestions (without authentication).
  204. FetchSnippetsNonAuthenticated(std::move(builder),
  205. std::move(wrapped_callback));
  206. }
  207. }
  208. void RemoteSuggestionsFetcherImpl::FetchSnippetsNonAuthenticated(
  209. JsonRequest::Builder builder,
  210. SnippetsAvailableCallback callback) {
  211. if (api_key_.empty() && !skip_api_key_check_for_testing_) {
  212. // If we don't have an API key, don't even try.
  213. FetchFinished(OptionalFetchedCategories(), std::move(callback),
  214. FetchResult::MISSING_API_KEY, std::string(),
  215. /*is_authenticated=*/false, std::string());
  216. return;
  217. }
  218. // When not providing OAuth token, we need to pass the Google API key.
  219. GURL url = net::AppendQueryParameter(fetch_url_, kApiKeyQueryParam, api_key_);
  220. url = AppendPriorityQueryParameterIfEnabled(url,
  221. builder.is_interactive_request());
  222. builder.SetUrl(url);
  223. StartRequest(std::move(builder), std::move(callback),
  224. /*is_authenticated=*/false, std::string());
  225. }
  226. void RemoteSuggestionsFetcherImpl::FetchSnippetsAuthenticated(
  227. JsonRequest::Builder builder,
  228. SnippetsAvailableCallback callback,
  229. const std::string& oauth_access_token) {
  230. GURL url = AppendPriorityQueryParameterIfEnabled(
  231. fetch_url_, builder.is_interactive_request());
  232. builder.SetUrl(url).SetAuthentication(
  233. base::StringPrintf(kAuthorizationRequestHeaderFormat,
  234. oauth_access_token.c_str()));
  235. StartRequest(std::move(builder), std::move(callback),
  236. /*is_authenticated=*/true, oauth_access_token);
  237. }
  238. void RemoteSuggestionsFetcherImpl::StartRequest(
  239. JsonRequest::Builder builder,
  240. SnippetsAvailableCallback callback,
  241. bool is_authenticated,
  242. std::string access_token) {
  243. std::unique_ptr<JsonRequest> request = builder.Build();
  244. JsonRequest* raw_request = request.get();
  245. raw_request->Start(base::BindOnce(
  246. &RemoteSuggestionsFetcherImpl::JsonRequestDone, base::Unretained(this),
  247. std::move(request), std::move(callback), is_authenticated, access_token));
  248. }
  249. void RemoteSuggestionsFetcherImpl::StartTokenRequest() {
  250. // If there is already an ongoing token request, just wait for that.
  251. if (token_fetcher_) {
  252. return;
  253. }
  254. base::Time token_start_time = clock_->Now();
  255. signin::ScopeSet scopes{kContentSuggestionsApiScope};
  256. token_fetcher_ = std::make_unique<signin::PrimaryAccountAccessTokenFetcher>(
  257. "ntp_snippets", identity_manager_, scopes,
  258. base::BindOnce(&RemoteSuggestionsFetcherImpl::AccessTokenFetchFinished,
  259. base::Unretained(this), token_start_time),
  260. signin::PrimaryAccountAccessTokenFetcher::Mode::kWaitUntilAvailable);
  261. }
  262. void RemoteSuggestionsFetcherImpl::AccessTokenFetchFinished(
  263. base::Time token_start_time,
  264. GoogleServiceAuthError error,
  265. signin::AccessTokenInfo access_token_info) {
  266. DCHECK(token_fetcher_);
  267. token_fetcher_.reset();
  268. UMA_HISTOGRAM_ENUMERATION("ContentSuggestions.Feed.Network.TokenFetchStatus",
  269. error.state(), GoogleServiceAuthError::NUM_STATES);
  270. base::TimeDelta token_duration = clock_->Now() - token_start_time;
  271. UMA_HISTOGRAM_MEDIUM_TIMES("ContentSuggestions.Feed.Network.TokenDuration",
  272. token_duration);
  273. if (error.state() != GoogleServiceAuthError::NONE) {
  274. AccessTokenError(error);
  275. return;
  276. }
  277. DCHECK(!access_token_info.token.empty());
  278. while (!pending_requests_.empty()) {
  279. std::pair<JsonRequest::Builder, SnippetsAvailableCallback>
  280. builder_and_callback = std::move(pending_requests_.front());
  281. pending_requests_.pop();
  282. FetchSnippetsAuthenticated(std::move(builder_and_callback.first),
  283. std::move(builder_and_callback.second),
  284. access_token_info.token);
  285. }
  286. }
  287. void RemoteSuggestionsFetcherImpl::AccessTokenError(
  288. const GoogleServiceAuthError& error) {
  289. DCHECK_NE(error.state(), GoogleServiceAuthError::NONE);
  290. DLOG(ERROR) << "Unable to get token: " << error.ToString();
  291. while (!pending_requests_.empty()) {
  292. std::pair<JsonRequest::Builder, SnippetsAvailableCallback>
  293. builder_and_callback = std::move(pending_requests_.front());
  294. FetchFinished(OptionalFetchedCategories(),
  295. std::move(builder_and_callback.second),
  296. FetchResult::OAUTH_TOKEN_ERROR,
  297. /*error_details=*/
  298. base::StringPrintf(" (%s)", error.ToString().c_str()),
  299. /*is_authenticated=*/true, std::string());
  300. pending_requests_.pop();
  301. }
  302. }
  303. void RemoteSuggestionsFetcherImpl::JsonRequestDone(
  304. std::unique_ptr<JsonRequest> request,
  305. SnippetsAvailableCallback callback,
  306. bool is_authenticated,
  307. std::string access_token,
  308. base::Value result,
  309. FetchResult status_code,
  310. const std::string& error_details) {
  311. DCHECK(request);
  312. // Record the time when request for fetching remote content snippets finished.
  313. const base::Time fetch_time = clock_->Now();
  314. last_fetch_json_ = request->GetResponseString();
  315. UMA_HISTOGRAM_TIMES("NewTabPage.Snippets.FetchTime",
  316. request->GetFetchDuration());
  317. if (result.is_none()) {
  318. FetchFinished(OptionalFetchedCategories(), std::move(callback), status_code,
  319. error_details, is_authenticated, access_token);
  320. return;
  321. }
  322. FetchedCategoriesVector categories;
  323. if (!JsonToCategories(result, &categories, fetch_time)) {
  324. LOG(WARNING) << "Received invalid snippets: " << last_fetch_json_;
  325. FetchFinished(OptionalFetchedCategories(), std::move(callback),
  326. FetchResult::INVALID_SNIPPET_CONTENT_ERROR, std::string(),
  327. is_authenticated, access_token);
  328. return;
  329. }
  330. // Filter out unwanted categories if necessary.
  331. // TODO(fhorschig): As soon as the server supports filtering by category,
  332. // adjust the request instead of over-fetching and filtering here.
  333. FilterCategories(&categories, request->exclusive_category());
  334. FetchFinished(std::move(categories), std::move(callback),
  335. FetchResult::SUCCESS, std::string(), is_authenticated,
  336. access_token);
  337. }
  338. void RemoteSuggestionsFetcherImpl::FetchFinished(
  339. OptionalFetchedCategories categories,
  340. SnippetsAvailableCallback callback,
  341. FetchResult fetch_result,
  342. const std::string& error_details,
  343. bool is_authenticated,
  344. std::string access_token) {
  345. DCHECK(fetch_result == FetchResult::SUCCESS || !categories.has_value());
  346. if (fetch_result == FetchResult::HTTP_ERROR_UNAUTHORIZED) {
  347. signin::ScopeSet scopes{kContentSuggestionsApiScope};
  348. CoreAccountId account_id =
  349. identity_manager_->GetPrimaryAccountId(signin::ConsentLevel::kSync);
  350. identity_manager_->RemoveAccessTokenFromCache(account_id, scopes,
  351. access_token);
  352. }
  353. last_status_ = FetchResultToString(fetch_result) + error_details;
  354. last_fetch_authenticated_ = is_authenticated;
  355. UMA_HISTOGRAM_ENUMERATION("NewTabPage.Snippets.FetchResult",
  356. static_cast<int>(fetch_result),
  357. static_cast<int>(FetchResult::RESULT_MAX));
  358. DVLOG(1) << "Fetch finished: " << last_status_;
  359. std::move(callback).Run(FetchResultToStatus(fetch_result),
  360. std::move(categories));
  361. }
  362. void RemoteSuggestionsFetcherImpl::EmitDurationAndInvokeCallback(
  363. base::Time start_time,
  364. SnippetsAvailableCallback callback,
  365. Status status,
  366. OptionalFetchedCategories fetched_categories) {
  367. base::TimeDelta duration = clock_->Now() - start_time;
  368. UMA_HISTOGRAM_MEDIUM_TIMES("ContentSuggestions.Feed.Network.Duration",
  369. duration);
  370. std::move(callback).Run(status, std::move(fetched_categories));
  371. }
  372. // static
  373. void RemoteSuggestionsFetcherImpl::set_skip_api_key_check_for_testing() {
  374. skip_api_key_check_for_testing_ = true;
  375. }
  376. } // namespace ntp_snippets