json_request.cc 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453
  1. // Copyright 2016 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/ntp_snippets/remote/json_request.h"
  5. #include <algorithm>
  6. #include <utility>
  7. #include <vector>
  8. #include "base/bind.h"
  9. #include "base/command_line.h"
  10. #include "base/feature_list.h"
  11. #include "base/json/json_writer.h"
  12. #include "base/metrics/histogram_functions.h"
  13. #include "base/metrics/histogram_macros.h"
  14. #include "base/strings/stringprintf.h"
  15. #include "base/time/clock.h"
  16. #include "base/values.h"
  17. #include "components/ntp_snippets/category_info.h"
  18. #include "components/ntp_snippets/features.h"
  19. #include "components/ntp_snippets/remote/request_params.h"
  20. #include "components/ntp_snippets/user_classifier.h"
  21. #include "components/strings/grit/components_strings.h"
  22. #include "components/variations/net/variations_http_headers.h"
  23. #include "components/variations/variations_associated_data.h"
  24. #include "net/base/load_flags.h"
  25. #include "net/http/http_response_headers.h"
  26. #include "net/http/http_status_code.h"
  27. #include "net/traffic_annotation/network_traffic_annotation.h"
  28. #include "services/network/public/cpp/resource_request.h"
  29. #include "services/network/public/cpp/shared_url_loader_factory.h"
  30. #include "services/network/public/cpp/simple_url_loader.h"
  31. #include "services/network/public/mojom/url_response_head.mojom.h"
  32. #include "third_party/icu/source/common/unicode/uloc.h"
  33. #include "third_party/icu/source/common/unicode/utypes.h"
  34. #include "ui/base/l10n/l10n_util.h"
  35. using language::UrlLanguageHistogram;
  36. namespace ntp_snippets {
  37. namespace internal {
  38. namespace {
  39. // Variation parameter for disabling the retry.
  40. const char kBackground5xxRetriesName[] = "background_5xx_retries_count";
  41. // Variation parameter for sending UrlLanguageHistogram info to the server.
  42. const char kSendTopLanguagesName[] = "send_top_languages";
  43. // Variation parameter for sending UserClassifier info to the server.
  44. const char kSendUserClassName[] = "send_user_class";
  45. bool IsSendingTopLanguagesEnabled() {
  46. return variations::GetVariationParamByFeatureAsBool(
  47. ntp_snippets::kArticleSuggestionsFeature, kSendTopLanguagesName,
  48. /*default_value=*/true);
  49. }
  50. bool IsSendingUserClassEnabled() {
  51. return variations::GetVariationParamByFeatureAsBool(
  52. ntp_snippets::kArticleSuggestionsFeature, kSendUserClassName,
  53. /*default_value=*/true);
  54. }
  55. bool IsSendingOptionalImagesCapabilityEnabled() {
  56. return base::FeatureList::IsEnabled(
  57. ntp_snippets::kOptionalImagesEnabledFeature);
  58. }
  59. // Translate the BCP 47 |language_code| into a posix locale string.
  60. std::string PosixLocaleFromBCP47Language(const std::string& language_code) {
  61. char locale[ULOC_FULLNAME_CAPACITY];
  62. UErrorCode error = U_ZERO_ERROR;
  63. // Translate the input to a posix locale.
  64. uloc_forLanguageTag(language_code.c_str(), locale, ULOC_FULLNAME_CAPACITY,
  65. nullptr, &error);
  66. if (error != U_ZERO_ERROR) {
  67. DLOG(WARNING) << "Error in translating language code to a locale string: "
  68. << error;
  69. return std::string();
  70. }
  71. return locale;
  72. }
  73. std::string ISO639FromPosixLocale(const std::string& locale) {
  74. char language[ULOC_LANG_CAPACITY];
  75. UErrorCode error = U_ZERO_ERROR;
  76. uloc_getLanguage(locale.c_str(), language, ULOC_LANG_CAPACITY, &error);
  77. if (error != U_ZERO_ERROR) {
  78. DLOG(WARNING)
  79. << "Error in translating locale string to a ISO639 language code: "
  80. << error;
  81. return std::string();
  82. }
  83. return language;
  84. }
  85. void AppendLanguageInfoToList(base::Value::List* list,
  86. const UrlLanguageHistogram::LanguageInfo& info) {
  87. base::Value::Dict lang;
  88. lang.Set("language", info.language_code);
  89. lang.Set("frequency", info.frequency);
  90. list->Append(std::move(lang));
  91. }
  92. std::string GetUserClassString(UserClassifier::UserClass user_class) {
  93. switch (user_class) {
  94. case UserClassifier::UserClass::RARE_NTP_USER:
  95. return "RARE_NTP_USER";
  96. case UserClassifier::UserClass::ACTIVE_NTP_USER:
  97. return "ACTIVE_NTP_USER";
  98. case UserClassifier::UserClass::ACTIVE_SUGGESTIONS_CONSUMER:
  99. return "ACTIVE_SUGGESTIONS_CONSUMER";
  100. }
  101. NOTREACHED();
  102. return std::string();
  103. }
  104. } // namespace
  105. JsonRequest::JsonRequest(
  106. absl::optional<Category> exclusive_category,
  107. const base::Clock* clock, // Needed until destruction of the request.
  108. const ParseJSONCallback& callback)
  109. : exclusive_category_(exclusive_category),
  110. clock_(clock),
  111. parse_json_callback_(callback) {
  112. creation_time_ = clock_->Now();
  113. }
  114. JsonRequest::~JsonRequest() {
  115. LOG_IF(DFATAL, !request_completed_callback_.is_null())
  116. << "The CompletionCallback was never called!";
  117. }
  118. void JsonRequest::Start(CompletedCallback callback) {
  119. DCHECK(simple_url_loader_);
  120. DCHECK(url_loader_factory_);
  121. request_completed_callback_ = std::move(callback);
  122. last_response_string_.clear();
  123. simple_url_loader_->SetAllowHttpErrorResults(true);
  124. simple_url_loader_->DownloadToStringOfUnboundedSizeUntilCrashAndDie(
  125. url_loader_factory_.get(),
  126. base::BindOnce(&JsonRequest::OnSimpleLoaderComplete,
  127. base::Unretained(this)));
  128. }
  129. // static
  130. int JsonRequest::Get5xxRetryCount(bool interactive_request) {
  131. if (interactive_request) {
  132. return 2;
  133. }
  134. return std::max(0, variations::GetVariationParamByFeatureAsInt(
  135. ntp_snippets::kArticleSuggestionsFeature,
  136. kBackground5xxRetriesName, 0));
  137. }
  138. base::TimeDelta JsonRequest::GetFetchDuration() const {
  139. return clock_->Now() - creation_time_;
  140. }
  141. std::string JsonRequest::GetResponseString() const {
  142. return last_response_string_;
  143. }
  144. void JsonRequest::OnSimpleLoaderComplete(
  145. std::unique_ptr<std::string> response_body) {
  146. int net_error = simple_url_loader_->NetError();
  147. int response_code = -1;
  148. if (simple_url_loader_->ResponseInfo() &&
  149. simple_url_loader_->ResponseInfo()->headers) {
  150. response_code =
  151. simple_url_loader_->ResponseInfo()->headers->response_code();
  152. }
  153. simple_url_loader_.reset();
  154. base::UmaHistogramSparse("NewTabPage.Snippets.FetchHttpResponseOrErrorCode",
  155. net_error == net::OK ? response_code : net_error);
  156. if (net_error != net::OK) {
  157. std::move(request_completed_callback_)
  158. .Run(/*result=*/base::Value(), FetchResult::URL_REQUEST_STATUS_ERROR,
  159. /*error_details=*/base::StringPrintf(" %d", net_error));
  160. } else if (response_code / 100 != 2) {
  161. FetchResult result = response_code == net::HTTP_UNAUTHORIZED
  162. ? FetchResult::HTTP_ERROR_UNAUTHORIZED
  163. : FetchResult::HTTP_ERROR;
  164. std::move(request_completed_callback_)
  165. .Run(/*result=*/base::Value(), result,
  166. /*error_details=*/base::StringPrintf(" %d", response_code));
  167. } else {
  168. last_response_string_ = std::move(*response_body);
  169. parse_json_callback_.Run(last_response_string_,
  170. base::BindOnce(&JsonRequest::OnJsonParsed,
  171. weak_ptr_factory_.GetWeakPtr()),
  172. base::BindOnce(&JsonRequest::OnJsonError,
  173. weak_ptr_factory_.GetWeakPtr()));
  174. }
  175. }
  176. void JsonRequest::OnJsonParsed(base::Value result) {
  177. std::move(request_completed_callback_)
  178. .Run(std::move(result), FetchResult::SUCCESS,
  179. /*error_details=*/std::string());
  180. }
  181. void JsonRequest::OnJsonError(const std::string& error) {
  182. LOG(WARNING) << "Received invalid JSON (" << error
  183. << "): " << last_response_string_;
  184. std::move(request_completed_callback_)
  185. .Run(/*result=*/base::Value(), FetchResult::JSON_PARSE_ERROR,
  186. /*error_details=*/base::StringPrintf(" (error %s)", error.c_str()));
  187. }
  188. JsonRequest::Builder::Builder() : language_histogram_(nullptr) {}
  189. JsonRequest::Builder::Builder(JsonRequest::Builder&&) = default;
  190. JsonRequest::Builder::~Builder() = default;
  191. std::unique_ptr<JsonRequest> JsonRequest::Builder::Build() const {
  192. DCHECK(!url_.is_empty());
  193. DCHECK(url_loader_factory_);
  194. DCHECK(clock_);
  195. auto request = std::make_unique<JsonRequest>(params_.exclusive_category,
  196. clock_, parse_json_callback_);
  197. std::string body = BuildBody();
  198. request->simple_url_loader_ = BuildURLLoader(body);
  199. request->url_loader_factory_ = std::move(url_loader_factory_);
  200. return request;
  201. }
  202. JsonRequest::Builder& JsonRequest::Builder::SetAuthentication(
  203. const std::string& auth_header) {
  204. auth_header_ = auth_header;
  205. return *this;
  206. }
  207. JsonRequest::Builder& JsonRequest::Builder::SetLanguageHistogram(
  208. const language::UrlLanguageHistogram* language_histogram) {
  209. language_histogram_ = language_histogram;
  210. return *this;
  211. }
  212. JsonRequest::Builder& JsonRequest::Builder::SetParams(
  213. const RequestParams& params) {
  214. params_ = params;
  215. return *this;
  216. }
  217. JsonRequest::Builder& JsonRequest::Builder::SetParseJsonCallback(
  218. ParseJSONCallback callback) {
  219. parse_json_callback_ = callback;
  220. return *this;
  221. }
  222. JsonRequest::Builder& JsonRequest::Builder::SetClock(const base::Clock* clock) {
  223. clock_ = clock;
  224. return *this;
  225. }
  226. JsonRequest::Builder& JsonRequest::Builder::SetUrl(const GURL& url) {
  227. url_ = url;
  228. return *this;
  229. }
  230. JsonRequest::Builder& JsonRequest::Builder::SetUrlLoaderFactory(
  231. scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory) {
  232. url_loader_factory_ = std::move(url_loader_factory);
  233. return *this;
  234. }
  235. JsonRequest::Builder& JsonRequest::Builder::SetUserClassifier(
  236. const UserClassifier& user_classifier) {
  237. if (IsSendingUserClassEnabled()) {
  238. user_class_ = GetUserClassString(user_classifier.GetUserClass());
  239. }
  240. return *this;
  241. }
  242. JsonRequest::Builder& JsonRequest::Builder::SetOptionalImagesCapability(
  243. bool supports_optional_images) {
  244. if (supports_optional_images && IsSendingOptionalImagesCapabilityEnabled()) {
  245. display_capability_ = "CAPABILITY_OPTIONAL_IMAGES";
  246. }
  247. return *this;
  248. }
  249. std::unique_ptr<network::ResourceRequest>
  250. JsonRequest::Builder::BuildResourceRequest() const {
  251. auto resource_request = std::make_unique<network::ResourceRequest>();
  252. resource_request->url = url_;
  253. resource_request->credentials_mode = network::mojom::CredentialsMode::kOmit;
  254. resource_request->method = "POST";
  255. resource_request->headers.SetHeader("Content-Type",
  256. "application/json; charset=UTF-8");
  257. if (!auth_header_.empty()) {
  258. resource_request->headers.SetHeader("Authorization", auth_header_);
  259. }
  260. // Add X-Client-Data header with experiment IDs from field trials.
  261. // TODO: We should call AppendVariationHeaders with explicit
  262. // variations::SignedIn::kNo If the auth_header_ is empty
  263. variations::AppendVariationsHeaderUnknownSignedIn(
  264. url_, variations::InIncognito::kNo, resource_request.get());
  265. return resource_request;
  266. }
  267. std::string JsonRequest::Builder::BuildBody() const {
  268. base::Value::Dict request;
  269. std::string user_locale = PosixLocaleFromBCP47Language(params_.language_code);
  270. if (!user_locale.empty()) {
  271. request.Set("uiLanguage", user_locale);
  272. }
  273. request.Set("priority", params_.interactive_request ? "USER_ACTION"
  274. : "BACKGROUND_PREFETCH");
  275. base::Value::List excluded;
  276. for (const auto& id : params_.excluded_ids) {
  277. excluded.Append(id);
  278. }
  279. request.Set("excludedSuggestionIds", std::move(excluded));
  280. if (!user_class_.empty()) {
  281. request.Set("userActivenessClass", user_class_);
  282. }
  283. if (!display_capability_.empty()) {
  284. request.Set("displayCapability", display_capability_);
  285. }
  286. language::UrlLanguageHistogram::LanguageInfo ui_language;
  287. language::UrlLanguageHistogram::LanguageInfo other_top_language;
  288. PrepareLanguages(&ui_language, &other_top_language);
  289. if (ui_language.frequency != 0 || other_top_language.frequency != 0) {
  290. base::Value::List language_list;
  291. if (ui_language.frequency > 0) {
  292. AppendLanguageInfoToList(&language_list, ui_language);
  293. }
  294. if (other_top_language.frequency > 0) {
  295. AppendLanguageInfoToList(&language_list, other_top_language);
  296. }
  297. request.Set("topLanguages", std::move(language_list));
  298. }
  299. // TODO(vitaliii): Support count_to_fetch without requiring
  300. // |exclusive_category|.
  301. if (params_.exclusive_category.has_value()) {
  302. base::Value::Dict exclusive_category_parameters;
  303. exclusive_category_parameters.Set("id",
  304. params_.exclusive_category->remote_id());
  305. exclusive_category_parameters.Set("numSuggestions", params_.count_to_fetch);
  306. base::Value::List category_parameters;
  307. category_parameters.Append(std::move(exclusive_category_parameters));
  308. request.Set("categoryParameters", std::move(category_parameters));
  309. }
  310. std::string request_json;
  311. bool success = base::JSONWriter::WriteWithOptions(
  312. request, base::JSONWriter::OPTIONS_PRETTY_PRINT, &request_json);
  313. DCHECK(success);
  314. return request_json;
  315. }
  316. std::unique_ptr<network::SimpleURLLoader> JsonRequest::Builder::BuildURLLoader(
  317. const std::string& body) const {
  318. net::NetworkTrafficAnnotationTag traffic_annotation =
  319. net::DefineNetworkTrafficAnnotation("ntp_snippets_fetch", R"(
  320. semantics {
  321. sender: "New Tab Page Content Suggestions Fetch"
  322. description:
  323. "Chromium can show content suggestions (e.g. news articles) on the "
  324. "New Tab page. For signed-in users, these may be personalized "
  325. "based on the user's synced browsing history."
  326. trigger:
  327. "Triggered periodically in the background, or upon explicit user "
  328. "request."
  329. data:
  330. "The Chromium UI language, as well as a second language the user "
  331. "understands, based on language::UrlLanguageHistogram. For "
  332. "signed-in users, the requests is authenticated."
  333. destination: GOOGLE_OWNED_SERVICE
  334. }
  335. policy {
  336. cookies_allowed: NO
  337. setting:
  338. "This feature cannot be disabled by settings now (but is requested "
  339. "to be implemented in crbug.com/695129)."
  340. chrome_policy {
  341. NTPContentSuggestionsEnabled {
  342. policy_options {mode: MANDATORY}
  343. NTPContentSuggestionsEnabled: false
  344. }
  345. }
  346. })");
  347. auto resource_request = BuildResourceRequest();
  348. // Log the request for debugging network issues.
  349. VLOG(1) << "Sending a NTP snippets request to " << url_ << ":\n"
  350. << resource_request->headers.ToString() << "\n"
  351. << body;
  352. auto loader = network::SimpleURLLoader::Create(std::move(resource_request),
  353. traffic_annotation);
  354. loader->AttachStringForUpload(body, "application/json");
  355. int max_retries = JsonRequest::Get5xxRetryCount(params_.interactive_request);
  356. if (max_retries > 0) {
  357. loader->SetRetryOptions(
  358. max_retries, network::SimpleURLLoader::RETRY_ON_5XX |
  359. network::SimpleURLLoader::RETRY_ON_NETWORK_CHANGE);
  360. }
  361. return loader;
  362. }
  363. void JsonRequest::Builder::PrepareLanguages(
  364. language::UrlLanguageHistogram::LanguageInfo* ui_language,
  365. language::UrlLanguageHistogram::LanguageInfo* other_top_language) const {
  366. // TODO(jkrcal): Add language model factory for iOS and add fakes to tests so
  367. // that |language_histogram| is never nullptr. Remove this check and add a
  368. // DCHECK into the constructor.
  369. if (!language_histogram_ || !IsSendingTopLanguagesEnabled()) {
  370. return;
  371. }
  372. // TODO(jkrcal): Is this back-and-forth converting necessary?
  373. ui_language->language_code = ISO639FromPosixLocale(
  374. PosixLocaleFromBCP47Language(params_.language_code));
  375. ui_language->frequency =
  376. language_histogram_->GetLanguageFrequency(ui_language->language_code);
  377. std::vector<UrlLanguageHistogram::LanguageInfo> top_languages =
  378. language_histogram_->GetTopLanguages();
  379. for (const UrlLanguageHistogram::LanguageInfo& info : top_languages) {
  380. if (info.language_code != ui_language->language_code) {
  381. *other_top_language = info;
  382. // Report to UMA how important the UI language is.
  383. DCHECK_GT(other_top_language->frequency, 0)
  384. << "GetTopLanguages() should not return languages with 0 frequency";
  385. float ratio_ui_in_both_languages =
  386. ui_language->frequency /
  387. (ui_language->frequency + other_top_language->frequency);
  388. UMA_HISTOGRAM_PERCENTAGE(
  389. "NewTabPage.Languages.UILanguageRatioInTwoTopLanguages",
  390. ratio_ui_in_both_languages * 100);
  391. break;
  392. }
  393. }
  394. }
  395. } // namespace internal
  396. } // namespace ntp_snippets