123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453 |
- // Copyright 2016 The Chromium Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style license that can be
- // found in the LICENSE file.
- #include "components/ntp_snippets/remote/json_request.h"
- #include <algorithm>
- #include <utility>
- #include <vector>
- #include "base/bind.h"
- #include "base/command_line.h"
- #include "base/feature_list.h"
- #include "base/json/json_writer.h"
- #include "base/metrics/histogram_functions.h"
- #include "base/metrics/histogram_macros.h"
- #include "base/strings/stringprintf.h"
- #include "base/time/clock.h"
- #include "base/values.h"
- #include "components/ntp_snippets/category_info.h"
- #include "components/ntp_snippets/features.h"
- #include "components/ntp_snippets/remote/request_params.h"
- #include "components/ntp_snippets/user_classifier.h"
- #include "components/strings/grit/components_strings.h"
- #include "components/variations/net/variations_http_headers.h"
- #include "components/variations/variations_associated_data.h"
- #include "net/base/load_flags.h"
- #include "net/http/http_response_headers.h"
- #include "net/http/http_status_code.h"
- #include "net/traffic_annotation/network_traffic_annotation.h"
- #include "services/network/public/cpp/resource_request.h"
- #include "services/network/public/cpp/shared_url_loader_factory.h"
- #include "services/network/public/cpp/simple_url_loader.h"
- #include "services/network/public/mojom/url_response_head.mojom.h"
- #include "third_party/icu/source/common/unicode/uloc.h"
- #include "third_party/icu/source/common/unicode/utypes.h"
- #include "ui/base/l10n/l10n_util.h"
- using language::UrlLanguageHistogram;
- namespace ntp_snippets {
- namespace internal {
- namespace {
- // Variation parameter for disabling the retry.
- const char kBackground5xxRetriesName[] = "background_5xx_retries_count";
- // Variation parameter for sending UrlLanguageHistogram info to the server.
- const char kSendTopLanguagesName[] = "send_top_languages";
- // Variation parameter for sending UserClassifier info to the server.
- const char kSendUserClassName[] = "send_user_class";
- bool IsSendingTopLanguagesEnabled() {
- return variations::GetVariationParamByFeatureAsBool(
- ntp_snippets::kArticleSuggestionsFeature, kSendTopLanguagesName,
- /*default_value=*/true);
- }
- bool IsSendingUserClassEnabled() {
- return variations::GetVariationParamByFeatureAsBool(
- ntp_snippets::kArticleSuggestionsFeature, kSendUserClassName,
- /*default_value=*/true);
- }
- bool IsSendingOptionalImagesCapabilityEnabled() {
- return base::FeatureList::IsEnabled(
- ntp_snippets::kOptionalImagesEnabledFeature);
- }
- // Translate the BCP 47 |language_code| into a posix locale string.
- std::string PosixLocaleFromBCP47Language(const std::string& language_code) {
- char locale[ULOC_FULLNAME_CAPACITY];
- UErrorCode error = U_ZERO_ERROR;
- // Translate the input to a posix locale.
- uloc_forLanguageTag(language_code.c_str(), locale, ULOC_FULLNAME_CAPACITY,
- nullptr, &error);
- if (error != U_ZERO_ERROR) {
- DLOG(WARNING) << "Error in translating language code to a locale string: "
- << error;
- return std::string();
- }
- return locale;
- }
- std::string ISO639FromPosixLocale(const std::string& locale) {
- char language[ULOC_LANG_CAPACITY];
- UErrorCode error = U_ZERO_ERROR;
- uloc_getLanguage(locale.c_str(), language, ULOC_LANG_CAPACITY, &error);
- if (error != U_ZERO_ERROR) {
- DLOG(WARNING)
- << "Error in translating locale string to a ISO639 language code: "
- << error;
- return std::string();
- }
- return language;
- }
- void AppendLanguageInfoToList(base::Value::List* list,
- const UrlLanguageHistogram::LanguageInfo& info) {
- base::Value::Dict lang;
- lang.Set("language", info.language_code);
- lang.Set("frequency", info.frequency);
- list->Append(std::move(lang));
- }
- std::string GetUserClassString(UserClassifier::UserClass user_class) {
- switch (user_class) {
- case UserClassifier::UserClass::RARE_NTP_USER:
- return "RARE_NTP_USER";
- case UserClassifier::UserClass::ACTIVE_NTP_USER:
- return "ACTIVE_NTP_USER";
- case UserClassifier::UserClass::ACTIVE_SUGGESTIONS_CONSUMER:
- return "ACTIVE_SUGGESTIONS_CONSUMER";
- }
- NOTREACHED();
- return std::string();
- }
- } // namespace
- JsonRequest::JsonRequest(
- absl::optional<Category> exclusive_category,
- const base::Clock* clock, // Needed until destruction of the request.
- const ParseJSONCallback& callback)
- : exclusive_category_(exclusive_category),
- clock_(clock),
- parse_json_callback_(callback) {
- creation_time_ = clock_->Now();
- }
- JsonRequest::~JsonRequest() {
- LOG_IF(DFATAL, !request_completed_callback_.is_null())
- << "The CompletionCallback was never called!";
- }
- void JsonRequest::Start(CompletedCallback callback) {
- DCHECK(simple_url_loader_);
- DCHECK(url_loader_factory_);
- request_completed_callback_ = std::move(callback);
- last_response_string_.clear();
- simple_url_loader_->SetAllowHttpErrorResults(true);
- simple_url_loader_->DownloadToStringOfUnboundedSizeUntilCrashAndDie(
- url_loader_factory_.get(),
- base::BindOnce(&JsonRequest::OnSimpleLoaderComplete,
- base::Unretained(this)));
- }
- // static
- int JsonRequest::Get5xxRetryCount(bool interactive_request) {
- if (interactive_request) {
- return 2;
- }
- return std::max(0, variations::GetVariationParamByFeatureAsInt(
- ntp_snippets::kArticleSuggestionsFeature,
- kBackground5xxRetriesName, 0));
- }
- base::TimeDelta JsonRequest::GetFetchDuration() const {
- return clock_->Now() - creation_time_;
- }
- std::string JsonRequest::GetResponseString() const {
- return last_response_string_;
- }
- void JsonRequest::OnSimpleLoaderComplete(
- std::unique_ptr<std::string> response_body) {
- int net_error = simple_url_loader_->NetError();
- int response_code = -1;
- if (simple_url_loader_->ResponseInfo() &&
- simple_url_loader_->ResponseInfo()->headers) {
- response_code =
- simple_url_loader_->ResponseInfo()->headers->response_code();
- }
- simple_url_loader_.reset();
- base::UmaHistogramSparse("NewTabPage.Snippets.FetchHttpResponseOrErrorCode",
- net_error == net::OK ? response_code : net_error);
- if (net_error != net::OK) {
- std::move(request_completed_callback_)
- .Run(/*result=*/base::Value(), FetchResult::URL_REQUEST_STATUS_ERROR,
- /*error_details=*/base::StringPrintf(" %d", net_error));
- } else if (response_code / 100 != 2) {
- FetchResult result = response_code == net::HTTP_UNAUTHORIZED
- ? FetchResult::HTTP_ERROR_UNAUTHORIZED
- : FetchResult::HTTP_ERROR;
- std::move(request_completed_callback_)
- .Run(/*result=*/base::Value(), result,
- /*error_details=*/base::StringPrintf(" %d", response_code));
- } else {
- last_response_string_ = std::move(*response_body);
- parse_json_callback_.Run(last_response_string_,
- base::BindOnce(&JsonRequest::OnJsonParsed,
- weak_ptr_factory_.GetWeakPtr()),
- base::BindOnce(&JsonRequest::OnJsonError,
- weak_ptr_factory_.GetWeakPtr()));
- }
- }
- void JsonRequest::OnJsonParsed(base::Value result) {
- std::move(request_completed_callback_)
- .Run(std::move(result), FetchResult::SUCCESS,
- /*error_details=*/std::string());
- }
- void JsonRequest::OnJsonError(const std::string& error) {
- LOG(WARNING) << "Received invalid JSON (" << error
- << "): " << last_response_string_;
- std::move(request_completed_callback_)
- .Run(/*result=*/base::Value(), FetchResult::JSON_PARSE_ERROR,
- /*error_details=*/base::StringPrintf(" (error %s)", error.c_str()));
- }
- JsonRequest::Builder::Builder() : language_histogram_(nullptr) {}
- JsonRequest::Builder::Builder(JsonRequest::Builder&&) = default;
- JsonRequest::Builder::~Builder() = default;
- std::unique_ptr<JsonRequest> JsonRequest::Builder::Build() const {
- DCHECK(!url_.is_empty());
- DCHECK(url_loader_factory_);
- DCHECK(clock_);
- auto request = std::make_unique<JsonRequest>(params_.exclusive_category,
- clock_, parse_json_callback_);
- std::string body = BuildBody();
- request->simple_url_loader_ = BuildURLLoader(body);
- request->url_loader_factory_ = std::move(url_loader_factory_);
- return request;
- }
- JsonRequest::Builder& JsonRequest::Builder::SetAuthentication(
- const std::string& auth_header) {
- auth_header_ = auth_header;
- return *this;
- }
- JsonRequest::Builder& JsonRequest::Builder::SetLanguageHistogram(
- const language::UrlLanguageHistogram* language_histogram) {
- language_histogram_ = language_histogram;
- return *this;
- }
- JsonRequest::Builder& JsonRequest::Builder::SetParams(
- const RequestParams& params) {
- params_ = params;
- return *this;
- }
- JsonRequest::Builder& JsonRequest::Builder::SetParseJsonCallback(
- ParseJSONCallback callback) {
- parse_json_callback_ = callback;
- return *this;
- }
- JsonRequest::Builder& JsonRequest::Builder::SetClock(const base::Clock* clock) {
- clock_ = clock;
- return *this;
- }
- JsonRequest::Builder& JsonRequest::Builder::SetUrl(const GURL& url) {
- url_ = url;
- return *this;
- }
- JsonRequest::Builder& JsonRequest::Builder::SetUrlLoaderFactory(
- scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory) {
- url_loader_factory_ = std::move(url_loader_factory);
- return *this;
- }
- JsonRequest::Builder& JsonRequest::Builder::SetUserClassifier(
- const UserClassifier& user_classifier) {
- if (IsSendingUserClassEnabled()) {
- user_class_ = GetUserClassString(user_classifier.GetUserClass());
- }
- return *this;
- }
- JsonRequest::Builder& JsonRequest::Builder::SetOptionalImagesCapability(
- bool supports_optional_images) {
- if (supports_optional_images && IsSendingOptionalImagesCapabilityEnabled()) {
- display_capability_ = "CAPABILITY_OPTIONAL_IMAGES";
- }
- return *this;
- }
- std::unique_ptr<network::ResourceRequest>
- JsonRequest::Builder::BuildResourceRequest() const {
- auto resource_request = std::make_unique<network::ResourceRequest>();
- resource_request->url = url_;
- resource_request->credentials_mode = network::mojom::CredentialsMode::kOmit;
- resource_request->method = "POST";
- resource_request->headers.SetHeader("Content-Type",
- "application/json; charset=UTF-8");
- if (!auth_header_.empty()) {
- resource_request->headers.SetHeader("Authorization", auth_header_);
- }
- // Add X-Client-Data header with experiment IDs from field trials.
- // TODO: We should call AppendVariationHeaders with explicit
- // variations::SignedIn::kNo If the auth_header_ is empty
- variations::AppendVariationsHeaderUnknownSignedIn(
- url_, variations::InIncognito::kNo, resource_request.get());
- return resource_request;
- }
- std::string JsonRequest::Builder::BuildBody() const {
- base::Value::Dict request;
- std::string user_locale = PosixLocaleFromBCP47Language(params_.language_code);
- if (!user_locale.empty()) {
- request.Set("uiLanguage", user_locale);
- }
- request.Set("priority", params_.interactive_request ? "USER_ACTION"
- : "BACKGROUND_PREFETCH");
- base::Value::List excluded;
- for (const auto& id : params_.excluded_ids) {
- excluded.Append(id);
- }
- request.Set("excludedSuggestionIds", std::move(excluded));
- if (!user_class_.empty()) {
- request.Set("userActivenessClass", user_class_);
- }
- if (!display_capability_.empty()) {
- request.Set("displayCapability", display_capability_);
- }
- language::UrlLanguageHistogram::LanguageInfo ui_language;
- language::UrlLanguageHistogram::LanguageInfo other_top_language;
- PrepareLanguages(&ui_language, &other_top_language);
- if (ui_language.frequency != 0 || other_top_language.frequency != 0) {
- base::Value::List language_list;
- if (ui_language.frequency > 0) {
- AppendLanguageInfoToList(&language_list, ui_language);
- }
- if (other_top_language.frequency > 0) {
- AppendLanguageInfoToList(&language_list, other_top_language);
- }
- request.Set("topLanguages", std::move(language_list));
- }
- // TODO(vitaliii): Support count_to_fetch without requiring
- // |exclusive_category|.
- if (params_.exclusive_category.has_value()) {
- base::Value::Dict exclusive_category_parameters;
- exclusive_category_parameters.Set("id",
- params_.exclusive_category->remote_id());
- exclusive_category_parameters.Set("numSuggestions", params_.count_to_fetch);
- base::Value::List category_parameters;
- category_parameters.Append(std::move(exclusive_category_parameters));
- request.Set("categoryParameters", std::move(category_parameters));
- }
- std::string request_json;
- bool success = base::JSONWriter::WriteWithOptions(
- request, base::JSONWriter::OPTIONS_PRETTY_PRINT, &request_json);
- DCHECK(success);
- return request_json;
- }
- std::unique_ptr<network::SimpleURLLoader> JsonRequest::Builder::BuildURLLoader(
- const std::string& body) const {
- net::NetworkTrafficAnnotationTag traffic_annotation =
- net::DefineNetworkTrafficAnnotation("ntp_snippets_fetch", R"(
- semantics {
- sender: "New Tab Page Content Suggestions Fetch"
- description:
- "Chromium can show content suggestions (e.g. news articles) on the "
- "New Tab page. For signed-in users, these may be personalized "
- "based on the user's synced browsing history."
- trigger:
- "Triggered periodically in the background, or upon explicit user "
- "request."
- data:
- "The Chromium UI language, as well as a second language the user "
- "understands, based on language::UrlLanguageHistogram. For "
- "signed-in users, the requests is authenticated."
- destination: GOOGLE_OWNED_SERVICE
- }
- policy {
- cookies_allowed: NO
- setting:
- "This feature cannot be disabled by settings now (but is requested "
- "to be implemented in crbug.com/695129)."
- chrome_policy {
- NTPContentSuggestionsEnabled {
- policy_options {mode: MANDATORY}
- NTPContentSuggestionsEnabled: false
- }
- }
- })");
- auto resource_request = BuildResourceRequest();
- // Log the request for debugging network issues.
- VLOG(1) << "Sending a NTP snippets request to " << url_ << ":\n"
- << resource_request->headers.ToString() << "\n"
- << body;
- auto loader = network::SimpleURLLoader::Create(std::move(resource_request),
- traffic_annotation);
- loader->AttachStringForUpload(body, "application/json");
- int max_retries = JsonRequest::Get5xxRetryCount(params_.interactive_request);
- if (max_retries > 0) {
- loader->SetRetryOptions(
- max_retries, network::SimpleURLLoader::RETRY_ON_5XX |
- network::SimpleURLLoader::RETRY_ON_NETWORK_CHANGE);
- }
- return loader;
- }
- void JsonRequest::Builder::PrepareLanguages(
- language::UrlLanguageHistogram::LanguageInfo* ui_language,
- language::UrlLanguageHistogram::LanguageInfo* other_top_language) const {
- // TODO(jkrcal): Add language model factory for iOS and add fakes to tests so
- // that |language_histogram| is never nullptr. Remove this check and add a
- // DCHECK into the constructor.
- if (!language_histogram_ || !IsSendingTopLanguagesEnabled()) {
- return;
- }
- // TODO(jkrcal): Is this back-and-forth converting necessary?
- ui_language->language_code = ISO639FromPosixLocale(
- PosixLocaleFromBCP47Language(params_.language_code));
- ui_language->frequency =
- language_histogram_->GetLanguageFrequency(ui_language->language_code);
- std::vector<UrlLanguageHistogram::LanguageInfo> top_languages =
- language_histogram_->GetTopLanguages();
- for (const UrlLanguageHistogram::LanguageInfo& info : top_languages) {
- if (info.language_code != ui_language->language_code) {
- *other_top_language = info;
- // Report to UMA how important the UI language is.
- DCHECK_GT(other_top_language->frequency, 0)
- << "GetTopLanguages() should not return languages with 0 frequency";
- float ratio_ui_in_both_languages =
- ui_language->frequency /
- (ui_language->frequency + other_top_language->frequency);
- UMA_HISTOGRAM_PERCENTAGE(
- "NewTabPage.Languages.UILanguageRatioInTwoTopLanguages",
- ratio_ui_in_both_languages * 100);
- break;
- }
- }
- }
- } // namespace internal
- } // namespace ntp_snippets
|