123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514 |
- // Copyright 2015 The Chromium Authors. All rights reserved.
- // Use of this source code is governed by a BSD-style license that can be
- // found in the LICENSE file.
- #include "components/ntp_tiles/popular_sites_impl.h"
- #include <stddef.h>
- #include <map>
- #include <memory>
- #include <utility>
- #include "base/bind.h"
- #include "base/command_line.h"
- #include "base/feature_list.h"
- #include "base/strings/string_number_conversions.h"
- #include "base/strings/string_util.h"
- #include "base/strings/stringprintf.h"
- #include "base/strings/utf_string_conversions.h"
- #include "base/time/time.h"
- #include "base/values.h"
- #include "build/branding_buildflags.h"
- #include "build/build_config.h"
- #include "components/google/core/common/google_util.h"
- #include "components/ntp_tiles/features.h"
- #include "components/ntp_tiles/pref_names.h"
- #include "components/ntp_tiles/switches.h"
- #include "components/pref_registry/pref_registry_syncable.h"
- #include "components/prefs/pref_service.h"
- #include "components/search_engines/search_engine_type.h"
- #include "components/search_engines/template_url_service.h"
- #include "components/variations/service/variations_service.h"
- #include "components/variations/variations_associated_data.h"
- #include "net/base/load_flags.h"
- #include "net/http/http_status_code.h"
- #include "net/traffic_annotation/network_traffic_annotation.h"
- #include "services/data_decoder/public/cpp/data_decoder.h"
- #include "services/network/public/cpp/resource_request.h"
- #include "services/network/public/cpp/shared_url_loader_factory.h"
- #include "services/network/public/cpp/simple_url_loader.h"
- #if BUILDFLAG(IS_ANDROID) || BUILDFLAG(IS_IOS)
- #include "base/json/json_reader.h"
- #include "components/grit/components_resources.h"
- #include "ui/base/resource/resource_bundle.h"
- #endif
- #if BUILDFLAG(IS_IOS)
- #include "components/ntp_tiles/country_code_ios.h"
- #endif
- using variations::VariationsService;
- namespace ntp_tiles {
- namespace {
- const char kPopularSitesURLFormat[] =
- "https://www.gstatic.com/%ssuggested_sites_%s_%s.json";
- const char kPopularSitesDefaultDirectory[] = "chrome/ntp/";
- const char kPopularSitesDefaultCountryCode[] = "DEFAULT";
- const char kPopularSitesDefaultVersion[] = "5";
- const int kSitesExplorationStartVersion = 6;
- const int kPopularSitesRedownloadIntervalHours = 24;
- GURL GetPopularSitesURL(const std::string& directory,
- const std::string& country,
- const std::string& version) {
- return GURL(base::StringPrintf(kPopularSitesURLFormat, directory.c_str(),
- country.c_str(), version.c_str()));
- }
- // Extract the country from the default search engine if the default search
- // engine is Google.
- std::string GetDefaultSearchEngineCountryCode(
- const TemplateURLService* template_url_service) {
- DCHECK(template_url_service);
- base::CommandLine* cmd_line = base::CommandLine::ForCurrentProcess();
- if (!cmd_line->HasSwitch(switches::kEnableNTPSearchEngineCountryDetection))
- return std::string();
- const TemplateURL* default_provider =
- template_url_service->GetDefaultSearchProvider();
- // It's possible to not have a default provider in the case that the default
- // search engine is defined by policy.
- if (default_provider) {
- bool is_google_search_engine =
- default_provider->GetEngineType(
- template_url_service->search_terms_data()) ==
- SearchEngineType::SEARCH_ENGINE_GOOGLE;
- if (is_google_search_engine) {
- GURL search_url = default_provider->GenerateSearchURL(
- template_url_service->search_terms_data());
- return google_util::GetGoogleCountryCode(search_url);
- }
- }
- return std::string();
- }
- std::string GetVariationCountry() {
- return variations::GetVariationParamValue(kPopularSitesFieldTrialName,
- "country");
- }
- std::string GetVariationVersion() {
- return variations::GetVariationParamValue(kPopularSitesFieldTrialName,
- "version");
- }
- std::string GetVariationDirectory() {
- return variations::GetVariationParamValue(kPopularSitesFieldTrialName,
- "directory");
- }
- PopularSites::SitesVector ParseSiteList(const base::Value::List& list) {
- PopularSites::SitesVector sites;
- for (const base::Value& item_value : list) {
- if (!item_value.is_dict())
- continue;
- const base::Value::Dict& item = item_value.GetDict();
- std::u16string title;
- if (const std::string* ptr = item.FindString("title"))
- title = base::UTF8ToUTF16(*ptr);
- else
- continue;
- std::string url;
- if (const std::string* ptr = item.FindString("url"))
- url = *ptr;
- else
- continue;
- std::string favicon_url;
- if (const std::string* ptr = item.FindString("favicon_url"))
- favicon_url = *ptr;
- std::string large_icon_url;
- if (const std::string* ptr = item.FindString("large_icon_url"))
- large_icon_url = *ptr;
- TileTitleSource title_source = TileTitleSource::UNKNOWN;
- absl::optional<int> title_source_int = item.FindInt("title_source");
- if (!title_source_int) {
- // Only v6 and later have "title_source". Earlier versions use title tags.
- title_source = TileTitleSource::TITLE_TAG;
- } else if (*title_source_int <= static_cast<int>(TileTitleSource::LAST) &&
- *title_source_int >= 0) {
- title_source = static_cast<TileTitleSource>(*title_source_int);
- }
- sites.emplace_back(title, GURL(url), GURL(favicon_url),
- GURL(large_icon_url), title_source);
- absl::optional<int> default_icon_resource =
- item.FindInt("default_icon_resource");
- if (default_icon_resource)
- sites.back().default_icon_resource = *default_icon_resource;
- absl::optional<bool> baked_in = item.FindBool("baked_in");
- if (baked_in.has_value())
- sites.back().baked_in = baked_in.value();
- }
- return sites;
- }
- std::map<SectionType, PopularSites::SitesVector> ParseVersion5(
- const base::Value::List& list) {
- return {{SectionType::PERSONALIZED, ParseSiteList(list)}};
- }
- std::map<SectionType, PopularSites::SitesVector> ParseVersion6OrAbove(
- const base::Value::List& list) {
- // Valid lists would have contained at least the PERSONALIZED section.
- std::map<SectionType, PopularSites::SitesVector> sections = {
- std::make_pair(SectionType::PERSONALIZED, PopularSites::SitesVector{})};
- for (size_t i = 0; i < list.size(); i++) {
- const base::Value::Dict* item_dict = list[i].GetIfDict();
- if (!item_dict) {
- LOG(WARNING) << "Parsed SitesExploration list contained an invalid "
- << "section at position " << i << ".";
- continue;
- }
- int section = item_dict->FindInt("section").value_or(-1);
- if (section < 0 || section > static_cast<int>(SectionType::LAST)) {
- LOG(WARNING) << "Parsed SitesExploration list contained a section with "
- << "invalid ID (" << section << ")";
- continue;
- }
- // Non-personalized site exploration tiles are no longer supported, so
- // ignore all other section types.
- SectionType section_type = static_cast<SectionType>(section);
- if (section_type != SectionType::PERSONALIZED)
- continue;
- const base::Value::List* sites_list = item_dict->FindList("sites");
- if (!sites_list)
- continue;
- sections[section_type] = ParseSiteList(*sites_list);
- }
- return sections;
- }
- std::map<SectionType, PopularSites::SitesVector> ParseSites(
- const base::Value::List& list,
- int version) {
- if (version >= kSitesExplorationStartVersion)
- return ParseVersion6OrAbove(list);
- return ParseVersion5(list);
- }
- #if BUILDFLAG(GOOGLE_CHROME_BRANDING) && \
- (BUILDFLAG(IS_ANDROID) || BUILDFLAG(IS_IOS))
- void SetDefaultResourceForSite(size_t index,
- int resource_id,
- base::Value::List& sites) {
- if (index >= sites.size() || !sites[index].is_dict())
- return;
- sites[index].GetDict().Set("default_icon_resource", resource_id);
- }
- #endif
- // Creates the list of popular sites based on a snapshot available for mobile.
- base::Value DefaultPopularSites() {
- #if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS)
- return base::Value(base::Value::Type::LIST);
- #else
- if (!base::FeatureList::IsEnabled(kPopularSitesBakedInContentFeature))
- return base::Value(base::Value::Type::LIST);
- absl::optional<base::Value> sites = base::JSONReader::Read(
- ui::ResourceBundle::GetSharedInstance().LoadDataResourceString(
- IDR_DEFAULT_POPULAR_SITES_JSON));
- for (base::Value& site : sites->GetList())
- site.GetDict().Set("baked_in", true);
- #if BUILDFLAG(GOOGLE_CHROME_BRANDING)
- size_t index = 0;
- for (int icon_resource :
- {IDR_DEFAULT_POPULAR_SITES_ICON0, IDR_DEFAULT_POPULAR_SITES_ICON1,
- IDR_DEFAULT_POPULAR_SITES_ICON2, IDR_DEFAULT_POPULAR_SITES_ICON3,
- IDR_DEFAULT_POPULAR_SITES_ICON4, IDR_DEFAULT_POPULAR_SITES_ICON5,
- IDR_DEFAULT_POPULAR_SITES_ICON6, IDR_DEFAULT_POPULAR_SITES_ICON7}) {
- SetDefaultResourceForSite(index++, icon_resource, sites->GetList());
- }
- #endif // BUILDFLAG(GOOGLE_CHROME_BRANDING)
- return std::move(sites.value());
- #endif // !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS)
- }
- } // namespace
- PopularSites::Site::Site(const std::u16string& title,
- const GURL& url,
- const GURL& favicon_url,
- const GURL& large_icon_url,
- TileTitleSource title_source)
- : title(title),
- url(url),
- favicon_url(favicon_url),
- large_icon_url(large_icon_url),
- title_source(title_source),
- baked_in(false),
- default_icon_resource(-1) {}
- PopularSites::Site::Site(const Site& other) = default;
- PopularSites::Site::~Site() {}
- PopularSitesImpl::PopularSitesImpl(
- PrefService* prefs,
- const TemplateURLService* template_url_service,
- VariationsService* variations_service,
- scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory)
- : prefs_(prefs),
- template_url_service_(template_url_service),
- variations_(variations_service),
- url_loader_factory_(std::move(url_loader_factory)),
- is_fallback_(false),
- sections_(
- ParseSites(prefs->GetValueList(prefs::kPopularSitesJsonPref),
- prefs_->GetInteger(prefs::kPopularSitesVersionPref))) {}
- PopularSitesImpl::~PopularSitesImpl() {}
- bool PopularSitesImpl::MaybeStartFetch(bool force_download,
- FinishedCallback callback) {
- DCHECK(!callback_);
- callback_ = std::move(callback);
- const base::Time last_download_time = base::Time::FromInternalValue(
- prefs_->GetInt64(prefs::kPopularSitesLastDownloadPref));
- const base::TimeDelta time_since_last_download =
- base::Time::Now() - last_download_time;
- const base::TimeDelta redownload_interval =
- base::Hours(kPopularSitesRedownloadIntervalHours);
- const bool download_time_is_future = base::Time::Now() < last_download_time;
- pending_url_ = GetURLToFetch();
- const bool url_changed =
- pending_url_.spec() != prefs_->GetString(prefs::kPopularSitesURLPref);
- // Download forced, or we need to download a new file.
- if (force_download || download_time_is_future ||
- (time_since_last_download > redownload_interval) || url_changed) {
- FetchPopularSites();
- return true;
- }
- return false;
- }
- const std::map<SectionType, PopularSitesImpl::SitesVector>&
- PopularSitesImpl::sections() const {
- return sections_;
- }
- GURL PopularSitesImpl::GetLastURLFetched() const {
- return GURL(prefs_->GetString(prefs::kPopularSitesURLPref));
- }
- GURL PopularSitesImpl::GetURLToFetch() {
- const std::string directory = GetDirectoryToFetch();
- const std::string country = GetCountryToFetch();
- const std::string version = GetVersionToFetch();
- if (!base::StringToInt(version, &version_in_pending_url_)) {
- // Parses the leading digits as version. Defaults to 0 if that failed.
- if (version_in_pending_url_ <= 0) {
- bool success = base::StringToInt(kPopularSitesDefaultVersion,
- &version_in_pending_url_);
- DLOG(WARNING) << "The set version \"" << version << "\" does not start "
- << "with a valid version number. Default version was used "
- << "instead (" << kPopularSitesDefaultVersion << ").";
- DCHECK(success);
- }
- }
- const GURL override_url =
- GURL(prefs_->GetString(prefs::kPopularSitesOverrideURL));
- return override_url.is_valid()
- ? override_url
- : GetPopularSitesURL(directory, country, version);
- }
- std::string PopularSitesImpl::GetDirectoryToFetch() {
- std::string directory =
- prefs_->GetString(prefs::kPopularSitesOverrideDirectory);
- if (directory.empty())
- directory = GetVariationDirectory();
- if (directory.empty())
- directory = kPopularSitesDefaultDirectory;
- return directory;
- }
- // Determine the country code to use. In order of precedence:
- // - The explicit "override country" pref set by the user.
- // - The country code from the field trial config (variation parameter).
- // - The Google country code if Google is the default search engine (and the
- // "--enable-ntp-search-engine-country-detection" switch is present).
- // - The country provided by the VariationsService.
- // - A default fallback.
- std::string PopularSitesImpl::GetCountryToFetch() {
- std::string country_code =
- prefs_->GetString(prefs::kPopularSitesOverrideCountry);
- if (country_code.empty())
- country_code = GetVariationCountry();
- if (country_code.empty())
- country_code = GetDefaultSearchEngineCountryCode(template_url_service_);
- if (country_code.empty() && variations_)
- country_code = variations_->GetStoredPermanentCountry();
- #if BUILDFLAG(IS_IOS)
- if (country_code.empty())
- country_code = GetDeviceCountryCode();
- #endif
- if (country_code.empty())
- country_code = kPopularSitesDefaultCountryCode;
- return base::ToUpperASCII(country_code);
- }
- // Determine the version to use. In order of precedence:
- // - The explicit "override version" pref set by the user.
- // - The version from the field trial config (variation parameter).
- // - A default fallback.
- std::string PopularSitesImpl::GetVersionToFetch() {
- std::string version = prefs_->GetString(prefs::kPopularSitesOverrideVersion);
- if (version.empty())
- version = GetVariationVersion();
- if (version.empty())
- version = kPopularSitesDefaultVersion;
- return version;
- }
- const base::ListValue* PopularSitesImpl::GetCachedJson() {
- return &base::Value::AsListValue(
- *prefs_->GetList(prefs::kPopularSitesJsonPref));
- }
- // static
- void PopularSitesImpl::RegisterProfilePrefs(
- user_prefs::PrefRegistrySyncable* user_prefs) {
- user_prefs->RegisterStringPref(prefs::kPopularSitesOverrideURL,
- std::string());
- user_prefs->RegisterStringPref(prefs::kPopularSitesOverrideDirectory,
- std::string());
- user_prefs->RegisterStringPref(prefs::kPopularSitesOverrideCountry,
- std::string());
- user_prefs->RegisterStringPref(prefs::kPopularSitesOverrideVersion,
- std::string());
- user_prefs->RegisterInt64Pref(prefs::kPopularSitesLastDownloadPref, 0);
- user_prefs->RegisterStringPref(prefs::kPopularSitesURLPref, std::string());
- user_prefs->RegisterListPref(prefs::kPopularSitesJsonPref,
- DefaultPopularSites());
- int version;
- base::StringToInt(kPopularSitesDefaultVersion, &version);
- user_prefs->RegisterIntegerPref(prefs::kPopularSitesVersionPref, version);
- }
- void PopularSitesImpl::FetchPopularSites() {
- net::NetworkTrafficAnnotationTag traffic_annotation =
- net::DefineNetworkTrafficAnnotation("popular_sites_fetch", R"(
- semantics {
- sender: "Popular Sites New Tab Fetch"
- description:
- "Google Chrome may display a list of regionally-popular web sites "
- "on the New Tab Page. This service fetches the list of these sites."
- trigger:
- "Once per day, unless no popular web sites are required because "
- "the New Tab Page is filled with suggestions based on the user's "
- "browsing history."
- data: "A two letter country code based on the user's location."
- destination: GOOGLE_OWNED_SERVICE
- }
- policy {
- cookies_allowed: NO
- setting: "This feature cannot be disabled in settings."
- policy_exception_justification:
- "Not implemented, considered not useful."
- })");
- auto resource_request = std::make_unique<network::ResourceRequest>();
- resource_request->url = pending_url_;
- resource_request->credentials_mode = network::mojom::CredentialsMode::kOmit;
- simple_url_loader_ = network::SimpleURLLoader::Create(
- std::move(resource_request), traffic_annotation);
- simple_url_loader_->SetRetryOptions(
- 1, network::SimpleURLLoader::RETRY_ON_NETWORK_CHANGE);
- simple_url_loader_->DownloadToStringOfUnboundedSizeUntilCrashAndDie(
- url_loader_factory_.get(),
- base::BindOnce(&PopularSitesImpl::OnSimpleLoaderComplete,
- base::Unretained(this)));
- }
- void PopularSitesImpl::OnSimpleLoaderComplete(
- std::unique_ptr<std::string> response_body) {
- simple_url_loader_.reset();
- if (!response_body) {
- OnDownloadFailed();
- return;
- }
- data_decoder::DataDecoder::ParseJsonIsolated(
- *response_body, base::BindOnce(&PopularSitesImpl::OnJsonParsed,
- weak_ptr_factory_.GetWeakPtr()));
- }
- void PopularSitesImpl::OnJsonParsed(
- data_decoder::DataDecoder::ValueOrError result) {
- if (!result.has_value()) {
- DLOG(WARNING) << "JSON parsing failed: " << result.error();
- OnDownloadFailed();
- return;
- }
- base::Value list = std::move(*result);
- if (!list.is_list()) {
- DLOG(WARNING) << "JSON is not a list";
- OnDownloadFailed();
- return;
- }
- sections_ = ParseSites(list.GetList(), version_in_pending_url_);
- prefs_->SetList(prefs::kPopularSitesJsonPref, std::move(list.GetList()));
- prefs_->SetInt64(prefs::kPopularSitesLastDownloadPref,
- base::Time::Now().ToInternalValue());
- prefs_->SetInteger(prefs::kPopularSitesVersionPref, version_in_pending_url_);
- prefs_->SetString(prefs::kPopularSitesURLPref, pending_url_.spec());
- std::move(callback_).Run(true);
- }
- void PopularSitesImpl::OnDownloadFailed() {
- if (!is_fallback_) {
- DLOG(WARNING) << "Download country site list failed";
- is_fallback_ = true;
- pending_url_ = GetPopularSitesURL(kPopularSitesDefaultDirectory,
- kPopularSitesDefaultCountryCode,
- kPopularSitesDefaultVersion);
- FetchPopularSites();
- } else {
- DLOG(WARNING) << "Download fallback site list failed";
- std::move(callback_).Run(false);
- }
- }
- } // namespace ntp_tiles
|