popular_sites_impl.cc 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514
  1. // Copyright 2015 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/ntp_tiles/popular_sites_impl.h"
  5. #include <stddef.h>
  6. #include <map>
  7. #include <memory>
  8. #include <utility>
  9. #include "base/bind.h"
  10. #include "base/command_line.h"
  11. #include "base/feature_list.h"
  12. #include "base/strings/string_number_conversions.h"
  13. #include "base/strings/string_util.h"
  14. #include "base/strings/stringprintf.h"
  15. #include "base/strings/utf_string_conversions.h"
  16. #include "base/time/time.h"
  17. #include "base/values.h"
  18. #include "build/branding_buildflags.h"
  19. #include "build/build_config.h"
  20. #include "components/google/core/common/google_util.h"
  21. #include "components/ntp_tiles/features.h"
  22. #include "components/ntp_tiles/pref_names.h"
  23. #include "components/ntp_tiles/switches.h"
  24. #include "components/pref_registry/pref_registry_syncable.h"
  25. #include "components/prefs/pref_service.h"
  26. #include "components/search_engines/search_engine_type.h"
  27. #include "components/search_engines/template_url_service.h"
  28. #include "components/variations/service/variations_service.h"
  29. #include "components/variations/variations_associated_data.h"
  30. #include "net/base/load_flags.h"
  31. #include "net/http/http_status_code.h"
  32. #include "net/traffic_annotation/network_traffic_annotation.h"
  33. #include "services/data_decoder/public/cpp/data_decoder.h"
  34. #include "services/network/public/cpp/resource_request.h"
  35. #include "services/network/public/cpp/shared_url_loader_factory.h"
  36. #include "services/network/public/cpp/simple_url_loader.h"
  37. #if BUILDFLAG(IS_ANDROID) || BUILDFLAG(IS_IOS)
  38. #include "base/json/json_reader.h"
  39. #include "components/grit/components_resources.h"
  40. #include "ui/base/resource/resource_bundle.h"
  41. #endif
  42. #if BUILDFLAG(IS_IOS)
  43. #include "components/ntp_tiles/country_code_ios.h"
  44. #endif
  45. using variations::VariationsService;
  46. namespace ntp_tiles {
  47. namespace {
  48. const char kPopularSitesURLFormat[] =
  49. "https://www.gstatic.com/%ssuggested_sites_%s_%s.json";
  50. const char kPopularSitesDefaultDirectory[] = "chrome/ntp/";
  51. const char kPopularSitesDefaultCountryCode[] = "DEFAULT";
  52. const char kPopularSitesDefaultVersion[] = "5";
  53. const int kSitesExplorationStartVersion = 6;
  54. const int kPopularSitesRedownloadIntervalHours = 24;
  55. GURL GetPopularSitesURL(const std::string& directory,
  56. const std::string& country,
  57. const std::string& version) {
  58. return GURL(base::StringPrintf(kPopularSitesURLFormat, directory.c_str(),
  59. country.c_str(), version.c_str()));
  60. }
  61. // Extract the country from the default search engine if the default search
  62. // engine is Google.
  63. std::string GetDefaultSearchEngineCountryCode(
  64. const TemplateURLService* template_url_service) {
  65. DCHECK(template_url_service);
  66. base::CommandLine* cmd_line = base::CommandLine::ForCurrentProcess();
  67. if (!cmd_line->HasSwitch(switches::kEnableNTPSearchEngineCountryDetection))
  68. return std::string();
  69. const TemplateURL* default_provider =
  70. template_url_service->GetDefaultSearchProvider();
  71. // It's possible to not have a default provider in the case that the default
  72. // search engine is defined by policy.
  73. if (default_provider) {
  74. bool is_google_search_engine =
  75. default_provider->GetEngineType(
  76. template_url_service->search_terms_data()) ==
  77. SearchEngineType::SEARCH_ENGINE_GOOGLE;
  78. if (is_google_search_engine) {
  79. GURL search_url = default_provider->GenerateSearchURL(
  80. template_url_service->search_terms_data());
  81. return google_util::GetGoogleCountryCode(search_url);
  82. }
  83. }
  84. return std::string();
  85. }
  86. std::string GetVariationCountry() {
  87. return variations::GetVariationParamValue(kPopularSitesFieldTrialName,
  88. "country");
  89. }
  90. std::string GetVariationVersion() {
  91. return variations::GetVariationParamValue(kPopularSitesFieldTrialName,
  92. "version");
  93. }
  94. std::string GetVariationDirectory() {
  95. return variations::GetVariationParamValue(kPopularSitesFieldTrialName,
  96. "directory");
  97. }
  98. PopularSites::SitesVector ParseSiteList(const base::Value::List& list) {
  99. PopularSites::SitesVector sites;
  100. for (const base::Value& item_value : list) {
  101. if (!item_value.is_dict())
  102. continue;
  103. const base::Value::Dict& item = item_value.GetDict();
  104. std::u16string title;
  105. if (const std::string* ptr = item.FindString("title"))
  106. title = base::UTF8ToUTF16(*ptr);
  107. else
  108. continue;
  109. std::string url;
  110. if (const std::string* ptr = item.FindString("url"))
  111. url = *ptr;
  112. else
  113. continue;
  114. std::string favicon_url;
  115. if (const std::string* ptr = item.FindString("favicon_url"))
  116. favicon_url = *ptr;
  117. std::string large_icon_url;
  118. if (const std::string* ptr = item.FindString("large_icon_url"))
  119. large_icon_url = *ptr;
  120. TileTitleSource title_source = TileTitleSource::UNKNOWN;
  121. absl::optional<int> title_source_int = item.FindInt("title_source");
  122. if (!title_source_int) {
  123. // Only v6 and later have "title_source". Earlier versions use title tags.
  124. title_source = TileTitleSource::TITLE_TAG;
  125. } else if (*title_source_int <= static_cast<int>(TileTitleSource::LAST) &&
  126. *title_source_int >= 0) {
  127. title_source = static_cast<TileTitleSource>(*title_source_int);
  128. }
  129. sites.emplace_back(title, GURL(url), GURL(favicon_url),
  130. GURL(large_icon_url), title_source);
  131. absl::optional<int> default_icon_resource =
  132. item.FindInt("default_icon_resource");
  133. if (default_icon_resource)
  134. sites.back().default_icon_resource = *default_icon_resource;
  135. absl::optional<bool> baked_in = item.FindBool("baked_in");
  136. if (baked_in.has_value())
  137. sites.back().baked_in = baked_in.value();
  138. }
  139. return sites;
  140. }
  141. std::map<SectionType, PopularSites::SitesVector> ParseVersion5(
  142. const base::Value::List& list) {
  143. return {{SectionType::PERSONALIZED, ParseSiteList(list)}};
  144. }
  145. std::map<SectionType, PopularSites::SitesVector> ParseVersion6OrAbove(
  146. const base::Value::List& list) {
  147. // Valid lists would have contained at least the PERSONALIZED section.
  148. std::map<SectionType, PopularSites::SitesVector> sections = {
  149. std::make_pair(SectionType::PERSONALIZED, PopularSites::SitesVector{})};
  150. for (size_t i = 0; i < list.size(); i++) {
  151. const base::Value::Dict* item_dict = list[i].GetIfDict();
  152. if (!item_dict) {
  153. LOG(WARNING) << "Parsed SitesExploration list contained an invalid "
  154. << "section at position " << i << ".";
  155. continue;
  156. }
  157. int section = item_dict->FindInt("section").value_or(-1);
  158. if (section < 0 || section > static_cast<int>(SectionType::LAST)) {
  159. LOG(WARNING) << "Parsed SitesExploration list contained a section with "
  160. << "invalid ID (" << section << ")";
  161. continue;
  162. }
  163. // Non-personalized site exploration tiles are no longer supported, so
  164. // ignore all other section types.
  165. SectionType section_type = static_cast<SectionType>(section);
  166. if (section_type != SectionType::PERSONALIZED)
  167. continue;
  168. const base::Value::List* sites_list = item_dict->FindList("sites");
  169. if (!sites_list)
  170. continue;
  171. sections[section_type] = ParseSiteList(*sites_list);
  172. }
  173. return sections;
  174. }
  175. std::map<SectionType, PopularSites::SitesVector> ParseSites(
  176. const base::Value::List& list,
  177. int version) {
  178. if (version >= kSitesExplorationStartVersion)
  179. return ParseVersion6OrAbove(list);
  180. return ParseVersion5(list);
  181. }
  182. #if BUILDFLAG(GOOGLE_CHROME_BRANDING) && \
  183. (BUILDFLAG(IS_ANDROID) || BUILDFLAG(IS_IOS))
  184. void SetDefaultResourceForSite(size_t index,
  185. int resource_id,
  186. base::Value::List& sites) {
  187. if (index >= sites.size() || !sites[index].is_dict())
  188. return;
  189. sites[index].GetDict().Set("default_icon_resource", resource_id);
  190. }
  191. #endif
  192. // Creates the list of popular sites based on a snapshot available for mobile.
  193. base::Value DefaultPopularSites() {
  194. #if !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS)
  195. return base::Value(base::Value::Type::LIST);
  196. #else
  197. if (!base::FeatureList::IsEnabled(kPopularSitesBakedInContentFeature))
  198. return base::Value(base::Value::Type::LIST);
  199. absl::optional<base::Value> sites = base::JSONReader::Read(
  200. ui::ResourceBundle::GetSharedInstance().LoadDataResourceString(
  201. IDR_DEFAULT_POPULAR_SITES_JSON));
  202. for (base::Value& site : sites->GetList())
  203. site.GetDict().Set("baked_in", true);
  204. #if BUILDFLAG(GOOGLE_CHROME_BRANDING)
  205. size_t index = 0;
  206. for (int icon_resource :
  207. {IDR_DEFAULT_POPULAR_SITES_ICON0, IDR_DEFAULT_POPULAR_SITES_ICON1,
  208. IDR_DEFAULT_POPULAR_SITES_ICON2, IDR_DEFAULT_POPULAR_SITES_ICON3,
  209. IDR_DEFAULT_POPULAR_SITES_ICON4, IDR_DEFAULT_POPULAR_SITES_ICON5,
  210. IDR_DEFAULT_POPULAR_SITES_ICON6, IDR_DEFAULT_POPULAR_SITES_ICON7}) {
  211. SetDefaultResourceForSite(index++, icon_resource, sites->GetList());
  212. }
  213. #endif // BUILDFLAG(GOOGLE_CHROME_BRANDING)
  214. return std::move(sites.value());
  215. #endif // !BUILDFLAG(IS_ANDROID) && !BUILDFLAG(IS_IOS)
  216. }
  217. } // namespace
  218. PopularSites::Site::Site(const std::u16string& title,
  219. const GURL& url,
  220. const GURL& favicon_url,
  221. const GURL& large_icon_url,
  222. TileTitleSource title_source)
  223. : title(title),
  224. url(url),
  225. favicon_url(favicon_url),
  226. large_icon_url(large_icon_url),
  227. title_source(title_source),
  228. baked_in(false),
  229. default_icon_resource(-1) {}
  230. PopularSites::Site::Site(const Site& other) = default;
  231. PopularSites::Site::~Site() {}
  232. PopularSitesImpl::PopularSitesImpl(
  233. PrefService* prefs,
  234. const TemplateURLService* template_url_service,
  235. VariationsService* variations_service,
  236. scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory)
  237. : prefs_(prefs),
  238. template_url_service_(template_url_service),
  239. variations_(variations_service),
  240. url_loader_factory_(std::move(url_loader_factory)),
  241. is_fallback_(false),
  242. sections_(
  243. ParseSites(prefs->GetValueList(prefs::kPopularSitesJsonPref),
  244. prefs_->GetInteger(prefs::kPopularSitesVersionPref))) {}
  245. PopularSitesImpl::~PopularSitesImpl() {}
  246. bool PopularSitesImpl::MaybeStartFetch(bool force_download,
  247. FinishedCallback callback) {
  248. DCHECK(!callback_);
  249. callback_ = std::move(callback);
  250. const base::Time last_download_time = base::Time::FromInternalValue(
  251. prefs_->GetInt64(prefs::kPopularSitesLastDownloadPref));
  252. const base::TimeDelta time_since_last_download =
  253. base::Time::Now() - last_download_time;
  254. const base::TimeDelta redownload_interval =
  255. base::Hours(kPopularSitesRedownloadIntervalHours);
  256. const bool download_time_is_future = base::Time::Now() < last_download_time;
  257. pending_url_ = GetURLToFetch();
  258. const bool url_changed =
  259. pending_url_.spec() != prefs_->GetString(prefs::kPopularSitesURLPref);
  260. // Download forced, or we need to download a new file.
  261. if (force_download || download_time_is_future ||
  262. (time_since_last_download > redownload_interval) || url_changed) {
  263. FetchPopularSites();
  264. return true;
  265. }
  266. return false;
  267. }
  268. const std::map<SectionType, PopularSitesImpl::SitesVector>&
  269. PopularSitesImpl::sections() const {
  270. return sections_;
  271. }
  272. GURL PopularSitesImpl::GetLastURLFetched() const {
  273. return GURL(prefs_->GetString(prefs::kPopularSitesURLPref));
  274. }
  275. GURL PopularSitesImpl::GetURLToFetch() {
  276. const std::string directory = GetDirectoryToFetch();
  277. const std::string country = GetCountryToFetch();
  278. const std::string version = GetVersionToFetch();
  279. if (!base::StringToInt(version, &version_in_pending_url_)) {
  280. // Parses the leading digits as version. Defaults to 0 if that failed.
  281. if (version_in_pending_url_ <= 0) {
  282. bool success = base::StringToInt(kPopularSitesDefaultVersion,
  283. &version_in_pending_url_);
  284. DLOG(WARNING) << "The set version \"" << version << "\" does not start "
  285. << "with a valid version number. Default version was used "
  286. << "instead (" << kPopularSitesDefaultVersion << ").";
  287. DCHECK(success);
  288. }
  289. }
  290. const GURL override_url =
  291. GURL(prefs_->GetString(prefs::kPopularSitesOverrideURL));
  292. return override_url.is_valid()
  293. ? override_url
  294. : GetPopularSitesURL(directory, country, version);
  295. }
  296. std::string PopularSitesImpl::GetDirectoryToFetch() {
  297. std::string directory =
  298. prefs_->GetString(prefs::kPopularSitesOverrideDirectory);
  299. if (directory.empty())
  300. directory = GetVariationDirectory();
  301. if (directory.empty())
  302. directory = kPopularSitesDefaultDirectory;
  303. return directory;
  304. }
  305. // Determine the country code to use. In order of precedence:
  306. // - The explicit "override country" pref set by the user.
  307. // - The country code from the field trial config (variation parameter).
  308. // - The Google country code if Google is the default search engine (and the
  309. // "--enable-ntp-search-engine-country-detection" switch is present).
  310. // - The country provided by the VariationsService.
  311. // - A default fallback.
  312. std::string PopularSitesImpl::GetCountryToFetch() {
  313. std::string country_code =
  314. prefs_->GetString(prefs::kPopularSitesOverrideCountry);
  315. if (country_code.empty())
  316. country_code = GetVariationCountry();
  317. if (country_code.empty())
  318. country_code = GetDefaultSearchEngineCountryCode(template_url_service_);
  319. if (country_code.empty() && variations_)
  320. country_code = variations_->GetStoredPermanentCountry();
  321. #if BUILDFLAG(IS_IOS)
  322. if (country_code.empty())
  323. country_code = GetDeviceCountryCode();
  324. #endif
  325. if (country_code.empty())
  326. country_code = kPopularSitesDefaultCountryCode;
  327. return base::ToUpperASCII(country_code);
  328. }
  329. // Determine the version to use. In order of precedence:
  330. // - The explicit "override version" pref set by the user.
  331. // - The version from the field trial config (variation parameter).
  332. // - A default fallback.
  333. std::string PopularSitesImpl::GetVersionToFetch() {
  334. std::string version = prefs_->GetString(prefs::kPopularSitesOverrideVersion);
  335. if (version.empty())
  336. version = GetVariationVersion();
  337. if (version.empty())
  338. version = kPopularSitesDefaultVersion;
  339. return version;
  340. }
  341. const base::ListValue* PopularSitesImpl::GetCachedJson() {
  342. return &base::Value::AsListValue(
  343. *prefs_->GetList(prefs::kPopularSitesJsonPref));
  344. }
  345. // static
  346. void PopularSitesImpl::RegisterProfilePrefs(
  347. user_prefs::PrefRegistrySyncable* user_prefs) {
  348. user_prefs->RegisterStringPref(prefs::kPopularSitesOverrideURL,
  349. std::string());
  350. user_prefs->RegisterStringPref(prefs::kPopularSitesOverrideDirectory,
  351. std::string());
  352. user_prefs->RegisterStringPref(prefs::kPopularSitesOverrideCountry,
  353. std::string());
  354. user_prefs->RegisterStringPref(prefs::kPopularSitesOverrideVersion,
  355. std::string());
  356. user_prefs->RegisterInt64Pref(prefs::kPopularSitesLastDownloadPref, 0);
  357. user_prefs->RegisterStringPref(prefs::kPopularSitesURLPref, std::string());
  358. user_prefs->RegisterListPref(prefs::kPopularSitesJsonPref,
  359. DefaultPopularSites());
  360. int version;
  361. base::StringToInt(kPopularSitesDefaultVersion, &version);
  362. user_prefs->RegisterIntegerPref(prefs::kPopularSitesVersionPref, version);
  363. }
  364. void PopularSitesImpl::FetchPopularSites() {
  365. net::NetworkTrafficAnnotationTag traffic_annotation =
  366. net::DefineNetworkTrafficAnnotation("popular_sites_fetch", R"(
  367. semantics {
  368. sender: "Popular Sites New Tab Fetch"
  369. description:
  370. "Google Chrome may display a list of regionally-popular web sites "
  371. "on the New Tab Page. This service fetches the list of these sites."
  372. trigger:
  373. "Once per day, unless no popular web sites are required because "
  374. "the New Tab Page is filled with suggestions based on the user's "
  375. "browsing history."
  376. data: "A two letter country code based on the user's location."
  377. destination: GOOGLE_OWNED_SERVICE
  378. }
  379. policy {
  380. cookies_allowed: NO
  381. setting: "This feature cannot be disabled in settings."
  382. policy_exception_justification:
  383. "Not implemented, considered not useful."
  384. })");
  385. auto resource_request = std::make_unique<network::ResourceRequest>();
  386. resource_request->url = pending_url_;
  387. resource_request->credentials_mode = network::mojom::CredentialsMode::kOmit;
  388. simple_url_loader_ = network::SimpleURLLoader::Create(
  389. std::move(resource_request), traffic_annotation);
  390. simple_url_loader_->SetRetryOptions(
  391. 1, network::SimpleURLLoader::RETRY_ON_NETWORK_CHANGE);
  392. simple_url_loader_->DownloadToStringOfUnboundedSizeUntilCrashAndDie(
  393. url_loader_factory_.get(),
  394. base::BindOnce(&PopularSitesImpl::OnSimpleLoaderComplete,
  395. base::Unretained(this)));
  396. }
  397. void PopularSitesImpl::OnSimpleLoaderComplete(
  398. std::unique_ptr<std::string> response_body) {
  399. simple_url_loader_.reset();
  400. if (!response_body) {
  401. OnDownloadFailed();
  402. return;
  403. }
  404. data_decoder::DataDecoder::ParseJsonIsolated(
  405. *response_body, base::BindOnce(&PopularSitesImpl::OnJsonParsed,
  406. weak_ptr_factory_.GetWeakPtr()));
  407. }
  408. void PopularSitesImpl::OnJsonParsed(
  409. data_decoder::DataDecoder::ValueOrError result) {
  410. if (!result.has_value()) {
  411. DLOG(WARNING) << "JSON parsing failed: " << result.error();
  412. OnDownloadFailed();
  413. return;
  414. }
  415. base::Value list = std::move(*result);
  416. if (!list.is_list()) {
  417. DLOG(WARNING) << "JSON is not a list";
  418. OnDownloadFailed();
  419. return;
  420. }
  421. sections_ = ParseSites(list.GetList(), version_in_pending_url_);
  422. prefs_->SetList(prefs::kPopularSitesJsonPref, std::move(list.GetList()));
  423. prefs_->SetInt64(prefs::kPopularSitesLastDownloadPref,
  424. base::Time::Now().ToInternalValue());
  425. prefs_->SetInteger(prefs::kPopularSitesVersionPref, version_in_pending_url_);
  426. prefs_->SetString(prefs::kPopularSitesURLPref, pending_url_.spec());
  427. std::move(callback_).Run(true);
  428. }
  429. void PopularSitesImpl::OnDownloadFailed() {
  430. if (!is_fallback_) {
  431. DLOG(WARNING) << "Download country site list failed";
  432. is_fallback_ = true;
  433. pending_url_ = GetPopularSitesURL(kPopularSitesDefaultDirectory,
  434. kPopularSitesDefaultCountryCode,
  435. kPopularSitesDefaultVersion);
  436. FetchPopularSites();
  437. } else {
  438. DLOG(WARNING) << "Download fallback site list failed";
  439. std::move(callback_).Run(false);
  440. }
  441. }
  442. } // namespace ntp_tiles