hints_fetcher_unittest.cc 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701
  1. // Copyright 2019 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/optimization_guide/core/hints_fetcher.h"
  5. #include <memory>
  6. #include "base/callback.h"
  7. #include "base/memory/raw_ptr.h"
  8. #include "base/memory/scoped_refptr.h"
  9. #include "base/run_loop.h"
  10. #include "base/strings/string_number_conversions.h"
  11. #include "base/test/metrics/histogram_tester.h"
  12. #include "base/test/scoped_feature_list.h"
  13. #include "base/test/simple_test_clock.h"
  14. #include "base/test/task_environment.h"
  15. #include "components/optimization_guide/core/hints_processing_util.h"
  16. #include "components/optimization_guide/core/optimization_guide_features.h"
  17. #include "components/optimization_guide/core/optimization_guide_prefs.h"
  18. #include "components/prefs/pref_service.h"
  19. #include "components/prefs/scoped_user_pref_update.h"
  20. #include "components/prefs/testing_pref_service.h"
  21. #include "components/variations/scoped_variations_ids_provider.h"
  22. #include "net/base/url_util.h"
  23. #include "services/network/public/cpp/shared_url_loader_factory.h"
  24. #include "services/network/public/cpp/weak_wrapper_shared_url_loader_factory.h"
  25. #include "services/network/test/test_url_loader_factory.h"
  26. #include "testing/gtest/include/gtest/gtest.h"
  27. #include "third_party/abseil-cpp/absl/types/optional.h"
  28. namespace optimization_guide {
  29. constexpr char optimization_guide_service_url[] = "https://hintsserver.com/";
  30. class HintsFetcherTest : public testing::Test,
  31. public testing::WithParamInterface<bool> {
  32. public:
  33. HintsFetcherTest()
  34. : task_environment_(base::test::TaskEnvironment::MainThreadType::UI,
  35. base::test::TaskEnvironment::TimeSource::MOCK_TIME),
  36. shared_url_loader_factory_(
  37. base::MakeRefCounted<network::WeakWrapperSharedURLLoaderFactory>(
  38. &test_url_loader_factory_)) {
  39. scoped_list_.InitWithFeaturesAndParameters(
  40. {{features::kRemoteOptimizationGuideFetching, {}},
  41. {features::kOptimizationHints,
  42. {{"persist_hints_to_disk",
  43. ShouldPersistHintsToDisk() ? "true" : "false"}}}},
  44. {});
  45. pref_service_ = std::make_unique<TestingPrefServiceSimple>();
  46. prefs::RegisterProfilePrefs(pref_service_->registry());
  47. hints_fetcher_ = std::make_unique<HintsFetcher>(
  48. shared_url_loader_factory_, GURL(optimization_guide_service_url),
  49. pref_service_.get(), /*optimization_guide_logger=*/nullptr);
  50. hints_fetcher_->SetTimeClockForTesting(task_environment_.GetMockClock());
  51. }
  52. HintsFetcherTest(const HintsFetcherTest&) = delete;
  53. HintsFetcherTest& operator=(const HintsFetcherTest&) = delete;
  54. ~HintsFetcherTest() override = default;
  55. void OnHintsFetched(absl::optional<std::unique_ptr<proto::GetHintsResponse>>
  56. get_hints_response) {
  57. if (get_hints_response)
  58. hints_fetched_ = true;
  59. }
  60. bool hints_fetched() const { return hints_fetched_; }
  61. // Updates the pref so that hints for each of the host in |hosts| are set to
  62. // expire at |host_invalid_time|.
  63. void SeedCoveredHosts(const std::vector<std::string>& hosts,
  64. base::Time host_invalid_time) {
  65. DictionaryPrefUpdate hosts_fetched(
  66. pref_service(), prefs::kHintsFetcherHostsSuccessfullyFetched);
  67. for (const std::string& host : hosts) {
  68. hosts_fetched->SetDoubleKey(
  69. HashHostForDictionary(host),
  70. host_invalid_time.ToDeltaSinceWindowsEpoch().InSecondsF());
  71. }
  72. }
  73. PrefService* pref_service() { return pref_service_.get(); }
  74. const base::Clock* GetMockClock() const {
  75. return task_environment_.GetMockClock();
  76. }
  77. void SetTimeClockForTesting(base::Clock* clock) {
  78. hints_fetcher_->SetTimeClockForTesting(clock);
  79. }
  80. bool ShouldPersistHintsToDisk() const { return GetParam(); }
  81. protected:
  82. bool FetchHints(const std::vector<std::string>& hosts,
  83. const std::vector<GURL>& urls) {
  84. bool status = hints_fetcher_->FetchOptimizationGuideServiceHints(
  85. hosts, urls, {optimization_guide::proto::NOSCRIPT},
  86. optimization_guide::proto::CONTEXT_BATCH_UPDATE_ACTIVE_TABS, "en-US",
  87. base::BindOnce(&HintsFetcherTest::OnHintsFetched,
  88. base::Unretained(this)));
  89. RunUntilIdle();
  90. return status;
  91. }
  92. // Return a 200 response with provided content to any pending requests.
  93. bool SimulateResponse(const std::string& content,
  94. net::HttpStatusCode http_status) {
  95. return test_url_loader_factory_.SimulateResponseForPendingRequest(
  96. optimization_guide_service_url, content, http_status,
  97. network::TestURLLoaderFactory::kUrlMatchPrefix);
  98. }
  99. void VerifyHasPendingFetchRequests() {
  100. EXPECT_GE(test_url_loader_factory_.NumPending(), 1);
  101. std::string key_value;
  102. for (const auto& pending_request :
  103. *test_url_loader_factory_.pending_requests()) {
  104. EXPECT_EQ(pending_request.request.method, "POST");
  105. EXPECT_TRUE(net::GetValueForKeyInQuery(pending_request.request.url, "key",
  106. &key_value));
  107. EXPECT_EQ(pending_request.request.request_body->elements()->size(), 1u);
  108. auto& element =
  109. pending_request.request.request_body->elements_mutable()->front();
  110. if (element.type() != network::DataElement::Tag::kBytes) {
  111. ADD_FAILURE() << "network::DataElement type mismatch";
  112. return;
  113. }
  114. last_request_body_ =
  115. std::string(element.As<network::DataElementBytes>().AsStringPiece());
  116. }
  117. }
  118. bool WasHostCoveredByFetch(const std::string& host) {
  119. return HintsFetcher::WasHostCoveredByFetch(pref_service(), host,
  120. GetMockClock());
  121. }
  122. void ResetHintsFetcher() { hints_fetcher_.reset(); }
  123. std::string last_request_body() const { return last_request_body_; }
  124. private:
  125. void RunUntilIdle() {
  126. task_environment_.RunUntilIdle();
  127. base::RunLoop().RunUntilIdle();
  128. }
  129. variations::ScopedVariationsIdsProvider scoped_variations_ids_provider_{
  130. variations::VariationsIdsProvider::Mode::kUseSignedInState};
  131. bool hints_fetched_ = false;
  132. base::test::ScopedFeatureList scoped_list_;
  133. base::test::TaskEnvironment task_environment_;
  134. std::unique_ptr<HintsFetcher> hints_fetcher_;
  135. std::unique_ptr<TestingPrefServiceSimple> pref_service_;
  136. scoped_refptr<network::SharedURLLoaderFactory> shared_url_loader_factory_;
  137. network::TestURLLoaderFactory test_url_loader_factory_;
  138. std::string last_request_body_;
  139. };
  140. INSTANTIATE_TEST_SUITE_P(WithPersistentStore,
  141. HintsFetcherTest,
  142. testing::Values(true, false));
  143. TEST_P(HintsFetcherTest,
  144. FetchOptimizationGuideServiceHintsLogsHistogramUponExiting) {
  145. base::HistogramTester histogram_tester;
  146. EXPECT_TRUE(FetchHints({"foo.com"}, {} /* urls */));
  147. VerifyHasPendingFetchRequests();
  148. ResetHintsFetcher();
  149. histogram_tester.ExpectUniqueSample(
  150. "OptimizationGuide.HintsFetcher.GetHintsRequest.ActiveRequestCanceled."
  151. "BatchUpdateActiveTabs",
  152. 1, 1);
  153. }
  154. TEST_P(HintsFetcherTest, FetchOptimizationGuideServiceHints) {
  155. base::HistogramTester histogram_tester;
  156. std::string response_content;
  157. EXPECT_TRUE(FetchHints({"foo.com"}, {} /* urls */));
  158. VerifyHasPendingFetchRequests();
  159. EXPECT_TRUE(SimulateResponse(response_content, net::HTTP_OK));
  160. EXPECT_TRUE(hints_fetched());
  161. histogram_tester.ExpectTotalCount(
  162. "OptimizationGuide.HintsFetcher.GetHintsRequest.FetchLatency", 1);
  163. histogram_tester.ExpectTotalCount(
  164. "OptimizationGuide.HintsFetcher.GetHintsRequest.FetchLatency."
  165. "BatchUpdateActiveTabs",
  166. 1);
  167. histogram_tester.ExpectUniqueSample(
  168. "OptimizationGuide.HintsFetcher.RequestStatus.BatchUpdateActiveTabs",
  169. HintsFetcherRequestStatus::kSuccess, 1);
  170. histogram_tester.ExpectTotalCount(
  171. "OptimizationGuide.HintsFetcher.GetHintsRequest.ActiveRequestCanceled."
  172. "BatchUpdateActiveTabs",
  173. 0);
  174. }
  175. // Tests to ensure that multiple hint fetches by the same object cannot be in
  176. // progress simultaneously.
  177. TEST_P(HintsFetcherTest, FetchInProgress) {
  178. base::SimpleTestClock test_clock;
  179. SetTimeClockForTesting(&test_clock);
  180. // Fetch back to back without waiting for Fetch to complete,
  181. // |fetch_in_progress_| should cause early exit.
  182. {
  183. base::HistogramTester histogram_tester;
  184. EXPECT_TRUE(FetchHints({"foo.com"}, {} /* urls */));
  185. EXPECT_FALSE(FetchHints({"bar.com"}, {} /* urls */));
  186. histogram_tester.ExpectUniqueSample(
  187. "OptimizationGuide.HintsFetcher.RequestStatus.BatchUpdateActiveTabs",
  188. HintsFetcherRequestStatus::kFetcherBusy, 1);
  189. }
  190. // Once response arrives, check to make sure a new fetch can start.
  191. {
  192. base::HistogramTester histogram_tester;
  193. std::string response_content;
  194. SimulateResponse(response_content, net::HTTP_OK);
  195. EXPECT_TRUE(FetchHints({"bar.com"}, {} /* urls */));
  196. histogram_tester.ExpectUniqueSample(
  197. "OptimizationGuide.HintsFetcher.RequestStatus.BatchUpdateActiveTabs",
  198. HintsFetcherRequestStatus::kSuccess, 1);
  199. }
  200. }
  201. // Tests that the hints are refreshed again for hosts for whom hints were
  202. // fetched recently.
  203. TEST_P(HintsFetcherTest, FetchInProgress_HostsHintsRefreshed) {
  204. if (!ShouldPersistHintsToDisk())
  205. return;
  206. base::SimpleTestClock test_clock;
  207. SetTimeClockForTesting(&test_clock);
  208. std::string response_content;
  209. // Fetch back to back without waiting for Fetch to complete,
  210. // |fetch_in_progress_| should cause early exit.
  211. EXPECT_TRUE(FetchHints({"foo.com"}, {} /* urls */));
  212. EXPECT_FALSE(FetchHints({"foo.com"}, {} /* urls */));
  213. // Once response arrives, check to make sure that the fetch for the same host
  214. // is not started again.
  215. SimulateResponse(response_content, net::HTTP_OK);
  216. EXPECT_FALSE(FetchHints({"foo.com"}, {} /* urls */));
  217. // Ensure a new fetch for a different host can start.
  218. EXPECT_TRUE(FetchHints({"bar.com"}, {} /* urls */));
  219. SimulateResponse(response_content, net::HTTP_OK);
  220. EXPECT_FALSE(FetchHints({"foo.com"}, {} /* urls */));
  221. EXPECT_FALSE(FetchHints({"bar.com"}, {} /* urls */));
  222. std::vector<std::string> hosts{"foo.com", "bar.com"};
  223. // Advancing the clock so that it's still one hour before the hints need to be
  224. // refreshed.
  225. test_clock.Advance(features::StoredFetchedHintsFreshnessDuration() -
  226. features::GetHostHintsFetchRefreshDuration() -
  227. base::Hours(1));
  228. EXPECT_FALSE(FetchHints({"foo.com"}, {} /* urls */));
  229. EXPECT_FALSE(FetchHints({"bar.com"}, {} /* urls */));
  230. // Advancing the clock by a little bit more than 1 hour so that the hints are
  231. // now due for refresh.
  232. test_clock.Advance(base::Minutes(61));
  233. EXPECT_TRUE(FetchHints({"foo.com"}, {} /* urls */));
  234. EXPECT_FALSE(FetchHints({"bar.com"}, {} /* urls */));
  235. SimulateResponse(response_content, net::HTTP_OK);
  236. // Hints should not be fetched again for foo.com since they were fetched
  237. // recently. Hints should still be fetched for bar.com.
  238. EXPECT_FALSE(FetchHints({"foo.com"}, {} /* urls */));
  239. EXPECT_TRUE(FetchHints({"bar.com"}, {} /* urls */));
  240. SimulateResponse(response_content, net::HTTP_OK);
  241. // Hints should not be fetched again for foo.com and bar.com since they were
  242. // fetched recently. For baz.com, hints should be fetched again.
  243. EXPECT_FALSE(FetchHints({"foo.com"}, {} /* urls */));
  244. EXPECT_FALSE(FetchHints({"bar.com"}, {} /* urls */));
  245. EXPECT_TRUE(FetchHints({"baz.com"}, {} /* urls */));
  246. proto::GetHintsResponse response;
  247. response.mutable_max_cache_duration()->set_seconds(60 * 60 * 24 * 20);
  248. response.SerializeToString(&response_content);
  249. SimulateResponse(response_content, net::HTTP_OK);
  250. // Advance clock for the default duration that the hint normally expires
  251. // under.
  252. test_clock.Advance(features::StoredFetchedHintsFreshnessDuration());
  253. // Max cache duration from response should be used for pref instead.
  254. EXPECT_FALSE(FetchHints({"baz.com"}, {} /* urls */));
  255. }
  256. // Tests 404 response from request.
  257. TEST_P(HintsFetcherTest, FetchReturned404) {
  258. base::HistogramTester histogram_tester;
  259. std::string response_content;
  260. EXPECT_TRUE(FetchHints({"foo.com"}, {} /* urls */));
  261. // Send a 404 to HintsFetcher.
  262. SimulateResponse(response_content, net::HTTP_NOT_FOUND);
  263. EXPECT_FALSE(hints_fetched());
  264. // Make sure histograms are recorded correctly on bad response.
  265. histogram_tester.ExpectTotalCount(
  266. "OptimizationGuide.HintsFetcher.GetHintsRequest.FetchLatency", 0);
  267. histogram_tester.ExpectUniqueSample(
  268. "OptimizationGuide.HintsFetcher.RequestStatus.BatchUpdateActiveTabs",
  269. HintsFetcherRequestStatus::kResponseError, 1);
  270. }
  271. TEST_P(HintsFetcherTest, FetchReturnBadResponse) {
  272. base::HistogramTester histogram_tester;
  273. std::string response_content = "not proto";
  274. EXPECT_TRUE(FetchHints({"foo.com"}, {} /* urls */));
  275. VerifyHasPendingFetchRequests();
  276. EXPECT_TRUE(SimulateResponse(response_content, net::HTTP_OK));
  277. EXPECT_FALSE(hints_fetched());
  278. // Make sure histograms are recorded correctly on bad response.
  279. histogram_tester.ExpectTotalCount(
  280. "OptimizationGuide.HintsFetcher.GetHintsRequest.FetchLatency", 0);
  281. histogram_tester.ExpectUniqueSample(
  282. "OptimizationGuide.HintsFetcher.RequestStatus.BatchUpdateActiveTabs",
  283. HintsFetcherRequestStatus::kResponseError, 1);
  284. }
  285. TEST_P(HintsFetcherTest, HintsFetchSuccessfulHostsRecorded) {
  286. std::vector<std::string> hosts{"host1.com", "host2.com"};
  287. std::string response_content;
  288. EXPECT_TRUE(FetchHints(hosts, {} /* urls */));
  289. VerifyHasPendingFetchRequests();
  290. EXPECT_TRUE(SimulateResponse(response_content, net::HTTP_OK));
  291. EXPECT_TRUE(hints_fetched());
  292. if (!ShouldPersistHintsToDisk())
  293. return;
  294. const base::Value::Dict& hosts_fetched = pref_service()->GetValueDict(
  295. prefs::kHintsFetcherHostsSuccessfullyFetched);
  296. absl::optional<double> value;
  297. for (const std::string& host : hosts) {
  298. value = hosts_fetched.FindDouble(HashHostForDictionary(host));
  299. // This reduces the necessary precision for the check on the expiry time for
  300. // the hosts stored in the pref. The exact time is not necessary, being
  301. // within 10 minutes is acceptable.
  302. EXPECT_NEAR((base::Time::FromDeltaSinceWindowsEpoch(base::Seconds(*value)) -
  303. GetMockClock()->Now())
  304. .InMinutes(),
  305. features::StoredFetchedHintsFreshnessDuration().InMinutes(),
  306. 10);
  307. }
  308. }
  309. TEST_P(HintsFetcherTest, HintsFetchFailsHostNotRecorded) {
  310. std::vector<std::string> hosts{"host1.com", "host2.com"};
  311. std::string response_content;
  312. EXPECT_TRUE(FetchHints(hosts, {} /* urls */));
  313. VerifyHasPendingFetchRequests();
  314. EXPECT_TRUE(SimulateResponse(response_content, net::HTTP_NOT_FOUND));
  315. EXPECT_FALSE(hints_fetched());
  316. if (!ShouldPersistHintsToDisk())
  317. return;
  318. const base::Value::Dict& hosts_fetched = pref_service()->GetValueDict(
  319. prefs::kHintsFetcherHostsSuccessfullyFetched);
  320. for (const std::string& host : hosts) {
  321. EXPECT_FALSE(hosts_fetched.FindDouble(HashHostForDictionary(host)));
  322. }
  323. }
  324. TEST_P(HintsFetcherTest, HintsFetchClearHostsSuccessfullyFetched) {
  325. std::vector<std::string> hosts{"host1.com", "host2.com"};
  326. std::string response_content;
  327. EXPECT_TRUE(FetchHints(hosts, {} /* urls */));
  328. VerifyHasPendingFetchRequests();
  329. EXPECT_TRUE(SimulateResponse(response_content, net::HTTP_OK));
  330. EXPECT_TRUE(hints_fetched());
  331. if (!ShouldPersistHintsToDisk())
  332. return;
  333. {
  334. const base::Value::Dict& hosts_fetched = pref_service()->GetValueDict(
  335. prefs::kHintsFetcherHostsSuccessfullyFetched);
  336. for (const std::string& host : hosts) {
  337. EXPECT_TRUE(hosts_fetched.FindDouble(HashHostForDictionary(host)));
  338. }
  339. }
  340. HintsFetcher::ClearHostsSuccessfullyFetched(pref_service());
  341. {
  342. const base::Value::Dict& hosts_fetched = pref_service()->GetValueDict(
  343. prefs::kHintsFetcherHostsSuccessfullyFetched);
  344. for (const std::string& host : hosts) {
  345. EXPECT_FALSE(hosts_fetched.FindDouble(HashHostForDictionary(host)));
  346. }
  347. }
  348. }
  349. TEST_P(HintsFetcherTest, HintsFetchClearSingleFetchedHost) {
  350. std::vector<std::string> hosts{"host1.com", "host2.com"};
  351. std::string response_content;
  352. EXPECT_TRUE(FetchHints(hosts, {} /* urls */));
  353. VerifyHasPendingFetchRequests();
  354. EXPECT_TRUE(SimulateResponse(response_content, net::HTTP_OK));
  355. EXPECT_TRUE(hints_fetched());
  356. if (!ShouldPersistHintsToDisk())
  357. return;
  358. {
  359. const base::Value::Dict& hosts_fetched = pref_service()->GetValueDict(
  360. prefs::kHintsFetcherHostsSuccessfullyFetched);
  361. for (const std::string& host : hosts) {
  362. EXPECT_TRUE(hosts_fetched.FindDouble(HashHostForDictionary(host)));
  363. }
  364. }
  365. HintsFetcher::ClearSingleFetchedHost(pref_service(), "host1.com");
  366. {
  367. const base::Value::Dict& hosts_fetched = pref_service()->GetValueDict(
  368. prefs::kHintsFetcherHostsSuccessfullyFetched);
  369. EXPECT_FALSE(hosts_fetched.FindDouble(HashHostForDictionary("host1.com")));
  370. EXPECT_TRUE(hosts_fetched.FindDouble(HashHostForDictionary("host2.com")));
  371. }
  372. }
  373. TEST_P(HintsFetcherTest, HintsFetcherHostsCovered) {
  374. if (!ShouldPersistHintsToDisk())
  375. return;
  376. std::vector<std::string> hosts{"host1.com", "host2.com"};
  377. base::Time host_invalid_time = base::Time::Now() + base::Hours(1);
  378. SeedCoveredHosts(hosts, host_invalid_time);
  379. EXPECT_TRUE(WasHostCoveredByFetch(hosts[0]));
  380. EXPECT_TRUE(WasHostCoveredByFetch(hosts[1]));
  381. }
  382. TEST_P(HintsFetcherTest, HintsFetcherCoveredHostExpired) {
  383. std::string response_content;
  384. std::vector<std::string> hosts{"host1.com", "host2.com"};
  385. base::Time host_invalid_time = GetMockClock()->Now() - base::Hours(1);
  386. SeedCoveredHosts(hosts, host_invalid_time);
  387. // Fetch hints for new hosts.
  388. std::vector<std::string> hosts_valid{"host3.com", "hosts4.com"};
  389. EXPECT_TRUE(FetchHints(hosts_valid, {} /* urls */));
  390. VerifyHasPendingFetchRequests();
  391. EXPECT_TRUE(SimulateResponse(response_content, net::HTTP_OK));
  392. EXPECT_TRUE(hints_fetched());
  393. if (!ShouldPersistHintsToDisk())
  394. return;
  395. // The first pair of hosts should be recorded as failed to be
  396. // covered by a recent hints fetcher as they have expired.
  397. EXPECT_FALSE(WasHostCoveredByFetch(hosts[0]));
  398. EXPECT_FALSE(WasHostCoveredByFetch(hosts[1]));
  399. // The first pair of hosts should be removed from the dictionary
  400. // pref as they have expired.
  401. DictionaryPrefUpdate hosts_fetched(
  402. pref_service(), prefs::kHintsFetcherHostsSuccessfullyFetched);
  403. EXPECT_EQ(2u, hosts_fetched->DictSize());
  404. // Navigations to the valid hosts should be recorded as successfully
  405. // covered.
  406. EXPECT_TRUE(WasHostCoveredByFetch(hosts_valid[0]));
  407. EXPECT_TRUE(WasHostCoveredByFetch(hosts_valid[1]));
  408. }
  409. TEST_P(HintsFetcherTest, HintsFetcherHostNotCovered) {
  410. std::vector<std::string> hosts{"host1.com", "host2.com"};
  411. base::Time host_invalid_time = base::Time::Now() + base::Hours(1);
  412. SeedCoveredHosts(hosts, host_invalid_time);
  413. DictionaryPrefUpdate hosts_fetched(
  414. pref_service(), prefs::kHintsFetcherHostsSuccessfullyFetched);
  415. EXPECT_EQ(2u, hosts_fetched->DictSize());
  416. if (!ShouldPersistHintsToDisk())
  417. return;
  418. EXPECT_TRUE(WasHostCoveredByFetch(hosts[0]));
  419. EXPECT_TRUE(WasHostCoveredByFetch(hosts[1]));
  420. EXPECT_FALSE(WasHostCoveredByFetch("newhost.com"));
  421. }
  422. TEST_P(HintsFetcherTest, HintsFetcherRemoveExpiredOnSuccessfullyFetched) {
  423. if (!ShouldPersistHintsToDisk())
  424. return;
  425. std::string response_content;
  426. std::vector<std::string> hosts_expired{"host1.com", "host2.com"};
  427. base::Time host_invalid_time = GetMockClock()->Now() - base::Hours(1);
  428. SeedCoveredHosts(hosts_expired, host_invalid_time);
  429. std::vector<std::string> hosts_valid{"host3.com", "host4.com"};
  430. EXPECT_TRUE(FetchHints(hosts_valid, {} /* urls */));
  431. VerifyHasPendingFetchRequests();
  432. EXPECT_TRUE(SimulateResponse(response_content, net::HTTP_OK));
  433. EXPECT_TRUE(hints_fetched());
  434. // The two expired hosts should be removed from the dictionary pref as they
  435. // have expired.
  436. DictionaryPrefUpdate hosts_fetched(
  437. pref_service(), prefs::kHintsFetcherHostsSuccessfullyFetched);
  438. EXPECT_EQ(2u, hosts_fetched->DictSize());
  439. EXPECT_FALSE(WasHostCoveredByFetch(hosts_expired[0]));
  440. EXPECT_FALSE(WasHostCoveredByFetch(hosts_expired[1]));
  441. EXPECT_TRUE(WasHostCoveredByFetch(hosts_valid[0]));
  442. EXPECT_TRUE(WasHostCoveredByFetch(hosts_valid[1]));
  443. }
  444. TEST_P(HintsFetcherTest, HintsFetcherSuccessfullyFetchedHostsFull) {
  445. if (!ShouldPersistHintsToDisk())
  446. return;
  447. std::string response_content;
  448. std::vector<std::string> hosts;
  449. size_t max_hosts =
  450. optimization_guide::features::MaxHostsForRecordingSuccessfullyCovered();
  451. for (size_t i = 0; i < max_hosts - 1; ++i) {
  452. hosts.push_back("host" + base::NumberToString(i) + ".com");
  453. }
  454. base::Time host_expiry_time = GetMockClock()->Now() + base::Hours(1);
  455. SeedCoveredHosts(hosts, host_expiry_time);
  456. std::vector<std::string> extra_hosts{"extra1.com", "extra2.com"};
  457. EXPECT_TRUE(FetchHints(extra_hosts, {} /* urls */));
  458. VerifyHasPendingFetchRequests();
  459. EXPECT_TRUE(SimulateResponse(response_content, net::HTTP_OK));
  460. EXPECT_TRUE(hints_fetched());
  461. // Navigations to both the extra hosts should be recorded.
  462. DictionaryPrefUpdate hosts_fetched(
  463. pref_service(), prefs::kHintsFetcherHostsSuccessfullyFetched);
  464. EXPECT_EQ(200u, hosts_fetched->DictSize());
  465. EXPECT_TRUE(WasHostCoveredByFetch(extra_hosts[0]));
  466. EXPECT_TRUE(WasHostCoveredByFetch(extra_hosts[1]));
  467. }
  468. TEST_P(HintsFetcherTest, MaxHostsForOptimizationGuideServiceHintsFetch) {
  469. base::HistogramTester histogram_tester;
  470. std::string response_content;
  471. std::vector<std::string> all_hosts;
  472. // Invalid hosts, IP addresses, and localhosts should be skipped.
  473. all_hosts.push_back("localhost");
  474. all_hosts.push_back("8.8.8.8");
  475. all_hosts.push_back("probably%20not%20Canonical");
  476. size_t max_hosts_in_fetch_request = optimization_guide::features::
  477. MaxHostsForOptimizationGuideServiceHintsFetch();
  478. for (size_t i = 0; i < max_hosts_in_fetch_request; ++i) {
  479. all_hosts.push_back("host" + base::NumberToString(i) + ".com");
  480. }
  481. all_hosts.push_back("extra1.com");
  482. all_hosts.push_back("extra2.com");
  483. EXPECT_TRUE(FetchHints(all_hosts, {} /* urls */));
  484. VerifyHasPendingFetchRequests();
  485. EXPECT_TRUE(SimulateResponse(response_content, net::HTTP_OK));
  486. EXPECT_TRUE(hints_fetched());
  487. if (!ShouldPersistHintsToDisk())
  488. return;
  489. DictionaryPrefUpdate hosts_fetched(
  490. pref_service(), prefs::kHintsFetcherHostsSuccessfullyFetched);
  491. EXPECT_EQ(max_hosts_in_fetch_request, hosts_fetched->DictSize());
  492. EXPECT_EQ(all_hosts.size(), max_hosts_in_fetch_request + 5);
  493. for (size_t i = 0; i < max_hosts_in_fetch_request; ++i) {
  494. EXPECT_TRUE(
  495. WasHostCoveredByFetch("host" + base::NumberToString(i) + ".com"));
  496. }
  497. // extra1.com and extra2.com should have been considered "dropped".
  498. histogram_tester.ExpectUniqueSample(
  499. "OptimizationGuide.HintsFetcher.GetHintsRequest.DroppedHosts."
  500. "BatchUpdateActiveTabs",
  501. 2, 1);
  502. }
  503. TEST_P(HintsFetcherTest, MaxUrlsForOptimizationGuideServiceHintsFetch) {
  504. base::HistogramTester histogram_tester;
  505. std::string response_content;
  506. std::vector<GURL> all_urls;
  507. // IP addresses, and localhosts should be skipped.
  508. all_urls.push_back(GURL("localhost"));
  509. all_urls.push_back(GURL("8.8.8.8"));
  510. size_t max_urls_in_fetch_request = optimization_guide::features::
  511. MaxUrlsForOptimizationGuideServiceHintsFetch();
  512. for (size_t i = 0; i < max_urls_in_fetch_request; ++i) {
  513. all_urls.push_back(GURL("https://url" + base::NumberToString(i) + ".com/"));
  514. }
  515. all_urls.push_back(GURL("https://notfetched.com/"));
  516. all_urls.push_back(GURL("https://notfetched-2.com/"));
  517. EXPECT_TRUE(FetchHints({} /* hosts */, all_urls));
  518. VerifyHasPendingFetchRequests();
  519. EXPECT_TRUE(SimulateResponse(response_content, net::HTTP_OK));
  520. EXPECT_TRUE(hints_fetched());
  521. histogram_tester.ExpectUniqueSample(
  522. "OptimizationGuide.HintsFetcher.GetHintsRequest.UrlCount",
  523. max_urls_in_fetch_request, 1);
  524. proto::GetHintsRequest last_request;
  525. last_request.ParseFromString(last_request_body());
  526. EXPECT_EQ(static_cast<size_t>(last_request.urls_size()),
  527. max_urls_in_fetch_request);
  528. for (size_t i = 0; i < max_urls_in_fetch_request; ++i) {
  529. EXPECT_EQ(last_request.urls(i).url(),
  530. "https://url" + base::NumberToString(i) + ".com/");
  531. }
  532. // notfetched.com and notfetched-2.com should have been considered "dropped".
  533. histogram_tester.ExpectUniqueSample(
  534. "OptimizationGuide.HintsFetcher.GetHintsRequest.DroppedUrls."
  535. "BatchUpdateActiveTabs",
  536. 2, 1);
  537. }
  538. TEST_P(HintsFetcherTest, OnlyURLsToFetch) {
  539. base::HistogramTester histogram_tester;
  540. std::string response_content;
  541. EXPECT_TRUE(FetchHints({}, {GURL("https://baz.com/r/werd")}));
  542. VerifyHasPendingFetchRequests();
  543. EXPECT_TRUE(SimulateResponse(response_content, net::HTTP_OK));
  544. EXPECT_TRUE(hints_fetched());
  545. histogram_tester.ExpectTotalCount(
  546. "OptimizationGuide.HintsFetcher.GetHintsRequest.FetchLatency", 1);
  547. histogram_tester.ExpectTotalCount(
  548. "OptimizationGuide.HintsFetcher.GetHintsRequest.FetchLatency."
  549. "BatchUpdateActiveTabs",
  550. 1);
  551. histogram_tester.ExpectUniqueSample(
  552. "OptimizationGuide.HintsFetcher.RequestStatus.BatchUpdateActiveTabs",
  553. static_cast<int>(HintsFetcherRequestStatus::kSuccess), 1);
  554. // Nothing was dropped so this shouldn't be recorded.
  555. histogram_tester.ExpectTotalCount(
  556. "OptimizationGuide.HintsFetcher.GetHintsRequest.DroppedHosts", 0);
  557. histogram_tester.ExpectTotalCount(
  558. "OptimizationGuide.HintsFetcher.GetHintsRequest.DroppedUrls", 0);
  559. }
  560. TEST_P(HintsFetcherTest, NoHostsOrURLsToFetch) {
  561. base::HistogramTester histogram_tester;
  562. std::string response_content;
  563. EXPECT_FALSE(FetchHints({} /* hosts */, {} /* urls */));
  564. EXPECT_FALSE(hints_fetched());
  565. histogram_tester.ExpectUniqueSample(
  566. "OptimizationGuide.HintsFetcher.RequestStatus.BatchUpdateActiveTabs",
  567. static_cast<int>(HintsFetcherRequestStatus::kNoHostsOrURLsToFetch), 1);
  568. }
  569. } // namespace optimization_guide