url_request_throttler_unittest.cc 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443
  1. // Copyright (c) 2012 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include <memory>
  5. #include "base/metrics/histogram_samples.h"
  6. #include "base/pickle.h"
  7. #include "base/strings/string_number_conversions.h"
  8. #include "base/test/metrics/histogram_tester.h"
  9. #include "base/time/time.h"
  10. #include "net/base/load_flags.h"
  11. #include "net/base/request_priority.h"
  12. #include "net/base/test_completion_callback.h"
  13. #include "net/test/test_with_task_environment.h"
  14. #include "net/traffic_annotation/network_traffic_annotation_test_helper.h"
  15. #include "net/url_request/url_request.h"
  16. #include "net/url_request/url_request_context.h"
  17. #include "net/url_request/url_request_context_builder.h"
  18. #include "net/url_request/url_request_test_util.h"
  19. #include "net/url_request/url_request_throttler_manager.h"
  20. #include "net/url_request/url_request_throttler_test_support.h"
  21. #include "testing/gtest/include/gtest/gtest.h"
  22. using base::TimeTicks;
  23. namespace net {
  24. namespace {
  25. const char kRequestThrottledHistogramName[] = "Throttling.RequestThrottled";
  26. class MockURLRequestThrottlerEntry : public URLRequestThrottlerEntry {
  27. public:
  28. explicit MockURLRequestThrottlerEntry(
  29. URLRequestThrottlerManager* manager)
  30. : URLRequestThrottlerEntry(manager, std::string()),
  31. backoff_entry_(&backoff_policy_, &fake_clock_) {
  32. InitPolicy();
  33. }
  34. MockURLRequestThrottlerEntry(
  35. URLRequestThrottlerManager* manager,
  36. const TimeTicks& exponential_backoff_release_time,
  37. const TimeTicks& sliding_window_release_time,
  38. const TimeTicks& fake_now)
  39. : URLRequestThrottlerEntry(manager, std::string()),
  40. fake_clock_(fake_now),
  41. backoff_entry_(&backoff_policy_, &fake_clock_) {
  42. InitPolicy();
  43. set_exponential_backoff_release_time(exponential_backoff_release_time);
  44. set_sliding_window_release_time(sliding_window_release_time);
  45. }
  46. void InitPolicy() {
  47. // Some tests become flaky if we have jitter.
  48. backoff_policy_.jitter_factor = 0.0;
  49. // This lets us avoid having to make multiple failures initially (this
  50. // logic is already tested in the BackoffEntry unit tests).
  51. backoff_policy_.num_errors_to_ignore = 0;
  52. }
  53. const BackoffEntry* GetBackoffEntry() const override {
  54. return &backoff_entry_;
  55. }
  56. BackoffEntry* GetBackoffEntry() override { return &backoff_entry_; }
  57. void ResetToBlank(const TimeTicks& time_now) {
  58. fake_clock_.set_now(time_now);
  59. GetBackoffEntry()->Reset();
  60. set_sliding_window_release_time(time_now);
  61. }
  62. // Overridden for tests.
  63. TimeTicks ImplGetTimeNow() const override { return fake_clock_.NowTicks(); }
  64. void set_fake_now(const TimeTicks& now) { fake_clock_.set_now(now); }
  65. void set_exponential_backoff_release_time(const TimeTicks& release_time) {
  66. GetBackoffEntry()->SetCustomReleaseTime(release_time);
  67. }
  68. TimeTicks sliding_window_release_time() const {
  69. return URLRequestThrottlerEntry::sliding_window_release_time();
  70. }
  71. void set_sliding_window_release_time(const TimeTicks& release_time) {
  72. URLRequestThrottlerEntry::set_sliding_window_release_time(release_time);
  73. }
  74. protected:
  75. ~MockURLRequestThrottlerEntry() override = default;
  76. private:
  77. mutable TestTickClock fake_clock_;
  78. BackoffEntry backoff_entry_;
  79. };
  80. class MockURLRequestThrottlerManager : public URLRequestThrottlerManager {
  81. public:
  82. MockURLRequestThrottlerManager() = default;
  83. // Method to process the URL using URLRequestThrottlerManager protected
  84. // method.
  85. std::string DoGetUrlIdFromUrl(const GURL& url) { return GetIdFromUrl(url); }
  86. // Method to use the garbage collecting method of URLRequestThrottlerManager.
  87. void DoGarbageCollectEntries() { GarbageCollectEntries(); }
  88. // Returns the number of entries in the map.
  89. int GetNumberOfEntries() const { return GetNumberOfEntriesForTests(); }
  90. void CreateEntry(bool is_outdated) {
  91. TimeTicks time = TimeTicks::Now();
  92. if (is_outdated) {
  93. time -= base::Milliseconds(
  94. MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs + 1000);
  95. }
  96. std::string fake_url_string("http://www.fakeurl.com/");
  97. fake_url_string.append(base::NumberToString(create_entry_index_++));
  98. GURL fake_url(fake_url_string);
  99. OverrideEntryForTests(fake_url,
  100. base::MakeRefCounted<MockURLRequestThrottlerEntry>(
  101. this, time, TimeTicks::Now(), TimeTicks::Now()));
  102. }
  103. private:
  104. int create_entry_index_ = 0;
  105. };
  106. struct TimeAndBool {
  107. TimeAndBool(const TimeTicks& time_value, bool expected, int line_num) {
  108. time = time_value;
  109. result = expected;
  110. line = line_num;
  111. }
  112. TimeTicks time;
  113. bool result;
  114. int line;
  115. };
  116. struct GurlAndString {
  117. GurlAndString(const GURL& url_value,
  118. const std::string& expected,
  119. int line_num) {
  120. url = url_value;
  121. result = expected;
  122. line = line_num;
  123. }
  124. GURL url;
  125. std::string result;
  126. int line;
  127. };
  128. } // namespace
  129. class URLRequestThrottlerEntryTest : public TestWithTaskEnvironment {
  130. protected:
  131. URLRequestThrottlerEntryTest()
  132. : context_(CreateTestURLRequestContextBuilder()->Build()),
  133. request_(context_->CreateRequest(GURL(),
  134. DEFAULT_PRIORITY,
  135. nullptr,
  136. TRAFFIC_ANNOTATION_FOR_TESTS)) {}
  137. void SetUp() override;
  138. TimeTicks now_;
  139. MockURLRequestThrottlerManager manager_; // Dummy object, not used.
  140. scoped_refptr<MockURLRequestThrottlerEntry> entry_;
  141. std::unique_ptr<URLRequestContext> context_;
  142. std::unique_ptr<URLRequest> request_;
  143. };
  144. void URLRequestThrottlerEntryTest::SetUp() {
  145. request_->SetLoadFlags(0);
  146. now_ = TimeTicks::Now();
  147. entry_ = base::MakeRefCounted<MockURLRequestThrottlerEntry>(&manager_);
  148. entry_->ResetToBlank(now_);
  149. }
  150. std::ostream& operator<<(std::ostream& out, const base::TimeTicks& time) {
  151. return out << time.ToInternalValue();
  152. }
  153. TEST_F(URLRequestThrottlerEntryTest, InterfaceDuringExponentialBackoff) {
  154. base::HistogramTester histogram_tester;
  155. entry_->set_exponential_backoff_release_time(entry_->ImplGetTimeNow() +
  156. base::Milliseconds(1));
  157. EXPECT_TRUE(entry_->ShouldRejectRequest(*request_));
  158. histogram_tester.ExpectBucketCount(kRequestThrottledHistogramName, 0, 0);
  159. histogram_tester.ExpectBucketCount(kRequestThrottledHistogramName, 1, 1);
  160. }
  161. TEST_F(URLRequestThrottlerEntryTest, InterfaceNotDuringExponentialBackoff) {
  162. base::HistogramTester histogram_tester;
  163. entry_->set_exponential_backoff_release_time(entry_->ImplGetTimeNow());
  164. EXPECT_FALSE(entry_->ShouldRejectRequest(*request_));
  165. entry_->set_exponential_backoff_release_time(entry_->ImplGetTimeNow() -
  166. base::Milliseconds(1));
  167. EXPECT_FALSE(entry_->ShouldRejectRequest(*request_));
  168. histogram_tester.ExpectBucketCount(kRequestThrottledHistogramName, 0, 2);
  169. histogram_tester.ExpectBucketCount(kRequestThrottledHistogramName, 1, 0);
  170. }
  171. TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateFailure) {
  172. entry_->UpdateWithResponse(503);
  173. EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(),
  174. entry_->ImplGetTimeNow())
  175. << "A failure should increase the release_time";
  176. }
  177. TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccess) {
  178. entry_->UpdateWithResponse(200);
  179. EXPECT_EQ(entry_->GetExponentialBackoffReleaseTime(),
  180. entry_->ImplGetTimeNow())
  181. << "A success should not add any delay";
  182. }
  183. TEST_F(URLRequestThrottlerEntryTest, InterfaceUpdateSuccessThenFailure) {
  184. entry_->UpdateWithResponse(200);
  185. entry_->UpdateWithResponse(503);
  186. EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(),
  187. entry_->ImplGetTimeNow())
  188. << "This scenario should add delay";
  189. entry_->UpdateWithResponse(200);
  190. }
  191. TEST_F(URLRequestThrottlerEntryTest, IsEntryReallyOutdated) {
  192. base::TimeDelta lifetime =
  193. base::Milliseconds(MockURLRequestThrottlerEntry::kDefaultEntryLifetimeMs);
  194. const base::TimeDelta kFiveMs = base::Milliseconds(5);
  195. TimeAndBool test_values[] = {
  196. TimeAndBool(now_, false, __LINE__),
  197. TimeAndBool(now_ - kFiveMs, false, __LINE__),
  198. TimeAndBool(now_ + kFiveMs, false, __LINE__),
  199. TimeAndBool(now_ - (lifetime - kFiveMs), false, __LINE__),
  200. TimeAndBool(now_ - lifetime, true, __LINE__),
  201. TimeAndBool(now_ - (lifetime + kFiveMs), true, __LINE__)};
  202. for (unsigned int i = 0; i < std::size(test_values); ++i) {
  203. entry_->set_exponential_backoff_release_time(test_values[i].time);
  204. EXPECT_EQ(entry_->IsEntryOutdated(), test_values[i].result) <<
  205. "Test case #" << i << " line " << test_values[i].line << " failed";
  206. }
  207. }
  208. TEST_F(URLRequestThrottlerEntryTest, MaxAllowedBackoff) {
  209. for (int i = 0; i < 30; ++i) {
  210. entry_->UpdateWithResponse(503);
  211. }
  212. base::TimeDelta delay = entry_->GetExponentialBackoffReleaseTime() - now_;
  213. EXPECT_EQ(delay.InMilliseconds(),
  214. MockURLRequestThrottlerEntry::kDefaultMaximumBackoffMs);
  215. }
  216. TEST_F(URLRequestThrottlerEntryTest, MalformedContent) {
  217. for (int i = 0; i < 5; ++i)
  218. entry_->UpdateWithResponse(503);
  219. TimeTicks release_after_failures = entry_->GetExponentialBackoffReleaseTime();
  220. // Inform the entry that a response body was malformed, which is supposed to
  221. // increase the back-off time. Note that we also submit a successful
  222. // UpdateWithResponse to pair with ReceivedContentWasMalformed() since that
  223. // is what happens in practice (if a body is received, then a non-500
  224. // response must also have been received).
  225. entry_->ReceivedContentWasMalformed(200);
  226. entry_->UpdateWithResponse(200);
  227. EXPECT_GT(entry_->GetExponentialBackoffReleaseTime(), release_after_failures);
  228. }
  229. TEST_F(URLRequestThrottlerEntryTest, SlidingWindow) {
  230. int max_send = URLRequestThrottlerEntry::kDefaultMaxSendThreshold;
  231. int sliding_window =
  232. URLRequestThrottlerEntry::kDefaultSlidingWindowPeriodMs;
  233. TimeTicks time_1 =
  234. entry_->ImplGetTimeNow() + base::Milliseconds(sliding_window / 3);
  235. TimeTicks time_2 =
  236. entry_->ImplGetTimeNow() + base::Milliseconds(2 * sliding_window / 3);
  237. TimeTicks time_3 =
  238. entry_->ImplGetTimeNow() + base::Milliseconds(sliding_window);
  239. TimeTicks time_4 =
  240. entry_->ImplGetTimeNow() +
  241. base::Milliseconds(sliding_window + 2 * sliding_window / 3);
  242. entry_->set_exponential_backoff_release_time(time_1);
  243. for (int i = 0; i < max_send / 2; ++i) {
  244. EXPECT_EQ(2 * sliding_window / 3,
  245. entry_->ReserveSendingTimeForNextRequest(time_2));
  246. }
  247. EXPECT_EQ(time_2, entry_->sliding_window_release_time());
  248. entry_->set_fake_now(time_3);
  249. for (int i = 0; i < (max_send + 1) / 2; ++i)
  250. EXPECT_EQ(0, entry_->ReserveSendingTimeForNextRequest(TimeTicks()));
  251. EXPECT_EQ(time_4, entry_->sliding_window_release_time());
  252. }
  253. class URLRequestThrottlerManagerTest : public TestWithTaskEnvironment {
  254. protected:
  255. URLRequestThrottlerManagerTest()
  256. : context_(CreateTestURLRequestContextBuilder()->Build()),
  257. request_(context_->CreateRequest(GURL(),
  258. DEFAULT_PRIORITY,
  259. nullptr,
  260. TRAFFIC_ANNOTATION_FOR_TESTS)) {}
  261. void SetUp() override { request_->SetLoadFlags(0); }
  262. // context_ must be declared before request_.
  263. std::unique_ptr<URLRequestContext> context_;
  264. std::unique_ptr<URLRequest> request_;
  265. };
  266. TEST_F(URLRequestThrottlerManagerTest, IsUrlStandardised) {
  267. MockURLRequestThrottlerManager manager;
  268. GurlAndString test_values[] = {
  269. GurlAndString(GURL("http://www.example.com"),
  270. std::string("http://www.example.com/"),
  271. __LINE__),
  272. GurlAndString(GURL("http://www.Example.com"),
  273. std::string("http://www.example.com/"),
  274. __LINE__),
  275. GurlAndString(GURL("http://www.ex4mple.com/Pr4c71c41"),
  276. std::string("http://www.ex4mple.com/pr4c71c41"),
  277. __LINE__),
  278. GurlAndString(GURL("http://www.example.com/0/token/false"),
  279. std::string("http://www.example.com/0/token/false"),
  280. __LINE__),
  281. GurlAndString(GURL("http://www.example.com/index.php?code=javascript"),
  282. std::string("http://www.example.com/index.php"),
  283. __LINE__),
  284. GurlAndString(GURL("http://www.example.com/index.php?code=1#superEntry"),
  285. std::string("http://www.example.com/index.php"),
  286. __LINE__),
  287. GurlAndString(GURL("http://www.example.com/index.php#superEntry"),
  288. std::string("http://www.example.com/index.php"),
  289. __LINE__),
  290. GurlAndString(GURL("http://www.example.com:1234/"),
  291. std::string("http://www.example.com:1234/"),
  292. __LINE__)};
  293. for (unsigned int i = 0; i < std::size(test_values); ++i) {
  294. std::string temp = manager.DoGetUrlIdFromUrl(test_values[i].url);
  295. EXPECT_EQ(temp, test_values[i].result) <<
  296. "Test case #" << i << " line " << test_values[i].line << " failed";
  297. }
  298. }
  299. TEST_F(URLRequestThrottlerManagerTest, AreEntriesBeingCollected) {
  300. MockURLRequestThrottlerManager manager;
  301. manager.CreateEntry(true); // true = Entry is outdated.
  302. manager.CreateEntry(true);
  303. manager.CreateEntry(true);
  304. manager.DoGarbageCollectEntries();
  305. EXPECT_EQ(0, manager.GetNumberOfEntries());
  306. manager.CreateEntry(false);
  307. manager.CreateEntry(false);
  308. manager.CreateEntry(false);
  309. manager.CreateEntry(true);
  310. manager.DoGarbageCollectEntries();
  311. EXPECT_EQ(3, manager.GetNumberOfEntries());
  312. }
  313. TEST_F(URLRequestThrottlerManagerTest, IsHostBeingRegistered) {
  314. MockURLRequestThrottlerManager manager;
  315. manager.RegisterRequestUrl(GURL("http://www.example.com/"));
  316. manager.RegisterRequestUrl(GURL("http://www.google.com/"));
  317. manager.RegisterRequestUrl(GURL("http://www.google.com/index/0"));
  318. manager.RegisterRequestUrl(GURL("http://www.google.com/index/0?code=1"));
  319. manager.RegisterRequestUrl(GURL("http://www.google.com/index/0#lolsaure"));
  320. EXPECT_EQ(3, manager.GetNumberOfEntries());
  321. }
  322. TEST_F(URLRequestThrottlerManagerTest, LocalHostOptedOut) {
  323. MockURLRequestThrottlerManager manager;
  324. // A localhost entry should always be opted out.
  325. scoped_refptr<URLRequestThrottlerEntryInterface> localhost_entry =
  326. manager.RegisterRequestUrl(GURL("http://localhost/hello"));
  327. EXPECT_FALSE(localhost_entry->ShouldRejectRequest(*request_));
  328. for (int i = 0; i < 10; ++i) {
  329. localhost_entry->UpdateWithResponse(503);
  330. }
  331. EXPECT_FALSE(localhost_entry->ShouldRejectRequest(*request_));
  332. // We're not mocking out GetTimeNow() in this scenario
  333. // so add a 100 ms buffer to avoid flakiness (that should always
  334. // give enough time to get from the TimeTicks::Now() call here
  335. // to the TimeTicks::Now() call in the entry class).
  336. EXPECT_GT(TimeTicks::Now() + base::Milliseconds(100),
  337. localhost_entry->GetExponentialBackoffReleaseTime());
  338. }
  339. TEST_F(URLRequestThrottlerManagerTest, ClearOnNetworkChange) {
  340. for (int i = 0; i < 3; ++i) {
  341. MockURLRequestThrottlerManager manager;
  342. scoped_refptr<URLRequestThrottlerEntryInterface> entry_before =
  343. manager.RegisterRequestUrl(GURL("http://www.example.com/"));
  344. for (int j = 0; j < 10; ++j) {
  345. entry_before->UpdateWithResponse(503);
  346. }
  347. EXPECT_TRUE(entry_before->ShouldRejectRequest(*request_));
  348. switch (i) {
  349. case 0:
  350. manager.OnIPAddressChanged();
  351. break;
  352. case 1:
  353. manager.OnConnectionTypeChanged(
  354. NetworkChangeNotifier::CONNECTION_UNKNOWN);
  355. break;
  356. case 2:
  357. manager.OnConnectionTypeChanged(NetworkChangeNotifier::CONNECTION_NONE);
  358. break;
  359. default:
  360. FAIL();
  361. }
  362. scoped_refptr<URLRequestThrottlerEntryInterface> entry_after =
  363. manager.RegisterRequestUrl(GURL("http://www.example.com/"));
  364. EXPECT_FALSE(entry_after->ShouldRejectRequest(*request_));
  365. }
  366. }
  367. } // namespace net