serial_worker_unittest.cc 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546
  1. // Copyright (c) 2011 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "net/dns/serial_worker.h"
  5. #include <memory>
  6. #include <utility>
  7. #include "base/bind.h"
  8. #include "base/callback.h"
  9. #include "base/check.h"
  10. #include "base/location.h"
  11. #include "base/memory/raw_ptr.h"
  12. #include "base/run_loop.h"
  13. #include "base/synchronization/lock.h"
  14. #include "base/synchronization/waitable_event.h"
  15. #include "base/task/current_thread.h"
  16. #include "base/task/single_thread_task_runner.h"
  17. #include "base/test/simple_test_tick_clock.h"
  18. #include "base/threading/thread_restrictions.h"
  19. #include "base/threading/thread_task_runner_handle.h"
  20. #include "base/time/time.h"
  21. #include "base/timer/timer.h"
  22. #include "net/base/backoff_entry.h"
  23. #include "net/test/test_with_task_environment.h"
  24. #include "testing/gtest/include/gtest/gtest.h"
  25. namespace net {
  26. namespace {
  27. constexpr base::TimeDelta kBackoffInitialDelay = base::Milliseconds(100);
  28. constexpr int kBackoffMultiplyFactor = 2;
  29. constexpr int kMaxRetries = 3;
  30. static const BackoffEntry::Policy kTestBackoffPolicy = {
  31. 0, // Number of initial errors to ignore without backoff.
  32. static_cast<int>(
  33. kBackoffInitialDelay
  34. .InMilliseconds()), // Initial delay for backoff in ms.
  35. kBackoffMultiplyFactor, // Factor to multiply for exponential backoff.
  36. 0, // Fuzzing percentage.
  37. static_cast<int>(
  38. base::Seconds(1).InMilliseconds()), // Maximum time to delay requests
  39. // in ms: 1 second.
  40. -1, // Don't discard entry.
  41. false // Don't use initial delay unless the last was an error.
  42. };
  43. class SerialWorkerTest : public TestWithTaskEnvironment {
  44. public:
  45. // The class under test
  46. class TestSerialWorker : public SerialWorker {
  47. public:
  48. class TestWorkItem : public SerialWorker::WorkItem {
  49. public:
  50. explicit TestWorkItem(SerialWorkerTest* test) : test_(test) {}
  51. void DoWork() override {
  52. ASSERT_TRUE(test_);
  53. test_->OnWork();
  54. }
  55. void FollowupWork(base::OnceClosure closure) override {
  56. ASSERT_TRUE(test_);
  57. test_->OnFollowup(std::move(closure));
  58. }
  59. private:
  60. raw_ptr<SerialWorkerTest> test_;
  61. };
  62. explicit TestSerialWorker(SerialWorkerTest* t)
  63. : SerialWorker(/*max_number_of_retries=*/kMaxRetries,
  64. &kTestBackoffPolicy),
  65. test_(t) {}
  66. ~TestSerialWorker() override = default;
  67. std::unique_ptr<SerialWorker::WorkItem> CreateWorkItem() override {
  68. return std::make_unique<TestWorkItem>(test_);
  69. }
  70. bool OnWorkFinished(
  71. std::unique_ptr<SerialWorker::WorkItem> work_item) override {
  72. CHECK(test_);
  73. return test_->OnWorkFinished();
  74. }
  75. private:
  76. raw_ptr<SerialWorkerTest> test_;
  77. };
  78. SerialWorkerTest(const SerialWorkerTest&) = delete;
  79. SerialWorkerTest& operator=(const SerialWorkerTest&) = delete;
  80. // Mocks
  81. void OnWork() {
  82. { // Check that OnWork is executed serially.
  83. base::AutoLock lock(work_lock_);
  84. EXPECT_FALSE(work_running_) << "`DoWork()` is not called serially!";
  85. work_running_ = true;
  86. }
  87. num_work_calls_observed_++;
  88. BreakNow("OnWork");
  89. {
  90. base::ScopedAllowBaseSyncPrimitivesForTesting
  91. scoped_allow_base_sync_primitives;
  92. work_allowed_.Wait();
  93. }
  94. // Calling from ThreadPool, but protected by work_allowed_/work_called_.
  95. output_value_ = input_value_;
  96. { // This lock might be destroyed after work_called_ is signalled.
  97. base::AutoLock lock(work_lock_);
  98. work_running_ = false;
  99. }
  100. work_called_.Signal();
  101. }
  102. void OnFollowup(base::OnceClosure closure) {
  103. EXPECT_TRUE(task_runner_->BelongsToCurrentThread());
  104. followup_closure_ = std::move(closure);
  105. BreakNow("OnFollowup");
  106. if (followup_immediately_)
  107. CompleteFollowup();
  108. }
  109. bool OnWorkFinished() {
  110. EXPECT_TRUE(task_runner_->BelongsToCurrentThread());
  111. EXPECT_EQ(output_value_, input_value_);
  112. ++work_finished_calls_;
  113. BreakNow("OnWorkFinished");
  114. return on_work_finished_should_report_success_;
  115. }
  116. protected:
  117. void BreakCallback(const std::string& breakpoint) {
  118. breakpoint_ = breakpoint;
  119. run_loop_->Quit();
  120. }
  121. void BreakNow(const std::string& b) {
  122. task_runner_->PostTask(FROM_HERE,
  123. base::BindOnce(&SerialWorkerTest::BreakCallback,
  124. base::Unretained(this), b));
  125. }
  126. void RunUntilBreak(const std::string& b) {
  127. base::RunLoop run_loop;
  128. ASSERT_FALSE(run_loop_);
  129. run_loop_ = &run_loop;
  130. run_loop_->Run();
  131. run_loop_ = nullptr;
  132. ASSERT_EQ(breakpoint_, b);
  133. }
  134. void CompleteFollowup() {
  135. ASSERT_TRUE(followup_closure_);
  136. task_runner_->PostTask(FROM_HERE, std::move(followup_closure_));
  137. }
  138. SerialWorkerTest()
  139. : TestWithTaskEnvironment(
  140. base::test::TaskEnvironment::TimeSource::MOCK_TIME),
  141. work_allowed_(base::WaitableEvent::ResetPolicy::AUTOMATIC,
  142. base::WaitableEvent::InitialState::NOT_SIGNALED),
  143. work_called_(base::WaitableEvent::ResetPolicy::AUTOMATIC,
  144. base::WaitableEvent::InitialState::NOT_SIGNALED) {}
  145. // Helpers for tests.
  146. // Lets OnWork run and waits for it to complete. Can only return if OnWork is
  147. // executed on a concurrent thread. Before calling, OnWork() must already have
  148. // been started and blocked (ensured by running `RunUntilBreak("OnWork")`).
  149. void UnblockWork() {
  150. ASSERT_TRUE(work_running_);
  151. work_allowed_.Signal();
  152. work_called_.Wait();
  153. }
  154. // test::Test methods
  155. void SetUp() override {
  156. task_runner_ = base::ThreadTaskRunnerHandle::Get();
  157. }
  158. void TearDown() override {
  159. // Cancel the worker to catch if it makes a late DoWork call.
  160. if (worker_)
  161. worker_->Cancel();
  162. // Check if OnWork is stalled.
  163. EXPECT_FALSE(work_running_) << "OnWork should be done by TearDown";
  164. // Release it for cleanliness.
  165. if (work_running_) {
  166. UnblockWork();
  167. }
  168. }
  169. // Input value read on WorkerPool.
  170. int input_value_ = 0;
  171. // Output value written on WorkerPool.
  172. int output_value_ = -1;
  173. // The number of times we saw an OnWork call.
  174. int num_work_calls_observed_ = 0;
  175. bool on_work_finished_should_report_success_ = true;
  176. // read is called on WorkerPool so we need to synchronize with it.
  177. base::WaitableEvent work_allowed_;
  178. base::WaitableEvent work_called_;
  179. // Protected by read_lock_. Used to verify that read calls are serialized.
  180. bool work_running_ = false;
  181. base::Lock work_lock_;
  182. int work_finished_calls_ = 0;
  183. // Task runner for this thread.
  184. scoped_refptr<base::SingleThreadTaskRunner> task_runner_;
  185. // WatcherDelegate under test.
  186. std::unique_ptr<TestSerialWorker> worker_ =
  187. std::make_unique<TestSerialWorker>(this);
  188. std::string breakpoint_;
  189. raw_ptr<base::RunLoop> run_loop_ = nullptr;
  190. bool followup_immediately_ = true;
  191. base::OnceClosure followup_closure_;
  192. };
  193. TEST_F(SerialWorkerTest, RunWorkMultipleTimes) {
  194. for (int i = 0; i < 3; ++i) {
  195. ++input_value_;
  196. worker_->WorkNow();
  197. RunUntilBreak("OnWork");
  198. EXPECT_EQ(work_finished_calls_, i);
  199. UnblockWork();
  200. RunUntilBreak("OnFollowup");
  201. RunUntilBreak("OnWorkFinished");
  202. EXPECT_EQ(work_finished_calls_, i + 1);
  203. EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
  204. }
  205. }
  206. TEST_F(SerialWorkerTest, TriggerTwoTimesBeforeRun) {
  207. // Schedule two calls. OnWork checks if it is called serially.
  208. ++input_value_;
  209. worker_->WorkNow();
  210. // Work is blocked, so this will have to induce re-work
  211. worker_->WorkNow();
  212. // Expect 2 cycles through work.
  213. RunUntilBreak("OnWork");
  214. UnblockWork();
  215. RunUntilBreak("OnWork");
  216. UnblockWork();
  217. RunUntilBreak("OnFollowup");
  218. RunUntilBreak("OnWorkFinished");
  219. EXPECT_EQ(work_finished_calls_, 1);
  220. // No more tasks should remain.
  221. EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
  222. }
  223. TEST_F(SerialWorkerTest, TriggerThreeTimesBeforeRun) {
  224. // Schedule two calls. OnWork checks if it is called serially.
  225. ++input_value_;
  226. worker_->WorkNow();
  227. // Work is blocked, so this will have to induce re-work
  228. worker_->WorkNow();
  229. // Repeat work is already scheduled, so this should be a noop.
  230. worker_->WorkNow();
  231. // Expect 2 cycles through work.
  232. RunUntilBreak("OnWork");
  233. UnblockWork();
  234. RunUntilBreak("OnWork");
  235. UnblockWork();
  236. RunUntilBreak("OnFollowup");
  237. RunUntilBreak("OnWorkFinished");
  238. EXPECT_EQ(work_finished_calls_, 1);
  239. // No more tasks should remain.
  240. EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
  241. }
  242. TEST_F(SerialWorkerTest, DelayFollowupCompletion) {
  243. followup_immediately_ = false;
  244. worker_->WorkNow();
  245. RunUntilBreak("OnWork");
  246. UnblockWork();
  247. RunUntilBreak("OnFollowup");
  248. EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
  249. CompleteFollowup();
  250. RunUntilBreak("OnWorkFinished");
  251. EXPECT_EQ(work_finished_calls_, 1);
  252. // No more tasks should remain.
  253. EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
  254. }
  255. TEST_F(SerialWorkerTest, RetriggerDuringRun) {
  256. // Trigger work and wait until blocked.
  257. worker_->WorkNow();
  258. RunUntilBreak("OnWork");
  259. worker_->WorkNow();
  260. worker_->WorkNow();
  261. // Expect a second work cycle after completion of current.
  262. UnblockWork();
  263. RunUntilBreak("OnWork");
  264. UnblockWork();
  265. RunUntilBreak("OnFollowup");
  266. RunUntilBreak("OnWorkFinished");
  267. EXPECT_EQ(work_finished_calls_, 1);
  268. // No more tasks should remain.
  269. EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
  270. }
  271. TEST_F(SerialWorkerTest, RetriggerDuringFollowup) {
  272. // Trigger work and wait until blocked on followup.
  273. followup_immediately_ = false;
  274. worker_->WorkNow();
  275. RunUntilBreak("OnWork");
  276. UnblockWork();
  277. RunUntilBreak("OnFollowup");
  278. worker_->WorkNow();
  279. worker_->WorkNow();
  280. // Expect a second work cycle after completion of followup.
  281. CompleteFollowup();
  282. RunUntilBreak("OnWork");
  283. UnblockWork();
  284. RunUntilBreak("OnFollowup");
  285. CompleteFollowup();
  286. RunUntilBreak("OnWorkFinished");
  287. EXPECT_EQ(work_finished_calls_, 1);
  288. // No more tasks should remain.
  289. EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
  290. }
  291. TEST_F(SerialWorkerTest, CancelDuringWork) {
  292. worker_->WorkNow();
  293. RunUntilBreak("OnWork");
  294. worker_->Cancel();
  295. UnblockWork();
  296. RunUntilIdle();
  297. EXPECT_EQ(breakpoint_, "OnWork");
  298. EXPECT_EQ(work_finished_calls_, 0);
  299. // No more tasks should remain.
  300. EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
  301. }
  302. TEST_F(SerialWorkerTest, CancelDuringFollowup) {
  303. followup_immediately_ = false;
  304. worker_->WorkNow();
  305. RunUntilBreak("OnWork");
  306. UnblockWork();
  307. RunUntilBreak("OnFollowup");
  308. worker_->Cancel();
  309. CompleteFollowup();
  310. RunUntilIdle();
  311. EXPECT_EQ(breakpoint_, "OnFollowup");
  312. EXPECT_EQ(work_finished_calls_, 0);
  313. // No more tasks should remain.
  314. EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
  315. }
  316. TEST_F(SerialWorkerTest, DeleteDuringWork) {
  317. worker_->WorkNow();
  318. RunUntilBreak("OnWork");
  319. worker_.reset();
  320. UnblockWork();
  321. RunUntilIdle();
  322. EXPECT_EQ(breakpoint_, "OnWork");
  323. EXPECT_EQ(work_finished_calls_, 0);
  324. // No more tasks should remain.
  325. EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
  326. }
  327. TEST_F(SerialWorkerTest, DeleteDuringFollowup) {
  328. followup_immediately_ = false;
  329. worker_->WorkNow();
  330. RunUntilBreak("OnWork");
  331. UnblockWork();
  332. RunUntilBreak("OnFollowup");
  333. worker_.reset();
  334. CompleteFollowup();
  335. RunUntilIdle();
  336. EXPECT_EQ(breakpoint_, "OnFollowup");
  337. EXPECT_EQ(work_finished_calls_, 0);
  338. // No more tasks should remain.
  339. EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
  340. }
  341. TEST_F(SerialWorkerTest, RetryAndThenSucceed) {
  342. ASSERT_EQ(0, worker_->GetBackoffEntryForTesting().failure_count());
  343. // Induce a failure.
  344. on_work_finished_should_report_success_ = false;
  345. ++input_value_;
  346. worker_->WorkNow();
  347. RunUntilBreak("OnWork");
  348. UnblockWork();
  349. RunUntilBreak("OnFollowup");
  350. RunUntilBreak("OnWorkFinished");
  351. // Confirm it failed and that a retry was scheduled.
  352. ASSERT_EQ(1, worker_->GetBackoffEntryForTesting().failure_count());
  353. EXPECT_EQ(kBackoffInitialDelay,
  354. worker_->GetBackoffEntryForTesting().GetTimeUntilRelease());
  355. // Make the subsequent attempt succeed.
  356. on_work_finished_should_report_success_ = true;
  357. RunUntilBreak("OnWork");
  358. UnblockWork();
  359. RunUntilBreak("OnFollowup");
  360. RunUntilBreak("OnWorkFinished");
  361. ASSERT_EQ(0, worker_->GetBackoffEntryForTesting().failure_count());
  362. EXPECT_EQ(2, num_work_calls_observed_);
  363. // No more tasks should remain.
  364. EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
  365. }
  366. TEST_F(SerialWorkerTest, ExternalWorkRequestResetsRetryState) {
  367. ASSERT_EQ(0, worker_->GetBackoffEntryForTesting().failure_count());
  368. // Induce a failure.
  369. on_work_finished_should_report_success_ = false;
  370. ++input_value_;
  371. worker_->WorkNow();
  372. RunUntilBreak("OnWork");
  373. UnblockWork();
  374. RunUntilBreak("OnFollowup");
  375. RunUntilBreak("OnWorkFinished");
  376. // Confirm it failed and that a retry was scheduled.
  377. ASSERT_EQ(1, worker_->GetBackoffEntryForTesting().failure_count());
  378. EXPECT_TRUE(worker_->GetRetryTimerForTesting().IsRunning());
  379. EXPECT_EQ(kBackoffInitialDelay,
  380. worker_->GetBackoffEntryForTesting().GetTimeUntilRelease());
  381. on_work_finished_should_report_success_ = true;
  382. // The retry state should be reset before we see OnWorkFinished.
  383. worker_->WorkNow();
  384. ASSERT_EQ(0, worker_->GetBackoffEntryForTesting().failure_count());
  385. EXPECT_FALSE(worker_->GetRetryTimerForTesting().IsRunning());
  386. EXPECT_EQ(base::TimeDelta(),
  387. worker_->GetBackoffEntryForTesting().GetTimeUntilRelease());
  388. RunUntilBreak("OnWork");
  389. UnblockWork();
  390. RunUntilBreak("OnFollowup");
  391. RunUntilBreak("OnWorkFinished");
  392. // No more tasks should remain.
  393. EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
  394. }
  395. TEST_F(SerialWorkerTest, MultipleFailureExponentialBackoff) {
  396. ASSERT_EQ(0, worker_->GetBackoffEntryForTesting().failure_count());
  397. // Induce a failure.
  398. on_work_finished_should_report_success_ = false;
  399. ++input_value_;
  400. worker_->WorkNow();
  401. RunUntilBreak("OnWork");
  402. UnblockWork();
  403. RunUntilBreak("OnFollowup");
  404. RunUntilBreak("OnWorkFinished");
  405. for (int retry_attempt_count = 1; retry_attempt_count <= kMaxRetries;
  406. retry_attempt_count++) {
  407. // Confirm it failed and that a retry was scheduled.
  408. ASSERT_EQ(retry_attempt_count,
  409. worker_->GetBackoffEntryForTesting().failure_count());
  410. EXPECT_TRUE(worker_->GetRetryTimerForTesting().IsRunning());
  411. base::TimeDelta expected_backoff_delay;
  412. if (retry_attempt_count == 1) {
  413. expected_backoff_delay = kBackoffInitialDelay;
  414. } else {
  415. expected_backoff_delay = kBackoffInitialDelay * kBackoffMultiplyFactor *
  416. (retry_attempt_count - 1);
  417. }
  418. EXPECT_EQ(expected_backoff_delay,
  419. worker_->GetBackoffEntryForTesting().GetTimeUntilRelease())
  420. << "retry_attempt_count=" << retry_attempt_count;
  421. // |on_work_finished_should_report_success_| is still false, so the retry
  422. // will fail too
  423. RunUntilBreak("OnWork");
  424. UnblockWork();
  425. RunUntilBreak("OnFollowup");
  426. RunUntilBreak("OnWorkFinished");
  427. }
  428. // The last retry attempt resets the retry state.
  429. ASSERT_EQ(0, worker_->GetBackoffEntryForTesting().failure_count());
  430. EXPECT_FALSE(worker_->GetRetryTimerForTesting().IsRunning());
  431. EXPECT_EQ(base::TimeDelta(),
  432. worker_->GetBackoffEntryForTesting().GetTimeUntilRelease());
  433. on_work_finished_should_report_success_ = true;
  434. // No more tasks should remain.
  435. EXPECT_TRUE(base::CurrentThread::Get()->IsIdleForTesting());
  436. }
  437. } // namespace
  438. } // namespace net