history_clusters_service_unittest.cc 58 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411
  1. // Copyright 2021 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/history_clusters/core/history_clusters_service.h"
  5. #include <memory>
  6. #include <string>
  7. #include <vector>
  8. #include "base/callback_forward.h"
  9. #include "base/callback_helpers.h"
  10. #include "base/files/scoped_temp_dir.h"
  11. #include "base/location.h"
  12. #include "base/run_loop.h"
  13. #include "base/strings/stringprintf.h"
  14. #include "base/task/cancelable_task_tracker.h"
  15. #include "base/test/bind.h"
  16. #include "base/test/gtest_util.h"
  17. #include "base/test/metrics/histogram_tester.h"
  18. #include "base/test/scoped_feature_list.h"
  19. #include "base/test/task_environment.h"
  20. #include "base/time/time.h"
  21. #include "build/build_config.h"
  22. #include "components/history/core/browser/history_context.h"
  23. #include "components/history/core/browser/history_service.h"
  24. #include "components/history/core/browser/history_types.h"
  25. #include "components/history/core/browser/url_row.h"
  26. #include "components/history/core/test/history_service_test_util.h"
  27. #include "components/history/core/test/visit_annotations_test_utils.h"
  28. #include "components/history_clusters/core/clustering_backend.h"
  29. #include "components/history_clusters/core/config.h"
  30. #include "components/history_clusters/core/features.h"
  31. #include "components/history_clusters/core/history_clusters_db_tasks.h"
  32. #include "components/history_clusters/core/history_clusters_service_test_api.h"
  33. #include "components/history_clusters/core/history_clusters_types.h"
  34. #include "components/history_clusters/core/history_clusters_util.h"
  35. #include "testing/gtest/include/gtest/gtest.h"
  36. #include "url/gurl.h"
  37. namespace history_clusters {
  38. namespace {
  39. base::Time DaysAgo(int days) {
  40. return base::Time::Now() - base::Days(days);
  41. };
  42. // Trivial backend to allow us to specifically test just the service behavior.
  43. class TestClusteringBackend : public ClusteringBackend {
  44. public:
  45. void GetClusters(ClusteringRequestSource clustering_request_source,
  46. ClustersCallback callback,
  47. std::vector<history::AnnotatedVisit> visits) override {
  48. callback_ = std::move(callback);
  49. last_clustered_visits_ = visits;
  50. std::move(wait_for_get_clusters_closure_).Run();
  51. }
  52. void FulfillCallback(const std::vector<history::Cluster>& clusters) {
  53. std::move(callback_).Run(clusters);
  54. }
  55. const std::vector<history::AnnotatedVisit>& LastClusteredVisits() const {
  56. return last_clustered_visits_;
  57. }
  58. // Fetches a scored visit by an ID. `visit_id` must be valid. This is a
  59. // convenience method used for constructing the fake response.
  60. history::ClusterVisit GetVisitById(int visit_id) {
  61. for (const auto& visit : last_clustered_visits_) {
  62. if (visit.visit_row.visit_id == visit_id)
  63. return AnnotatedVisitToClusterVisit(visit);
  64. }
  65. NOTREACHED()
  66. << "TestClusteringBackend::GetVisitById() could not find visit_id: "
  67. << visit_id;
  68. return {};
  69. }
  70. // Should be invoked before `GetClusters()` is invoked.
  71. void WaitForGetClustersCall() {
  72. base::RunLoop loop;
  73. wait_for_get_clusters_closure_ = loop.QuitClosure();
  74. loop.Run();
  75. }
  76. private:
  77. base::OnceClosure wait_for_get_clusters_closure_;
  78. ClustersCallback callback_;
  79. std::vector<history::AnnotatedVisit> last_clustered_visits_;
  80. };
  81. class HistoryClustersServiceTestBase : public testing::Test {
  82. public:
  83. HistoryClustersServiceTestBase()
  84. : task_environment_(
  85. base::test::SingleThreadTaskEnvironment::TimeSource::MOCK_TIME),
  86. run_loop_quit_(run_loop_.QuitClosure()) {}
  87. void SetUp() override {
  88. CHECK(history_dir_.CreateUniqueTempDir());
  89. history_service_ =
  90. history::CreateHistoryService(history_dir_.GetPath(), true);
  91. history_clusters_service_ = std::make_unique<HistoryClustersService>(
  92. "en-US", history_service_.get(),
  93. /*entity_metadata_provider=*/nullptr,
  94. /*url_loader_factory=*/nullptr,
  95. /*engagement_score_provider=*/nullptr,
  96. /*optimization_guide_decider=*/nullptr);
  97. history_clusters_service_test_api_ =
  98. std::make_unique<HistoryClustersServiceTestApi>(
  99. history_clusters_service_.get(), history_service_.get());
  100. auto test_backend = std::make_unique<TestClusteringBackend>();
  101. test_clustering_backend_ = test_backend.get();
  102. history_clusters_service_test_api_->SetClusteringBackendForTest(
  103. std::move(test_backend));
  104. }
  105. HistoryClustersServiceTestBase(const HistoryClustersServiceTestBase&) =
  106. delete;
  107. HistoryClustersServiceTestBase& operator=(
  108. const HistoryClustersServiceTestBase&) = delete;
  109. // Add hardcoded completed visits with context annotations to the history
  110. // database.
  111. void AddHardcodedTestDataToHistoryService() {
  112. for (auto& visit : GetHardcodedTestVisits())
  113. AddCompleteVisit(visit);
  114. }
  115. // Add a complete visit with context annotations to the history database.
  116. void AddCompleteVisit(const history::AnnotatedVisit& visit) {
  117. static const history::ContextID context_id =
  118. reinterpret_cast<history::ContextID>(1);
  119. history::HistoryAddPageArgs add_page_args;
  120. add_page_args.context_id = context_id;
  121. add_page_args.nav_entry_id = next_navigation_id_;
  122. add_page_args.url = visit.url_row.url();
  123. add_page_args.title = visit.url_row.title();
  124. add_page_args.time = visit.visit_row.visit_time;
  125. add_page_args.visit_source = visit.source;
  126. history_service_->AddPage(add_page_args);
  127. history_service_->UpdateWithPageEndTime(
  128. context_id, next_navigation_id_, visit.url_row.url(),
  129. visit.visit_row.visit_time + visit.visit_row.visit_duration);
  130. auto& incomplete_visit_context_annotations =
  131. history_clusters_service_->GetOrCreateIncompleteVisitContextAnnotations(
  132. next_navigation_id_);
  133. incomplete_visit_context_annotations.visit_row = visit.visit_row;
  134. incomplete_visit_context_annotations.url_row = visit.url_row;
  135. incomplete_visit_context_annotations.context_annotations =
  136. visit.context_annotations;
  137. incomplete_visit_context_annotations.status.history_rows = true;
  138. incomplete_visit_context_annotations.status.navigation_ended = true;
  139. incomplete_visit_context_annotations.status.navigation_end_signals = true;
  140. history_clusters_service_->CompleteVisitContextAnnotationsIfReady(
  141. next_navigation_id_);
  142. next_navigation_id_++;
  143. }
  144. // Like `AddCompleteVisit()` above but with less input provided.
  145. void AddCompleteVisit(history::VisitID visit_id, base::Time visit_time) {
  146. history::AnnotatedVisit visit;
  147. visit.url_row.set_id(1);
  148. visit.visit_row.visit_id = visit_id;
  149. visit.visit_row.visit_time = visit_time;
  150. visit.source = history::VisitSource::SOURCE_BROWSED;
  151. AddCompleteVisit(visit);
  152. }
  153. // Add an incomplete visit context annotations to the in memory incomplete
  154. // visit map. Does not touch the history database.
  155. void AddIncompleteVisit(
  156. history::URLID url_id,
  157. history::VisitID visit_id,
  158. base::Time visit_time,
  159. ui::PageTransition transition = ui::PageTransitionFromInt(
  160. ui::PAGE_TRANSITION_LINK | ui::PAGE_TRANSITION_CHAIN_START |
  161. ui::PAGE_TRANSITION_CHAIN_END)) {
  162. // It's not possible to have an incomplete visit with URL or visit set but
  163. // not the other. The IDs must either both be 0 or both be non-zero.
  164. ASSERT_FALSE(url_id ^ visit_id);
  165. auto& incomplete_visit_context_annotations =
  166. history_clusters_service_->GetOrCreateIncompleteVisitContextAnnotations(
  167. next_navigation_id_);
  168. incomplete_visit_context_annotations.url_row.set_id(url_id);
  169. incomplete_visit_context_annotations.visit_row.visit_id = visit_id;
  170. incomplete_visit_context_annotations.visit_row.visit_time = visit_time;
  171. incomplete_visit_context_annotations.visit_row.transition = transition;
  172. incomplete_visit_context_annotations.status.history_rows = url_id;
  173. next_navigation_id_++;
  174. }
  175. void AddCluster(std::vector<history::VisitID> visit_ids) {
  176. base::CancelableTaskTracker task_tracker;
  177. history_service_->ReplaceClusters({}, {history::CreateCluster(visit_ids)},
  178. base::DoNothing(), &task_tracker);
  179. history::BlockUntilHistoryProcessesPendingRequests(history_service_.get());
  180. }
  181. // Verifies that the hardcoded visits were passed to the clustering backend.
  182. void AwaitAndVerifyTestClusteringBackendRequest() {
  183. test_clustering_backend_->WaitForGetClustersCall();
  184. std::vector<history::AnnotatedVisit> visits =
  185. test_clustering_backend_->LastClusteredVisits();
  186. // Visits 2, 3, and 5 are 1-day-old; visit 3 is a synced visit and therefore
  187. // excluded.
  188. ASSERT_EQ(visits.size(), 2u);
  189. auto& visit = visits[0];
  190. EXPECT_EQ(visit.visit_row.visit_id, 5);
  191. EXPECT_EQ(visit.visit_row.visit_time,
  192. GetHardcodedTestVisits()[4].visit_row.visit_time);
  193. EXPECT_EQ(visit.visit_row.visit_duration, base::Seconds(20));
  194. EXPECT_EQ(visit.url_row.url(), "https://second-1-day-old-visit.com/");
  195. EXPECT_EQ(visit.context_annotations.page_end_reason, 5);
  196. visit = visits[1];
  197. EXPECT_EQ(visit.visit_row.visit_id, 2);
  198. EXPECT_EQ(visit.visit_row.visit_time,
  199. GetHardcodedTestVisits()[1].visit_row.visit_time);
  200. EXPECT_EQ(visit.visit_row.visit_duration, base::Seconds(20));
  201. EXPECT_EQ(visit.url_row.url(), "https://github.com/");
  202. EXPECT_EQ(visit.context_annotations.page_end_reason, 5);
  203. // TODO(tommycli): Add back visit.referring_visit_id() check after updating
  204. // the HistoryService test methods to support that field.
  205. }
  206. // Helper to repeatedly call `QueryClusters` and return the clusters it
  207. // returns as well as the visits that were sent to `ClusteringBackend`. Will
  208. // verify a request to the clustering backend is or is NOT made depending on
  209. // `expect_clustering_backend_call`.
  210. std::pair<std::vector<history::Cluster>, std::vector<history::AnnotatedVisit>>
  211. NextQueryClusters(QueryClustersContinuationParams& continuation_params,
  212. bool expect_clustering_backend_call = true) {
  213. std::vector<history::Cluster> clusters;
  214. base::RunLoop loop;
  215. const auto task = history_clusters_service_->QueryClusters(
  216. ClusteringRequestSource::kJourneysPage,
  217. /*begin_time=*/base::Time(), continuation_params, /*recluster=*/false,
  218. base::BindLambdaForTesting(
  219. [&](std::vector<history::Cluster> clusters_temp,
  220. QueryClustersContinuationParams continuation_params_temp) {
  221. loop.Quit();
  222. clusters = clusters_temp;
  223. continuation_params = continuation_params_temp;
  224. }));
  225. // If we expect a clustering call, expect a request and return no clusters.
  226. if (expect_clustering_backend_call) {
  227. test_clustering_backend_->WaitForGetClustersCall();
  228. test_clustering_backend_->FulfillCallback({});
  229. }
  230. // Wait for all the async stuff to complete.
  231. loop.Run();
  232. // Give history a chance to flush out the task to avoid memory leaks.
  233. history::BlockUntilHistoryProcessesPendingRequests(history_service_.get());
  234. // Persisted visits are ordered before incomplete visits. Persisted visits
  235. // are ordered newest first. Incomplete visits are ordered the same as they
  236. // were sent to the `HistoryClustersService`.
  237. return {clusters, expect_clustering_backend_call
  238. ? test_clustering_backend_->LastClusteredVisits()
  239. : std::vector<history::AnnotatedVisit>{}};
  240. };
  241. // Helper to repeatedly schedule a `GetAnnotatedVisitsToCluster` and return
  242. // the clusters and visits it returns.
  243. std::pair<std::vector<int64_t>, std::vector<history::AnnotatedVisit>>
  244. NextVisits(QueryClustersContinuationParams& continuation_params,
  245. bool recent_first,
  246. int days_of_clustered_visits) {
  247. std::vector<int64_t> old_clusters;
  248. std::vector<history::AnnotatedVisit> visits;
  249. base::CancelableTaskTracker task_tracker;
  250. history_service_->ScheduleDBTask(
  251. FROM_HERE,
  252. std::make_unique<GetAnnotatedVisitsToCluster>(
  253. IncompleteVisitMap{}, base::Time(), continuation_params,
  254. recent_first, days_of_clustered_visits, /*recluster=*/false,
  255. base::BindLambdaForTesting(
  256. [&](std::vector<int64_t> old_clusters_temp,
  257. std::vector<history::AnnotatedVisit> visits_temp,
  258. QueryClustersContinuationParams continuation_params_temp) {
  259. old_clusters = old_clusters_temp;
  260. visits = visits_temp;
  261. continuation_params = continuation_params_temp;
  262. })),
  263. &task_tracker);
  264. history::BlockUntilHistoryProcessesPendingRequests(history_service_.get());
  265. return {old_clusters, visits};
  266. }
  267. // Helper to flush out the multiple history and cluster backend requests made
  268. // by `Does[Query|URL]MatchAnyCluster()`. It won't populate the cache until
  269. // all its requests have been completed. It makes 1 request (to each) per
  270. // unique day with at least 1 visit; i.e. `number_of_days_with_visits`.
  271. void FlushKeywordRequests(std::vector<history::Cluster> clusters,
  272. size_t number_of_days_with_visits) {
  273. // `Does[Query|URL]MatchAnyCluster()` will continue making history and
  274. // cluster backend requests until it has exhausted history. We have to flush
  275. // out these requests before it will populate the cache.
  276. for (size_t i = 0; i < number_of_days_with_visits; ++i) {
  277. test_clustering_backend_->WaitForGetClustersCall();
  278. test_clustering_backend_->FulfillCallback(
  279. i == 0 ? clusters : std::vector<history::Cluster>{});
  280. }
  281. // Flush out the last, empty history requests. There'll be 2 history
  282. // requests: the 1st to exhaust visits to cluster requests, and the 2nd to
  283. // exhaust persisted cluster requests.
  284. history::BlockUntilHistoryProcessesPendingRequests(history_service_.get());
  285. history::BlockUntilHistoryProcessesPendingRequests(history_service_.get());
  286. };
  287. protected:
  288. // ScopedFeatureList needs to be declared before TaskEnvironment, so that it
  289. // is destroyed after the TaskEnvironment is destroyed, preventing other
  290. // threads from accessing the feature list while it's being destroyed.
  291. base::test::ScopedFeatureList scoped_feature_list_;
  292. base::test::TaskEnvironment task_environment_;
  293. // Used to construct a `HistoryClustersService`.
  294. base::ScopedTempDir history_dir_;
  295. std::unique_ptr<history::HistoryService> history_service_;
  296. std::unique_ptr<HistoryClustersService> history_clusters_service_;
  297. std::unique_ptr<HistoryClustersServiceTestApi>
  298. history_clusters_service_test_api_;
  299. // Non-owning pointer. The actual owner is `history_clusters_service_`.
  300. TestClusteringBackend* test_clustering_backend_;
  301. // Used to verify the async callback is invoked.
  302. base::RunLoop run_loop_;
  303. base::RepeatingClosure run_loop_quit_;
  304. // Tracks the next available navigation ID to be associated with visits.
  305. int64_t next_navigation_id_ = 0;
  306. };
  307. class HistoryClustersServiceTest : public HistoryClustersServiceTestBase {
  308. public:
  309. HistoryClustersServiceTest() {
  310. scoped_feature_list_.InitAndEnableFeature(internal::kJourneys);
  311. Config config;
  312. config.persist_clusters_in_history_db = true;
  313. SetConfigForTesting(config);
  314. }
  315. };
  316. TEST_F(HistoryClustersServiceTest, HardCapOnVisitsFetchedFromHistory) {
  317. Config config;
  318. config.is_journeys_enabled_no_locale_check = true;
  319. config.max_visits_to_cluster = 20;
  320. SetConfigForTesting(config);
  321. history::ContextID context_id = reinterpret_cast<history::ContextID>(1);
  322. auto visit = GetHardcodedTestVisits()[0];
  323. for (size_t i = 0; i < 100; ++i) {
  324. // Visit IDs start at 1.
  325. visit.visit_row.visit_id = i + 1;
  326. history::HistoryAddPageArgs add_page_args;
  327. add_page_args.context_id = context_id;
  328. add_page_args.nav_entry_id = next_navigation_id_;
  329. add_page_args.url = visit.url_row.url();
  330. add_page_args.title = visit.url_row.title();
  331. add_page_args.time = visit.visit_row.visit_time;
  332. add_page_args.visit_source = visit.source;
  333. history_service_->AddPage(add_page_args);
  334. history_service_->UpdateWithPageEndTime(
  335. context_id, next_navigation_id_, visit.url_row.url(),
  336. visit.visit_row.visit_time + visit.visit_row.visit_duration);
  337. auto& incomplete_visit_context_annotations =
  338. history_clusters_service_->GetOrCreateIncompleteVisitContextAnnotations(
  339. next_navigation_id_);
  340. incomplete_visit_context_annotations.visit_row = visit.visit_row;
  341. incomplete_visit_context_annotations.url_row = visit.url_row;
  342. incomplete_visit_context_annotations.context_annotations =
  343. visit.context_annotations;
  344. incomplete_visit_context_annotations.status.history_rows = true;
  345. incomplete_visit_context_annotations.status.navigation_ended = true;
  346. incomplete_visit_context_annotations.status.navigation_end_signals = true;
  347. history_clusters_service_->CompleteVisitContextAnnotationsIfReady(
  348. next_navigation_id_);
  349. next_navigation_id_++;
  350. }
  351. history::BlockUntilHistoryProcessesPendingRequests(history_service_.get());
  352. const auto task = history_clusters_service_->QueryClusters(
  353. ClusteringRequestSource::kKeywordCacheGeneration,
  354. /*begin_time=*/base::Time(), /*continuation_params=*/{},
  355. /*recluster=*/false,
  356. base::DoNothing() // Only need to verify the correct request is sent
  357. );
  358. test_clustering_backend_->WaitForGetClustersCall();
  359. history::BlockUntilHistoryProcessesPendingRequests(history_service_.get());
  360. EXPECT_EQ(test_clustering_backend_->LastClusteredVisits().size(), 20U);
  361. }
  362. TEST_F(HistoryClustersServiceTest, QueryClusters_IncompleteAndPersistedVisits) {
  363. // Create 5 persisted visits with visit times 2, 1, 1, 60, and 1 days ago.
  364. AddHardcodedTestDataToHistoryService();
  365. // Create incomplete visits; only 6 & 7 should be returned by the query.
  366. AddIncompleteVisit(6, 6, DaysAgo(1));
  367. AddIncompleteVisit(0, 0, DaysAgo(1)); // Missing history rows.
  368. AddIncompleteVisit(7, 7, DaysAgo(90));
  369. AddIncompleteVisit(8, 8, DaysAgo(0)); // Too recent.
  370. AddIncompleteVisit(9, 9, DaysAgo(93)); // Too old.
  371. AddIncompleteVisit(
  372. 10, 10, DaysAgo(1),
  373. ui::PageTransitionFromInt(805306372)); // Non-visible page transition.
  374. QueryClustersContinuationParams continuation_params = {};
  375. continuation_params.continuation_time = base::Time::Now();
  376. // 1st query should return visits 2, 5, & 6, the good, 1-day-old visits.
  377. // Visits 3, 0, and 10, also 1-day-old, are excluded since they're synced,
  378. // missing history rows, and non-visible transition respectively.
  379. {
  380. const auto [clusters, visits] = NextQueryClusters(continuation_params);
  381. EXPECT_THAT(GetClusterIds(clusters), testing::ElementsAre());
  382. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre(5, 2, 6));
  383. EXPECT_TRUE(continuation_params.is_continuation);
  384. EXPECT_FALSE(continuation_params.is_partial_day);
  385. }
  386. // 2nd query should return visit 1, a 2-day-old complete visit.
  387. {
  388. const auto [clusters, visits] = NextQueryClusters(continuation_params);
  389. EXPECT_THAT(GetClusterIds(clusters), testing::ElementsAre());
  390. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre(1));
  391. }
  392. // 3rd query should return visit 4, a 30-day-old complete visit, since there
  393. // are no 3-to-29-day-old visits.
  394. {
  395. const auto [clusters, visits] = NextQueryClusters(continuation_params);
  396. EXPECT_THAT(GetClusterIds(clusters), testing::ElementsAre());
  397. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre(4));
  398. EXPECT_FALSE(continuation_params.exhausted_unclustered_visits);
  399. EXPECT_FALSE(continuation_params.exhausted_all_visits);
  400. }
  401. // 4th query should return visit 7, a 90-day-old incomplete visit, since there
  402. // are no 31-to-89-day-old visits.
  403. {
  404. const auto [clusters, visits] = NextQueryClusters(continuation_params);
  405. EXPECT_THAT(GetClusterIds(clusters), testing::ElementsAre());
  406. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre(7));
  407. EXPECT_TRUE(continuation_params.exhausted_unclustered_visits);
  408. EXPECT_TRUE(continuation_params.exhausted_all_visits);
  409. }
  410. }
  411. TEST_F(HistoryClustersServiceTest,
  412. QueryClusters_PersistedClusters_NoMixedDays) {
  413. // Test the case where there are persisted clusters but none on a day also
  414. // containing unclustered visits.
  415. // 2 unclustered visits.
  416. AddCompleteVisit(1, DaysAgo(1));
  417. AddCompleteVisit(2, DaysAgo(2));
  418. // 2 clustered visits; i.e. persisted clusters.
  419. AddCompleteVisit(3, DaysAgo(3));
  420. AddCompleteVisit(4, DaysAgo(4));
  421. AddCluster({3});
  422. AddCluster({4});
  423. // Another clustered visit with a gap.
  424. AddCompleteVisit(5, DaysAgo(10));
  425. AddCluster({5});
  426. // The DB looks like:
  427. // Days ago: 10 9 8 7 6 5 4 3 2 1
  428. // Visit: C C C U U
  429. // Where C & U are clustered & unclustered visits.
  430. QueryClustersContinuationParams continuation_params = {};
  431. continuation_params.continuation_time = base::Time::Now();
  432. // 1st 2 queries should return the 2 unclustered visits.
  433. {
  434. const auto [clusters, visits] = NextQueryClusters(continuation_params);
  435. EXPECT_THAT(GetClusterIds(clusters), testing::ElementsAre());
  436. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre(1));
  437. EXPECT_FALSE(continuation_params.exhausted_unclustered_visits);
  438. EXPECT_FALSE(continuation_params.exhausted_all_visits);
  439. }
  440. {
  441. const auto [clusters, visits] = NextQueryClusters(continuation_params);
  442. EXPECT_THAT(GetClusterIds(clusters), testing::ElementsAre());
  443. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre(2));
  444. EXPECT_FALSE(continuation_params.exhausted_unclustered_visits);
  445. EXPECT_FALSE(continuation_params.exhausted_all_visits);
  446. }
  447. // Next 3 queries should return the persisted clusters. They should not make
  448. // requests to the clustering backend. And they should set
  449. // `exhausted_unclustered_visits`.
  450. {
  451. const auto [clusters, visits] =
  452. NextQueryClusters(continuation_params, false);
  453. ASSERT_THAT(GetClusterIds(clusters), testing::ElementsAre(1));
  454. EXPECT_THAT(GetVisitIds(clusters[0].visits), testing::ElementsAre(3));
  455. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre());
  456. EXPECT_TRUE(continuation_params.exhausted_unclustered_visits);
  457. EXPECT_FALSE(continuation_params.exhausted_all_visits);
  458. }
  459. {
  460. const auto [clusters, visits] =
  461. NextQueryClusters(continuation_params, false);
  462. ASSERT_THAT(GetClusterIds(clusters), testing::ElementsAre(2));
  463. EXPECT_THAT(GetVisitIds(clusters[0].visits), testing::ElementsAre(4));
  464. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre());
  465. EXPECT_TRUE(continuation_params.exhausted_unclustered_visits);
  466. EXPECT_FALSE(continuation_params.exhausted_all_visits);
  467. }
  468. {
  469. const auto [clusters, visits] =
  470. NextQueryClusters(continuation_params, false);
  471. ASSERT_THAT(GetClusterIds(clusters), testing::ElementsAre(3));
  472. EXPECT_THAT(GetVisitIds(clusters[0].visits), testing::ElementsAre(5));
  473. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre());
  474. EXPECT_TRUE(continuation_params.exhausted_unclustered_visits);
  475. EXPECT_FALSE(continuation_params.exhausted_all_visits);
  476. }
  477. // The last query should set `exhausted_all_visits`.
  478. {
  479. const auto [clusters, visits] =
  480. NextQueryClusters(continuation_params, false);
  481. EXPECT_THAT(GetClusterIds(clusters), testing::ElementsAre());
  482. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre());
  483. EXPECT_TRUE(continuation_params.exhausted_unclustered_visits);
  484. EXPECT_TRUE(continuation_params.exhausted_all_visits);
  485. }
  486. }
  487. TEST_F(HistoryClustersServiceTest,
  488. QueryClusters_PersistedClusters_PersistenceDisabled) {
  489. // Test the case where there are persisted clusters but persistence is
  490. // disabled to check users who were in an enabled then disabled group
  491. // don't encounter weirdness.
  492. Config config;
  493. config.persist_clusters_in_history_db = false;
  494. SetConfigForTesting(config);
  495. // Unclustered visit.
  496. AddCompleteVisit(1, DaysAgo(1));
  497. // Clustered visit; i.e. persisted cluster.
  498. AddCompleteVisit(2, DaysAgo(2));
  499. AddCluster({2});
  500. QueryClustersContinuationParams continuation_params = {};
  501. continuation_params.continuation_time = base::Time::Now();
  502. // 2 queries should return the 2 visits and treat both as unclustered.
  503. {
  504. const auto [clusters, visits] = NextQueryClusters(continuation_params);
  505. EXPECT_THAT(GetClusterIds(clusters), testing::ElementsAre());
  506. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre(1));
  507. EXPECT_FALSE(continuation_params.exhausted_unclustered_visits);
  508. EXPECT_FALSE(continuation_params.exhausted_all_visits);
  509. }
  510. {
  511. const auto [clusters, visits] = NextQueryClusters(continuation_params);
  512. EXPECT_THAT(GetClusterIds(clusters), testing::ElementsAre());
  513. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre(2));
  514. EXPECT_FALSE(continuation_params.exhausted_unclustered_visits);
  515. EXPECT_FALSE(continuation_params.exhausted_all_visits);
  516. }
  517. // 3rd query should consider history exhausted.
  518. {
  519. const auto [clusters, visits] =
  520. NextQueryClusters(continuation_params, false);
  521. EXPECT_THAT(GetClusterIds(clusters), testing::ElementsAre());
  522. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre());
  523. EXPECT_TRUE(continuation_params.exhausted_unclustered_visits);
  524. EXPECT_TRUE(continuation_params.exhausted_all_visits);
  525. }
  526. }
  527. TEST_F(HistoryClustersServiceTest, QueryClusters_PersistedClusters_Today) {
  528. // Test the case where there is a persisted cluster today. The task rewinds
  529. // the query bounds when it reaches a clustered visit, and this should be done
  530. // correctly even if it's at the edge.
  531. // Can't use `Now()`, as the task only searches [now-90, now).
  532. const auto today = base::Time::Now() - base::Hours(1);
  533. // A clustered and unclustered visit, both today.
  534. AddCompleteVisit(1, today);
  535. AddCompleteVisit(2, today);
  536. AddCluster({2});
  537. QueryClustersContinuationParams continuation_params = {};
  538. continuation_params.continuation_time = base::Time::Now();
  539. // 1st query should return the 1st unclustered visits only and set
  540. // `exhausted_unclustered_visits`.
  541. {
  542. const auto [clusters, visits] = NextQueryClusters(continuation_params);
  543. EXPECT_THAT(GetClusterIds(clusters), testing::ElementsAre());
  544. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre(1));
  545. EXPECT_TRUE(continuation_params.exhausted_unclustered_visits);
  546. EXPECT_FALSE(continuation_params.exhausted_all_visits);
  547. }
  548. // 2nd query should return the cluster.
  549. {
  550. const auto [clusters, visits] =
  551. NextQueryClusters(continuation_params, false);
  552. ASSERT_THAT(GetClusterIds(clusters), testing::ElementsAre(1));
  553. EXPECT_THAT(GetVisitIds(clusters[0].visits), testing::ElementsAre(2));
  554. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre());
  555. EXPECT_TRUE(continuation_params.exhausted_unclustered_visits);
  556. EXPECT_FALSE(continuation_params.exhausted_all_visits);
  557. }
  558. // The last query should set `exhausted_all_visits`.
  559. {
  560. const auto [clusters, visits] =
  561. NextQueryClusters(continuation_params, false);
  562. EXPECT_THAT(GetClusterIds(clusters), testing::ElementsAre());
  563. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre());
  564. EXPECT_TRUE(continuation_params.exhausted_unclustered_visits);
  565. EXPECT_TRUE(continuation_params.exhausted_all_visits);
  566. }
  567. }
  568. TEST_F(HistoryClustersServiceTest, QueryClusters_PersistedClusters_MixedDay) {
  569. // Test the case where there are persisted clusters on a day also containing
  570. // unclustered visits.
  571. // 2 unclustered visits.
  572. AddCompleteVisit(1, DaysAgo(1));
  573. AddCompleteVisit(2, DaysAgo(2));
  574. // 2 clustered visits; i.e. persisted clusters.
  575. AddCompleteVisit(3, DaysAgo(2));
  576. AddCompleteVisit(4, DaysAgo(3));
  577. AddCluster({3});
  578. AddCluster({4});
  579. // The DB looks like:
  580. // Days ago: 3 2 1
  581. // Visit: C M U
  582. // Where C, U, & M are days containing clustered, unclustered, and mixed
  583. // visits.
  584. QueryClustersContinuationParams continuation_params = {};
  585. continuation_params.continuation_time = base::Time::Now();
  586. // 1st query should return the unclustered visit.
  587. {
  588. const auto [clusters, visits] = NextQueryClusters(continuation_params);
  589. EXPECT_THAT(GetClusterIds(clusters), testing::ElementsAre());
  590. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre(1));
  591. EXPECT_FALSE(continuation_params.exhausted_unclustered_visits);
  592. EXPECT_FALSE(continuation_params.exhausted_all_visits);
  593. }
  594. // 2nd query should return only the unclustered visit. Should also set
  595. // `exhausted_unclustered_visits`.
  596. {
  597. const auto [clusters, visits] = NextQueryClusters(continuation_params);
  598. EXPECT_THAT(GetClusterIds(clusters), testing::ElementsAre());
  599. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre(2));
  600. EXPECT_TRUE(continuation_params.exhausted_unclustered_visits);
  601. EXPECT_FALSE(continuation_params.exhausted_all_visits);
  602. }
  603. // 3rd query should return the 1st cluster from 2 days ago; it shouldn't be
  604. // skipped even though the 2nd query already returned a visit from 2 days ago.
  605. {
  606. const auto [clusters, visits] =
  607. NextQueryClusters(continuation_params, false);
  608. ASSERT_THAT(GetClusterIds(clusters), testing::ElementsAre(1));
  609. EXPECT_THAT(GetVisitIds(clusters[0].visits), testing::ElementsAre(3));
  610. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre());
  611. EXPECT_TRUE(continuation_params.exhausted_unclustered_visits);
  612. EXPECT_FALSE(continuation_params.exhausted_all_visits);
  613. }
  614. // 4th query should return the non-mixed cluster.
  615. {
  616. const auto [clusters, visits] =
  617. NextQueryClusters(continuation_params, false);
  618. ASSERT_THAT(GetClusterIds(clusters), testing::ElementsAre(2));
  619. EXPECT_THAT(GetVisitIds(clusters[0].visits), testing::ElementsAre(4));
  620. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre());
  621. EXPECT_TRUE(continuation_params.exhausted_unclustered_visits);
  622. EXPECT_FALSE(continuation_params.exhausted_all_visits);
  623. }
  624. // Last query should set `exhausted_all_visits`.
  625. {
  626. const auto [clusters, visits] =
  627. NextQueryClusters(continuation_params, false);
  628. EXPECT_THAT(GetClusterIds(clusters), testing::ElementsAre());
  629. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre());
  630. EXPECT_TRUE(continuation_params.exhausted_unclustered_visits);
  631. EXPECT_TRUE(continuation_params.exhausted_all_visits);
  632. }
  633. }
  634. TEST_F(HistoryClustersServiceTest, QueryVisits_OldestFirst) {
  635. // Create 5 persisted visits with visit times 2, 1, 1, 60, and 1 days ago.
  636. AddHardcodedTestDataToHistoryService();
  637. // Add a sync visit on a day without other visits in order to verify a day
  638. // with only sync visits doesn't interrupt `GetAnnotatedVisitsToCluster`'s
  639. // intention of iterating until a visit is found.
  640. history::AnnotatedVisit sync_visit;
  641. sync_visit.url_row.set_id(1);
  642. sync_visit.visit_row.visit_id = 10;
  643. sync_visit.visit_row.visit_time = base::Time::Now() - base::Days(15);
  644. sync_visit.source = history::VisitSource::SOURCE_SYNCED;
  645. AddCompleteVisit(sync_visit);
  646. // Helper to repeatedly schedule a `GetAnnotatedVisitsToCluster`, with the
  647. // continuation time returned from the previous task, and return the visits
  648. // it returns.
  649. QueryClustersContinuationParams continuation_params = {};
  650. {
  651. // 1st query should return the oldest, 60-day-old visit.
  652. const auto [clusters, visits] = NextVisits(continuation_params, false, 0);
  653. EXPECT_TRUE(clusters.empty());
  654. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre(4));
  655. EXPECT_TRUE(continuation_params.is_continuation);
  656. EXPECT_FALSE(continuation_params.is_partial_day);
  657. EXPECT_FALSE(continuation_params.exhausted_unclustered_visits);
  658. EXPECT_FALSE(continuation_params.exhausted_all_visits);
  659. }
  660. {
  661. // 2nd query should return the next oldest, 2-day-old visit.
  662. const auto [clusters, visits] = NextVisits(continuation_params, false, 0);
  663. EXPECT_TRUE(clusters.empty());
  664. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre(1));
  665. EXPECT_TRUE(continuation_params.is_continuation);
  666. EXPECT_FALSE(continuation_params.exhausted_unclustered_visits);
  667. EXPECT_FALSE(continuation_params.exhausted_all_visits);
  668. }
  669. {
  670. // 3rd query should return the next oldest, 1-day-old visits. Visit 3 is
  671. // excluded as it's from sync.
  672. const auto [clusters, visits] = NextVisits(continuation_params, false, 0);
  673. EXPECT_TRUE(clusters.empty());
  674. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre(5, 2));
  675. EXPECT_TRUE(continuation_params.is_continuation);
  676. EXPECT_FALSE(continuation_params.exhausted_unclustered_visits);
  677. EXPECT_FALSE(continuation_params.exhausted_all_visits);
  678. }
  679. {
  680. // 4th query should return no visits; all visits were exhausted.
  681. const auto [clusters, visits] = NextVisits(continuation_params, false, 0);
  682. EXPECT_TRUE(clusters.empty());
  683. EXPECT_TRUE(visits.empty());
  684. EXPECT_TRUE(continuation_params.exhausted_unclustered_visits);
  685. EXPECT_TRUE(continuation_params.exhausted_all_visits);
  686. }
  687. }
  688. TEST_F(HistoryClustersServiceTest, QueryClusteredVisits) {
  689. // Create unclustered visits 1, 2, 3, and 4 days-old.
  690. AddCompleteVisit(1, DaysAgo(1));
  691. AddCompleteVisit(2, DaysAgo(2));
  692. AddCompleteVisit(3, DaysAgo(3));
  693. AddCompleteVisit(4, DaysAgo(4));
  694. // Create clustered visits 3 and 4 days-old.
  695. AddCompleteVisit(5, DaysAgo(3));
  696. AddCompleteVisit(6, DaysAgo(4));
  697. AddCluster({5});
  698. AddCluster({6});
  699. QueryClustersContinuationParams continuation_params = {};
  700. {
  701. // 1st query should get the newest, 1-day-old, visit. There are no adjacent
  702. // clusters to get.
  703. const auto [clusters, visits] = NextVisits(continuation_params, true, 1);
  704. EXPECT_TRUE(clusters.empty());
  705. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre(1));
  706. EXPECT_TRUE(continuation_params.is_continuation);
  707. EXPECT_FALSE(continuation_params.is_partial_day);
  708. EXPECT_FALSE(continuation_params.exhausted_unclustered_visits);
  709. EXPECT_FALSE(continuation_params.exhausted_all_visits);
  710. }
  711. {
  712. // 2nd query should get the 2-day-old visit and the adjacent
  713. // 3-day-old clustered visit. Should not get the 3-day-old or older
  714. // unclustered visits.
  715. const auto [clusters, visits] = NextVisits(continuation_params, true, 1);
  716. EXPECT_THAT(clusters, testing::ElementsAre(1));
  717. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre(2, 5));
  718. EXPECT_TRUE(continuation_params.exhausted_unclustered_visits);
  719. EXPECT_FALSE(continuation_params.exhausted_all_visits);
  720. }
  721. }
  722. TEST_F(HistoryClustersServiceTest, EndToEndWithBackend) {
  723. base::HistogramTester histogram_tester;
  724. AddHardcodedTestDataToHistoryService();
  725. base::RunLoop run_loop;
  726. auto run_loop_quit = run_loop.QuitClosure();
  727. const auto task = history_clusters_service_->QueryClusters(
  728. ClusteringRequestSource::kJourneysPage,
  729. /*begin_time=*/base::Time(),
  730. /*continuation_params=*/{},
  731. /*recluster=*/false,
  732. // This "expect" block is not run until after the fake response is
  733. // sent further down in this method.
  734. base::BindLambdaForTesting([&](std::vector<history::Cluster> clusters,
  735. QueryClustersContinuationParams) {
  736. ASSERT_EQ(clusters.size(), 2U);
  737. auto& cluster = clusters[0];
  738. auto& visits = cluster.visits;
  739. ASSERT_EQ(visits.size(), 2u);
  740. EXPECT_EQ(visits[0].annotated_visit.url_row.url(),
  741. "https://github.com/");
  742. EXPECT_EQ(visits[0].annotated_visit.visit_row.visit_time,
  743. GetHardcodedTestVisits()[1].visit_row.visit_time);
  744. EXPECT_EQ(visits[0].annotated_visit.url_row.title(),
  745. u"Code Storage Title");
  746. EXPECT_FALSE(
  747. visits[0].annotated_visit.context_annotations.is_new_bookmark);
  748. EXPECT_TRUE(visits[0]
  749. .annotated_visit.context_annotations
  750. .is_existing_part_of_tab_group);
  751. EXPECT_FLOAT_EQ(visits[0].score, 0.5);
  752. EXPECT_EQ(visits[1].annotated_visit.url_row.url(),
  753. "https://second-1-day-old-visit.com/");
  754. EXPECT_EQ(visits[1].annotated_visit.visit_row.visit_time,
  755. GetHardcodedTestVisits()[4].visit_row.visit_time);
  756. EXPECT_EQ(visits[1].annotated_visit.url_row.title(),
  757. u"second-1-day-old-visit");
  758. EXPECT_TRUE(
  759. visits[1].annotated_visit.context_annotations.is_new_bookmark);
  760. EXPECT_FALSE(visits[1]
  761. .annotated_visit.context_annotations
  762. .is_existing_part_of_tab_group);
  763. EXPECT_FLOAT_EQ(visits[1].score, 0.5);
  764. ASSERT_EQ(cluster.keyword_to_data_map.size(), 2u);
  765. EXPECT_TRUE(cluster.keyword_to_data_map.contains(u"apples"));
  766. EXPECT_TRUE(cluster.keyword_to_data_map.contains(u"Red Oranges"));
  767. cluster = clusters[1];
  768. visits = cluster.visits;
  769. ASSERT_EQ(visits.size(), 1u);
  770. EXPECT_EQ(visits[0].annotated_visit.url_row.url(),
  771. "https://github.com/");
  772. EXPECT_EQ(visits[0].annotated_visit.visit_row.visit_time,
  773. GetHardcodedTestVisits()[1].visit_row.visit_time);
  774. EXPECT_EQ(visits[0].annotated_visit.url_row.title(),
  775. u"Code Storage Title");
  776. EXPECT_TRUE(cluster.keyword_to_data_map.empty());
  777. run_loop_quit.Run();
  778. }));
  779. AwaitAndVerifyTestClusteringBackendRequest();
  780. std::vector<history::Cluster> clusters;
  781. clusters.push_back(
  782. history::Cluster(0,
  783. {
  784. test_clustering_backend_->GetVisitById(2),
  785. test_clustering_backend_->GetVisitById(5),
  786. },
  787. {{u"apples", history::ClusterKeywordData()},
  788. {u"Red Oranges", history::ClusterKeywordData()}},
  789. /*should_show_on_prominent_ui_surfaces=*/true));
  790. clusters.push_back(
  791. history::Cluster(0,
  792. {
  793. test_clustering_backend_->GetVisitById(2),
  794. },
  795. {},
  796. /*should_show_on_prominent_ui_surfaces=*/true));
  797. test_clustering_backend_->FulfillCallback(clusters);
  798. // Verify the callback is invoked.
  799. run_loop.Run();
  800. history::BlockUntilHistoryProcessesPendingRequests(history_service_.get());
  801. histogram_tester.ExpectBucketCount(
  802. "History.Clusters.Backend.NumClustersReturned", 2, 1);
  803. histogram_tester.ExpectBucketCount(
  804. "History.Clusters.Backend.NumVisitsToCluster", 2, 1);
  805. histogram_tester.ExpectTotalCount(
  806. "History.Clusters.Backend.GetClustersLatency", 1);
  807. }
  808. TEST_F(HistoryClustersServiceTest, CompleteVisitContextAnnotationsIfReady) {
  809. auto test = [&](RecordingStatus status, bool expected_complete) {
  810. auto& incomplete_visit_context_annotations =
  811. history_clusters_service_->GetOrCreateIncompleteVisitContextAnnotations(
  812. 0);
  813. incomplete_visit_context_annotations.url_row.set_id(1);
  814. incomplete_visit_context_annotations.visit_row.visit_id = 1;
  815. incomplete_visit_context_annotations.status = status;
  816. history_clusters_service_->CompleteVisitContextAnnotationsIfReady(0);
  817. EXPECT_NE(
  818. history_clusters_service_->HasIncompleteVisitContextAnnotations(0),
  819. expected_complete);
  820. };
  821. // Complete cases:
  822. {
  823. SCOPED_TRACE("Complete without UKM");
  824. test({true, true, true}, true);
  825. }
  826. {
  827. SCOPED_TRACE("Complete with UKM");
  828. test({true, true, true, true, true}, true);
  829. }
  830. // Incomplete without UKM cases:
  831. {
  832. SCOPED_TRACE("Incomplete, missing history rows");
  833. test({false, true}, false);
  834. }
  835. {
  836. SCOPED_TRACE("Incomplete, navigation hasn't ended");
  837. test({true}, false);
  838. }
  839. {
  840. SCOPED_TRACE("Incomplete, navigation end metrics haven't been recorded");
  841. test({true, true}, false);
  842. }
  843. // Incomplete with UKM cases:
  844. {
  845. SCOPED_TRACE("Incomplete, missing history rows");
  846. test({false, true, false, true, true}, false);
  847. }
  848. {
  849. SCOPED_TRACE("Incomplete, navigation hasn't ended");
  850. test({true, false, false, true, true}, false);
  851. }
  852. {
  853. SCOPED_TRACE("Incomplete, navigation end metrics haven't been recorded");
  854. test({true, true, false, true, true}, false);
  855. }
  856. {
  857. SCOPED_TRACE("Incomplete, UKM page end missing");
  858. test({true, true, true, true, false}, false);
  859. }
  860. auto test_dcheck = [&](RecordingStatus status) {
  861. auto& incomplete_visit_context_annotations =
  862. history_clusters_service_->GetOrCreateIncompleteVisitContextAnnotations(
  863. 0);
  864. incomplete_visit_context_annotations.url_row.set_id(1);
  865. incomplete_visit_context_annotations.visit_row.visit_id = 1;
  866. incomplete_visit_context_annotations.status = status;
  867. EXPECT_DCHECK_DEATH(
  868. history_clusters_service_->CompleteVisitContextAnnotationsIfReady(0));
  869. EXPECT_TRUE(
  870. history_clusters_service_->HasIncompleteVisitContextAnnotations(0));
  871. };
  872. // Impossible cases:
  873. {
  874. SCOPED_TRACE(
  875. "Impossible, navigation end signals recorded before navigation ended");
  876. test_dcheck({true, false, true});
  877. }
  878. {
  879. SCOPED_TRACE(
  880. "Impossible, navigation end signals recorded before history rows");
  881. test_dcheck({false, true, true});
  882. }
  883. {
  884. SCOPED_TRACE("Impossible, unexpected UKM page end recorded");
  885. test_dcheck({false, false, false, false, true});
  886. }
  887. }
  888. class HistoryClustersServiceJourneysDisabledTest
  889. : public HistoryClustersServiceTestBase {
  890. public:
  891. HistoryClustersServiceJourneysDisabledTest() {
  892. scoped_feature_list_.InitWithFeatures(
  893. /*enabled_features=*/{},
  894. /*disabled_features=*/{
  895. internal::kJourneys,
  896. internal::kPersistContextAnnotationsInHistoryDb,
  897. });
  898. }
  899. };
  900. TEST_F(HistoryClustersServiceJourneysDisabledTest,
  901. CompleteVisitContextAnnotationsIfReadyWhenFeatureDisabled) {
  902. // When the feature is disabled, the `IncompleteVisitContextAnnotations`
  903. // should be removed but not added to visits.
  904. auto& incomplete_visit_context_annotations =
  905. history_clusters_service_->GetOrCreateIncompleteVisitContextAnnotations(
  906. 0);
  907. incomplete_visit_context_annotations.url_row.set_id(1);
  908. incomplete_visit_context_annotations.visit_row.visit_id = 1;
  909. incomplete_visit_context_annotations.status = {true, true, true};
  910. history_clusters_service_->CompleteVisitContextAnnotationsIfReady(0);
  911. EXPECT_FALSE(
  912. history_clusters_service_->HasIncompleteVisitContextAnnotations(0));
  913. }
  914. TEST_F(HistoryClustersServiceTest,
  915. CompleteVisitContextAnnotationsIfReadyWhenFeatureEnabled) {
  916. // When the feature is enabled, the `IncompleteVisitContextAnnotations`
  917. // should be removed and added to visits.
  918. auto& incomplete_visit_context_annotations =
  919. history_clusters_service_->GetOrCreateIncompleteVisitContextAnnotations(
  920. 0);
  921. incomplete_visit_context_annotations.url_row.set_id(1);
  922. incomplete_visit_context_annotations.visit_row.visit_id = 1;
  923. incomplete_visit_context_annotations.status = {true, true, true};
  924. history_clusters_service_->CompleteVisitContextAnnotationsIfReady(0);
  925. EXPECT_FALSE(
  926. history_clusters_service_->HasIncompleteVisitContextAnnotations(0));
  927. }
  928. TEST_F(HistoryClustersServiceTest, DoesQueryMatchAnyCluster) {
  929. AddHardcodedTestDataToHistoryService();
  930. // Verify that initially, the test keyword doesn't match anything, but this
  931. // query should have kicked off a cache population request.
  932. EXPECT_FALSE(history_clusters_service_->DoesQueryMatchAnyCluster("apples"));
  933. std::vector<history::Cluster> clusters;
  934. clusters.push_back(history::Cluster(
  935. 0,
  936. {
  937. GetHardcodedClusterVisit(5),
  938. GetHardcodedClusterVisit(2),
  939. },
  940. {{u"apples", history::ClusterKeywordData(
  941. history::ClusterKeywordData::kEntity, 5.0f, {})},
  942. {u"oranges", history::ClusterKeywordData()},
  943. {u"z", history::ClusterKeywordData()},
  944. {u"apples bananas", history::ClusterKeywordData()}},
  945. /*should_show_on_prominent_ui_surfaces=*/true));
  946. clusters.push_back(history::Cluster(
  947. 0,
  948. {
  949. GetHardcodedClusterVisit(5),
  950. GetHardcodedClusterVisit(2),
  951. },
  952. {
  953. {u"apples",
  954. history::ClusterKeywordData(
  955. history::ClusterKeywordData::kSearchTerms, 100.0f, {})},
  956. },
  957. /*should_show_on_prominent_ui_surfaces=*/true));
  958. clusters.push_back(
  959. history::Cluster(0,
  960. {
  961. GetHardcodedClusterVisit(5),
  962. GetHardcodedClusterVisit(2),
  963. },
  964. {{u"sensitive", history::ClusterKeywordData()}},
  965. /*should_show_on_prominent_ui_surfaces=*/false));
  966. clusters.push_back(
  967. history::Cluster(0,
  968. {
  969. GetHardcodedClusterVisit(5),
  970. },
  971. {{u"singlevisit", history::ClusterKeywordData()}},
  972. /*should_show_on_prominent_ui_surfaces=*/true));
  973. // Hardcoded test visits span 3 days (1-day-old, 2-days-old, and 60-day-old).
  974. FlushKeywordRequests(clusters, 3);
  975. // Now the exact query should match the populated cache.
  976. const auto keyword_data =
  977. history_clusters_service_->DoesQueryMatchAnyCluster("apples");
  978. EXPECT_TRUE(keyword_data);
  979. // Its keyword data type is kSearchTerms as it has a higher score.
  980. EXPECT_EQ(keyword_data,
  981. history::ClusterKeywordData(
  982. history::ClusterKeywordData::kSearchTerms, 100.0f, {}));
  983. // Check that clusters that shouldn't be shown on prominent UI surfaces don't
  984. // have their keywords inserted into the keyword bag.
  985. EXPECT_FALSE(
  986. history_clusters_service_->DoesQueryMatchAnyCluster("sensitive"));
  987. // Ignore clusters with fewer than two visits.
  988. EXPECT_FALSE(
  989. history_clusters_service_->DoesQueryMatchAnyCluster("singlevisit"));
  990. // Too-short prefix queries rejected.
  991. EXPECT_FALSE(history_clusters_service_->DoesQueryMatchAnyCluster("ap"));
  992. // Single character exact queries are also rejected.
  993. EXPECT_FALSE(history_clusters_service_->DoesQueryMatchAnyCluster("z"));
  994. // Non-exact (substring) matches are rejected too.
  995. EXPECT_FALSE(history_clusters_service_->DoesQueryMatchAnyCluster("appl"));
  996. // Adding a second non-exact query word also should make it no longer match.
  997. EXPECT_FALSE(
  998. history_clusters_service_->DoesQueryMatchAnyCluster("apples oran"));
  999. // A multi-word phrase shouldn't be considered a match against two separate
  1000. // keywords: "apples oranges" can't match keywords ["apples", "oranges"].
  1001. EXPECT_FALSE(
  1002. history_clusters_service_->DoesQueryMatchAnyCluster("apples oranges"));
  1003. // But a multi-word phrases can still match against a keyword with multiple
  1004. // words: "apples bananas" matches ["apples bananas"].
  1005. EXPECT_TRUE(
  1006. history_clusters_service_->DoesQueryMatchAnyCluster("apples bananas"));
  1007. // Deleting a history entry should clear the keyword cache.
  1008. history_service_->DeleteURLs({GURL{"https://google.com/"}});
  1009. history::BlockUntilHistoryProcessesPendingRequests(history_service_.get());
  1010. EXPECT_FALSE(history_clusters_service_->DoesQueryMatchAnyCluster("apples"));
  1011. // Visits now span 2 days (1-day-old and 60-day-old) since we deleted the only
  1012. // 2-day-old visit.
  1013. FlushKeywordRequests(clusters, 2);
  1014. // The keyword cache should be repopulated.
  1015. EXPECT_TRUE(history_clusters_service_->DoesQueryMatchAnyCluster("apples"));
  1016. }
  1017. TEST_F(HistoryClustersServiceTest, DoesQueryMatchAnyClusterSecondaryCache) {
  1018. auto minutes_ago = [](int minutes) {
  1019. return base::Time::Now() - base::Minutes(minutes);
  1020. };
  1021. // Set up the cache timestamps.
  1022. history_clusters_service_test_api_->SetAllKeywordsCacheTimestamp(
  1023. minutes_ago(60));
  1024. history_clusters_service_test_api_->SetShortKeywordCacheTimestamp(
  1025. minutes_ago(15));
  1026. // Set up the visit timestamps.
  1027. // Visits newer than both cache timestamps should be reclustered.
  1028. AddIncompleteVisit(1, 1, minutes_ago(5));
  1029. // Visits older than the secondary cache timestamp should be reclustered.
  1030. AddIncompleteVisit(2, 2, minutes_ago(30));
  1031. // Visits older than the primary cache timestamp should not be reclustered.
  1032. AddIncompleteVisit(3, 3, minutes_ago(70));
  1033. // Kick off cluster request and verify the correct visits are sent.
  1034. EXPECT_FALSE(history_clusters_service_->DoesQueryMatchAnyCluster("peach"));
  1035. test_clustering_backend_->WaitForGetClustersCall();
  1036. std::vector<history::AnnotatedVisit> visits =
  1037. test_clustering_backend_->LastClusteredVisits();
  1038. EXPECT_THAT(GetVisitIds(visits), testing::ElementsAre(1, 2));
  1039. // Send the cluster response and verify the keyword was cached.
  1040. std::vector<history::Cluster> clusters2;
  1041. clusters2.push_back(
  1042. history::Cluster(0,
  1043. {
  1044. test_clustering_backend_->GetVisitById(1),
  1045. test_clustering_backend_->GetVisitById(2),
  1046. },
  1047. {{u"peach", history::ClusterKeywordData()},
  1048. {u"", history::ClusterKeywordData()}},
  1049. /*should_show_on_prominent_ui_surfaces=*/true));
  1050. test_clustering_backend_->FulfillCallback(clusters2);
  1051. history::BlockUntilHistoryProcessesPendingRequests(history_service_.get());
  1052. EXPECT_TRUE(history_clusters_service_->DoesQueryMatchAnyCluster("peach"));
  1053. }
  1054. TEST_F(HistoryClustersServiceTest, DoesURLMatchAnyClusterWithNoisyURLs) {
  1055. Config config;
  1056. config.omnibox_action_on_urls = true;
  1057. config.omnibox_action_on_noisy_urls = true;
  1058. SetConfigForTesting(config);
  1059. AddHardcodedTestDataToHistoryService();
  1060. // Verify that initially, the test URL doesn't match anything, but this
  1061. // query should have kicked off a cache population request. This is the URL
  1062. // for visit 5.
  1063. EXPECT_FALSE(history_clusters_service_->DoesURLMatchAnyCluster(
  1064. ComputeURLKeywordForLookup(GURL("https://second-1-day-old-visit.com/"))));
  1065. std::vector<history::Cluster> clusters;
  1066. clusters.push_back(history::Cluster(
  1067. 0,
  1068. {
  1069. GetHardcodedClusterVisit(5),
  1070. GetHardcodedClusterVisit(
  1071. /*visit_id=*/2, /*score=*/0.0, /*engagement_score=*/20.0),
  1072. },
  1073. {{u"apples", history::ClusterKeywordData()},
  1074. {u"oranges", history::ClusterKeywordData()},
  1075. {u"z", history::ClusterKeywordData()},
  1076. {u"apples bananas", history::ClusterKeywordData()}},
  1077. /*should_show_on_prominent_ui_surfaces=*/true));
  1078. clusters.push_back(
  1079. history::Cluster(0,
  1080. {
  1081. GetHardcodedClusterVisit(5),
  1082. GetHardcodedClusterVisit(2),
  1083. },
  1084. {{u"sensitive", history::ClusterKeywordData()}},
  1085. /*should_show_on_prominent_ui_surfaces=*/false));
  1086. clusters.push_back(
  1087. history::Cluster(0,
  1088. {
  1089. GetHardcodedClusterVisit(2),
  1090. },
  1091. {{u"singlevisit", history::ClusterKeywordData()}},
  1092. /*should_show_on_prominent_ui_surfaces=*/true));
  1093. // Hardcoded test visits span 3 days (1-day-old, 2-days-old, and 60-day-old).
  1094. FlushKeywordRequests(clusters, 3);
  1095. // Now the exact query should match the populated cache.
  1096. EXPECT_TRUE(history_clusters_service_->DoesURLMatchAnyCluster(
  1097. ComputeURLKeywordForLookup(GURL("https://second-1-day-old-visit.com/"))));
  1098. // Github should be shown since we are including visits from noisy URLs.
  1099. EXPECT_TRUE(history_clusters_service_->DoesURLMatchAnyCluster(
  1100. ComputeURLKeywordForLookup(GURL("https://github.com/"))));
  1101. // Deleting a history entry should clear the keyword cache.
  1102. history_service_->DeleteURLs({GURL{"https://google.com/"}});
  1103. history::BlockUntilHistoryProcessesPendingRequests(history_service_.get());
  1104. EXPECT_FALSE(history_clusters_service_->DoesURLMatchAnyCluster(
  1105. ComputeURLKeywordForLookup(GURL("https://second-1-day-old-visit.com/"))));
  1106. // Visits now span 2 days (1-day-old and 60-day-old) since we deleted the only
  1107. // 2-day-old visit.
  1108. FlushKeywordRequests(clusters, 2);
  1109. // The keyword cache should be repopulated.
  1110. EXPECT_TRUE(history_clusters_service_->DoesURLMatchAnyCluster(
  1111. ComputeURLKeywordForLookup(GURL("https://second-1-day-old-visit.com/"))));
  1112. }
  1113. TEST_F(HistoryClustersServiceTest, DoesURLMatchAnyClusterNoNoisyURLs) {
  1114. Config config;
  1115. config.omnibox_action_on_urls = true;
  1116. config.omnibox_action_on_noisy_urls = false;
  1117. SetConfigForTesting(config);
  1118. AddHardcodedTestDataToHistoryService();
  1119. // Verify that initially, the test URL doesn't match anything, but this
  1120. // query should have kicked off a cache population request. This is the URL
  1121. // for visit 5.
  1122. EXPECT_FALSE(history_clusters_service_->DoesURLMatchAnyCluster(
  1123. ComputeURLKeywordForLookup(GURL("https://second-1-day-old-visit.com/"))));
  1124. std::vector<history::Cluster> clusters;
  1125. clusters.push_back(history::Cluster(
  1126. 0,
  1127. {
  1128. GetHardcodedClusterVisit(5),
  1129. GetHardcodedClusterVisit(
  1130. /*visit_id=*/2, /*score=*/0.0, /*engagement_score=*/20.0),
  1131. },
  1132. {{u"apples", history::ClusterKeywordData()},
  1133. {u"oranges", history::ClusterKeywordData()},
  1134. {u"z", history::ClusterKeywordData()},
  1135. {u"apples bananas", history::ClusterKeywordData()}},
  1136. /*should_show_on_prominent_ui_surfaces=*/true));
  1137. clusters.push_back(
  1138. history::Cluster(0,
  1139. {
  1140. GetHardcodedClusterVisit(5),
  1141. GetHardcodedClusterVisit(2),
  1142. },
  1143. {{u"sensitive", history::ClusterKeywordData()}},
  1144. /*should_show_on_prominent_ui_surfaces=*/false));
  1145. clusters.push_back(
  1146. history::Cluster(0,
  1147. {
  1148. GetHardcodedClusterVisit(2),
  1149. },
  1150. {{u"singlevisit", history::ClusterKeywordData()}},
  1151. /*should_show_on_prominent_ui_surfaces=*/true));
  1152. // Hardcoded test visits span 3 days (1-day-old, 2-days-old, and 60-day-old).
  1153. FlushKeywordRequests(clusters, 3);
  1154. // Now the exact query should match the populated cache.
  1155. EXPECT_TRUE(history_clusters_service_->DoesURLMatchAnyCluster(
  1156. ComputeURLKeywordForLookup(GURL("https://second-1-day-old-visit.com/"))));
  1157. // Github should never be shown (highly-engaged for cluster 1, sensitive for
  1158. // cluster 2, single visit cluster for cluster 3).
  1159. EXPECT_FALSE(history_clusters_service_->DoesURLMatchAnyCluster(
  1160. ComputeURLKeywordForLookup(GURL("https://github.com/"))));
  1161. // Deleting a history entry should clear the keyword cache.
  1162. history_service_->DeleteURLs({GURL{"https://google.com/"}});
  1163. history::BlockUntilHistoryProcessesPendingRequests(history_service_.get());
  1164. EXPECT_FALSE(history_clusters_service_->DoesURLMatchAnyCluster(
  1165. ComputeURLKeywordForLookup(GURL("https://second-1-day-old-visit.com/"))));
  1166. // Visits now span 2 days (1-day-old and 60-day-old) since we deleted the only
  1167. // 2-day-old visit.
  1168. FlushKeywordRequests(clusters, 2);
  1169. // The keyword cache should be repopulated.
  1170. EXPECT_TRUE(history_clusters_service_->DoesURLMatchAnyCluster(
  1171. ComputeURLKeywordForLookup(GURL("https://second-1-day-old-visit.com/"))));
  1172. }
  1173. class HistoryClustersServiceMaxKeywordsTest
  1174. : public HistoryClustersServiceTestBase {
  1175. public:
  1176. HistoryClustersServiceMaxKeywordsTest() {
  1177. // Set the max keyword phrases to 5.
  1178. config_.is_journeys_enabled_no_locale_check = true;
  1179. config_.max_keyword_phrases = 5;
  1180. SetConfigForTesting(config_);
  1181. }
  1182. private:
  1183. Config config_;
  1184. };
  1185. TEST_F(HistoryClustersServiceMaxKeywordsTest,
  1186. DoesQueryMatchAnyClusterMaxKeywordPhrases) {
  1187. base::HistogramTester histogram_tester;
  1188. // Add visits.
  1189. const auto yesterday = base::Time::Now() - base::Days(1);
  1190. AddIncompleteVisit(1, 1, yesterday);
  1191. AddIncompleteVisit(2, 2, yesterday);
  1192. AddIncompleteVisit(3, 3, yesterday);
  1193. AddIncompleteVisit(4, 4, yesterday);
  1194. AddIncompleteVisit(5, 5, yesterday);
  1195. AddIncompleteVisit(6, 6, yesterday);
  1196. AddIncompleteVisit(7, 7, yesterday);
  1197. // Create 4 clusters:
  1198. std::vector<history::AnnotatedVisit> visits =
  1199. test_clustering_backend_->LastClusteredVisits();
  1200. std::vector<history::Cluster> clusters;
  1201. // 1) A cluster with 4 phrases and 6 words. The next cluster's keywords should
  1202. // also be cached since we have less than 5 phrases.
  1203. clusters.push_back(
  1204. history::Cluster(0, {{}, {}},
  1205. {{u"one", history::ClusterKeywordData()},
  1206. {u"two", history::ClusterKeywordData()},
  1207. {u"three", history::ClusterKeywordData()},
  1208. {u"four five six", history::ClusterKeywordData()}},
  1209. /*should_show_on_prominent_ui_surfaces=*/true));
  1210. // 2) The 2nd cluster has only 1 visit. Since it's keywords won't be cached,
  1211. // they should not affect the max.
  1212. clusters.push_back(history::Cluster(
  1213. 0, {{}},
  1214. {{u"ignored not cached", history::ClusterKeywordData()},
  1215. {u"elephant penguin kangaroo", history::ClusterKeywordData()}},
  1216. /*should_show_on_prominent_ui_surfaces=*/true));
  1217. // 3) With this 3rd cluster, we'll have 5 phrases and 7 words. Now that we've
  1218. // reached 5 phrases, the next cluster's keywords should not be cached.
  1219. clusters.push_back(
  1220. history::Cluster(0, {{}, {}}, {{u"seven", history::ClusterKeywordData()}},
  1221. /*should_show_on_prominent_ui_surfaces=*/true));
  1222. // 4) The 4th cluster's keywords should not be cached since we've reached 5
  1223. // phrases.
  1224. clusters.push_back(
  1225. history::Cluster(0, {{}, {}}, {{u"eight", history::ClusterKeywordData()}},
  1226. /*should_show_on_prominent_ui_surfaces=*/true));
  1227. // Kick off cluster request.
  1228. EXPECT_FALSE(history_clusters_service_->DoesQueryMatchAnyCluster("peach"));
  1229. FlushKeywordRequests(clusters, 1);
  1230. ASSERT_EQ(test_clustering_backend_->LastClusteredVisits().size(), 7u);
  1231. // The 1st cluster's phrases should always be cached.
  1232. EXPECT_TRUE(history_clusters_service_->DoesQueryMatchAnyCluster("one"));
  1233. EXPECT_TRUE(
  1234. history_clusters_service_->DoesQueryMatchAnyCluster("four five six"));
  1235. // Phrases should be cached if we haven't reached 5 phrases even if we've
  1236. // reached 5 words.
  1237. EXPECT_TRUE(history_clusters_service_->DoesQueryMatchAnyCluster("seven"));
  1238. // Phrases after the first 5 won't be cached.
  1239. EXPECT_FALSE(history_clusters_service_->DoesQueryMatchAnyCluster("eight"));
  1240. // Phrases of cluster's with 1 visit won't be cached.
  1241. EXPECT_FALSE(history_clusters_service_->DoesQueryMatchAnyCluster("penguin"));
  1242. histogram_tester.ExpectUniqueSample(
  1243. "History.Clusters.Backend.KeywordCache.AllKeywordsCount", 5, 1);
  1244. histogram_tester.ExpectTotalCount(
  1245. "History.Clusters.Backend.KeywordCache.ShortKeywordsCount", 0);
  1246. }
  1247. } // namespace
  1248. } // namespace history_clusters