variations_seed_simulator_unittest.cc 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415
  1. // Copyright 2014 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. #include "components/variations/variations_seed_simulator.h"
  5. #include <stdint.h>
  6. #include <map>
  7. #include "base/strings/stringprintf.h"
  8. #include "base/test/mock_entropy_provider.h"
  9. #include "base/time/time.h"
  10. #include "components/variations/processed_study.h"
  11. #include "components/variations/proto/study.pb.h"
  12. #include "components/variations/variations_associated_data.h"
  13. #include "testing/gtest/include/gtest/gtest.h"
  14. namespace variations {
  15. namespace {
  16. // Converts |time| to Study proto format.
  17. int64_t TimeToProtoTime(const base::Time& time) {
  18. return (time - base::Time::UnixEpoch()).InSeconds();
  19. }
  20. // Creates and activates a single-group field trial with name |trial_name| and
  21. // group |group_name| and variations |params| (if not null).
  22. void CreateTrial(const std::string& trial_name,
  23. const std::string& group_name,
  24. const std::map<std::string, std::string>* params) {
  25. base::FieldTrialList::CreateFieldTrial(trial_name, group_name);
  26. if (params != nullptr)
  27. AssociateVariationParams(trial_name, group_name, *params);
  28. base::FieldTrialList::FindFullName(trial_name);
  29. }
  30. // Creates a study with the given |study_name| and |consistency|.
  31. Study CreateStudy(const std::string& study_name,
  32. Study_Consistency consistency) {
  33. Study study;
  34. study.set_name(study_name);
  35. study.set_consistency(consistency);
  36. return study;
  37. }
  38. // Adds an experiment to |study| with the specified |experiment_name| and
  39. // |probability| values and sets it as the study's default experiment.
  40. Study_Experiment* AddExperiment(const std::string& experiment_name,
  41. int probability,
  42. Study* study) {
  43. Study_Experiment* experiment = study->add_experiment();
  44. experiment->set_name(experiment_name);
  45. experiment->set_probability_weight(probability);
  46. study->set_default_experiment_name(experiment_name);
  47. return experiment;
  48. }
  49. // Add an experiment param with |param_name| and |param_value| to |experiment|.
  50. Study_Experiment_Param* AddExperimentParam(const std::string& param_name,
  51. const std::string& param_value,
  52. Study_Experiment* experiment) {
  53. Study_Experiment_Param* param = experiment->add_param();
  54. param->set_name(param_name);
  55. param->set_value(param_value);
  56. return param;
  57. }
  58. } // namespace
  59. class VariationsSeedSimulatorTest : public ::testing::Test {
  60. public:
  61. VariationsSeedSimulatorTest() {}
  62. VariationsSeedSimulatorTest(const VariationsSeedSimulatorTest&) = delete;
  63. VariationsSeedSimulatorTest& operator=(const VariationsSeedSimulatorTest&) =
  64. delete;
  65. ~VariationsSeedSimulatorTest() override {
  66. // Ensure that the maps are cleared between tests, since they are stored as
  67. // process singletons.
  68. testing::ClearAllVariationIDs();
  69. testing::ClearAllVariationParams();
  70. }
  71. // Uses a VariationsSeedSimulator to simulate the differences between
  72. // |studies| and the current field trial state.
  73. VariationsSeedSimulator::Result SimulateDifferences(
  74. const std::vector<ProcessedStudy>& studies) {
  75. // Should pick the first group that has non-zero probability weight.
  76. base::MockEntropyProvider default_provider(0);
  77. // Should pick default groups, if they have non-zero probability weight.
  78. base::MockEntropyProvider low_provider(1.0 - 1e-8);
  79. VariationsSeedSimulator seed_simulator(default_provider, low_provider);
  80. return seed_simulator.ComputeDifferences(studies);
  81. }
  82. // Simulates the differences between |study| and the current field trial
  83. // state, returning a string like "1 2 3", where 1 is the number of regular
  84. // group changes, 2 is the number of "kill best effort" group changes and 3
  85. // is the number of "kill critical" group changes.
  86. std::string SimulateStudyDifferences(const Study* study) {
  87. ProcessedStudy processed_study;
  88. if (!processed_study.Init(study, false))
  89. return "invalid study";
  90. std::vector<ProcessedStudy> studies = {processed_study};
  91. return ConvertSimulationResultToString(SimulateDifferences(studies));
  92. }
  93. // Simulates the differences between expired |study| and the current field
  94. // trial state, returning a string like "1 2 3", where 1 is the number of
  95. // regular group changes, 2 is the number of "kill best effort" group changes
  96. // and 3 is the number of "kill critical" group changes.
  97. std::string SimulateStudyDifferencesExpired(const Study* study) {
  98. ProcessedStudy processed_study;
  99. if (!processed_study.Init(study, true))
  100. return "invalid study";
  101. if (!processed_study.is_expired())
  102. return "not expired";
  103. std::vector<ProcessedStudy> studies = {processed_study};
  104. return ConvertSimulationResultToString(SimulateDifferences(studies));
  105. }
  106. // Formats |result| as a string with format "1 2 3", where 1 is the number of
  107. // regular group changes, 2 is the number of "kill best effort" group changes
  108. // and 3 is the number of "kill critical" group changes.
  109. std::string ConvertSimulationResultToString(
  110. const VariationsSeedSimulator::Result& result) {
  111. return base::StringPrintf("%d %d %d",
  112. result.normal_group_change_count,
  113. result.kill_best_effort_group_change_count,
  114. result.kill_critical_group_change_count);
  115. }
  116. };
  117. TEST_F(VariationsSeedSimulatorTest, PermanentNoChanges) {
  118. CreateTrial("A", "B", nullptr);
  119. std::vector<ProcessedStudy> processed_studies;
  120. Study study = CreateStudy("A", Study_Consistency_PERMANENT);
  121. Study_Experiment* experiment = AddExperiment("B", 100, &study);
  122. EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
  123. experiment->set_type(Study_Experiment_Type_NORMAL);
  124. EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
  125. experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
  126. EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
  127. experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
  128. EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
  129. experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
  130. EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
  131. }
  132. TEST_F(VariationsSeedSimulatorTest, PermanentGroupChange) {
  133. CreateTrial("A", "B", nullptr);
  134. Study study = CreateStudy("A", Study_Consistency_PERMANENT);
  135. Study_Experiment* experiment = AddExperiment("C", 100, &study);
  136. EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
  137. // Changing "C" group type should not affect the type of change. (Since the
  138. // type is evaluated for the "old" group.)
  139. //
  140. // Note: The current (i.e. old) group is checked for the type since that group
  141. // is the one that should be annotated with the type when killing it.
  142. experiment->set_type(Study_Experiment_Type_NORMAL);
  143. EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
  144. experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
  145. EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
  146. experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
  147. EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
  148. experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
  149. EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
  150. }
  151. TEST_F(VariationsSeedSimulatorTest, PermanentGroupChangeDueToExperimentID) {
  152. CreateTrial("A", "B", nullptr);
  153. CreateTrial("X", "Y", nullptr);
  154. Study study = CreateStudy("A", Study_Consistency_PERMANENT);
  155. Study_Experiment* experiment_b = AddExperiment("B", 50, &study);
  156. AddExperiment("Default", 50, &study);
  157. EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
  158. // Adding a google_web_experiment_id will cause the low entropy provider to be
  159. // used, causing a group change.
  160. experiment_b->set_google_web_experiment_id(1234);
  161. EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
  162. }
  163. TEST_F(VariationsSeedSimulatorTest, PermanentExpired) {
  164. CreateTrial("A", "B", nullptr);
  165. Study study = CreateStudy("A", Study_Consistency_PERMANENT);
  166. Study_Experiment* experiment = AddExperiment("B", 1, &study);
  167. AddExperiment("C", 0, &study);
  168. // There should be a difference because the study is expired, which should
  169. // result in the default group "C" being chosen.
  170. EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
  171. experiment->set_type(Study_Experiment_Type_NORMAL);
  172. EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
  173. experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
  174. EXPECT_EQ("0 0 0", SimulateStudyDifferencesExpired(&study));
  175. experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
  176. EXPECT_EQ("0 1 0", SimulateStudyDifferencesExpired(&study));
  177. experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
  178. EXPECT_EQ("0 0 1", SimulateStudyDifferencesExpired(&study));
  179. }
  180. TEST_F(VariationsSeedSimulatorTest, SessionRandomized) {
  181. CreateTrial("A", "B", nullptr);
  182. Study study = CreateStudy("A", Study_Consistency_SESSION);
  183. Study_Experiment* experiment = AddExperiment("B", 1, &study);
  184. AddExperiment("C", 1, &study);
  185. AddExperiment("D", 1, &study);
  186. // There should be no differences, since a session randomized study can result
  187. // in any of the groups being chosen on startup.
  188. EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
  189. experiment->set_type(Study_Experiment_Type_NORMAL);
  190. EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
  191. experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
  192. EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
  193. experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
  194. EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
  195. experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
  196. EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
  197. }
  198. TEST_F(VariationsSeedSimulatorTest, SessionRandomizedGroupRemoved) {
  199. CreateTrial("A", "B", nullptr);
  200. Study study = CreateStudy("A", Study_Consistency_SESSION);
  201. AddExperiment("C", 1, &study);
  202. AddExperiment("D", 1, &study);
  203. // There should be a difference since there is no group "B" in the new config.
  204. EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
  205. }
  206. TEST_F(VariationsSeedSimulatorTest, SessionRandomizedGroupProbabilityZero) {
  207. CreateTrial("A", "B", nullptr);
  208. Study study = CreateStudy("A", Study_Consistency_SESSION);
  209. Study_Experiment* experiment = AddExperiment("B", 0, &study);
  210. AddExperiment("C", 1, &study);
  211. AddExperiment("D", 1, &study);
  212. // There should be a difference since group "B" has probability 0.
  213. EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
  214. experiment->set_type(Study_Experiment_Type_NORMAL);
  215. EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
  216. experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
  217. EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
  218. experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
  219. EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
  220. experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
  221. EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
  222. }
  223. TEST_F(VariationsSeedSimulatorTest, SessionRandomizedExpired) {
  224. CreateTrial("A", "B", nullptr);
  225. Study study = CreateStudy("A", Study_Consistency_SESSION);
  226. Study_Experiment* experiment = AddExperiment("B", 1, &study);
  227. AddExperiment("C", 1, &study);
  228. AddExperiment("D", 1, &study);
  229. // There should be a difference because the study is expired, which should
  230. // result in the default group "D" being chosen.
  231. EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
  232. experiment->set_type(Study_Experiment_Type_NORMAL);
  233. EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
  234. experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
  235. EXPECT_EQ("0 0 0", SimulateStudyDifferencesExpired(&study));
  236. experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
  237. EXPECT_EQ("0 1 0", SimulateStudyDifferencesExpired(&study));
  238. experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
  239. EXPECT_EQ("0 0 1", SimulateStudyDifferencesExpired(&study));
  240. }
  241. TEST_F(VariationsSeedSimulatorTest, ParamsUnchanged) {
  242. std::map<std::string, std::string> params;
  243. params["p1"] = "x";
  244. params["p2"] = "y";
  245. params["p3"] = "z";
  246. CreateTrial("A", "B", &params);
  247. std::vector<ProcessedStudy> processed_studies;
  248. Study study = CreateStudy("A", Study_Consistency_PERMANENT);
  249. Study_Experiment* experiment = AddExperiment("B", 100, &study);
  250. AddExperimentParam("p2", "y", experiment);
  251. AddExperimentParam("p1", "x", experiment);
  252. AddExperimentParam("p3", "z", experiment);
  253. EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
  254. experiment->set_type(Study_Experiment_Type_NORMAL);
  255. EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
  256. experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
  257. EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
  258. experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
  259. EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
  260. experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
  261. EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
  262. }
  263. TEST_F(VariationsSeedSimulatorTest, ParamsChanged) {
  264. std::map<std::string, std::string> params;
  265. params["p1"] = "x";
  266. params["p2"] = "y";
  267. params["p3"] = "z";
  268. CreateTrial("A", "B", &params);
  269. std::vector<ProcessedStudy> processed_studies;
  270. Study study = CreateStudy("A", Study_Consistency_PERMANENT);
  271. Study_Experiment* experiment = AddExperiment("B", 100, &study);
  272. AddExperimentParam("p2", "test", experiment);
  273. AddExperimentParam("p1", "x", experiment);
  274. AddExperimentParam("p3", "z", experiment);
  275. // The param lists differ.
  276. EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
  277. experiment->set_type(Study_Experiment_Type_NORMAL);
  278. EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
  279. experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
  280. EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
  281. experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
  282. EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
  283. experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
  284. EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
  285. }
  286. TEST_F(VariationsSeedSimulatorTest, ParamsRemoved) {
  287. std::map<std::string, std::string> params;
  288. params["p1"] = "x";
  289. params["p2"] = "y";
  290. params["p3"] = "z";
  291. CreateTrial("A", "B", &params);
  292. std::vector<ProcessedStudy> processed_studies;
  293. Study study = CreateStudy("A", Study_Consistency_PERMANENT);
  294. Study_Experiment* experiment = AddExperiment("B", 100, &study);
  295. // The current group has params, but the new config doesn't have any.
  296. EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
  297. experiment->set_type(Study_Experiment_Type_NORMAL);
  298. EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
  299. experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
  300. EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
  301. experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
  302. EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
  303. experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
  304. EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
  305. }
  306. TEST_F(VariationsSeedSimulatorTest, ParamsAdded) {
  307. CreateTrial("A", "B", nullptr);
  308. std::vector<ProcessedStudy> processed_studies;
  309. Study study = CreateStudy("A", Study_Consistency_PERMANENT);
  310. Study_Experiment* experiment = AddExperiment("B", 100, &study);
  311. AddExperimentParam("p2", "y", experiment);
  312. AddExperimentParam("p1", "x", experiment);
  313. AddExperimentParam("p3", "z", experiment);
  314. // The current group has no params, but the config has added some.
  315. EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
  316. experiment->set_type(Study_Experiment_Type_NORMAL);
  317. EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
  318. experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
  319. EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
  320. experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
  321. EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
  322. experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
  323. EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
  324. }
  325. // Tests that simulating an expired trial without a default group doesn't crash.
  326. // This is very much an edge case which should generally not be encountered due
  327. // to server-side, but we should ensure that it still doesn't cause a client
  328. // side crash.
  329. TEST_F(VariationsSeedSimulatorTest, NoDefaultGroup) {
  330. static struct base::Feature kFeature {
  331. "FeatureName", base::FEATURE_ENABLED_BY_DEFAULT
  332. };
  333. CreateTrial("Study1", "VariationsDefaultExperiment", nullptr);
  334. Study study;
  335. study.set_consistency(Study::PERMANENT);
  336. study.set_name("Study1");
  337. const base::Time year_ago = base::Time::Now() - base::Days(365);
  338. study.set_expiry_date(TimeToProtoTime(year_ago));
  339. auto* exp1 = AddExperiment("A", 1, &study);
  340. study.clear_default_experiment_name();
  341. exp1->mutable_feature_association()->add_enable_feature(kFeature.name);
  342. EXPECT_FALSE(study.has_default_experiment_name());
  343. EXPECT_EQ("0 0 0", SimulateStudyDifferencesExpired(&study));
  344. }
  345. } // namespace variations