variations_runner.py 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217
  1. # Copyright 2021 The Chromium Authors. All rights reserved.
  2. # Use of this source code is governed by a BSD-style license that can be
  3. # found in the LICENSE file.
  4. """Runner class for variations smoke tests."""
  5. from datetime import datetime
  6. import logging
  7. import os
  8. import subprocess
  9. import sys
  10. import iossim_util
  11. import test_apps
  12. import test_runner
  13. from test_result_util import ResultCollection, TestResult, TestStatus
  14. from xcodebuild_runner import SimulatorParallelTestRunner
  15. import xcode_log_parser
  16. _THIS_DIR = os.path.dirname(os.path.abspath(__file__))
  17. _SRC_DIR = os.path.join(_THIS_DIR, os.path.pardir, os.path.pardir,
  18. os.path.pardir, os.path.pardir)
  19. _VARIATIONS_SMOKE_TEST_DIR = os.path.join(_SRC_DIR, 'testing', 'scripts')
  20. sys.path.insert(0, _VARIATIONS_SMOKE_TEST_DIR)
  21. import variations_seed_access_helper as seed_helper
  22. # Constants around the variation keys.
  23. _LOCAL_STATE_VARIATIONS_LAST_FETCH_TIME_KEY = 'variations_last_fetch_time'
  24. # Test argument to make EG2 test verify the fetch happens in current app launch.
  25. _VERIFY_FETCHED_IN_CURRENT_LAUNCH_ARG = '--verify-fetched-in-current-launch'
  26. LOGGER = logging.getLogger(__name__)
  27. class VariationsSimulatorParallelTestRunner(SimulatorParallelTestRunner):
  28. """Variations simulator runner."""
  29. def __init__(self, app_path, host_app_path, iossim_path, version, platform,
  30. out_dir, variations_seed_path, **kwargs):
  31. super(VariationsSimulatorParallelTestRunner,
  32. self).__init__(app_path, host_app_path, iossim_path, version,
  33. platform, out_dir, **kwargs)
  34. self.variations_seed_path = variations_seed_path
  35. self.host_app_bundle_id = test_apps.get_bundle_id(self.host_app_path)
  36. self.log_parser = xcode_log_parser.get_parser()
  37. self.test_app = self.get_launch_test_app()
  38. def _user_data_dir(self):
  39. """Returns path to user data dir containing "Local State" file.
  40. Note: The path is under app data directory of host Chrome app under test.
  41. The path changes each time launching app but the content is consistent.
  42. """
  43. # This is required for next cmd to work.
  44. iossim_util.boot_simulator_if_not_booted(self.udid)
  45. app_data_path = iossim_util.get_app_data_directory(self.host_app_bundle_id,
  46. self.udid)
  47. return os.path.join(app_data_path, 'Library', 'Application Support',
  48. 'Google', 'Chrome')
  49. def _write_accepted_eula(self):
  50. """Writes eula accepted to Local State.
  51. This is needed once. Chrome host app doesn't have it accepted and variations
  52. seed fetching requires it.
  53. """
  54. seed_helper.update_local_state(self._user_data_dir(),
  55. {'EulaAccepted': True})
  56. LOGGER.info('Wrote EulaAccepted: true to Local State.')
  57. def _reset_last_fetch_time(self):
  58. """Resets last fetch time to one day before so the next fetch can happen.
  59. On mobile devices the fetch will only happen 30 min after last fetch by
  60. checking |variations_last_fetch_time| key in Local State.
  61. """
  62. # Last fetch time in local state uses win timestamp in microseconds.
  63. win_delta = datetime.utcnow() - datetime(1601, 1, 1)
  64. win_now = int(win_delta.total_seconds())
  65. win_one_day_before = win_now - 60 * 60 * 24
  66. win_one_day_before_microseconds = win_one_day_before * 1000000
  67. seed_helper.update_local_state(
  68. self._user_data_dir(), {
  69. _LOCAL_STATE_VARIATIONS_LAST_FETCH_TIME_KEY:
  70. str(win_one_day_before_microseconds)
  71. })
  72. LOGGER.info('Reset last fetch time to %s in Local State.' %
  73. win_one_day_before_microseconds)
  74. def _launch_app_once(self, out_sub_dir, verify_fetched_within_launch=False):
  75. """Launches app once.
  76. Args:
  77. out_sub_dir: (str) Sub dir under |self.out_dir| for this attempt output.
  78. verify_fetched_within_launch: (bool) Whether to verify that the fetch
  79. would happens in current launch.
  80. Returns:
  81. (test_result_util.ResultCollection): Raw EG test result of the launch.
  82. """
  83. launch_out_dir = os.path.join(self.out_dir, out_sub_dir)
  84. if verify_fetched_within_launch:
  85. self.test_app.test_args.append(_VERIFY_FETCHED_IN_CURRENT_LAUNCH_ARG)
  86. cmd = self.test_app.command(launch_out_dir, 'id=%s' % self.udid, 1)
  87. proc = subprocess.Popen(
  88. cmd,
  89. env=self.env_vars or {},
  90. stdout=subprocess.PIPE,
  91. stderr=subprocess.STDOUT,
  92. )
  93. output = test_runner.print_process_output(proc, self.readline_timeout)
  94. if _VERIFY_FETCHED_IN_CURRENT_LAUNCH_ARG in self.test_app.test_args:
  95. self.test_app.test_args.remove(_VERIFY_FETCHED_IN_CURRENT_LAUNCH_ARG)
  96. return self.log_parser.collect_test_results(launch_out_dir, output)
  97. def _launch_variations_smoke_test(self):
  98. """Runs variations smoke test logic which involves multiple test launches.
  99. Returns:
  100. Tuple of (bool, str) Success status and reason.
  101. """
  102. # Launch app once to install app and create Local State file.
  103. first_launch_result = self._launch_app_once('first_launch')
  104. # Test will fail because there isn't EulaAccepted pref in Local State and no
  105. # fetch will happen.
  106. if first_launch_result.passed_tests():
  107. log = 'Test passed (expected to fail) at first launch (to install app).'
  108. LOGGER.error(log)
  109. return False, log
  110. self._write_accepted_eula()
  111. # Launch app to make it fetch seed from server.
  112. fetch_launch_result = self._launch_app_once(
  113. 'fetch_launch', verify_fetched_within_launch=True)
  114. if not fetch_launch_result.passed_tests():
  115. log = 'Test failure at app launch to fetch variations seed.'
  116. LOGGER.error(log)
  117. return False, log
  118. # Verify a production version of variations seed was fetched successfully.
  119. current_seed, current_signature = seed_helper.get_current_seed(
  120. self._user_data_dir())
  121. if not current_seed or not current_signature:
  122. log = 'Failed to fetch variations seed on initial fetch launch.'
  123. LOGGER.error(log)
  124. return False, log
  125. # Inject the test seed.
  126. # |seed_helper.load_test_seed_from_file()| tries to find a seed file under
  127. # src root first. If it doesn't exist, it will fallback to the one in
  128. # |self.variations_seed_path|.
  129. seed, signature = seed_helper.load_test_seed_from_file(
  130. self.variations_seed_path)
  131. if not seed or not signature:
  132. log = ('Ill-formed test seed json file: "%s" and "%s" are required',
  133. seed_helper.LOCAL_STATE_SEED_NAME,
  134. seed_helper.LOCAL_STATE_SEED_SIGNATURE_NAME)
  135. return False, log
  136. if not seed_helper.inject_test_seed(seed, signature, self._user_data_dir()):
  137. log = 'Failed to inject test seed.'
  138. LOGGER.error(log)
  139. return False, log
  140. # Launch app with injected seed.
  141. injected_launch_result = self._launch_app_once('injected_launch')
  142. if not injected_launch_result.passed_tests():
  143. log = 'Test failure at app launch after the seed is injected.'
  144. LOGGER.error(log)
  145. return False, log
  146. # Reset last fetch timestamp to one day before now. On mobile devices a
  147. # fetch will only happen after 30 min of last fetch.
  148. self._reset_last_fetch_time()
  149. # Launch app again to refetch and update the injected seed with a delta.
  150. update_launch_result = self._launch_app_once(
  151. 'update_launch', verify_fetched_within_launch=True)
  152. if not update_launch_result.passed_tests():
  153. log = 'Test failure at app launch to update seed with a delta.'
  154. LOGGER.error(log)
  155. return False, log
  156. # Verify seed has been updated successfully and it's different from the
  157. # injected test seed.
  158. #
  159. # TODO(crbug.com/1234171): This test expectation may not work correctly when
  160. # a field trial config under test does not affect a platform, so it requires
  161. # more investigations to figure out the correct behavior.
  162. current_seed, current_signature = seed_helper.get_current_seed(
  163. self._user_data_dir())
  164. if current_seed == seed or current_signature == signature:
  165. log = 'Failed to update seed with a delta'
  166. LOGGER.error(log)
  167. return False, log
  168. return True, 'Variations smoke test passed all steps!'
  169. def launch(self):
  170. """Entrance to launch tests in this runner."""
  171. success, log = self._launch_variations_smoke_test()
  172. test_status = TestStatus.PASS if success else TestStatus.FAIL
  173. # Report a single test named |VariationsSmokeTest| as part of runner output.
  174. overall_result = ResultCollection(test_results=[
  175. TestResult('VariationsSmokeTest', test_status, test_log=log)
  176. ])
  177. overall_result.report_to_result_sink()
  178. self.test_results = overall_result.standard_json_output(path_delimiter='/')
  179. self.logs.update(overall_result.test_runner_logs())
  180. self.tear_down()
  181. return success