test_buildbucket_api_gpu_use_cases.py 4.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134
  1. #!/usr/bin/env python
  2. # Copyright 2019 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. from __future__ import print_function
  6. import argparse
  7. import json
  8. import os
  9. import sys
  10. # Add src/testing/ into sys.path for importing common without pylint errors.
  11. sys.path.append(
  12. os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)))
  13. from scripts import common
  14. # Add src/content/test/gpu into sys.path for importing common.
  15. sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__),
  16. os.path.pardir, os.path.pardir, 'content',
  17. 'test', 'gpu')))
  18. import gather_power_measurement_results
  19. import gather_swarming_json_results
  20. class BuildBucketApiGpuUseCaseTests:
  21. @classmethod
  22. def GenerateTests(cls):
  23. return [
  24. 'TestGatherPowerMeasurementResultsFromLatestGreenBuild',
  25. 'TestGatherWebGL2TestTimesFromLatestGreenBuild',
  26. ]
  27. @staticmethod
  28. def TestGatherPowerMeasurementResultsFromLatestGreenBuild():
  29. # Verify we can get power measurement test data from latest successful
  30. # build, including the swarming bot that runs the test, and actual test
  31. # results.
  32. bot = 'Win10 FYI x64 Release (Intel HD 630)'
  33. step = 'power_measurement_test'
  34. build_id = gather_power_measurement_results.GetLatestGreenBuild(bot)
  35. build_json = gather_power_measurement_results.GetJsonForBuildSteps(
  36. bot, build_id)
  37. if 'steps' not in build_json:
  38. return '"steps" is missing from the build json'
  39. stdout_url = gather_power_measurement_results.FindStepLogURL(
  40. build_json['steps'], step, 'stdout')
  41. if not stdout_url:
  42. return 'Unable to find stdout from step %s' % step
  43. results = { 'number': build_id, 'tests': [] }
  44. gather_power_measurement_results.ProcessStepStdout(stdout_url, results)
  45. if 'bot' not in results or not results['bot'].startswith('BUILD'):
  46. return 'Failed to find bot name as BUILD*'
  47. if not results['tests']:
  48. return 'Failed to find power measurment test data'
  49. return None
  50. @staticmethod
  51. def TestGatherWebGL2TestTimesFromLatestGreenBuild():
  52. # Verify we can get more than 2000 WebGL2 tests running time from the
  53. # latest successful build.
  54. extracted_times, _ = gather_swarming_json_results.GatherResults(
  55. bot='Linux FYI Release (NVIDIA)',
  56. build=None, # Use the latest green build
  57. step='webgl2_conformance_validating_tests')
  58. if 'times' not in extracted_times:
  59. return '"times" is missing from the extracted dict'
  60. num_of_tests = len(extracted_times['times'])
  61. # From local run, there are 2700+ tests. This is sanity check that we
  62. # get reasonable data.
  63. if num_of_tests < 2000:
  64. return 'expected 2000+ tests, got %d tests' % num_of_tests
  65. return None
  66. def main(argv):
  67. parser = argparse.ArgumentParser()
  68. parser.add_argument(
  69. '--isolated-script-test-output', type=str)
  70. parser.add_argument(
  71. '--isolated-script-test-chartjson-output', type=str,
  72. required=False)
  73. parser.add_argument(
  74. '--isolated-script-test-perf-output', type=str,
  75. required=False)
  76. parser.add_argument(
  77. '--isolated-script-test-filter', type=str,
  78. required=False)
  79. args = parser.parse_args(argv)
  80. # Run actual tests
  81. failures = []
  82. retval = 1
  83. for test_name in BuildBucketApiGpuUseCaseTests.GenerateTests():
  84. test = getattr(BuildBucketApiGpuUseCaseTests, test_name)
  85. error_msg = test()
  86. if error_msg is not None:
  87. result = '%s: %s' % (test_name, error_msg)
  88. print('FAIL: %s' % result)
  89. failures.append(result)
  90. if not failures:
  91. print('PASS: test_buildbucket_api_gpu_use_cases ran successfully.')
  92. retval = 0
  93. if args.isolated_script_test_output:
  94. with open(args.isolated_script_test_output, 'w') as json_file:
  95. json.dump({
  96. 'valid': True,
  97. 'failures': failures,
  98. }, json_file)
  99. return retval
  100. # This is not really a "script test" so does not need to manually add
  101. # any additional compile targets.
  102. def main_compile_targets(args):
  103. json.dump([], args.output)
  104. if __name__ == '__main__':
  105. # Conform minimally to the protocol defined by ScriptTest.
  106. if 'compile_targets' in sys.argv:
  107. funcs = {
  108. 'run': None,
  109. 'compile_targets': main_compile_targets,
  110. }
  111. sys.exit(common.run_script(sys.argv[1:], funcs))
  112. sys.exit(main(sys.argv[1:]))