find_commit_with_best_gold_results.py 3.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122
  1. #! /usr/bin/env python
  2. # Copyright 2019 Google LLC.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. import json
  6. import os
  7. import re
  8. import subprocess
  9. import sys
  10. import threading
  11. import urllib
  12. import urllib2
  13. assert '/' in [os.sep, os.altsep]
  14. skia_directory = os.path.abspath(os.path.dirname(__file__) + '/../..')
  15. def get_jobs():
  16. path = skia_directory + '/infra/bots/jobs.json'
  17. reg = re.compile('Test-(?P<os>[A-Za-z0-9_]+)-'
  18. '(?P<compiler>[A-Za-z0-9_]+)-'
  19. '(?P<model>[A-Za-z0-9_]+)-GPU-'
  20. '(?P<cpu_or_gpu_value>[A-Za-z0-9_]+)-'
  21. '(?P<arch>[A-Za-z0-9_]+)-'
  22. '(?P<configuration>[A-Za-z0-9_]+)-'
  23. 'All(-(?P<extra_config>[A-Za-z0-9_]+)|)')
  24. keys = ['os', 'compiler', 'model', 'cpu_or_gpu_value', 'arch',
  25. 'configuration', 'extra_config']
  26. def fmt(s):
  27. return s.encode('utf-8') if s is not None else ''
  28. with open(path) as f:
  29. jobs = json.load(f)
  30. for job in jobs:
  31. m = reg.match(job)
  32. if m is not None:
  33. yield [(k, fmt(m.group(k))) for k in keys]
  34. def gold_export_url(job, config, first_commit, last_commit):
  35. qq = [('source_type', 'gm'), ('config', config)] + job
  36. query = [
  37. ('fbegin', first_commit),
  38. ('fend', last_commit),
  39. ('query', urllib.urlencode(qq)),
  40. ('pos', 'true'),
  41. ('neg', 'false'),
  42. ('unt', 'false'),
  43. ('head', 'true')
  44. ]
  45. return 'https://public-gold.skia.org/json/export?' + urllib.urlencode(query)
  46. def get_results_for_commit(commit, jobs):
  47. sys.stderr.write('%s\n' % commit)
  48. sys.stderr.flush()
  49. CONFIGS = ['gles', 'vk']
  50. passing_tests_for_all_jobs = []
  51. def process(url):
  52. testResults = json.load(urllib2.urlopen(url))
  53. sys.stderr.write('.')
  54. sys.stderr.flush()
  55. passing_tests = 0
  56. for t in testResults:
  57. assert t['digests']
  58. passing_tests += 1
  59. passing_tests_for_all_jobs.append(passing_tests)
  60. all_urls = [gold_export_url(job, config, commit, commit)
  61. for job in jobs for config in CONFIGS]
  62. threads = [threading.Thread(target=process, args=(url,)) for url in all_urls]
  63. for t in threads:
  64. t.start()
  65. for t in threads:
  66. t.join()
  67. result = sum(passing_tests_for_all_jobs)
  68. sys.stderr.write('\n%d\n' % result)
  69. sys.stderr.flush()
  70. return result
  71. def find_best_commit(commits):
  72. jobs = [j for j in get_jobs()]
  73. results = []
  74. for commit_name in commits:
  75. commit_hash = subprocess.check_output(['git', 'rev-parse', commit_name]).strip()
  76. results.append((commit_hash, get_results_for_commit(commit_hash, jobs)))
  77. best_result = max(r for h, r in results)
  78. for h, r in results:
  79. if r == best_result:
  80. return h
  81. return None
  82. def generate_commit_list(args):
  83. return subprocess.check_output(['git', 'log', '--format=%H'] + args).splitlines()
  84. def main(args):
  85. os.chdir(skia_directory)
  86. subprocess.check_call(['git', 'fetch', 'origin'])
  87. sys.stderr.write('%s\n' % ' '.join(args))
  88. commits = generate_commit_list(args)
  89. sys.stderr.write('%d\n' % len(commits))
  90. best = find_best_commit(commits)
  91. sys.stderr.write('DONE:\n')
  92. sys.stderr.flush()
  93. sys.stdout.write('%s\n' % best)
  94. usage = '''Example usage:
  95. python %s origin/master ^origin/skqp/dev < /dev/null > LOG 2>&1 & disown
  96. '''
  97. if __name__ == '__main__':
  98. if len(sys.argv) < 2:
  99. sys.stderr.write(usage % sys.argv[0])
  100. sys.exit(1)
  101. main(sys.argv[1:])