cut_release.py 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168
  1. #! /usr/bin/env python
  2. # Copyright 2018 Google LLC.
  3. # Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.
  4. import json
  5. import md5
  6. import multiprocessing
  7. import os
  8. import shutil
  9. import sys
  10. import tempfile
  11. import urllib
  12. import urllib2
  13. from subprocess import check_call, check_output
  14. assert '/' in [os.sep, os.altsep] and os.pardir == '..'
  15. ASSETS = 'platform_tools/android/apps/skqp/src/main/assets'
  16. BUCKET = 'skia-skqp-assets'
  17. def make_skqp_model(arg):
  18. name, urls, exe = arg
  19. tmp = tempfile.mkdtemp()
  20. for url in urls:
  21. urllib.urlretrieve(url, tmp + '/' + url[url.rindex('/') + 1:])
  22. check_call([exe, tmp, ASSETS + '/gmkb/' + name])
  23. shutil.rmtree(tmp)
  24. sys.stdout.write(name + ' ')
  25. sys.stdout.flush()
  26. def goldgetter(meta, exe):
  27. assert os.path.exists(exe)
  28. jobs = []
  29. for rec in meta:
  30. urls = [d['URL'] for d in rec['digests']
  31. if d['status'] == 'positive' and
  32. (set(d['paramset']['config']) & set(['vk', 'gles']))]
  33. if urls:
  34. jobs.append((rec['testName'], urls, exe))
  35. pool = multiprocessing.Pool(processes=20)
  36. pool.map(make_skqp_model, jobs)
  37. sys.stdout.write('\n')
  38. return set((n for n, _, _ in jobs))
  39. def gold(first_commit, last_commit):
  40. c1, c2 = (check_output(['git', 'rev-parse', c]).strip()
  41. for c in (first_commit, last_commit))
  42. f = urllib2.urlopen('https://public-gold.skia.org/json/export?' + urllib.urlencode([
  43. ('fbegin', c1),
  44. ('fend', c2),
  45. ('query', 'config=gles&config=vk&source_type=gm'),
  46. ('pos', 'true'),
  47. ('neg', 'false'),
  48. ('unt', 'false')
  49. ]))
  50. j = json.load(f)
  51. f.close()
  52. return j
  53. def gset(path):
  54. s = set()
  55. if os.path.isfile(path):
  56. with open(path, 'r') as f:
  57. for line in f:
  58. s.add(line.strip())
  59. return s
  60. def make_rendertest_list(models, good, bad):
  61. assert good.isdisjoint(bad)
  62. do_score = good & models
  63. no_score = bad | (good - models)
  64. to_delete = models & bad
  65. for d in to_delete:
  66. path = ASSETS + '/gmkb/' + d
  67. if os.path.isdir(path):
  68. shutil.rmtree(path)
  69. results = dict()
  70. for n in do_score:
  71. results[n] = 0
  72. for n in no_score:
  73. results[n] = -1
  74. return ''.join('%s,%d\n' % (n, results[n]) for n in sorted(results))
  75. def get_digest(path):
  76. m = md5.new()
  77. with open(path, 'r') as f:
  78. m.update(f.read())
  79. return m.hexdigest()
  80. def upload_cmd(path, digest):
  81. return ['gsutil', 'cp', path, 'gs://%s/%s' % (BUCKET, digest)]
  82. def upload_model():
  83. bucket_url = 'gs://%s/' % BUCKET
  84. extant = set((u.replace(bucket_url, '', 1)
  85. for u in check_output(['gsutil', 'ls', bucket_url]).splitlines() if u))
  86. cmds = []
  87. filelist = []
  88. for dirpath, _, filenames in os.walk(ASSETS + '/gmkb'):
  89. for filename in filenames:
  90. path = os.path.join(dirpath, filename)
  91. digest = get_digest(path)
  92. if digest not in extant:
  93. cmds.append(upload_cmd(path, digest))
  94. filelist.append('%s;%s\n' % (digest, os.path.relpath(path, ASSETS)))
  95. tmp = tempfile.mkdtemp()
  96. filelist_path = tmp + '/x'
  97. with open(filelist_path, 'w') as o:
  98. for l in filelist:
  99. o.write(l)
  100. filelist_digest = get_digest(filelist_path)
  101. if filelist_digest not in extant:
  102. cmds.append(upload_cmd(filelist_path, filelist_digest))
  103. pool = multiprocessing.Pool(processes=20)
  104. pool.map(check_call, cmds)
  105. shutil.rmtree(tmp)
  106. return filelist_digest
  107. def remove(x):
  108. if os.path.isdir(x) and not os.path.islink(x):
  109. shutil.rmtree(x)
  110. if os.path.exists(x):
  111. os.remove(x)
  112. def main(first_commit, last_commit):
  113. check_call(upload_cmd('/dev/null', get_digest('/dev/null')))
  114. os.chdir(os.path.dirname(__file__) + '/../..')
  115. remove(ASSETS + '/files.checksum')
  116. for d in [ASSETS + '/gmkb', ASSETS + '/skqp', ]:
  117. remove(d)
  118. os.mkdir(d)
  119. check_call([sys.executable, 'tools/git-sync-deps'],
  120. env=dict(os.environ, GIT_SYNC_DEPS_QUIET='T'))
  121. build = 'out/ndebug'
  122. check_call(['bin/gn', 'gen', build,
  123. '--args=cc="clang" cxx="clang++" is_debug=false'])
  124. check_call(['ninja', '-C', build,
  125. 'jitter_gms', 'list_gpu_unit_tests', 'make_skqp_model'])
  126. models = goldgetter(gold(first_commit, last_commit), build + '/make_skqp_model')
  127. check_call([build + '/jitter_gms', 'tools/skqp/bad_gms.txt'])
  128. with open(ASSETS + '/skqp/rendertests.txt', 'w') as o:
  129. o.write(make_rendertest_list(models, gset('good.txt'), gset('bad.txt')))
  130. remove('good.txt')
  131. remove('bad.txt')
  132. with open(ASSETS + '/skqp/unittests.txt', 'w') as o:
  133. o.write(check_output([build + '/list_gpu_unit_tests']))
  134. with open(ASSETS + '/files.checksum', 'w') as o:
  135. o.write(upload_model() + '\n')
  136. sys.stdout.write(ASSETS + '/files.checksum\n')
  137. sys.stdout.write(ASSETS + '/skqp/rendertests.txt\n')
  138. sys.stdout.write(ASSETS + '/skqp/unittests.txt\n')
  139. if __name__ == '__main__':
  140. if len(sys.argv) != 3:
  141. sys.stderr.write('Usage:\n %s C1 C2\n\n' % sys.argv[0])
  142. sys.exit(1)
  143. main(sys.argv[1], sys.argv[2])