bulk-download.js 3.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129
  1. // Copyright 2021 The Chromium Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style license that can be
  3. // found in the LICENSE file.
  4. 'use strict';
  5. const bulider = require('./builder.js');
  6. const gsutil = require('./gsutil.js');
  7. const tryrequire = require('./try-require.js');
  8. const fs = require('fs');
  9. const yargs = tryrequire.tryrequire('yargs');
  10. if (!yargs) {
  11. console.error('Please install the `yargs` package from npm (`npm i yargs`).');
  12. return;
  13. }
  14. function printUpdate(msg) {
  15. console.log('*** ' + msg);
  16. }
  17. async function main() {
  18. if (!gsutil.exists()) {
  19. console.error('Command `gsutil` (used to download the files) not found.');
  20. console.error('You may need to install `google-cloud-sdk` package.');
  21. return;
  22. }
  23. const argv =
  24. yargs
  25. .option('benchmark', {
  26. alias: 'b',
  27. description: 'The benchmark to download traces for.',
  28. type: 'string'
  29. })
  30. .option('output', {
  31. alias: 'o',
  32. description: 'The output location for the downloaded trace files.',
  33. type: 'string'
  34. })
  35. .usage('Usage: $0 -b <benchmark> -o <download-path> <build-url>')
  36. .example(
  37. '$0 -b rendering.mobile -o /tmp/foo https://ci.chromium.org/ui/p/chrome/builders/ci/android-pixel2_webview-perf/24015/overview')
  38. .wrap(null)
  39. .argv;
  40. if (!argv.benchmark) {
  41. console.error('Please specify the name of a benchmark (using -b).');
  42. return;
  43. }
  44. if (!argv.output) {
  45. console.error(
  46. 'Please specify the location to download the files to (using -o).');
  47. return;
  48. }
  49. if (!fs.existsSync(argv.output)) {
  50. console.error(`Create output location (${argv.output}) first.`);
  51. return;
  52. }
  53. const builder = new bulider.Build(argv._[0]);
  54. const task = builder.findSwarmingTask();
  55. printUpdate(
  56. `Found swarming task ${task.task_id}. Looking for child tasks ... `);
  57. const children = task.findChildTasks();
  58. printUpdate(`Found ${children.length} child tasks.`);
  59. const all = {};
  60. const promises = [];
  61. let taskno = 0;
  62. for (const child of children) {
  63. const p = child.downloadJSONFile('output.json');
  64. promises.push(p);
  65. p.then((output) => {
  66. if (argv.benchmark in output.tests) {
  67. const tests = Object.keys(output.tests[argv.benchmark]);
  68. printUpdate(
  69. `${++taskno}/${children.length} ${tests.length} tests found for ${
  70. argv.benchmark} in ${child.task_id}.`);
  71. for (const t of tests) {
  72. const artifacts = output.tests[argv.benchmark][t].artifacts;
  73. if (artifacts && artifacts['trace.html']) {
  74. const trace = artifacts['trace.html'];
  75. const url = (typeof (trace) === 'object' &&
  76. typeof (trace[0]) === 'string') ?
  77. trace[0] :
  78. trace;
  79. if (t in all) {
  80. all[t].push(url);
  81. } else {
  82. all[t] = [url];
  83. }
  84. }
  85. }
  86. } else {
  87. printUpdate(`${++taskno}/${children.length} 0 tests found for ${
  88. argv.benchmark} in ${child.task_id}.`);
  89. }
  90. });
  91. }
  92. await Promise.all(promises);
  93. // Maps a gs:// url to a local filename.
  94. const map = {};
  95. const names = Object.keys(all);
  96. while (names.length > 0) {
  97. const orig = names.shift();
  98. const name = orig.replace('/', '_').replace('\.', '_');
  99. const urls = all[orig];
  100. if (urls.length === 1) {
  101. map[urls[0]] = name + '.html';
  102. } else {
  103. for (let i = 0; i < urls.length; ++i) {
  104. map[urls[i]] = `${name}_${i}.html`;
  105. }
  106. }
  107. }
  108. const total = Object.keys(map).length;
  109. printUpdate(`There are ${total} files to download.`);
  110. await gsutil.downloadFiles(map, argv.output);
  111. process.stdout.write('\nDone.\n');
  112. }
  113. main();