testharnessreport.js 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372
  1. /*
  2. * This file is intended for vendors to implement code needed to integrate
  3. * testharness.js tests with their own test systems.
  4. *
  5. * Typically such integration will attach callbacks when each test is
  6. * has run, using add_result_callback(callback(test)), or when the whole test
  7. * file has completed, using add_completion_callback(callback(tests,
  8. * harness_status)).
  9. *
  10. * For more documentation about the callback functions and the
  11. * parameters they are called with, see testharness.js, or the docs at:
  12. * https://web-platform-tests.org/writing-tests/testharness-api.html
  13. */
  14. (function() {
  15. let outputDocument = document;
  16. let didDispatchLoadEvent = false;
  17. let localPathRegExp = undefined;
  18. // Setup for Blink JavaScript tests. self.testRunner is expected to be
  19. // present when tests are run.
  20. if (self.testRunner) {
  21. testRunner.dumpAsText();
  22. testRunner.waitUntilDone();
  23. testRunner.setPopupBlockingEnabled(false);
  24. testRunner.setDumpJavaScriptDialogs(false);
  25. // Some tests intentionally load mixed content in order to test the
  26. // referrer policy, so WebKitAllowRunningInsecureContent must be set
  27. // to true or else the load would be blocked.
  28. const paths = [
  29. 'service-workers/service-worker/fetch-event-referrer-policy.https.html',
  30. 'beacon/headers/header-referrer-no-referrer-when-downgrade.https.html',
  31. 'beacon/headers/header-referrer-strict-origin-when-cross-origin.https.html',
  32. 'beacon/headers/header-referrer-strict-origin.https.html',
  33. 'beacon/headers/header-referrer-unsafe-url.https.html',
  34. ];
  35. for (const path of paths) {
  36. if (document.URL.endsWith(path)) {
  37. testRunner.overridePreference('WebKitAllowRunningInsecureContent', true);
  38. break;
  39. }
  40. }
  41. }
  42. if (document.URL.startsWith('file:///')) {
  43. const index = document.URL.indexOf('/external/wpt');
  44. if (index >= 0) {
  45. const localPath = document.URL.substring(
  46. 'file:///'.length, index + '/external/wpt'.length);
  47. localPathRegExp = new RegExp(localPath.replace(/(\W)/g, '\\$1'), 'g');
  48. }
  49. }
  50. window.addEventListener('load', loadCallback, {'once': true});
  51. setup({
  52. // The default output formats test results into an HTML table, but for
  53. // the Blink layout test runner, we dump the results as text in the
  54. // completion callback, so we disable the default output.
  55. 'output': false,
  56. // The Blink layout test runner has its own timeout mechanism.
  57. 'explicit_timeout': true
  58. });
  59. add_start_callback(startCallback);
  60. add_completion_callback(completionCallback);
  61. /** Loads an automation script if necessary. */
  62. function loadCallback() {
  63. didDispatchLoadEvent = true;
  64. if (isWPTManualTest()) {
  65. setTimeout(loadAutomationScript, 0);
  66. }
  67. }
  68. /** Checks whether the current path is a manual test in WPT. */
  69. function isWPTManualTest() {
  70. // Here we assume that if wptserve is running, then the hostname
  71. // is web-platform.test.
  72. const path = location.pathname;
  73. if (location.hostname == 'web-platform.test' &&
  74. /.*-manual(\.sub)?(\.https)?(\.tentative)?\.html$/.test(path)) {
  75. return true;
  76. }
  77. // If the file is loaded locally via file://, it must include
  78. // the wpt directory in the path.
  79. return /\/external\/wpt\/.*-manual(\.sub)?(\.https)?(\.tentative)?\.html$/.test(path);
  80. }
  81. /** Loads the WPT automation script for the current test, if applicable. */
  82. function loadAutomationScript() {
  83. const pathAndBase = pathAndBaseNameInWPT();
  84. if (!pathAndBase) {
  85. return;
  86. }
  87. let automationPath = location.pathname.replace(
  88. /\/external\/wpt\/.*$/, '/external/wpt_automation');
  89. if (location.hostname == 'web-platform.test') {
  90. automationPath = '/wpt_automation';
  91. }
  92. // Export importAutomationScript for use by the automation scripts.
  93. window.importAutomationScript = function(relativePath) {
  94. const script = document.createElement('script');
  95. script.src = automationPath + relativePath;
  96. document.head.appendChild(script);
  97. };
  98. let src;
  99. if (pathAndBase.startsWith('/fullscreen/')) {
  100. // Fullscreen tests all use the same automation script.
  101. src = automationPath + '/fullscreen/auto-click.js';
  102. } else if (pathAndBase.startsWith('/file-system-access/local_')) {
  103. // local_ File System Access tests all use the same automation script.
  104. src = automationPath + '/file-system-access/auto-pick-folder.js';
  105. } else if (pathAndBase.startsWith('/file-system-access/')) {
  106. // Per-test automation scripts.
  107. src = automationPath + pathAndBase + '-automation.sub.js';
  108. } else if (
  109. pathAndBase.startsWith('/css/') ||
  110. pathAndBase.startsWith('/pointerevents/') ||
  111. pathAndBase.startsWith('/uievents/') ||
  112. pathAndBase.startsWith('/html/') ||
  113. pathAndBase.startsWith('/input-events/') ||
  114. pathAndBase.startsWith('/css/selectors/') ||
  115. pathAndBase.startsWith('/css/cssom-view/') ||
  116. pathAndBase.startsWith('/css/css-scroll-snap/') ||
  117. pathAndBase.startsWith('/dom/events/') ||
  118. pathAndBase.startsWith('/feature-policy/experimental-features/') ||
  119. pathAndBase.startsWith('/permissions-policy/experimental-features/')) {
  120. // Per-test automation scripts.
  121. src = automationPath + pathAndBase + '-automation.js';
  122. } else {
  123. return;
  124. }
  125. const script = document.createElement('script');
  126. script.src = src;
  127. document.head.appendChild(script);
  128. }
  129. /**
  130. * Sets the output document based on the given properties.
  131. * Usually the output document is the current document, but it could be
  132. * a separate document in some cases.
  133. */
  134. function startCallback(properties) {
  135. if (properties.output_document) {
  136. outputDocument = properties.output_document;
  137. }
  138. }
  139. /**
  140. * Adds results to the page in a manner that allows dumpAsText to produce
  141. * readable test results.
  142. */
  143. function completionCallback(tests, harness_status) {
  144. const xhtmlNS = 'http://www.w3.org/1999/xhtml';
  145. // Create element to hold results.
  146. const resultsElement = outputDocument.createElementNS(xhtmlNS, 'pre');
  147. resultsElement.style.whiteSpace = 'pre-wrap';
  148. resultsElement.style.lineHeight = '1.5';
  149. // Declare result string.
  150. let resultStr = 'This is a testharness.js-based test.\n';
  151. // Check harness_status. If it is not 0, tests did not execute
  152. // correctly, output the error code and message.
  153. if (harness_status.status != 0) {
  154. resultStr += `Harness Error. ` +
  155. `harness_status.status = ${harness_status.status} , ` +
  156. `harness_status.message = ${harness_status.message}\n`;
  157. }
  158. // Output failure metrics if there are many.
  159. const resultCounts = countResultTypes(tests);
  160. if (outputDocument.URL.indexOf('://web-platform.test') >= 0 &&
  161. tests.length >= 50 &&
  162. (resultCounts[1] || resultCounts[2] || resultCounts[3])) {
  163. resultStr += failureMetricSummary(resultCounts);
  164. }
  165. if (outputDocument.URL.indexOf('/html/dom/reflection') >= 0) {
  166. resultStr += compactTestOutput(tests);
  167. } else {
  168. resultStr += testOutput(tests);
  169. }
  170. resultStr += 'Harness: the test ran to completion.\n';
  171. resultsElement.textContent = resultStr;
  172. function done() {
  173. let body = null;
  174. if (outputDocument.body && outputDocument.body.tagName == 'BODY' &&
  175. outputDocument.body.namespaceURI == xhtmlNS) {
  176. body = outputDocument.body;
  177. }
  178. // A temporary workaround since |window.self| property lookup starts
  179. // failing if the frame is detached. |outputDocument| may be an
  180. // ancestor of |self| so clearing |textContent| may detach |self|.
  181. // To get around this, cache window.self now and use the cached
  182. // value.
  183. // TODO(dcheng): Remove this hack after fixing window/self/frames
  184. // lookup in https://crbug.com/618672
  185. const cachedSelf = window.self;
  186. if (cachedSelf.testRunner) {
  187. // The following DOM operations may show console messages. We
  188. // suppress them because they are not related to the running
  189. // test.
  190. testRunner.setDumpConsoleMessages(false);
  191. // Anything in the body isn't part of the output and so should
  192. // be hidden from the text dump.
  193. if (body) {
  194. body.textContent = '';
  195. }
  196. }
  197. // Add the results element to the output document.
  198. if (!body) {
  199. // |outputDocument| might be an SVG document.
  200. if (outputDocument.documentElement) {
  201. outputDocument.documentElement.remove();
  202. }
  203. let html = outputDocument.createElementNS(xhtmlNS, 'html');
  204. outputDocument.appendChild(html);
  205. body = outputDocument.createElementNS(xhtmlNS, 'body');
  206. body.setAttribute('style', 'white-space:pre;');
  207. html.appendChild(body);
  208. }
  209. outputDocument.body.appendChild(resultsElement);
  210. // IFrames running tests should not complete the harness as the parent
  211. // page will.
  212. let shouldCompleteHarness = (window.self == window.top);
  213. if (cachedSelf.testRunner && shouldCompleteHarness) {
  214. testRunner.notifyDone();
  215. }
  216. }
  217. if (didDispatchLoadEvent || outputDocument.readyState != 'loading') {
  218. // This function might not be the last completion callback, and
  219. // another completion callback might generate more results.
  220. // So, we don't dump the results immediately.
  221. setTimeout(done, 0);
  222. } else {
  223. // Parsing the test HTML isn't finished yet.
  224. window.addEventListener('load', done);
  225. }
  226. }
  227. /**
  228. * Returns a directory part relative to WPT root and a basename part of the
  229. * current test. e.g.
  230. * Current test: file:///.../LayoutTests/external/wpt/pointerevents/foobar.html
  231. * Output: "/pointerevents/foobar"
  232. */
  233. function pathAndBaseNameInWPT() {
  234. const path = location.pathname;
  235. let matches;
  236. if (location.hostname == 'web-platform.test') {
  237. matches = path.match(/^(\/.*)\.html$/);
  238. return matches ? matches[1] : null;
  239. }
  240. matches = path.match(/external\/wpt(\/.*)\.html$/);
  241. return matches ? matches[1] : null;
  242. }
  243. /** Converts the testharness test status into the corresponding string. */
  244. function convertResult(resultStatus) {
  245. switch (resultStatus) {
  246. case 0:
  247. return 'PASS';
  248. case 1:
  249. return 'FAIL';
  250. case 2:
  251. return 'TIMEOUT';
  252. default:
  253. return 'NOTRUN';
  254. }
  255. }
  256. /**
  257. * Returns a compact output for reflection test results.
  258. *
  259. * The reflection tests contain a large number of tests.
  260. * This test output merges PASS lines to make baselines smaller.
  261. */
  262. function compactTestOutput(tests) {
  263. let testResults = [];
  264. for (let i = 0; i < tests.length; ++i) {
  265. if (tests[i].status == 0) {
  266. const colon = tests[i].name.indexOf(':');
  267. if (colon > 0) {
  268. const prefix = tests[i].name.substring(0, colon + 1);
  269. let j = i + 1;
  270. for (; j < tests.length; ++j) {
  271. if (!tests[j].name.startsWith(prefix) ||
  272. tests[j].status != 0)
  273. break;
  274. }
  275. const numPasses = j - i;
  276. if (numPasses > 1) {
  277. testResults.push(
  278. `${convertResult(tests[i].status)} ` +
  279. `${sanitize(prefix)} ${numPasses} tests\n`);
  280. i = j - 1;
  281. continue;
  282. }
  283. }
  284. }
  285. testResults.push(resultLine(tests[i]));
  286. }
  287. return testResults.join('');
  288. }
  289. function testOutput(tests) {
  290. let testResults = '';
  291. window.tests = tests;
  292. for (let test of tests) {
  293. testResults += resultLine(test);
  294. }
  295. return testResults;
  296. }
  297. function resultLine(test) {
  298. let result = `${convertResult(test.status)} ${sanitize(test.name)}`;
  299. if (test.message) {
  300. result += ' ' + sanitize(test.message).trim();
  301. }
  302. return result + '\n';
  303. }
  304. /** Prepares the given text for display in test results. */
  305. function sanitize(text) {
  306. if (!text) {
  307. return '';
  308. }
  309. // Escape null characters, otherwise diff will think the file is binary.
  310. text = text.replace(/\0/g, '\\0');
  311. // Escape some special characters to improve readability of the output.
  312. text = text.replace(/\r/g, '\\r');
  313. // Replace machine-dependent path with "...".
  314. if (localPathRegExp) {
  315. text = text.replace(localPathRegExp, '...');
  316. }
  317. return text;
  318. }
  319. function countResultTypes(tests) {
  320. const resultCounts = [0, 0, 0, 0];
  321. for (let test of tests) {
  322. resultCounts[test.status]++;
  323. }
  324. return resultCounts;
  325. }
  326. function failureMetricSummary(resultCounts) {
  327. const total = resultCounts[0] + resultCounts[1] + resultCounts[2] + resultCounts[3];
  328. return `Found ${total} tests;` +
  329. ` ${resultCounts[0]} PASS,` +
  330. ` ${resultCounts[1]} FAIL,` +
  331. ` ${resultCounts[2]} TIMEOUT,` +
  332. ` ${resultCounts[3]} NOTRUN.\n`;
  333. }
  334. })();