nocompile_driver.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524
  1. #!/usr/bin/env python3
  2. # Copyright (c) 2011 The Chromium Authors. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """Implements a simple "negative compile" test for C++ on linux.
  6. Sometimes a C++ API needs to ensure that various usages cannot compile. To
  7. enable unittesting of these assertions, we use this python script to
  8. invoke the compiler on a source file and assert that compilation fails.
  9. For more info, see:
  10. http://dev.chromium.org/developers/testing/no-compile-tests
  11. """
  12. from __future__ import print_function
  13. import ast
  14. import io
  15. import os
  16. import re
  17. import select
  18. import subprocess
  19. import sys
  20. import tempfile
  21. import time
  22. # Matches lines that start with #if and have the substring TEST in the
  23. # conditional. Also extracts the comment. This allows us to search for
  24. # lines like the following:
  25. #
  26. # #ifdef NCTEST_NAME_OF_TEST // [r'expected output']
  27. # #if defined(NCTEST_NAME_OF_TEST) // [r'expected output']
  28. # #if NCTEST_NAME_OF_TEST // [r'expected output']
  29. # #elif NCTEST_NAME_OF_TEST // [r'expected output']
  30. # #elif DISABLED_NCTEST_NAME_OF_TEST // [r'expected output']
  31. #
  32. # inside the unittest file.
  33. NCTEST_CONFIG_RE = re.compile(r'^#(?:el)?if.*\s+(\S*NCTEST\S*)\s*(//.*)?')
  34. # Matches and removes the defined() preprocesor predicate. This is useful
  35. # for test cases that use the preprocessor if-statement form:
  36. #
  37. # #if defined(NCTEST_NAME_OF_TEST)
  38. #
  39. # Should be used to post-process the results found by NCTEST_CONFIG_RE.
  40. STRIP_DEFINED_RE = re.compile(r'defined\((.*)\)')
  41. # Used to grab the expectation from comment at the end of an #ifdef. See
  42. # NCTEST_CONFIG_RE's comment for examples of what the format should look like.
  43. #
  44. # The extracted substring should be a python array of regular expressions.
  45. EXTRACT_EXPECTATION_RE = re.compile(r'//\s*(\[.*\])')
  46. # The header for the result file so that it can be compiled.
  47. RESULT_FILE_HEADER = """
  48. // This file is generated by the no compile test from:
  49. // %s
  50. #include "base/logging.h"
  51. #include "testing/gtest/include/gtest/gtest.h"
  52. """
  53. # The log message on a test completion.
  54. LOG_TEMPLATE = """
  55. TEST(%s, %s) took %f secs. Started at %f, ended at %f.
  56. """
  57. # The GUnit test function to output for a successful or disabled test.
  58. GUNIT_TEMPLATE = """
  59. TEST(%s, %s) { }
  60. """
  61. # Timeout constants.
  62. NCTEST_TERMINATE_TIMEOUT_SEC = 120
  63. NCTEST_KILL_TIMEOUT_SEC = NCTEST_TERMINATE_TIMEOUT_SEC + 2
  64. BUSY_LOOP_MAX_TIME_SEC = NCTEST_KILL_TIMEOUT_SEC * 2
  65. def ValidateInput(compiler, parallelism, sourcefile_path, cflags,
  66. resultfile_path):
  67. """Make sure the arguments being passed in are sane."""
  68. assert os.path.isfile(compiler)
  69. assert parallelism >= 1
  70. assert type(sourcefile_path) is str
  71. assert type(cflags) is list
  72. for flag in cflags:
  73. assert type(flag) is str
  74. assert type(resultfile_path) is str
  75. def ParseExpectation(expectation_string):
  76. """Extracts expectation definition from the trailing comment on the ifdef.
  77. See the comment on NCTEST_CONFIG_RE for examples of the format we are parsing.
  78. Args:
  79. expectation_string: A string like "// [r'some_regex']"
  80. Returns:
  81. A list of compiled regular expressions indicating all possible valid
  82. compiler outputs. If the list is empty, all outputs are considered valid.
  83. """
  84. assert expectation_string is not None
  85. match = EXTRACT_EXPECTATION_RE.match(expectation_string)
  86. assert match
  87. raw_expectation = ast.literal_eval(match.group(1))
  88. assert type(raw_expectation) is list
  89. expectation = []
  90. for regex_str in raw_expectation:
  91. assert type(regex_str) is str
  92. expectation.append(re.compile(regex_str))
  93. return expectation
  94. def ExtractTestConfigs(sourcefile_path, suite_name):
  95. """Parses the source file for test configurations.
  96. Each no-compile test in the file is separated by an ifdef macro. We scan
  97. the source file with the NCTEST_CONFIG_RE to find all ifdefs that look like
  98. they demark one no-compile test and try to extract the test configuration
  99. from that.
  100. Args:
  101. sourcefile_path: The path to the source file.
  102. suite_name: The name of the test suite.
  103. Returns:
  104. A list of test configurations. Each test configuration is a dictionary of
  105. the form:
  106. { name: 'NCTEST_NAME'
  107. suite_name: 'SOURCE_FILE_NAME'
  108. expectations: [re.Pattern, re.Pattern] }
  109. The |suite_name| is used to generate a pretty gtest output on successful
  110. completion of the no compile test.
  111. The compiled regexps in |expectations| define the valid outputs of the
  112. compiler. If any one of the listed patterns matches either the stderr or
  113. stdout from the compilation, and the compilation failed, then the test is
  114. considered to have succeeded. If the list is empty, than we ignore the
  115. compiler output and just check for failed compilation. If |expectations|
  116. is actually None, then this specifies a compiler sanity check test, which
  117. should expect a SUCCESSFUL compilation.
  118. """
  119. sourcefile = open(sourcefile_path, 'r')
  120. # Start with at least the compiler sanity test. You need to always have one
  121. # sanity test to show that compiler flags and configuration are not just
  122. # wrong. Otherwise, having a misconfigured compiler, or an error in the
  123. # shared portions of the .nc file would cause all tests to erroneously pass.
  124. test_configs = []
  125. for line in sourcefile:
  126. match_result = NCTEST_CONFIG_RE.match(line)
  127. if not match_result:
  128. continue
  129. groups = match_result.groups()
  130. # Grab the name and remove the defined() predicate if there is one.
  131. name = groups[0]
  132. strip_result = STRIP_DEFINED_RE.match(name)
  133. if strip_result:
  134. name = strip_result.group(1)
  135. # Read expectations if there are any.
  136. test_configs.append({'name': name,
  137. 'suite_name': suite_name,
  138. 'expectations': ParseExpectation(groups[1])})
  139. sourcefile.close()
  140. return test_configs
  141. def StartTest(compiler, sourcefile_path, tempfile_dir, cflags, config):
  142. """Start one negative compile test.
  143. Args:
  144. sourcefile_path: The path to the source file.
  145. tempfile_dir: A directory to store temporary data from tests.
  146. cflags: An array of strings with all the CFLAGS to give to gcc.
  147. config: A dictionary describing the test. See ExtractTestConfigs
  148. for a description of the config format.
  149. Returns:
  150. A dictionary containing all the information about the started test. The
  151. fields in the dictionary are as follows:
  152. { 'proc': A subprocess object representing the compiler run.
  153. 'cmdline': The executed command line.
  154. 'name': The name of the test.
  155. 'suite_name': The suite name to use when generating the gunit test
  156. result.
  157. 'terminate_timeout': The timestamp in seconds since the epoch after
  158. which the test should be terminated.
  159. 'kill_timeout': The timestamp in seconds since the epoch after which
  160. the test should be given a hard kill signal.
  161. 'started_at': A timestamp in seconds since the epoch for when this test
  162. was started.
  163. 'aborted_at': A timestamp in seconds since the epoch for when this test
  164. was aborted. If the test completed successfully,
  165. this value is 0.
  166. 'finished_at': A timestamp in seconds since the epoch for when this
  167. test was successfully complete. If the test is aborted,
  168. or running, this value is 0.
  169. 'expectations': A dictionary with the test expectations. See
  170. ParseExpectation() for the structure.
  171. }
  172. """
  173. cmdline = [compiler]
  174. cmdline.extend(cflags)
  175. name = config['name']
  176. expectations = config['expectations']
  177. if expectations is not None:
  178. cmdline.append('-D%s' % name)
  179. cmdline.extend(['-o', '/dev/null', '-c', '-x', 'c++',
  180. sourcefile_path])
  181. test_stdout = tempfile.TemporaryFile(dir=tempfile_dir, mode='w+')
  182. test_stderr = tempfile.TemporaryFile(dir=tempfile_dir, mode='w+')
  183. process = subprocess.Popen(cmdline, stdout=test_stdout, stderr=test_stderr)
  184. now = time.time()
  185. return {'proc': process,
  186. 'cmdline': ' '.join(cmdline),
  187. 'stdout': test_stdout,
  188. 'stderr': test_stderr,
  189. 'name': name,
  190. 'suite_name': config['suite_name'],
  191. 'terminate_timeout': now + NCTEST_TERMINATE_TIMEOUT_SEC,
  192. 'kill_timeout': now + NCTEST_KILL_TIMEOUT_SEC,
  193. 'started_at': now,
  194. 'aborted_at': 0,
  195. 'finished_at': 0,
  196. 'expectations': expectations}
  197. def PassTest(resultfile, resultlog, test):
  198. """Logs the result of a test started by StartTest(), or a disabled test
  199. configuration.
  200. Args:
  201. resultfile: File object for .cc file that results are written to.
  202. resultlog: File object for the log file.
  203. test: An instance of the dictionary returned by StartTest(), a
  204. configuration from ExtractTestConfigs().
  205. """
  206. resultfile.write(GUNIT_TEMPLATE % (
  207. test['suite_name'], test['name']))
  208. # The 'started_at' key is only added if a test has been started.
  209. if 'started_at' in test:
  210. resultlog.write(LOG_TEMPLATE % (
  211. test['suite_name'], test['name'],
  212. test['finished_at'] - test['started_at'],
  213. test['started_at'], test['finished_at']))
  214. def FailTest(resultfile, test, error, stdout=None, stderr=None):
  215. """Logs the result of a test started by StartTest()
  216. Args:
  217. resultfile: File object for .cc file that results are written to.
  218. test: An instance of the dictionary returned by StartTest()
  219. error: The printable reason for the failure.
  220. stdout: The test's output to stdout.
  221. stderr: The test's output to stderr.
  222. """
  223. resultfile.write('#error "%s Failed: %s"\n' % (test['name'], error))
  224. resultfile.write('#error "compile line: %s"\n' % test['cmdline'])
  225. if stdout and len(stdout) != 0:
  226. resultfile.write('#error "%s stdout:"\n' % test['name'])
  227. for line in stdout.split('\n'):
  228. resultfile.write('#error " %s:"\n' % line)
  229. if stderr and len(stderr) != 0:
  230. resultfile.write('#error "%s stderr:"\n' % test['name'])
  231. for line in stderr.split('\n'):
  232. resultfile.write('#error " %s"\n' % line)
  233. resultfile.write('\n')
  234. def WriteStats(resultlog, suite_name, timings):
  235. """Logs the peformance timings for each stage of the script.
  236. Args:
  237. resultlog: File object for the log file.
  238. suite_name: The name of the GUnit suite this test belongs to.
  239. timings: Dictionary with timestamps for each stage of the script run.
  240. """
  241. stats_template = """
  242. TEST(%s): Started %f, Ended %f, Total %fs, Extract %fs, Compile %fs, Process %fs
  243. """
  244. total_secs = timings['results_processed'] - timings['started']
  245. extract_secs = timings['extract_done'] - timings['started']
  246. compile_secs = timings['compile_done'] - timings['extract_done']
  247. process_secs = timings['results_processed'] - timings['compile_done']
  248. resultlog.write(stats_template % (
  249. suite_name, timings['started'], timings['results_processed'], total_secs,
  250. extract_secs, compile_secs, process_secs))
  251. def ExtractTestOutputAndCleanup(test):
  252. """Test output is in temp files. Read those and delete them.
  253. Returns: A tuple (stderr, stdout).
  254. """
  255. outputs = [None, None]
  256. for i, stream_name in ((0, "stdout"), (1, "stderr")):
  257. stream = test[stream_name]
  258. stream.seek(0)
  259. outputs[i] = stream.read()
  260. stream.close()
  261. return outputs
  262. def ProcessTestResult(resultfile, resultlog, test):
  263. """Interprets and logs the result of a test started by StartTest()
  264. Args:
  265. resultfile: File object for .cc file that results are written to.
  266. resultlog: File object for the log file.
  267. test: The dictionary from StartTest() to process.
  268. """
  269. proc = test['proc']
  270. proc.wait()
  271. (stdout, stderr) = ExtractTestOutputAndCleanup(test)
  272. if test['aborted_at'] != 0:
  273. FailTest(resultfile, test, "Compile timed out. Started %f ended %f." %
  274. (test['started_at'], test['aborted_at']))
  275. return
  276. if proc.poll() == 0:
  277. # Handle failure due to successful compile.
  278. FailTest(resultfile, test,
  279. 'Unexpected successful compilation.',
  280. stdout, stderr)
  281. return
  282. else:
  283. # Check the output has the right expectations. If there are no
  284. # expectations, then we just consider the output "matched" by default.
  285. if len(test['expectations']) == 0:
  286. PassTest(resultfile, resultlog, test)
  287. return
  288. # Otherwise test against all expectations.
  289. for regexp in test['expectations']:
  290. if (regexp.search(stdout) is not None or
  291. regexp.search(stderr) is not None):
  292. PassTest(resultfile, resultlog, test)
  293. return
  294. expectation_str = ', '.join(
  295. ["r'%s'" % regexp.pattern for regexp in test['expectations']])
  296. FailTest(resultfile, test,
  297. 'Expectations [%s] did not match output.' % expectation_str,
  298. stdout, stderr)
  299. return
  300. def CompleteAtLeastOneTest(executing_tests):
  301. """Blocks until at least one task is removed from executing_tests.
  302. This function removes completed tests from executing_tests, logging failures
  303. and output. If no tests can be removed, it will enter a poll-loop until one
  304. test finishes or times out. On a timeout, this function is responsible for
  305. terminating the process in the appropriate fashion.
  306. Args:
  307. executing_tests: A dict mapping a string containing the test name to the
  308. test dict return from StartTest().
  309. Returns:
  310. A list of tests that have finished.
  311. """
  312. finished_tests = []
  313. busy_loop_timeout = time.time() + BUSY_LOOP_MAX_TIME_SEC
  314. while len(finished_tests) == 0:
  315. # If we don't make progress for too long, assume the code is just dead.
  316. assert busy_loop_timeout > time.time()
  317. # Select on the output files to block until we have something to
  318. # do. We ignore the return value from select and just poll all
  319. # processes.
  320. read_set = []
  321. for test in executing_tests.values():
  322. read_set.extend([test['stdout'], test['stderr']])
  323. select.select(read_set, [], read_set, NCTEST_TERMINATE_TIMEOUT_SEC)
  324. # Now attempt to process results.
  325. now = time.time()
  326. for test in executing_tests.values():
  327. proc = test['proc']
  328. if proc.poll() is not None:
  329. test['finished_at'] = now
  330. finished_tests.append(test)
  331. elif test['terminate_timeout'] < now:
  332. proc.terminate()
  333. test['aborted_at'] = now
  334. elif test['kill_timeout'] < now:
  335. proc.kill()
  336. test['aborted_at'] = now
  337. if len(finished_tests) == 0:
  338. # We had output from some process but no process had
  339. # finished. To avoid busy looping while waiting for a process to
  340. # finish, insert a small 100 ms delay here.
  341. time.sleep(0.1)
  342. for test in finished_tests:
  343. del executing_tests[test['name']]
  344. return finished_tests
  345. def main():
  346. if len(sys.argv) < 6 or sys.argv[5] != '--':
  347. print('Usage: %s <compiler> <parallelism> <sourcefile> <resultfile> '
  348. '-- <cflags...>' % sys.argv[0])
  349. sys.exit(1)
  350. # Force us into the "C" locale so the compiler doesn't localize its output.
  351. # In particular, this stops gcc from using smart quotes when in english UTF-8
  352. # locales. This makes the expectation writing much easier.
  353. os.environ['LC_ALL'] = 'C'
  354. compiler = sys.argv[1]
  355. parallelism = int(sys.argv[2])
  356. sourcefile_path = sys.argv[3]
  357. resultfile_path = sys.argv[4]
  358. cflags = sys.argv[6:]
  359. timings = {'started': time.time()}
  360. ValidateInput(compiler, parallelism, sourcefile_path, cflags, resultfile_path)
  361. # Convert filename from underscores to CamelCase.
  362. words = os.path.splitext(os.path.basename(sourcefile_path))[0].split('_')
  363. words = [w.capitalize() for w in words]
  364. suite_name = 'NoCompile' + ''.join(words)
  365. test_configs = ExtractTestConfigs(sourcefile_path, suite_name)
  366. timings['extract_done'] = time.time()
  367. resultfile = io.StringIO()
  368. resultlog = io.StringIO()
  369. resultfile.write(RESULT_FILE_HEADER % sourcefile_path)
  370. # Run the no-compile tests, but ensure we do not run more than |parallelism|
  371. # tests at once.
  372. timings['header_written'] = time.time()
  373. executing_tests = {}
  374. finished_tests = []
  375. cflags.extend(['-MMD', '-MF', resultfile_path + '.d', '-MT', resultfile_path])
  376. test = StartTest(
  377. compiler,
  378. sourcefile_path,
  379. os.path.dirname(resultfile_path),
  380. cflags,
  381. { 'name': 'NCTEST_SANITY',
  382. 'suite_name': suite_name,
  383. 'expectations': None,
  384. })
  385. executing_tests[test['name']] = test
  386. for config in test_configs:
  387. # CompleteAtLeastOneTest blocks until at least one test finishes. Thus, this
  388. # acts as a semaphore. We cannot use threads + a real semaphore because
  389. # subprocess forks, which can cause all sorts of hilarity with threads.
  390. if len(executing_tests) >= parallelism:
  391. finished_tests.extend(CompleteAtLeastOneTest(executing_tests))
  392. if config['name'].startswith('DISABLED_'):
  393. PassTest(resultfile, resultlog, config)
  394. else:
  395. test = StartTest(compiler, sourcefile_path,
  396. os.path.dirname(resultfile_path), cflags, config)
  397. assert test['name'] not in executing_tests
  398. executing_tests[test['name']] = test
  399. # If there are no more test to start, we still need to drain the running
  400. # ones.
  401. while len(executing_tests) > 0:
  402. finished_tests.extend(CompleteAtLeastOneTest(executing_tests))
  403. timings['compile_done'] = time.time()
  404. finished_tests = sorted(finished_tests, key=lambda test: test['name'])
  405. for test in finished_tests:
  406. if test['name'] == 'NCTEST_SANITY':
  407. test['proc'].wait()
  408. (stdout, stderr) = ExtractTestOutputAndCleanup(test)
  409. return_code = test['proc'].returncode
  410. if return_code != 0:
  411. sys.stdout.write(stdout)
  412. sys.stderr.write(stderr)
  413. continue
  414. ProcessTestResult(resultfile, resultlog, test)
  415. timings['results_processed'] = time.time()
  416. WriteStats(resultlog, suite_name, timings)
  417. with open(resultfile_path + '.log', 'w') as fd:
  418. fd.write(resultlog.getvalue())
  419. if return_code == 0:
  420. with open(resultfile_path, 'w') as fd:
  421. fd.write(resultfile.getvalue())
  422. resultfile.close()
  423. if return_code != 0:
  424. print("No-compile driver failure with return_code %d. Result log:" %
  425. return_code)
  426. print(resultlog.getvalue())
  427. sys.exit(return_code)
  428. if __name__ == '__main__':
  429. main()