runner.py 7.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203
  1. #!/usr/bin/python
  2. # ex:ts=4:sw=4:sts=4:et
  3. # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
  4. #
  5. # Copyright (C) 2015 Alexandru Damian for Intel Corp.
  6. #
  7. # This program is free software; you can redistribute it and/or modify
  8. # it under the terms of the GNU General Public License version 2 as
  9. # published by the Free Software Foundation.
  10. #
  11. # This program is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. # GNU General Public License for more details.
  15. #
  16. # You should have received a copy of the GNU General Public License along
  17. # with this program; if not, write to the Free Software Foundation, Inc.,
  18. # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  19. # This is the main test execution controller. It is designed to be run
  20. # manually from the command line, or to be called from a different program
  21. # that schedules test execution.
  22. #
  23. # Execute runner.py -h for help.
  24. from __future__ import print_function
  25. import sys, os
  26. import unittest, importlib
  27. import logging, pprint, json
  28. from shellutils import ShellCmdException, mkdirhier, run_shell_cmd
  29. import config
  30. # we also log to a file, in addition to console, because our output is important
  31. __log_file_name__ = os.path.join(os.path.dirname(__file__), "log/tts_%d.log" % config.OWN_PID)
  32. mkdirhier(os.path.dirname(__log_file_name__))
  33. __log_file__ = open(__log_file_name__, "w")
  34. __file_handler__ = logging.StreamHandler(__log_file__)
  35. __file_handler__.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s"))
  36. config.logger.addHandler(__file_handler__)
  37. # set up log directory
  38. try:
  39. if not os.path.exists(config.LOGDIR):
  40. os.mkdir(config.LOGDIR)
  41. else:
  42. if not os.path.isdir(config.LOGDIR):
  43. raise Exception("Expected log dir '%s' is not actually a directory." % config.LOGDIR)
  44. except OSError as exc:
  45. raise exc
  46. # creates the under-test-branch as a separate directory
  47. def set_up_test_branch(settings, branch_name):
  48. testdir = "%s/%s.%d" % (settings['workdir'], config.TEST_DIR_NAME, config.OWN_PID)
  49. # creates the host dir
  50. if os.path.exists(testdir):
  51. raise Exception("Test dir '%s'is already there, aborting" % testdir)
  52. os.mkdir(testdir)
  53. # copies over the .git from the localclone
  54. run_shell_cmd("cp -a '%s'/.git '%s'" % (settings['localclone'], testdir))
  55. # add the remote if it doesn't exist
  56. crt_remotes = run_shell_cmd("git remote -v", cwd=testdir)
  57. remotes = [word for line in crt_remotes.split("\n") for word in line.split()]
  58. if not config.CONTRIB_REPO in remotes:
  59. remote_name = "tts_contrib"
  60. run_shell_cmd("git remote add %s %s" % (remote_name, config.CONTRIB_REPO), cwd=testdir)
  61. else:
  62. remote_name = remotes[remotes.index(config.CONTRIB_REPO) - 1]
  63. # do the fetch
  64. run_shell_cmd("git fetch %s -p" % remote_name, cwd=testdir)
  65. # do the checkout
  66. run_shell_cmd("git checkout origin/master && git branch -D %s; git checkout %s/%s -b %s && git reset --hard" % (branch_name, remote_name, branch_name, branch_name), cwd=testdir)
  67. return testdir
  68. def __search_for_tests():
  69. # we find all classes that can run, and run them
  70. tests = []
  71. for _, _, files_list in os.walk(os.path.dirname(os.path.abspath(__file__))):
  72. for module_file in [f[:-3] for f in files_list if f.endswith(".py") and not f.startswith("__init__")]:
  73. config.logger.debug("Inspecting module %s", module_file)
  74. current_module = importlib.import_module(module_file)
  75. crtclass_names = vars(current_module)
  76. for name in crtclass_names:
  77. tested_value = crtclass_names[name]
  78. if isinstance(tested_value, type(unittest.TestCase)) and issubclass(tested_value, unittest.TestCase):
  79. tests.append((module_file, name))
  80. break
  81. return tests
  82. # boilerplate to self discover tests and run them
  83. def execute_tests(dir_under_test, testname):
  84. if testname is not None and "." in testname:
  85. tests = []
  86. tests.append(tuple(testname.split(".", 2)))
  87. else:
  88. tests = __search_for_tests()
  89. # let's move to the directory under test
  90. crt_dir = os.getcwd()
  91. os.chdir(dir_under_test)
  92. # execute each module
  93. # pylint: disable=broad-except
  94. # we disable the broad-except because we want to actually catch all possible exceptions
  95. try:
  96. config.logger.debug("Discovered test clases: %s", pprint.pformat(tests))
  97. suite = unittest.TestSuite()
  98. loader = unittest.TestLoader()
  99. result = unittest.TestResult()
  100. for module_file, name in tests:
  101. suite.addTest(loader.loadTestsFromName("%s.%s" % (module_file, name)))
  102. config.logger.info("Running %d test(s)", suite.countTestCases())
  103. suite.run(result)
  104. for error in result.errors:
  105. config.logger.error("Exception on test: %s\n%s", error[0],
  106. "\n".join(["-- %s" % x for x in error[1].split("\n")]))
  107. for failure in result.failures:
  108. config.logger.error("Failed test: %s:\n%s\n", failure[0],
  109. "\n".join(["-- %s" % x for x in failure[1].split("\n")]))
  110. config.logger.info("Test results: %d ran, %d errors, %d failures", result.testsRun, len(result.errors), len(result.failures))
  111. except Exception as exc:
  112. import traceback
  113. config.logger.error("Exception while running test. Tracedump: \n%s", traceback.format_exc(exc))
  114. finally:
  115. os.chdir(crt_dir)
  116. return len(result.failures)
  117. # verify that we had a branch-under-test name as parameter
  118. def validate_args():
  119. from optparse import OptionParser
  120. parser = OptionParser(usage="usage: %prog [options] branch_under_test")
  121. parser.add_option("-t", "--test-dir", dest="testdir", default=None, help="Use specified directory to run tests, inhibits the checkout.")
  122. parser.add_option("-s", "--single", dest="singletest", default=None, help="Run only the specified test")
  123. (options, args) = parser.parse_args()
  124. if len(args) < 1:
  125. raise Exception("Please specify the branch to run on. Use option '-h' when in doubt.")
  126. return (options, args)
  127. # load the configuration options
  128. def read_settings():
  129. if not os.path.exists(config.SETTINGS_FILE) or not os.path.isfile(config.SETTINGS_FILE):
  130. raise Exception("Config file '%s' cannot be openend" % config.SETTINGS_FILE)
  131. return json.loads(open(config.SETTINGS_FILE, "r").read())
  132. # cleanup !
  133. def clean_up(testdir):
  134. run_shell_cmd("rm -rf -- '%s'" % testdir)
  135. def main():
  136. (options, args) = validate_args()
  137. settings = read_settings()
  138. need_cleanup = False
  139. testdir = None
  140. no_failures = 1
  141. try:
  142. if options.testdir is not None and os.path.exists(options.testdir):
  143. testdir = os.path.abspath(options.testdir)
  144. config.logger.info("No checkout, using %s", testdir)
  145. else:
  146. need_cleanup = True
  147. testdir = set_up_test_branch(settings, args[0]) # we expect a branch name as first argument
  148. config.TESTDIR = testdir # we let tests know where to run
  149. no_failures = execute_tests(testdir, options.singletest)
  150. except ShellCmdException as exc:
  151. import traceback
  152. config.logger.error("Error while setting up testing. Traceback: \n%s", traceback.format_exc(exc))
  153. finally:
  154. if need_cleanup and testdir is not None:
  155. clean_up(testdir)
  156. sys.exit(no_failures)
  157. if __name__ == "__main__":
  158. main()