check-xinclude-test-suite.py 5.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220
  1. #!/usr/bin/env python
  2. import sys
  3. import time
  4. import os
  5. sys.path.insert(0, "python")
  6. import libxml2
  7. #
  8. # the testsuite description
  9. #
  10. DIR="xinclude-test-suite"
  11. CONF="testdescr.xml"
  12. LOG="check-xinclude-test-suite.log"
  13. log = open(LOG, "w")
  14. os.chdir(DIR)
  15. test_nr = 0
  16. test_succeed = 0
  17. test_failed = 0
  18. test_error = 0
  19. #
  20. # Error and warning handlers
  21. #
  22. error_nr = 0
  23. error_msg = ''
  24. def errorHandler(ctx, str):
  25. global error_nr
  26. global error_msg
  27. if str.find("error:") >= 0:
  28. error_nr = error_nr + 1
  29. if len(error_msg) < 300:
  30. if len(error_msg) == 0 or error_msg[-1] == '\n':
  31. error_msg = error_msg + " >>" + str
  32. else:
  33. error_msg = error_msg + str
  34. libxml2.registerErrorHandler(errorHandler, None)
  35. def testXInclude(filename, id):
  36. global error_nr
  37. global error_msg
  38. global log
  39. error_nr = 0
  40. error_msg = ''
  41. print("testXInclude(%s, %s)" % (filename, id))
  42. return 1
  43. def runTest(test, basedir):
  44. global test_nr
  45. global test_failed
  46. global test_error
  47. global test_succeed
  48. global error_msg
  49. global log
  50. fatal_error = 0
  51. uri = test.prop('href')
  52. id = test.prop('id')
  53. type = test.prop('type')
  54. if uri is None:
  55. print("Test without ID:", uri)
  56. return -1
  57. if id is None:
  58. print("Test without URI:", id)
  59. return -1
  60. if type is None:
  61. print("Test without URI:", id)
  62. return -1
  63. if basedir != None:
  64. URI = basedir + "/" + uri
  65. else:
  66. URI = uri
  67. if os.access(URI, os.R_OK) == 0:
  68. print("Test %s missing: base %s uri %s" % (URI, basedir, uri))
  69. return -1
  70. expected = None
  71. outputfile = None
  72. diff = None
  73. if type != 'error':
  74. output = test.xpathEval('string(output)')
  75. if output == 'No output file.':
  76. output = None
  77. if output == '':
  78. output = None
  79. if output != None:
  80. if basedir != None:
  81. output = basedir + "/" + output
  82. if os.access(output, os.R_OK) == 0:
  83. print("Result for %s missing: %s" % (id, output))
  84. output = None
  85. else:
  86. try:
  87. f = open(output)
  88. expected = f.read()
  89. outputfile = output
  90. except:
  91. print("Result for %s unreadable: %s" % (id, output))
  92. try:
  93. # print("testing %s" % (URI))
  94. doc = libxml2.parseFile(URI)
  95. except:
  96. doc = None
  97. if doc != None:
  98. res = doc.xincludeProcess()
  99. if res >= 0 and expected != None:
  100. result = doc.serialize()
  101. if result != expected:
  102. print("Result for %s differs" % (id))
  103. open("xinclude.res", "w").write(result)
  104. diff = os.popen("diff %s xinclude.res" % outputfile).read()
  105. doc.freeDoc()
  106. else:
  107. print("Failed to parse %s" % (URI))
  108. res = -1
  109. test_nr = test_nr + 1
  110. if type == 'success':
  111. if res > 0:
  112. test_succeed = test_succeed + 1
  113. elif res == 0:
  114. test_failed = test_failed + 1
  115. print("Test %s: no substitution done ???" % (id))
  116. elif res < 0:
  117. test_error = test_error + 1
  118. print("Test %s: failed valid XInclude processing" % (id))
  119. elif type == 'error':
  120. if res > 0:
  121. test_error = test_error + 1
  122. print("Test %s: failed to detect invalid XInclude processing" % (id))
  123. elif res == 0:
  124. test_failed = test_failed + 1
  125. print("Test %s: Invalid but no substitution done" % (id))
  126. elif res < 0:
  127. test_succeed = test_succeed + 1
  128. elif type == 'optional':
  129. if res > 0:
  130. test_succeed = test_succeed + 1
  131. else:
  132. print("Test %s: failed optional test" % (id))
  133. # Log the ontext
  134. if res != 1:
  135. log.write("Test ID %s\n" % (id))
  136. log.write(" File: %s\n" % (URI))
  137. content = test.content.strip()
  138. while content[-1] == '\n':
  139. content = content[0:-1]
  140. log.write(" %s:%s\n\n" % (type, content))
  141. if error_msg != '':
  142. log.write(" ----\n%s ----\n" % (error_msg))
  143. error_msg = ''
  144. log.write("\n")
  145. if diff != None:
  146. log.write("diff from test %s:\n" %(id))
  147. log.write(" -----------\n%s\n -----------\n" % (diff));
  148. return 0
  149. def runTestCases(case):
  150. creator = case.prop('creator')
  151. if creator != None:
  152. print("=>", creator)
  153. base = case.getBase(None)
  154. basedir = case.prop('basedir')
  155. if basedir != None:
  156. base = libxml2.buildURI(basedir, base)
  157. test = case.children
  158. while test != None:
  159. if test.name == 'testcase':
  160. runTest(test, base)
  161. if test.name == 'testcases':
  162. runTestCases(test)
  163. test = test.next
  164. conf = libxml2.parseFile(CONF)
  165. if conf is None:
  166. print("Unable to load %s" % CONF)
  167. sys.exit(1)
  168. testsuite = conf.getRootElement()
  169. if testsuite.name != 'testsuite':
  170. print("Expecting TESTSUITE root element: aborting")
  171. sys.exit(1)
  172. profile = testsuite.prop('PROFILE')
  173. if profile != None:
  174. print(profile)
  175. start = time.time()
  176. case = testsuite.children
  177. while case != None:
  178. if case.name == 'testcases':
  179. old_test_nr = test_nr
  180. old_test_succeed = test_succeed
  181. old_test_failed = test_failed
  182. old_test_error = test_error
  183. runTestCases(case)
  184. print(" Ran %d tests: %d succeeded, %d failed and %d generated an error" % (
  185. test_nr - old_test_nr, test_succeed - old_test_succeed,
  186. test_failed - old_test_failed, test_error - old_test_error))
  187. case = case.next
  188. conf.freeDoc()
  189. log.close()
  190. print("Ran %d tests: %d succeeded, %d failed and %d generated an error in %.2f s." % (
  191. test_nr, test_succeed, test_failed, test_error, time.time() - start))