check-relaxng-test-suite2.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422
  1. #!/usr/bin/env python
  2. import sys
  3. import time
  4. import os
  5. try:
  6. # Python 2
  7. from StringIO import StringIO
  8. except ImportError:
  9. # Python 3
  10. from io import StringIO
  11. sys.path.insert(0, "python")
  12. import libxml2
  13. # Memory debug specific
  14. libxml2.debugMemory(1)
  15. debug = 0
  16. quiet = 1
  17. #
  18. # the testsuite description
  19. #
  20. CONF=os.path.join(os.path.dirname(__file__), "test/relaxng/testsuite.xml")
  21. LOG="check-relaxng-test-suite2.log"
  22. log = open(LOG, "w")
  23. nb_schemas_tests = 0
  24. nb_schemas_success = 0
  25. nb_schemas_failed = 0
  26. nb_instances_tests = 0
  27. nb_instances_success = 0
  28. nb_instances_failed = 0
  29. libxml2.lineNumbersDefault(1)
  30. #
  31. # Resolver callback
  32. #
  33. resources = {}
  34. def resolver(URL, ID, ctxt):
  35. global resources
  36. if URL in resources:
  37. return(StringIO(resources[URL]))
  38. log.write("Resolver failure: asked %s\n" % (URL))
  39. log.write("resources: %s\n" % (resources))
  40. return None
  41. #
  42. # Load the previous results
  43. #
  44. #results = {}
  45. #previous = {}
  46. #
  47. #try:
  48. # res = libxml2.parseFile(RES)
  49. #except:
  50. # log.write("Could not parse %s" % (RES))
  51. #
  52. # handle a valid instance
  53. #
  54. def handle_valid(node, schema):
  55. global log
  56. global nb_instances_success
  57. global nb_instances_failed
  58. instance = node.prop("dtd")
  59. if instance is None:
  60. instance = ""
  61. child = node.children
  62. while child != None:
  63. if child.type != 'text':
  64. instance = instance + child.serialize()
  65. child = child.next
  66. # mem = libxml2.debugMemory(1);
  67. try:
  68. doc = libxml2.parseDoc(instance)
  69. except:
  70. doc = None
  71. if doc is None:
  72. log.write("\nFailed to parse correct instance:\n-----\n")
  73. log.write(instance)
  74. log.write("\n-----\n")
  75. nb_instances_failed = nb_instances_failed + 1
  76. return
  77. if debug:
  78. print("instance line %d" % (node.lineNo()))
  79. try:
  80. ctxt = schema.relaxNGNewValidCtxt()
  81. ret = doc.relaxNGValidateDoc(ctxt)
  82. del ctxt
  83. except:
  84. ret = -1
  85. doc.freeDoc()
  86. # if mem != libxml2.debugMemory(1):
  87. # print("validating instance %d line %d leaks" % (
  88. # nb_instances_tests, node.lineNo()))
  89. if ret != 0:
  90. log.write("\nFailed to validate correct instance:\n-----\n")
  91. log.write(instance)
  92. log.write("\n-----\n")
  93. nb_instances_failed = nb_instances_failed + 1
  94. else:
  95. nb_instances_success = nb_instances_success + 1
  96. #
  97. # handle an invalid instance
  98. #
  99. def handle_invalid(node, schema):
  100. global log
  101. global nb_instances_success
  102. global nb_instances_failed
  103. instance = node.prop("dtd")
  104. if instance is None:
  105. instance = ""
  106. child = node.children
  107. while child != None:
  108. if child.type != 'text':
  109. instance = instance + child.serialize()
  110. child = child.next
  111. # mem = libxml2.debugMemory(1);
  112. try:
  113. doc = libxml2.parseDoc(instance)
  114. except:
  115. doc = None
  116. if doc is None:
  117. log.write("\nStrange: failed to parse incorrect instance:\n-----\n")
  118. log.write(instance)
  119. log.write("\n-----\n")
  120. return
  121. if debug:
  122. print("instance line %d" % (node.lineNo()))
  123. try:
  124. ctxt = schema.relaxNGNewValidCtxt()
  125. ret = doc.relaxNGValidateDoc(ctxt)
  126. del ctxt
  127. except:
  128. ret = -1
  129. doc.freeDoc()
  130. # mem2 = libxml2.debugMemory(1)
  131. # if mem != mem2:
  132. # print("validating instance %d line %d leaks %d bytes" % (
  133. # nb_instances_tests, node.lineNo(), mem2 - mem))
  134. if ret == 0:
  135. log.write("\nFailed to detect validation problem in instance:\n-----\n")
  136. log.write(instance)
  137. log.write("\n-----\n")
  138. nb_instances_failed = nb_instances_failed + 1
  139. else:
  140. nb_instances_success = nb_instances_success + 1
  141. #
  142. # handle an incorrect test
  143. #
  144. def handle_correct(node):
  145. global log
  146. global nb_schemas_success
  147. global nb_schemas_failed
  148. schema = ""
  149. child = node.children
  150. while child != None:
  151. if child.type != 'text':
  152. schema = schema + child.serialize()
  153. child = child.next
  154. try:
  155. rngp = libxml2.relaxNGNewMemParserCtxt(schema, len(schema))
  156. rngs = rngp.relaxNGParse()
  157. except:
  158. rngs = None
  159. if rngs is None:
  160. log.write("\nFailed to compile correct schema:\n-----\n")
  161. log.write(schema)
  162. log.write("\n-----\n")
  163. nb_schemas_failed = nb_schemas_failed + 1
  164. else:
  165. nb_schemas_success = nb_schemas_success + 1
  166. return rngs
  167. def handle_incorrect(node):
  168. global log
  169. global nb_schemas_success
  170. global nb_schemas_failed
  171. schema = ""
  172. child = node.children
  173. while child != None:
  174. if child.type != 'text':
  175. schema = schema + child.serialize()
  176. child = child.next
  177. try:
  178. rngp = libxml2.relaxNGNewMemParserCtxt(schema, len(schema))
  179. rngs = rngp.relaxNGParse()
  180. except:
  181. rngs = None
  182. if rngs != None:
  183. log.write("\nFailed to detect schema error in:\n-----\n")
  184. log.write(schema)
  185. log.write("\n-----\n")
  186. nb_schemas_failed = nb_schemas_failed + 1
  187. else:
  188. # log.write("\nSuccess detecting schema error in:\n-----\n")
  189. # log.write(schema)
  190. # log.write("\n-----\n")
  191. nb_schemas_success = nb_schemas_success + 1
  192. return None
  193. #
  194. # resource handling: keep a dictionary of URL->string mappings
  195. #
  196. def handle_resource(node, dir):
  197. global resources
  198. try:
  199. name = node.prop('name')
  200. except:
  201. name = None
  202. if name is None or name == '':
  203. log.write("resource has no name")
  204. return;
  205. if dir != None:
  206. # name = libxml2.buildURI(name, dir)
  207. name = dir + '/' + name
  208. res = ""
  209. child = node.children
  210. while child != None:
  211. if child.type != 'text':
  212. res = res + child.serialize()
  213. child = child.next
  214. resources[name] = res
  215. #
  216. # dir handling: pseudo directory resources
  217. #
  218. def handle_dir(node, dir):
  219. try:
  220. name = node.prop('name')
  221. except:
  222. name = None
  223. if name is None or name == '':
  224. log.write("resource has no name")
  225. return;
  226. if dir != None:
  227. # name = libxml2.buildURI(name, dir)
  228. name = dir + '/' + name
  229. dirs = node.xpathEval('dir')
  230. for dir in dirs:
  231. handle_dir(dir, name)
  232. res = node.xpathEval('resource')
  233. for r in res:
  234. handle_resource(r, name)
  235. #
  236. # handle a testCase element
  237. #
  238. def handle_testCase(node):
  239. global nb_schemas_tests
  240. global nb_instances_tests
  241. global resources
  242. sections = node.xpathEval('string(section)')
  243. log.write("\n ======== test %d line %d section %s ==========\n" % (
  244. nb_schemas_tests, node.lineNo(), sections))
  245. resources = {}
  246. if debug:
  247. print("test %d line %d" % (nb_schemas_tests, node.lineNo()))
  248. dirs = node.xpathEval('dir')
  249. for dir in dirs:
  250. handle_dir(dir, None)
  251. res = node.xpathEval('resource')
  252. for r in res:
  253. handle_resource(r, None)
  254. tsts = node.xpathEval('incorrect')
  255. if tsts != []:
  256. if len(tsts) != 1:
  257. print("warning test line %d has more than one <incorrect> example" %(node.lineNo()))
  258. schema = handle_incorrect(tsts[0])
  259. else:
  260. tsts = node.xpathEval('correct')
  261. if tsts != []:
  262. if len(tsts) != 1:
  263. print("warning test line %d has more than one <correct> example"% (node.lineNo()))
  264. schema = handle_correct(tsts[0])
  265. else:
  266. print("warning <testCase> line %d has no <correct> nor <incorrect> child" % (node.lineNo()))
  267. nb_schemas_tests = nb_schemas_tests + 1;
  268. valids = node.xpathEval('valid')
  269. invalids = node.xpathEval('invalid')
  270. nb_instances_tests = nb_instances_tests + len(valids) + len(invalids)
  271. if schema != None:
  272. for valid in valids:
  273. handle_valid(valid, schema)
  274. for invalid in invalids:
  275. handle_invalid(invalid, schema)
  276. #
  277. # handle a testSuite element
  278. #
  279. def handle_testSuite(node, level = 0):
  280. global nb_schemas_tests, nb_schemas_success, nb_schemas_failed
  281. global nb_instances_tests, nb_instances_success, nb_instances_failed
  282. if level >= 1:
  283. old_schemas_tests = nb_schemas_tests
  284. old_schemas_success = nb_schemas_success
  285. old_schemas_failed = nb_schemas_failed
  286. old_instances_tests = nb_instances_tests
  287. old_instances_success = nb_instances_success
  288. old_instances_failed = nb_instances_failed
  289. docs = node.xpathEval('documentation')
  290. authors = node.xpathEval('author')
  291. if docs != []:
  292. msg = ""
  293. for doc in docs:
  294. msg = msg + doc.content + " "
  295. if authors != []:
  296. msg = msg + "written by "
  297. for author in authors:
  298. msg = msg + author.content + " "
  299. if quiet == 0:
  300. print(msg)
  301. sections = node.xpathEval('section')
  302. if sections != [] and level <= 0:
  303. msg = ""
  304. for section in sections:
  305. msg = msg + section.content + " "
  306. if quiet == 0:
  307. print("Tests for section %s" % (msg))
  308. for test in node.xpathEval('testCase'):
  309. handle_testCase(test)
  310. for test in node.xpathEval('testSuite'):
  311. handle_testSuite(test, level + 1)
  312. if level >= 1 and sections != []:
  313. msg = ""
  314. for section in sections:
  315. msg = msg + section.content + " "
  316. print("Result of tests for section %s" % (msg))
  317. if nb_schemas_tests != old_schemas_tests:
  318. print("found %d test schemas: %d success %d failures" % (
  319. nb_schemas_tests - old_schemas_tests,
  320. nb_schemas_success - old_schemas_success,
  321. nb_schemas_failed - old_schemas_failed))
  322. if nb_instances_tests != old_instances_tests:
  323. print("found %d test instances: %d success %d failures" % (
  324. nb_instances_tests - old_instances_tests,
  325. nb_instances_success - old_instances_success,
  326. nb_instances_failed - old_instances_failed))
  327. #
  328. # Parse the conf file
  329. #
  330. libxml2.substituteEntitiesDefault(1);
  331. testsuite = libxml2.parseFile(CONF)
  332. #
  333. # Error and warnng callbacks
  334. #
  335. def callback(ctx, str):
  336. global log
  337. log.write("%s%s" % (ctx, str))
  338. libxml2.registerErrorHandler(callback, "")
  339. libxml2.setEntityLoader(resolver)
  340. root = testsuite.getRootElement()
  341. if root.name != 'testSuite':
  342. print("%s doesn't start with a testSuite element, aborting" % (CONF))
  343. sys.exit(1)
  344. if quiet == 0:
  345. print("Running Relax NG testsuite")
  346. handle_testSuite(root)
  347. if quiet == 0:
  348. print("\nTOTAL:\n")
  349. if quiet == 0 or nb_schemas_failed != 0:
  350. print("found %d test schemas: %d success %d failures" % (
  351. nb_schemas_tests, nb_schemas_success, nb_schemas_failed))
  352. if quiet == 0 or nb_instances_failed != 0:
  353. print("found %d test instances: %d success %d failures" % (
  354. nb_instances_tests, nb_instances_success, nb_instances_failed))
  355. log.close()
  356. testsuite.freeDoc()
  357. # Memory debug specific
  358. libxml2.relaxNGCleanupTypes()
  359. libxml2.cleanupParser()
  360. if libxml2.debugMemory(1) == 0:
  361. if quiet == 0:
  362. print("OK")
  363. else:
  364. print("Memory leak %d bytes" % (libxml2.debugMemory(1)))
  365. libxml2.dumpMemory()