check-xsddata-test-suite.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424
  1. #!/usr/bin/env python
  2. import sys
  3. import time
  4. import os
  5. try:
  6. # Python 2
  7. from StringIO import StringIO
  8. except ImportError:
  9. # Python 3
  10. from io import StringIO
  11. sys.path.insert(0, "python")
  12. import libxml2
  13. # Memory debug specific
  14. libxml2.debugMemory(1)
  15. debug = 0
  16. verbose = 0
  17. quiet = 1
  18. #
  19. # the testsuite description
  20. #
  21. CONF=os.path.join(os.path.dirname(__file__), "test/xsdtest/xsdtestsuite.xml")
  22. LOG="check-xsddata-test-suite.log"
  23. log = open(LOG, "w")
  24. nb_schemas_tests = 0
  25. nb_schemas_success = 0
  26. nb_schemas_failed = 0
  27. nb_instances_tests = 0
  28. nb_instances_success = 0
  29. nb_instances_failed = 0
  30. libxml2.lineNumbersDefault(1)
  31. #
  32. # Error and warnng callbacks
  33. #
  34. def callback(ctx, str):
  35. global log
  36. log.write("%s%s" % (ctx, str))
  37. libxml2.registerErrorHandler(callback, "")
  38. #
  39. # Resolver callback
  40. #
  41. resources = {}
  42. def resolver(URL, ID, ctxt):
  43. global resources
  44. if URL in resources:
  45. return(StringIO(resources[URL]))
  46. log.write("Resolver failure: asked %s\n" % (URL))
  47. log.write("resources: %s\n" % (resources))
  48. return None
  49. #
  50. # handle a valid instance
  51. #
  52. def handle_valid(node, schema):
  53. global log
  54. global nb_instances_success
  55. global nb_instances_failed
  56. instance = node.prop("dtd")
  57. if instance is None:
  58. instance = ""
  59. child = node.children
  60. while child != None:
  61. if child.type != 'text':
  62. instance = instance + child.serialize()
  63. child = child.next
  64. mem = libxml2.debugMemory(1);
  65. try:
  66. doc = libxml2.parseDoc(instance)
  67. except:
  68. doc = None
  69. if doc is None:
  70. log.write("\nFailed to parse correct instance:\n-----\n")
  71. log.write(instance)
  72. log.write("\n-----\n")
  73. nb_instances_failed = nb_instances_failed + 1
  74. return
  75. if debug:
  76. print("instance line %d" % (node.lineNo()))
  77. try:
  78. ctxt = schema.relaxNGNewValidCtxt()
  79. ret = doc.relaxNGValidateDoc(ctxt)
  80. del ctxt
  81. except:
  82. ret = -1
  83. doc.freeDoc()
  84. if mem != libxml2.debugMemory(1):
  85. print("validating instance %d line %d leaks" % (
  86. nb_instances_tests, node.lineNo()))
  87. if ret != 0:
  88. log.write("\nFailed to validate correct instance:\n-----\n")
  89. log.write(instance)
  90. log.write("\n-----\n")
  91. nb_instances_failed = nb_instances_failed + 1
  92. else:
  93. nb_instances_success = nb_instances_success + 1
  94. #
  95. # handle an invalid instance
  96. #
  97. def handle_invalid(node, schema):
  98. global log
  99. global nb_instances_success
  100. global nb_instances_failed
  101. instance = node.prop("dtd")
  102. if instance is None:
  103. instance = ""
  104. child = node.children
  105. while child != None:
  106. if child.type != 'text':
  107. instance = instance + child.serialize()
  108. child = child.next
  109. # mem = libxml2.debugMemory(1);
  110. try:
  111. doc = libxml2.parseDoc(instance)
  112. except:
  113. doc = None
  114. if doc is None:
  115. log.write("\nStrange: failed to parse incorrect instance:\n-----\n")
  116. log.write(instance)
  117. log.write("\n-----\n")
  118. return
  119. if debug:
  120. print("instance line %d" % (node.lineNo()))
  121. try:
  122. ctxt = schema.relaxNGNewValidCtxt()
  123. ret = doc.relaxNGValidateDoc(ctxt)
  124. del ctxt
  125. except:
  126. ret = -1
  127. doc.freeDoc()
  128. # if mem != libxml2.debugMemory(1):
  129. # print("validating instance %d line %d leaks" % (
  130. # nb_instances_tests, node.lineNo()))
  131. if ret == 0:
  132. log.write("\nFailed to detect validation problem in instance:\n-----\n")
  133. log.write(instance)
  134. log.write("\n-----\n")
  135. nb_instances_failed = nb_instances_failed + 1
  136. else:
  137. nb_instances_success = nb_instances_success + 1
  138. #
  139. # handle an incorrect test
  140. #
  141. def handle_correct(node):
  142. global log
  143. global nb_schemas_success
  144. global nb_schemas_failed
  145. schema = ""
  146. child = node.children
  147. while child != None:
  148. if child.type != 'text':
  149. schema = schema + child.serialize()
  150. child = child.next
  151. try:
  152. rngp = libxml2.relaxNGNewMemParserCtxt(schema, len(schema))
  153. rngs = rngp.relaxNGParse()
  154. except:
  155. rngs = None
  156. if rngs is None:
  157. log.write("\nFailed to compile correct schema:\n-----\n")
  158. log.write(schema)
  159. log.write("\n-----\n")
  160. nb_schemas_failed = nb_schemas_failed + 1
  161. else:
  162. nb_schemas_success = nb_schemas_success + 1
  163. return rngs
  164. def handle_incorrect(node):
  165. global log
  166. global nb_schemas_success
  167. global nb_schemas_failed
  168. schema = ""
  169. child = node.children
  170. while child != None:
  171. if child.type != 'text':
  172. schema = schema + child.serialize()
  173. child = child.next
  174. try:
  175. rngp = libxml2.relaxNGNewMemParserCtxt(schema, len(schema))
  176. rngs = rngp.relaxNGParse()
  177. except:
  178. rngs = None
  179. if rngs != None:
  180. log.write("\nFailed to detect schema error in:\n-----\n")
  181. log.write(schema)
  182. log.write("\n-----\n")
  183. nb_schemas_failed = nb_schemas_failed + 1
  184. else:
  185. # log.write("\nSuccess detecting schema error in:\n-----\n")
  186. # log.write(schema)
  187. # log.write("\n-----\n")
  188. nb_schemas_success = nb_schemas_success + 1
  189. return None
  190. #
  191. # resource handling: keep a dictionary of URL->string mappings
  192. #
  193. def handle_resource(node, dir):
  194. global resources
  195. try:
  196. name = node.prop('name')
  197. except:
  198. name = None
  199. if name is None or name == '':
  200. log.write("resource has no name")
  201. return;
  202. if dir != None:
  203. # name = libxml2.buildURI(name, dir)
  204. name = dir + '/' + name
  205. res = ""
  206. child = node.children
  207. while child != None:
  208. if child.type != 'text':
  209. res = res + child.serialize()
  210. child = child.next
  211. resources[name] = res
  212. #
  213. # dir handling: pseudo directory resources
  214. #
  215. def handle_dir(node, dir):
  216. try:
  217. name = node.prop('name')
  218. except:
  219. name = None
  220. if name is None or name == '':
  221. log.write("resource has no name")
  222. return;
  223. if dir != None:
  224. # name = libxml2.buildURI(name, dir)
  225. name = dir + '/' + name
  226. dirs = node.xpathEval('dir')
  227. for dir in dirs:
  228. handle_dir(dir, name)
  229. res = node.xpathEval('resource')
  230. for r in res:
  231. handle_resource(r, name)
  232. #
  233. # handle a testCase element
  234. #
  235. def handle_testCase(node):
  236. global nb_schemas_tests
  237. global nb_instances_tests
  238. global resources
  239. sections = node.xpathEval('string(section)')
  240. log.write("\n ======== test %d line %d section %s ==========\n" % (
  241. nb_schemas_tests, node.lineNo(), sections))
  242. resources = {}
  243. if debug:
  244. print("test %d line %d" % (nb_schemas_tests, node.lineNo()))
  245. dirs = node.xpathEval('dir')
  246. for dir in dirs:
  247. handle_dir(dir, None)
  248. res = node.xpathEval('resource')
  249. for r in res:
  250. handle_resource(r, None)
  251. tsts = node.xpathEval('incorrect')
  252. if tsts != []:
  253. if len(tsts) != 1:
  254. print("warning test line %d has more than one <incorrect> example" %(node.lineNo()))
  255. schema = handle_incorrect(tsts[0])
  256. else:
  257. tsts = node.xpathEval('correct')
  258. if tsts != []:
  259. if len(tsts) != 1:
  260. print("warning test line %d has more than one <correct> example"% (node.lineNo()))
  261. schema = handle_correct(tsts[0])
  262. else:
  263. print("warning <testCase> line %d has no <correct> nor <incorrect> child" % (node.lineNo()))
  264. nb_schemas_tests = nb_schemas_tests + 1;
  265. valids = node.xpathEval('valid')
  266. invalids = node.xpathEval('invalid')
  267. nb_instances_tests = nb_instances_tests + len(valids) + len(invalids)
  268. if schema != None:
  269. for valid in valids:
  270. handle_valid(valid, schema)
  271. for invalid in invalids:
  272. handle_invalid(invalid, schema)
  273. #
  274. # handle a testSuite element
  275. #
  276. def handle_testSuite(node, level = 0):
  277. global nb_schemas_tests, nb_schemas_success, nb_schemas_failed
  278. global nb_instances_tests, nb_instances_success, nb_instances_failed
  279. if verbose and level >= 0:
  280. old_schemas_tests = nb_schemas_tests
  281. old_schemas_success = nb_schemas_success
  282. old_schemas_failed = nb_schemas_failed
  283. old_instances_tests = nb_instances_tests
  284. old_instances_success = nb_instances_success
  285. old_instances_failed = nb_instances_failed
  286. docs = node.xpathEval('documentation')
  287. authors = node.xpathEval('author')
  288. if docs != []:
  289. msg = ""
  290. for doc in docs:
  291. msg = msg + doc.content + " "
  292. if authors != []:
  293. msg = msg + "written by "
  294. for author in authors:
  295. msg = msg + author.content + " "
  296. if quiet == 0:
  297. print(msg)
  298. sections = node.xpathEval('section')
  299. if verbose and sections != [] and level <= 0:
  300. msg = ""
  301. for section in sections:
  302. msg = msg + section.content + " "
  303. if quiet == 0:
  304. print("Tests for section %s" % (msg))
  305. for test in node.xpathEval('testCase'):
  306. handle_testCase(test)
  307. for test in node.xpathEval('testSuite'):
  308. handle_testSuite(test, level + 1)
  309. if verbose and level >= 0 :
  310. if sections != []:
  311. msg = ""
  312. for section in sections:
  313. msg = msg + section.content + " "
  314. print("Result of tests for section %s" % (msg))
  315. elif docs != []:
  316. msg = ""
  317. for doc in docs:
  318. msg = msg + doc.content + " "
  319. print("Result of tests for %s" % (msg))
  320. if nb_schemas_tests != old_schemas_tests:
  321. print("found %d test schemas: %d success %d failures" % (
  322. nb_schemas_tests - old_schemas_tests,
  323. nb_schemas_success - old_schemas_success,
  324. nb_schemas_failed - old_schemas_failed))
  325. if nb_instances_tests != old_instances_tests:
  326. print("found %d test instances: %d success %d failures" % (
  327. nb_instances_tests - old_instances_tests,
  328. nb_instances_success - old_instances_success,
  329. nb_instances_failed - old_instances_failed))
  330. #
  331. # Parse the conf file
  332. #
  333. libxml2.substituteEntitiesDefault(1);
  334. testsuite = libxml2.parseFile(CONF)
  335. #
  336. # Error and warnng callbacks
  337. #
  338. def callback(ctx, str):
  339. global log
  340. log.write("%s%s" % (ctx, str))
  341. libxml2.registerErrorHandler(callback, "")
  342. libxml2.setEntityLoader(resolver)
  343. root = testsuite.getRootElement()
  344. if root.name != 'testSuite':
  345. print("%s doesn't start with a testSuite element, aborting" % (CONF))
  346. sys.exit(1)
  347. if quiet == 0:
  348. print("Running Relax NG testsuite")
  349. handle_testSuite(root)
  350. if quiet == 0 or nb_schemas_failed != 0:
  351. print("\nTOTAL:\nfound %d test schemas: %d success %d failures" % (
  352. nb_schemas_tests, nb_schemas_success, nb_schemas_failed))
  353. if quiet == 0 or nb_instances_failed != 0:
  354. print("found %d test instances: %d success %d failures" % (
  355. nb_instances_tests, nb_instances_success, nb_instances_failed))
  356. testsuite.freeDoc()
  357. # Memory debug specific
  358. libxml2.relaxNGCleanupTypes()
  359. libxml2.cleanupParser()
  360. if libxml2.debugMemory(1) == 0:
  361. if quiet == 0:
  362. print("OK")
  363. else:
  364. print("Memory leak %d bytes" % (libxml2.debugMemory(1)))
  365. libxml2.dumpMemory()