tap-driver.expect 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266
  1. #!/usr/bin/env expect
  2. # Push a file to the device, run it, and watch the tests run
  3. #
  4. # A typical invocation looks like:
  5. # TCLLIBPATH=./expectnmcu ./tap-driver.expect -serial /dev/ttyUSB3 ./mispec.lua ./mispec_file.lua
  6. #
  7. # For debugging the driver itself, it may be useful to invoke expect with -d,
  8. # which will give a great deal of diagnostic information about the expect state
  9. # machine's internals:
  10. #
  11. # TCLLIBPATH=./expectnmcu expect -d ./tap-driver.expect ...
  12. #
  13. # The -debug option will turn on some additional reporting from this driver program, as well.
  14. package require expectnmcu::core
  15. package require expectnmcu::xfer
  16. package require cmdline
  17. set cmd_parameters {
  18. { serial.arg "/dev/ttyUSB0" "Set the serial interface name" }
  19. { tpfx.arg "TAP: " "Set the expected TAP test prefix" }
  20. { lfs.arg "" "Flash a file to LFS" }
  21. { noxfer "Do not send files, just run script" }
  22. { runfunc "Last argument is function, not file" }
  23. { notests "Don't run tests, just xfer files" }
  24. { nontestshim "Don't shim NTest when testing" }
  25. { debug "Enable debugging reporting" }
  26. }
  27. set cmd_usage "- A NodeMCU Lua-based-test runner"
  28. if {[catch {array set cmdopts [cmdline::getoptions ::argv $cmd_parameters $cmd_usage]}]} {
  29. send_user [cmdline::usage $cmd_parameters $cmd_usage]
  30. send_user "\n Additional arguments should be files be transferred\n"
  31. send_user " The last file transferred will be run with `dofile`\n"
  32. exit 0
  33. }
  34. if { ${cmdopts(noxfer)} } {
  35. if { [ llength ${::argv} ] > 1 } {
  36. send_user "No point in more than one argument if noxfer given\n"
  37. exit 1
  38. }
  39. } {
  40. set xfers ${::argv}
  41. if { ${cmdopts(runfunc)} } {
  42. # Last argument is command, not file to xfer
  43. set xfers [lreplace xfers end end]
  44. }
  45. foreach arg ${xfers} {
  46. if { ! [file exists ${arg}] } {
  47. send_user "File ${arg} does not exist\n"
  48. exit 1
  49. }
  50. }
  51. }
  52. if { ${cmdopts(lfs)} ne "" } {
  53. if { ! [file exists ${cmdopts(lfs)}] } {
  54. send_user "LFS file does not exist\n"
  55. exit 1
  56. }
  57. }
  58. proc sus { what } { send_user "\n===> ${what} <===\n" }
  59. proc sui { what } { send_user "\n---> ${what} <---\n" }
  60. proc sud { what } {
  61. upvar 1 cmdopts cmdopts
  62. if { ${cmdopts(debug)} } { send_user "\n~~~> ${what} <~~~\n" }
  63. }
  64. set victim [::expectnmcu::core::connect ${cmdopts(serial)}]
  65. sus "Machine has booted"
  66. if { ${cmdopts(lfs)} ne "" } {
  67. ::expectnmcu::xfer::sendfile ${victim} ${cmdopts(lfs)} "tap-driver.lfs"
  68. send -i ${victim} "=node.LFS.reload(\"tap-driver.lfs\")\n"
  69. ::expectnmcu::core::waitboot ${victim}
  70. }
  71. if { ! ${cmdopts(noxfer)} } {
  72. foreach arg ${xfers} {
  73. ::expectnmcu::xfer::sendfile ${victim} ${arg} [file tail ${arg}]
  74. }
  75. }
  76. set tfn [file tail [lindex ${::argv} end ] ]
  77. if { ${cmdopts(notests)} || ${tfn} eq "" } {
  78. sus "No tests requested, and so operations are completed"
  79. exit 0
  80. }
  81. sus "Files transferred; running ${tfn}"
  82. if { ! ${cmdopts(nontestshim)} } {
  83. ::expectnmcu::core::send_exp_prompt_c ${victim} "function ntshim(...)"
  84. ::expectnmcu::core::send_exp_prompt_c ${victim} " local test = (require \"NTest\")(...)"
  85. ::expectnmcu::core::send_exp_prompt_c ${victim} " test.outputhandler = require\"NTestTapOut\""
  86. ::expectnmcu::core::send_exp_prompt_c ${victim} " return test"
  87. ::expectnmcu::core::send_exp_prompt ${victim} "end"
  88. } else {
  89. sui "Not shimming NTest output; test must report its own TAP messages"
  90. }
  91. # ntshim may be nil at this point if -nontestshim was given; that's fine
  92. if { ${cmdopts(runfunc)} } {
  93. send -i ${victim} "[ lindex ${::argv} end ](ntshim)\n"
  94. expect -i ${victim} -re "\\(ntshim\\)\[\r\n\]+" { }
  95. } else {
  96. send -i ${victim} "assert(loadfile(\"${tfn}\"))(ntshim)\n"
  97. expect -i ${victim} -re "assert\\(loadfile\\(\"${tfn}\"\\)\\)\\(ntshim\\)\[\r\n\]+" { }
  98. }
  99. set tpfx ${cmdopts(tpfx)}
  100. set toeol "\[^\n\]*(?=\n)"
  101. # Wait for the test to start and tell us how many
  102. # success lines we should expect
  103. set ntests 0
  104. set timeout 10
  105. expect {
  106. -i ${victim} -re "${tpfx}1\\.\\.(\\d+)(?=\r?\n)" {
  107. global ntests
  108. set ntests $expect_out(1,string)
  109. }
  110. -i ${victim} -re "${tpfx}Bail out!${toeol}" {
  111. sus "Bail out before start"
  112. exit 2
  113. }
  114. -i ${victim} -re ${::expectnmcu::core::panicre} {
  115. sus "Panic!"
  116. exit 2
  117. }
  118. # A prefixed line other than a plan (1..N) or bailout means we've not got
  119. # a plan. Leave ${ntests} at 0 and proceed to run the protocol.
  120. -i ${victim} -notransfer -re "${tpfx}${toeol}" { }
  121. # -i ${victim} -ex "\n> " {
  122. # sus "Prompt before start!"
  123. # exit 2
  124. # }
  125. # Consume other outputs and discard as if they were comments
  126. # This must come as the last pattern that looks at input
  127. -i ${victim} -re "(?p).${toeol}" { exp_continue }
  128. timeout {
  129. send_user "Failure: time out getting started\n"
  130. exit 2
  131. }
  132. }
  133. if { ${ntests} == 0 } {
  134. sus "System did not report plan; will look for summary at end"
  135. } else {
  136. sus "Expecting ${ntests} test results"
  137. }
  138. set timeout 60
  139. set exitwith 0
  140. set failures 0
  141. for {set this 1} {${ntests} == 0 || ${this} <= ${ntests}} {incr this} {
  142. expect {
  143. -i ${victim} -re "${tpfx}#${toeol}" {
  144. sud "Harness got comment: ${expect_out(buffer)}"
  145. exp_continue
  146. }
  147. -i ${victim} -re "${tpfx}ok (\\d+)\\D${toeol}" {
  148. sud "Harness acknowledge OK! ${this} ${expect_out(1,string)}"
  149. set tid ${expect_out(1,string)}
  150. if { ${tid} != "" && ${tid} != ${this} } {
  151. sui "WARNING: Test reporting misaligned at ${this} (got ${tid})"
  152. }
  153. }
  154. -i ${victim} -re "${tpfx}ok #${toeol}" {
  155. sud "Harness acknowledge anonymous ok! ${this}"
  156. }
  157. -i ${victim} -re "${tpfx}not ok (\\d+)\\D${toeol}" {
  158. sud "Failure in simulation after ${this} ${expect_out(1,string)}"
  159. set tid ${expect_out(1,string)}
  160. if { ${tid} != "" && ${tid} != ${this} } {
  161. sui "WARNING: Test reporting misaligned at ${this}"
  162. }
  163. set exitwith [expr max(${exitwith},1)]
  164. incr failures
  165. }
  166. -i ${victim} -re "${tpfx}not ok #${toeol}" {
  167. sud "Failure (anonymous) in simulation after ${this}"
  168. set exitwith [expr max(${exitwith},1)]
  169. incr failures
  170. }
  171. -i ${victim} -re "${tpfx}Bail out!${toeol}" {
  172. sus "Bail out after ${this} tests"
  173. exit 2
  174. }
  175. -i ${victim} -re "${tpfx}POST 1\\.\\.(\\d+)(?=\r?\n)" {
  176. # A post-factual plan; this must be the end of testing
  177. global ntests
  178. set ntests ${expect_out(1,string)}
  179. if { ${ntests} != ${this} } {
  180. sus "Postfix plan claimed ${ntests} but we saw ${this}"
  181. set exitwith [expr max(${exitwith},2)]
  182. incr failures
  183. }
  184. # break out of for loop
  185. set this ${ntests}
  186. }
  187. -i ${victim} -re "${tpfx}${toeol}" {
  188. sus "TAP line not understood!"
  189. exit 2
  190. }
  191. # -i ${victim} -ex ${::expectnmcu::core::promptstr} {
  192. # sus "Prompt while running tests!"
  193. # exit 2
  194. # }
  195. -i ${victim} -re ${::expectnmcu::core::panicre} {
  196. sus "Panic!"
  197. exit 2
  198. }
  199. # Consume other outputs and discard as if they were comments
  200. # This must come as the last pattern that looks at input
  201. -re "(?p).${toeol}" { exp_continue }
  202. timeout {
  203. send_user "Failure: time out\n"
  204. exit 2
  205. }
  206. }
  207. }
  208. # We think we're done running tests; send a final command for synchronization
  209. send -i ${victim} "print(\"f\",\"i\",\"n\")\n"
  210. expect -i ${victim} -re "print\\(\"f\",\"i\",\"n\"\\)\[\r\n\]+" { }
  211. expect {
  212. -i ${victim} -ex "f\ti\tn" { }
  213. -i ${victim} -re "${tpfx}#${toeol}" {
  214. sud "Harness got comment: ${expect_out(buffer)}"
  215. exp_continue
  216. }
  217. -i ${victim} -re "${tpfx}Bail out!${toeol}" {
  218. sus "Bail out after all tests finished"
  219. exit 2
  220. }
  221. -i ${victim} -re "${tpfx}${toeol}" {
  222. sus "Unexpected TAP output after tests finished"
  223. exit 2
  224. }
  225. -i ${victim} -re ${::expectnmcu::core::panicre} {
  226. sus "Panic!"
  227. exit 2
  228. }
  229. -re "(?p).${toeol}" { exp_continue }
  230. timeout {
  231. send_user "Failure: time out\n"
  232. exit 2
  233. }
  234. }
  235. if { ${exitwith} == 0 } {
  236. sus "All tests reported in OK"
  237. } else {
  238. sus "${failures} TEST FAILURES; REVIEW LOGS"
  239. }
  240. exit ${exitwith}