Browse Source

More NTest prep work for eventual test harness (#3353)

* Rename to tests/README.md

* Expand tests/README.md a bit

* NTest: remove report() in favor of named fields

Use a metatable to provide defaults which can be shadowed by the calling
code.

* NTest: remove old interface flag

I think we have few enough tests that we can verify not needing this
alert for ourselves.

* NTest tests: new standard prelude

Allow for NTest constructor to be passed in to the test itself.
The test harness can use this to provide a wrapper that will
pre-configure NTest itself.

* NTest output handler for TAP messages

* expect tests: core library functions

* expect tests: file xfer TCL module

* expect tests: add TAP-based test runner

* Begin documenting TCL goo

* Add .gitattributes to make sure lineends are correct ...

... if checked out under windows and executed under linux (say docker)

* tests/README: enumerate dependencies

* tests: more README.md

Co-authored-by: Gregor Hartmann <HHHartmann@users.noreply.github.com>
Nathaniel Wesley Filardo 3 years ago
parent
commit
6316b33296

+ 13 - 0
tests/.gitattributes

@@ -0,0 +1,13 @@
+# Enforce Unix newlines
+*.css   text eol=lf
+*.html  text eol=lf
+*.js    text eol=lf
+*.json  text eol=lf
+*.less  text eol=lf
+*.md    text eol=lf
+*.svg   text eol=lf
+*.yml   text eol=lf
+*.py    text eol=lf
+*.sh    text eol=lf
+*.tcl   text eol=lf
+*.expect text eol=lf

+ 21 - 25
tests/NTest/NTest.lua

@@ -171,22 +171,26 @@ local function fail(handler, name, func, expected, msg)
   handler('pass', name, msg)
 end
 
-local function NTest(testrunname, failoldinterface)
+local nmt = {
+  env = _G,
+  outputhandler = TERMINAL_HANDLER
+}
+nmt.__index = nmt
 
-  if failoldinterface then error("The interface has changed. Please see documentstion.") end
+return function(testrunname)
 
   local pendingtests = {}
-  local env = _G
-  local outputhandler = TERMINAL_HANDLER
   local started
 
+  local N = setmetatable({}, nmt)
+
   local function runpending()
     if pendingtests[1] ~= nil then
       node.task.post(node.task.LOW_PRIORITY, function()
         pendingtests[1](runpending)
       end)
     else
-      outputhandler('finish', testrunname)
+      N.outputhandler('finish', testrunname)
     end
   end
 
@@ -202,9 +206,9 @@ local function NTest(testrunname, failoldinterface)
     local testfn = function(next)
 
       local prev = {}
-      copyenv(prev, env)
+      copyenv(prev, N.env)
 
-      local handler = outputhandler
+      local handler = N.outputhandler
 
       local restore = function(err)
         if err then
@@ -214,8 +218,8 @@ local function NTest(testrunname, failoldinterface)
           end
         end
         if node then node.setonerror() end
-        copyenv(env, prev)
-        outputhandler('end', name)
+        copyenv(N.env, prev)
+        handler('end', name)
         table.remove(pendingtests, 1)
         collectgarbage()
         if next then next() end
@@ -233,6 +237,7 @@ local function NTest(testrunname, failoldinterface)
         restore()
       end
 
+      local env = N.env
       env.eq = deepeq
       env.spy = spy
       env.ok = function (cond, msg) wrap(assertok, false, cond, msg) end
@@ -258,7 +263,7 @@ local function NTest(testrunname, failoldinterface)
     end
 
     if not started then
-      outputhandler('start', testrunname)
+      N.outputhandler('start', testrunname)
       started = true
     end
 
@@ -270,25 +275,20 @@ local function NTest(testrunname, failoldinterface)
     end
   end
 
-  local function test(name, f)
+  function N.test(name, f)
     testimpl(name, f)
   end
 
-  local function testasync(name, f)
+  function N.testasync(name, f)
     testimpl(name, f, true)
   end
 
-  local function report(f, envP)
-    outputhandler = f or outputhandler
-    env = envP or env
-  end
-
   local currentCoName
 
-  local function testco(name, func)
+  function N.testco(name, func)
   --  local t = tmr.create();
     local co
-    testasync(name, function(Next)
+    N.testasync(name, function(Next)
       currentCoName = name
 
       local function getCB(cbName)
@@ -299,7 +299,7 @@ local function NTest(testrunname, failoldinterface)
                       currentCoName = nil
                       Next(err)
                     else
-                      outputhandler('fail', name, "Found stray Callback '"..cbName.."' from test '"..name.."'")
+                      N.outputhandler('fail', name, "Found stray Callback '"..cbName.."' from test '"..name.."'")
                     end
                   elseif coroutine.status(co) == "dead" then
                     currentCoName = nil
@@ -327,9 +327,5 @@ local function NTest(testrunname, failoldinterface)
     end)
   end
 
-
-  return {test = test, testasync = testasync, testco = testco, report = report}
+  return N
 end
-
-return NTest
-

+ 5 - 6
tests/NTest/NTest.md

@@ -145,7 +145,7 @@ ok(f.errors[3] ~= nil)
 
 ## Reports
 
-Another useful feature is that you can customize test reports as you need. The default `reports` just more or less prints out a basic report. You can easily override this behavior as well as add any other information you need (number of passed/failed assertions, time the test took etc):
+Another useful feature is that you can customize test reports as you need. The default `outputhandler` just more or less prints out a basic report. You can easily override (or augment by wrapping, e.g.) this behavior as well as add any other information you need (number of passed/failed assertions, time the test took etc):
 
 Events are:
 `start`   when testing starts
@@ -161,7 +161,7 @@ Events are:
 local passed = 0
 local failed = 0
 
-tests.report(function(event, testfunc, msg)
+tests.outputhandler = function(event, testfunc, msg)
   if event == 'begin' then
     print('Started test', testfunc)
     passed = 0
@@ -176,7 +176,7 @@ tests.report(function(event, testfunc, msg)
   elseif event == 'except' then
     print('ERROR', testfunc, msg)
   end
-end)
+end
 ```
 
 Additionally, you can pass a different environment to keep `_G` unpolluted:
@@ -184,8 +184,7 @@ You need to set it, so the helper functions mentioned above can be added before
 
 ``` Lua
 local myenv = {}
-
-tests.report(function() ... end, myenv)
+tests.env = myenv
 
 tests.test('Some test', function()
   myenv.ok(myenv.eq(...))
@@ -193,7 +192,7 @@ tests.test('Some test', function()
 end)
 ```
 
-You can set any of the parameters to `nil` to leave the value unchanged. 
+You can restore `env` or `outputhandler` to their defaults by setting their values to `nil`.
 
 
 ## Appendix

+ 2 - 2
tests/NTest/NTest_NTest.lua

@@ -434,7 +434,7 @@ end
 
 local pass
 -- Set meta test handler
-N.report(function(e, test, msg, errormsg)
+N.outputhandler = function(e, test, msg, errormsg)
   local function consumemsg(msg, area) -- luacheck: ignore
     if not expected[1][area][1] then
       print("--- FAIL "..expected[1].name..' ('..area..'ed): unexpected "'..
@@ -487,7 +487,7 @@ N.report(function(e, test, msg, errormsg)
   else
     print("Extra output: ", e, test, msg, errormsg)
   end
-end)
+end
 
 local async_queue = {}
 async = function(f) table.insert(async_queue, cbWrap(f)) end

+ 2 - 1
tests/NTest_adc_env.lua

@@ -1,7 +1,8 @@
 -- Walk the ADC through a stepped triangle wave using the attached voltage
 -- divider and I2C GPIO expander.
 
-local N = require('NTest')("adc-env")
+local N = ...
+N = (N or require "NTest")("adc-env")
 
 -- TODO: Preflight test that we are in the correct environment with an I2C
 -- expander in the right place with the right connections.

+ 2 - 1
tests/NTest_file.lua

@@ -1,4 +1,5 @@
-local N = require('NTest')("file")
+local N = ...
+N = (N or require "NTest")("file")
 
 local function cleanup()
     file.remove("testfile")

+ 2 - 1
tests/NTest_gpio_env.lua

@@ -3,7 +3,8 @@
 --   Node GPIO 13 (index 7) is connected to I2C expander channel B6; node OUT
 --   Node GPIO 15 (index 8) is connected to I2C expander channel B7; node IN
 
-local N = require('NTest')("gpio-env")
+local N = ...
+N = (N or require "NTest")("gpio-env")
 
 -- TODO: Preflight test that we are in the correct environment with an I2C
 -- expander in the right place with the right connections.

+ 2 - 1
tests/NTest_tmr.lua

@@ -1,4 +1,5 @@
-local N = require('NTest')("tmr")
+local N = ...
+N = (N or require "NTest")("tmr")
 
 N.testasync('SINGLE alarm', function(next)
   local t = tmr.create();

+ 0 - 106
tests/NodeMCU_Test_Environment.md

@@ -1,106 +0,0 @@
-NodeMCU Testing Environment
-===========================
-
-Herein we define the environment our testing framework expects to see
-when it runs. It is composed of two ESP8266 devices, each capable of
-holding an entire NodeMCU firmware, LFS image, and SPIFFS file system,
-as well as additional peripheral hardware. It is designed to fit
-comfortably on a breadboard and so should be easily replicated and
-integrated into any firmware validation testing.
-
-The test harness runs from a dedicated host computer, which is expected
-to have reset- and programming-capable UART links to both ESP8266
-devices, as found on almost all ESP8266 boards with USB to UART
-adapters, but the host does not necessarily need to use USB to connect,
-so long as TXD, RXD, DTR, and RTS are wired across.
-
-Peripherals
------------
-
-### I2C Bus
-
-There is an I2C bus hanging off DUT 0. Attached hardware is used both as
-tests of modules directly and also to facilitate testing other modules
-(e.g., gpio).
-
-#### MCP23017: I/O Expander
-
-At address 0x20. An 16-bit tristate GPIO expander, this chip is used to
-test I2C, GPIO, and ADC functionality. This chip's interconnections are
-as follows:
-
-MPC23017 | Purpose
----------|--------------------------------------------------------------
-/RESET   |DUT0 reset. This resets the chip whenever the host computer resets DUT 0 over its serial link (using DTR/RTS).
-B 0      |4K7 resistor to DUT 0 ADC.
-B 1      |2K2 resistor to DUT 0 ADC.
-B 5      |DUT1 GPIO16/WAKE via 4K7 resitor
-B 6      |DUT0 GPIO13 via 4K7 resistor and DUT1 GPIO15 via 4K7 resistor
-B 7      |DUT0 GPIO15 via 4K7 resistor and DUT1 GPIO13 via 4K7 resistor
-
-Notes:
-
--   DUT 0's ADC pin is connected via a 2K2 reistor to this chip's port
-    B, pin 1 and via a 4K7 resistor to port B, pin 0. This gives us the
-    ability to produce approximately 0 (both pins low), 1.1 (pin 0 high,
-    pin 1 low), 2.2 (pin 1 high, pin 0 low), and 3.3V (both pins high)
-    on the ADC pin.
--   Port B pins 6 and 7 sit on the UART cross-wiring between DUT 0 and
-    DUT 1. The 23017 will be tristated for inter-DUT UART tests, but
-    these
--   Port B pins 2, 3, and 4, as well as all of port A, remain available
-    for expansion.
--   The interrupt pins are not yet routed, but could be. We reserve DUT
-    0 GPIO 2 for this purpose with the understanding that the 23017's
-    interrupt functionality will be disabled (INTA, INTB set to
-    open-drain, GPINTEN set to 0) when not explicitly under test.
-
-ESP8266 Device 0 Connections
-----------------------------
-
-ESP       | Usage
-----------|----------------------------------------------------------
-GPIO 0    |Used to enter programming mode; otherwise unused in test environment.
-GPIO 1    |Primary UART transmit; reserved for host communication
-GPIO 2    |[reserved for 1-Wire] [+ reserved for 23017 INT[AB] connections]
-GPIO 3    |Primary UART recieve; reserved for host communication
-GPIO 4    |I2C SDA
-GPIO 5    |I2C SCL
-GPIO 6    |[Reserved for on-chip flash]
-GPIO 7    |[Reserved for on-chip flash]
-GPIO 8    |[Reserved for on-chip flash]
-GPIO 9    |[Reserved for on-chip flash]
-GPIO 10   |[Reserved for on-chip flash]
-GPIO 11   |[Reserved for on-chip flash]
-GPIO 12   |
-GPIO 13   |Secondary UART RX; DUT 1 GPIO 15, I/O expander B 6
-GPIO 14   |
-GPIO 15   |Secondary UART TX; DUT 1 GPIO 13, I/O expander B 7
-GPIO 16   |
-ADC 0     |Resistor divider with I/O expander
-
-ESP8266 Device 1 Connections
-----------------------------
-
-ESP       | Usage
-----------|----------------------------------------------------------
-GPIO 0    |Used to enter programming mode; otherwise unused in test environment.
-GPIO 1    |Primary UART transmit; reserved for host communication
-GPIO 2    |[Reserved for WS2812]
-GPIO 3    |Primary UART recieve; reserved for host communication
-GPIO 4    |
-GPIO 5    |
-GPIO 6    |[Reserved for on-chip flash]
-GPIO 7    |[Reserved for on-chip flash]
-GPIO 8    |[Reserved for on-chip flash]
-GPIO 9    |[Reserved for on-chip flash]
-GPIO 10   |[Reserved for on-chip flash]
-GPIO 11   |[Reserved for on-chip flash]
-GPIO 12   |HSPI MISO
-GPIO 13   |Secondary UART RX; DUT 0 GPIO 15, I/O exp B 7 via 4K7 Also used as HSPI MOSI for SPI tests
-GPIO 14   |HSPI CLK
-GPIO 15   |Secondary UART TX; DUT 0 GPIO 13, I/O exp B 6 via 4K7 Also used as HSPI /CS for SPI tests
-GPIO 16   |I/O expander B 5 via 4K7 resistor, for deep-sleep tests
-ADC 0     |
-
-

+ 215 - 0
tests/README.md

@@ -0,0 +1,215 @@
+# Introduction
+
+Welcome to the NodeMCU self-test suite.  Here you will find our growing effort
+to ensure that our software behaves as we think it should and that we do not
+regress against earlier versions.
+
+Our tests are written using [NTest](./NTest/NTest.md), a lightweight yet
+featureful framework for specifying unit tests.
+
+# Building and Running Test Software on NodeMCU Devices
+
+Naturally, to test NodeMCU on its intended hardware, you will need one or more
+NodeMCU-capable boards.  At present, the test environment is specified using
+two ESP8266 Devices Under Test (DUTs), but we envision expanding this to mixed
+ESP8266/ESP32 environments as well.
+
+Test programs live beside this file.  While many test programs run on the
+NodeMCU DUTs, but there is reason to want to orchestrate DUTs and the
+environment using the host.  Files matching the glob `NTest_*.lua` are intended
+for on-DUT execution.
+
+## Manual Test Invocation
+
+At the moment, the testing regime and host-based orchestration is still in
+development, and so things are a little more manual than perhaps desired.  The
+`NTest`-based test programs all assume that they can `require "NTest"`, and so
+the easiest route to success is to
+
+* build an LFS image containing
+
+  * [package.loader support for LFS](../lua_examples/lfs/_init.lua)
+
+  * [NTest itself](./NTest/NTest.lua)
+
+  * Any additional Lua support modules required (e.g., [mcp23017
+  support](../lua_modules/mcp23017/mcp23017.lua) )
+
+* build a firmware with the appropriate C modules
+
+* program the board with your firmware and LFS images
+
+* ensure that `package.loader` is patched appropriately on startup
+
+* transfer the `NTest_foo` program you wish to run to the device SPIFFS
+  (or have included it in the LFS).
+
+* at the interpreter prompt, say `dofile("NTest_foo.lua")` (or
+  `node.LFS.get("NTest_foo")()`) to run the `foo` test program.
+
+## Experimental Host Orchestration
+
+Enthusiastic testers are encouraged to try using our very new, very
+experimental host test runner, [tap-driver.expect](./tap-driver.expect).  To
+use this program, in addition to the above, the LFS environment should contain
+[NTestTapOut](./tests/utils/NTestTapOut.lua), an output adapter for `NTest`,
+making it speak a slight variant of the [Test Anything
+Protocol](https://testanything.org/).  This structured output is scanned for
+by the script on the host.
+
+You'll need `expect` and TCL and some TCL libraries available; on Debian, that
+amounts to
+
+    apt install tcl tcllib tclx8.4 expect
+
+This program should be invoked from beside this file with something like
+
+    TCLLIBPATH=./expectnmcu ./tap-driver.expect -serial /dev/ttyUSB3 -lfs ./lfs.img NTest_file.lua
+
+This will...
+
+* transfer and install the specified LFS module (and reboot the device to load LFS)
+
+* transfer the test program
+
+* run the test program with `NTest` shimmed to use the `NTestTapOut` output
+  handler
+
+* summarize the results
+
+* return 0 if and only if all tests have passed
+
+This tool is quite flexible and takes a number of other options and flags
+controlling aspects of its behavior:
+
+* Additional files, Lua or otherwise, may be transferred by specifing them
+  before the test to run (e.g., `./tap-driver.expect a.lua b.lua
+  NTest_foo.lua`); dually, a `-noxfer` flag will suppress transferring even the
+  last file.  All transferred files are moved byte-for-byte to the DUT's
+  SPIFFS with names, but not directory components, preserved.
+
+* The `-lfs LFS.img` option need not be specified and, if not given, any
+  existing `LFS` image will remain on the device for use by the test.
+
+* A `-nontestshim` flag will skip attempting to shim the given test program
+  with `NTestTapOut`; the test program is expected to provide its own TAP
+  output.  The `-tpfx` argument can be used to override the leading `TAP: `
+  sigil used by the `NTestTapOut` output handler.
+
+* A `-runfunc` option indicates that the last argument is not a file to
+  transfer but rather a function to be run.  It will be invoked at the REPL
+  with a single argument, the shimmed `NTest` constructor, unless `-nontestshim`
+  is given, in which case the argument will be `nil`.
+
+* A `-notests` option suppresses running tests (making the tool merely another
+  option for loading files to the device).
+
+Transfers will be significantly faster if
+[pipeutils](../lua_examples/pipeutils.lua) is available to `require` on the
+DUT, but a fallback strategy exists if not.  We suggest either including
+`pipeutils` in LFS images, in SPIFFS, or as the first file to be transferred.
+
+# NodeMCU Testing Environment
+
+Herein we define the environment our testing framework expects to see
+when it runs. It is composed of two ESP8266 devices, each capable of
+holding an entire NodeMCU firmware, LFS image, and SPIFFS file system,
+as well as additional peripheral hardware. It is designed to fit
+comfortably on a breadboard and so should be easily replicated and
+integrated into any firmware validation testing.
+
+The test harness runs from a dedicated host computer, which is expected
+to have reset- and programming-capable UART links to both ESP8266
+devices, as found on almost all ESP8266 boards with USB to UART
+adapters, but the host does not necessarily need to use USB to connect,
+so long as TXD, RXD, DTR, and RTS are wired across.
+
+## Peripherals
+
+### I2C Bus
+
+There is an I2C bus hanging off DUT 0. Attached hardware is used both as
+tests of modules directly and also to facilitate testing other modules
+(e.g., gpio).
+
+#### MCP23017: I/O Expander
+
+At address 0x20. An 16-bit tristate GPIO expander, this chip is used to
+test I2C, GPIO, and ADC functionality. This chip's interconnections are
+as follows:
+
+MPC23017 | Purpose
+---------|--------------------------------------------------------------
+/RESET   |DUT0 reset. This resets the chip whenever the host computer resets DUT 0 over its serial link (using DTR/RTS).
+B 0      |4K7 resistor to DUT 0 ADC.
+B 1      |2K2 resistor to DUT 0 ADC.
+B 5      |DUT1 GPIO16/WAKE via 4K7 resitor
+B 6      |DUT0 GPIO13 via 4K7 resistor and DUT1 GPIO15 via 4K7 resistor
+B 7      |DUT0 GPIO15 via 4K7 resistor and DUT1 GPIO13 via 4K7 resistor
+
+Notes:
+
+-   DUT 0's ADC pin is connected via a 2K2 reistor to this chip's port
+    B, pin 1 and via a 4K7 resistor to port B, pin 0. This gives us the
+    ability to produce approximately 0 (both pins low), 1.1 (pin 0 high,
+    pin 1 low), 2.2 (pin 1 high, pin 0 low), and 3.3V (both pins high)
+    on the ADC pin.
+-   Port B pins 6 and 7 sit on the UART cross-wiring between DUT 0 and
+    DUT 1. The 23017 will be tristated for inter-DUT UART tests, but
+    these
+-   Port B pins 2, 3, and 4, as well as all of port A, remain available
+    for expansion.
+-   The interrupt pins are not yet routed, but could be. We reserve DUT
+    0 GPIO 2 for this purpose with the understanding that the 23017's
+    interrupt functionality will be disabled (INTA, INTB set to
+    open-drain, GPINTEN set to 0) when not explicitly under test.
+
+ESP8266 Device 0 Connections
+----------------------------
+
+ESP       | Usage
+----------|----------------------------------------------------------
+GPIO 0    |Used to enter programming mode; otherwise unused in test environment.
+GPIO 1    |Primary UART transmit; reserved for host communication
+GPIO 2    |[reserved for 1-Wire] [+ reserved for 23017 INT[AB] connections]
+GPIO 3    |Primary UART recieve; reserved for host communication
+GPIO 4    |I2C SDA
+GPIO 5    |I2C SCL
+GPIO 6    |[Reserved for on-chip flash]
+GPIO 7    |[Reserved for on-chip flash]
+GPIO 8    |[Reserved for on-chip flash]
+GPIO 9    |[Reserved for on-chip flash]
+GPIO 10   |[Reserved for on-chip flash]
+GPIO 11   |[Reserved for on-chip flash]
+GPIO 12   |
+GPIO 13   |Secondary UART RX; DUT 1 GPIO 15, I/O expander B 6
+GPIO 14   |
+GPIO 15   |Secondary UART TX; DUT 1 GPIO 13, I/O expander B 7
+GPIO 16   |
+ADC 0     |Resistor divider with I/O expander
+
+ESP8266 Device 1 Connections
+----------------------------
+
+ESP       | Usage
+----------|----------------------------------------------------------
+GPIO 0    |Used to enter programming mode; otherwise unused in test environment.
+GPIO 1    |Primary UART transmit; reserved for host communication
+GPIO 2    |[Reserved for WS2812]
+GPIO 3    |Primary UART recieve; reserved for host communication
+GPIO 4    |
+GPIO 5    |
+GPIO 6    |[Reserved for on-chip flash]
+GPIO 7    |[Reserved for on-chip flash]
+GPIO 8    |[Reserved for on-chip flash]
+GPIO 9    |[Reserved for on-chip flash]
+GPIO 10   |[Reserved for on-chip flash]
+GPIO 11   |[Reserved for on-chip flash]
+GPIO 12   |HSPI MISO
+GPIO 13   |Secondary UART RX; DUT 0 GPIO 15, I/O exp B 7 via 4K7 Also used as HSPI MOSI for SPI tests
+GPIO 14   |HSPI CLK
+GPIO 15   |Secondary UART TX; DUT 0 GPIO 13, I/O exp B 6 via 4K7 Also used as HSPI /CS for SPI tests
+GPIO 16   |I/O expander B 5 via 4K7 resistor, for deep-sleep tests
+ADC 0     |
+
+

+ 136 - 0
tests/expectnmcu/core.tcl

@@ -0,0 +1,136 @@
+namespace eval expectnmcu::core {
+  set panicre "powered by Lua \[0-9.\]+ on SDK \[0-9.\]+"
+  set promptstr "\n> "
+
+  namespace export reboot waitboot connect
+  namespace export send_exp_prompt send_exp_res_prompt send_exp_prompt_c
+}
+
+package require cmdline
+
+# Use DTR/RTS signaling to reboot the device
+## I'm not sure why we have to keep resetting the mode, but so it goes.
+proc ::expectnmcu::core::reboot { dev } {
+  set victimfd [open ${dev} ]
+  set mode [fconfigure ${victimfd} -mode ]
+  fconfigure ${victimfd} -mode ${mode} -ttycontrol {DTR 0 RTS 1}
+  sleep 0.1
+  fconfigure ${victimfd} -mode ${mode} -ttycontrol {DTR 0 RTS 0}
+  close ${victimfd}
+}
+
+proc ::expectnmcu::core::waitboot { victim } {
+  expect {
+    -i ${victim} "Formatting file system" {
+      set timeout 120
+      exp_continue
+    }
+    -i ${victim} "powered by Lua" { }
+    timeout { return -code error "Timeout" }
+  }
+  # Catch nwf's system bootup, in case we're testing an existing system,
+  # rather than a blank firmware.
+  expect {
+    -i ${victim} -re "Reset delay!.*${::expectnmcu::core::promptstr}" {
+      send -i ${victim} "stop(true)\n"
+      expect -i ${victim} -ex ${::expectnmcu::core::promptstr}
+    }
+    -i ${victim} -ex ${::expectnmcu::core::promptstr} { }
+    timeout { return -code error "Timeout" }
+  }
+
+  # Do a little more active synchronization with the DUT: send it a command
+  # and wait for the side-effect of that command to happen, thereby ensuring
+  # that the next prompt we see is after this point in the input.
+  send -i ${victim} "print(\"a\",\"z\")\n"
+  expect {
+    -i ${victim} -ex "a\tz" { }
+  }
+  expect {
+    -i ${victim} -ex ${::expectnmcu::core::promptstr} { }
+    timeout { return -code error "Timeout" }
+  }
+}
+
+# Establish a serial connection to the device via socat.  Takes
+# -baud=N, -reboot=0/1/dontwait, -waitboot=0/1 optional parameters
+proc ::expectnmcu::core::connect { dev args } {
+  set opts {
+    { baud.arg 115200 }
+    { reboot.arg 1 }
+  }
+  array set arg [::cmdline::getoptions args $opts]
+
+  spawn "socat" "STDIO" "${dev},b${arg(baud)},raw,crnl"
+  close -onexec 1 -i ${spawn_id}
+  set victim ${spawn_id}
+
+  # XXX?
+  set victimfd [open ${dev} ]
+  set mode [fconfigure ${victimfd} -mode ${arg(baud)},n,8,1 ]
+
+  if { ${arg(reboot)} != 0 } {
+    ::expectnmcu::core::reboot ${dev}
+    if { ${arg(reboot)} != "dontwait" } {
+      ::expectnmcu::core::waitboot ${victim}
+    }
+  }
+
+  close ${victimfd}
+
+  return ${victim}
+}
+
+# This one is somewhat "for experts only" -- it expects that you have either
+# consumed whatever command you flung at the node or that you have some reason
+# to not be concerned with its echo (and return)
+proc ::expectnmcu::core::exp_prompt { sid } {
+  expect {
+   -i ${sid} -ex ${::expectnmcu::core::promptstr} { }
+   -i ${sid} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" }
+   timeout { return -code error "Timeout" }
+  }
+}
+
+proc ::expectnmcu::core::send_exp_prompt { sid cmd } {
+  send -i ${sid} -- "${cmd}\n"
+  expect {
+   -i ${sid} -ex "${cmd}" { }
+   timeout { return -code error "Timeout" }
+  }
+  expect {
+   -i ${sid} -ex ${::expectnmcu::core::promptstr} { }
+   -i ${sid} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" }
+   timeout { return -code error "Timeout" }
+  }
+}
+
+proc ::expectnmcu::core::send_exp_res_prompt { sid cmd res } {
+  send -i ${sid} -- "${cmd}\n"
+  expect {
+   -i ${sid} -ex "${cmd}" { }
+   timeout { return -code error "Timeout" }
+  }
+  expect {
+   -i ${sid} -re "${res}.*${::expectnmcu::core::promptstr}" { }
+   -i ${sid} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" }
+   -i ${sid} -ex ${::expectnmcu::core::promptstr} { return -code error "Prompt before expected response" }
+   timeout { return -code error "Timeout" }
+  }
+}
+
+proc ::expectnmcu::core::send_exp_prompt_c { sid cmd } {
+  send -i ${sid} -- "${cmd}\n"
+  expect {
+   -i ${sid} -ex "${cmd}" { }
+   timeout { return -code error "Timeout" }
+  }
+  expect {
+   -i ${sid} -ex "\n>> " { }
+   -i ${sid} -ex ${::expectnmcu::core::promptstr} { return -code error "Non-continuation prompt" }
+   -i ${sid} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" }
+   timeout { return -code error "Timeout" }
+  }
+}
+
+package provide expectnmcu::core 1.0

+ 12 - 0
tests/expectnmcu/pkgIndex.tcl

@@ -0,0 +1,12 @@
+# Tcl package index file, version 1.1
+# This file is generated by the "pkg_mkIndex" command
+# and sourced either when an application starts up or
+# by a "package unknown" script.  It invokes the
+# "package ifneeded" command to set up package-related
+# information so that packages will be loaded automatically
+# in response to "package require" commands.  When this
+# script is sourced, the variable $dir must contain the
+# full path name of this file's directory.
+
+package ifneeded expectnmcu::core 1.0 [list source [file join $dir core.tcl]]
+package ifneeded expectnmcu::xfer 1.0 [list source [file join $dir xfer.tcl]]

+ 148 - 0
tests/expectnmcu/xfer.tcl

@@ -0,0 +1,148 @@
+namespace eval expectnmcu::xfer {
+}
+
+package require expectnmcu::core
+
+# Open remote file `which` on `dev` in `mode` as Lua object `dfh`
+proc ::expectnmcu::xfer::open { dev dfh which mode } {
+  ::expectnmcu::core::send_exp_prompt ${dev} "${dfh} = nil"
+  ::expectnmcu::core::send_exp_prompt ${dev} "${dfh} = file.open(\"${which}\",\"${mode}\")"
+  ::expectnmcu::core::send_exp_res_prompt ${dev} "=type(${dfh})" "userdata"
+}
+
+# Close Lua file object `dfh` on `dev`
+proc ::expectnmcu::xfer::close { dev dfh } {
+  ::expectnmcu::core::send_exp_prompt ${dev} "${dfh}:close()"
+}
+
+# Write to `dfh` on `dev` at `where` `what`, using base64 as transport
+#
+# This does not split lines; write only short amounts of data.
+proc ::expectnmcu::xfer::pwrite { dev dfh where what } {
+  send -i ${dev} -- [string cat \
+    "do local d,e = encoder.fromBase64(\"[binary encode base64 -maxlen 0 ${what}]\");" \
+    "${dfh}:seek(\"set\",${where});" \
+    "print(${dfh}:write(d));" \
+    "end\n" \
+  ]
+  expect {
+    -i ${dev} -re "true\[\r\n\]+> " { }
+    -i ${dev} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" }
+    -i ${dev} -ex "\n> " { return -code error "Bad result from pwrite" }
+    timeout { return -code error "Timeout while waiting for pwrite" }
+  }
+}
+
+# Read `howmuch` byetes from `dfh` on `dev` at `where`, using base64
+# as transport.  This buffers the whole data and its base64 encoding
+# in device RAM; read only short strings.
+proc ::expectnmcu::xfer::pread { dev dfh where howmuch } {
+  send -i ${dev} -- "${dfh}:seek(\"set\",${where}); print(encoder.toBase64(${dfh}:read(${howmuch})))\n"
+  expect {
+    -i ${dev} -re "\\)\\)\\)\[\r\n\]+(\[^\r\n\]+)\[\r\n\]+> " {
+      return [binary decode base64 ${expect_out(1,string)}]
+    }
+    -i ${dev} -ex "\n> " { return -code error "No reply to pread" }
+    -i ${dev} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" }
+    timeout { return -code error "Timeout while pread-ing" }
+  }
+}
+
+# Check for pipeutils on the target device
+proc ::expectnmcu::xfer::haspipeutils { dev } {
+  send -i ${dev} -- "local ok, pu = pcall(require, \"pipeutils\"); print(ok and type(pu) == \"table\" and pu.chunker and pu.debase64 and true or false)\n"
+  expect {
+    -i ${dev} -re "\[\r\n\]+false\[\r\n\]+> " { return 0 }
+    -i ${dev} -re "\[\r\n\]+true\[\r\n\]+> " { return 1 }
+    -i ${dev} -ex "\n> " { return -code error "No reply to pipeutils probe" }
+    -i ${dev} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" }
+    timeout { return -code error "Timeout while probing for pipeutils" }
+  }
+}
+
+# Send local file `lfn` to the remote filesystem on `dev` and name it `rfn`.
+# Use `dfo` as the Lua handle to the remote file for the duration of writing,
+# (and `nil` it out afterwards)
+proc ::expectnmcu::xfer::sendfile { dev lfn rfn {dfo "xfo"} } {
+  package require sha256
+
+  set has_pipeutils [::expectnmcu::xfer::haspipeutils ${dev} ]
+
+  set ltf [::open ${lfn} ]
+  fconfigure ${ltf} -translation binary
+  file stat ${lfn} lfstat
+  ::expectnmcu::xfer::open ${dev} ${dfo} "${rfn}.sf" "w+"
+
+  if { ${has_pipeutils} } {
+    # Send over a loader program
+    ::expectnmcu::core::send_exp_prompt_c ${dev} "do"
+    ::expectnmcu::core::send_exp_prompt_c ${dev} " local pu = require \"pipeutils\""
+    ::expectnmcu::core::send_exp_prompt_c ${dev} " local ch = pu.chunker(function(d) ${dfo}:write(d) end, 256)"
+    ::expectnmcu::core::send_exp_prompt_c ${dev} " local db = pu.debase64(ch.write, function(ed,ee)"
+    ::expectnmcu::core::send_exp_prompt_c ${dev} "  if ed:match(\"^%.\[\\r\\n\]*$\") then ch.flush() print(\"F I N\")"
+    ::expectnmcu::core::send_exp_prompt_c ${dev} "  else print(\"ABORT\", ee, ed) end"
+    ::expectnmcu::core::send_exp_prompt_c ${dev} "  uart.on(\"data\") end)"
+    # TODO: make echo use CRC not full string; probably best add to crypto module
+    ::expectnmcu::core::send_exp_prompt_c ${dev} " uart.on(\"data\", \"\\n\", function(x) db.write(x); uart.write(0, \"OK: \", x) end, 0)"
+    ::expectnmcu::core::send_exp_prompt ${dev}   "end"
+    set xln 90
+  } else {
+    set xln 48
+  }
+
+  set lho [sha2::SHA256Init]
+
+  set fpos 0
+  while { 1 } {
+    send_user ">> xfer ${fpos} of ${lfstat(size)}\n"
+    set data [read ${ltf} ${xln}]
+    sha2::SHA256Update ${lho} ${data}
+    if { ${has_pipeutils} } {
+      set estr [binary encode base64 -maxlen 0 ${data}]
+      send -i ${dev} -- "${estr}\n"
+      expect {
+        -i ${dev} -ex "OK: ${estr}" { expect -i ${dev} -re "\[\r\n\]+" {} }
+        -i ${dev} -ex "\n> " { return -code error "Prompt while sending data" }
+        -i ${dev} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" }
+        timeout { return -code error "Timeout while sending data" }
+      }
+    } else {
+      ::expectnmcu::xfer::pwrite ${dev} ${dfo} ${fpos} ${data}
+    }
+    set fpos [expr $fpos + ${xln}]
+    if { [string length ${data}] != ${xln} } { break }
+  }
+
+  if { ${has_pipeutils} } {
+    send -i ${dev} -- ".\n"
+    expect {
+      -i ${dev} -re "F I N\[\r\n\]+" { }
+      -i ${dev} -ex "\n> " { return -code error "Prompt while awaiting acknowledgement" }
+      -i ${dev} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" }
+      timeout { return -code error "Timeout while awaiting acknowledgement" }
+    }
+  }
+
+  ::close ${ltf}
+  ::expectnmcu::xfer::close ${dev} ${dfo}
+  ::expectnmcu::core::send_exp_prompt ${dev} "${dfo} = nil"
+
+  set exphash [sha2::Hex [sha2::SHA256Final ${lho}]]
+
+  send -i ${dev} "=encoder.toHex(crypto.fhash(\"sha256\",\"${rfn}.sf\"))\n"
+  expect {
+    -i ${dev} -re "\[\r\n\]+(\[a-f0-9\]+)\[\r\n\]+> " {
+      if { ${expect_out(1,string)} != ${exphash} } {
+        return -code error \
+          "Sendfile checksum mismatch: ${expect_out(1,string)} != ${exphash}"
+      }
+    }
+    -i ${dev} -re ${::expectnmcu::core::panicre} { return -code error "Panic!" }
+    timeout { return -code error "Timeout while verifying checksum" }
+  }
+
+  ::expectnmcu::core::send_exp_prompt ${dev} "file.remove(\"${rfn}\")"
+  ::expectnmcu::core::send_exp_res_prompt ${dev} "=file.rename(\"${rfn}.sf\", \"${rfn}\")" "true"
+}
+
+package provide expectnmcu::xfer 1.0

+ 266 - 0
tests/tap-driver.expect

@@ -0,0 +1,266 @@
+#!/usr/bin/env expect
+
+# Push a file to the device, run it, and watch the tests run
+#
+# A typical invocation looks like:
+#   TCLLIBPATH=./expectnmcu ./tap-driver.expect -serial /dev/ttyUSB3 ./mispec.lua ./mispec_file.lua
+#
+# For debugging the driver itself, it may be useful to invoke expect with -d,
+# which will give a great deal of diagnostic information about the expect state
+# machine's internals:
+#
+#   TCLLIBPATH=./expectnmcu expect -d ./tap-driver.expect ...
+#
+# The -debug option will turn on some additional reporting from this driver program, as well.
+
+
+package require expectnmcu::core
+package require expectnmcu::xfer
+
+package require cmdline
+set cmd_parameters {
+  { serial.arg "/dev/ttyUSB0"   "Set the serial interface name" }
+  { tpfx.arg   "TAP: "          "Set the expected TAP test prefix" }
+  { lfs.arg    ""               "Flash a file to LFS" }
+  { noxfer                      "Do not send files, just run script" }
+  { runfunc                     "Last argument is function, not file" }
+  { notests                     "Don't run tests, just xfer files" }
+  { nontestshim                 "Don't shim NTest when testing" }
+  { debug                       "Enable debugging reporting" }
+}
+set cmd_usage "- A NodeMCU Lua-based-test runner"
+if {[catch {array set cmdopts [cmdline::getoptions ::argv $cmd_parameters $cmd_usage]}]} {
+  send_user [cmdline::usage $cmd_parameters $cmd_usage]
+  send_user "\n  Additional arguments should be files be transferred\n"
+  send_user "  The last file transferred will be run with `dofile`\n"
+  exit 0
+}
+
+if { ${cmdopts(noxfer)} } {
+  if { [ llength ${::argv} ] > 1 } {
+    send_user "No point in more than one argument if noxfer given\n"
+    exit 1
+  }
+} {
+  set xfers ${::argv}
+
+  if { ${cmdopts(runfunc)} } {
+    # Last argument is command, not file to xfer
+    set xfers [lreplace xfers end end]
+  }
+
+  foreach arg ${xfers} {
+    if { ! [file exists ${arg}] } {
+      send_user "File ${arg} does not exist\n"
+      exit 1
+    }
+  }
+}
+
+if { ${cmdopts(lfs)} ne "" } {
+  if { ! [file exists ${cmdopts(lfs)}] } {
+    send_user "LFS file does not exist\n"
+    exit 1
+  }
+}
+
+proc sus { what } { send_user "\n===> ${what} <===\n" }
+proc sui { what } { send_user "\n---> ${what} <---\n" }
+proc sud { what } {
+  upvar 1 cmdopts cmdopts
+  if { ${cmdopts(debug)} } { send_user "\n~~~> ${what} <~~~\n" }
+}
+
+set victim [::expectnmcu::core::connect ${cmdopts(serial)}]
+sus "Machine has booted"
+
+if { ${cmdopts(lfs)} ne "" } {
+  ::expectnmcu::xfer::sendfile ${victim} ${cmdopts(lfs)} "tap-driver.lfs"
+  send -i ${victim} "=node.LFS.reload(\"tap-driver.lfs\")\n"
+  ::expectnmcu::core::waitboot ${victim}
+}
+
+if { ! ${cmdopts(noxfer)} } {
+  foreach arg ${xfers} {
+    ::expectnmcu::xfer::sendfile ${victim} ${arg} [file tail ${arg}]
+  }
+}
+
+set tfn [file tail [lindex ${::argv} end ] ]
+
+if { ${cmdopts(notests)} || ${tfn} eq "" } {
+  sus "No tests requested, and so operations are completed"
+  exit 0
+}
+
+sus "Files transferred; running ${tfn}"
+
+if { ! ${cmdopts(nontestshim)} } {
+  ::expectnmcu::core::send_exp_prompt_c ${victim} "function ntshim(...)"
+  ::expectnmcu::core::send_exp_prompt_c ${victim} "  local test = (require \"NTest\")(...)"
+  ::expectnmcu::core::send_exp_prompt_c ${victim} "  test.outputhandler = require\"NTestTapOut\""
+  ::expectnmcu::core::send_exp_prompt_c ${victim} "  return test"
+  ::expectnmcu::core::send_exp_prompt   ${victim} "end"
+} else {
+  sui "Not shimming NTest output; test must report its own TAP messages"
+}
+
+# ntshim may be nil at this point if -nontestshim was given; that's fine
+if { ${cmdopts(runfunc)} } {
+  send -i ${victim} "[ lindex ${::argv} end ](ntshim)\n"
+  expect -i ${victim} -re "\\(ntshim\\)\[\r\n\]+" { }
+} else {
+  send -i ${victim} "assert(loadfile(\"${tfn}\"))(ntshim)\n"
+  expect -i ${victim} -re "assert\\(loadfile\\(\"${tfn}\"\\)\\)\\(ntshim\\)\[\r\n\]+" { }
+}
+
+set tpfx ${cmdopts(tpfx)}
+set toeol "\[^\n\]*(?=\n)"
+
+# Wait for the test to start and tell us how many
+# success lines we should expect
+set ntests 0
+set timeout 10
+expect {
+  -i ${victim} -re "${tpfx}1\\.\\.(\\d+)(?=\r?\n)" {
+    global ntests
+    set ntests $expect_out(1,string)
+  }
+  -i ${victim} -re "${tpfx}Bail out!${toeol}" {
+    sus "Bail out before start"
+    exit 2
+  }
+  -i ${victim} -re ${::expectnmcu::core::panicre} {
+    sus "Panic!"
+    exit 2
+  }
+  # A prefixed line other than a plan (1..N) or bailout means we've not got
+  # a plan.  Leave ${ntests} at 0 and proceed to run the protocol.
+  -i ${victim} -notransfer -re "${tpfx}${toeol}" { }
+  # -i ${victim} -ex "\n> " {
+  #   sus "Prompt before start!"
+  #   exit 2
+  # }
+  # Consume other outputs and discard as if they were comments
+  # This must come as the last pattern that looks at input
+  -i ${victim} -re "(?p).${toeol}" { exp_continue }
+  timeout {
+    send_user "Failure: time out getting started\n"
+    exit 2
+  }
+}
+
+if { ${ntests} == 0 } {
+  sus "System did not report plan; will look for summary at end"
+} else {
+  sus "Expecting ${ntests} test results"
+}
+
+set timeout 60
+set exitwith 0
+set failures 0
+for {set this 1} {${ntests} == 0 || ${this} <= ${ntests}} {incr this} {
+  expect {
+    -i ${victim} -re "${tpfx}#${toeol}" {
+      sud "Harness got comment: ${expect_out(buffer)}"
+      exp_continue
+    }
+    -i ${victim} -re "${tpfx}ok (\\d+)\\D${toeol}" {
+      sud "Harness acknowledge OK! ${this} ${expect_out(1,string)}"
+      set tid ${expect_out(1,string)}
+      if { ${tid} != "" && ${tid} != ${this} } {
+        sui "WARNING: Test reporting misaligned at ${this} (got ${tid})"
+      }
+    }
+    -i ${victim} -re "${tpfx}ok #${toeol}" {
+      sud "Harness acknowledge anonymous ok! ${this}"
+    }
+    -i ${victim} -re "${tpfx}not ok (\\d+)\\D${toeol}" {
+      sud "Failure in simulation after ${this} ${expect_out(1,string)}"
+      set tid ${expect_out(1,string)}
+      if { ${tid} != "" && ${tid} != ${this} } {
+        sui "WARNING: Test reporting misaligned at ${this}"
+      }
+      set exitwith [expr max(${exitwith},1)]
+      incr failures
+    }
+    -i ${victim} -re "${tpfx}not ok #${toeol}" {
+      sud "Failure (anonymous) in simulation after ${this}"
+      set exitwith [expr max(${exitwith},1)]
+      incr failures
+    }
+    -i ${victim} -re "${tpfx}Bail out!${toeol}" {
+      sus "Bail out after ${this} tests"
+      exit 2
+    }
+    -i ${victim} -re "${tpfx}POST 1\\.\\.(\\d+)(?=\r?\n)" {
+      # A post-factual plan; this must be the end of testing
+      global ntests
+      set ntests ${expect_out(1,string)}
+      if { ${ntests} != ${this} } {
+        sus "Postfix plan claimed ${ntests} but we saw ${this}"
+        set exitwith [expr max(${exitwith},2)]
+        incr failures
+      }
+      # break out of for loop
+      set this ${ntests}
+    }
+    -i ${victim} -re "${tpfx}${toeol}" {
+        sus "TAP line not understood!"
+        exit 2
+    }
+    # -i ${victim} -ex ${::expectnmcu::core::promptstr} {
+    #   sus "Prompt while running tests!"
+    #   exit 2
+    # }
+    -i ${victim} -re ${::expectnmcu::core::panicre} {
+      sus "Panic!"
+      exit 2
+    }
+    # Consume other outputs and discard as if they were comments
+    # This must come as the last pattern that looks at input
+    -re "(?p).${toeol}" { exp_continue }
+    timeout {
+      send_user "Failure: time out\n"
+      exit 2
+    }
+  }
+}
+
+# We think we're done running tests; send a final command for synchronization
+send -i ${victim} "print(\"f\",\"i\",\"n\")\n"
+expect -i ${victim} -re "print\\(\"f\",\"i\",\"n\"\\)\[\r\n\]+" { }
+expect {
+  -i ${victim} -ex "f\ti\tn" { }
+
+  -i ${victim} -re "${tpfx}#${toeol}" {
+    sud "Harness got comment: ${expect_out(buffer)}"
+    exp_continue
+  }
+
+  -i ${victim} -re "${tpfx}Bail out!${toeol}" {
+    sus "Bail out after all tests finished"
+    exit 2
+  }
+  -i ${victim} -re "${tpfx}${toeol}" {
+    sus "Unexpected TAP output after tests finished"
+    exit 2
+  }
+  -i ${victim} -re ${::expectnmcu::core::panicre} {
+    sus "Panic!"
+    exit 2
+  }
+
+  -re "(?p).${toeol}" { exp_continue }
+  timeout {
+    send_user "Failure: time out\n"
+    exit 2
+  }
+}
+
+if { ${exitwith} == 0 } {
+  sus "All tests reported in OK"
+} else {
+  sus "${failures} TEST FAILURES; REVIEW LOGS"
+}
+exit ${exitwith}

+ 30 - 0
tests/utils/NTestTapOut.lua

@@ -0,0 +1,30 @@
+-- This is a NTest output handler that formats its output in a way that
+-- resembles the Test Anything Protocol (though prefixed with "TAP: " so we can
+-- more readily find it in comingled output streams).
+
+local nrun
+return function(e, test, msg, err)
+  msg = msg or ""
+  err = err or ""
+  if     e == "pass" then
+    print(("\nTAP: ok %d %s # %s"):format(nrun, test, msg))
+    nrun = nrun + 1
+  elseif e == "fail" then
+    print(("\nTAP: not ok %d %s # %s: %s"):format(nrun, test, msg, err))
+    nrun = nrun + 1
+  elseif e == "except" then
+    print(("\nTAP: not ok %d %s # exn; %s: %s"):format(nrun, test, msg, err))
+    nrun = nrun + 1
+  elseif e == "abort" then
+    print(("\nTAP: Bail out! %d %s # exn; %s: %s"):format(nrun, test, msg, err))
+  elseif e == "start" then
+    -- We don't know how many tests we plan to run, so emit a comment instead
+    print(("\nTAP: # STARTUP %s"):format(test))
+    nrun = 1
+  elseif e == "finish" then
+    -- Ah, now, here we go; we know how many tests we ran, so signal completion
+    print(("\nTAP: POST 1..%d"):format(nrun))
+  elseif #msg ~= 0 or #err ~= 0 then
+    print(("\nTAP: # %s: %s: %s"):format(test, msg, err))
+  end
+end