Rewrite the testcrypt.c macro system.

Yesterday's commit 52ee636b09 which further extended the huge
pile of arity-specific annoying wrapper macros pushed me over the edge
and inspired me to give some harder thought to finding a way to handle
all arities at once. And this time I found one!

The new technique changes the syntax of the function specifications in
testcrypt.h. In particular, they now have to specify a _name_ for each
parameter as well as a type, because the macros generating the C
marshalling wrappers will need a structure field for each parameter
and cpp isn't flexible enough to generate names for those fields
automatically. Rather than tediously name them arg1, arg2 etc, I've
reused the names of the parameters from the prototypes or definitions
of the underlying real functions (via a one-off auto-extraction
process starting from the output of 'clang -Xclang -dump-ast' plus
some manual polishing), which means testcrypt.h is now a bit more
self-documenting.

The testcrypt.py end of the mechanism is rewritten to eat the new
format. Since it's got more complicated syntax and nested parens and
things, I've written something a bit like a separated lexer/parser
system in place of the previous crude regex matcher, which should
enforce that the whole header file really does conform to the
restricted syntax it has to fit into.

The new system uses a lot less code in testcrypt.c, but I've made up
for that by also writing a long comment explaining how it works, which
was another thing the previous system lacked! Similarly, the new
testcrypt.h has some long-overdue instructions at the top.

test/testcrypt.py

@@ -3,6 +3,7 @@ import os
 import numbers
 import subprocess
 import re
+import string
 import struct
 from binascii import hexlify
 
@@ -269,10 +270,95 @@ class Function(object):
             return retvals[0]
         return tuple(retvals)
 
-def _setup(scope):
-    header_file = os.path.join(putty_srcdir, "testcrypt.h")
+def _lex_testcrypt_header(header):
+    pat = re.compile(
+        # Skip any combination of whitespace and comments
+        '(?:{})*'.format('|'.join((
+            '[ \t\n]',             # whitespace
+            '/\\*(?:.|\n)*?\\*/',  # C90-style /* ... */ comment, ended eagerly
+            '//[^\n]*\n',          # C99-style comment to end-of-line
+        ))) +
+        # And then match a token
+        '({})'.format('|'.join((
+            # Punctuation
+            '\(',
+            '\)',
+            ',',
+            # Identifier
+            '[A-Za-z_][A-Za-z0-9_]*',
+            # End of string
+            '$',
+        )))
+    )
 
-    linere = re.compile(r'^FUNC\d+\((.*)\)$')
+    pos = 0
+    end = len(header)
+    while pos < end:
+        m = pat.match(header, pos)
+        assert m is not None, (
+            "Failed to lex testcrypt.h at byte position {:d}".format(pos))
+
+        pos = m.end()
+        tok = m.group(1)
+        if len(tok) == 0:
+            assert pos == end, (
+                "Empty token should only be returned at end of string")
+        yield tok, m.start(1)
+
+def _parse_testcrypt_header(tokens):
+    def is_id(tok):
+        return tok[0] in string.ascii_letters+"_"
+    def expect(what, why, eof_ok=False):
+        tok, pos = next(tokens)
+        if tok == '' and eof_ok:
+            return None
+        if hasattr(what, '__call__'):
+            description = lambda: ""
+            ok = what(tok)
+        elif isinstance(what, set):
+            description = lambda: " or ".join("'"+x+"' " for x in sorted(what))
+            ok = tok in what
+        else:
+            description = lambda: "'"+what+"' "
+            ok = tok == what
+        if not ok:
+            sys.exit("testcrypt.h:{:d}: expected {}{}".format(
+                pos, description(), why))
+        return tok
+
+    while True:
+        tok = expect("FUNC", "at start of function specification", eof_ok=True)
+        if tok is None:
+            break
+
+        expect("(", "after FUNC")
+        rettype = expect(is_id, "return type")
+        expect(",", "after return type")
+        funcname = expect(is_id, "function name")
+        expect(",", "after function name")
+        expect("(", "to begin argument list")
+        args = []
+        firstargkind = expect({"ARG", "VOID"}, "at start of argument list")
+        if firstargkind == "VOID":
+            expect(")", "after VOID")
+        else:
+            while True:
+                # Every time we come back to the top of this loop, we've
+                # just seen 'ARG'
+                expect("(", "after ARG")
+                argtype = expect(is_id, "argument type")
+                expect(",", "after argument type")
+                argname = expect(is_id, "argument name")
+                args.append((argtype, argname))
+                expect(")", "at end of ARG")
+                punct = expect({",", ")"}, "after argument")
+                if punct == ")":
+                    break
+                expect("ARG", "to begin next argument")
+        expect(")", "at end of FUNC")
+        yield funcname, rettype, args
+
+def _setup(scope):
     valprefix = "val_"
     outprefix = "out_"
     optprefix = "opt_"
@@ -288,36 +374,34 @@ def _setup(scope):
             arg = arg[:arg.index("_", len(valprefix))]
         return arg
 
-    with open(header_file) as f:
-        for line in iter(f.readline, ""):
-            line = line.rstrip("\r\n").replace(" ", "")
-            m = linere.match(line)
-            if m is not None:
-                words = m.group(1).split(",")
-                function = words[1]
-                rettypes = []
-                argtypes = []
-                argsconsumed = []
-                if words[0] != "void":
-                    rettypes.append(trim_argtype(words[0]))
-                for arg in words[2:]:
-                    if arg.startswith(outprefix):
-                        rettypes.append(trim_argtype(arg[len(outprefix):]))
-                    else:
-                        consumed = False
-                        if arg.startswith(consprefix):
-                            arg = arg[len(consprefix):]
-                            consumed = True
-                        arg = trim_argtype(arg)
-                        argtypes.append((arg, consumed))
-                func = Function(function, rettypes, argtypes)
-                scope[function] = func
-                if len(argtypes) > 0:
-                    t = argtypes[0][0]
-                    if (t in method_prefixes and
-                        function.startswith(method_prefixes[t])):
-                        methodname = function[len(method_prefixes[t]):]
-                        method_lists[t].append((methodname, func))
+    with open(os.path.join(putty_srcdir, "testcrypt.h")) as f:
+        header = f.read()
+    tokens = _lex_testcrypt_header(header)
+    for function, rettype, arglist in _parse_testcrypt_header(tokens):
+        rettypes = []
+        if rettype != "void":
+            rettypes.append(trim_argtype(rettype))
+
+        argtypes = []
+        argsconsumed = []
+        for arg, argname in arglist:
+            if arg.startswith(outprefix):
+                rettypes.append(trim_argtype(arg[len(outprefix):]))
+            else:
+                consumed = False
+                if arg.startswith(consprefix):
+                    arg = arg[len(consprefix):]
+                    consumed = True
+                arg = trim_argtype(arg)
+                argtypes.append((arg, consumed))
+        func = Function(function, rettypes, argtypes)
+        scope[function] = func
+        if len(argtypes) > 0:
+            t = argtypes[0][0]
+            if (t in method_prefixes and
+                function.startswith(method_prefixes[t])):
+                methodname = function[len(method_prefixes[t]):]
+                method_lists[t].append((methodname, func))
 
 _setup(globals())
 del _setup