Implement the SHA-3 family.

These aren't used _directly_ by SSH at present, but an instance of
SHAKE-256 is required by the recently standardised Ed448.

Recipe

@@ -257,7 +257,7 @@ NONSSH   = telnet raw rlogin ldisc pinger
 
 # SSH back end (putty, plink, pscp, psftp).
 ARITH    = mpint ecc
-SSHCRYPTO = ARITH sshmd5 sshsha sshsh256 sshsh512
+SSHCRYPTO = ARITH sshmd5 sshsha sshsh256 sshsh512 sshsha3
 	 + sshrsa sshdss sshecc
          + sshdes sshblowf sshaes sshccp ssharcf
          + sshdh sshcrc sshcrcda sshauxcrypt

ssh.h

@@ -929,7 +929,7 @@ struct ssh2_userkey {
 };
 
 /* The maximum length of any hash algorithm. (bytes) */
-#define MAX_HASH_LEN (64)              /* longest is SHA-512 */
+#define MAX_HASH_LEN (114) /* longest is SHAKE256 with 114-byte output */
 
 extern const ssh_cipheralg ssh_3des_ssh1;
 extern const ssh_cipheralg ssh_blowfish_ssh1;
@@ -975,6 +975,11 @@ extern const ssh_hashalg ssh_sha256_hw;
 extern const ssh_hashalg ssh_sha256_sw;
 extern const ssh_hashalg ssh_sha384;
 extern const ssh_hashalg ssh_sha512;
+extern const ssh_hashalg ssh_sha3_224;
+extern const ssh_hashalg ssh_sha3_256;
+extern const ssh_hashalg ssh_sha3_384;
+extern const ssh_hashalg ssh_sha3_512;
+extern const ssh_hashalg ssh_shake256_114bytes;
 extern const ssh_kexes ssh_diffiehellman_group1;
 extern const ssh_kexes ssh_diffiehellman_group14;
 extern const ssh_kexes ssh_diffiehellman_gex;

sshsha3.c

@@ -0,0 +1,320 @@
+/*
+ * SHA-3, as defined in FIPS PUB 202.
+ */
+
+#include <assert.h>
+#include <string.h>
+#include "ssh.h"
+
+static inline uint64_t rol(uint64_t x, unsigned shift)
+{
+    unsigned L = (+shift) & 63;
+    unsigned R = (-shift) & 63;
+    return (x << L) | (x >> R);
+}
+
+/*
+ * General Keccak is defined such that its state is a 5x5 array of
+ * words which can be any power-of-2 size from 1 up to 64. SHA-3 fixes
+ * on 64, and so do we.
+ *
+ * The number of rounds is defined as 12 + 2k if the word size is 2^k.
+ * Here we have 64-bit words only, so k=6, so 24 rounds always.
+ */
+typedef uint64_t keccak_core_state[5][5];
+#define NROUNDS 24               /* would differ for other word sizes */
+static const uint64_t round_constants[NROUNDS];
+static const unsigned rotation_counts[5][5];
+
+/*
+ * Core Keccak transform: just squodge the state around internally,
+ * without adding or extracting any data from it.
+ */
+static void keccak_transform(keccak_core_state A)
+{
+    union {
+        uint64_t C[5];
+        uint64_t B[5][5];
+    } u;
+
+    for (unsigned round = 0; round < NROUNDS; round++) {
+        /* theta step */
+        for (unsigned x = 0; x < 5; x++)
+            u.C[x] = A[x][0] ^ A[x][1] ^ A[x][2] ^ A[x][3] ^ A[x][4];
+        for (unsigned x = 0; x < 5; x++) {
+            uint64_t D = rol(u.C[(x+1) % 5], 1) ^ u.C[(x+4) % 5];
+            for (unsigned y = 0; y < 5; y++)
+                A[x][y] ^= D;
+        }
+
+        /* rho and pi steps */
+        for (unsigned x = 0; x < 5; x++)
+            for (unsigned y = 0; y < 5; y++)
+                u.B[y][(2*x+3*y) % 5] = rol(A[x][y], rotation_counts[x][y]);
+
+        /* chi step */
+        for (unsigned x = 0; x < 5; x++)
+            for (unsigned y = 0; y < 5; y++)
+                A[x][y] = u.B[x][y] ^ (u.B[(x+2)%5][y] & ~u.B[(x+1)%5][y]);
+
+        /* iota step */
+        A[0][0] ^= round_constants[round];
+    }
+
+    smemclr(&u, sizeof(u));
+}
+
+typedef struct {
+    keccak_core_state A;
+    unsigned char bytes[25*8];
+    unsigned char first_pad_byte;
+    size_t bytes_got, bytes_wanted, hash_bytes;
+} keccak_state;
+
+/*
+ * Keccak accumulation function: given a piece of message, add it to
+ * the hash.
+ */
+static void keccak_accumulate(keccak_state *s, const void *vdata, size_t len)
+{
+    const unsigned char *data = (const unsigned char *)vdata;
+
+    while (len >= s->bytes_wanted - s->bytes_got) {
+        size_t b = s->bytes_wanted - s->bytes_got;
+        memcpy(s->bytes + s->bytes_got, data, b);
+        len -= b;
+        data += b;
+
+        size_t n = 0;
+        for (unsigned y = 0; y < 5; y++) {
+            for (unsigned x = 0; x < 5; x++) {
+                if (n >= s->bytes_wanted)
+                    break;
+
+                s->A[x][y] ^= GET_64BIT_LSB_FIRST(s->bytes + n);
+                n += 8;
+            }
+        }
+        keccak_transform(s->A);
+
+        s->bytes_got = 0;
+    }
+
+    memcpy(s->bytes + s->bytes_got, data, len);
+    s->bytes_got += len;
+}
+
+/*
+ * Keccak output function.
+ */
+static void keccak_output(keccak_state *s, void *voutput)
+{
+    unsigned char *output = (unsigned char *)voutput;
+
+    /*
+     * Add message padding.
+     */
+    {
+        unsigned char padding[25*8];
+        size_t len = s->bytes_wanted - s->bytes_got;
+        if (len == 0)
+            len = s->bytes_wanted;
+        memset(padding, 0, len);
+        padding[0] |= s->first_pad_byte;
+        padding[len-1] |= 0x80;
+        keccak_accumulate(s, padding, len);
+    }
+
+    size_t n = 0;
+    for (unsigned y = 0; y < 5; y++) {
+        for (unsigned x = 0; x < 5; x++) {
+            size_t to_copy = s->hash_bytes - n;
+            if (to_copy == 0)
+                break;
+            if (to_copy > 8)
+                to_copy = 8;
+            unsigned char outbytes[8];
+            PUT_64BIT_LSB_FIRST(outbytes, s->A[x][y]);
+            memcpy(output + n, outbytes, to_copy);
+            n += to_copy;
+        }
+    }
+}
+
+static void keccak_init(keccak_state *s, unsigned hashbits, unsigned ratebits,
+                        unsigned char first_pad_byte)
+{
+    int x, y;
+
+    assert(hashbits % 8 == 0);
+    assert(ratebits % 8 == 0);
+
+    s->hash_bytes = hashbits / 8;
+    s->bytes_wanted = (25 * 64 - ratebits) / 8;
+    s->bytes_got = 0;
+    s->first_pad_byte = first_pad_byte;
+
+    assert(s->bytes_wanted % 8 == 0);
+
+    for (y = 0; y < 5; y++)
+        for (x = 0; x < 5; x++)
+            s->A[x][y] = 0;
+}
+
+static void keccak_sha3_init(keccak_state *s, int hashbits)
+{
+    keccak_init(s, hashbits, hashbits * 2, 0x06);
+}
+
+static void keccak_shake_init(keccak_state *s, int parambits, int hashbits)
+{
+    keccak_init(s, hashbits, parambits * 2, 0x1f);
+}
+
+/*
+ * Keccak round constants, generated via the LFSR specified in the
+ * Keccak reference by the following piece of Python:
+
+import textwrap
+from functools import reduce
+
+rbytes = [1]
+while len(rbytes) < 7*24:
+    k = rbytes[-1] * 2
+    rbytes.append(k ^ (0x171 * (k >> 8)))
+
+rbits = [byte & 1 for byte in rbytes]
+
+rwords = [sum(rbits[i+j] << ((1 << j) - 1) for j in range(7))
+          for i in range(0, len(rbits), 7)]
+
+print(textwrap.indent("\n".join(textwrap.wrap(", ".join(
+    map("0x{:016x}".format, rwords)))), " "*4))
+
+*/
+
+static const uint64_t round_constants[24] = {
+    0x0000000000000001, 0x0000000000008082, 0x800000000000808a,
+    0x8000000080008000, 0x000000000000808b, 0x0000000080000001,
+    0x8000000080008081, 0x8000000000008009, 0x000000000000008a,
+    0x0000000000000088, 0x0000000080008009, 0x000000008000000a,
+    0x000000008000808b, 0x800000000000008b, 0x8000000000008089,
+    0x8000000000008003, 0x8000000000008002, 0x8000000000000080,
+    0x000000000000800a, 0x800000008000000a, 0x8000000080008081,
+    0x8000000000008080, 0x0000000080000001, 0x8000000080008008
+};
+
+/*
+ * Keccak per-element rotation counts, generated from the matrix
+ * formula in the Keccak reference by the following piece of Python:
+
+coords = [1, 0]
+while len(coords) < 26:
+    coords.append((2*coords[-2] + 3*coords[-1]) % 5)
+
+matrix = { (coords[i], coords[i+1]) : i for i in range(24) }
+matrix[0,0] = -1
+
+f = lambda t: (t+1) * (t+2) // 2 % 64
+
+for y in range(5):
+    print("    {{{}}},".format(", ".join("{:2d}".format(f(matrix[y,x]))
+                                         for x in range(5))))
+
+*/
+static const unsigned rotation_counts[5][5] = {
+    { 0, 36,  3, 41, 18},
+    { 1, 44, 10, 45,  2},
+    {62,  6, 43, 15, 61},
+    {28, 55, 25, 21, 56},
+    {27, 20, 39,  8, 14},
+};
+
+/*
+ * The PuTTY ssh_hashalg abstraction.
+ */
+struct keccak_hash {
+    keccak_state state;
+    ssh_hash hash;
+    BinarySink_IMPLEMENTATION;
+};
+
+static void keccak_BinarySink_write(BinarySink *bs, const void *p, size_t len)
+{
+    struct keccak_hash *kh = BinarySink_DOWNCAST(bs, struct keccak_hash);
+    keccak_accumulate(&kh->state, p, len);
+}
+
+static ssh_hash *keccak_new(const ssh_hashalg *alg)
+{
+    struct keccak_hash *kh = snew(struct keccak_hash);
+    kh->hash.vt = alg;
+    BinarySink_INIT(kh, keccak_BinarySink_write);
+    BinarySink_DELEGATE_INIT(&kh->hash, kh);
+    return ssh_hash_reset(&kh->hash);
+}
+
+static void keccak_free(ssh_hash *hash)
+{
+    struct keccak_hash *kh = container_of(hash, struct keccak_hash, hash);
+    smemclr(kh, sizeof(*kh));
+    sfree(kh);
+}
+
+static void keccak_copyfrom(ssh_hash *hnew, ssh_hash *hold)
+{
+    struct keccak_hash *khold = container_of(hold, struct keccak_hash, hash);
+    struct keccak_hash *khnew = container_of(hnew, struct keccak_hash, hash);
+    khnew->state = khold->state;
+}
+
+static void keccak_digest(ssh_hash *hash, unsigned char *output)
+{
+    struct keccak_hash *kh = container_of(hash, struct keccak_hash, hash);
+    keccak_output(&kh->state, output);
+}
+
+static void sha3_reset(ssh_hash *hash)
+{
+    struct keccak_hash *kh = container_of(hash, struct keccak_hash, hash);
+    keccak_sha3_init(&kh->state, hash->vt->hlen * 8);
+}
+
+#define DEFINE_SHA3(bits)                               \
+    const ssh_hashalg ssh_sha3_##bits = {               \
+        keccak_new, sha3_reset, keccak_copyfrom,        \
+        keccak_digest, keccak_free,                     \
+        bits/8, 200 - 2*(bits/8),                       \
+        HASHALG_NAMES_BARE("SHA3-" #bits),              \
+    }
+
+DEFINE_SHA3(224);
+DEFINE_SHA3(256);
+DEFINE_SHA3(384);
+DEFINE_SHA3(512);
+
+static void shake256_reset(ssh_hash *hash)
+{
+    struct keccak_hash *kh = container_of(hash, struct keccak_hash, hash);
+    keccak_shake_init(&kh->state, 256, hash->vt->hlen * 8);
+}
+
+/*
+ * There is some confusion over the output length parameter for the
+ * SHAKE functions. By my reading, FIPS PUB 202 defines SHAKE256(M,d)
+ * to generate d _bits_ of output. But RFC 8032 (defining Ed448) talks
+ * about "SHAKE256(x,114)" in a context where it definitely means
+ * generating 114 _bytes_ of output.
+ *
+ * Our internal ID therefore suffixes the output length with "bytes",
+ * to be clear which we're talking about
+ */
+
+#define DEFINE_SHAKE(param, hashbytes)                                  \
+    const ssh_hashalg ssh_shake##param##_##hashbytes##bytes = {         \
+        keccak_new, shake##param##_reset, keccak_copyfrom,              \
+        keccak_digest, keccak_free,                                     \
+        hashbytes, 0, HASHALG_NAMES_BARE("SHAKE" #param),               \
+    }
+
+DEFINE_SHAKE(256, 114);

test/cryptsuite.py

@@ -2341,6 +2341,22 @@ class standard_test_vectors(MyTestBase):
             'c665befb36da189d78822d10528cbf3b12b3eef726039909c1a16a270d487193'
             '77966b957a878e720584779a62825c18da26415e49a7176a894e7510fd1451f5'))
 
+    def testSHA3(self):
+        # Source: all the SHA-3 test strings from
+        # https://csrc.nist.gov/projects/cryptographic-standards-and-guidelines/example-values#aHashing
+        # which are a multiple of 8 bits long.
+
+        self.assertEqualBin(hash_str('sha3_224', ''), unhex("6b4e03423667dbb73b6e15454f0eb1abd4597f9a1b078e3f5b5a6bc7"))
+        self.assertEqualBin(hash_str('sha3_224', unhex('a3')*200), unhex("9376816aba503f72f96ce7eb65ac095deee3be4bf9bbc2a1cb7e11e0"))
+        self.assertEqualBin(hash_str('sha3_256', ''), unhex("a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a"))
+        self.assertEqualBin(hash_str('sha3_256', unhex('a3')*200), unhex("79f38adec5c20307a98ef76e8324afbfd46cfd81b22e3973c65fa1bd9de31787"))
+        self.assertEqualBin(hash_str('sha3_384', ''), unhex("0c63a75b845e4f7d01107d852e4c2485c51a50aaaa94fc61995e71bbee983a2ac3713831264adb47fb6bd1e058d5f004"))
+        self.assertEqualBin(hash_str('sha3_384', unhex('a3')*200), unhex("1881de2ca7e41ef95dc4732b8f5f002b189cc1e42b74168ed1732649ce1dbcdd76197a31fd55ee989f2d7050dd473e8f"))
+        self.assertEqualBin(hash_str('sha3_512', ''), unhex("a69f73cca23a9ac5c8b567dc185a756e97c982164fe25859e0d1dcc1475c80a615b2123af1f5f94c11e3e9402c3ac558f500199d95b6d3e301758586281dcd26"))
+        self.assertEqualBin(hash_str('sha3_512', unhex('a3')*200), unhex("e76dfad22084a8b1467fcf2ffa58361bec7628edf5f3fdc0e4805dc48caeeca81b7c13c30adf52a3659584739a2df46be589c51ca1a4a8416df6545a1ce8ba00"))
+        self.assertEqualBin(hash_str('shake256_114bytes', ''), unhex("46b9dd2b0ba88d13233b3feb743eeb243fcd52ea62b81b82b50c27646ed5762fd75dc4ddd8c0f200cb05019d67b592f6fc821c49479ab48640292eacb3b7c4be141e96616fb13957692cc7edd0b45ae3dc07223c8e92937bef84bc0eab862853349ec75546f58fb7c2775c38462c5010d846"))
+        self.assertEqualBin(hash_str('shake256_114bytes', unhex('a3')*200), unhex("cd8a920ed141aa0407a22d59288652e9d9f1a7ee0c1e7c1ca699424da84a904d2d700caae7396ece96604440577da4f3aa22aeb8857f961c4cd8e06f0ae6610b1048a7f64e1074cd629e85ad7566048efc4fb500b486a3309a8f26724c0ed628001a1099422468de726f1061d99eb9e93604"))
+
     def testHmacSHA(self):
         # Test cases from RFC 6234 section 8.5.
         def vector(key, message, s1=None, s256=None):

testcrypt.c

@@ -216,6 +216,11 @@ static const ssh_hashalg *get_hashalg(BinarySource *in)
         {"sha256_hw", &ssh_sha256_hw},
         {"sha384", &ssh_sha384},
         {"sha512", &ssh_sha512},
+        {"sha3_224", &ssh_sha3_224},
+        {"sha3_256", &ssh_sha3_256},
+        {"sha3_384", &ssh_sha3_384},
+        {"sha3_512", &ssh_sha3_512},
+        {"shake256_114bytes", &ssh_shake256_114bytes},
     };
 
     ptrlen name = get_word(in);

testsc.c

@@ -272,6 +272,11 @@ VOLATILE_WRAPPED_DEFN(static, size_t, looplimit, (size_t x))
     X(Y, ssh_sha256_sw)                         \
     X(Y, ssh_sha384)                            \
     X(Y, ssh_sha512)                            \
+    X(Y, ssh_sha3_224)                          \
+    X(Y, ssh_sha3_256)                          \
+    X(Y, ssh_sha3_384)                          \
+    X(Y, ssh_sha3_512)                          \
+    X(Y, ssh_shake256_114bytes)                 \
     /* end of list */
 
 #define HASH_TESTLIST(X, name) X(hash_ ## name)