Add more variants of SHAKE.

This adds a ssh_hashalg defining SHAKE256 with a 32-byte output, in
addition to the 114-byte output we already have.

Also, it defines a new API for using SHAKE128 and SHAKE256 in the more
general form of an extendable output function, which is to say that
you still have to put in all the input before reading any output, but
once you start reading output you can just keep going until you have
enough.

Both of these will be needed in an upcoming commit implementing ML-KEM.

crypto/sha3.c

@@ -326,4 +326,76 @@ static void shake256_reset(ssh_hash *hash)
         HASHALG_NAMES_BARE("SHAKE" #param),                     \
     }
 
-DEFINE_SHAKE(256, 114);
+DEFINE_SHAKE(256, 114);                /* used by Ed448 */
+DEFINE_SHAKE(256, 32);                 /* used by ML-KEM */
+
+struct ShakeXOF {
+    keccak_state state;
+    unsigned char *buf;
+    size_t bytes_per_transform, pos;
+};
+
+static ShakeXOF *shake_xof_from_input(unsigned bits, ptrlen data)
+{
+    ShakeXOF *sx = snew_plus(ShakeXOF, 200 * 64);
+    sx->buf = snew_plus_get_aux(sx);
+
+    /* Initialise as if we were generating 0 bytes of hash. That way,
+     * keccak_output will do the final accumulation but generate no data. */
+    keccak_shake_init(&sx->state, bits, 0);
+    keccak_accumulate(&sx->state, data.ptr, data.len);
+    keccak_output(&sx->state, NULL);
+
+    sx->bytes_per_transform = 200 - bits/4;
+    sx->pos = 0;
+
+    return sx;
+}
+
+ShakeXOF *shake128_xof_from_input(ptrlen data)
+{
+    return shake_xof_from_input(128, data);
+}
+
+ShakeXOF *shake256_xof_from_input(ptrlen data)
+{
+    return shake_xof_from_input(256, data);
+}
+
+void shake_xof_read(ShakeXOF *sx, void *output_v, size_t size)
+{
+    unsigned char *output = (unsigned char *)output_v;
+
+    while (size > 0) {
+        if (sx->pos == 0) {
+            /* Copy the 64-bit words from the Keccak state into the
+             * output buffer of bytes */
+            for (unsigned y = 0; y < 5; y++)
+                for (unsigned x = 0; x < 5; x++)
+                    PUT_64BIT_LSB_FIRST(sx->buf + 8 * (5*y+x),
+                                        sx->state.A[x][y]);
+        }
+
+        /* Read a chunk from the byte buffer */
+        size_t this_size = sx->bytes_per_transform - sx->pos;
+        if (this_size > size)
+            this_size = size;
+        memcpy(output, sx->buf + sx->pos, this_size);
+        sx->pos += this_size;
+        output += this_size;
+        size -= this_size;
+
+        /* Retransform the Keccak state if we've run out of data */
+        if (sx->pos >= sx->bytes_per_transform) {
+            keccak_transform(sx->state.A);
+            sx->pos = 0;
+        }
+    }
+}
+
+void shake_xof_free(ShakeXOF *sx)
+{
+    smemclr(sx->buf, 200 * 64);
+    smemclr(sx, sizeof(*sx));
+    sfree(sx);
+}

defs.h

@@ -193,6 +193,7 @@ typedef struct NTRUKeyPair NTRUKeyPair;
 typedef struct NTRUEncodeSchedule NTRUEncodeSchedule;
 typedef struct RFC6979 RFC6979;
 typedef struct RFC6979Result RFC6979Result;
+typedef struct ShakeXOF ShakeXOF;
 
 typedef struct dlgparam dlgparam;
 typedef struct dlgcontrol dlgcontrol;

ssh.h

@@ -1213,6 +1213,7 @@ extern const ssh_hashalg ssh_sha3_224;
 extern const ssh_hashalg ssh_sha3_256;
 extern const ssh_hashalg ssh_sha3_384;
 extern const ssh_hashalg ssh_sha3_512;
+extern const ssh_hashalg ssh_shake256_32bytes;
 extern const ssh_hashalg ssh_shake256_114bytes;
 extern const ssh_hashalg ssh_blake2b;
 extern const ssh_kexes ssh_diffiehellman_group1;

test/cryptsuite.py

@@ -3853,6 +3853,22 @@ class standard_test_vectors(MyTestBase):
         self.assertEqualBin(hash_str('shake256_114bytes', ''), unhex("46b9dd2b0ba88d13233b3feb743eeb243fcd52ea62b81b82b50c27646ed5762fd75dc4ddd8c0f200cb05019d67b592f6fc821c49479ab48640292eacb3b7c4be141e96616fb13957692cc7edd0b45ae3dc07223c8e92937bef84bc0eab862853349ec75546f58fb7c2775c38462c5010d846"))
         self.assertEqualBin(hash_str('shake256_114bytes', unhex('a3')*200), unhex("cd8a920ed141aa0407a22d59288652e9d9f1a7ee0c1e7c1ca699424da84a904d2d700caae7396ece96604440577da4f3aa22aeb8857f961c4cd8e06f0ae6610b1048a7f64e1074cd629e85ad7566048efc4fb500b486a3309a8f26724c0ed628001a1099422468de726f1061d99eb9e93604"))
 
+    def testSHA3XOF(self):
+        # Cherry-picked examples from CAVS 19.0, testing both SHAKE128
+        # and SHAKE256, each with a long input and a long output.
+
+        xof = shake128_xof_from_input(unhex('a6fe00064257aa318b621c5eb311d32bb8004c2fa1a969d205d71762cc5d2e633907992629d1b69d9557ff6d5e8deb454ab00f6e497c89a4fea09e257a6fa2074bd818ceb5981b3e3faefd6e720f2d1edd9c5e4a5c51e5009abf636ed5bca53fe159c8287014a1bd904f5c8a7501625f79ac81eb618f478ce21cae6664acffb30572f059e1ad0fc2912264e8f1ca52af26c8bf78e09d75f3dd9fc734afa8770abe0bd78c90cc2ff448105fb16dd2c5b7edd8611a62e537db9331f5023e16d6ec150cc6e706d7c7fcbfff930c7281831fd5c4aff86ece57ed0db882f59a5fe403105d0592ca38a081fed84922873f538ee774f13b8cc09bd0521db4374aec69f4bae6dcb66455822c0b84c91a3474ffac2ad06f0a4423cd2c6a49d4f0d6242d6a1890937b5d9835a5f0ea5b1d01884d22a6c1718e1f60b3ab5e232947c76ef70b344171083c688093b5f1475377e3069863'))
+        self.assertEqualBin(shake_xof_read(xof, 128//8), unhex("3109d9472ca436e805c6b3db2251a9bc"))
+
+        xof = shake128_xof_from_input(unhex('0a13ad2c7a239b4ba73ea6592ae84ea9'))
+        self.assertEqualBin(shake_xof_read(xof, 1120//8), unhex("5feaf99c15f48851943ff9baa6e5055d8377f0dd347aa4dbece51ad3a6d9ce0c01aee9fe2260b80a4673a909b532adcdd1e421c32d6460535b5fe392a58d2634979a5a104d6c470aa3306c400b061db91c463b2848297bca2bc26d1864ba49d7ff949ebca50fbf79a5e63716dc82b600bd52ca7437ed774d169f6bf02e46487956fba2230f34cd2a0485484d"))
+
+        xof = shake256_xof_from_input(unhex('dc5a100fa16df1583c79722a0d72833d3bf22c109b8889dbd35213c6bfce205813edae3242695cfd9f59b9a1c203c1b72ef1a5423147cb990b5316a85266675894e2644c3f9578cebe451a09e58c53788fe77a9e850943f8a275f830354b0593a762bac55e984db3e0661eca3cb83f67a6fb348e6177f7dee2df40c4322602f094953905681be3954fe44c4c902c8f6bba565a788b38f13411ba76ce0f9f6756a2a2687424c5435a51e62df7a8934b6e141f74c6ccf539e3782d22b5955d3baf1ab2cf7b5c3f74ec2f9447344e937957fd7f0bdfec56d5d25f61cde18c0986e244ecf780d6307e313117256948d4230ebb9ea62bb302cfe80d7dfebabc4a51d7687967ed5b416a139e974c005fff507a96'))
+        self.assertEqualBin(shake_xof_read(xof, 256//8), unhex("2bac5716803a9cda8f9e84365ab0a681327b5ba34fdedfb1c12e6e807f45284b"))
+
+        xof = shake256_xof_from_input(unhex('8d8001e2c096f1b88e7c9224a086efd4797fbf74a8033a2d422a2b6b8f6747e4'))
+        self.assertEqualBin(shake_xof_read(xof, 2000//8), unhex("2e975f6a8a14f0704d51b13667d8195c219f71e6345696c49fa4b9d08e9225d3d39393425152c97e71dd24601c11abcfa0f12f53c680bd3ae757b8134a9c10d429615869217fdd5885c4db174985703a6d6de94a667eac3023443a8337ae1bc601b76d7d38ec3c34463105f0d3949d78e562a039e4469548b609395de5a4fd43c46ca9fd6ee29ada5efc07d84d553249450dab4a49c483ded250c9338f85cd937ae66bb436f3b4026e859fda1ca571432f3bfc09e7c03ca4d183b741111ca0483d0edabc03feb23b17ee48e844ba2408d9dcfd0139d2e8c7310125aee801c61ab7900d1efc47c078281766f361c5e6111346235e1dc38325666c"))
+
     def testBLAKE2b(self):
         # Test case from RFC 7693 appendix A.
         self.assertEqualBin(hash_str('blake2b', b'abc'), unhex(

test/testcrypt-func.h

@@ -264,6 +264,10 @@ FUNC(void, ssh_hash_update, ARG(val_hash, h), ARG(val_string_ptrlen, data))
 
 FUNC(opt_val_hash, blake2b_new_general, ARG(uint, hashlen))
 
+FUNC(val_shakexof, shake128_xof_from_input, ARG(val_string_ptrlen, input))
+FUNC(val_shakexof, shake256_xof_from_input, ARG(val_string_ptrlen, input))
+FUNC_WRAPPED(val_string, shake_xof_read, ARG(val_shakexof, xof), ARG(uint, size))
+
 /*
  * The ssh2_mac abstraction. Note the optional ssh_cipher parameter
  * to ssh2_mac_new. Also, again, I've invented an ssh2_mac_update so

test/testcrypt.c

@@ -99,6 +99,7 @@ uint64_t prng_reseed_time_ms(void)
     X(millerrabin, MillerRabin *, miller_rabin_free(v))                 \
     X(ntrukeypair, NTRUKeyPair *, ntru_keypair_free(v))                 \
     X(ntruencodeschedule, NTRUEncodeSchedule *, ntru_encode_schedule_free(v)) \
+    X(shakexof, ShakeXOF *, shake_xof_free(v))                          \
     /* end of list */
 
 typedef struct Value Value;
@@ -743,6 +744,14 @@ strbuf *ssh_hash_final_wrapper(ssh_hash *h)
     return sb;
 }
 
+strbuf *shake_xof_read_wrapper(ShakeXOF *sx, TD_uint size)
+{
+    strbuf *sb = strbuf_new();
+    void *p = strbuf_append(sb, size);
+    shake_xof_read(sx, p, size);
+    return sb;
+}
+
 void ssh_cipher_setiv_wrapper(ssh_cipher *c, ptrlen iv)
 {
     if (iv.len != ssh_cipher_alg(c)->blksize)