/* * Hardware-accelerated implementation of SHA-512 using Arm NEON. */ #include "ssh.h" #include "sha512.h" #if USE_ARM64_NEON_H #include #else #include #endif static bool sha512_neon_available(void) { /* * For Arm, we delegate to a per-platform detection function (see * explanation in aes-neon.c). */ return platform_sha512_neon_available(); } #if !HAVE_NEON_SHA512_INTRINSICS /* * clang 12 and before do not provide the SHA-512 NEON intrinsics, but * do provide assembler support for the underlying instructions. So I * define the intrinsic functions myself, using inline assembler. */ static inline uint64x2_t vsha512su0q_u64(uint64x2_t x, uint64x2_t y) { __asm__("sha512su0 %0.2D,%1.2D" : "+w" (x) : "w" (y)); return x; } static inline uint64x2_t vsha512su1q_u64(uint64x2_t x, uint64x2_t y, uint64x2_t z) { __asm__("sha512su1 %0.2D,%1.2D,%2.2D" : "+w" (x) : "w" (y), "w" (z)); return x; } static inline uint64x2_t vsha512hq_u64(uint64x2_t x, uint64x2_t y, uint64x2_t z) { __asm__("sha512h %0,%1,%2.2D" : "+w" (x) : "w" (y), "w" (z)); return x; } static inline uint64x2_t vsha512h2q_u64(uint64x2_t x, uint64x2_t y, uint64x2_t z) { __asm__("sha512h2 %0,%1,%2.2D" : "+w" (x) : "w" (y), "w" (z)); return x; } #endif /* HAVE_NEON_SHA512_INTRINSICS */ typedef struct sha512_neon_core sha512_neon_core; struct sha512_neon_core { uint64x2_t ab, cd, ef, gh; }; static inline uint64x2_t sha512_neon_load_input(const uint8_t *p) { return vreinterpretq_u64_u8(vrev64q_u8(vld1q_u8(p))); } static inline uint64x2_t sha512_neon_schedule_update( uint64x2_t m8, uint64x2_t m7, uint64x2_t m4, uint64x2_t m3, uint64x2_t m1) { /* * vsha512su0q_u64() takes words from a long way back in the * schedule and performs the sigma_0 half of the computation of * the next two 64-bit message-schedule words. * * vsha512su1q_u64() combines the result of that with the sigma_1 * steps, to output the finished version of those two words. The * total amount of input data it requires fits nicely into three * 128-bit vector registers, but one of those registers is * misaligned compared to the 128-bit chunks that the message * schedule is stored in. So we use vextq_u64 to make one of its * input words out of the second half of m4 and the first half of * m3. */ return vsha512su1q_u64(vsha512su0q_u64(m8, m7), m1, vextq_u64(m4, m3, 1)); } static inline void sha512_neon_round2( unsigned round_index, uint64x2_t schedule_words, uint64x2_t *ab, uint64x2_t *cd, uint64x2_t *ef, uint64x2_t *gh) { /* * vsha512hq_u64 performs the Sigma_1 and Ch half of the * computation of two rounds of SHA-512 (including feeding back * one of the outputs from the first of those half-rounds into the * second one). * * vsha512h2q_u64 combines the result of that with the Sigma_0 and * Maj steps, and outputs one 128-bit vector that replaces the gh * piece of the input hash state, and a second that updates cd by * addition. * * Similarly to vsha512su1q_u64 above, some of the input registers * expected by these instructions are misaligned by 64 bits * relative to the chunks we've divided the hash state into, so we * have to start by making 'de' and 'fg' words out of our input * cd,ef,gh, using vextq_u64. * * Also, one of the inputs to vsha512hq_u64 is expected to contain * the results of summing gh + two round constants + two words of * message schedule, but the two words of the message schedule * have to be the opposite way round in the vector register from * the way that vsha512su1q_u64 output them. Hence, there's * another vextq_u64 in here that swaps the two halves of the * initial_sum vector register. * * (This also means that I don't have to prepare a specially * reordered version of the sha512_round_constants[] array: as * long as I'm unavoidably doing a swap at run time _anyway_, I * can load from the normally ordered version of that array, and * just take care to fold in that data _before_ the swap rather * than after.) */ /* Load two round constants, with the first one in the low half */ uint64x2_t round_constants = vld1q_u64( sha512_round_constants + round_index); /* Add schedule words to round constants */ uint64x2_t initial_sum = vaddq_u64(schedule_words, round_constants); /* Swap that sum around so the word used in the first of the two * rounds is in the _high_ half of the vector, matching where h * lives in the gh vector */ uint64x2_t swapped_initial_sum = vextq_u64(initial_sum, initial_sum, 1); /* Add gh to that, now that they're matching ways round */ uint64x2_t sum = vaddq_u64(swapped_initial_sum, *gh); /* Make the misaligned de and fg words */ uint64x2_t de = vextq_u64(*cd, *ef, 1); uint64x2_t fg = vextq_u64(*ef, *gh, 1); /* Now we're ready to put all the pieces together. The output from * vsha512h2q_u64 can be used directly as the new gh, and the * output from vsha512hq_u64 is simultaneously the intermediate * value passed to h2 and the thing you have to add on to cd. */ uint64x2_t intermed = vsha512hq_u64(sum, fg, de); *gh = vsha512h2q_u64(intermed, *cd, *ab); *cd = vaddq_u64(*cd, intermed); } static inline void sha512_neon_block(sha512_neon_core *core, const uint8_t *p) { uint64x2_t s0, s1, s2, s3, s4, s5, s6, s7; uint64x2_t ab = core->ab, cd = core->cd, ef = core->ef, gh = core->gh; s0 = sha512_neon_load_input(p + 16*0); sha512_neon_round2(0, s0, &ab, &cd, &ef, &gh); s1 = sha512_neon_load_input(p + 16*1); sha512_neon_round2(2, s1, &gh, &ab, &cd, &ef); s2 = sha512_neon_load_input(p + 16*2); sha512_neon_round2(4, s2, &ef, &gh, &ab, &cd); s3 = sha512_neon_load_input(p + 16*3); sha512_neon_round2(6, s3, &cd, &ef, &gh, &ab); s4 = sha512_neon_load_input(p + 16*4); sha512_neon_round2(8, s4, &ab, &cd, &ef, &gh); s5 = sha512_neon_load_input(p + 16*5); sha512_neon_round2(10, s5, &gh, &ab, &cd, &ef); s6 = sha512_neon_load_input(p + 16*6); sha512_neon_round2(12, s6, &ef, &gh, &ab, &cd); s7 = sha512_neon_load_input(p + 16*7); sha512_neon_round2(14, s7, &cd, &ef, &gh, &ab); s0 = sha512_neon_schedule_update(s0, s1, s4, s5, s7); sha512_neon_round2(16, s0, &ab, &cd, &ef, &gh); s1 = sha512_neon_schedule_update(s1, s2, s5, s6, s0); sha512_neon_round2(18, s1, &gh, &ab, &cd, &ef); s2 = sha512_neon_schedule_update(s2, s3, s6, s7, s1); sha512_neon_round2(20, s2, &ef, &gh, &ab, &cd); s3 = sha512_neon_schedule_update(s3, s4, s7, s0, s2); sha512_neon_round2(22, s3, &cd, &ef, &gh, &ab); s4 = sha512_neon_schedule_update(s4, s5, s0, s1, s3); sha512_neon_round2(24, s4, &ab, &cd, &ef, &gh); s5 = sha512_neon_schedule_update(s5, s6, s1, s2, s4); sha512_neon_round2(26, s5, &gh, &ab, &cd, &ef); s6 = sha512_neon_schedule_update(s6, s7, s2, s3, s5); sha512_neon_round2(28, s6, &ef, &gh, &ab, &cd); s7 = sha512_neon_schedule_update(s7, s0, s3, s4, s6); sha512_neon_round2(30, s7, &cd, &ef, &gh, &ab); s0 = sha512_neon_schedule_update(s0, s1, s4, s5, s7); sha512_neon_round2(32, s0, &ab, &cd, &ef, &gh); s1 = sha512_neon_schedule_update(s1, s2, s5, s6, s0); sha512_neon_round2(34, s1, &gh, &ab, &cd, &ef); s2 = sha512_neon_schedule_update(s2, s3, s6, s7, s1); sha512_neon_round2(36, s2, &ef, &gh, &ab, &cd); s3 = sha512_neon_schedule_update(s3, s4, s7, s0, s2); sha512_neon_round2(38, s3, &cd, &ef, &gh, &ab); s4 = sha512_neon_schedule_update(s4, s5, s0, s1, s3); sha512_neon_round2(40, s4, &ab, &cd, &ef, &gh); s5 = sha512_neon_schedule_update(s5, s6, s1, s2, s4); sha512_neon_round2(42, s5, &gh, &ab, &cd, &ef); s6 = sha512_neon_schedule_update(s6, s7, s2, s3, s5); sha512_neon_round2(44, s6, &ef, &gh, &ab, &cd); s7 = sha512_neon_schedule_update(s7, s0, s3, s4, s6); sha512_neon_round2(46, s7, &cd, &ef, &gh, &ab); s0 = sha512_neon_schedule_update(s0, s1, s4, s5, s7); sha512_neon_round2(48, s0, &ab, &cd, &ef, &gh); s1 = sha512_neon_schedule_update(s1, s2, s5, s6, s0); sha512_neon_round2(50, s1, &gh, &ab, &cd, &ef); s2 = sha512_neon_schedule_update(s2, s3, s6, s7, s1); sha512_neon_round2(52, s2, &ef, &gh, &ab, &cd); s3 = sha512_neon_schedule_update(s3, s4, s7, s0, s2); sha512_neon_round2(54, s3, &cd, &ef, &gh, &ab); s4 = sha512_neon_schedule_update(s4, s5, s0, s1, s3); sha512_neon_round2(56, s4, &ab, &cd, &ef, &gh); s5 = sha512_neon_schedule_update(s5, s6, s1, s2, s4); sha512_neon_round2(58, s5, &gh, &ab, &cd, &ef); s6 = sha512_neon_schedule_update(s6, s7, s2, s3, s5); sha512_neon_round2(60, s6, &ef, &gh, &ab, &cd); s7 = sha512_neon_schedule_update(s7, s0, s3, s4, s6); sha512_neon_round2(62, s7, &cd, &ef, &gh, &ab); s0 = sha512_neon_schedule_update(s0, s1, s4, s5, s7); sha512_neon_round2(64, s0, &ab, &cd, &ef, &gh); s1 = sha512_neon_schedule_update(s1, s2, s5, s6, s0); sha512_neon_round2(66, s1, &gh, &ab, &cd, &ef); s2 = sha512_neon_schedule_update(s2, s3, s6, s7, s1); sha512_neon_round2(68, s2, &ef, &gh, &ab, &cd); s3 = sha512_neon_schedule_update(s3, s4, s7, s0, s2); sha512_neon_round2(70, s3, &cd, &ef, &gh, &ab); s4 = sha512_neon_schedule_update(s4, s5, s0, s1, s3); sha512_neon_round2(72, s4, &ab, &cd, &ef, &gh); s5 = sha512_neon_schedule_update(s5, s6, s1, s2, s4); sha512_neon_round2(74, s5, &gh, &ab, &cd, &ef); s6 = sha512_neon_schedule_update(s6, s7, s2, s3, s5); sha512_neon_round2(76, s6, &ef, &gh, &ab, &cd); s7 = sha512_neon_schedule_update(s7, s0, s3, s4, s6); sha512_neon_round2(78, s7, &cd, &ef, &gh, &ab); core->ab = vaddq_u64(core->ab, ab); core->cd = vaddq_u64(core->cd, cd); core->ef = vaddq_u64(core->ef, ef); core->gh = vaddq_u64(core->gh, gh); } typedef struct sha512_neon { sha512_neon_core core; sha512_block blk; BinarySink_IMPLEMENTATION; ssh_hash hash; } sha512_neon; static void sha512_neon_write(BinarySink *bs, const void *vp, size_t len); static ssh_hash *sha512_neon_new(const ssh_hashalg *alg) { const struct sha512_extra *extra = (const struct sha512_extra *)alg->extra; if (!check_availability(extra)) return NULL; sha512_neon *s = snew(sha512_neon); s->hash.vt = alg; BinarySink_INIT(s, sha512_neon_write); BinarySink_DELEGATE_INIT(&s->hash, s); return &s->hash; } static void sha512_neon_reset(ssh_hash *hash) { sha512_neon *s = container_of(hash, sha512_neon, hash); const struct sha512_extra *extra = (const struct sha512_extra *)hash->vt->extra; s->core.ab = vld1q_u64(extra->initial_state); s->core.cd = vld1q_u64(extra->initial_state+2); s->core.ef = vld1q_u64(extra->initial_state+4); s->core.gh = vld1q_u64(extra->initial_state+6); sha512_block_setup(&s->blk); } static void sha512_neon_copyfrom(ssh_hash *hcopy, ssh_hash *horig) { sha512_neon *copy = container_of(hcopy, sha512_neon, hash); sha512_neon *orig = container_of(horig, sha512_neon, hash); *copy = *orig; /* structure copy */ BinarySink_COPIED(copy); BinarySink_DELEGATE_INIT(©->hash, copy); } static void sha512_neon_free(ssh_hash *hash) { sha512_neon *s = container_of(hash, sha512_neon, hash); smemclr(s, sizeof(*s)); sfree(s); } static void sha512_neon_write(BinarySink *bs, const void *vp, size_t len) { sha512_neon *s = BinarySink_DOWNCAST(bs, sha512_neon); while (len > 0) if (sha512_block_write(&s->blk, &vp, &len)) sha512_neon_block(&s->core, s->blk.block); } static void sha512_neon_digest(ssh_hash *hash, uint8_t *digest) { sha512_neon *s = container_of(hash, sha512_neon, hash); sha512_block_pad(&s->blk, BinarySink_UPCAST(s)); vst1q_u8(digest, vrev64q_u8(vreinterpretq_u8_u64(s->core.ab))); vst1q_u8(digest+16, vrev64q_u8(vreinterpretq_u8_u64(s->core.cd))); vst1q_u8(digest+32, vrev64q_u8(vreinterpretq_u8_u64(s->core.ef))); vst1q_u8(digest+48, vrev64q_u8(vreinterpretq_u8_u64(s->core.gh))); } static void sha384_neon_digest(ssh_hash *hash, uint8_t *digest) { sha512_neon *s = container_of(hash, sha512_neon, hash); sha512_block_pad(&s->blk, BinarySink_UPCAST(s)); vst1q_u8(digest, vrev64q_u8(vreinterpretq_u8_u64(s->core.ab))); vst1q_u8(digest+16, vrev64q_u8(vreinterpretq_u8_u64(s->core.cd))); vst1q_u8(digest+32, vrev64q_u8(vreinterpretq_u8_u64(s->core.ef))); } SHA512_VTABLES(neon, "NEON accelerated");