blake2s-core.S 9.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306
  1. /* SPDX-License-Identifier: GPL-2.0-or-later */
  2. /*
  3. * BLAKE2s digest algorithm, ARM scalar implementation
  4. *
  5. * Copyright 2020 Google LLC
  6. *
  7. * Author: Eric Biggers <ebiggers@google.com>
  8. */
  9. #include <linux/linkage.h>
  10. #include <asm/assembler.h>
  11. // Registers used to hold message words temporarily. There aren't
  12. // enough ARM registers to hold the whole message block, so we have to
  13. // load the words on-demand.
  14. M_0 .req r12
  15. M_1 .req r14
  16. // The BLAKE2s initialization vector
  17. .Lblake2s_IV:
  18. .word 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A
  19. .word 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19
  20. .macro __ldrd a, b, src, offset
  21. #if __LINUX_ARM_ARCH__ >= 6
  22. ldrd \a, \b, [\src, #\offset]
  23. #else
  24. ldr \a, [\src, #\offset]
  25. ldr \b, [\src, #\offset + 4]
  26. #endif
  27. .endm
  28. .macro __strd a, b, dst, offset
  29. #if __LINUX_ARM_ARCH__ >= 6
  30. strd \a, \b, [\dst, #\offset]
  31. #else
  32. str \a, [\dst, #\offset]
  33. str \b, [\dst, #\offset + 4]
  34. #endif
  35. .endm
  36. .macro _le32_bswap a, tmp
  37. #ifdef __ARMEB__
  38. rev_l \a, \tmp
  39. #endif
  40. .endm
  41. .macro _le32_bswap_8x a, b, c, d, e, f, g, h, tmp
  42. _le32_bswap \a, \tmp
  43. _le32_bswap \b, \tmp
  44. _le32_bswap \c, \tmp
  45. _le32_bswap \d, \tmp
  46. _le32_bswap \e, \tmp
  47. _le32_bswap \f, \tmp
  48. _le32_bswap \g, \tmp
  49. _le32_bswap \h, \tmp
  50. .endm
  51. // Execute a quarter-round of BLAKE2s by mixing two columns or two diagonals.
  52. // (a0, b0, c0, d0) and (a1, b1, c1, d1) give the registers containing the two
  53. // columns/diagonals. s0-s1 are the word offsets to the message words the first
  54. // column/diagonal needs, and likewise s2-s3 for the second column/diagonal.
  55. // M_0 and M_1 are free to use, and the message block can be found at sp + 32.
  56. //
  57. // Note that to save instructions, the rotations don't happen when the
  58. // pseudocode says they should, but rather they are delayed until the values are
  59. // used. See the comment above _blake2s_round().
  60. .macro _blake2s_quarterround a0, b0, c0, d0, a1, b1, c1, d1, s0, s1, s2, s3
  61. ldr M_0, [sp, #32 + 4 * \s0]
  62. ldr M_1, [sp, #32 + 4 * \s2]
  63. // a += b + m[blake2s_sigma[r][2*i + 0]];
  64. add \a0, \a0, \b0, ror #brot
  65. add \a1, \a1, \b1, ror #brot
  66. add \a0, \a0, M_0
  67. add \a1, \a1, M_1
  68. // d = ror32(d ^ a, 16);
  69. eor \d0, \a0, \d0, ror #drot
  70. eor \d1, \a1, \d1, ror #drot
  71. // c += d;
  72. add \c0, \c0, \d0, ror #16
  73. add \c1, \c1, \d1, ror #16
  74. // b = ror32(b ^ c, 12);
  75. eor \b0, \c0, \b0, ror #brot
  76. eor \b1, \c1, \b1, ror #brot
  77. ldr M_0, [sp, #32 + 4 * \s1]
  78. ldr M_1, [sp, #32 + 4 * \s3]
  79. // a += b + m[blake2s_sigma[r][2*i + 1]];
  80. add \a0, \a0, \b0, ror #12
  81. add \a1, \a1, \b1, ror #12
  82. add \a0, \a0, M_0
  83. add \a1, \a1, M_1
  84. // d = ror32(d ^ a, 8);
  85. eor \d0, \a0, \d0, ror#16
  86. eor \d1, \a1, \d1, ror#16
  87. // c += d;
  88. add \c0, \c0, \d0, ror#8
  89. add \c1, \c1, \d1, ror#8
  90. // b = ror32(b ^ c, 7);
  91. eor \b0, \c0, \b0, ror#12
  92. eor \b1, \c1, \b1, ror#12
  93. .endm
  94. // Execute one round of BLAKE2s by updating the state matrix v[0..15]. v[0..9]
  95. // are in r0..r9. The stack pointer points to 8 bytes of scratch space for
  96. // spilling v[8..9], then to v[9..15], then to the message block. r10-r12 and
  97. // r14 are free to use. The macro arguments s0-s15 give the order in which the
  98. // message words are used in this round.
  99. //
  100. // All rotates are performed using the implicit rotate operand accepted by the
  101. // 'add' and 'eor' instructions. This is faster than using explicit rotate
  102. // instructions. To make this work, we allow the values in the second and last
  103. // rows of the BLAKE2s state matrix (rows 'b' and 'd') to temporarily have the
  104. // wrong rotation amount. The rotation amount is then fixed up just in time
  105. // when the values are used. 'brot' is the number of bits the values in row 'b'
  106. // need to be rotated right to arrive at the correct values, and 'drot'
  107. // similarly for row 'd'. (brot, drot) start out as (0, 0) but we make it such
  108. // that they end up as (7, 8) after every round.
  109. .macro _blake2s_round s0, s1, s2, s3, s4, s5, s6, s7, \
  110. s8, s9, s10, s11, s12, s13, s14, s15
  111. // Mix first two columns:
  112. // (v[0], v[4], v[8], v[12]) and (v[1], v[5], v[9], v[13]).
  113. __ldrd r10, r11, sp, 16 // load v[12] and v[13]
  114. _blake2s_quarterround r0, r4, r8, r10, r1, r5, r9, r11, \
  115. \s0, \s1, \s2, \s3
  116. __strd r8, r9, sp, 0
  117. __strd r10, r11, sp, 16
  118. // Mix second two columns:
  119. // (v[2], v[6], v[10], v[14]) and (v[3], v[7], v[11], v[15]).
  120. __ldrd r8, r9, sp, 8 // load v[10] and v[11]
  121. __ldrd r10, r11, sp, 24 // load v[14] and v[15]
  122. _blake2s_quarterround r2, r6, r8, r10, r3, r7, r9, r11, \
  123. \s4, \s5, \s6, \s7
  124. str r10, [sp, #24] // store v[14]
  125. // v[10], v[11], and v[15] are used below, so no need to store them yet.
  126. .set brot, 7
  127. .set drot, 8
  128. // Mix first two diagonals:
  129. // (v[0], v[5], v[10], v[15]) and (v[1], v[6], v[11], v[12]).
  130. ldr r10, [sp, #16] // load v[12]
  131. _blake2s_quarterround r0, r5, r8, r11, r1, r6, r9, r10, \
  132. \s8, \s9, \s10, \s11
  133. __strd r8, r9, sp, 8
  134. str r11, [sp, #28]
  135. str r10, [sp, #16]
  136. // Mix second two diagonals:
  137. // (v[2], v[7], v[8], v[13]) and (v[3], v[4], v[9], v[14]).
  138. __ldrd r8, r9, sp, 0 // load v[8] and v[9]
  139. __ldrd r10, r11, sp, 20 // load v[13] and v[14]
  140. _blake2s_quarterround r2, r7, r8, r10, r3, r4, r9, r11, \
  141. \s12, \s13, \s14, \s15
  142. __strd r10, r11, sp, 20
  143. .endm
  144. //
  145. // void blake2s_compress_arch(struct blake2s_state *state,
  146. // const u8 *block, size_t nblocks, u32 inc);
  147. //
  148. // Only the first three fields of struct blake2s_state are used:
  149. // u32 h[8]; (inout)
  150. // u32 t[2]; (inout)
  151. // u32 f[2]; (in)
  152. //
  153. .align 5
  154. ENTRY(blake2s_compress_arch)
  155. push {r0-r2,r4-r11,lr} // keep this an even number
  156. .Lnext_block:
  157. // r0 is 'state'
  158. // r1 is 'block'
  159. // r3 is 'inc'
  160. // Load and increment the counter t[0..1].
  161. __ldrd r10, r11, r0, 32
  162. adds r10, r10, r3
  163. adc r11, r11, #0
  164. __strd r10, r11, r0, 32
  165. // _blake2s_round is very short on registers, so copy the message block
  166. // to the stack to save a register during the rounds. This also has the
  167. // advantage that misalignment only needs to be dealt with in one place.
  168. sub sp, sp, #64
  169. mov r12, sp
  170. tst r1, #3
  171. bne .Lcopy_block_misaligned
  172. ldmia r1!, {r2-r9}
  173. _le32_bswap_8x r2, r3, r4, r5, r6, r7, r8, r9, r14
  174. stmia r12!, {r2-r9}
  175. ldmia r1!, {r2-r9}
  176. _le32_bswap_8x r2, r3, r4, r5, r6, r7, r8, r9, r14
  177. stmia r12, {r2-r9}
  178. .Lcopy_block_done:
  179. str r1, [sp, #68] // Update message pointer
  180. // Calculate v[8..15]. Push v[9..15] onto the stack, and leave space
  181. // for spilling v[8..9]. Leave v[8..9] in r8-r9.
  182. mov r14, r0 // r14 = state
  183. adr r12, .Lblake2s_IV
  184. ldmia r12!, {r8-r9} // load IV[0..1]
  185. __ldrd r0, r1, r14, 40 // load f[0..1]
  186. ldm r12, {r2-r7} // load IV[3..7]
  187. eor r4, r4, r10 // v[12] = IV[4] ^ t[0]
  188. eor r5, r5, r11 // v[13] = IV[5] ^ t[1]
  189. eor r6, r6, r0 // v[14] = IV[6] ^ f[0]
  190. eor r7, r7, r1 // v[15] = IV[7] ^ f[1]
  191. push {r2-r7} // push v[9..15]
  192. sub sp, sp, #8 // leave space for v[8..9]
  193. // Load h[0..7] == v[0..7].
  194. ldm r14, {r0-r7}
  195. // Execute the rounds. Each round is provided the order in which it
  196. // needs to use the message words.
  197. .set brot, 0
  198. .set drot, 0
  199. _blake2s_round 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
  200. _blake2s_round 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3
  201. _blake2s_round 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4
  202. _blake2s_round 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8
  203. _blake2s_round 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13
  204. _blake2s_round 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9
  205. _blake2s_round 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11
  206. _blake2s_round 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10
  207. _blake2s_round 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5
  208. _blake2s_round 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0
  209. // Fold the final state matrix into the hash chaining value:
  210. //
  211. // for (i = 0; i < 8; i++)
  212. // h[i] ^= v[i] ^ v[i + 8];
  213. //
  214. ldr r14, [sp, #96] // r14 = &h[0]
  215. add sp, sp, #8 // v[8..9] are already loaded.
  216. pop {r10-r11} // load v[10..11]
  217. eor r0, r0, r8
  218. eor r1, r1, r9
  219. eor r2, r2, r10
  220. eor r3, r3, r11
  221. ldm r14, {r8-r11} // load h[0..3]
  222. eor r0, r0, r8
  223. eor r1, r1, r9
  224. eor r2, r2, r10
  225. eor r3, r3, r11
  226. stmia r14!, {r0-r3} // store new h[0..3]
  227. ldm r14, {r0-r3} // load old h[4..7]
  228. pop {r8-r11} // load v[12..15]
  229. eor r0, r0, r4, ror #brot
  230. eor r1, r1, r5, ror #brot
  231. eor r2, r2, r6, ror #brot
  232. eor r3, r3, r7, ror #brot
  233. eor r0, r0, r8, ror #drot
  234. eor r1, r1, r9, ror #drot
  235. eor r2, r2, r10, ror #drot
  236. eor r3, r3, r11, ror #drot
  237. add sp, sp, #64 // skip copy of message block
  238. stm r14, {r0-r3} // store new h[4..7]
  239. // Advance to the next block, if there is one. Note that if there are
  240. // multiple blocks, then 'inc' (the counter increment amount) must be
  241. // 64. So we can simply set it to 64 without re-loading it.
  242. ldm sp, {r0, r1, r2} // load (state, block, nblocks)
  243. mov r3, #64 // set 'inc'
  244. subs r2, r2, #1 // nblocks--
  245. str r2, [sp, #8]
  246. bne .Lnext_block // nblocks != 0?
  247. pop {r0-r2,r4-r11,pc}
  248. // The next message block (pointed to by r1) isn't 4-byte aligned, so it
  249. // can't be loaded using ldmia. Copy it to the stack buffer (pointed to
  250. // by r12) using an alternative method. r2-r9 are free to use.
  251. .Lcopy_block_misaligned:
  252. mov r2, #64
  253. 1:
  254. #ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
  255. ldr r3, [r1], #4
  256. _le32_bswap r3, r4
  257. #else
  258. ldrb r3, [r1, #0]
  259. ldrb r4, [r1, #1]
  260. ldrb r5, [r1, #2]
  261. ldrb r6, [r1, #3]
  262. add r1, r1, #4
  263. orr r3, r3, r4, lsl #8
  264. orr r3, r3, r5, lsl #16
  265. orr r3, r3, r6, lsl #24
  266. #endif
  267. subs r2, r2, #4
  268. str r3, [r12], #4
  269. bne 1b
  270. b .Lcopy_block_done
  271. ENDPROC(blake2s_compress_arch)