ghash-ce-glue.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Accelerated GHASH implementation with ARMv8 PMULL instructions.
  4. *
  5. * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
  6. */
  7. #include <asm/neon.h>
  8. #include <asm/simd.h>
  9. #include <asm/unaligned.h>
  10. #include <crypto/aes.h>
  11. #include <crypto/algapi.h>
  12. #include <crypto/b128ops.h>
  13. #include <crypto/gf128mul.h>
  14. #include <crypto/internal/aead.h>
  15. #include <crypto/internal/hash.h>
  16. #include <crypto/internal/simd.h>
  17. #include <crypto/internal/skcipher.h>
  18. #include <crypto/scatterwalk.h>
  19. #include <linux/cpufeature.h>
  20. #include <linux/crypto.h>
  21. #include <linux/module.h>
  22. MODULE_DESCRIPTION("GHASH and AES-GCM using ARMv8 Crypto Extensions");
  23. MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
  24. MODULE_LICENSE("GPL v2");
  25. MODULE_ALIAS_CRYPTO("ghash");
  26. #define GHASH_BLOCK_SIZE 16
  27. #define GHASH_DIGEST_SIZE 16
  28. #define GCM_IV_SIZE 12
  29. struct ghash_key {
  30. be128 k;
  31. u64 h[][2];
  32. };
  33. struct ghash_desc_ctx {
  34. u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
  35. u8 buf[GHASH_BLOCK_SIZE];
  36. u32 count;
  37. };
  38. struct gcm_aes_ctx {
  39. struct crypto_aes_ctx aes_key;
  40. struct ghash_key ghash_key;
  41. };
  42. asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
  43. u64 const h[][2], const char *head);
  44. asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src,
  45. u64 const h[][2], const char *head);
  46. asmlinkage void pmull_gcm_encrypt(int bytes, u8 dst[], const u8 src[],
  47. u64 const h[][2], u64 dg[], u8 ctr[],
  48. u32 const rk[], int rounds, u8 tag[]);
  49. asmlinkage int pmull_gcm_decrypt(int bytes, u8 dst[], const u8 src[],
  50. u64 const h[][2], u64 dg[], u8 ctr[],
  51. u32 const rk[], int rounds, const u8 l[],
  52. const u8 tag[], u64 authsize);
  53. static int ghash_init(struct shash_desc *desc)
  54. {
  55. struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
  56. *ctx = (struct ghash_desc_ctx){};
  57. return 0;
  58. }
  59. static void ghash_do_update(int blocks, u64 dg[], const char *src,
  60. struct ghash_key *key, const char *head)
  61. {
  62. be128 dst = { cpu_to_be64(dg[1]), cpu_to_be64(dg[0]) };
  63. do {
  64. const u8 *in = src;
  65. if (head) {
  66. in = head;
  67. blocks++;
  68. head = NULL;
  69. } else {
  70. src += GHASH_BLOCK_SIZE;
  71. }
  72. crypto_xor((u8 *)&dst, in, GHASH_BLOCK_SIZE);
  73. gf128mul_lle(&dst, &key->k);
  74. } while (--blocks);
  75. dg[0] = be64_to_cpu(dst.b);
  76. dg[1] = be64_to_cpu(dst.a);
  77. }
  78. static __always_inline
  79. void ghash_do_simd_update(int blocks, u64 dg[], const char *src,
  80. struct ghash_key *key, const char *head,
  81. void (*simd_update)(int blocks, u64 dg[],
  82. const char *src,
  83. u64 const h[][2],
  84. const char *head))
  85. {
  86. if (likely(crypto_simd_usable())) {
  87. kernel_neon_begin();
  88. simd_update(blocks, dg, src, key->h, head);
  89. kernel_neon_end();
  90. } else {
  91. ghash_do_update(blocks, dg, src, key, head);
  92. }
  93. }
  94. /* avoid hogging the CPU for too long */
  95. #define MAX_BLOCKS (SZ_64K / GHASH_BLOCK_SIZE)
  96. static int ghash_update(struct shash_desc *desc, const u8 *src,
  97. unsigned int len)
  98. {
  99. struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
  100. unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
  101. ctx->count += len;
  102. if ((partial + len) >= GHASH_BLOCK_SIZE) {
  103. struct ghash_key *key = crypto_shash_ctx(desc->tfm);
  104. int blocks;
  105. if (partial) {
  106. int p = GHASH_BLOCK_SIZE - partial;
  107. memcpy(ctx->buf + partial, src, p);
  108. src += p;
  109. len -= p;
  110. }
  111. blocks = len / GHASH_BLOCK_SIZE;
  112. len %= GHASH_BLOCK_SIZE;
  113. do {
  114. int chunk = min(blocks, MAX_BLOCKS);
  115. ghash_do_simd_update(chunk, ctx->digest, src, key,
  116. partial ? ctx->buf : NULL,
  117. pmull_ghash_update_p8);
  118. blocks -= chunk;
  119. src += chunk * GHASH_BLOCK_SIZE;
  120. partial = 0;
  121. } while (unlikely(blocks > 0));
  122. }
  123. if (len)
  124. memcpy(ctx->buf + partial, src, len);
  125. return 0;
  126. }
  127. static int ghash_final(struct shash_desc *desc, u8 *dst)
  128. {
  129. struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
  130. unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
  131. if (partial) {
  132. struct ghash_key *key = crypto_shash_ctx(desc->tfm);
  133. memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial);
  134. ghash_do_simd_update(1, ctx->digest, ctx->buf, key, NULL,
  135. pmull_ghash_update_p8);
  136. }
  137. put_unaligned_be64(ctx->digest[1], dst);
  138. put_unaligned_be64(ctx->digest[0], dst + 8);
  139. *ctx = (struct ghash_desc_ctx){};
  140. return 0;
  141. }
  142. static void ghash_reflect(u64 h[], const be128 *k)
  143. {
  144. u64 carry = be64_to_cpu(k->a) & BIT(63) ? 1 : 0;
  145. h[0] = (be64_to_cpu(k->b) << 1) | carry;
  146. h[1] = (be64_to_cpu(k->a) << 1) | (be64_to_cpu(k->b) >> 63);
  147. if (carry)
  148. h[1] ^= 0xc200000000000000UL;
  149. }
  150. static int ghash_setkey(struct crypto_shash *tfm,
  151. const u8 *inkey, unsigned int keylen)
  152. {
  153. struct ghash_key *key = crypto_shash_ctx(tfm);
  154. if (keylen != GHASH_BLOCK_SIZE)
  155. return -EINVAL;
  156. /* needed for the fallback */
  157. memcpy(&key->k, inkey, GHASH_BLOCK_SIZE);
  158. ghash_reflect(key->h[0], &key->k);
  159. return 0;
  160. }
  161. static struct shash_alg ghash_alg = {
  162. .base.cra_name = "ghash",
  163. .base.cra_driver_name = "ghash-neon",
  164. .base.cra_priority = 150,
  165. .base.cra_blocksize = GHASH_BLOCK_SIZE,
  166. .base.cra_ctxsize = sizeof(struct ghash_key) + sizeof(u64[2]),
  167. .base.cra_module = THIS_MODULE,
  168. .digestsize = GHASH_DIGEST_SIZE,
  169. .init = ghash_init,
  170. .update = ghash_update,
  171. .final = ghash_final,
  172. .setkey = ghash_setkey,
  173. .descsize = sizeof(struct ghash_desc_ctx),
  174. };
  175. static int num_rounds(struct crypto_aes_ctx *ctx)
  176. {
  177. /*
  178. * # of rounds specified by AES:
  179. * 128 bit key 10 rounds
  180. * 192 bit key 12 rounds
  181. * 256 bit key 14 rounds
  182. * => n byte key => 6 + (n/4) rounds
  183. */
  184. return 6 + ctx->key_length / 4;
  185. }
  186. static int gcm_setkey(struct crypto_aead *tfm, const u8 *inkey,
  187. unsigned int keylen)
  188. {
  189. struct gcm_aes_ctx *ctx = crypto_aead_ctx(tfm);
  190. u8 key[GHASH_BLOCK_SIZE];
  191. be128 h;
  192. int ret;
  193. ret = aes_expandkey(&ctx->aes_key, inkey, keylen);
  194. if (ret)
  195. return -EINVAL;
  196. aes_encrypt(&ctx->aes_key, key, (u8[AES_BLOCK_SIZE]){});
  197. /* needed for the fallback */
  198. memcpy(&ctx->ghash_key.k, key, GHASH_BLOCK_SIZE);
  199. ghash_reflect(ctx->ghash_key.h[0], &ctx->ghash_key.k);
  200. h = ctx->ghash_key.k;
  201. gf128mul_lle(&h, &ctx->ghash_key.k);
  202. ghash_reflect(ctx->ghash_key.h[1], &h);
  203. gf128mul_lle(&h, &ctx->ghash_key.k);
  204. ghash_reflect(ctx->ghash_key.h[2], &h);
  205. gf128mul_lle(&h, &ctx->ghash_key.k);
  206. ghash_reflect(ctx->ghash_key.h[3], &h);
  207. return 0;
  208. }
  209. static int gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
  210. {
  211. switch (authsize) {
  212. case 4:
  213. case 8:
  214. case 12 ... 16:
  215. break;
  216. default:
  217. return -EINVAL;
  218. }
  219. return 0;
  220. }
  221. static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[],
  222. int *buf_count, struct gcm_aes_ctx *ctx)
  223. {
  224. if (*buf_count > 0) {
  225. int buf_added = min(count, GHASH_BLOCK_SIZE - *buf_count);
  226. memcpy(&buf[*buf_count], src, buf_added);
  227. *buf_count += buf_added;
  228. src += buf_added;
  229. count -= buf_added;
  230. }
  231. if (count >= GHASH_BLOCK_SIZE || *buf_count == GHASH_BLOCK_SIZE) {
  232. int blocks = count / GHASH_BLOCK_SIZE;
  233. ghash_do_simd_update(blocks, dg, src, &ctx->ghash_key,
  234. *buf_count ? buf : NULL,
  235. pmull_ghash_update_p64);
  236. src += blocks * GHASH_BLOCK_SIZE;
  237. count %= GHASH_BLOCK_SIZE;
  238. *buf_count = 0;
  239. }
  240. if (count > 0) {
  241. memcpy(buf, src, count);
  242. *buf_count = count;
  243. }
  244. }
  245. static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[])
  246. {
  247. struct crypto_aead *aead = crypto_aead_reqtfm(req);
  248. struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
  249. u8 buf[GHASH_BLOCK_SIZE];
  250. struct scatter_walk walk;
  251. u32 len = req->assoclen;
  252. int buf_count = 0;
  253. scatterwalk_start(&walk, req->src);
  254. do {
  255. u32 n = scatterwalk_clamp(&walk, len);
  256. u8 *p;
  257. if (!n) {
  258. scatterwalk_start(&walk, sg_next(walk.sg));
  259. n = scatterwalk_clamp(&walk, len);
  260. }
  261. p = scatterwalk_map(&walk);
  262. gcm_update_mac(dg, p, n, buf, &buf_count, ctx);
  263. len -= n;
  264. scatterwalk_unmap(p);
  265. scatterwalk_advance(&walk, n);
  266. scatterwalk_done(&walk, 0, len);
  267. } while (len);
  268. if (buf_count) {
  269. memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count);
  270. ghash_do_simd_update(1, dg, buf, &ctx->ghash_key, NULL,
  271. pmull_ghash_update_p64);
  272. }
  273. }
  274. static int gcm_encrypt(struct aead_request *req)
  275. {
  276. struct crypto_aead *aead = crypto_aead_reqtfm(req);
  277. struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
  278. int nrounds = num_rounds(&ctx->aes_key);
  279. struct skcipher_walk walk;
  280. u8 buf[AES_BLOCK_SIZE];
  281. u8 iv[AES_BLOCK_SIZE];
  282. u64 dg[2] = {};
  283. be128 lengths;
  284. u8 *tag;
  285. int err;
  286. lengths.a = cpu_to_be64(req->assoclen * 8);
  287. lengths.b = cpu_to_be64(req->cryptlen * 8);
  288. if (req->assoclen)
  289. gcm_calculate_auth_mac(req, dg);
  290. memcpy(iv, req->iv, GCM_IV_SIZE);
  291. put_unaligned_be32(2, iv + GCM_IV_SIZE);
  292. err = skcipher_walk_aead_encrypt(&walk, req, false);
  293. if (likely(crypto_simd_usable())) {
  294. do {
  295. const u8 *src = walk.src.virt.addr;
  296. u8 *dst = walk.dst.virt.addr;
  297. int nbytes = walk.nbytes;
  298. tag = (u8 *)&lengths;
  299. if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) {
  300. src = dst = memcpy(buf + sizeof(buf) - nbytes,
  301. src, nbytes);
  302. } else if (nbytes < walk.total) {
  303. nbytes &= ~(AES_BLOCK_SIZE - 1);
  304. tag = NULL;
  305. }
  306. kernel_neon_begin();
  307. pmull_gcm_encrypt(nbytes, dst, src, ctx->ghash_key.h,
  308. dg, iv, ctx->aes_key.key_enc, nrounds,
  309. tag);
  310. kernel_neon_end();
  311. if (unlikely(!nbytes))
  312. break;
  313. if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE))
  314. memcpy(walk.dst.virt.addr,
  315. buf + sizeof(buf) - nbytes, nbytes);
  316. err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
  317. } while (walk.nbytes);
  318. } else {
  319. while (walk.nbytes >= AES_BLOCK_SIZE) {
  320. int blocks = walk.nbytes / AES_BLOCK_SIZE;
  321. const u8 *src = walk.src.virt.addr;
  322. u8 *dst = walk.dst.virt.addr;
  323. int remaining = blocks;
  324. do {
  325. aes_encrypt(&ctx->aes_key, buf, iv);
  326. crypto_xor_cpy(dst, src, buf, AES_BLOCK_SIZE);
  327. crypto_inc(iv, AES_BLOCK_SIZE);
  328. dst += AES_BLOCK_SIZE;
  329. src += AES_BLOCK_SIZE;
  330. } while (--remaining > 0);
  331. ghash_do_update(blocks, dg, walk.dst.virt.addr,
  332. &ctx->ghash_key, NULL);
  333. err = skcipher_walk_done(&walk,
  334. walk.nbytes % AES_BLOCK_SIZE);
  335. }
  336. /* handle the tail */
  337. if (walk.nbytes) {
  338. aes_encrypt(&ctx->aes_key, buf, iv);
  339. crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr,
  340. buf, walk.nbytes);
  341. memcpy(buf, walk.dst.virt.addr, walk.nbytes);
  342. memset(buf + walk.nbytes, 0, sizeof(buf) - walk.nbytes);
  343. }
  344. tag = (u8 *)&lengths;
  345. ghash_do_update(1, dg, tag, &ctx->ghash_key,
  346. walk.nbytes ? buf : NULL);
  347. if (walk.nbytes)
  348. err = skcipher_walk_done(&walk, 0);
  349. put_unaligned_be64(dg[1], tag);
  350. put_unaligned_be64(dg[0], tag + 8);
  351. put_unaligned_be32(1, iv + GCM_IV_SIZE);
  352. aes_encrypt(&ctx->aes_key, iv, iv);
  353. crypto_xor(tag, iv, AES_BLOCK_SIZE);
  354. }
  355. if (err)
  356. return err;
  357. /* copy authtag to end of dst */
  358. scatterwalk_map_and_copy(tag, req->dst, req->assoclen + req->cryptlen,
  359. crypto_aead_authsize(aead), 1);
  360. return 0;
  361. }
  362. static int gcm_decrypt(struct aead_request *req)
  363. {
  364. struct crypto_aead *aead = crypto_aead_reqtfm(req);
  365. struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
  366. unsigned int authsize = crypto_aead_authsize(aead);
  367. int nrounds = num_rounds(&ctx->aes_key);
  368. struct skcipher_walk walk;
  369. u8 otag[AES_BLOCK_SIZE];
  370. u8 buf[AES_BLOCK_SIZE];
  371. u8 iv[AES_BLOCK_SIZE];
  372. u64 dg[2] = {};
  373. be128 lengths;
  374. u8 *tag;
  375. int err;
  376. lengths.a = cpu_to_be64(req->assoclen * 8);
  377. lengths.b = cpu_to_be64((req->cryptlen - authsize) * 8);
  378. if (req->assoclen)
  379. gcm_calculate_auth_mac(req, dg);
  380. memcpy(iv, req->iv, GCM_IV_SIZE);
  381. put_unaligned_be32(2, iv + GCM_IV_SIZE);
  382. scatterwalk_map_and_copy(otag, req->src,
  383. req->assoclen + req->cryptlen - authsize,
  384. authsize, 0);
  385. err = skcipher_walk_aead_decrypt(&walk, req, false);
  386. if (likely(crypto_simd_usable())) {
  387. int ret;
  388. do {
  389. const u8 *src = walk.src.virt.addr;
  390. u8 *dst = walk.dst.virt.addr;
  391. int nbytes = walk.nbytes;
  392. tag = (u8 *)&lengths;
  393. if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) {
  394. src = dst = memcpy(buf + sizeof(buf) - nbytes,
  395. src, nbytes);
  396. } else if (nbytes < walk.total) {
  397. nbytes &= ~(AES_BLOCK_SIZE - 1);
  398. tag = NULL;
  399. }
  400. kernel_neon_begin();
  401. ret = pmull_gcm_decrypt(nbytes, dst, src,
  402. ctx->ghash_key.h,
  403. dg, iv, ctx->aes_key.key_enc,
  404. nrounds, tag, otag, authsize);
  405. kernel_neon_end();
  406. if (unlikely(!nbytes))
  407. break;
  408. if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE))
  409. memcpy(walk.dst.virt.addr,
  410. buf + sizeof(buf) - nbytes, nbytes);
  411. err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
  412. } while (walk.nbytes);
  413. if (err)
  414. return err;
  415. if (ret)
  416. return -EBADMSG;
  417. } else {
  418. while (walk.nbytes >= AES_BLOCK_SIZE) {
  419. int blocks = walk.nbytes / AES_BLOCK_SIZE;
  420. const u8 *src = walk.src.virt.addr;
  421. u8 *dst = walk.dst.virt.addr;
  422. ghash_do_update(blocks, dg, walk.src.virt.addr,
  423. &ctx->ghash_key, NULL);
  424. do {
  425. aes_encrypt(&ctx->aes_key, buf, iv);
  426. crypto_xor_cpy(dst, src, buf, AES_BLOCK_SIZE);
  427. crypto_inc(iv, AES_BLOCK_SIZE);
  428. dst += AES_BLOCK_SIZE;
  429. src += AES_BLOCK_SIZE;
  430. } while (--blocks > 0);
  431. err = skcipher_walk_done(&walk,
  432. walk.nbytes % AES_BLOCK_SIZE);
  433. }
  434. /* handle the tail */
  435. if (walk.nbytes) {
  436. memcpy(buf, walk.src.virt.addr, walk.nbytes);
  437. memset(buf + walk.nbytes, 0, sizeof(buf) - walk.nbytes);
  438. }
  439. tag = (u8 *)&lengths;
  440. ghash_do_update(1, dg, tag, &ctx->ghash_key,
  441. walk.nbytes ? buf : NULL);
  442. if (walk.nbytes) {
  443. aes_encrypt(&ctx->aes_key, buf, iv);
  444. crypto_xor_cpy(walk.dst.virt.addr, walk.src.virt.addr,
  445. buf, walk.nbytes);
  446. err = skcipher_walk_done(&walk, 0);
  447. }
  448. if (err)
  449. return err;
  450. put_unaligned_be64(dg[1], tag);
  451. put_unaligned_be64(dg[0], tag + 8);
  452. put_unaligned_be32(1, iv + GCM_IV_SIZE);
  453. aes_encrypt(&ctx->aes_key, iv, iv);
  454. crypto_xor(tag, iv, AES_BLOCK_SIZE);
  455. if (crypto_memneq(tag, otag, authsize)) {
  456. memzero_explicit(tag, AES_BLOCK_SIZE);
  457. return -EBADMSG;
  458. }
  459. }
  460. return 0;
  461. }
  462. static struct aead_alg gcm_aes_alg = {
  463. .ivsize = GCM_IV_SIZE,
  464. .chunksize = AES_BLOCK_SIZE,
  465. .maxauthsize = AES_BLOCK_SIZE,
  466. .setkey = gcm_setkey,
  467. .setauthsize = gcm_setauthsize,
  468. .encrypt = gcm_encrypt,
  469. .decrypt = gcm_decrypt,
  470. .base.cra_name = "gcm(aes)",
  471. .base.cra_driver_name = "gcm-aes-ce",
  472. .base.cra_priority = 300,
  473. .base.cra_blocksize = 1,
  474. .base.cra_ctxsize = sizeof(struct gcm_aes_ctx) +
  475. 4 * sizeof(u64[2]),
  476. .base.cra_module = THIS_MODULE,
  477. };
  478. static int __init ghash_ce_mod_init(void)
  479. {
  480. if (!cpu_have_named_feature(ASIMD))
  481. return -ENODEV;
  482. if (cpu_have_named_feature(PMULL))
  483. return crypto_register_aead(&gcm_aes_alg);
  484. return crypto_register_shash(&ghash_alg);
  485. }
  486. static void __exit ghash_ce_mod_exit(void)
  487. {
  488. if (cpu_have_named_feature(PMULL))
  489. crypto_unregister_aead(&gcm_aes_alg);
  490. else
  491. crypto_unregister_shash(&ghash_alg);
  492. }
  493. static const struct cpu_feature ghash_cpu_feature[] = {
  494. { cpu_feature(PMULL) }, { }
  495. };
  496. MODULE_DEVICE_TABLE(cpu, ghash_cpu_feature);
  497. module_init(ghash_ce_mod_init);
  498. module_exit(ghash_ce_mod_exit);