crc32c-vpmsum_glue.c 4.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. #include <linux/crc32.h>
  3. #include <crypto/internal/hash.h>
  4. #include <crypto/internal/simd.h>
  5. #include <linux/init.h>
  6. #include <linux/module.h>
  7. #include <linux/string.h>
  8. #include <linux/kernel.h>
  9. #include <linux/cpufeature.h>
  10. #include <asm/simd.h>
  11. #include <asm/switch_to.h>
  12. #define CHKSUM_BLOCK_SIZE 1
  13. #define CHKSUM_DIGEST_SIZE 4
  14. #define VMX_ALIGN 16
  15. #define VMX_ALIGN_MASK (VMX_ALIGN-1)
  16. #define VECTOR_BREAKPOINT 512
  17. u32 __crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len);
  18. static u32 crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len)
  19. {
  20. unsigned int prealign;
  21. unsigned int tail;
  22. if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || !crypto_simd_usable())
  23. return __crc32c_le(crc, p, len);
  24. if ((unsigned long)p & VMX_ALIGN_MASK) {
  25. prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK);
  26. crc = __crc32c_le(crc, p, prealign);
  27. len -= prealign;
  28. p += prealign;
  29. }
  30. if (len & ~VMX_ALIGN_MASK) {
  31. preempt_disable();
  32. pagefault_disable();
  33. enable_kernel_altivec();
  34. crc = __crc32c_vpmsum(crc, p, len & ~VMX_ALIGN_MASK);
  35. disable_kernel_altivec();
  36. pagefault_enable();
  37. preempt_enable();
  38. }
  39. tail = len & VMX_ALIGN_MASK;
  40. if (tail) {
  41. p += len & ~VMX_ALIGN_MASK;
  42. crc = __crc32c_le(crc, p, tail);
  43. }
  44. return crc;
  45. }
  46. static int crc32c_vpmsum_cra_init(struct crypto_tfm *tfm)
  47. {
  48. u32 *key = crypto_tfm_ctx(tfm);
  49. *key = ~0;
  50. return 0;
  51. }
  52. /*
  53. * Setting the seed allows arbitrary accumulators and flexible XOR policy
  54. * If your algorithm starts with ~0, then XOR with ~0 before you set
  55. * the seed.
  56. */
  57. static int crc32c_vpmsum_setkey(struct crypto_shash *hash, const u8 *key,
  58. unsigned int keylen)
  59. {
  60. u32 *mctx = crypto_shash_ctx(hash);
  61. if (keylen != sizeof(u32))
  62. return -EINVAL;
  63. *mctx = le32_to_cpup((__le32 *)key);
  64. return 0;
  65. }
  66. static int crc32c_vpmsum_init(struct shash_desc *desc)
  67. {
  68. u32 *mctx = crypto_shash_ctx(desc->tfm);
  69. u32 *crcp = shash_desc_ctx(desc);
  70. *crcp = *mctx;
  71. return 0;
  72. }
  73. static int crc32c_vpmsum_update(struct shash_desc *desc, const u8 *data,
  74. unsigned int len)
  75. {
  76. u32 *crcp = shash_desc_ctx(desc);
  77. *crcp = crc32c_vpmsum(*crcp, data, len);
  78. return 0;
  79. }
  80. static int __crc32c_vpmsum_finup(u32 *crcp, const u8 *data, unsigned int len,
  81. u8 *out)
  82. {
  83. *(__le32 *)out = ~cpu_to_le32(crc32c_vpmsum(*crcp, data, len));
  84. return 0;
  85. }
  86. static int crc32c_vpmsum_finup(struct shash_desc *desc, const u8 *data,
  87. unsigned int len, u8 *out)
  88. {
  89. return __crc32c_vpmsum_finup(shash_desc_ctx(desc), data, len, out);
  90. }
  91. static int crc32c_vpmsum_final(struct shash_desc *desc, u8 *out)
  92. {
  93. u32 *crcp = shash_desc_ctx(desc);
  94. *(__le32 *)out = ~cpu_to_le32p(crcp);
  95. return 0;
  96. }
  97. static int crc32c_vpmsum_digest(struct shash_desc *desc, const u8 *data,
  98. unsigned int len, u8 *out)
  99. {
  100. return __crc32c_vpmsum_finup(crypto_shash_ctx(desc->tfm), data, len,
  101. out);
  102. }
  103. static struct shash_alg alg = {
  104. .setkey = crc32c_vpmsum_setkey,
  105. .init = crc32c_vpmsum_init,
  106. .update = crc32c_vpmsum_update,
  107. .final = crc32c_vpmsum_final,
  108. .finup = crc32c_vpmsum_finup,
  109. .digest = crc32c_vpmsum_digest,
  110. .descsize = sizeof(u32),
  111. .digestsize = CHKSUM_DIGEST_SIZE,
  112. .base = {
  113. .cra_name = "crc32c",
  114. .cra_driver_name = "crc32c-vpmsum",
  115. .cra_priority = 200,
  116. .cra_flags = CRYPTO_ALG_OPTIONAL_KEY,
  117. .cra_blocksize = CHKSUM_BLOCK_SIZE,
  118. .cra_ctxsize = sizeof(u32),
  119. .cra_module = THIS_MODULE,
  120. .cra_init = crc32c_vpmsum_cra_init,
  121. }
  122. };
  123. static int __init crc32c_vpmsum_mod_init(void)
  124. {
  125. if (!cpu_has_feature(CPU_FTR_ARCH_207S))
  126. return -ENODEV;
  127. return crypto_register_shash(&alg);
  128. }
  129. static void __exit crc32c_vpmsum_mod_fini(void)
  130. {
  131. crypto_unregister_shash(&alg);
  132. }
  133. module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, crc32c_vpmsum_mod_init);
  134. module_exit(crc32c_vpmsum_mod_fini);
  135. MODULE_AUTHOR("Anton Blanchard <anton@samba.org>");
  136. MODULE_DESCRIPTION("CRC32C using vector polynomial multiply-sum instructions");
  137. MODULE_LICENSE("GPL");
  138. MODULE_ALIAS_CRYPTO("crc32c");
  139. MODULE_ALIAS_CRYPTO("crc32c-vpmsum");