recov_neon.c 2.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 2012 Intel Corporation
  4. * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
  5. */
  6. #include <linux/raid/pq.h>
  7. #ifdef __KERNEL__
  8. #include <asm/neon.h>
  9. #else
  10. #define kernel_neon_begin()
  11. #define kernel_neon_end()
  12. #define cpu_has_neon() (1)
  13. #endif
  14. static int raid6_has_neon(void)
  15. {
  16. return cpu_has_neon();
  17. }
  18. void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
  19. uint8_t *dq, const uint8_t *pbmul,
  20. const uint8_t *qmul);
  21. void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
  22. const uint8_t *qmul);
  23. static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
  24. int failb, void **ptrs)
  25. {
  26. u8 *p, *q, *dp, *dq;
  27. const u8 *pbmul; /* P multiplier table for B data */
  28. const u8 *qmul; /* Q multiplier table (for both) */
  29. p = (u8 *)ptrs[disks - 2];
  30. q = (u8 *)ptrs[disks - 1];
  31. /*
  32. * Compute syndrome with zero for the missing data pages
  33. * Use the dead data pages as temporary storage for
  34. * delta p and delta q
  35. */
  36. dp = (u8 *)ptrs[faila];
  37. ptrs[faila] = (void *)raid6_empty_zero_page;
  38. ptrs[disks - 2] = dp;
  39. dq = (u8 *)ptrs[failb];
  40. ptrs[failb] = (void *)raid6_empty_zero_page;
  41. ptrs[disks - 1] = dq;
  42. raid6_call.gen_syndrome(disks, bytes, ptrs);
  43. /* Restore pointer table */
  44. ptrs[faila] = dp;
  45. ptrs[failb] = dq;
  46. ptrs[disks - 2] = p;
  47. ptrs[disks - 1] = q;
  48. /* Now, pick the proper data tables */
  49. pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
  50. qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
  51. raid6_gfexp[failb]]];
  52. kernel_neon_begin();
  53. __raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul);
  54. kernel_neon_end();
  55. }
  56. static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
  57. void **ptrs)
  58. {
  59. u8 *p, *q, *dq;
  60. const u8 *qmul; /* Q multiplier table */
  61. p = (u8 *)ptrs[disks - 2];
  62. q = (u8 *)ptrs[disks - 1];
  63. /*
  64. * Compute syndrome with zero for the missing data page
  65. * Use the dead data page as temporary storage for delta q
  66. */
  67. dq = (u8 *)ptrs[faila];
  68. ptrs[faila] = (void *)raid6_empty_zero_page;
  69. ptrs[disks - 1] = dq;
  70. raid6_call.gen_syndrome(disks, bytes, ptrs);
  71. /* Restore pointer table */
  72. ptrs[faila] = dq;
  73. ptrs[disks - 1] = q;
  74. /* Now, pick the proper data tables */
  75. qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
  76. kernel_neon_begin();
  77. __raid6_datap_recov_neon(bytes, p, q, dq, qmul);
  78. kernel_neon_end();
  79. }
  80. const struct raid6_recov_calls raid6_recov_neon = {
  81. .data2 = raid6_2data_recov_neon,
  82. .datap = raid6_datap_recov_neon,
  83. .valid = raid6_has_neon,
  84. .name = "neon",
  85. .priority = 10,
  86. };