mmx.c 3.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /* -*- linux-c -*- ------------------------------------------------------- *
  3. *
  4. * Copyright 2002 H. Peter Anvin - All Rights Reserved
  5. *
  6. * ----------------------------------------------------------------------- */
  7. /*
  8. * raid6/mmx.c
  9. *
  10. * MMX implementation of RAID-6 syndrome functions
  11. */
  12. #ifdef CONFIG_X86_32
  13. #include <linux/raid/pq.h>
  14. #include "x86.h"
  15. /* Shared with raid6/sse1.c */
  16. const struct raid6_mmx_constants {
  17. u64 x1d;
  18. } raid6_mmx_constants = {
  19. 0x1d1d1d1d1d1d1d1dULL,
  20. };
  21. static int raid6_have_mmx(void)
  22. {
  23. /* Not really "boot_cpu" but "all_cpus" */
  24. return boot_cpu_has(X86_FEATURE_MMX);
  25. }
  26. /*
  27. * Plain MMX implementation
  28. */
  29. static void raid6_mmx1_gen_syndrome(int disks, size_t bytes, void **ptrs)
  30. {
  31. u8 **dptr = (u8 **)ptrs;
  32. u8 *p, *q;
  33. int d, z, z0;
  34. z0 = disks - 3; /* Highest data disk */
  35. p = dptr[z0+1]; /* XOR parity */
  36. q = dptr[z0+2]; /* RS syndrome */
  37. kernel_fpu_begin();
  38. asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
  39. asm volatile("pxor %mm5,%mm5"); /* Zero temp */
  40. for ( d = 0 ; d < bytes ; d += 8 ) {
  41. asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
  42. asm volatile("movq %mm2,%mm4"); /* Q[0] */
  43. for ( z = z0-1 ; z >= 0 ; z-- ) {
  44. asm volatile("movq %0,%%mm6" : : "m" (dptr[z][d]));
  45. asm volatile("pcmpgtb %mm4,%mm5");
  46. asm volatile("paddb %mm4,%mm4");
  47. asm volatile("pand %mm0,%mm5");
  48. asm volatile("pxor %mm5,%mm4");
  49. asm volatile("pxor %mm5,%mm5");
  50. asm volatile("pxor %mm6,%mm2");
  51. asm volatile("pxor %mm6,%mm4");
  52. }
  53. asm volatile("movq %%mm2,%0" : "=m" (p[d]));
  54. asm volatile("pxor %mm2,%mm2");
  55. asm volatile("movq %%mm4,%0" : "=m" (q[d]));
  56. asm volatile("pxor %mm4,%mm4");
  57. }
  58. kernel_fpu_end();
  59. }
  60. const struct raid6_calls raid6_mmxx1 = {
  61. raid6_mmx1_gen_syndrome,
  62. NULL, /* XOR not yet implemented */
  63. raid6_have_mmx,
  64. "mmxx1",
  65. 0
  66. };
  67. /*
  68. * Unrolled-by-2 MMX implementation
  69. */
  70. static void raid6_mmx2_gen_syndrome(int disks, size_t bytes, void **ptrs)
  71. {
  72. u8 **dptr = (u8 **)ptrs;
  73. u8 *p, *q;
  74. int d, z, z0;
  75. z0 = disks - 3; /* Highest data disk */
  76. p = dptr[z0+1]; /* XOR parity */
  77. q = dptr[z0+2]; /* RS syndrome */
  78. kernel_fpu_begin();
  79. asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
  80. asm volatile("pxor %mm5,%mm5"); /* Zero temp */
  81. asm volatile("pxor %mm7,%mm7"); /* Zero temp */
  82. for ( d = 0 ; d < bytes ; d += 16 ) {
  83. asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
  84. asm volatile("movq %0,%%mm3" : : "m" (dptr[z0][d+8]));
  85. asm volatile("movq %mm2,%mm4"); /* Q[0] */
  86. asm volatile("movq %mm3,%mm6"); /* Q[1] */
  87. for ( z = z0-1 ; z >= 0 ; z-- ) {
  88. asm volatile("pcmpgtb %mm4,%mm5");
  89. asm volatile("pcmpgtb %mm6,%mm7");
  90. asm volatile("paddb %mm4,%mm4");
  91. asm volatile("paddb %mm6,%mm6");
  92. asm volatile("pand %mm0,%mm5");
  93. asm volatile("pand %mm0,%mm7");
  94. asm volatile("pxor %mm5,%mm4");
  95. asm volatile("pxor %mm7,%mm6");
  96. asm volatile("movq %0,%%mm5" : : "m" (dptr[z][d]));
  97. asm volatile("movq %0,%%mm7" : : "m" (dptr[z][d+8]));
  98. asm volatile("pxor %mm5,%mm2");
  99. asm volatile("pxor %mm7,%mm3");
  100. asm volatile("pxor %mm5,%mm4");
  101. asm volatile("pxor %mm7,%mm6");
  102. asm volatile("pxor %mm5,%mm5");
  103. asm volatile("pxor %mm7,%mm7");
  104. }
  105. asm volatile("movq %%mm2,%0" : "=m" (p[d]));
  106. asm volatile("movq %%mm3,%0" : "=m" (p[d+8]));
  107. asm volatile("movq %%mm4,%0" : "=m" (q[d]));
  108. asm volatile("movq %%mm6,%0" : "=m" (q[d+8]));
  109. }
  110. kernel_fpu_end();
  111. }
  112. const struct raid6_calls raid6_mmxx2 = {
  113. raid6_mmx2_gen_syndrome,
  114. NULL, /* XOR not yet implemented */
  115. raid6_have_mmx,
  116. "mmxx2",
  117. 0
  118. };
  119. #endif