checksum.c 4.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Checksum functions for Hexagon
  4. *
  5. * Copyright (c) 2010-2011, The Linux Foundation. All rights reserved.
  6. */
  7. /* This was derived from arch/alpha/lib/checksum.c */
  8. #include <linux/module.h>
  9. #include <linux/string.h>
  10. #include <asm/byteorder.h>
  11. #include <net/checksum.h>
  12. #include <linux/uaccess.h>
  13. #include <asm/intrinsics.h>
  14. /* Vector value operations */
  15. #define SIGN(x, y) ((0x8000ULL*x)<<y)
  16. #define CARRY(x, y) ((0x0002ULL*x)<<y)
  17. #define SELECT(x, y) ((0x0001ULL*x)<<y)
  18. #define VR_NEGATE(a, b, c, d) (SIGN(a, 48) + SIGN(b, 32) + SIGN(c, 16) \
  19. + SIGN(d, 0))
  20. #define VR_CARRY(a, b, c, d) (CARRY(a, 48) + CARRY(b, 32) + CARRY(c, 16) \
  21. + CARRY(d, 0))
  22. #define VR_SELECT(a, b, c, d) (SELECT(a, 48) + SELECT(b, 32) + SELECT(c, 16) \
  23. + SELECT(d, 0))
  24. /* optimized HEXAGON V3 intrinsic version */
  25. static inline unsigned short from64to16(u64 x)
  26. {
  27. u64 sum;
  28. sum = HEXAGON_P_vrmpyh_PP(x^VR_NEGATE(1, 1, 1, 1),
  29. VR_SELECT(1, 1, 1, 1));
  30. sum += VR_CARRY(0, 0, 1, 0);
  31. sum = HEXAGON_P_vrmpyh_PP(sum, VR_SELECT(0, 0, 1, 1));
  32. return 0xFFFF & sum;
  33. }
  34. /*
  35. * computes the checksum of the TCP/UDP pseudo-header
  36. * returns a 16-bit checksum, already complemented.
  37. */
  38. __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr,
  39. __u32 len, __u8 proto, __wsum sum)
  40. {
  41. return (__force __sum16)~from64to16(
  42. (__force u64)saddr + (__force u64)daddr +
  43. (__force u64)sum + ((len + proto) << 8));
  44. }
  45. __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr,
  46. __u32 len, __u8 proto, __wsum sum)
  47. {
  48. u64 result;
  49. result = (__force u64)saddr + (__force u64)daddr +
  50. (__force u64)sum + ((len + proto) << 8);
  51. /* Fold down to 32-bits so we don't lose in the typedef-less
  52. network stack. */
  53. /* 64 to 33 */
  54. result = (result & 0xffffffffUL) + (result >> 32);
  55. /* 33 to 32 */
  56. result = (result & 0xffffffffUL) + (result >> 32);
  57. return (__force __wsum)result;
  58. }
  59. EXPORT_SYMBOL(csum_tcpudp_nofold);
  60. /*
  61. * Do a 64-bit checksum on an arbitrary memory area..
  62. *
  63. * This isn't a great routine, but it's not _horrible_ either. The
  64. * inner loop could be unrolled a bit further, and there are better
  65. * ways to do the carry, but this is reasonable.
  66. */
  67. /* optimized HEXAGON intrinsic version, with over read fixed */
  68. unsigned int do_csum(const void *voidptr, int len)
  69. {
  70. u64 sum0, sum1, x0, x1, *ptr8_o, *ptr8_e, *ptr8;
  71. int i, start, mid, end, mask;
  72. const char *ptr = voidptr;
  73. unsigned short *ptr2;
  74. unsigned int *ptr4;
  75. if (len <= 0)
  76. return 0;
  77. start = 0xF & (16-(((int) ptr) & 0xF)) ;
  78. mask = 0x7fffffffUL >> HEXAGON_R_cl0_R(len);
  79. start = start & mask ;
  80. mid = len - start;
  81. end = mid & 0xF;
  82. mid = mid>>4;
  83. sum0 = mid << 18;
  84. sum1 = 0;
  85. if (start & 1)
  86. sum0 += (u64) (ptr[0] << 8);
  87. ptr2 = (unsigned short *) &ptr[start & 1];
  88. if (start & 2)
  89. sum1 += (u64) ptr2[0];
  90. ptr4 = (unsigned int *) &ptr[start & 3];
  91. if (start & 4) {
  92. sum0 = HEXAGON_P_vrmpyhacc_PP(sum0,
  93. VR_NEGATE(0, 0, 1, 1)^((u64)ptr4[0]),
  94. VR_SELECT(0, 0, 1, 1));
  95. sum0 += VR_SELECT(0, 0, 1, 0);
  96. }
  97. ptr8 = (u64 *) &ptr[start & 7];
  98. if (start & 8) {
  99. sum1 = HEXAGON_P_vrmpyhacc_PP(sum1,
  100. VR_NEGATE(1, 1, 1, 1)^(ptr8[0]),
  101. VR_SELECT(1, 1, 1, 1));
  102. sum1 += VR_CARRY(0, 0, 1, 0);
  103. }
  104. ptr8_o = (u64 *) (ptr + start);
  105. ptr8_e = (u64 *) (ptr + start + 8);
  106. if (mid) {
  107. x0 = *ptr8_e; ptr8_e += 2;
  108. x1 = *ptr8_o; ptr8_o += 2;
  109. if (mid > 1)
  110. for (i = 0; i < mid-1; i++) {
  111. sum0 = HEXAGON_P_vrmpyhacc_PP(sum0,
  112. x0^VR_NEGATE(1, 1, 1, 1),
  113. VR_SELECT(1, 1, 1, 1));
  114. sum1 = HEXAGON_P_vrmpyhacc_PP(sum1,
  115. x1^VR_NEGATE(1, 1, 1, 1),
  116. VR_SELECT(1, 1, 1, 1));
  117. x0 = *ptr8_e; ptr8_e += 2;
  118. x1 = *ptr8_o; ptr8_o += 2;
  119. }
  120. sum0 = HEXAGON_P_vrmpyhacc_PP(sum0, x0^VR_NEGATE(1, 1, 1, 1),
  121. VR_SELECT(1, 1, 1, 1));
  122. sum1 = HEXAGON_P_vrmpyhacc_PP(sum1, x1^VR_NEGATE(1, 1, 1, 1),
  123. VR_SELECT(1, 1, 1, 1));
  124. }
  125. ptr4 = (unsigned int *) &ptr[start + (mid * 16) + (end & 8)];
  126. if (end & 4) {
  127. sum1 = HEXAGON_P_vrmpyhacc_PP(sum1,
  128. VR_NEGATE(0, 0, 1, 1)^((u64)ptr4[0]),
  129. VR_SELECT(0, 0, 1, 1));
  130. sum1 += VR_SELECT(0, 0, 1, 0);
  131. }
  132. ptr2 = (unsigned short *) &ptr[start + (mid * 16) + (end & 12)];
  133. if (end & 2)
  134. sum0 += (u64) ptr2[0];
  135. if (end & 1)
  136. sum1 += (u64) ptr[start + (mid * 16) + (end & 14)];
  137. ptr8 = (u64 *) &ptr[start + (mid * 16)];
  138. if (end & 8) {
  139. sum0 = HEXAGON_P_vrmpyhacc_PP(sum0,
  140. VR_NEGATE(1, 1, 1, 1)^(ptr8[0]),
  141. VR_SELECT(1, 1, 1, 1));
  142. sum0 += VR_CARRY(0, 0, 1, 0);
  143. }
  144. sum0 = HEXAGON_P_vrmpyh_PP((sum0+sum1)^VR_NEGATE(0, 0, 0, 1),
  145. VR_SELECT(0, 0, 1, 1));
  146. sum0 += VR_NEGATE(0, 0, 0, 1);
  147. sum0 = HEXAGON_P_vrmpyh_PP(sum0, VR_SELECT(0, 0, 1, 1));
  148. if (start & 1)
  149. sum0 = (sum0 << 8) | (0xFF & (sum0 >> 8));
  150. return 0xFFFF & sum0;
  151. }