checksum.S 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459
  1. /*
  2. * INET An implementation of the TCP/IP protocol suite for the LINUX
  3. * operating system. INET is implemented using the BSD Socket
  4. * interface as the means of communication with the user level.
  5. *
  6. * IP/TCP/UDP checksumming routines
  7. *
  8. * Authors: Jorge Cwik, <jorge@laser.satlink.net>
  9. * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
  10. * Tom May, <ftom@netcom.com>
  11. * Pentium Pro/II routines:
  12. * Alexander Kjeldaas <astor@guardian.no>
  13. * Finn Arne Gangstad <finnag@guardian.no>
  14. * Lots of code moved from tcp.c and ip.c; see those files
  15. * for more names.
  16. *
  17. * Changes: Ingo Molnar, converted csum_partial_copy() to 2.1 exception
  18. * handling.
  19. * Andi Kleen, add zeroing on error
  20. * converted to pure assembler
  21. *
  22. * This program is free software; you can redistribute it and/or
  23. * modify it under the terms of the GNU General Public License
  24. * as published by the Free Software Foundation; either version
  25. * 2 of the License, or (at your option) any later version.
  26. */
  27. #include <asm/errno.h>
  28. /*
  29. * computes a partial checksum, e.g. for TCP/UDP fragments
  30. */
  31. /*
  32. unsigned int csum_partial(const unsigned char * buff, int len, unsigned int sum)
  33. */
  34. .text
  35. .align 4
  36. .globl csum_partial
  37. #ifndef CONFIG_X86_USE_PPRO_CHECKSUM
  38. /*
  39. * Experiments with Ethernet and SLIP connections show that buff
  40. * is aligned on either a 2-byte or 4-byte boundary. We get at
  41. * least a twofold speedup on 486 and Pentium if it is 4-byte aligned.
  42. * Fortunately, it is easy to convert 2-byte alignment to 4-byte
  43. * alignment for the unrolled loop.
  44. */
  45. csum_partial:
  46. pushl %esi
  47. pushl %ebx
  48. movl 20(%esp),%eax # Function arg: unsigned int sum
  49. movl 16(%esp),%ecx # Function arg: int len
  50. movl 12(%esp),%esi # Function arg: unsigned char *buff
  51. testl $2, %esi # Check alignment.
  52. jz 2f # Jump if alignment is ok.
  53. subl $2, %ecx # Alignment uses up two bytes.
  54. jae 1f # Jump if we had at least two bytes.
  55. addl $2, %ecx # ecx was < 2. Deal with it.
  56. jmp 4f
  57. 1: movw (%esi), %bx
  58. addl $2, %esi
  59. addw %bx, %ax
  60. adcl $0, %eax
  61. 2:
  62. movl %ecx, %edx
  63. shrl $5, %ecx
  64. jz 2f
  65. testl %esi, %esi
  66. 1: movl (%esi), %ebx
  67. adcl %ebx, %eax
  68. movl 4(%esi), %ebx
  69. adcl %ebx, %eax
  70. movl 8(%esi), %ebx
  71. adcl %ebx, %eax
  72. movl 12(%esi), %ebx
  73. adcl %ebx, %eax
  74. movl 16(%esi), %ebx
  75. adcl %ebx, %eax
  76. movl 20(%esi), %ebx
  77. adcl %ebx, %eax
  78. movl 24(%esi), %ebx
  79. adcl %ebx, %eax
  80. movl 28(%esi), %ebx
  81. adcl %ebx, %eax
  82. lea 32(%esi), %esi
  83. dec %ecx
  84. jne 1b
  85. adcl $0, %eax
  86. 2: movl %edx, %ecx
  87. andl $0x1c, %edx
  88. je 4f
  89. shrl $2, %edx # This clears CF
  90. 3: adcl (%esi), %eax
  91. lea 4(%esi), %esi
  92. dec %edx
  93. jne 3b
  94. adcl $0, %eax
  95. 4: andl $3, %ecx
  96. jz 7f
  97. cmpl $2, %ecx
  98. jb 5f
  99. movw (%esi),%cx
  100. leal 2(%esi),%esi
  101. je 6f
  102. shll $16,%ecx
  103. 5: movb (%esi),%cl
  104. 6: addl %ecx,%eax
  105. adcl $0, %eax
  106. 7:
  107. popl %ebx
  108. popl %esi
  109. ret
  110. #else
  111. /* Version for PentiumII/PPro */
  112. csum_partial:
  113. pushl %esi
  114. pushl %ebx
  115. movl 20(%esp),%eax # Function arg: unsigned int sum
  116. movl 16(%esp),%ecx # Function arg: int len
  117. movl 12(%esp),%esi # Function arg: const unsigned char *buf
  118. testl $2, %esi
  119. jnz 30f
  120. 10:
  121. movl %ecx, %edx
  122. movl %ecx, %ebx
  123. andl $0x7c, %ebx
  124. shrl $7, %ecx
  125. addl %ebx,%esi
  126. shrl $2, %ebx
  127. negl %ebx
  128. lea 45f(%ebx,%ebx,2), %ebx
  129. testl %esi, %esi
  130. jmp *%ebx
  131. # Handle 2-byte-aligned regions
  132. 20: addw (%esi), %ax
  133. lea 2(%esi), %esi
  134. adcl $0, %eax
  135. jmp 10b
  136. 30: subl $2, %ecx
  137. ja 20b
  138. je 32f
  139. movzbl (%esi),%ebx # csumming 1 byte, 2-aligned
  140. addl %ebx, %eax
  141. adcl $0, %eax
  142. jmp 80f
  143. 32:
  144. addw (%esi), %ax # csumming 2 bytes, 2-aligned
  145. adcl $0, %eax
  146. jmp 80f
  147. 40:
  148. addl -128(%esi), %eax
  149. adcl -124(%esi), %eax
  150. adcl -120(%esi), %eax
  151. adcl -116(%esi), %eax
  152. adcl -112(%esi), %eax
  153. adcl -108(%esi), %eax
  154. adcl -104(%esi), %eax
  155. adcl -100(%esi), %eax
  156. adcl -96(%esi), %eax
  157. adcl -92(%esi), %eax
  158. adcl -88(%esi), %eax
  159. adcl -84(%esi), %eax
  160. adcl -80(%esi), %eax
  161. adcl -76(%esi), %eax
  162. adcl -72(%esi), %eax
  163. adcl -68(%esi), %eax
  164. adcl -64(%esi), %eax
  165. adcl -60(%esi), %eax
  166. adcl -56(%esi), %eax
  167. adcl -52(%esi), %eax
  168. adcl -48(%esi), %eax
  169. adcl -44(%esi), %eax
  170. adcl -40(%esi), %eax
  171. adcl -36(%esi), %eax
  172. adcl -32(%esi), %eax
  173. adcl -28(%esi), %eax
  174. adcl -24(%esi), %eax
  175. adcl -20(%esi), %eax
  176. adcl -16(%esi), %eax
  177. adcl -12(%esi), %eax
  178. adcl -8(%esi), %eax
  179. adcl -4(%esi), %eax
  180. 45:
  181. lea 128(%esi), %esi
  182. adcl $0, %eax
  183. dec %ecx
  184. jge 40b
  185. movl %edx, %ecx
  186. 50: andl $3, %ecx
  187. jz 80f
  188. # Handle the last 1-3 bytes without jumping
  189. notl %ecx # 1->2, 2->1, 3->0, higher bits are masked
  190. movl $0xffffff,%ebx # by the shll and shrl instructions
  191. shll $3,%ecx
  192. shrl %cl,%ebx
  193. andl -128(%esi),%ebx # esi is 4-aligned so should be ok
  194. addl %ebx,%eax
  195. adcl $0,%eax
  196. 80:
  197. popl %ebx
  198. popl %esi
  199. ret
  200. #endif
  201. /*
  202. unsigned int csum_partial_copy_generic (const char *src, char *dst,
  203. int len, int sum, int *src_err_ptr, int *dst_err_ptr)
  204. */
  205. /*
  206. * Copy from ds while checksumming, otherwise like csum_partial
  207. *
  208. * The macros SRC and DST specify the type of access for the instruction.
  209. * thus we can call a custom exception handler for all access types.
  210. *
  211. * FIXME: could someone double-check whether I haven't mixed up some SRC and
  212. * DST definitions? It's damn hard to trigger all cases. I hope I got
  213. * them all but there's no guarantee.
  214. */
  215. #define SRC(y...) \
  216. 9999: y; \
  217. .section __ex_table, "a"; \
  218. .long 9999b, 6001f ; \
  219. .previous
  220. #define DST(y...) \
  221. 9999: y; \
  222. .section __ex_table, "a"; \
  223. .long 9999b, 6002f ; \
  224. .previous
  225. .align 4
  226. .globl csum_partial_copy_generic_i386
  227. #ifndef CONFIG_X86_USE_PPRO_CHECKSUM
  228. #define ARGBASE 16
  229. #define FP 12
  230. csum_partial_copy_generic_i386:
  231. subl $4,%esp
  232. pushl %edi
  233. pushl %esi
  234. pushl %ebx
  235. movl ARGBASE+16(%esp),%eax # sum
  236. movl ARGBASE+12(%esp),%ecx # len
  237. movl ARGBASE+4(%esp),%esi # src
  238. movl ARGBASE+8(%esp),%edi # dst
  239. testl $2, %edi # Check alignment.
  240. jz 2f # Jump if alignment is ok.
  241. subl $2, %ecx # Alignment uses up two bytes.
  242. jae 1f # Jump if we had at least two bytes.
  243. addl $2, %ecx # ecx was < 2. Deal with it.
  244. jmp 4f
  245. SRC(1: movw (%esi), %bx )
  246. addl $2, %esi
  247. DST( movw %bx, (%edi) )
  248. addl $2, %edi
  249. addw %bx, %ax
  250. adcl $0, %eax
  251. 2:
  252. movl %ecx, FP(%esp)
  253. shrl $5, %ecx
  254. jz 2f
  255. testl %esi, %esi
  256. SRC(1: movl (%esi), %ebx )
  257. SRC( movl 4(%esi), %edx )
  258. adcl %ebx, %eax
  259. DST( movl %ebx, (%edi) )
  260. adcl %edx, %eax
  261. DST( movl %edx, 4(%edi) )
  262. SRC( movl 8(%esi), %ebx )
  263. SRC( movl 12(%esi), %edx )
  264. adcl %ebx, %eax
  265. DST( movl %ebx, 8(%edi) )
  266. adcl %edx, %eax
  267. DST( movl %edx, 12(%edi) )
  268. SRC( movl 16(%esi), %ebx )
  269. SRC( movl 20(%esi), %edx )
  270. adcl %ebx, %eax
  271. DST( movl %ebx, 16(%edi) )
  272. adcl %edx, %eax
  273. DST( movl %edx, 20(%edi) )
  274. SRC( movl 24(%esi), %ebx )
  275. SRC( movl 28(%esi), %edx )
  276. adcl %ebx, %eax
  277. DST( movl %ebx, 24(%edi) )
  278. adcl %edx, %eax
  279. DST( movl %edx, 28(%edi) )
  280. lea 32(%esi), %esi
  281. lea 32(%edi), %edi
  282. dec %ecx
  283. jne 1b
  284. adcl $0, %eax
  285. 2: movl FP(%esp), %edx
  286. movl %edx, %ecx
  287. andl $0x1c, %edx
  288. je 4f
  289. shrl $2, %edx # This clears CF
  290. SRC(3: movl (%esi), %ebx )
  291. adcl %ebx, %eax
  292. DST( movl %ebx, (%edi) )
  293. lea 4(%esi), %esi
  294. lea 4(%edi), %edi
  295. dec %edx
  296. jne 3b
  297. adcl $0, %eax
  298. 4: andl $3, %ecx
  299. jz 7f
  300. cmpl $2, %ecx
  301. jb 5f
  302. SRC( movw (%esi), %cx )
  303. leal 2(%esi), %esi
  304. DST( movw %cx, (%edi) )
  305. leal 2(%edi), %edi
  306. je 6f
  307. shll $16,%ecx
  308. SRC(5: movb (%esi), %cl )
  309. DST( movb %cl, (%edi) )
  310. 6: addl %ecx, %eax
  311. adcl $0, %eax
  312. 7:
  313. 5000:
  314. # Exception handler:
  315. .section .fixup, "ax"
  316. 6001:
  317. movl ARGBASE+20(%esp), %ebx # src_err_ptr
  318. movl $-EFAULT, (%ebx)
  319. # zero the complete destination - computing the rest
  320. # is too much work
  321. movl ARGBASE+8(%esp), %edi # dst
  322. movl ARGBASE+12(%esp), %ecx # len
  323. xorl %eax,%eax
  324. rep ; stosb
  325. jmp 5000b
  326. 6002:
  327. movl ARGBASE+24(%esp), %ebx # dst_err_ptr
  328. movl $-EFAULT,(%ebx)
  329. jmp 5000b
  330. .previous
  331. popl %ebx
  332. popl %esi
  333. popl %edi
  334. popl %ecx # equivalent to addl $4,%esp
  335. ret
  336. #else
  337. /* Version for PentiumII/PPro */
  338. #define ROUND1(x) \
  339. SRC(movl x(%esi), %ebx ) ; \
  340. addl %ebx, %eax ; \
  341. DST(movl %ebx, x(%edi) ) ;
  342. #define ROUND(x) \
  343. SRC(movl x(%esi), %ebx ) ; \
  344. adcl %ebx, %eax ; \
  345. DST(movl %ebx, x(%edi) ) ;
  346. #define ARGBASE 12
  347. csum_partial_copy_generic_i386:
  348. pushl %ebx
  349. pushl %edi
  350. pushl %esi
  351. movl ARGBASE+4(%esp),%esi #src
  352. movl ARGBASE+8(%esp),%edi #dst
  353. movl ARGBASE+12(%esp),%ecx #len
  354. movl ARGBASE+16(%esp),%eax #sum
  355. # movl %ecx, %edx
  356. movl %ecx, %ebx
  357. movl %esi, %edx
  358. shrl $6, %ecx
  359. andl $0x3c, %ebx
  360. negl %ebx
  361. subl %ebx, %esi
  362. subl %ebx, %edi
  363. lea -1(%esi),%edx
  364. andl $-32,%edx
  365. lea 3f(%ebx,%ebx), %ebx
  366. testl %esi, %esi
  367. jmp *%ebx
  368. 1: addl $64,%esi
  369. addl $64,%edi
  370. SRC(movb -32(%edx),%bl) ; SRC(movb (%edx),%bl)
  371. ROUND1(-64) ROUND(-60) ROUND(-56) ROUND(-52)
  372. ROUND (-48) ROUND(-44) ROUND(-40) ROUND(-36)
  373. ROUND (-32) ROUND(-28) ROUND(-24) ROUND(-20)
  374. ROUND (-16) ROUND(-12) ROUND(-8) ROUND(-4)
  375. 3: adcl $0,%eax
  376. addl $64, %edx
  377. dec %ecx
  378. jge 1b
  379. 4: movl ARGBASE+12(%esp),%edx #len
  380. andl $3, %edx
  381. jz 7f
  382. cmpl $2, %edx
  383. jb 5f
  384. SRC( movw (%esi), %dx )
  385. leal 2(%esi), %esi
  386. DST( movw %dx, (%edi) )
  387. leal 2(%edi), %edi
  388. je 6f
  389. shll $16,%edx
  390. 5:
  391. SRC( movb (%esi), %dl )
  392. DST( movb %dl, (%edi) )
  393. 6: addl %edx, %eax
  394. adcl $0, %eax
  395. 7:
  396. .section .fixup, "ax"
  397. 6001: movl ARGBASE+20(%esp), %ebx # src_err_ptr
  398. movl $-EFAULT, (%ebx)
  399. # zero the complete destination (computing the rest is too much work)
  400. movl ARGBASE+8(%esp),%edi # dst
  401. movl ARGBASE+12(%esp),%ecx # len
  402. xorl %eax,%eax
  403. rep; stosb
  404. jmp 7b
  405. 6002: movl ARGBASE+24(%esp), %ebx # dst_err_ptr
  406. movl $-EFAULT, (%ebx)
  407. jmp 7b
  408. .previous
  409. popl %esi
  410. popl %edi
  411. popl %ebx
  412. ret
  413. #undef ROUND
  414. #undef ROUND1
  415. #endif