sse2-memset32-atom.S 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513
  1. /*
  2. * Copyright (C) 2010 The Android Open Source Project
  3. *
  4. * Licensed under the Apache License, Version 2.0 (the "License");
  5. * you may not use this file except in compliance with the License.
  6. * You may obtain a copy of the License at
  7. *
  8. * http://www.apache.org/licenses/LICENSE-2.0
  9. *
  10. * Unless required by applicable law or agreed to in writing, software
  11. * distributed under the License is distributed on an "AS IS" BASIS,
  12. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. * See the License for the specific language governing permissions and
  14. * limitations under the License.
  15. */
  16. /*
  17. * Contributed by: Intel Corporation
  18. */
  19. #ifndef L
  20. # define L(label) .L##label
  21. #endif
  22. #ifndef ALIGN
  23. # define ALIGN(n) .p2align n
  24. #endif
  25. #ifndef cfi_startproc
  26. # define cfi_startproc .cfi_startproc
  27. #endif
  28. #ifndef cfi_endproc
  29. # define cfi_endproc .cfi_endproc
  30. #endif
  31. #ifndef cfi_rel_offset
  32. # define cfi_rel_offset(reg, off) .cfi_rel_offset reg, off
  33. #endif
  34. #ifndef cfi_restore
  35. # define cfi_restore(reg) .cfi_restore reg
  36. #endif
  37. #ifndef cfi_adjust_cfa_offset
  38. # define cfi_adjust_cfa_offset(off) .cfi_adjust_cfa_offset off
  39. #endif
  40. #ifndef ENTRY
  41. # define ENTRY(name) \
  42. .type name, @function; \
  43. .globl name; \
  44. .p2align 4; \
  45. name: \
  46. cfi_startproc
  47. #endif
  48. #ifndef END
  49. # define END(name) \
  50. cfi_endproc; \
  51. .size name, .-name
  52. #endif
  53. #define CFI_PUSH(REG) \
  54. cfi_adjust_cfa_offset (4); \
  55. cfi_rel_offset (REG, 0)
  56. #define CFI_POP(REG) \
  57. cfi_adjust_cfa_offset (-4); \
  58. cfi_restore (REG)
  59. #define PUSH(REG) pushl REG; CFI_PUSH (REG)
  60. #define POP(REG) popl REG; CFI_POP (REG)
  61. #ifdef USE_AS_BZERO32
  62. # define DEST PARMS
  63. # define LEN DEST+4
  64. #else
  65. # define DEST PARMS
  66. # define DWDS DEST+4
  67. # define LEN DWDS+4
  68. #endif
  69. #ifdef USE_AS_WMEMSET32
  70. # define SETRTNVAL movl DEST(%esp), %eax
  71. #else
  72. # define SETRTNVAL
  73. #endif
  74. #ifdef SHARED
  75. # define ENTRANCE PUSH (%ebx);
  76. # define RETURN_END POP (%ebx); ret
  77. # define RETURN RETURN_END; CFI_PUSH (%ebx)
  78. # define PARMS 8 /* Preserve EBX. */
  79. # define JMPTBL(I, B) I - B
  80. /* Load an entry in a jump table into EBX and branch to it. TABLE is a
  81. jump table with relative offsets. */
  82. # define BRANCH_TO_JMPTBL_ENTRY(TABLE) \
  83. /* We first load PC into EBX. */ \
  84. call __i686.get_pc_thunk.bx; \
  85. /* Get the address of the jump table. */ \
  86. add $(TABLE - .), %ebx; \
  87. /* Get the entry and convert the relative offset to the \
  88. absolute address. */ \
  89. add (%ebx,%ecx,4), %ebx; \
  90. /* We loaded the jump table and adjuested EDX. Go. */ \
  91. jmp *%ebx
  92. .section .gnu.linkonce.t.__i686.get_pc_thunk.bx,"ax",@progbits
  93. .globl __i686.get_pc_thunk.bx
  94. .hidden __i686.get_pc_thunk.bx
  95. ALIGN (4)
  96. .type __i686.get_pc_thunk.bx,@function
  97. __i686.get_pc_thunk.bx:
  98. movl (%esp), %ebx
  99. ret
  100. #else
  101. # define ENTRANCE
  102. # define RETURN_END ret
  103. # define RETURN RETURN_END
  104. # define PARMS 4
  105. # define JMPTBL(I, B) I
  106. /* Branch to an entry in a jump table. TABLE is a jump table with
  107. absolute offsets. */
  108. # define BRANCH_TO_JMPTBL_ENTRY(TABLE) \
  109. jmp *TABLE(,%ecx,4)
  110. #endif
  111. .section .text.sse2,"ax",@progbits
  112. ALIGN (4)
  113. ENTRY (sse2_memset32_atom)
  114. ENTRANCE
  115. movl LEN(%esp), %ecx
  116. #ifdef USE_AS_ANDROID
  117. shr $2, %ecx
  118. #endif
  119. #ifdef USE_AS_BZERO32
  120. xor %eax, %eax
  121. #else
  122. mov DWDS(%esp), %eax
  123. mov %eax, %edx
  124. #endif
  125. movl DEST(%esp), %edx
  126. cmp $16, %ecx
  127. jae L(16dbwordsormore)
  128. L(write_less16dbwords):
  129. lea (%edx, %ecx, 4), %edx
  130. BRANCH_TO_JMPTBL_ENTRY (L(table_less16dbwords))
  131. .pushsection .rodata.sse2,"a",@progbits
  132. ALIGN (2)
  133. L(table_less16dbwords):
  134. .int JMPTBL (L(write_0dbwords), L(table_less16dbwords))
  135. .int JMPTBL (L(write_1dbwords), L(table_less16dbwords))
  136. .int JMPTBL (L(write_2dbwords), L(table_less16dbwords))
  137. .int JMPTBL (L(write_3dbwords), L(table_less16dbwords))
  138. .int JMPTBL (L(write_4dbwords), L(table_less16dbwords))
  139. .int JMPTBL (L(write_5dbwords), L(table_less16dbwords))
  140. .int JMPTBL (L(write_6dbwords), L(table_less16dbwords))
  141. .int JMPTBL (L(write_7dbwords), L(table_less16dbwords))
  142. .int JMPTBL (L(write_8dbwords), L(table_less16dbwords))
  143. .int JMPTBL (L(write_9dbwords), L(table_less16dbwords))
  144. .int JMPTBL (L(write_10dbwords), L(table_less16dbwords))
  145. .int JMPTBL (L(write_11dbwords), L(table_less16dbwords))
  146. .int JMPTBL (L(write_12dbwords), L(table_less16dbwords))
  147. .int JMPTBL (L(write_13dbwords), L(table_less16dbwords))
  148. .int JMPTBL (L(write_14dbwords), L(table_less16dbwords))
  149. .int JMPTBL (L(write_15dbwords), L(table_less16dbwords))
  150. .popsection
  151. ALIGN (4)
  152. L(write_15dbwords):
  153. movl %eax, -60(%edx)
  154. L(write_14dbwords):
  155. movl %eax, -56(%edx)
  156. L(write_13dbwords):
  157. movl %eax, -52(%edx)
  158. L(write_12dbwords):
  159. movl %eax, -48(%edx)
  160. L(write_11dbwords):
  161. movl %eax, -44(%edx)
  162. L(write_10dbwords):
  163. movl %eax, -40(%edx)
  164. L(write_9dbwords):
  165. movl %eax, -36(%edx)
  166. L(write_8dbwords):
  167. movl %eax, -32(%edx)
  168. L(write_7dbwords):
  169. movl %eax, -28(%edx)
  170. L(write_6dbwords):
  171. movl %eax, -24(%edx)
  172. L(write_5dbwords):
  173. movl %eax, -20(%edx)
  174. L(write_4dbwords):
  175. movl %eax, -16(%edx)
  176. L(write_3dbwords):
  177. movl %eax, -12(%edx)
  178. L(write_2dbwords):
  179. movl %eax, -8(%edx)
  180. L(write_1dbwords):
  181. movl %eax, -4(%edx)
  182. L(write_0dbwords):
  183. SETRTNVAL
  184. RETURN
  185. ALIGN (4)
  186. L(16dbwordsormore):
  187. test $3, %edx
  188. jz L(aligned4bytes)
  189. mov %eax, (%edx)
  190. mov %eax, -4(%edx, %ecx, 4)
  191. sub $1, %ecx
  192. rol $24, %eax
  193. add $1, %edx
  194. test $3, %edx
  195. jz L(aligned4bytes)
  196. ror $8, %eax
  197. add $1, %edx
  198. test $3, %edx
  199. jz L(aligned4bytes)
  200. ror $8, %eax
  201. add $1, %edx
  202. L(aligned4bytes):
  203. shl $2, %ecx
  204. #ifdef USE_AS_BZERO32
  205. pxor %xmm0, %xmm0
  206. #else
  207. movd %eax, %xmm0
  208. pshufd $0, %xmm0, %xmm0
  209. #endif
  210. testl $0xf, %edx
  211. jz L(aligned_16)
  212. /* ECX > 32 and EDX is not 16 byte aligned. */
  213. L(not_aligned_16):
  214. movdqu %xmm0, (%edx)
  215. movl %edx, %eax
  216. and $-16, %edx
  217. add $16, %edx
  218. sub %edx, %eax
  219. add %eax, %ecx
  220. movd %xmm0, %eax
  221. ALIGN (4)
  222. L(aligned_16):
  223. cmp $128, %ecx
  224. jae L(128bytesormore)
  225. L(aligned_16_less128bytes):
  226. add %ecx, %edx
  227. shr $2, %ecx
  228. BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes))
  229. ALIGN (4)
  230. L(128bytesormore):
  231. #ifdef SHARED_CACHE_SIZE
  232. PUSH (%ebx)
  233. mov $SHARED_CACHE_SIZE, %ebx
  234. #else
  235. # ifdef SHARED
  236. call __i686.get_pc_thunk.bx
  237. add $_GLOBAL_OFFSET_TABLE_, %ebx
  238. mov __x86_shared_cache_size@GOTOFF(%ebx), %ebx
  239. # else
  240. PUSH (%ebx)
  241. mov __x86_shared_cache_size, %ebx
  242. # endif
  243. #endif
  244. cmp %ebx, %ecx
  245. jae L(128bytesormore_nt_start)
  246. #ifdef DATA_CACHE_SIZE
  247. POP (%ebx)
  248. # define RESTORE_EBX_STATE CFI_PUSH (%ebx)
  249. cmp $DATA_CACHE_SIZE, %ecx
  250. #else
  251. # ifdef SHARED
  252. # define RESTORE_EBX_STATE
  253. call __i686.get_pc_thunk.bx
  254. add $_GLOBAL_OFFSET_TABLE_, %ebx
  255. cmp __x86_data_cache_size@GOTOFF(%ebx), %ecx
  256. # else
  257. POP (%ebx)
  258. # define RESTORE_EBX_STATE CFI_PUSH (%ebx)
  259. cmp __x86_data_cache_size, %ecx
  260. # endif
  261. #endif
  262. jae L(128bytes_L2_normal)
  263. subl $128, %ecx
  264. L(128bytesormore_normal):
  265. sub $128, %ecx
  266. movdqa %xmm0, (%edx)
  267. movdqa %xmm0, 0x10(%edx)
  268. movdqa %xmm0, 0x20(%edx)
  269. movdqa %xmm0, 0x30(%edx)
  270. movdqa %xmm0, 0x40(%edx)
  271. movdqa %xmm0, 0x50(%edx)
  272. movdqa %xmm0, 0x60(%edx)
  273. movdqa %xmm0, 0x70(%edx)
  274. lea 128(%edx), %edx
  275. jb L(128bytesless_normal)
  276. sub $128, %ecx
  277. movdqa %xmm0, (%edx)
  278. movdqa %xmm0, 0x10(%edx)
  279. movdqa %xmm0, 0x20(%edx)
  280. movdqa %xmm0, 0x30(%edx)
  281. movdqa %xmm0, 0x40(%edx)
  282. movdqa %xmm0, 0x50(%edx)
  283. movdqa %xmm0, 0x60(%edx)
  284. movdqa %xmm0, 0x70(%edx)
  285. lea 128(%edx), %edx
  286. jae L(128bytesormore_normal)
  287. L(128bytesless_normal):
  288. lea 128(%ecx), %ecx
  289. add %ecx, %edx
  290. shr $2, %ecx
  291. BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes))
  292. ALIGN (4)
  293. L(128bytes_L2_normal):
  294. prefetcht0 0x380(%edx)
  295. prefetcht0 0x3c0(%edx)
  296. sub $128, %ecx
  297. movdqa %xmm0, (%edx)
  298. movaps %xmm0, 0x10(%edx)
  299. movaps %xmm0, 0x20(%edx)
  300. movaps %xmm0, 0x30(%edx)
  301. movaps %xmm0, 0x40(%edx)
  302. movaps %xmm0, 0x50(%edx)
  303. movaps %xmm0, 0x60(%edx)
  304. movaps %xmm0, 0x70(%edx)
  305. add $128, %edx
  306. cmp $128, %ecx
  307. jae L(128bytes_L2_normal)
  308. L(128bytesless_L2_normal):
  309. add %ecx, %edx
  310. shr $2, %ecx
  311. BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes))
  312. RESTORE_EBX_STATE
  313. L(128bytesormore_nt_start):
  314. sub %ebx, %ecx
  315. mov %ebx, %eax
  316. and $0x7f, %eax
  317. add %eax, %ecx
  318. movd %xmm0, %eax
  319. ALIGN (4)
  320. L(128bytesormore_shared_cache_loop):
  321. prefetcht0 0x3c0(%edx)
  322. prefetcht0 0x380(%edx)
  323. sub $0x80, %ebx
  324. movdqa %xmm0, (%edx)
  325. movdqa %xmm0, 0x10(%edx)
  326. movdqa %xmm0, 0x20(%edx)
  327. movdqa %xmm0, 0x30(%edx)
  328. movdqa %xmm0, 0x40(%edx)
  329. movdqa %xmm0, 0x50(%edx)
  330. movdqa %xmm0, 0x60(%edx)
  331. movdqa %xmm0, 0x70(%edx)
  332. add $0x80, %edx
  333. cmp $0x80, %ebx
  334. jae L(128bytesormore_shared_cache_loop)
  335. cmp $0x80, %ecx
  336. jb L(shared_cache_loop_end)
  337. ALIGN (4)
  338. L(128bytesormore_nt):
  339. sub $0x80, %ecx
  340. movntdq %xmm0, (%edx)
  341. movntdq %xmm0, 0x10(%edx)
  342. movntdq %xmm0, 0x20(%edx)
  343. movntdq %xmm0, 0x30(%edx)
  344. movntdq %xmm0, 0x40(%edx)
  345. movntdq %xmm0, 0x50(%edx)
  346. movntdq %xmm0, 0x60(%edx)
  347. movntdq %xmm0, 0x70(%edx)
  348. add $0x80, %edx
  349. cmp $0x80, %ecx
  350. jae L(128bytesormore_nt)
  351. sfence
  352. L(shared_cache_loop_end):
  353. #if defined DATA_CACHE_SIZE || !defined SHARED
  354. POP (%ebx)
  355. #endif
  356. add %ecx, %edx
  357. shr $2, %ecx
  358. BRANCH_TO_JMPTBL_ENTRY (L(table_16_128bytes))
  359. .pushsection .rodata.sse2,"a",@progbits
  360. ALIGN (2)
  361. L(table_16_128bytes):
  362. .int JMPTBL (L(aligned_16_0bytes), L(table_16_128bytes))
  363. .int JMPTBL (L(aligned_16_4bytes), L(table_16_128bytes))
  364. .int JMPTBL (L(aligned_16_8bytes), L(table_16_128bytes))
  365. .int JMPTBL (L(aligned_16_12bytes), L(table_16_128bytes))
  366. .int JMPTBL (L(aligned_16_16bytes), L(table_16_128bytes))
  367. .int JMPTBL (L(aligned_16_20bytes), L(table_16_128bytes))
  368. .int JMPTBL (L(aligned_16_24bytes), L(table_16_128bytes))
  369. .int JMPTBL (L(aligned_16_28bytes), L(table_16_128bytes))
  370. .int JMPTBL (L(aligned_16_32bytes), L(table_16_128bytes))
  371. .int JMPTBL (L(aligned_16_36bytes), L(table_16_128bytes))
  372. .int JMPTBL (L(aligned_16_40bytes), L(table_16_128bytes))
  373. .int JMPTBL (L(aligned_16_44bytes), L(table_16_128bytes))
  374. .int JMPTBL (L(aligned_16_48bytes), L(table_16_128bytes))
  375. .int JMPTBL (L(aligned_16_52bytes), L(table_16_128bytes))
  376. .int JMPTBL (L(aligned_16_56bytes), L(table_16_128bytes))
  377. .int JMPTBL (L(aligned_16_60bytes), L(table_16_128bytes))
  378. .int JMPTBL (L(aligned_16_64bytes), L(table_16_128bytes))
  379. .int JMPTBL (L(aligned_16_68bytes), L(table_16_128bytes))
  380. .int JMPTBL (L(aligned_16_72bytes), L(table_16_128bytes))
  381. .int JMPTBL (L(aligned_16_76bytes), L(table_16_128bytes))
  382. .int JMPTBL (L(aligned_16_80bytes), L(table_16_128bytes))
  383. .int JMPTBL (L(aligned_16_84bytes), L(table_16_128bytes))
  384. .int JMPTBL (L(aligned_16_88bytes), L(table_16_128bytes))
  385. .int JMPTBL (L(aligned_16_92bytes), L(table_16_128bytes))
  386. .int JMPTBL (L(aligned_16_96bytes), L(table_16_128bytes))
  387. .int JMPTBL (L(aligned_16_100bytes), L(table_16_128bytes))
  388. .int JMPTBL (L(aligned_16_104bytes), L(table_16_128bytes))
  389. .int JMPTBL (L(aligned_16_108bytes), L(table_16_128bytes))
  390. .int JMPTBL (L(aligned_16_112bytes), L(table_16_128bytes))
  391. .int JMPTBL (L(aligned_16_116bytes), L(table_16_128bytes))
  392. .int JMPTBL (L(aligned_16_120bytes), L(table_16_128bytes))
  393. .int JMPTBL (L(aligned_16_124bytes), L(table_16_128bytes))
  394. .popsection
  395. ALIGN (4)
  396. L(aligned_16_112bytes):
  397. movdqa %xmm0, -112(%edx)
  398. L(aligned_16_96bytes):
  399. movdqa %xmm0, -96(%edx)
  400. L(aligned_16_80bytes):
  401. movdqa %xmm0, -80(%edx)
  402. L(aligned_16_64bytes):
  403. movdqa %xmm0, -64(%edx)
  404. L(aligned_16_48bytes):
  405. movdqa %xmm0, -48(%edx)
  406. L(aligned_16_32bytes):
  407. movdqa %xmm0, -32(%edx)
  408. L(aligned_16_16bytes):
  409. movdqa %xmm0, -16(%edx)
  410. L(aligned_16_0bytes):
  411. SETRTNVAL
  412. RETURN
  413. ALIGN (4)
  414. L(aligned_16_116bytes):
  415. movdqa %xmm0, -116(%edx)
  416. L(aligned_16_100bytes):
  417. movdqa %xmm0, -100(%edx)
  418. L(aligned_16_84bytes):
  419. movdqa %xmm0, -84(%edx)
  420. L(aligned_16_68bytes):
  421. movdqa %xmm0, -68(%edx)
  422. L(aligned_16_52bytes):
  423. movdqa %xmm0, -52(%edx)
  424. L(aligned_16_36bytes):
  425. movdqa %xmm0, -36(%edx)
  426. L(aligned_16_20bytes):
  427. movdqa %xmm0, -20(%edx)
  428. L(aligned_16_4bytes):
  429. movl %eax, -4(%edx)
  430. SETRTNVAL
  431. RETURN
  432. ALIGN (4)
  433. L(aligned_16_120bytes):
  434. movdqa %xmm0, -120(%edx)
  435. L(aligned_16_104bytes):
  436. movdqa %xmm0, -104(%edx)
  437. L(aligned_16_88bytes):
  438. movdqa %xmm0, -88(%edx)
  439. L(aligned_16_72bytes):
  440. movdqa %xmm0, -72(%edx)
  441. L(aligned_16_56bytes):
  442. movdqa %xmm0, -56(%edx)
  443. L(aligned_16_40bytes):
  444. movdqa %xmm0, -40(%edx)
  445. L(aligned_16_24bytes):
  446. movdqa %xmm0, -24(%edx)
  447. L(aligned_16_8bytes):
  448. movq %xmm0, -8(%edx)
  449. SETRTNVAL
  450. RETURN
  451. ALIGN (4)
  452. L(aligned_16_124bytes):
  453. movdqa %xmm0, -124(%edx)
  454. L(aligned_16_108bytes):
  455. movdqa %xmm0, -108(%edx)
  456. L(aligned_16_92bytes):
  457. movdqa %xmm0, -92(%edx)
  458. L(aligned_16_76bytes):
  459. movdqa %xmm0, -76(%edx)
  460. L(aligned_16_60bytes):
  461. movdqa %xmm0, -60(%edx)
  462. L(aligned_16_44bytes):
  463. movdqa %xmm0, -44(%edx)
  464. L(aligned_16_28bytes):
  465. movdqa %xmm0, -28(%edx)
  466. L(aligned_16_12bytes):
  467. movq %xmm0, -12(%edx)
  468. movl %eax, -4(%edx)
  469. SETRTNVAL
  470. RETURN
  471. END (sse2_memset32_atom)