sun4v_ivec.S 8.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /* sun4v_ivec.S: Sun4v interrupt vector handling.
  3. *
  4. * Copyright (C) 2006 <davem@davemloft.net>
  5. */
  6. #include <asm/cpudata.h>
  7. #include <asm/intr_queue.h>
  8. #include <asm/pil.h>
  9. .text
  10. .align 32
  11. sun4v_cpu_mondo:
  12. /* Head offset in %g2, tail offset in %g4.
  13. * If they are the same, no work.
  14. */
  15. mov INTRQ_CPU_MONDO_HEAD, %g2
  16. ldxa [%g2] ASI_QUEUE, %g2
  17. mov INTRQ_CPU_MONDO_TAIL, %g4
  18. ldxa [%g4] ASI_QUEUE, %g4
  19. cmp %g2, %g4
  20. be,pn %xcc, sun4v_cpu_mondo_queue_empty
  21. nop
  22. /* Get &trap_block[smp_processor_id()] into %g4. */
  23. ldxa [%g0] ASI_SCRATCHPAD, %g4
  24. sub %g4, TRAP_PER_CPU_FAULT_INFO, %g4
  25. /* Get smp_processor_id() into %g3 */
  26. sethi %hi(trap_block), %g5
  27. or %g5, %lo(trap_block), %g5
  28. sub %g4, %g5, %g3
  29. srlx %g3, TRAP_BLOCK_SZ_SHIFT, %g3
  30. /* Increment cpu_mondo_counter[smp_processor_id()] */
  31. sethi %hi(cpu_mondo_counter), %g5
  32. or %g5, %lo(cpu_mondo_counter), %g5
  33. sllx %g3, 3, %g3
  34. add %g5, %g3, %g5
  35. ldx [%g5], %g3
  36. add %g3, 1, %g3
  37. stx %g3, [%g5]
  38. /* Get CPU mondo queue base phys address into %g7. */
  39. ldx [%g4 + TRAP_PER_CPU_CPU_MONDO_PA], %g7
  40. /* Now get the cross-call arguments and handler PC, same
  41. * layout as sun4u:
  42. *
  43. * 1st 64-bit word: low half is 32-bit PC, put into %g3 and jmpl to it
  44. * high half is context arg to MMU flushes, into %g5
  45. * 2nd 64-bit word: 64-bit arg, load into %g1
  46. * 3rd 64-bit word: 64-bit arg, load into %g7
  47. */
  48. ldxa [%g7 + %g2] ASI_PHYS_USE_EC, %g3
  49. add %g2, 0x8, %g2
  50. srlx %g3, 32, %g5
  51. ldxa [%g7 + %g2] ASI_PHYS_USE_EC, %g1
  52. add %g2, 0x8, %g2
  53. srl %g3, 0, %g3
  54. ldxa [%g7 + %g2] ASI_PHYS_USE_EC, %g7
  55. add %g2, 0x40 - 0x8 - 0x8, %g2
  56. /* Update queue head pointer. */
  57. lduw [%g4 + TRAP_PER_CPU_CPU_MONDO_QMASK], %g4
  58. and %g2, %g4, %g2
  59. mov INTRQ_CPU_MONDO_HEAD, %g4
  60. stxa %g2, [%g4] ASI_QUEUE
  61. membar #Sync
  62. jmpl %g3, %g0
  63. nop
  64. sun4v_cpu_mondo_queue_empty:
  65. retry
  66. sun4v_dev_mondo:
  67. /* Head offset in %g2, tail offset in %g4. */
  68. mov INTRQ_DEVICE_MONDO_HEAD, %g2
  69. ldxa [%g2] ASI_QUEUE, %g2
  70. mov INTRQ_DEVICE_MONDO_TAIL, %g4
  71. ldxa [%g4] ASI_QUEUE, %g4
  72. cmp %g2, %g4
  73. be,pn %xcc, sun4v_dev_mondo_queue_empty
  74. nop
  75. /* Get &trap_block[smp_processor_id()] into %g4. */
  76. ldxa [%g0] ASI_SCRATCHPAD, %g4
  77. sub %g4, TRAP_PER_CPU_FAULT_INFO, %g4
  78. /* Get DEV mondo queue base phys address into %g5. */
  79. ldx [%g4 + TRAP_PER_CPU_DEV_MONDO_PA], %g5
  80. /* Load IVEC into %g3. */
  81. ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3
  82. add %g2, 0x40, %g2
  83. /* XXX There can be a full 64-byte block of data here.
  84. * XXX This is how we can get at MSI vector data.
  85. * XXX Current we do not capture this, but when we do we'll
  86. * XXX need to add a 64-byte storage area in the struct ino_bucket
  87. * XXX or the struct irq_desc.
  88. */
  89. /* Update queue head pointer, this frees up some registers. */
  90. lduw [%g4 + TRAP_PER_CPU_DEV_MONDO_QMASK], %g4
  91. and %g2, %g4, %g2
  92. mov INTRQ_DEVICE_MONDO_HEAD, %g4
  93. stxa %g2, [%g4] ASI_QUEUE
  94. membar #Sync
  95. TRAP_LOAD_IRQ_WORK_PA(%g1, %g4)
  96. /* For VIRQs, cookie is encoded as ~bucket_phys_addr */
  97. brlz,pt %g3, 1f
  98. xnor %g3, %g0, %g4
  99. /* Get __pa(&ivector_table[IVEC]) into %g4. */
  100. sethi %hi(ivector_table_pa), %g4
  101. ldx [%g4 + %lo(ivector_table_pa)], %g4
  102. sllx %g3, 4, %g3
  103. add %g4, %g3, %g4
  104. 1: ldx [%g1], %g2
  105. stxa %g2, [%g4] ASI_PHYS_USE_EC
  106. stx %g4, [%g1]
  107. /* Signal the interrupt by setting (1 << pil) in %softint. */
  108. wr %g0, 1 << PIL_DEVICE_IRQ, %set_softint
  109. sun4v_dev_mondo_queue_empty:
  110. retry
  111. sun4v_res_mondo:
  112. /* Head offset in %g2, tail offset in %g4. */
  113. mov INTRQ_RESUM_MONDO_HEAD, %g2
  114. ldxa [%g2] ASI_QUEUE, %g2
  115. mov INTRQ_RESUM_MONDO_TAIL, %g4
  116. ldxa [%g4] ASI_QUEUE, %g4
  117. cmp %g2, %g4
  118. be,pn %xcc, sun4v_res_mondo_queue_empty
  119. nop
  120. /* Get &trap_block[smp_processor_id()] into %g3. */
  121. ldxa [%g0] ASI_SCRATCHPAD, %g3
  122. sub %g3, TRAP_PER_CPU_FAULT_INFO, %g3
  123. /* Get RES mondo queue base phys address into %g5. */
  124. ldx [%g3 + TRAP_PER_CPU_RESUM_MONDO_PA], %g5
  125. /* Get RES kernel buffer base phys address into %g7. */
  126. ldx [%g3 + TRAP_PER_CPU_RESUM_KBUF_PA], %g7
  127. /* If the first word is non-zero, queue is full. */
  128. ldxa [%g7 + %g2] ASI_PHYS_USE_EC, %g1
  129. brnz,pn %g1, sun4v_res_mondo_queue_full
  130. nop
  131. lduw [%g3 + TRAP_PER_CPU_RESUM_QMASK], %g4
  132. /* Remember this entry's offset in %g1. */
  133. mov %g2, %g1
  134. /* Copy 64-byte queue entry into kernel buffer. */
  135. ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3
  136. stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC
  137. add %g2, 0x08, %g2
  138. ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3
  139. stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC
  140. add %g2, 0x08, %g2
  141. ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3
  142. stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC
  143. add %g2, 0x08, %g2
  144. ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3
  145. stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC
  146. add %g2, 0x08, %g2
  147. ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3
  148. stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC
  149. add %g2, 0x08, %g2
  150. ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3
  151. stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC
  152. add %g2, 0x08, %g2
  153. ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3
  154. stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC
  155. add %g2, 0x08, %g2
  156. ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3
  157. stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC
  158. add %g2, 0x08, %g2
  159. /* Update queue head pointer. */
  160. and %g2, %g4, %g2
  161. mov INTRQ_RESUM_MONDO_HEAD, %g4
  162. stxa %g2, [%g4] ASI_QUEUE
  163. membar #Sync
  164. /* Disable interrupts and save register state so we can call
  165. * C code. The etrap handling will leave %g4 in %l4 for us
  166. * when it's done.
  167. */
  168. rdpr %pil, %g2
  169. wrpr %g0, PIL_NORMAL_MAX, %pil
  170. mov %g1, %g4
  171. ba,pt %xcc, etrap_irq
  172. rd %pc, %g7
  173. #ifdef CONFIG_TRACE_IRQFLAGS
  174. call trace_hardirqs_off
  175. nop
  176. #endif
  177. /* Log the event. */
  178. add %sp, PTREGS_OFF, %o0
  179. call sun4v_resum_error
  180. mov %l4, %o1
  181. /* Return from trap. */
  182. ba,pt %xcc, rtrap_irq
  183. nop
  184. sun4v_res_mondo_queue_empty:
  185. retry
  186. sun4v_res_mondo_queue_full:
  187. /* The queue is full, consolidate our damage by setting
  188. * the head equal to the tail. We'll just trap again otherwise.
  189. * Call C code to log the event.
  190. */
  191. mov INTRQ_RESUM_MONDO_HEAD, %g2
  192. stxa %g4, [%g2] ASI_QUEUE
  193. membar #Sync
  194. rdpr %pil, %g2
  195. wrpr %g0, PIL_NORMAL_MAX, %pil
  196. ba,pt %xcc, etrap_irq
  197. rd %pc, %g7
  198. #ifdef CONFIG_TRACE_IRQFLAGS
  199. call trace_hardirqs_off
  200. nop
  201. #endif
  202. call sun4v_resum_overflow
  203. add %sp, PTREGS_OFF, %o0
  204. ba,pt %xcc, rtrap_irq
  205. nop
  206. sun4v_nonres_mondo:
  207. /* Head offset in %g2, tail offset in %g4. */
  208. mov INTRQ_NONRESUM_MONDO_HEAD, %g2
  209. ldxa [%g2] ASI_QUEUE, %g2
  210. mov INTRQ_NONRESUM_MONDO_TAIL, %g4
  211. ldxa [%g4] ASI_QUEUE, %g4
  212. cmp %g2, %g4
  213. be,pn %xcc, sun4v_nonres_mondo_queue_empty
  214. nop
  215. /* Get &trap_block[smp_processor_id()] into %g3. */
  216. ldxa [%g0] ASI_SCRATCHPAD, %g3
  217. sub %g3, TRAP_PER_CPU_FAULT_INFO, %g3
  218. /* Get RES mondo queue base phys address into %g5. */
  219. ldx [%g3 + TRAP_PER_CPU_NONRESUM_MONDO_PA], %g5
  220. /* Get RES kernel buffer base phys address into %g7. */
  221. ldx [%g3 + TRAP_PER_CPU_NONRESUM_KBUF_PA], %g7
  222. /* If the first word is non-zero, queue is full. */
  223. ldxa [%g7 + %g2] ASI_PHYS_USE_EC, %g1
  224. brnz,pn %g1, sun4v_nonres_mondo_queue_full
  225. nop
  226. lduw [%g3 + TRAP_PER_CPU_NONRESUM_QMASK], %g4
  227. /* Remember this entry's offset in %g1. */
  228. mov %g2, %g1
  229. /* Copy 64-byte queue entry into kernel buffer. */
  230. ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3
  231. stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC
  232. add %g2, 0x08, %g2
  233. ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3
  234. stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC
  235. add %g2, 0x08, %g2
  236. ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3
  237. stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC
  238. add %g2, 0x08, %g2
  239. ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3
  240. stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC
  241. add %g2, 0x08, %g2
  242. ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3
  243. stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC
  244. add %g2, 0x08, %g2
  245. ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3
  246. stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC
  247. add %g2, 0x08, %g2
  248. ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3
  249. stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC
  250. add %g2, 0x08, %g2
  251. ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3
  252. stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC
  253. add %g2, 0x08, %g2
  254. /* Update queue head pointer. */
  255. and %g2, %g4, %g2
  256. mov INTRQ_NONRESUM_MONDO_HEAD, %g4
  257. stxa %g2, [%g4] ASI_QUEUE
  258. membar #Sync
  259. /* Disable interrupts and save register state so we can call
  260. * C code. The etrap handling will leave %g4 in %l4 for us
  261. * when it's done.
  262. */
  263. rdpr %pil, %g2
  264. wrpr %g0, PIL_NORMAL_MAX, %pil
  265. mov %g1, %g4
  266. ba,pt %xcc, etrap_irq
  267. rd %pc, %g7
  268. #ifdef CONFIG_TRACE_IRQFLAGS
  269. call trace_hardirqs_off
  270. nop
  271. #endif
  272. /* Log the event. */
  273. add %sp, PTREGS_OFF, %o0
  274. call sun4v_nonresum_error
  275. mov %l4, %o1
  276. /* Return from trap. */
  277. ba,pt %xcc, rtrap_irq
  278. nop
  279. sun4v_nonres_mondo_queue_empty:
  280. retry
  281. sun4v_nonres_mondo_queue_full:
  282. /* The queue is full, consolidate our damage by setting
  283. * the head equal to the tail. We'll just trap again otherwise.
  284. * Call C code to log the event.
  285. */
  286. mov INTRQ_NONRESUM_MONDO_HEAD, %g2
  287. stxa %g4, [%g2] ASI_QUEUE
  288. membar #Sync
  289. rdpr %pil, %g2
  290. wrpr %g0, PIL_NORMAL_MAX, %pil
  291. ba,pt %xcc, etrap_irq
  292. rd %pc, %g7
  293. #ifdef CONFIG_TRACE_IRQFLAGS
  294. call trace_hardirqs_off
  295. nop
  296. #endif
  297. call sun4v_nonresum_overflow
  298. add %sp, PTREGS_OFF, %o0
  299. ba,pt %xcc, rtrap_irq
  300. nop