vha_mmu.c 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261
  1. /*
  2. *****************************************************************************
  3. * Copyright (c) Imagination Technologies Ltd.
  4. *
  5. * The contents of this file are subject to the MIT license as set out below.
  6. *
  7. * Permission is hereby granted, free of charge, to any person obtaining a
  8. * copy of this software and associated documentation files (the "Software"),
  9. * to deal in the Software without restriction, including without limitation
  10. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11. * and/or sell copies of the Software, and to permit persons to whom the
  12. * Software is furnished to do so, subject to the following conditions:
  13. *
  14. * The above copyright notice and this permission notice shall be included in
  15. * all copies or substantial portions of the Software.
  16. *
  17. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  20. * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22. * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  23. * THE SOFTWARE.
  24. *
  25. * Alternatively, the contents of this file may be used under the terms of the
  26. * GNU General Public License Version 2 ("GPL")in which case the provisions of
  27. * GPL are applicable instead of those above.
  28. *
  29. * If you wish to allow use of your version of this file only under the terms
  30. * of GPL, and not to allow others to use your version of this file under the
  31. * terms of the MIT license, indicate your decision by deleting the provisions
  32. * above and replace them with the notice and other provisions required by GPL
  33. * as set out in the file called "GPLHEADER" included in this distribution. If
  34. * you do not delete the provisions above, a recipient may use your version of
  35. * this file under the terms of either the MIT license or GPL.
  36. *
  37. * This License is also included in this distribution in the file called
  38. * "MIT_COPYING".
  39. *
  40. *****************************************************************************/
  41. #include <linux/moduleparam.h>
  42. #include <linux/delay.h>
  43. #include <uapi/vha.h>
  44. #include "vha_common.h"
  45. #include "vha_plat.h"
  46. #include <vha_regs.h>
  47. int vha_mmu_flush_ctx(struct vha_dev *vha, int ctx_id)
  48. {
  49. int ret;
  50. uint64_t inval =
  51. VHA_SET_FIELD_SIMPLE_VAL(OS0_MMU_CTRL_INVAL, PC, EN) |
  52. VHA_SET_FIELD_SIMPLE_VAL(OS0_MMU_CTRL_INVAL, PD, EN) |
  53. VHA_SET_FIELD_SIMPLE_VAL(OS0_MMU_CTRL_INVAL, PT, EN);
  54. uint64_t pend = VHA_SET_FIELD_SIMPLE_VAL(OS0_MMU_CTRL_INVAL_STATUS, PENDING, EN);
  55. /* No need to handle mmu cache, when core is already offline */
  56. if (vha->state == VHA_STATE_OFF)
  57. return 0;
  58. #ifdef VHA_SCF
  59. if (vha->hw_props.supported.parity && !vha->parity_disable) {
  60. /* If pending bit is set then parity bit must be set as well ! */
  61. pend |= VHA_SET_FIELD_SIMPLE_VAL(OS0_MMU_CTRL_INVAL_STATUS, PARITY, EN);
  62. }
  63. #endif
  64. ret = IOPOLL64_CR_PDUMP_PARITY(0, 30, 150, pend, OS0_MMU_CTRL_INVAL_STATUS);
  65. if (ret) {
  66. dev_err(vha->dev, "Error during MMU ctx %d flush\n", ctx_id);
  67. } else {
  68. if (unlikely(ctx_id == VHA_INVALID_ID))
  69. inval |= VHA_SET_FIELD_SIMPLE_VAL(OS0_MMU_CTRL_INVAL, ALL_CONTEXTS, EN);
  70. else {
  71. inval |= VHA_CR_SETBITS(OS0_MMU_CTRL_INVAL, CONTEXT, (uint64_t)ctx_id);
  72. }
  73. inval |= VHA_CR_SETBITS(OS0_MMU_CTRL_INVAL, CONTEXT, (uint64_t)ctx_id);
  74. dev_dbg(vha->dev, "%s: ctx_id:%d (0x%llx)\n", __func__, ctx_id, inval);
  75. img_pdump_printf("-- MMU invalidate TLB caches\n");
  76. IOWRITE64_CR_PDUMP(inval, OS0_MMU_CTRL_INVAL);
  77. }
  78. return ret;
  79. }
  80. /* this function is called from img_mmu, to handle cache issues */
  81. int vha_mmu_callback(enum img_mmu_callback_type callback_type,
  82. int buf_id, void *data)
  83. {
  84. struct vha_session *session = data;
  85. struct vha_dev *vha = session->vha;
  86. int ctx_id;
  87. int ret = 0;
  88. if (!vha)
  89. return 0;
  90. for (ctx_id = 0; ctx_id < ARRAY_SIZE(session->mmu_ctxs); ctx_id++)
  91. ret |= vha_mmu_flush_ctx(vha, session->mmu_ctxs[ctx_id].hw_id);
  92. if (ret) {
  93. dev_err(vha->dev, "Error during MMU flush (2), resetting the HW...\n");
  94. /* Rollback commands being processed */
  95. vha_rollback_cmds(vha);
  96. /* Perform full reset */
  97. vha_dev_stop(vha, true);
  98. /* Reschedule */
  99. vha_chk_cmd_queues(vha, true);
  100. }
  101. return ret;
  102. }
  103. static void do_mmu_ctx_setup(struct vha_dev *vha,
  104. uint8_t hw_id, int pc_bufid, uint32_t pc_baddr)
  105. {
  106. img_pdump_printf("-- Setup MMU context:%d\n", hw_id);
  107. IOWRITE64_CR_PDUMP(hw_id, OS0_MMU_CBASE_MAPPING_CONTEXT);
  108. if (!vha->mmu_base_pf_test) {
  109. IOWRITE64(vha->reg_base, VHA_CR_OS0_MMU_CBASE_MAPPING, pc_baddr);
  110. /* This is physical address so we need use MEM_OS0:BLOCK tag
  111. * when pdump'ing. */
  112. img_pdump_printf("-- Setup MMU base address\n"
  113. "WRW "_PMEM_":$0 "_PMEM_":BLOCK_%d:0 -- 'PC'\n"
  114. "SHR "_PMEM_":$0 "_PMEM_":$0 %d\n"
  115. "WRW64 :REG:%#x "_PMEM_":$0\n", pc_bufid,
  116. IMG_MMU_PC_ADDR_SHIFT,
  117. VHA_CR_OS0_MMU_CBASE_MAPPING);
  118. dev_dbg(vha->dev, "%s: setting hardware ctx id:%u\n", __func__, hw_id);
  119. } else
  120. dev_info(vha->dev, "Bringup test: force MMU base page fault\n");
  121. }
  122. int vha_mmu_setup(struct vha_session *session)
  123. {
  124. struct vha_dev *vha = session->vha;
  125. int ctx_id;
  126. int ret = 0;
  127. for (ctx_id = 0; ctx_id < ARRAY_SIZE(session->mmu_ctxs); ctx_id++)
  128. dev_dbg(vha->dev,
  129. "%s: mode:%d session ctxid:%x active ctxid:%x\n",
  130. __func__, vha->mmu_mode,
  131. session->mmu_ctxs[ctx_id].id,
  132. vha->active_mmu_ctx);
  133. if (vha->mmu_mode == VHA_MMU_DISABLED) {
  134. img_pdump_printf("-- MMU bypass ON\n");
  135. IOWRITE64_PDUMP(VHA_CR_OS(MMU_CTRL_BYPASS_EN),
  136. VHA_CR_OS(MMU_CTRL));
  137. return 0;
  138. }
  139. /* Using model context to track active context */
  140. if (session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].id == vha->active_mmu_ctx)
  141. return 0;
  142. img_pdump_printf("-- MMU_SETUP_BEGIN\n");
  143. img_pdump_printf("-- MMU bypass OFF\n");
  144. IOWRITE64_PDUMP(0, VHA_CR_OS(MMU_CTRL));
  145. for (ctx_id = 0; ctx_id < ARRAY_SIZE(session->mmu_ctxs); ctx_id++) {
  146. do_mmu_ctx_setup(vha, session->mmu_ctxs[ctx_id].hw_id,
  147. session->mmu_ctxs[ctx_id].pc_bufid,
  148. session->mmu_ctxs[ctx_id].pc_baddr);
  149. /* If there are multiple sessions using the same mmu hardware context
  150. * we need to flush caches for the old context (id is the same).
  151. * This will happen when number of processes is > VHA_MMU_MAX_HW_CTXS */
  152. if (vha->mmu_ctxs[session->mmu_ctxs[ctx_id].hw_id] > 1) {
  153. dev_dbg(vha->dev, "%s: flushing shared ctx id:%u\n",
  154. __func__, session->mmu_ctxs[ctx_id].hw_id);
  155. ret = vha_mmu_flush_ctx(vha, session->mmu_ctxs[ctx_id].hw_id);
  156. if (ret) {
  157. dev_err(vha->dev, "Error during MMU flush, resetting the HW...\n");
  158. goto mmu_setup_err;
  159. }
  160. }
  161. }
  162. /* Using model context to track context change */
  163. vha->active_mmu_ctx = session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].id;
  164. dev_dbg(vha->dev, "%s: update ctx id active:%x pc:%#x\n",
  165. __func__, vha->active_mmu_ctx,
  166. session->mmu_ctxs[VHA_MMU_REQ_MODEL_CTXID].pc_baddr <<
  167. VHA_CR_OS0_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT);
  168. mmu_setup_err:
  169. img_pdump_printf("-- MMU_SETUP_END\n");
  170. return ret;
  171. }
  172. void vha_mmu_status(struct vha_dev *vha, uint8_t core_mask)
  173. {
  174. const char levels[][5] = {"PT", "PD", "PC", "BASE"};
  175. uint32_t core_mmu_fault_reg_set_base = VHA_CR_CORE0_MMU_FAULT_STATUS1;
  176. uint32_t core_mmu_fault_reg_set_offset =
  177. VHA_CR_CORE1_MMU_FAULT_STATUS1 - VHA_CR_CORE0_MMU_FAULT_STATUS1;
  178. uint32_t core_mmu_fault_status1_offset =
  179. VHA_CR_CORE0_MMU_FAULT_STATUS1 - core_mmu_fault_reg_set_base;
  180. uint32_t core_mmu_fault_status2_offset =
  181. VHA_CR_CORE0_MMU_FAULT_STATUS2 - core_mmu_fault_reg_set_base;
  182. while (core_mask) {
  183. uint8_t id = ffs(core_mask) - 1;
  184. uint64_t status1 = IOREAD64_REGIO(core_mmu_fault_reg_set_base +
  185. id* core_mmu_fault_reg_set_offset +
  186. core_mmu_fault_status1_offset);
  187. uint64_t status2 = IOREAD64_REGIO(core_mmu_fault_reg_set_base +
  188. id* core_mmu_fault_reg_set_offset +
  189. core_mmu_fault_status2_offset);
  190. #define MMU_FAULT_GETBITS(sreg, field, val) \
  191. _get_bits(val, VHA_CR_CORE0_MMU_FAULT_ ## sreg ## _ ## field ## _SHIFT, \
  192. ~VHA_CR_CORE0_MMU_FAULT_ ## sreg ## _ ## field ## _CLRMSK)
  193. uint64_t addr = MMU_FAULT_GETBITS(STATUS1, ADDRESS, status1);
  194. uint8_t level = MMU_FAULT_GETBITS(STATUS1, LEVEL, status1);
  195. uint8_t req_id = MMU_FAULT_GETBITS(STATUS1, REQ_ID, status1);
  196. uint8_t ctx = MMU_FAULT_GETBITS(STATUS1, CONTEXT, status1);
  197. uint8_t rnw = MMU_FAULT_GETBITS(STATUS1, RNW, status1);
  198. uint8_t type = MMU_FAULT_GETBITS(STATUS1, TYPE, status1);
  199. uint8_t fault = MMU_FAULT_GETBITS(STATUS1, FAULT, status1);
  200. uint8_t bif_id = MMU_FAULT_GETBITS(STATUS2, BIF_ID, status2);
  201. uint8_t tlb_entry = MMU_FAULT_GETBITS(STATUS2, TLB_ENTRY, status2);
  202. uint8_t slc_bank = MMU_FAULT_GETBITS(STATUS2, BANK, status2);
  203. uint64_t mapping = 0;
  204. #undef MMU_FAULT_GETBITS
  205. /* Select context and read current pc */
  206. IOWRITE64_CR_REGIO(ctx, OS0_MMU_CBASE_MAPPING_CONTEXT);
  207. mapping = IOREAD64_CR_REGIO(OS0_MMU_CBASE_MAPPING);
  208. /* false alarm ? */
  209. if (!fault)
  210. return;
  211. dev_dbg(vha->dev, "%s: Core%u MMU FAULT: s1:%llx s2:%llx\n",
  212. __func__, id, status1, status2);
  213. dev_warn(vha->dev, "%s: MMU fault while %s @ 0x%llx\n",
  214. __func__, (rnw) ? "reading" : "writing", addr << 4);
  215. dev_warn(vha->dev, "%s: level:%s Requestor:%x Context:%x Type:%s\n",
  216. __func__, levels[level], req_id, ctx,
  217. (type == 0) ? "VALID" :
  218. (type == 2) ? "READ-ONLY" :
  219. "UNKNOWN");
  220. dev_warn(vha->dev, "%s: bif_id:%x tlb_entry:%x slc_bank:%x\n",
  221. __func__, bif_id, tlb_entry, slc_bank);
  222. dev_warn(vha->dev, "%s: current mapping@context%d:%#llx\n",
  223. __func__, ctx,
  224. mapping <<
  225. VHA_CR_OS0_MMU_CBASE_MAPPING_BASE_ADDR_ALIGNSHIFT);
  226. core_mask &= ~(VHA_CORE_ID_TO_MASK(id));
  227. }
  228. }