fault.c 8.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Copyright 2014 IBM Corp.
  4. */
  5. #include <linux/workqueue.h>
  6. #include <linux/sched/signal.h>
  7. #include <linux/sched/mm.h>
  8. #include <linux/pid.h>
  9. #include <linux/mm.h>
  10. #include <linux/moduleparam.h>
  11. #undef MODULE_PARAM_PREFIX
  12. #define MODULE_PARAM_PREFIX "cxl" "."
  13. #include <asm/current.h>
  14. #include <asm/copro.h>
  15. #include <asm/mmu.h>
  16. #include "cxl.h"
  17. #include "trace.h"
  18. static bool sste_matches(struct cxl_sste *sste, struct copro_slb *slb)
  19. {
  20. return ((sste->vsid_data == cpu_to_be64(slb->vsid)) &&
  21. (sste->esid_data == cpu_to_be64(slb->esid)));
  22. }
  23. /*
  24. * This finds a free SSTE for the given SLB, or returns NULL if it's already in
  25. * the segment table.
  26. */
  27. static struct cxl_sste *find_free_sste(struct cxl_context *ctx,
  28. struct copro_slb *slb)
  29. {
  30. struct cxl_sste *primary, *sste, *ret = NULL;
  31. unsigned int mask = (ctx->sst_size >> 7) - 1; /* SSTP0[SegTableSize] */
  32. unsigned int entry;
  33. unsigned int hash;
  34. if (slb->vsid & SLB_VSID_B_1T)
  35. hash = (slb->esid >> SID_SHIFT_1T) & mask;
  36. else /* 256M */
  37. hash = (slb->esid >> SID_SHIFT) & mask;
  38. primary = ctx->sstp + (hash << 3);
  39. for (entry = 0, sste = primary; entry < 8; entry++, sste++) {
  40. if (!ret && !(be64_to_cpu(sste->esid_data) & SLB_ESID_V))
  41. ret = sste;
  42. if (sste_matches(sste, slb))
  43. return NULL;
  44. }
  45. if (ret)
  46. return ret;
  47. /* Nothing free, select an entry to cast out */
  48. ret = primary + ctx->sst_lru;
  49. ctx->sst_lru = (ctx->sst_lru + 1) & 0x7;
  50. return ret;
  51. }
  52. static void cxl_load_segment(struct cxl_context *ctx, struct copro_slb *slb)
  53. {
  54. /* mask is the group index, we search primary and secondary here. */
  55. struct cxl_sste *sste;
  56. unsigned long flags;
  57. spin_lock_irqsave(&ctx->sste_lock, flags);
  58. sste = find_free_sste(ctx, slb);
  59. if (!sste)
  60. goto out_unlock;
  61. pr_devel("CXL Populating SST[%li]: %#llx %#llx\n",
  62. sste - ctx->sstp, slb->vsid, slb->esid);
  63. trace_cxl_ste_write(ctx, sste - ctx->sstp, slb->esid, slb->vsid);
  64. sste->vsid_data = cpu_to_be64(slb->vsid);
  65. sste->esid_data = cpu_to_be64(slb->esid);
  66. out_unlock:
  67. spin_unlock_irqrestore(&ctx->sste_lock, flags);
  68. }
  69. static int cxl_fault_segment(struct cxl_context *ctx, struct mm_struct *mm,
  70. u64 ea)
  71. {
  72. struct copro_slb slb = {0,0};
  73. int rc;
  74. if (!(rc = copro_calculate_slb(mm, ea, &slb))) {
  75. cxl_load_segment(ctx, &slb);
  76. }
  77. return rc;
  78. }
  79. static void cxl_ack_ae(struct cxl_context *ctx)
  80. {
  81. unsigned long flags;
  82. cxl_ops->ack_irq(ctx, CXL_PSL_TFC_An_AE, 0);
  83. spin_lock_irqsave(&ctx->lock, flags);
  84. ctx->pending_fault = true;
  85. ctx->fault_addr = ctx->dar;
  86. ctx->fault_dsisr = ctx->dsisr;
  87. spin_unlock_irqrestore(&ctx->lock, flags);
  88. wake_up_all(&ctx->wq);
  89. }
  90. static int cxl_handle_segment_miss(struct cxl_context *ctx,
  91. struct mm_struct *mm, u64 ea)
  92. {
  93. int rc;
  94. pr_devel("CXL interrupt: Segment fault pe: %i ea: %#llx\n", ctx->pe, ea);
  95. trace_cxl_ste_miss(ctx, ea);
  96. if ((rc = cxl_fault_segment(ctx, mm, ea)))
  97. cxl_ack_ae(ctx);
  98. else {
  99. mb(); /* Order seg table write to TFC MMIO write */
  100. cxl_ops->ack_irq(ctx, CXL_PSL_TFC_An_R, 0);
  101. }
  102. return IRQ_HANDLED;
  103. }
  104. int cxl_handle_mm_fault(struct mm_struct *mm, u64 dsisr, u64 dar)
  105. {
  106. vm_fault_t flt = 0;
  107. int result;
  108. unsigned long access, flags, inv_flags = 0;
  109. /*
  110. * Add the fault handling cpu to task mm cpumask so that we
  111. * can do a safe lockless page table walk when inserting the
  112. * hash page table entry. This function get called with a
  113. * valid mm for user space addresses. Hence using the if (mm)
  114. * check is sufficient here.
  115. */
  116. if (mm && !cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm))) {
  117. cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm));
  118. /*
  119. * We need to make sure we walk the table only after
  120. * we update the cpumask. The other side of the barrier
  121. * is explained in serialize_against_pte_lookup()
  122. */
  123. smp_mb();
  124. }
  125. if ((result = copro_handle_mm_fault(mm, dar, dsisr, &flt))) {
  126. pr_devel("copro_handle_mm_fault failed: %#x\n", result);
  127. return result;
  128. }
  129. if (!radix_enabled()) {
  130. /*
  131. * update_mmu_cache() will not have loaded the hash since current->trap
  132. * is not a 0x400 or 0x300, so just call hash_page_mm() here.
  133. */
  134. access = _PAGE_PRESENT | _PAGE_READ;
  135. if (dsisr & CXL_PSL_DSISR_An_S)
  136. access |= _PAGE_WRITE;
  137. if (!mm && (get_region_id(dar) != USER_REGION_ID))
  138. access |= _PAGE_PRIVILEGED;
  139. if (dsisr & DSISR_NOHPTE)
  140. inv_flags |= HPTE_NOHPTE_UPDATE;
  141. local_irq_save(flags);
  142. hash_page_mm(mm, dar, access, 0x300, inv_flags);
  143. local_irq_restore(flags);
  144. }
  145. return 0;
  146. }
  147. static void cxl_handle_page_fault(struct cxl_context *ctx,
  148. struct mm_struct *mm,
  149. u64 dsisr, u64 dar)
  150. {
  151. trace_cxl_pte_miss(ctx, dsisr, dar);
  152. if (cxl_handle_mm_fault(mm, dsisr, dar)) {
  153. cxl_ack_ae(ctx);
  154. } else {
  155. pr_devel("Page fault successfully handled for pe: %i!\n", ctx->pe);
  156. cxl_ops->ack_irq(ctx, CXL_PSL_TFC_An_R, 0);
  157. }
  158. }
  159. /*
  160. * Returns the mm_struct corresponding to the context ctx.
  161. * mm_users == 0, the context may be in the process of being closed.
  162. */
  163. static struct mm_struct *get_mem_context(struct cxl_context *ctx)
  164. {
  165. if (ctx->mm == NULL)
  166. return NULL;
  167. if (!atomic_inc_not_zero(&ctx->mm->mm_users))
  168. return NULL;
  169. return ctx->mm;
  170. }
  171. static bool cxl_is_segment_miss(struct cxl_context *ctx, u64 dsisr)
  172. {
  173. if ((cxl_is_power8() && (dsisr & CXL_PSL_DSISR_An_DS)))
  174. return true;
  175. return false;
  176. }
  177. static bool cxl_is_page_fault(struct cxl_context *ctx, u64 dsisr)
  178. {
  179. if ((cxl_is_power8()) && (dsisr & CXL_PSL_DSISR_An_DM))
  180. return true;
  181. if (cxl_is_power9())
  182. return true;
  183. return false;
  184. }
  185. void cxl_handle_fault(struct work_struct *fault_work)
  186. {
  187. struct cxl_context *ctx =
  188. container_of(fault_work, struct cxl_context, fault_work);
  189. u64 dsisr = ctx->dsisr;
  190. u64 dar = ctx->dar;
  191. struct mm_struct *mm = NULL;
  192. if (cpu_has_feature(CPU_FTR_HVMODE)) {
  193. if (cxl_p2n_read(ctx->afu, CXL_PSL_DSISR_An) != dsisr ||
  194. cxl_p2n_read(ctx->afu, CXL_PSL_DAR_An) != dar ||
  195. cxl_p2n_read(ctx->afu, CXL_PSL_PEHandle_An) != ctx->pe) {
  196. /* Most likely explanation is harmless - a dedicated
  197. * process has detached and these were cleared by the
  198. * PSL purge, but warn about it just in case
  199. */
  200. dev_notice(&ctx->afu->dev, "cxl_handle_fault: Translation fault regs changed\n");
  201. return;
  202. }
  203. }
  204. /* Early return if the context is being / has been detached */
  205. if (ctx->status == CLOSED) {
  206. cxl_ack_ae(ctx);
  207. return;
  208. }
  209. pr_devel("CXL BOTTOM HALF handling fault for afu pe: %i. "
  210. "DSISR: %#llx DAR: %#llx\n", ctx->pe, dsisr, dar);
  211. if (!ctx->kernel) {
  212. mm = get_mem_context(ctx);
  213. if (mm == NULL) {
  214. pr_devel("%s: unable to get mm for pe=%d pid=%i\n",
  215. __func__, ctx->pe, pid_nr(ctx->pid));
  216. cxl_ack_ae(ctx);
  217. return;
  218. } else {
  219. pr_devel("Handling page fault for pe=%d pid=%i\n",
  220. ctx->pe, pid_nr(ctx->pid));
  221. }
  222. }
  223. if (cxl_is_segment_miss(ctx, dsisr))
  224. cxl_handle_segment_miss(ctx, mm, dar);
  225. else if (cxl_is_page_fault(ctx, dsisr))
  226. cxl_handle_page_fault(ctx, mm, dsisr, dar);
  227. else
  228. WARN(1, "cxl_handle_fault has nothing to handle\n");
  229. if (mm)
  230. mmput(mm);
  231. }
  232. static void cxl_prefault_one(struct cxl_context *ctx, u64 ea)
  233. {
  234. struct mm_struct *mm;
  235. mm = get_mem_context(ctx);
  236. if (mm == NULL) {
  237. pr_devel("cxl_prefault_one unable to get mm %i\n",
  238. pid_nr(ctx->pid));
  239. return;
  240. }
  241. cxl_fault_segment(ctx, mm, ea);
  242. mmput(mm);
  243. }
  244. static u64 next_segment(u64 ea, u64 vsid)
  245. {
  246. if (vsid & SLB_VSID_B_1T)
  247. ea |= (1ULL << 40) - 1;
  248. else
  249. ea |= (1ULL << 28) - 1;
  250. return ea + 1;
  251. }
  252. static void cxl_prefault_vma(struct cxl_context *ctx)
  253. {
  254. u64 ea, last_esid = 0;
  255. struct copro_slb slb;
  256. struct vm_area_struct *vma;
  257. int rc;
  258. struct mm_struct *mm;
  259. mm = get_mem_context(ctx);
  260. if (mm == NULL) {
  261. pr_devel("cxl_prefault_vm unable to get mm %i\n",
  262. pid_nr(ctx->pid));
  263. return;
  264. }
  265. mmap_read_lock(mm);
  266. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  267. for (ea = vma->vm_start; ea < vma->vm_end;
  268. ea = next_segment(ea, slb.vsid)) {
  269. rc = copro_calculate_slb(mm, ea, &slb);
  270. if (rc)
  271. continue;
  272. if (last_esid == slb.esid)
  273. continue;
  274. cxl_load_segment(ctx, &slb);
  275. last_esid = slb.esid;
  276. }
  277. }
  278. mmap_read_unlock(mm);
  279. mmput(mm);
  280. }
  281. void cxl_prefault(struct cxl_context *ctx, u64 wed)
  282. {
  283. switch (ctx->afu->prefault_mode) {
  284. case CXL_PREFAULT_WED:
  285. cxl_prefault_one(ctx, wed);
  286. break;
  287. case CXL_PREFAULT_ALL:
  288. cxl_prefault_vma(ctx);
  289. break;
  290. default:
  291. break;
  292. }
  293. }