context.c 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Copyright 2014 IBM Corp.
  4. */
  5. #include <linux/module.h>
  6. #include <linux/kernel.h>
  7. #include <linux/bitmap.h>
  8. #include <linux/sched.h>
  9. #include <linux/pid.h>
  10. #include <linux/fs.h>
  11. #include <linux/mm.h>
  12. #include <linux/debugfs.h>
  13. #include <linux/slab.h>
  14. #include <linux/idr.h>
  15. #include <linux/sched/mm.h>
  16. #include <linux/mmu_context.h>
  17. #include <asm/cputable.h>
  18. #include <asm/current.h>
  19. #include <asm/copro.h>
  20. #include "cxl.h"
  21. /*
  22. * Allocates space for a CXL context.
  23. */
  24. struct cxl_context *cxl_context_alloc(void)
  25. {
  26. return kzalloc(sizeof(struct cxl_context), GFP_KERNEL);
  27. }
  28. /*
  29. * Initialises a CXL context.
  30. */
  31. int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master)
  32. {
  33. int i;
  34. ctx->afu = afu;
  35. ctx->master = master;
  36. ctx->pid = NULL; /* Set in start work ioctl */
  37. mutex_init(&ctx->mapping_lock);
  38. ctx->mapping = NULL;
  39. ctx->tidr = 0;
  40. ctx->assign_tidr = false;
  41. if (cxl_is_power8()) {
  42. spin_lock_init(&ctx->sste_lock);
  43. /*
  44. * Allocate the segment table before we put it in the IDR so that we
  45. * can always access it when dereferenced from IDR. For the same
  46. * reason, the segment table is only destroyed after the context is
  47. * removed from the IDR. Access to this in the IOCTL is protected by
  48. * Linux filesytem symantics (can't IOCTL until open is complete).
  49. */
  50. i = cxl_alloc_sst(ctx);
  51. if (i)
  52. return i;
  53. }
  54. INIT_WORK(&ctx->fault_work, cxl_handle_fault);
  55. init_waitqueue_head(&ctx->wq);
  56. spin_lock_init(&ctx->lock);
  57. ctx->irq_bitmap = NULL;
  58. ctx->pending_irq = false;
  59. ctx->pending_fault = false;
  60. ctx->pending_afu_err = false;
  61. INIT_LIST_HEAD(&ctx->irq_names);
  62. /*
  63. * When we have to destroy all contexts in cxl_context_detach_all() we
  64. * end up with afu_release_irqs() called from inside a
  65. * idr_for_each_entry(). Hence we need to make sure that anything
  66. * dereferenced from this IDR is ok before we allocate the IDR here.
  67. * This clears out the IRQ ranges to ensure this.
  68. */
  69. for (i = 0; i < CXL_IRQ_RANGES; i++)
  70. ctx->irqs.range[i] = 0;
  71. mutex_init(&ctx->status_mutex);
  72. ctx->status = OPENED;
  73. /*
  74. * Allocating IDR! We better make sure everything's setup that
  75. * dereferences from it.
  76. */
  77. mutex_lock(&afu->contexts_lock);
  78. idr_preload(GFP_KERNEL);
  79. i = idr_alloc(&ctx->afu->contexts_idr, ctx, 0,
  80. ctx->afu->num_procs, GFP_NOWAIT);
  81. idr_preload_end();
  82. mutex_unlock(&afu->contexts_lock);
  83. if (i < 0)
  84. return i;
  85. ctx->pe = i;
  86. if (cpu_has_feature(CPU_FTR_HVMODE)) {
  87. ctx->elem = &ctx->afu->native->spa[i];
  88. ctx->external_pe = ctx->pe;
  89. } else {
  90. ctx->external_pe = -1; /* assigned when attaching */
  91. }
  92. ctx->pe_inserted = false;
  93. /*
  94. * take a ref on the afu so that it stays alive at-least till
  95. * this context is reclaimed inside reclaim_ctx.
  96. */
  97. cxl_afu_get(afu);
  98. return 0;
  99. }
  100. void cxl_context_set_mapping(struct cxl_context *ctx,
  101. struct address_space *mapping)
  102. {
  103. mutex_lock(&ctx->mapping_lock);
  104. ctx->mapping = mapping;
  105. mutex_unlock(&ctx->mapping_lock);
  106. }
  107. static vm_fault_t cxl_mmap_fault(struct vm_fault *vmf)
  108. {
  109. struct vm_area_struct *vma = vmf->vma;
  110. struct cxl_context *ctx = vma->vm_file->private_data;
  111. u64 area, offset;
  112. vm_fault_t ret;
  113. offset = vmf->pgoff << PAGE_SHIFT;
  114. pr_devel("%s: pe: %i address: 0x%lx offset: 0x%llx\n",
  115. __func__, ctx->pe, vmf->address, offset);
  116. if (ctx->afu->current_mode == CXL_MODE_DEDICATED) {
  117. area = ctx->afu->psn_phys;
  118. if (offset >= ctx->afu->adapter->ps_size)
  119. return VM_FAULT_SIGBUS;
  120. } else {
  121. area = ctx->psn_phys;
  122. if (offset >= ctx->psn_size)
  123. return VM_FAULT_SIGBUS;
  124. }
  125. mutex_lock(&ctx->status_mutex);
  126. if (ctx->status != STARTED) {
  127. mutex_unlock(&ctx->status_mutex);
  128. pr_devel("%s: Context not started, failing problem state access\n", __func__);
  129. if (ctx->mmio_err_ff) {
  130. if (!ctx->ff_page) {
  131. ctx->ff_page = alloc_page(GFP_USER);
  132. if (!ctx->ff_page)
  133. return VM_FAULT_OOM;
  134. memset(page_address(ctx->ff_page), 0xff, PAGE_SIZE);
  135. }
  136. get_page(ctx->ff_page);
  137. vmf->page = ctx->ff_page;
  138. vma->vm_page_prot = pgprot_cached(vma->vm_page_prot);
  139. return 0;
  140. }
  141. return VM_FAULT_SIGBUS;
  142. }
  143. ret = vmf_insert_pfn(vma, vmf->address, (area + offset) >> PAGE_SHIFT);
  144. mutex_unlock(&ctx->status_mutex);
  145. return ret;
  146. }
  147. static const struct vm_operations_struct cxl_mmap_vmops = {
  148. .fault = cxl_mmap_fault,
  149. };
  150. /*
  151. * Map a per-context mmio space into the given vma.
  152. */
  153. int cxl_context_iomap(struct cxl_context *ctx, struct vm_area_struct *vma)
  154. {
  155. u64 start = vma->vm_pgoff << PAGE_SHIFT;
  156. u64 len = vma->vm_end - vma->vm_start;
  157. if (ctx->afu->current_mode == CXL_MODE_DEDICATED) {
  158. if (start + len > ctx->afu->adapter->ps_size)
  159. return -EINVAL;
  160. if (cxl_is_power9()) {
  161. /*
  162. * Make sure there is a valid problem state
  163. * area space for this AFU.
  164. */
  165. if (ctx->master && !ctx->afu->psa) {
  166. pr_devel("AFU doesn't support mmio space\n");
  167. return -EINVAL;
  168. }
  169. /* Can't mmap until the AFU is enabled */
  170. if (!ctx->afu->enabled)
  171. return -EBUSY;
  172. }
  173. } else {
  174. if (start + len > ctx->psn_size)
  175. return -EINVAL;
  176. /* Make sure there is a valid per process space for this AFU */
  177. if ((ctx->master && !ctx->afu->psa) || (!ctx->afu->pp_psa)) {
  178. pr_devel("AFU doesn't support mmio space\n");
  179. return -EINVAL;
  180. }
  181. /* Can't mmap until the AFU is enabled */
  182. if (!ctx->afu->enabled)
  183. return -EBUSY;
  184. }
  185. pr_devel("%s: mmio physical: %llx pe: %i master:%i\n", __func__,
  186. ctx->psn_phys, ctx->pe , ctx->master);
  187. vma->vm_flags |= VM_IO | VM_PFNMAP;
  188. vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
  189. vma->vm_ops = &cxl_mmap_vmops;
  190. return 0;
  191. }
  192. /*
  193. * Detach a context from the hardware. This disables interrupts and doesn't
  194. * return until all outstanding interrupts for this context have completed. The
  195. * hardware should no longer access *ctx after this has returned.
  196. */
  197. int __detach_context(struct cxl_context *ctx)
  198. {
  199. enum cxl_context_status status;
  200. mutex_lock(&ctx->status_mutex);
  201. status = ctx->status;
  202. ctx->status = CLOSED;
  203. mutex_unlock(&ctx->status_mutex);
  204. if (status != STARTED)
  205. return -EBUSY;
  206. /* Only warn if we detached while the link was OK.
  207. * If detach fails when hw is down, we don't care.
  208. */
  209. WARN_ON(cxl_ops->detach_process(ctx) &&
  210. cxl_ops->link_ok(ctx->afu->adapter, ctx->afu));
  211. flush_work(&ctx->fault_work); /* Only needed for dedicated process */
  212. /*
  213. * Wait until no further interrupts are presented by the PSL
  214. * for this context.
  215. */
  216. if (cxl_ops->irq_wait)
  217. cxl_ops->irq_wait(ctx);
  218. /* release the reference to the group leader and mm handling pid */
  219. put_pid(ctx->pid);
  220. cxl_ctx_put();
  221. /* Decrease the attached context count on the adapter */
  222. cxl_adapter_context_put(ctx->afu->adapter);
  223. /* Decrease the mm count on the context */
  224. cxl_context_mm_count_put(ctx);
  225. if (ctx->mm)
  226. mm_context_remove_copro(ctx->mm);
  227. ctx->mm = NULL;
  228. return 0;
  229. }
  230. /*
  231. * Detach the given context from the AFU. This doesn't actually
  232. * free the context but it should stop the context running in hardware
  233. * (ie. prevent this context from generating any further interrupts
  234. * so that it can be freed).
  235. */
  236. void cxl_context_detach(struct cxl_context *ctx)
  237. {
  238. int rc;
  239. rc = __detach_context(ctx);
  240. if (rc)
  241. return;
  242. afu_release_irqs(ctx, ctx);
  243. wake_up_all(&ctx->wq);
  244. }
  245. /*
  246. * Detach all contexts on the given AFU.
  247. */
  248. void cxl_context_detach_all(struct cxl_afu *afu)
  249. {
  250. struct cxl_context *ctx;
  251. int tmp;
  252. mutex_lock(&afu->contexts_lock);
  253. idr_for_each_entry(&afu->contexts_idr, ctx, tmp) {
  254. /*
  255. * Anything done in here needs to be setup before the IDR is
  256. * created and torn down after the IDR removed
  257. */
  258. cxl_context_detach(ctx);
  259. /*
  260. * We are force detaching - remove any active PSA mappings so
  261. * userspace cannot interfere with the card if it comes back.
  262. * Easiest way to exercise this is to unbind and rebind the
  263. * driver via sysfs while it is in use.
  264. */
  265. mutex_lock(&ctx->mapping_lock);
  266. if (ctx->mapping)
  267. unmap_mapping_range(ctx->mapping, 0, 0, 1);
  268. mutex_unlock(&ctx->mapping_lock);
  269. }
  270. mutex_unlock(&afu->contexts_lock);
  271. }
  272. static void reclaim_ctx(struct rcu_head *rcu)
  273. {
  274. struct cxl_context *ctx = container_of(rcu, struct cxl_context, rcu);
  275. if (cxl_is_power8())
  276. free_page((u64)ctx->sstp);
  277. if (ctx->ff_page)
  278. __free_page(ctx->ff_page);
  279. ctx->sstp = NULL;
  280. kfree(ctx->irq_bitmap);
  281. /* Drop ref to the afu device taken during cxl_context_init */
  282. cxl_afu_put(ctx->afu);
  283. kfree(ctx);
  284. }
  285. void cxl_context_free(struct cxl_context *ctx)
  286. {
  287. if (ctx->kernelapi && ctx->mapping)
  288. cxl_release_mapping(ctx);
  289. mutex_lock(&ctx->afu->contexts_lock);
  290. idr_remove(&ctx->afu->contexts_idr, ctx->pe);
  291. mutex_unlock(&ctx->afu->contexts_lock);
  292. call_rcu(&ctx->rcu, reclaim_ctx);
  293. }
  294. void cxl_context_mm_count_get(struct cxl_context *ctx)
  295. {
  296. if (ctx->mm)
  297. mmgrab(ctx->mm);
  298. }
  299. void cxl_context_mm_count_put(struct cxl_context *ctx)
  300. {
  301. if (ctx->mm)
  302. mmdrop(ctx->mm);
  303. }