context.c 7.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294
  1. // SPDX-License-Identifier: GPL-2.0+
  2. // Copyright 2017 IBM Corp.
  3. #include <linux/sched/mm.h>
  4. #include "trace.h"
  5. #include "ocxl_internal.h"
  6. int ocxl_context_alloc(struct ocxl_context **context, struct ocxl_afu *afu,
  7. struct address_space *mapping)
  8. {
  9. int pasid;
  10. struct ocxl_context *ctx;
  11. ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
  12. if (!ctx)
  13. return -ENOMEM;
  14. ctx->afu = afu;
  15. mutex_lock(&afu->contexts_lock);
  16. pasid = idr_alloc(&afu->contexts_idr, ctx, afu->pasid_base,
  17. afu->pasid_base + afu->pasid_max, GFP_KERNEL);
  18. if (pasid < 0) {
  19. mutex_unlock(&afu->contexts_lock);
  20. kfree(ctx);
  21. return pasid;
  22. }
  23. afu->pasid_count++;
  24. mutex_unlock(&afu->contexts_lock);
  25. ctx->pasid = pasid;
  26. ctx->status = OPENED;
  27. mutex_init(&ctx->status_mutex);
  28. ctx->mapping = mapping;
  29. mutex_init(&ctx->mapping_lock);
  30. init_waitqueue_head(&ctx->events_wq);
  31. mutex_init(&ctx->xsl_error_lock);
  32. mutex_init(&ctx->irq_lock);
  33. idr_init(&ctx->irq_idr);
  34. ctx->tidr = 0;
  35. /*
  36. * Keep a reference on the AFU to make sure it's valid for the
  37. * duration of the life of the context
  38. */
  39. ocxl_afu_get(afu);
  40. *context = ctx;
  41. return 0;
  42. }
  43. EXPORT_SYMBOL_GPL(ocxl_context_alloc);
  44. /*
  45. * Callback for when a translation fault triggers an error
  46. * data: a pointer to the context which triggered the fault
  47. * addr: the address that triggered the error
  48. * dsisr: the value of the PPC64 dsisr register
  49. */
  50. static void xsl_fault_error(void *data, u64 addr, u64 dsisr)
  51. {
  52. struct ocxl_context *ctx = (struct ocxl_context *) data;
  53. mutex_lock(&ctx->xsl_error_lock);
  54. ctx->xsl_error.addr = addr;
  55. ctx->xsl_error.dsisr = dsisr;
  56. ctx->xsl_error.count++;
  57. mutex_unlock(&ctx->xsl_error_lock);
  58. wake_up_all(&ctx->events_wq);
  59. }
  60. int ocxl_context_attach(struct ocxl_context *ctx, u64 amr, struct mm_struct *mm)
  61. {
  62. int rc;
  63. unsigned long pidr = 0;
  64. // Locks both status & tidr
  65. mutex_lock(&ctx->status_mutex);
  66. if (ctx->status != OPENED) {
  67. rc = -EIO;
  68. goto out;
  69. }
  70. if (mm)
  71. pidr = mm->context.id;
  72. rc = ocxl_link_add_pe(ctx->afu->fn->link, ctx->pasid, pidr, ctx->tidr,
  73. amr, mm, xsl_fault_error, ctx);
  74. if (rc)
  75. goto out;
  76. ctx->status = ATTACHED;
  77. out:
  78. mutex_unlock(&ctx->status_mutex);
  79. return rc;
  80. }
  81. EXPORT_SYMBOL_GPL(ocxl_context_attach);
  82. static vm_fault_t map_afu_irq(struct vm_area_struct *vma, unsigned long address,
  83. u64 offset, struct ocxl_context *ctx)
  84. {
  85. u64 trigger_addr;
  86. int irq_id = ocxl_irq_offset_to_id(ctx, offset);
  87. trigger_addr = ocxl_afu_irq_get_addr(ctx, irq_id);
  88. if (!trigger_addr)
  89. return VM_FAULT_SIGBUS;
  90. return vmf_insert_pfn(vma, address, trigger_addr >> PAGE_SHIFT);
  91. }
  92. static vm_fault_t map_pp_mmio(struct vm_area_struct *vma, unsigned long address,
  93. u64 offset, struct ocxl_context *ctx)
  94. {
  95. u64 pp_mmio_addr;
  96. int pasid_off;
  97. vm_fault_t ret;
  98. if (offset >= ctx->afu->config.pp_mmio_stride)
  99. return VM_FAULT_SIGBUS;
  100. mutex_lock(&ctx->status_mutex);
  101. if (ctx->status != ATTACHED) {
  102. mutex_unlock(&ctx->status_mutex);
  103. pr_debug("%s: Context not attached, failing mmio mmap\n",
  104. __func__);
  105. return VM_FAULT_SIGBUS;
  106. }
  107. pasid_off = ctx->pasid - ctx->afu->pasid_base;
  108. pp_mmio_addr = ctx->afu->pp_mmio_start +
  109. pasid_off * ctx->afu->config.pp_mmio_stride +
  110. offset;
  111. ret = vmf_insert_pfn(vma, address, pp_mmio_addr >> PAGE_SHIFT);
  112. mutex_unlock(&ctx->status_mutex);
  113. return ret;
  114. }
  115. static vm_fault_t ocxl_mmap_fault(struct vm_fault *vmf)
  116. {
  117. struct vm_area_struct *vma = vmf->vma;
  118. struct ocxl_context *ctx = vma->vm_file->private_data;
  119. u64 offset;
  120. vm_fault_t ret;
  121. offset = vmf->pgoff << PAGE_SHIFT;
  122. pr_debug("%s: pasid %d address 0x%lx offset 0x%llx\n", __func__,
  123. ctx->pasid, vmf->address, offset);
  124. if (offset < ctx->afu->irq_base_offset)
  125. ret = map_pp_mmio(vma, vmf->address, offset, ctx);
  126. else
  127. ret = map_afu_irq(vma, vmf->address, offset, ctx);
  128. return ret;
  129. }
  130. static const struct vm_operations_struct ocxl_vmops = {
  131. .fault = ocxl_mmap_fault,
  132. };
  133. static int check_mmap_afu_irq(struct ocxl_context *ctx,
  134. struct vm_area_struct *vma)
  135. {
  136. int irq_id = ocxl_irq_offset_to_id(ctx, vma->vm_pgoff << PAGE_SHIFT);
  137. /* only one page */
  138. if (vma_pages(vma) != 1)
  139. return -EINVAL;
  140. /* check offset validty */
  141. if (!ocxl_afu_irq_get_addr(ctx, irq_id))
  142. return -EINVAL;
  143. /*
  144. * trigger page should only be accessible in write mode.
  145. *
  146. * It's a bit theoretical, as a page mmaped with only
  147. * PROT_WRITE is currently readable, but it doesn't hurt.
  148. */
  149. if ((vma->vm_flags & VM_READ) || (vma->vm_flags & VM_EXEC) ||
  150. !(vma->vm_flags & VM_WRITE))
  151. return -EINVAL;
  152. vma->vm_flags &= ~(VM_MAYREAD | VM_MAYEXEC);
  153. return 0;
  154. }
  155. static int check_mmap_mmio(struct ocxl_context *ctx,
  156. struct vm_area_struct *vma)
  157. {
  158. if ((vma_pages(vma) + vma->vm_pgoff) >
  159. (ctx->afu->config.pp_mmio_stride >> PAGE_SHIFT))
  160. return -EINVAL;
  161. return 0;
  162. }
  163. int ocxl_context_mmap(struct ocxl_context *ctx, struct vm_area_struct *vma)
  164. {
  165. int rc;
  166. if ((vma->vm_pgoff << PAGE_SHIFT) < ctx->afu->irq_base_offset)
  167. rc = check_mmap_mmio(ctx, vma);
  168. else
  169. rc = check_mmap_afu_irq(ctx, vma);
  170. if (rc)
  171. return rc;
  172. vma->vm_flags |= VM_IO | VM_PFNMAP;
  173. vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
  174. vma->vm_ops = &ocxl_vmops;
  175. return 0;
  176. }
  177. int ocxl_context_detach(struct ocxl_context *ctx)
  178. {
  179. struct pci_dev *dev;
  180. int afu_control_pos;
  181. enum ocxl_context_status status;
  182. int rc;
  183. mutex_lock(&ctx->status_mutex);
  184. status = ctx->status;
  185. ctx->status = CLOSED;
  186. mutex_unlock(&ctx->status_mutex);
  187. if (status != ATTACHED)
  188. return 0;
  189. dev = to_pci_dev(ctx->afu->fn->dev.parent);
  190. afu_control_pos = ctx->afu->config.dvsec_afu_control_pos;
  191. mutex_lock(&ctx->afu->afu_control_lock);
  192. rc = ocxl_config_terminate_pasid(dev, afu_control_pos, ctx->pasid);
  193. mutex_unlock(&ctx->afu->afu_control_lock);
  194. trace_ocxl_terminate_pasid(ctx->pasid, rc);
  195. if (rc) {
  196. /*
  197. * If we timeout waiting for the AFU to terminate the
  198. * pasid, then it's dangerous to clean up the Process
  199. * Element entry in the SPA, as it may be referenced
  200. * in the future by the AFU. In which case, we would
  201. * checkstop because of an invalid PE access (FIR
  202. * register 2, bit 42). So leave the PE
  203. * defined. Caller shouldn't free the context so that
  204. * PASID remains allocated.
  205. *
  206. * A link reset will be required to cleanup the AFU
  207. * and the SPA.
  208. */
  209. if (rc == -EBUSY)
  210. return rc;
  211. }
  212. rc = ocxl_link_remove_pe(ctx->afu->fn->link, ctx->pasid);
  213. if (rc) {
  214. dev_warn(&dev->dev,
  215. "Couldn't remove PE entry cleanly: %d\n", rc);
  216. }
  217. return 0;
  218. }
  219. EXPORT_SYMBOL_GPL(ocxl_context_detach);
  220. void ocxl_context_detach_all(struct ocxl_afu *afu)
  221. {
  222. struct ocxl_context *ctx;
  223. int tmp;
  224. mutex_lock(&afu->contexts_lock);
  225. idr_for_each_entry(&afu->contexts_idr, ctx, tmp) {
  226. ocxl_context_detach(ctx);
  227. /*
  228. * We are force detaching - remove any active mmio
  229. * mappings so userspace cannot interfere with the
  230. * card if it comes back. Easiest way to exercise
  231. * this is to unbind and rebind the driver via sysfs
  232. * while it is in use.
  233. */
  234. mutex_lock(&ctx->mapping_lock);
  235. if (ctx->mapping)
  236. unmap_mapping_range(ctx->mapping, 0, 0, 1);
  237. mutex_unlock(&ctx->mapping_lock);
  238. }
  239. mutex_unlock(&afu->contexts_lock);
  240. }
  241. void ocxl_context_free(struct ocxl_context *ctx)
  242. {
  243. mutex_lock(&ctx->afu->contexts_lock);
  244. ctx->afu->pasid_count--;
  245. idr_remove(&ctx->afu->contexts_idr, ctx->pasid);
  246. mutex_unlock(&ctx->afu->contexts_lock);
  247. ocxl_afu_irq_free_all(ctx);
  248. idr_destroy(&ctx->irq_idr);
  249. /* reference to the AFU taken in ocxl_context_alloc() */
  250. ocxl_afu_put(ctx->afu);
  251. kfree(ctx);
  252. }
  253. EXPORT_SYMBOL_GPL(ocxl_context_free);