cxllib.c 6.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Copyright 2017 IBM Corp.
  4. */
  5. #include <linux/hugetlb.h>
  6. #include <linux/sched/mm.h>
  7. #include <asm/pnv-pci.h>
  8. #include <misc/cxllib.h>
  9. #include "cxl.h"
  10. #define CXL_INVALID_DRA ~0ull
  11. #define CXL_DUMMY_READ_SIZE 128
  12. #define CXL_DUMMY_READ_ALIGN 8
  13. #define CXL_CAPI_WINDOW_START 0x2000000000000ull
  14. #define CXL_CAPI_WINDOW_LOG_SIZE 48
  15. #define CXL_XSL_CONFIG_CURRENT_VERSION CXL_XSL_CONFIG_VERSION1
  16. bool cxllib_slot_is_supported(struct pci_dev *dev, unsigned long flags)
  17. {
  18. int rc;
  19. u32 phb_index;
  20. u64 chip_id, capp_unit_id;
  21. /* No flags currently supported */
  22. if (flags)
  23. return false;
  24. if (!cpu_has_feature(CPU_FTR_HVMODE))
  25. return false;
  26. if (!cxl_is_power9())
  27. return false;
  28. if (cxl_slot_is_switched(dev))
  29. return false;
  30. /* on p9, some pci slots are not connected to a CAPP unit */
  31. rc = cxl_calc_capp_routing(dev, &chip_id, &phb_index, &capp_unit_id);
  32. if (rc)
  33. return false;
  34. return true;
  35. }
  36. EXPORT_SYMBOL_GPL(cxllib_slot_is_supported);
  37. static DEFINE_MUTEX(dra_mutex);
  38. static u64 dummy_read_addr = CXL_INVALID_DRA;
  39. static int allocate_dummy_read_buf(void)
  40. {
  41. u64 buf, vaddr;
  42. size_t buf_size;
  43. /*
  44. * Dummy read buffer is 128-byte long, aligned on a
  45. * 256-byte boundary and we need the physical address.
  46. */
  47. buf_size = CXL_DUMMY_READ_SIZE + (1ull << CXL_DUMMY_READ_ALIGN);
  48. buf = (u64) kzalloc(buf_size, GFP_KERNEL);
  49. if (!buf)
  50. return -ENOMEM;
  51. vaddr = (buf + (1ull << CXL_DUMMY_READ_ALIGN) - 1) &
  52. (~0ull << CXL_DUMMY_READ_ALIGN);
  53. WARN((vaddr + CXL_DUMMY_READ_SIZE) > (buf + buf_size),
  54. "Dummy read buffer alignment issue");
  55. dummy_read_addr = virt_to_phys((void *) vaddr);
  56. return 0;
  57. }
  58. int cxllib_get_xsl_config(struct pci_dev *dev, struct cxllib_xsl_config *cfg)
  59. {
  60. int rc;
  61. u32 phb_index;
  62. u64 chip_id, capp_unit_id;
  63. if (!cpu_has_feature(CPU_FTR_HVMODE))
  64. return -EINVAL;
  65. mutex_lock(&dra_mutex);
  66. if (dummy_read_addr == CXL_INVALID_DRA) {
  67. rc = allocate_dummy_read_buf();
  68. if (rc) {
  69. mutex_unlock(&dra_mutex);
  70. return rc;
  71. }
  72. }
  73. mutex_unlock(&dra_mutex);
  74. rc = cxl_calc_capp_routing(dev, &chip_id, &phb_index, &capp_unit_id);
  75. if (rc)
  76. return rc;
  77. rc = cxl_get_xsl9_dsnctl(dev, capp_unit_id, &cfg->dsnctl);
  78. if (rc)
  79. return rc;
  80. cfg->version = CXL_XSL_CONFIG_CURRENT_VERSION;
  81. cfg->log_bar_size = CXL_CAPI_WINDOW_LOG_SIZE;
  82. cfg->bar_addr = CXL_CAPI_WINDOW_START;
  83. cfg->dra = dummy_read_addr;
  84. return 0;
  85. }
  86. EXPORT_SYMBOL_GPL(cxllib_get_xsl_config);
  87. int cxllib_switch_phb_mode(struct pci_dev *dev, enum cxllib_mode mode,
  88. unsigned long flags)
  89. {
  90. int rc = 0;
  91. if (!cpu_has_feature(CPU_FTR_HVMODE))
  92. return -EINVAL;
  93. switch (mode) {
  94. case CXL_MODE_PCI:
  95. /*
  96. * We currently don't support going back to PCI mode
  97. * However, we'll turn the invalidations off, so that
  98. * the firmware doesn't have to ack them and can do
  99. * things like reset, etc.. with no worries.
  100. * So always return EPERM (can't go back to PCI) or
  101. * EBUSY if we couldn't even turn off snooping
  102. */
  103. rc = pnv_phb_to_cxl_mode(dev, OPAL_PHB_CAPI_MODE_SNOOP_OFF);
  104. if (rc)
  105. rc = -EBUSY;
  106. else
  107. rc = -EPERM;
  108. break;
  109. case CXL_MODE_CXL:
  110. /* DMA only supported on TVT1 for the time being */
  111. if (flags != CXL_MODE_DMA_TVT1)
  112. return -EINVAL;
  113. rc = pnv_phb_to_cxl_mode(dev, OPAL_PHB_CAPI_MODE_DMA_TVT1);
  114. if (rc)
  115. return rc;
  116. rc = pnv_phb_to_cxl_mode(dev, OPAL_PHB_CAPI_MODE_SNOOP_ON);
  117. break;
  118. default:
  119. rc = -EINVAL;
  120. }
  121. return rc;
  122. }
  123. EXPORT_SYMBOL_GPL(cxllib_switch_phb_mode);
  124. /*
  125. * When switching the PHB to capi mode, the TVT#1 entry for
  126. * the Partitionable Endpoint is set in bypass mode, like
  127. * in PCI mode.
  128. * Configure the device dma to use TVT#1, which is done
  129. * by calling dma_set_mask() with a mask large enough.
  130. */
  131. int cxllib_set_device_dma(struct pci_dev *dev, unsigned long flags)
  132. {
  133. int rc;
  134. if (flags)
  135. return -EINVAL;
  136. rc = dma_set_mask(&dev->dev, DMA_BIT_MASK(64));
  137. return rc;
  138. }
  139. EXPORT_SYMBOL_GPL(cxllib_set_device_dma);
  140. int cxllib_get_PE_attributes(struct task_struct *task,
  141. unsigned long translation_mode,
  142. struct cxllib_pe_attributes *attr)
  143. {
  144. struct mm_struct *mm = NULL;
  145. if (translation_mode != CXL_TRANSLATED_MODE &&
  146. translation_mode != CXL_REAL_MODE)
  147. return -EINVAL;
  148. attr->sr = cxl_calculate_sr(false,
  149. task == NULL,
  150. translation_mode == CXL_REAL_MODE,
  151. true);
  152. attr->lpid = mfspr(SPRN_LPID);
  153. if (task) {
  154. mm = get_task_mm(task);
  155. if (mm == NULL)
  156. return -EINVAL;
  157. /*
  158. * Caller is keeping a reference on mm_users for as long
  159. * as XSL uses the memory context
  160. */
  161. attr->pid = mm->context.id;
  162. mmput(mm);
  163. attr->tid = task->thread.tidr;
  164. } else {
  165. attr->pid = 0;
  166. attr->tid = 0;
  167. }
  168. return 0;
  169. }
  170. EXPORT_SYMBOL_GPL(cxllib_get_PE_attributes);
  171. static int get_vma_info(struct mm_struct *mm, u64 addr,
  172. u64 *vma_start, u64 *vma_end,
  173. unsigned long *page_size)
  174. {
  175. struct vm_area_struct *vma = NULL;
  176. int rc = 0;
  177. mmap_read_lock(mm);
  178. vma = find_vma(mm, addr);
  179. if (!vma) {
  180. rc = -EFAULT;
  181. goto out;
  182. }
  183. *page_size = vma_kernel_pagesize(vma);
  184. *vma_start = vma->vm_start;
  185. *vma_end = vma->vm_end;
  186. out:
  187. mmap_read_unlock(mm);
  188. return rc;
  189. }
  190. int cxllib_handle_fault(struct mm_struct *mm, u64 addr, u64 size, u64 flags)
  191. {
  192. int rc;
  193. u64 dar, vma_start, vma_end;
  194. unsigned long page_size;
  195. if (mm == NULL)
  196. return -EFAULT;
  197. /*
  198. * The buffer we have to process can extend over several pages
  199. * and may also cover several VMAs.
  200. * We iterate over all the pages. The page size could vary
  201. * between VMAs.
  202. */
  203. rc = get_vma_info(mm, addr, &vma_start, &vma_end, &page_size);
  204. if (rc)
  205. return rc;
  206. for (dar = (addr & ~(page_size - 1)); dar < (addr + size);
  207. dar += page_size) {
  208. if (dar < vma_start || dar >= vma_end) {
  209. /*
  210. * We don't hold mm->mmap_lock while iterating, since
  211. * the lock is required by one of the lower-level page
  212. * fault processing functions and it could
  213. * create a deadlock.
  214. *
  215. * It means the VMAs can be altered between 2
  216. * loop iterations and we could theoretically
  217. * miss a page (however unlikely). But that's
  218. * not really a problem, as the driver will
  219. * retry access, get another page fault on the
  220. * missing page and call us again.
  221. */
  222. rc = get_vma_info(mm, dar, &vma_start, &vma_end,
  223. &page_size);
  224. if (rc)
  225. return rc;
  226. }
  227. rc = cxl_handle_mm_fault(mm, flags, dar);
  228. if (rc)
  229. return -EFAULT;
  230. }
  231. return 0;
  232. }
  233. EXPORT_SYMBOL_GPL(cxllib_handle_fault);