ats.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * PCI Express I/O Virtualization (IOV) support
  4. * Address Translation Service 1.0
  5. * Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com>
  6. * PASID support added by Joerg Roedel <joerg.roedel@amd.com>
  7. *
  8. * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>
  9. * Copyright (C) 2011 Advanced Micro Devices,
  10. */
  11. #include <linux/export.h>
  12. #include <linux/pci-ats.h>
  13. #include <linux/pci.h>
  14. #include <linux/slab.h>
  15. #include "pci.h"
  16. void pci_ats_init(struct pci_dev *dev)
  17. {
  18. int pos;
  19. if (pci_ats_disabled())
  20. return;
  21. pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS);
  22. if (!pos)
  23. return;
  24. dev->ats_cap = pos;
  25. }
  26. /**
  27. * pci_ats_supported - check if the device can use ATS
  28. * @dev: the PCI device
  29. *
  30. * Returns true if the device supports ATS and is allowed to use it, false
  31. * otherwise.
  32. */
  33. bool pci_ats_supported(struct pci_dev *dev)
  34. {
  35. if (!dev->ats_cap)
  36. return false;
  37. return (dev->untrusted == 0);
  38. }
  39. EXPORT_SYMBOL_GPL(pci_ats_supported);
  40. /**
  41. * pci_enable_ats - enable the ATS capability
  42. * @dev: the PCI device
  43. * @ps: the IOMMU page shift
  44. *
  45. * Returns 0 on success, or negative on failure.
  46. */
  47. int pci_enable_ats(struct pci_dev *dev, int ps)
  48. {
  49. u16 ctrl;
  50. struct pci_dev *pdev;
  51. if (!pci_ats_supported(dev))
  52. return -EINVAL;
  53. if (WARN_ON(dev->ats_enabled))
  54. return -EBUSY;
  55. if (ps < PCI_ATS_MIN_STU)
  56. return -EINVAL;
  57. /*
  58. * Note that enabling ATS on a VF fails unless it's already enabled
  59. * with the same STU on the PF.
  60. */
  61. ctrl = PCI_ATS_CTRL_ENABLE;
  62. if (dev->is_virtfn) {
  63. pdev = pci_physfn(dev);
  64. if (pdev->ats_stu != ps)
  65. return -EINVAL;
  66. } else {
  67. dev->ats_stu = ps;
  68. ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
  69. }
  70. pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
  71. dev->ats_enabled = 1;
  72. return 0;
  73. }
  74. EXPORT_SYMBOL_GPL(pci_enable_ats);
  75. /**
  76. * pci_disable_ats - disable the ATS capability
  77. * @dev: the PCI device
  78. */
  79. void pci_disable_ats(struct pci_dev *dev)
  80. {
  81. u16 ctrl;
  82. if (WARN_ON(!dev->ats_enabled))
  83. return;
  84. pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl);
  85. ctrl &= ~PCI_ATS_CTRL_ENABLE;
  86. pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
  87. dev->ats_enabled = 0;
  88. }
  89. EXPORT_SYMBOL_GPL(pci_disable_ats);
  90. void pci_restore_ats_state(struct pci_dev *dev)
  91. {
  92. u16 ctrl;
  93. if (!dev->ats_enabled)
  94. return;
  95. ctrl = PCI_ATS_CTRL_ENABLE;
  96. if (!dev->is_virtfn)
  97. ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
  98. pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
  99. }
  100. /**
  101. * pci_ats_queue_depth - query the ATS Invalidate Queue Depth
  102. * @dev: the PCI device
  103. *
  104. * Returns the queue depth on success, or negative on failure.
  105. *
  106. * The ATS spec uses 0 in the Invalidate Queue Depth field to
  107. * indicate that the function can accept 32 Invalidate Request.
  108. * But here we use the `real' values (i.e. 1~32) for the Queue
  109. * Depth; and 0 indicates the function shares the Queue with
  110. * other functions (doesn't exclusively own a Queue).
  111. */
  112. int pci_ats_queue_depth(struct pci_dev *dev)
  113. {
  114. u16 cap;
  115. if (!dev->ats_cap)
  116. return -EINVAL;
  117. if (dev->is_virtfn)
  118. return 0;
  119. pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap);
  120. return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : PCI_ATS_MAX_QDEP;
  121. }
  122. /**
  123. * pci_ats_page_aligned - Return Page Aligned Request bit status.
  124. * @pdev: the PCI device
  125. *
  126. * Returns 1, if the Untranslated Addresses generated by the device
  127. * are always aligned or 0 otherwise.
  128. *
  129. * Per PCIe spec r4.0, sec 10.5.1.2, if the Page Aligned Request bit
  130. * is set, it indicates the Untranslated Addresses generated by the
  131. * device are always aligned to a 4096 byte boundary.
  132. */
  133. int pci_ats_page_aligned(struct pci_dev *pdev)
  134. {
  135. u16 cap;
  136. if (!pdev->ats_cap)
  137. return 0;
  138. pci_read_config_word(pdev, pdev->ats_cap + PCI_ATS_CAP, &cap);
  139. if (cap & PCI_ATS_CAP_PAGE_ALIGNED)
  140. return 1;
  141. return 0;
  142. }
  143. #ifdef CONFIG_PCI_PRI
  144. void pci_pri_init(struct pci_dev *pdev)
  145. {
  146. u16 status;
  147. pdev->pri_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
  148. if (!pdev->pri_cap)
  149. return;
  150. pci_read_config_word(pdev, pdev->pri_cap + PCI_PRI_STATUS, &status);
  151. if (status & PCI_PRI_STATUS_PASID)
  152. pdev->pasid_required = 1;
  153. }
  154. /**
  155. * pci_enable_pri - Enable PRI capability
  156. * @pdev: PCI device structure
  157. * @reqs: outstanding requests
  158. *
  159. * Returns 0 on success, negative value on error
  160. */
  161. int pci_enable_pri(struct pci_dev *pdev, u32 reqs)
  162. {
  163. u16 control, status;
  164. u32 max_requests;
  165. int pri = pdev->pri_cap;
  166. /*
  167. * VFs must not implement the PRI Capability. If their PF
  168. * implements PRI, it is shared by the VFs, so if the PF PRI is
  169. * enabled, it is also enabled for the VF.
  170. */
  171. if (pdev->is_virtfn) {
  172. if (pci_physfn(pdev)->pri_enabled)
  173. return 0;
  174. return -EINVAL;
  175. }
  176. if (WARN_ON(pdev->pri_enabled))
  177. return -EBUSY;
  178. if (!pri)
  179. return -EINVAL;
  180. pci_read_config_word(pdev, pri + PCI_PRI_STATUS, &status);
  181. if (!(status & PCI_PRI_STATUS_STOPPED))
  182. return -EBUSY;
  183. pci_read_config_dword(pdev, pri + PCI_PRI_MAX_REQ, &max_requests);
  184. reqs = min(max_requests, reqs);
  185. pdev->pri_reqs_alloc = reqs;
  186. pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs);
  187. control = PCI_PRI_CTRL_ENABLE;
  188. pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
  189. pdev->pri_enabled = 1;
  190. return 0;
  191. }
  192. /**
  193. * pci_disable_pri - Disable PRI capability
  194. * @pdev: PCI device structure
  195. *
  196. * Only clears the enabled-bit, regardless of its former value
  197. */
  198. void pci_disable_pri(struct pci_dev *pdev)
  199. {
  200. u16 control;
  201. int pri = pdev->pri_cap;
  202. /* VFs share the PF PRI */
  203. if (pdev->is_virtfn)
  204. return;
  205. if (WARN_ON(!pdev->pri_enabled))
  206. return;
  207. if (!pri)
  208. return;
  209. pci_read_config_word(pdev, pri + PCI_PRI_CTRL, &control);
  210. control &= ~PCI_PRI_CTRL_ENABLE;
  211. pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
  212. pdev->pri_enabled = 0;
  213. }
  214. EXPORT_SYMBOL_GPL(pci_disable_pri);
  215. /**
  216. * pci_restore_pri_state - Restore PRI
  217. * @pdev: PCI device structure
  218. */
  219. void pci_restore_pri_state(struct pci_dev *pdev)
  220. {
  221. u16 control = PCI_PRI_CTRL_ENABLE;
  222. u32 reqs = pdev->pri_reqs_alloc;
  223. int pri = pdev->pri_cap;
  224. if (pdev->is_virtfn)
  225. return;
  226. if (!pdev->pri_enabled)
  227. return;
  228. if (!pri)
  229. return;
  230. pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs);
  231. pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
  232. }
  233. /**
  234. * pci_reset_pri - Resets device's PRI state
  235. * @pdev: PCI device structure
  236. *
  237. * The PRI capability must be disabled before this function is called.
  238. * Returns 0 on success, negative value on error.
  239. */
  240. int pci_reset_pri(struct pci_dev *pdev)
  241. {
  242. u16 control;
  243. int pri = pdev->pri_cap;
  244. if (pdev->is_virtfn)
  245. return 0;
  246. if (WARN_ON(pdev->pri_enabled))
  247. return -EBUSY;
  248. if (!pri)
  249. return -EINVAL;
  250. control = PCI_PRI_CTRL_RESET;
  251. pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
  252. return 0;
  253. }
  254. /**
  255. * pci_prg_resp_pasid_required - Return PRG Response PASID Required bit
  256. * status.
  257. * @pdev: PCI device structure
  258. *
  259. * Returns 1 if PASID is required in PRG Response Message, 0 otherwise.
  260. */
  261. int pci_prg_resp_pasid_required(struct pci_dev *pdev)
  262. {
  263. if (pdev->is_virtfn)
  264. pdev = pci_physfn(pdev);
  265. return pdev->pasid_required;
  266. }
  267. /**
  268. * pci_pri_supported - Check if PRI is supported.
  269. * @pdev: PCI device structure
  270. *
  271. * Returns true if PRI capability is present, false otherwise.
  272. */
  273. bool pci_pri_supported(struct pci_dev *pdev)
  274. {
  275. /* VFs share the PF PRI */
  276. if (pci_physfn(pdev)->pri_cap)
  277. return true;
  278. return false;
  279. }
  280. EXPORT_SYMBOL_GPL(pci_pri_supported);
  281. #endif /* CONFIG_PCI_PRI */
  282. #ifdef CONFIG_PCI_PASID
  283. void pci_pasid_init(struct pci_dev *pdev)
  284. {
  285. pdev->pasid_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
  286. }
  287. /**
  288. * pci_enable_pasid - Enable the PASID capability
  289. * @pdev: PCI device structure
  290. * @features: Features to enable
  291. *
  292. * Returns 0 on success, negative value on error. This function checks
  293. * whether the features are actually supported by the device and returns
  294. * an error if not.
  295. */
  296. int pci_enable_pasid(struct pci_dev *pdev, int features)
  297. {
  298. u16 control, supported;
  299. int pasid = pdev->pasid_cap;
  300. /*
  301. * VFs must not implement the PASID Capability, but if a PF
  302. * supports PASID, its VFs share the PF PASID configuration.
  303. */
  304. if (pdev->is_virtfn) {
  305. if (pci_physfn(pdev)->pasid_enabled)
  306. return 0;
  307. return -EINVAL;
  308. }
  309. if (WARN_ON(pdev->pasid_enabled))
  310. return -EBUSY;
  311. if (!pdev->eetlp_prefix_path)
  312. return -EINVAL;
  313. if (!pasid)
  314. return -EINVAL;
  315. pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
  316. supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;
  317. /* User wants to enable anything unsupported? */
  318. if ((supported & features) != features)
  319. return -EINVAL;
  320. control = PCI_PASID_CTRL_ENABLE | features;
  321. pdev->pasid_features = features;
  322. pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);
  323. pdev->pasid_enabled = 1;
  324. return 0;
  325. }
  326. EXPORT_SYMBOL_GPL(pci_enable_pasid);
  327. /**
  328. * pci_disable_pasid - Disable the PASID capability
  329. * @pdev: PCI device structure
  330. */
  331. void pci_disable_pasid(struct pci_dev *pdev)
  332. {
  333. u16 control = 0;
  334. int pasid = pdev->pasid_cap;
  335. /* VFs share the PF PASID configuration */
  336. if (pdev->is_virtfn)
  337. return;
  338. if (WARN_ON(!pdev->pasid_enabled))
  339. return;
  340. if (!pasid)
  341. return;
  342. pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);
  343. pdev->pasid_enabled = 0;
  344. }
  345. EXPORT_SYMBOL_GPL(pci_disable_pasid);
  346. /**
  347. * pci_restore_pasid_state - Restore PASID capabilities
  348. * @pdev: PCI device structure
  349. */
  350. void pci_restore_pasid_state(struct pci_dev *pdev)
  351. {
  352. u16 control;
  353. int pasid = pdev->pasid_cap;
  354. if (pdev->is_virtfn)
  355. return;
  356. if (!pdev->pasid_enabled)
  357. return;
  358. if (!pasid)
  359. return;
  360. control = PCI_PASID_CTRL_ENABLE | pdev->pasid_features;
  361. pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);
  362. }
  363. /**
  364. * pci_pasid_features - Check which PASID features are supported
  365. * @pdev: PCI device structure
  366. *
  367. * Returns a negative value when no PASI capability is present.
  368. * Otherwise is returns a bitmask with supported features. Current
  369. * features reported are:
  370. * PCI_PASID_CAP_EXEC - Execute permission supported
  371. * PCI_PASID_CAP_PRIV - Privileged mode supported
  372. */
  373. int pci_pasid_features(struct pci_dev *pdev)
  374. {
  375. u16 supported;
  376. int pasid;
  377. if (pdev->is_virtfn)
  378. pdev = pci_physfn(pdev);
  379. pasid = pdev->pasid_cap;
  380. if (!pasid)
  381. return -EINVAL;
  382. pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
  383. supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;
  384. return supported;
  385. }
  386. EXPORT_SYMBOL_GPL(pci_pasid_features);
  387. #define PASID_NUMBER_SHIFT 8
  388. #define PASID_NUMBER_MASK (0x1f << PASID_NUMBER_SHIFT)
  389. /**
  390. * pci_max_pasid - Get maximum number of PASIDs supported by device
  391. * @pdev: PCI device structure
  392. *
  393. * Returns negative value when PASID capability is not present.
  394. * Otherwise it returns the number of supported PASIDs.
  395. */
  396. int pci_max_pasids(struct pci_dev *pdev)
  397. {
  398. u16 supported;
  399. int pasid;
  400. if (pdev->is_virtfn)
  401. pdev = pci_physfn(pdev);
  402. pasid = pdev->pasid_cap;
  403. if (!pasid)
  404. return -EINVAL;
  405. pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
  406. supported = (supported & PASID_NUMBER_MASK) >> PASID_NUMBER_SHIFT;
  407. return (1 << supported);
  408. }
  409. EXPORT_SYMBOL_GPL(pci_max_pasids);
  410. #endif /* CONFIG_PCI_PASID */