dca.c 8.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Intel I/OAT DMA Linux driver
  4. * Copyright(c) 2007 - 2009 Intel Corporation.
  5. */
  6. #include <linux/kernel.h>
  7. #include <linux/pci.h>
  8. #include <linux/smp.h>
  9. #include <linux/interrupt.h>
  10. #include <linux/dca.h>
  11. /* either a kernel change is needed, or we need something like this in kernel */
  12. #ifndef CONFIG_SMP
  13. #include <asm/smp.h>
  14. #undef cpu_physical_id
  15. #define cpu_physical_id(cpu) (cpuid_ebx(1) >> 24)
  16. #endif
  17. #include "dma.h"
  18. #include "registers.h"
  19. /*
  20. * Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6
  21. * contain the bit number of the APIC ID to map into the DCA tag. If the valid
  22. * bit is not set, then the value must be 0 or 1 and defines the bit in the tag.
  23. */
  24. #define DCA_TAG_MAP_VALID 0x80
  25. #define DCA3_TAG_MAP_BIT_TO_INV 0x80
  26. #define DCA3_TAG_MAP_BIT_TO_SEL 0x40
  27. #define DCA3_TAG_MAP_LITERAL_VAL 0x1
  28. #define DCA_TAG_MAP_MASK 0xDF
  29. /* expected tag map bytes for I/OAT ver.2 */
  30. #define DCA2_TAG_MAP_BYTE0 0x80
  31. #define DCA2_TAG_MAP_BYTE1 0x0
  32. #define DCA2_TAG_MAP_BYTE2 0x81
  33. #define DCA2_TAG_MAP_BYTE3 0x82
  34. #define DCA2_TAG_MAP_BYTE4 0x82
  35. /*
  36. * "Legacy" DCA systems do not implement the DCA register set in the
  37. * I/OAT device. Software needs direct support for their tag mappings.
  38. */
  39. #define APICID_BIT(x) (DCA_TAG_MAP_VALID | (x))
  40. #define IOAT_TAG_MAP_LEN 8
  41. /* pack PCI B/D/F into a u16 */
  42. static inline u16 dcaid_from_pcidev(struct pci_dev *pci)
  43. {
  44. return (pci->bus->number << 8) | pci->devfn;
  45. }
  46. static int dca_enabled_in_bios(struct pci_dev *pdev)
  47. {
  48. /* CPUID level 9 returns DCA configuration */
  49. /* Bit 0 indicates DCA enabled by the BIOS */
  50. unsigned long cpuid_level_9;
  51. int res;
  52. cpuid_level_9 = cpuid_eax(9);
  53. res = test_bit(0, &cpuid_level_9);
  54. if (!res)
  55. dev_dbg(&pdev->dev, "DCA is disabled in BIOS\n");
  56. return res;
  57. }
  58. int system_has_dca_enabled(struct pci_dev *pdev)
  59. {
  60. if (boot_cpu_has(X86_FEATURE_DCA))
  61. return dca_enabled_in_bios(pdev);
  62. dev_dbg(&pdev->dev, "boot cpu doesn't have X86_FEATURE_DCA\n");
  63. return 0;
  64. }
  65. struct ioat_dca_slot {
  66. struct pci_dev *pdev; /* requester device */
  67. u16 rid; /* requester id, as used by IOAT */
  68. };
  69. #define IOAT_DCA_MAX_REQ 6
  70. #define IOAT3_DCA_MAX_REQ 2
  71. struct ioat_dca_priv {
  72. void __iomem *iobase;
  73. void __iomem *dca_base;
  74. int max_requesters;
  75. int requester_count;
  76. u8 tag_map[IOAT_TAG_MAP_LEN];
  77. struct ioat_dca_slot req_slots[];
  78. };
  79. static int ioat_dca_dev_managed(struct dca_provider *dca,
  80. struct device *dev)
  81. {
  82. struct ioat_dca_priv *ioatdca = dca_priv(dca);
  83. struct pci_dev *pdev;
  84. int i;
  85. pdev = to_pci_dev(dev);
  86. for (i = 0; i < ioatdca->max_requesters; i++) {
  87. if (ioatdca->req_slots[i].pdev == pdev)
  88. return 1;
  89. }
  90. return 0;
  91. }
  92. static int ioat_dca_add_requester(struct dca_provider *dca, struct device *dev)
  93. {
  94. struct ioat_dca_priv *ioatdca = dca_priv(dca);
  95. struct pci_dev *pdev;
  96. int i;
  97. u16 id;
  98. u16 global_req_table;
  99. /* This implementation only supports PCI-Express */
  100. if (!dev_is_pci(dev))
  101. return -ENODEV;
  102. pdev = to_pci_dev(dev);
  103. id = dcaid_from_pcidev(pdev);
  104. if (ioatdca->requester_count == ioatdca->max_requesters)
  105. return -ENODEV;
  106. for (i = 0; i < ioatdca->max_requesters; i++) {
  107. if (ioatdca->req_slots[i].pdev == NULL) {
  108. /* found an empty slot */
  109. ioatdca->requester_count++;
  110. ioatdca->req_slots[i].pdev = pdev;
  111. ioatdca->req_slots[i].rid = id;
  112. global_req_table =
  113. readw(ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET);
  114. writel(id | IOAT_DCA_GREQID_VALID,
  115. ioatdca->iobase + global_req_table + (i * 4));
  116. return i;
  117. }
  118. }
  119. /* Error, ioatdma->requester_count is out of whack */
  120. return -EFAULT;
  121. }
  122. static int ioat_dca_remove_requester(struct dca_provider *dca,
  123. struct device *dev)
  124. {
  125. struct ioat_dca_priv *ioatdca = dca_priv(dca);
  126. struct pci_dev *pdev;
  127. int i;
  128. u16 global_req_table;
  129. /* This implementation only supports PCI-Express */
  130. if (!dev_is_pci(dev))
  131. return -ENODEV;
  132. pdev = to_pci_dev(dev);
  133. for (i = 0; i < ioatdca->max_requesters; i++) {
  134. if (ioatdca->req_slots[i].pdev == pdev) {
  135. global_req_table =
  136. readw(ioatdca->dca_base + IOAT3_DCA_GREQID_OFFSET);
  137. writel(0, ioatdca->iobase + global_req_table + (i * 4));
  138. ioatdca->req_slots[i].pdev = NULL;
  139. ioatdca->req_slots[i].rid = 0;
  140. ioatdca->requester_count--;
  141. return i;
  142. }
  143. }
  144. return -ENODEV;
  145. }
  146. static u8 ioat_dca_get_tag(struct dca_provider *dca,
  147. struct device *dev,
  148. int cpu)
  149. {
  150. u8 tag;
  151. struct ioat_dca_priv *ioatdca = dca_priv(dca);
  152. int i, apic_id, bit, value;
  153. u8 entry;
  154. tag = 0;
  155. apic_id = cpu_physical_id(cpu);
  156. for (i = 0; i < IOAT_TAG_MAP_LEN; i++) {
  157. entry = ioatdca->tag_map[i];
  158. if (entry & DCA3_TAG_MAP_BIT_TO_SEL) {
  159. bit = entry &
  160. ~(DCA3_TAG_MAP_BIT_TO_SEL | DCA3_TAG_MAP_BIT_TO_INV);
  161. value = (apic_id & (1 << bit)) ? 1 : 0;
  162. } else if (entry & DCA3_TAG_MAP_BIT_TO_INV) {
  163. bit = entry & ~DCA3_TAG_MAP_BIT_TO_INV;
  164. value = (apic_id & (1 << bit)) ? 0 : 1;
  165. } else {
  166. value = (entry & DCA3_TAG_MAP_LITERAL_VAL) ? 1 : 0;
  167. }
  168. tag |= (value << i);
  169. }
  170. return tag;
  171. }
  172. static const struct dca_ops ioat_dca_ops = {
  173. .add_requester = ioat_dca_add_requester,
  174. .remove_requester = ioat_dca_remove_requester,
  175. .get_tag = ioat_dca_get_tag,
  176. .dev_managed = ioat_dca_dev_managed,
  177. };
  178. static int ioat_dca_count_dca_slots(void *iobase, u16 dca_offset)
  179. {
  180. int slots = 0;
  181. u32 req;
  182. u16 global_req_table;
  183. global_req_table = readw(iobase + dca_offset + IOAT3_DCA_GREQID_OFFSET);
  184. if (global_req_table == 0)
  185. return 0;
  186. do {
  187. req = readl(iobase + global_req_table + (slots * sizeof(u32)));
  188. slots++;
  189. } while ((req & IOAT_DCA_GREQID_LASTID) == 0);
  190. return slots;
  191. }
  192. static inline int dca3_tag_map_invalid(u8 *tag_map)
  193. {
  194. /*
  195. * If the tag map is not programmed by the BIOS the default is:
  196. * 0x80 0x80 0x80 0x80 0x80 0x00 0x00 0x00
  197. *
  198. * This an invalid map and will result in only 2 possible tags
  199. * 0x1F and 0x00. 0x00 is an invalid DCA tag so we know that
  200. * this entire definition is invalid.
  201. */
  202. return ((tag_map[0] == DCA_TAG_MAP_VALID) &&
  203. (tag_map[1] == DCA_TAG_MAP_VALID) &&
  204. (tag_map[2] == DCA_TAG_MAP_VALID) &&
  205. (tag_map[3] == DCA_TAG_MAP_VALID) &&
  206. (tag_map[4] == DCA_TAG_MAP_VALID));
  207. }
  208. struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
  209. {
  210. struct dca_provider *dca;
  211. struct ioat_dca_priv *ioatdca;
  212. int slots;
  213. int i;
  214. int err;
  215. u16 dca_offset;
  216. u16 csi_fsb_control;
  217. u16 pcie_control;
  218. u8 bit;
  219. union {
  220. u64 full;
  221. struct {
  222. u32 low;
  223. u32 high;
  224. };
  225. } tag_map;
  226. if (!system_has_dca_enabled(pdev))
  227. return NULL;
  228. dca_offset = readw(iobase + IOAT_DCAOFFSET_OFFSET);
  229. if (dca_offset == 0)
  230. return NULL;
  231. slots = ioat_dca_count_dca_slots(iobase, dca_offset);
  232. if (slots == 0)
  233. return NULL;
  234. dca = alloc_dca_provider(&ioat_dca_ops,
  235. struct_size(ioatdca, req_slots, slots));
  236. if (!dca)
  237. return NULL;
  238. ioatdca = dca_priv(dca);
  239. ioatdca->iobase = iobase;
  240. ioatdca->dca_base = iobase + dca_offset;
  241. ioatdca->max_requesters = slots;
  242. /* some bios might not know to turn these on */
  243. csi_fsb_control = readw(ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET);
  244. if ((csi_fsb_control & IOAT3_CSI_CONTROL_PREFETCH) == 0) {
  245. csi_fsb_control |= IOAT3_CSI_CONTROL_PREFETCH;
  246. writew(csi_fsb_control,
  247. ioatdca->dca_base + IOAT3_CSI_CONTROL_OFFSET);
  248. }
  249. pcie_control = readw(ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET);
  250. if ((pcie_control & IOAT3_PCI_CONTROL_MEMWR) == 0) {
  251. pcie_control |= IOAT3_PCI_CONTROL_MEMWR;
  252. writew(pcie_control,
  253. ioatdca->dca_base + IOAT3_PCI_CONTROL_OFFSET);
  254. }
  255. /* TODO version, compatibility and configuration checks */
  256. /* copy out the APIC to DCA tag map */
  257. tag_map.low =
  258. readl(ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_LOW);
  259. tag_map.high =
  260. readl(ioatdca->dca_base + IOAT3_APICID_TAG_MAP_OFFSET_HIGH);
  261. for (i = 0; i < 8; i++) {
  262. bit = tag_map.full >> (8 * i);
  263. ioatdca->tag_map[i] = bit & DCA_TAG_MAP_MASK;
  264. }
  265. if (dca3_tag_map_invalid(ioatdca->tag_map)) {
  266. add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
  267. pr_warn_once("%s %s: APICID_TAG_MAP set incorrectly by BIOS, disabling DCA\n",
  268. dev_driver_string(&pdev->dev),
  269. dev_name(&pdev->dev));
  270. free_dca_provider(dca);
  271. return NULL;
  272. }
  273. err = register_dca_provider(dca, &pdev->dev);
  274. if (err) {
  275. free_dca_provider(dca);
  276. return NULL;
  277. }
  278. return dca;
  279. }