vc.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * PCI Virtual Channel support
  4. *
  5. * Copyright (C) 2013 Red Hat, Inc. All rights reserved.
  6. * Author: Alex Williamson <alex.williamson@redhat.com>
  7. */
  8. #include <linux/device.h>
  9. #include <linux/kernel.h>
  10. #include <linux/module.h>
  11. #include <linux/pci.h>
  12. #include <linux/pci_regs.h>
  13. #include <linux/types.h>
  14. #include "pci.h"
  15. /**
  16. * pci_vc_save_restore_dwords - Save or restore a series of dwords
  17. * @dev: device
  18. * @pos: starting config space position
  19. * @buf: buffer to save to or restore from
  20. * @dwords: number of dwords to save/restore
  21. * @save: whether to save or restore
  22. */
  23. static void pci_vc_save_restore_dwords(struct pci_dev *dev, int pos,
  24. u32 *buf, int dwords, bool save)
  25. {
  26. int i;
  27. for (i = 0; i < dwords; i++, buf++) {
  28. if (save)
  29. pci_read_config_dword(dev, pos + (i * 4), buf);
  30. else
  31. pci_write_config_dword(dev, pos + (i * 4), *buf);
  32. }
  33. }
  34. /**
  35. * pci_vc_load_arb_table - load and wait for VC arbitration table
  36. * @dev: device
  37. * @pos: starting position of VC capability (VC/VC9/MFVC)
  38. *
  39. * Set Load VC Arbitration Table bit requesting hardware to apply the VC
  40. * Arbitration Table (previously loaded). When the VC Arbitration Table
  41. * Status clears, hardware has latched the table into VC arbitration logic.
  42. */
  43. static void pci_vc_load_arb_table(struct pci_dev *dev, int pos)
  44. {
  45. u16 ctrl;
  46. pci_read_config_word(dev, pos + PCI_VC_PORT_CTRL, &ctrl);
  47. pci_write_config_word(dev, pos + PCI_VC_PORT_CTRL,
  48. ctrl | PCI_VC_PORT_CTRL_LOAD_TABLE);
  49. if (pci_wait_for_pending(dev, pos + PCI_VC_PORT_STATUS,
  50. PCI_VC_PORT_STATUS_TABLE))
  51. return;
  52. pci_err(dev, "VC arbitration table failed to load\n");
  53. }
  54. /**
  55. * pci_vc_load_port_arb_table - Load and wait for VC port arbitration table
  56. * @dev: device
  57. * @pos: starting position of VC capability (VC/VC9/MFVC)
  58. * @res: VC resource number, ie. VCn (0-7)
  59. *
  60. * Set Load Port Arbitration Table bit requesting hardware to apply the Port
  61. * Arbitration Table (previously loaded). When the Port Arbitration Table
  62. * Status clears, hardware has latched the table into port arbitration logic.
  63. */
  64. static void pci_vc_load_port_arb_table(struct pci_dev *dev, int pos, int res)
  65. {
  66. int ctrl_pos, status_pos;
  67. u32 ctrl;
  68. ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF);
  69. status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF);
  70. pci_read_config_dword(dev, ctrl_pos, &ctrl);
  71. pci_write_config_dword(dev, ctrl_pos,
  72. ctrl | PCI_VC_RES_CTRL_LOAD_TABLE);
  73. if (pci_wait_for_pending(dev, status_pos, PCI_VC_RES_STATUS_TABLE))
  74. return;
  75. pci_err(dev, "VC%d port arbitration table failed to load\n", res);
  76. }
  77. /**
  78. * pci_vc_enable - Enable virtual channel
  79. * @dev: device
  80. * @pos: starting position of VC capability (VC/VC9/MFVC)
  81. * @res: VC res number, ie. VCn (0-7)
  82. *
  83. * A VC is enabled by setting the enable bit in matching resource control
  84. * registers on both sides of a link. We therefore need to find the opposite
  85. * end of the link. To keep this simple we enable from the downstream device.
  86. * RC devices do not have an upstream device, nor does it seem that VC9 do
  87. * (spec is unclear). Once we find the upstream device, match the VC ID to
  88. * get the correct resource, disable and enable on both ends.
  89. */
  90. static void pci_vc_enable(struct pci_dev *dev, int pos, int res)
  91. {
  92. int ctrl_pos, status_pos, id, pos2, evcc, i, ctrl_pos2, status_pos2;
  93. u32 ctrl, header, cap1, ctrl2;
  94. struct pci_dev *link = NULL;
  95. /* Enable VCs from the downstream device */
  96. if (!pci_is_pcie(dev) || !pcie_downstream_port(dev))
  97. return;
  98. ctrl_pos = pos + PCI_VC_RES_CTRL + (res * PCI_CAP_VC_PER_VC_SIZEOF);
  99. status_pos = pos + PCI_VC_RES_STATUS + (res * PCI_CAP_VC_PER_VC_SIZEOF);
  100. pci_read_config_dword(dev, ctrl_pos, &ctrl);
  101. id = ctrl & PCI_VC_RES_CTRL_ID;
  102. pci_read_config_dword(dev, pos, &header);
  103. /* If there is no opposite end of the link, skip to enable */
  104. if (PCI_EXT_CAP_ID(header) == PCI_EXT_CAP_ID_VC9 ||
  105. pci_is_root_bus(dev->bus))
  106. goto enable;
  107. pos2 = pci_find_ext_capability(dev->bus->self, PCI_EXT_CAP_ID_VC);
  108. if (!pos2)
  109. goto enable;
  110. pci_read_config_dword(dev->bus->self, pos2 + PCI_VC_PORT_CAP1, &cap1);
  111. evcc = cap1 & PCI_VC_CAP1_EVCC;
  112. /* VC0 is hardwired enabled, so we can start with 1 */
  113. for (i = 1; i < evcc + 1; i++) {
  114. ctrl_pos2 = pos2 + PCI_VC_RES_CTRL +
  115. (i * PCI_CAP_VC_PER_VC_SIZEOF);
  116. status_pos2 = pos2 + PCI_VC_RES_STATUS +
  117. (i * PCI_CAP_VC_PER_VC_SIZEOF);
  118. pci_read_config_dword(dev->bus->self, ctrl_pos2, &ctrl2);
  119. if ((ctrl2 & PCI_VC_RES_CTRL_ID) == id) {
  120. link = dev->bus->self;
  121. break;
  122. }
  123. }
  124. if (!link)
  125. goto enable;
  126. /* Disable if enabled */
  127. if (ctrl2 & PCI_VC_RES_CTRL_ENABLE) {
  128. ctrl2 &= ~PCI_VC_RES_CTRL_ENABLE;
  129. pci_write_config_dword(link, ctrl_pos2, ctrl2);
  130. }
  131. /* Enable on both ends */
  132. ctrl2 |= PCI_VC_RES_CTRL_ENABLE;
  133. pci_write_config_dword(link, ctrl_pos2, ctrl2);
  134. enable:
  135. ctrl |= PCI_VC_RES_CTRL_ENABLE;
  136. pci_write_config_dword(dev, ctrl_pos, ctrl);
  137. if (!pci_wait_for_pending(dev, status_pos, PCI_VC_RES_STATUS_NEGO))
  138. pci_err(dev, "VC%d negotiation stuck pending\n", id);
  139. if (link && !pci_wait_for_pending(link, status_pos2,
  140. PCI_VC_RES_STATUS_NEGO))
  141. pci_err(link, "VC%d negotiation stuck pending\n", id);
  142. }
  143. /**
  144. * pci_vc_do_save_buffer - Size, save, or restore VC state
  145. * @dev: device
  146. * @pos: starting position of VC capability (VC/VC9/MFVC)
  147. * @save_state: buffer for save/restore
  148. * @save: if provided a buffer, this indicates what to do with it
  149. *
  150. * Walking Virtual Channel config space to size, save, or restore it
  151. * is complicated, so we do it all from one function to reduce code and
  152. * guarantee ordering matches in the buffer. When called with NULL
  153. * @save_state, return the size of the necessary save buffer. When called
  154. * with a non-NULL @save_state, @save determines whether we save to the
  155. * buffer or restore from it.
  156. */
  157. static int pci_vc_do_save_buffer(struct pci_dev *dev, int pos,
  158. struct pci_cap_saved_state *save_state,
  159. bool save)
  160. {
  161. u32 cap1;
  162. char evcc, lpevcc, parb_size;
  163. int i, len = 0;
  164. u8 *buf = save_state ? (u8 *)save_state->cap.data : NULL;
  165. /* Sanity check buffer size for save/restore */
  166. if (buf && save_state->cap.size !=
  167. pci_vc_do_save_buffer(dev, pos, NULL, save)) {
  168. pci_err(dev, "VC save buffer size does not match @0x%x\n", pos);
  169. return -ENOMEM;
  170. }
  171. pci_read_config_dword(dev, pos + PCI_VC_PORT_CAP1, &cap1);
  172. /* Extended VC Count (not counting VC0) */
  173. evcc = cap1 & PCI_VC_CAP1_EVCC;
  174. /* Low Priority Extended VC Count (not counting VC0) */
  175. lpevcc = (cap1 & PCI_VC_CAP1_LPEVCC) >> 4;
  176. /* Port Arbitration Table Entry Size (bits) */
  177. parb_size = 1 << ((cap1 & PCI_VC_CAP1_ARB_SIZE) >> 10);
  178. /*
  179. * Port VC Control Register contains VC Arbitration Select, which
  180. * cannot be modified when more than one LPVC is in operation. We
  181. * therefore save/restore it first, as only VC0 should be enabled
  182. * after device reset.
  183. */
  184. if (buf) {
  185. if (save)
  186. pci_read_config_word(dev, pos + PCI_VC_PORT_CTRL,
  187. (u16 *)buf);
  188. else
  189. pci_write_config_word(dev, pos + PCI_VC_PORT_CTRL,
  190. *(u16 *)buf);
  191. buf += 4;
  192. }
  193. len += 4;
  194. /*
  195. * If we have any Low Priority VCs and a VC Arbitration Table Offset
  196. * in Port VC Capability Register 2 then save/restore it next.
  197. */
  198. if (lpevcc) {
  199. u32 cap2;
  200. int vcarb_offset;
  201. pci_read_config_dword(dev, pos + PCI_VC_PORT_CAP2, &cap2);
  202. vcarb_offset = ((cap2 & PCI_VC_CAP2_ARB_OFF) >> 24) * 16;
  203. if (vcarb_offset) {
  204. int size, vcarb_phases = 0;
  205. if (cap2 & PCI_VC_CAP2_128_PHASE)
  206. vcarb_phases = 128;
  207. else if (cap2 & PCI_VC_CAP2_64_PHASE)
  208. vcarb_phases = 64;
  209. else if (cap2 & PCI_VC_CAP2_32_PHASE)
  210. vcarb_phases = 32;
  211. /* Fixed 4 bits per phase per lpevcc (plus VC0) */
  212. size = ((lpevcc + 1) * vcarb_phases * 4) / 8;
  213. if (size && buf) {
  214. pci_vc_save_restore_dwords(dev,
  215. pos + vcarb_offset,
  216. (u32 *)buf,
  217. size / 4, save);
  218. /*
  219. * On restore, we need to signal hardware to
  220. * re-load the VC Arbitration Table.
  221. */
  222. if (!save)
  223. pci_vc_load_arb_table(dev, pos);
  224. buf += size;
  225. }
  226. len += size;
  227. }
  228. }
  229. /*
  230. * In addition to each VC Resource Control Register, we may have a
  231. * Port Arbitration Table attached to each VC. The Port Arbitration
  232. * Table Offset in each VC Resource Capability Register tells us if
  233. * it exists. The entry size is global from the Port VC Capability
  234. * Register1 above. The number of phases is determined per VC.
  235. */
  236. for (i = 0; i < evcc + 1; i++) {
  237. u32 cap;
  238. int parb_offset;
  239. pci_read_config_dword(dev, pos + PCI_VC_RES_CAP +
  240. (i * PCI_CAP_VC_PER_VC_SIZEOF), &cap);
  241. parb_offset = ((cap & PCI_VC_RES_CAP_ARB_OFF) >> 24) * 16;
  242. if (parb_offset) {
  243. int size, parb_phases = 0;
  244. if (cap & PCI_VC_RES_CAP_256_PHASE)
  245. parb_phases = 256;
  246. else if (cap & (PCI_VC_RES_CAP_128_PHASE |
  247. PCI_VC_RES_CAP_128_PHASE_TB))
  248. parb_phases = 128;
  249. else if (cap & PCI_VC_RES_CAP_64_PHASE)
  250. parb_phases = 64;
  251. else if (cap & PCI_VC_RES_CAP_32_PHASE)
  252. parb_phases = 32;
  253. size = (parb_size * parb_phases) / 8;
  254. if (size && buf) {
  255. pci_vc_save_restore_dwords(dev,
  256. pos + parb_offset,
  257. (u32 *)buf,
  258. size / 4, save);
  259. buf += size;
  260. }
  261. len += size;
  262. }
  263. /* VC Resource Control Register */
  264. if (buf) {
  265. int ctrl_pos = pos + PCI_VC_RES_CTRL +
  266. (i * PCI_CAP_VC_PER_VC_SIZEOF);
  267. if (save)
  268. pci_read_config_dword(dev, ctrl_pos,
  269. (u32 *)buf);
  270. else {
  271. u32 tmp, ctrl = *(u32 *)buf;
  272. /*
  273. * For an FLR case, the VC config may remain.
  274. * Preserve enable bit, restore the rest.
  275. */
  276. pci_read_config_dword(dev, ctrl_pos, &tmp);
  277. tmp &= PCI_VC_RES_CTRL_ENABLE;
  278. tmp |= ctrl & ~PCI_VC_RES_CTRL_ENABLE;
  279. pci_write_config_dword(dev, ctrl_pos, tmp);
  280. /* Load port arbitration table if used */
  281. if (ctrl & PCI_VC_RES_CTRL_ARB_SELECT)
  282. pci_vc_load_port_arb_table(dev, pos, i);
  283. /* Re-enable if needed */
  284. if ((ctrl ^ tmp) & PCI_VC_RES_CTRL_ENABLE)
  285. pci_vc_enable(dev, pos, i);
  286. }
  287. buf += 4;
  288. }
  289. len += 4;
  290. }
  291. return buf ? 0 : len;
  292. }
  293. static struct {
  294. u16 id;
  295. const char *name;
  296. } vc_caps[] = { { PCI_EXT_CAP_ID_MFVC, "MFVC" },
  297. { PCI_EXT_CAP_ID_VC, "VC" },
  298. { PCI_EXT_CAP_ID_VC9, "VC9" } };
  299. /**
  300. * pci_save_vc_state - Save VC state to pre-allocate save buffer
  301. * @dev: device
  302. *
  303. * For each type of VC capability, VC/VC9/MFVC, find the capability and
  304. * save it to the pre-allocated save buffer.
  305. */
  306. int pci_save_vc_state(struct pci_dev *dev)
  307. {
  308. int i;
  309. for (i = 0; i < ARRAY_SIZE(vc_caps); i++) {
  310. int pos, ret;
  311. struct pci_cap_saved_state *save_state;
  312. pos = pci_find_ext_capability(dev, vc_caps[i].id);
  313. if (!pos)
  314. continue;
  315. save_state = pci_find_saved_ext_cap(dev, vc_caps[i].id);
  316. if (!save_state) {
  317. pci_err(dev, "%s buffer not found in %s\n",
  318. vc_caps[i].name, __func__);
  319. return -ENOMEM;
  320. }
  321. ret = pci_vc_do_save_buffer(dev, pos, save_state, true);
  322. if (ret) {
  323. pci_err(dev, "%s save unsuccessful %s\n",
  324. vc_caps[i].name, __func__);
  325. return ret;
  326. }
  327. }
  328. return 0;
  329. }
  330. /**
  331. * pci_restore_vc_state - Restore VC state from save buffer
  332. * @dev: device
  333. *
  334. * For each type of VC capability, VC/VC9/MFVC, find the capability and
  335. * restore it from the previously saved buffer.
  336. */
  337. void pci_restore_vc_state(struct pci_dev *dev)
  338. {
  339. int i;
  340. for (i = 0; i < ARRAY_SIZE(vc_caps); i++) {
  341. int pos;
  342. struct pci_cap_saved_state *save_state;
  343. pos = pci_find_ext_capability(dev, vc_caps[i].id);
  344. save_state = pci_find_saved_ext_cap(dev, vc_caps[i].id);
  345. if (!save_state || !pos)
  346. continue;
  347. pci_vc_do_save_buffer(dev, pos, save_state, false);
  348. }
  349. }
  350. /**
  351. * pci_allocate_vc_save_buffers - Allocate save buffers for VC caps
  352. * @dev: device
  353. *
  354. * For each type of VC capability, VC/VC9/MFVC, find the capability, size
  355. * it, and allocate a buffer for save/restore.
  356. */
  357. void pci_allocate_vc_save_buffers(struct pci_dev *dev)
  358. {
  359. int i;
  360. for (i = 0; i < ARRAY_SIZE(vc_caps); i++) {
  361. int len, pos = pci_find_ext_capability(dev, vc_caps[i].id);
  362. if (!pos)
  363. continue;
  364. len = pci_vc_do_save_buffer(dev, pos, NULL, false);
  365. if (pci_add_ext_cap_save_buffer(dev, vc_caps[i].id, len))
  366. pci_err(dev, "unable to preallocate %s save buffer\n",
  367. vc_caps[i].name);
  368. }
  369. }