access.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include <linux/pci.h>
  3. #include <linux/module.h>
  4. #include <linux/slab.h>
  5. #include <linux/ioport.h>
  6. #include <linux/wait.h>
  7. #include "pci.h"
  8. /*
  9. * This interrupt-safe spinlock protects all accesses to PCI
  10. * configuration space.
  11. */
  12. DEFINE_RAW_SPINLOCK(pci_lock);
  13. /*
  14. * Wrappers for all PCI configuration access functions. They just check
  15. * alignment, do locking and call the low-level functions pointed to
  16. * by pci_dev->ops.
  17. */
  18. #define PCI_byte_BAD 0
  19. #define PCI_word_BAD (pos & 1)
  20. #define PCI_dword_BAD (pos & 3)
  21. #ifdef CONFIG_PCI_LOCKLESS_CONFIG
  22. # define pci_lock_config(f) do { (void)(f); } while (0)
  23. # define pci_unlock_config(f) do { (void)(f); } while (0)
  24. #else
  25. # define pci_lock_config(f) raw_spin_lock_irqsave(&pci_lock, f)
  26. # define pci_unlock_config(f) raw_spin_unlock_irqrestore(&pci_lock, f)
  27. #endif
  28. #define PCI_OP_READ(size, type, len) \
  29. int noinline pci_bus_read_config_##size \
  30. (struct pci_bus *bus, unsigned int devfn, int pos, type *value) \
  31. { \
  32. int res; \
  33. unsigned long flags; \
  34. u32 data = 0; \
  35. if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER; \
  36. pci_lock_config(flags); \
  37. res = bus->ops->read(bus, devfn, pos, len, &data); \
  38. *value = (type)data; \
  39. pci_unlock_config(flags); \
  40. return res; \
  41. }
  42. #define PCI_OP_WRITE(size, type, len) \
  43. int noinline pci_bus_write_config_##size \
  44. (struct pci_bus *bus, unsigned int devfn, int pos, type value) \
  45. { \
  46. int res; \
  47. unsigned long flags; \
  48. if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER; \
  49. pci_lock_config(flags); \
  50. res = bus->ops->write(bus, devfn, pos, len, value); \
  51. pci_unlock_config(flags); \
  52. return res; \
  53. }
  54. PCI_OP_READ(byte, u8, 1)
  55. PCI_OP_READ(word, u16, 2)
  56. PCI_OP_READ(dword, u32, 4)
  57. PCI_OP_WRITE(byte, u8, 1)
  58. PCI_OP_WRITE(word, u16, 2)
  59. PCI_OP_WRITE(dword, u32, 4)
  60. EXPORT_SYMBOL(pci_bus_read_config_byte);
  61. EXPORT_SYMBOL(pci_bus_read_config_word);
  62. EXPORT_SYMBOL(pci_bus_read_config_dword);
  63. EXPORT_SYMBOL(pci_bus_write_config_byte);
  64. EXPORT_SYMBOL(pci_bus_write_config_word);
  65. EXPORT_SYMBOL(pci_bus_write_config_dword);
  66. int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn,
  67. int where, int size, u32 *val)
  68. {
  69. void __iomem *addr;
  70. addr = bus->ops->map_bus(bus, devfn, where);
  71. if (!addr) {
  72. *val = ~0;
  73. return PCIBIOS_DEVICE_NOT_FOUND;
  74. }
  75. if (size == 1)
  76. *val = readb(addr);
  77. else if (size == 2)
  78. *val = readw(addr);
  79. else
  80. *val = readl(addr);
  81. return PCIBIOS_SUCCESSFUL;
  82. }
  83. EXPORT_SYMBOL_GPL(pci_generic_config_read);
  84. int pci_generic_config_write(struct pci_bus *bus, unsigned int devfn,
  85. int where, int size, u32 val)
  86. {
  87. void __iomem *addr;
  88. addr = bus->ops->map_bus(bus, devfn, where);
  89. if (!addr)
  90. return PCIBIOS_DEVICE_NOT_FOUND;
  91. if (size == 1)
  92. writeb(val, addr);
  93. else if (size == 2)
  94. writew(val, addr);
  95. else
  96. writel(val, addr);
  97. return PCIBIOS_SUCCESSFUL;
  98. }
  99. EXPORT_SYMBOL_GPL(pci_generic_config_write);
  100. int pci_generic_config_read32(struct pci_bus *bus, unsigned int devfn,
  101. int where, int size, u32 *val)
  102. {
  103. void __iomem *addr;
  104. addr = bus->ops->map_bus(bus, devfn, where & ~0x3);
  105. if (!addr) {
  106. *val = ~0;
  107. return PCIBIOS_DEVICE_NOT_FOUND;
  108. }
  109. *val = readl(addr);
  110. if (size <= 2)
  111. *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1);
  112. return PCIBIOS_SUCCESSFUL;
  113. }
  114. EXPORT_SYMBOL_GPL(pci_generic_config_read32);
  115. int pci_generic_config_write32(struct pci_bus *bus, unsigned int devfn,
  116. int where, int size, u32 val)
  117. {
  118. void __iomem *addr;
  119. u32 mask, tmp;
  120. addr = bus->ops->map_bus(bus, devfn, where & ~0x3);
  121. if (!addr)
  122. return PCIBIOS_DEVICE_NOT_FOUND;
  123. if (size == 4) {
  124. writel(val, addr);
  125. return PCIBIOS_SUCCESSFUL;
  126. }
  127. /*
  128. * In general, hardware that supports only 32-bit writes on PCI is
  129. * not spec-compliant. For example, software may perform a 16-bit
  130. * write. If the hardware only supports 32-bit accesses, we must
  131. * do a 32-bit read, merge in the 16 bits we intend to write,
  132. * followed by a 32-bit write. If the 16 bits we *don't* intend to
  133. * write happen to have any RW1C (write-one-to-clear) bits set, we
  134. * just inadvertently cleared something we shouldn't have.
  135. */
  136. dev_warn_ratelimited(&bus->dev, "%d-byte config write to %04x:%02x:%02x.%d offset %#x may corrupt adjacent RW1C bits\n",
  137. size, pci_domain_nr(bus), bus->number,
  138. PCI_SLOT(devfn), PCI_FUNC(devfn), where);
  139. mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
  140. tmp = readl(addr) & mask;
  141. tmp |= val << ((where & 0x3) * 8);
  142. writel(tmp, addr);
  143. return PCIBIOS_SUCCESSFUL;
  144. }
  145. EXPORT_SYMBOL_GPL(pci_generic_config_write32);
  146. /**
  147. * pci_bus_set_ops - Set raw operations of pci bus
  148. * @bus: pci bus struct
  149. * @ops: new raw operations
  150. *
  151. * Return previous raw operations
  152. */
  153. struct pci_ops *pci_bus_set_ops(struct pci_bus *bus, struct pci_ops *ops)
  154. {
  155. struct pci_ops *old_ops;
  156. unsigned long flags;
  157. raw_spin_lock_irqsave(&pci_lock, flags);
  158. old_ops = bus->ops;
  159. bus->ops = ops;
  160. raw_spin_unlock_irqrestore(&pci_lock, flags);
  161. return old_ops;
  162. }
  163. EXPORT_SYMBOL(pci_bus_set_ops);
  164. /*
  165. * The following routines are to prevent the user from accessing PCI config
  166. * space when it's unsafe to do so. Some devices require this during BIST and
  167. * we're required to prevent it during D-state transitions.
  168. *
  169. * We have a bit per device to indicate it's blocked and a global wait queue
  170. * for callers to sleep on until devices are unblocked.
  171. */
  172. static DECLARE_WAIT_QUEUE_HEAD(pci_cfg_wait);
  173. static noinline void pci_wait_cfg(struct pci_dev *dev)
  174. __must_hold(&pci_lock)
  175. {
  176. do {
  177. raw_spin_unlock_irq(&pci_lock);
  178. wait_event(pci_cfg_wait, !dev->block_cfg_access);
  179. raw_spin_lock_irq(&pci_lock);
  180. } while (dev->block_cfg_access);
  181. }
  182. /* Returns 0 on success, negative values indicate error. */
  183. #define PCI_USER_READ_CONFIG(size, type) \
  184. int pci_user_read_config_##size \
  185. (struct pci_dev *dev, int pos, type *val) \
  186. { \
  187. int ret = PCIBIOS_SUCCESSFUL; \
  188. u32 data = -1; \
  189. if (PCI_##size##_BAD) \
  190. return -EINVAL; \
  191. raw_spin_lock_irq(&pci_lock); \
  192. if (unlikely(dev->block_cfg_access)) \
  193. pci_wait_cfg(dev); \
  194. ret = dev->bus->ops->read(dev->bus, dev->devfn, \
  195. pos, sizeof(type), &data); \
  196. raw_spin_unlock_irq(&pci_lock); \
  197. *val = (type)data; \
  198. return pcibios_err_to_errno(ret); \
  199. } \
  200. EXPORT_SYMBOL_GPL(pci_user_read_config_##size);
  201. /* Returns 0 on success, negative values indicate error. */
  202. #define PCI_USER_WRITE_CONFIG(size, type) \
  203. int pci_user_write_config_##size \
  204. (struct pci_dev *dev, int pos, type val) \
  205. { \
  206. int ret = PCIBIOS_SUCCESSFUL; \
  207. if (PCI_##size##_BAD) \
  208. return -EINVAL; \
  209. raw_spin_lock_irq(&pci_lock); \
  210. if (unlikely(dev->block_cfg_access)) \
  211. pci_wait_cfg(dev); \
  212. ret = dev->bus->ops->write(dev->bus, dev->devfn, \
  213. pos, sizeof(type), val); \
  214. raw_spin_unlock_irq(&pci_lock); \
  215. return pcibios_err_to_errno(ret); \
  216. } \
  217. EXPORT_SYMBOL_GPL(pci_user_write_config_##size);
  218. PCI_USER_READ_CONFIG(byte, u8)
  219. PCI_USER_READ_CONFIG(word, u16)
  220. PCI_USER_READ_CONFIG(dword, u32)
  221. PCI_USER_WRITE_CONFIG(byte, u8)
  222. PCI_USER_WRITE_CONFIG(word, u16)
  223. PCI_USER_WRITE_CONFIG(dword, u32)
  224. /**
  225. * pci_cfg_access_lock - Lock PCI config reads/writes
  226. * @dev: pci device struct
  227. *
  228. * When access is locked, any userspace reads or writes to config
  229. * space and concurrent lock requests will sleep until access is
  230. * allowed via pci_cfg_access_unlock() again.
  231. */
  232. void pci_cfg_access_lock(struct pci_dev *dev)
  233. {
  234. might_sleep();
  235. raw_spin_lock_irq(&pci_lock);
  236. if (dev->block_cfg_access)
  237. pci_wait_cfg(dev);
  238. dev->block_cfg_access = 1;
  239. raw_spin_unlock_irq(&pci_lock);
  240. }
  241. EXPORT_SYMBOL_GPL(pci_cfg_access_lock);
  242. /**
  243. * pci_cfg_access_trylock - try to lock PCI config reads/writes
  244. * @dev: pci device struct
  245. *
  246. * Same as pci_cfg_access_lock, but will return 0 if access is
  247. * already locked, 1 otherwise. This function can be used from
  248. * atomic contexts.
  249. */
  250. bool pci_cfg_access_trylock(struct pci_dev *dev)
  251. {
  252. unsigned long flags;
  253. bool locked = true;
  254. raw_spin_lock_irqsave(&pci_lock, flags);
  255. if (dev->block_cfg_access)
  256. locked = false;
  257. else
  258. dev->block_cfg_access = 1;
  259. raw_spin_unlock_irqrestore(&pci_lock, flags);
  260. return locked;
  261. }
  262. EXPORT_SYMBOL_GPL(pci_cfg_access_trylock);
  263. /**
  264. * pci_cfg_access_unlock - Unlock PCI config reads/writes
  265. * @dev: pci device struct
  266. *
  267. * This function allows PCI config accesses to resume.
  268. */
  269. void pci_cfg_access_unlock(struct pci_dev *dev)
  270. {
  271. unsigned long flags;
  272. raw_spin_lock_irqsave(&pci_lock, flags);
  273. /*
  274. * This indicates a problem in the caller, but we don't need
  275. * to kill them, unlike a double-block above.
  276. */
  277. WARN_ON(!dev->block_cfg_access);
  278. dev->block_cfg_access = 0;
  279. raw_spin_unlock_irqrestore(&pci_lock, flags);
  280. wake_up_all(&pci_cfg_wait);
  281. }
  282. EXPORT_SYMBOL_GPL(pci_cfg_access_unlock);
  283. static inline int pcie_cap_version(const struct pci_dev *dev)
  284. {
  285. return pcie_caps_reg(dev) & PCI_EXP_FLAGS_VERS;
  286. }
  287. bool pcie_cap_has_lnkctl(const struct pci_dev *dev)
  288. {
  289. int type = pci_pcie_type(dev);
  290. return type == PCI_EXP_TYPE_ENDPOINT ||
  291. type == PCI_EXP_TYPE_LEG_END ||
  292. type == PCI_EXP_TYPE_ROOT_PORT ||
  293. type == PCI_EXP_TYPE_UPSTREAM ||
  294. type == PCI_EXP_TYPE_DOWNSTREAM ||
  295. type == PCI_EXP_TYPE_PCI_BRIDGE ||
  296. type == PCI_EXP_TYPE_PCIE_BRIDGE;
  297. }
  298. static inline bool pcie_cap_has_sltctl(const struct pci_dev *dev)
  299. {
  300. return pcie_downstream_port(dev) &&
  301. pcie_caps_reg(dev) & PCI_EXP_FLAGS_SLOT;
  302. }
  303. bool pcie_cap_has_rtctl(const struct pci_dev *dev)
  304. {
  305. int type = pci_pcie_type(dev);
  306. return type == PCI_EXP_TYPE_ROOT_PORT ||
  307. type == PCI_EXP_TYPE_RC_EC;
  308. }
  309. static bool pcie_capability_reg_implemented(struct pci_dev *dev, int pos)
  310. {
  311. if (!pci_is_pcie(dev))
  312. return false;
  313. switch (pos) {
  314. case PCI_EXP_FLAGS:
  315. return true;
  316. case PCI_EXP_DEVCAP:
  317. case PCI_EXP_DEVCTL:
  318. case PCI_EXP_DEVSTA:
  319. return true;
  320. case PCI_EXP_LNKCAP:
  321. case PCI_EXP_LNKCTL:
  322. case PCI_EXP_LNKSTA:
  323. return pcie_cap_has_lnkctl(dev);
  324. case PCI_EXP_SLTCAP:
  325. case PCI_EXP_SLTCTL:
  326. case PCI_EXP_SLTSTA:
  327. return pcie_cap_has_sltctl(dev);
  328. case PCI_EXP_RTCTL:
  329. case PCI_EXP_RTCAP:
  330. case PCI_EXP_RTSTA:
  331. return pcie_cap_has_rtctl(dev);
  332. case PCI_EXP_DEVCAP2:
  333. case PCI_EXP_DEVCTL2:
  334. case PCI_EXP_LNKCAP2:
  335. case PCI_EXP_LNKCTL2:
  336. case PCI_EXP_LNKSTA2:
  337. return pcie_cap_version(dev) > 1;
  338. default:
  339. return false;
  340. }
  341. }
  342. /*
  343. * Note that these accessor functions are only for the "PCI Express
  344. * Capability" (see PCIe spec r3.0, sec 7.8). They do not apply to the
  345. * other "PCI Express Extended Capabilities" (AER, VC, ACS, MFVC, etc.)
  346. */
  347. int pcie_capability_read_word(struct pci_dev *dev, int pos, u16 *val)
  348. {
  349. int ret;
  350. *val = 0;
  351. if (pos & 1)
  352. return PCIBIOS_BAD_REGISTER_NUMBER;
  353. if (pcie_capability_reg_implemented(dev, pos)) {
  354. ret = pci_read_config_word(dev, pci_pcie_cap(dev) + pos, val);
  355. /*
  356. * Reset *val to 0 if pci_read_config_word() fails, it may
  357. * have been written as 0xFFFF if hardware error happens
  358. * during pci_read_config_word().
  359. */
  360. if (ret)
  361. *val = 0;
  362. return ret;
  363. }
  364. /*
  365. * For Functions that do not implement the Slot Capabilities,
  366. * Slot Status, and Slot Control registers, these spaces must
  367. * be hardwired to 0b, with the exception of the Presence Detect
  368. * State bit in the Slot Status register of Downstream Ports,
  369. * which must be hardwired to 1b. (PCIe Base Spec 3.0, sec 7.8)
  370. */
  371. if (pci_is_pcie(dev) && pcie_downstream_port(dev) &&
  372. pos == PCI_EXP_SLTSTA)
  373. *val = PCI_EXP_SLTSTA_PDS;
  374. return 0;
  375. }
  376. EXPORT_SYMBOL(pcie_capability_read_word);
  377. int pcie_capability_read_dword(struct pci_dev *dev, int pos, u32 *val)
  378. {
  379. int ret;
  380. *val = 0;
  381. if (pos & 3)
  382. return PCIBIOS_BAD_REGISTER_NUMBER;
  383. if (pcie_capability_reg_implemented(dev, pos)) {
  384. ret = pci_read_config_dword(dev, pci_pcie_cap(dev) + pos, val);
  385. /*
  386. * Reset *val to 0 if pci_read_config_dword() fails, it may
  387. * have been written as 0xFFFFFFFF if hardware error happens
  388. * during pci_read_config_dword().
  389. */
  390. if (ret)
  391. *val = 0;
  392. return ret;
  393. }
  394. if (pci_is_pcie(dev) && pcie_downstream_port(dev) &&
  395. pos == PCI_EXP_SLTSTA)
  396. *val = PCI_EXP_SLTSTA_PDS;
  397. return 0;
  398. }
  399. EXPORT_SYMBOL(pcie_capability_read_dword);
  400. int pcie_capability_write_word(struct pci_dev *dev, int pos, u16 val)
  401. {
  402. if (pos & 1)
  403. return PCIBIOS_BAD_REGISTER_NUMBER;
  404. if (!pcie_capability_reg_implemented(dev, pos))
  405. return 0;
  406. return pci_write_config_word(dev, pci_pcie_cap(dev) + pos, val);
  407. }
  408. EXPORT_SYMBOL(pcie_capability_write_word);
  409. int pcie_capability_write_dword(struct pci_dev *dev, int pos, u32 val)
  410. {
  411. if (pos & 3)
  412. return PCIBIOS_BAD_REGISTER_NUMBER;
  413. if (!pcie_capability_reg_implemented(dev, pos))
  414. return 0;
  415. return pci_write_config_dword(dev, pci_pcie_cap(dev) + pos, val);
  416. }
  417. EXPORT_SYMBOL(pcie_capability_write_dword);
  418. int pcie_capability_clear_and_set_word(struct pci_dev *dev, int pos,
  419. u16 clear, u16 set)
  420. {
  421. int ret;
  422. u16 val;
  423. ret = pcie_capability_read_word(dev, pos, &val);
  424. if (!ret) {
  425. val &= ~clear;
  426. val |= set;
  427. ret = pcie_capability_write_word(dev, pos, val);
  428. }
  429. return ret;
  430. }
  431. EXPORT_SYMBOL(pcie_capability_clear_and_set_word);
  432. int pcie_capability_clear_and_set_dword(struct pci_dev *dev, int pos,
  433. u32 clear, u32 set)
  434. {
  435. int ret;
  436. u32 val;
  437. ret = pcie_capability_read_dword(dev, pos, &val);
  438. if (!ret) {
  439. val &= ~clear;
  440. val |= set;
  441. ret = pcie_capability_write_dword(dev, pos, val);
  442. }
  443. return ret;
  444. }
  445. EXPORT_SYMBOL(pcie_capability_clear_and_set_dword);
  446. int pci_read_config_byte(const struct pci_dev *dev, int where, u8 *val)
  447. {
  448. if (pci_dev_is_disconnected(dev)) {
  449. *val = ~0;
  450. return PCIBIOS_DEVICE_NOT_FOUND;
  451. }
  452. return pci_bus_read_config_byte(dev->bus, dev->devfn, where, val);
  453. }
  454. EXPORT_SYMBOL(pci_read_config_byte);
  455. int pci_read_config_word(const struct pci_dev *dev, int where, u16 *val)
  456. {
  457. if (pci_dev_is_disconnected(dev)) {
  458. *val = ~0;
  459. return PCIBIOS_DEVICE_NOT_FOUND;
  460. }
  461. return pci_bus_read_config_word(dev->bus, dev->devfn, where, val);
  462. }
  463. EXPORT_SYMBOL(pci_read_config_word);
  464. int pci_read_config_dword(const struct pci_dev *dev, int where,
  465. u32 *val)
  466. {
  467. if (pci_dev_is_disconnected(dev)) {
  468. *val = ~0;
  469. return PCIBIOS_DEVICE_NOT_FOUND;
  470. }
  471. return pci_bus_read_config_dword(dev->bus, dev->devfn, where, val);
  472. }
  473. EXPORT_SYMBOL(pci_read_config_dword);
  474. int pci_write_config_byte(const struct pci_dev *dev, int where, u8 val)
  475. {
  476. if (pci_dev_is_disconnected(dev))
  477. return PCIBIOS_DEVICE_NOT_FOUND;
  478. return pci_bus_write_config_byte(dev->bus, dev->devfn, where, val);
  479. }
  480. EXPORT_SYMBOL(pci_write_config_byte);
  481. int pci_write_config_word(const struct pci_dev *dev, int where, u16 val)
  482. {
  483. if (pci_dev_is_disconnected(dev))
  484. return PCIBIOS_DEVICE_NOT_FOUND;
  485. return pci_bus_write_config_word(dev->bus, dev->devfn, where, val);
  486. }
  487. EXPORT_SYMBOL(pci_write_config_word);
  488. int pci_write_config_dword(const struct pci_dev *dev, int where,
  489. u32 val)
  490. {
  491. if (pci_dev_is_disconnected(dev))
  492. return PCIBIOS_DEVICE_NOT_FOUND;
  493. return pci_bus_write_config_dword(dev->bus, dev->devfn, where, val);
  494. }
  495. EXPORT_SYMBOL(pci_write_config_dword);