pcie-rockchip-host.c 28 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066
  1. // SPDX-License-Identifier: GPL-2.0+
  2. /*
  3. * Rockchip AXI PCIe host controller driver
  4. *
  5. * Copyright (c) 2016 Rockchip, Inc.
  6. *
  7. * Author: Shawn Lin <shawn.lin@rock-chips.com>
  8. * Wenrui Li <wenrui.li@rock-chips.com>
  9. *
  10. * Bits taken from Synopsys DesignWare Host controller driver and
  11. * ARM PCI Host generic driver.
  12. */
  13. #include <linux/bitrev.h>
  14. #include <linux/clk.h>
  15. #include <linux/delay.h>
  16. #include <linux/gpio/consumer.h>
  17. #include <linux/init.h>
  18. #include <linux/interrupt.h>
  19. #include <linux/iopoll.h>
  20. #include <linux/irq.h>
  21. #include <linux/irqchip/chained_irq.h>
  22. #include <linux/irqdomain.h>
  23. #include <linux/kernel.h>
  24. #include <linux/mfd/syscon.h>
  25. #include <linux/module.h>
  26. #include <linux/of_address.h>
  27. #include <linux/of_device.h>
  28. #include <linux/of_pci.h>
  29. #include <linux/of_platform.h>
  30. #include <linux/of_irq.h>
  31. #include <linux/pci.h>
  32. #include <linux/pci_ids.h>
  33. #include <linux/phy/phy.h>
  34. #include <linux/platform_device.h>
  35. #include <linux/reset.h>
  36. #include <linux/regmap.h>
  37. #include "../pci.h"
  38. #include "pcie-rockchip.h"
  39. static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip)
  40. {
  41. u32 status;
  42. status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
  43. status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE);
  44. rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
  45. }
  46. static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip)
  47. {
  48. u32 status;
  49. status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
  50. status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16;
  51. rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
  52. }
  53. static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip)
  54. {
  55. u32 val;
  56. /* Update Tx credit maximum update interval */
  57. val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1);
  58. val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK;
  59. val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000); /* ns */
  60. rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1);
  61. }
  62. static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip,
  63. struct pci_bus *bus, int dev)
  64. {
  65. /*
  66. * Access only one slot on each root port.
  67. * Do not read more than one device on the bus directly attached
  68. * to RC's downstream side.
  69. */
  70. if (pci_is_root_bus(bus) || pci_is_root_bus(bus->parent))
  71. return dev == 0;
  72. return 1;
  73. }
  74. static u8 rockchip_pcie_lane_map(struct rockchip_pcie *rockchip)
  75. {
  76. u32 val;
  77. u8 map;
  78. if (rockchip->legacy_phy)
  79. return GENMASK(MAX_LANE_NUM - 1, 0);
  80. val = rockchip_pcie_read(rockchip, PCIE_CORE_LANE_MAP);
  81. map = val & PCIE_CORE_LANE_MAP_MASK;
  82. /* The link may be using a reverse-indexed mapping. */
  83. if (val & PCIE_CORE_LANE_MAP_REVERSE)
  84. map = bitrev8(map) >> 4;
  85. return map;
  86. }
  87. static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip,
  88. int where, int size, u32 *val)
  89. {
  90. void __iomem *addr;
  91. addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + where;
  92. if (!IS_ALIGNED((uintptr_t)addr, size)) {
  93. *val = 0;
  94. return PCIBIOS_BAD_REGISTER_NUMBER;
  95. }
  96. if (size == 4) {
  97. *val = readl(addr);
  98. } else if (size == 2) {
  99. *val = readw(addr);
  100. } else if (size == 1) {
  101. *val = readb(addr);
  102. } else {
  103. *val = 0;
  104. return PCIBIOS_BAD_REGISTER_NUMBER;
  105. }
  106. return PCIBIOS_SUCCESSFUL;
  107. }
  108. static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip,
  109. int where, int size, u32 val)
  110. {
  111. u32 mask, tmp, offset;
  112. void __iomem *addr;
  113. offset = where & ~0x3;
  114. addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + offset;
  115. if (size == 4) {
  116. writel(val, addr);
  117. return PCIBIOS_SUCCESSFUL;
  118. }
  119. mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
  120. /*
  121. * N.B. This read/modify/write isn't safe in general because it can
  122. * corrupt RW1C bits in adjacent registers. But the hardware
  123. * doesn't support smaller writes.
  124. */
  125. tmp = readl(addr) & mask;
  126. tmp |= val << ((where & 0x3) * 8);
  127. writel(tmp, addr);
  128. return PCIBIOS_SUCCESSFUL;
  129. }
  130. static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip,
  131. struct pci_bus *bus, u32 devfn,
  132. int where, int size, u32 *val)
  133. {
  134. u32 busdev;
  135. busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn),
  136. PCI_FUNC(devfn), where);
  137. if (!IS_ALIGNED(busdev, size)) {
  138. *val = 0;
  139. return PCIBIOS_BAD_REGISTER_NUMBER;
  140. }
  141. if (pci_is_root_bus(bus->parent))
  142. rockchip_pcie_cfg_configuration_accesses(rockchip,
  143. AXI_WRAPPER_TYPE0_CFG);
  144. else
  145. rockchip_pcie_cfg_configuration_accesses(rockchip,
  146. AXI_WRAPPER_TYPE1_CFG);
  147. if (size == 4) {
  148. *val = readl(rockchip->reg_base + busdev);
  149. } else if (size == 2) {
  150. *val = readw(rockchip->reg_base + busdev);
  151. } else if (size == 1) {
  152. *val = readb(rockchip->reg_base + busdev);
  153. } else {
  154. *val = 0;
  155. return PCIBIOS_BAD_REGISTER_NUMBER;
  156. }
  157. return PCIBIOS_SUCCESSFUL;
  158. }
  159. static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip,
  160. struct pci_bus *bus, u32 devfn,
  161. int where, int size, u32 val)
  162. {
  163. u32 busdev;
  164. busdev = PCIE_ECAM_ADDR(bus->number, PCI_SLOT(devfn),
  165. PCI_FUNC(devfn), where);
  166. if (!IS_ALIGNED(busdev, size))
  167. return PCIBIOS_BAD_REGISTER_NUMBER;
  168. if (pci_is_root_bus(bus->parent))
  169. rockchip_pcie_cfg_configuration_accesses(rockchip,
  170. AXI_WRAPPER_TYPE0_CFG);
  171. else
  172. rockchip_pcie_cfg_configuration_accesses(rockchip,
  173. AXI_WRAPPER_TYPE1_CFG);
  174. if (size == 4)
  175. writel(val, rockchip->reg_base + busdev);
  176. else if (size == 2)
  177. writew(val, rockchip->reg_base + busdev);
  178. else if (size == 1)
  179. writeb(val, rockchip->reg_base + busdev);
  180. else
  181. return PCIBIOS_BAD_REGISTER_NUMBER;
  182. return PCIBIOS_SUCCESSFUL;
  183. }
  184. static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where,
  185. int size, u32 *val)
  186. {
  187. struct rockchip_pcie *rockchip = bus->sysdata;
  188. if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) {
  189. *val = 0xffffffff;
  190. return PCIBIOS_DEVICE_NOT_FOUND;
  191. }
  192. if (pci_is_root_bus(bus))
  193. return rockchip_pcie_rd_own_conf(rockchip, where, size, val);
  194. return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size,
  195. val);
  196. }
  197. static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn,
  198. int where, int size, u32 val)
  199. {
  200. struct rockchip_pcie *rockchip = bus->sysdata;
  201. if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn)))
  202. return PCIBIOS_DEVICE_NOT_FOUND;
  203. if (pci_is_root_bus(bus))
  204. return rockchip_pcie_wr_own_conf(rockchip, where, size, val);
  205. return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size,
  206. val);
  207. }
  208. static struct pci_ops rockchip_pcie_ops = {
  209. .read = rockchip_pcie_rd_conf,
  210. .write = rockchip_pcie_wr_conf,
  211. };
  212. static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip)
  213. {
  214. int curr;
  215. u32 status, scale, power;
  216. if (IS_ERR(rockchip->vpcie3v3))
  217. return;
  218. /*
  219. * Set RC's captured slot power limit and scale if
  220. * vpcie3v3 available. The default values are both zero
  221. * which means the software should set these two according
  222. * to the actual power supply.
  223. */
  224. curr = regulator_get_current_limit(rockchip->vpcie3v3);
  225. if (curr <= 0)
  226. return;
  227. scale = 3; /* 0.001x */
  228. curr = curr / 1000; /* convert to mA */
  229. power = (curr * 3300) / 1000; /* milliwatt */
  230. while (power > PCIE_RC_CONFIG_DCR_CSPL_LIMIT) {
  231. if (!scale) {
  232. dev_warn(rockchip->dev, "invalid power supply\n");
  233. return;
  234. }
  235. scale--;
  236. power = power / 10;
  237. }
  238. status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCR);
  239. status |= (power << PCIE_RC_CONFIG_DCR_CSPL_SHIFT) |
  240. (scale << PCIE_RC_CONFIG_DCR_CPLS_SHIFT);
  241. rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCR);
  242. }
  243. /**
  244. * rockchip_pcie_host_init_port - Initialize hardware
  245. * @rockchip: PCIe port information
  246. */
  247. static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip)
  248. {
  249. struct device *dev = rockchip->dev;
  250. int err, i = MAX_LANE_NUM;
  251. u32 status;
  252. gpiod_set_value_cansleep(rockchip->ep_gpio, 0);
  253. err = rockchip_pcie_init_port(rockchip);
  254. if (err)
  255. return err;
  256. /* Fix the transmitted FTS count desired to exit from L0s. */
  257. status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1);
  258. status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) |
  259. (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT);
  260. rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1);
  261. rockchip_pcie_set_power_limit(rockchip);
  262. /* Set RC's clock architecture as common clock */
  263. status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
  264. status |= PCI_EXP_LNKSTA_SLC << 16;
  265. rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
  266. /* Set RC's RCB to 128 */
  267. status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
  268. status |= PCI_EXP_LNKCTL_RCB;
  269. rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
  270. /* Enable Gen1 training */
  271. rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE,
  272. PCIE_CLIENT_CONFIG);
  273. gpiod_set_value_cansleep(rockchip->ep_gpio, 1);
  274. /* 500ms timeout value should be enough for Gen1/2 training */
  275. err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1,
  276. status, PCIE_LINK_UP(status), 20,
  277. 500 * USEC_PER_MSEC);
  278. if (err) {
  279. dev_err(dev, "PCIe link training gen1 timeout!\n");
  280. goto err_power_off_phy;
  281. }
  282. if (rockchip->link_gen == 2) {
  283. /*
  284. * Enable retrain for gen2. This should be configured only after
  285. * gen1 finished.
  286. */
  287. status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LCS);
  288. status |= PCI_EXP_LNKCTL_RL;
  289. rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LCS);
  290. err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL,
  291. status, PCIE_LINK_IS_GEN2(status), 20,
  292. 500 * USEC_PER_MSEC);
  293. if (err)
  294. dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n");
  295. }
  296. /* Check the final link width from negotiated lane counter from MGMT */
  297. status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL);
  298. status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >>
  299. PCIE_CORE_PL_CONF_LANE_SHIFT);
  300. dev_dbg(dev, "current link width is x%d\n", status);
  301. /* Power off unused lane(s) */
  302. rockchip->lanes_map = rockchip_pcie_lane_map(rockchip);
  303. for (i = 0; i < MAX_LANE_NUM; i++) {
  304. if (!(rockchip->lanes_map & BIT(i))) {
  305. dev_dbg(dev, "idling lane %d\n", i);
  306. phy_power_off(rockchip->phys[i]);
  307. }
  308. }
  309. rockchip_pcie_write(rockchip, ROCKCHIP_VENDOR_ID,
  310. PCIE_CORE_CONFIG_VENDOR);
  311. rockchip_pcie_write(rockchip,
  312. PCI_CLASS_BRIDGE_PCI << PCIE_RC_CONFIG_SCC_SHIFT,
  313. PCIE_RC_CONFIG_RID_CCR);
  314. /* Clear THP cap's next cap pointer to remove L1 substate cap */
  315. status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP);
  316. status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK;
  317. rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP);
  318. /* Clear L0s from RC's link cap */
  319. if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) {
  320. status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_LINK_CAP);
  321. status &= ~PCIE_RC_CONFIG_LINK_CAP_L0S;
  322. rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_LINK_CAP);
  323. }
  324. status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_DCSR);
  325. status &= ~PCIE_RC_CONFIG_DCSR_MPS_MASK;
  326. status |= PCIE_RC_CONFIG_DCSR_MPS_256;
  327. rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_DCSR);
  328. return 0;
  329. err_power_off_phy:
  330. while (i--)
  331. phy_power_off(rockchip->phys[i]);
  332. i = MAX_LANE_NUM;
  333. while (i--)
  334. phy_exit(rockchip->phys[i]);
  335. return err;
  336. }
  337. static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg)
  338. {
  339. struct rockchip_pcie *rockchip = arg;
  340. struct device *dev = rockchip->dev;
  341. u32 reg;
  342. u32 sub_reg;
  343. reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
  344. if (reg & PCIE_CLIENT_INT_LOCAL) {
  345. dev_dbg(dev, "local interrupt received\n");
  346. sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS);
  347. if (sub_reg & PCIE_CORE_INT_PRFPE)
  348. dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n");
  349. if (sub_reg & PCIE_CORE_INT_CRFPE)
  350. dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n");
  351. if (sub_reg & PCIE_CORE_INT_RRPE)
  352. dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n");
  353. if (sub_reg & PCIE_CORE_INT_PRFO)
  354. dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n");
  355. if (sub_reg & PCIE_CORE_INT_CRFO)
  356. dev_dbg(dev, "overflow occurred in the completion receive FIFO\n");
  357. if (sub_reg & PCIE_CORE_INT_RT)
  358. dev_dbg(dev, "replay timer timed out\n");
  359. if (sub_reg & PCIE_CORE_INT_RTR)
  360. dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n");
  361. if (sub_reg & PCIE_CORE_INT_PE)
  362. dev_dbg(dev, "phy error detected on receive side\n");
  363. if (sub_reg & PCIE_CORE_INT_MTR)
  364. dev_dbg(dev, "malformed TLP received from the link\n");
  365. if (sub_reg & PCIE_CORE_INT_UCR)
  366. dev_dbg(dev, "malformed TLP received from the link\n");
  367. if (sub_reg & PCIE_CORE_INT_FCE)
  368. dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n");
  369. if (sub_reg & PCIE_CORE_INT_CT)
  370. dev_dbg(dev, "a request timed out waiting for completion\n");
  371. if (sub_reg & PCIE_CORE_INT_UTC)
  372. dev_dbg(dev, "unmapped TC error\n");
  373. if (sub_reg & PCIE_CORE_INT_MMVC)
  374. dev_dbg(dev, "MSI mask register changes\n");
  375. rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS);
  376. } else if (reg & PCIE_CLIENT_INT_PHY) {
  377. dev_dbg(dev, "phy link changes\n");
  378. rockchip_pcie_update_txcredit_mui(rockchip);
  379. rockchip_pcie_clr_bw_int(rockchip);
  380. }
  381. rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL,
  382. PCIE_CLIENT_INT_STATUS);
  383. return IRQ_HANDLED;
  384. }
  385. static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg)
  386. {
  387. struct rockchip_pcie *rockchip = arg;
  388. struct device *dev = rockchip->dev;
  389. u32 reg;
  390. reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
  391. if (reg & PCIE_CLIENT_INT_LEGACY_DONE)
  392. dev_dbg(dev, "legacy done interrupt received\n");
  393. if (reg & PCIE_CLIENT_INT_MSG)
  394. dev_dbg(dev, "message done interrupt received\n");
  395. if (reg & PCIE_CLIENT_INT_HOT_RST)
  396. dev_dbg(dev, "hot reset interrupt received\n");
  397. if (reg & PCIE_CLIENT_INT_DPA)
  398. dev_dbg(dev, "dpa interrupt received\n");
  399. if (reg & PCIE_CLIENT_INT_FATAL_ERR)
  400. dev_dbg(dev, "fatal error interrupt received\n");
  401. if (reg & PCIE_CLIENT_INT_NFATAL_ERR)
  402. dev_dbg(dev, "no fatal error interrupt received\n");
  403. if (reg & PCIE_CLIENT_INT_CORR_ERR)
  404. dev_dbg(dev, "correctable error interrupt received\n");
  405. if (reg & PCIE_CLIENT_INT_PHY)
  406. dev_dbg(dev, "phy interrupt received\n");
  407. rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE |
  408. PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST |
  409. PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR |
  410. PCIE_CLIENT_INT_NFATAL_ERR |
  411. PCIE_CLIENT_INT_CORR_ERR |
  412. PCIE_CLIENT_INT_PHY),
  413. PCIE_CLIENT_INT_STATUS);
  414. return IRQ_HANDLED;
  415. }
  416. static void rockchip_pcie_legacy_int_handler(struct irq_desc *desc)
  417. {
  418. struct irq_chip *chip = irq_desc_get_chip(desc);
  419. struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc);
  420. struct device *dev = rockchip->dev;
  421. u32 reg;
  422. u32 hwirq;
  423. u32 virq;
  424. chained_irq_enter(chip, desc);
  425. reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS);
  426. reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT;
  427. while (reg) {
  428. hwirq = ffs(reg) - 1;
  429. reg &= ~BIT(hwirq);
  430. virq = irq_find_mapping(rockchip->irq_domain, hwirq);
  431. if (virq)
  432. generic_handle_irq(virq);
  433. else
  434. dev_err(dev, "unexpected IRQ, INT%d\n", hwirq);
  435. }
  436. chained_irq_exit(chip, desc);
  437. }
  438. static int rockchip_pcie_setup_irq(struct rockchip_pcie *rockchip)
  439. {
  440. int irq, err;
  441. struct device *dev = rockchip->dev;
  442. struct platform_device *pdev = to_platform_device(dev);
  443. irq = platform_get_irq_byname(pdev, "sys");
  444. if (irq < 0)
  445. return irq;
  446. err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler,
  447. IRQF_SHARED, "pcie-sys", rockchip);
  448. if (err) {
  449. dev_err(dev, "failed to request PCIe subsystem IRQ\n");
  450. return err;
  451. }
  452. irq = platform_get_irq_byname(pdev, "legacy");
  453. if (irq < 0)
  454. return irq;
  455. irq_set_chained_handler_and_data(irq,
  456. rockchip_pcie_legacy_int_handler,
  457. rockchip);
  458. irq = platform_get_irq_byname(pdev, "client");
  459. if (irq < 0)
  460. return irq;
  461. err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler,
  462. IRQF_SHARED, "pcie-client", rockchip);
  463. if (err) {
  464. dev_err(dev, "failed to request PCIe client IRQ\n");
  465. return err;
  466. }
  467. return 0;
  468. }
  469. /**
  470. * rockchip_pcie_parse_host_dt - Parse Device Tree
  471. * @rockchip: PCIe port information
  472. *
  473. * Return: '0' on success and error value on failure
  474. */
  475. static int rockchip_pcie_parse_host_dt(struct rockchip_pcie *rockchip)
  476. {
  477. struct device *dev = rockchip->dev;
  478. int err;
  479. err = rockchip_pcie_parse_dt(rockchip);
  480. if (err)
  481. return err;
  482. rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v");
  483. if (IS_ERR(rockchip->vpcie12v)) {
  484. if (PTR_ERR(rockchip->vpcie12v) != -ENODEV)
  485. return PTR_ERR(rockchip->vpcie12v);
  486. dev_info(dev, "no vpcie12v regulator found\n");
  487. }
  488. rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3");
  489. if (IS_ERR(rockchip->vpcie3v3)) {
  490. if (PTR_ERR(rockchip->vpcie3v3) != -ENODEV)
  491. return PTR_ERR(rockchip->vpcie3v3);
  492. dev_info(dev, "no vpcie3v3 regulator found\n");
  493. }
  494. rockchip->vpcie1v8 = devm_regulator_get(dev, "vpcie1v8");
  495. if (IS_ERR(rockchip->vpcie1v8))
  496. return PTR_ERR(rockchip->vpcie1v8);
  497. rockchip->vpcie0v9 = devm_regulator_get(dev, "vpcie0v9");
  498. if (IS_ERR(rockchip->vpcie0v9))
  499. return PTR_ERR(rockchip->vpcie0v9);
  500. return 0;
  501. }
  502. static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip)
  503. {
  504. struct device *dev = rockchip->dev;
  505. int err;
  506. if (!IS_ERR(rockchip->vpcie12v)) {
  507. err = regulator_enable(rockchip->vpcie12v);
  508. if (err) {
  509. dev_err(dev, "fail to enable vpcie12v regulator\n");
  510. goto err_out;
  511. }
  512. }
  513. if (!IS_ERR(rockchip->vpcie3v3)) {
  514. err = regulator_enable(rockchip->vpcie3v3);
  515. if (err) {
  516. dev_err(dev, "fail to enable vpcie3v3 regulator\n");
  517. goto err_disable_12v;
  518. }
  519. }
  520. err = regulator_enable(rockchip->vpcie1v8);
  521. if (err) {
  522. dev_err(dev, "fail to enable vpcie1v8 regulator\n");
  523. goto err_disable_3v3;
  524. }
  525. err = regulator_enable(rockchip->vpcie0v9);
  526. if (err) {
  527. dev_err(dev, "fail to enable vpcie0v9 regulator\n");
  528. goto err_disable_1v8;
  529. }
  530. return 0;
  531. err_disable_1v8:
  532. regulator_disable(rockchip->vpcie1v8);
  533. err_disable_3v3:
  534. if (!IS_ERR(rockchip->vpcie3v3))
  535. regulator_disable(rockchip->vpcie3v3);
  536. err_disable_12v:
  537. if (!IS_ERR(rockchip->vpcie12v))
  538. regulator_disable(rockchip->vpcie12v);
  539. err_out:
  540. return err;
  541. }
  542. static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip)
  543. {
  544. rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) &
  545. (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK);
  546. rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT),
  547. PCIE_CORE_INT_MASK);
  548. rockchip_pcie_enable_bw_int(rockchip);
  549. }
  550. static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq,
  551. irq_hw_number_t hwirq)
  552. {
  553. irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq);
  554. irq_set_chip_data(irq, domain->host_data);
  555. return 0;
  556. }
  557. static const struct irq_domain_ops intx_domain_ops = {
  558. .map = rockchip_pcie_intx_map,
  559. };
  560. static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip)
  561. {
  562. struct device *dev = rockchip->dev;
  563. struct device_node *intc = of_get_next_child(dev->of_node, NULL);
  564. if (!intc) {
  565. dev_err(dev, "missing child interrupt-controller node\n");
  566. return -EINVAL;
  567. }
  568. rockchip->irq_domain = irq_domain_add_linear(intc, PCI_NUM_INTX,
  569. &intx_domain_ops, rockchip);
  570. of_node_put(intc);
  571. if (!rockchip->irq_domain) {
  572. dev_err(dev, "failed to get a INTx IRQ domain\n");
  573. return -EINVAL;
  574. }
  575. return 0;
  576. }
  577. static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip,
  578. int region_no, int type, u8 num_pass_bits,
  579. u32 lower_addr, u32 upper_addr)
  580. {
  581. u32 ob_addr_0;
  582. u32 ob_addr_1;
  583. u32 ob_desc_0;
  584. u32 aw_offset;
  585. if (region_no >= MAX_AXI_WRAPPER_REGION_NUM)
  586. return -EINVAL;
  587. if (num_pass_bits + 1 < 8)
  588. return -EINVAL;
  589. if (num_pass_bits > 63)
  590. return -EINVAL;
  591. if (region_no == 0) {
  592. if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits))
  593. return -EINVAL;
  594. }
  595. if (region_no != 0) {
  596. if (AXI_REGION_SIZE < (2ULL << num_pass_bits))
  597. return -EINVAL;
  598. }
  599. aw_offset = (region_no << OB_REG_SIZE_SHIFT);
  600. ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS;
  601. ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR;
  602. ob_addr_1 = upper_addr;
  603. ob_desc_0 = (1 << 23 | type);
  604. rockchip_pcie_write(rockchip, ob_addr_0,
  605. PCIE_CORE_OB_REGION_ADDR0 + aw_offset);
  606. rockchip_pcie_write(rockchip, ob_addr_1,
  607. PCIE_CORE_OB_REGION_ADDR1 + aw_offset);
  608. rockchip_pcie_write(rockchip, ob_desc_0,
  609. PCIE_CORE_OB_REGION_DESC0 + aw_offset);
  610. rockchip_pcie_write(rockchip, 0,
  611. PCIE_CORE_OB_REGION_DESC1 + aw_offset);
  612. return 0;
  613. }
  614. static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip,
  615. int region_no, u8 num_pass_bits,
  616. u32 lower_addr, u32 upper_addr)
  617. {
  618. u32 ib_addr_0;
  619. u32 ib_addr_1;
  620. u32 aw_offset;
  621. if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM)
  622. return -EINVAL;
  623. if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED)
  624. return -EINVAL;
  625. if (num_pass_bits > 63)
  626. return -EINVAL;
  627. aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT);
  628. ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS;
  629. ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR;
  630. ib_addr_1 = upper_addr;
  631. rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset);
  632. rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset);
  633. return 0;
  634. }
  635. static int rockchip_pcie_cfg_atu(struct rockchip_pcie *rockchip)
  636. {
  637. struct device *dev = rockchip->dev;
  638. struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip);
  639. struct resource_entry *entry;
  640. u64 pci_addr, size;
  641. int offset;
  642. int err;
  643. int reg_no;
  644. rockchip_pcie_cfg_configuration_accesses(rockchip,
  645. AXI_WRAPPER_TYPE0_CFG);
  646. entry = resource_list_first_type(&bridge->windows, IORESOURCE_MEM);
  647. if (!entry)
  648. return -ENODEV;
  649. size = resource_size(entry->res);
  650. pci_addr = entry->res->start - entry->offset;
  651. rockchip->msg_bus_addr = pci_addr;
  652. for (reg_no = 0; reg_no < (size >> 20); reg_no++) {
  653. err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1,
  654. AXI_WRAPPER_MEM_WRITE,
  655. 20 - 1,
  656. pci_addr + (reg_no << 20),
  657. 0);
  658. if (err) {
  659. dev_err(dev, "program RC mem outbound ATU failed\n");
  660. return err;
  661. }
  662. }
  663. err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0);
  664. if (err) {
  665. dev_err(dev, "program RC mem inbound ATU failed\n");
  666. return err;
  667. }
  668. entry = resource_list_first_type(&bridge->windows, IORESOURCE_IO);
  669. if (!entry)
  670. return -ENODEV;
  671. /* store the register number offset to program RC io outbound ATU */
  672. offset = size >> 20;
  673. size = resource_size(entry->res);
  674. pci_addr = entry->res->start - entry->offset;
  675. for (reg_no = 0; reg_no < (size >> 20); reg_no++) {
  676. err = rockchip_pcie_prog_ob_atu(rockchip,
  677. reg_no + 1 + offset,
  678. AXI_WRAPPER_IO_WRITE,
  679. 20 - 1,
  680. pci_addr + (reg_no << 20),
  681. 0);
  682. if (err) {
  683. dev_err(dev, "program RC io outbound ATU failed\n");
  684. return err;
  685. }
  686. }
  687. /* assign message regions */
  688. rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1 + offset,
  689. AXI_WRAPPER_NOR_MSG,
  690. 20 - 1, 0, 0);
  691. rockchip->msg_bus_addr += ((reg_no + offset) << 20);
  692. return err;
  693. }
  694. static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip)
  695. {
  696. u32 value;
  697. int err;
  698. /* send PME_TURN_OFF message */
  699. writel(0x0, rockchip->msg_region + PCIE_RC_SEND_PME_OFF);
  700. /* read LTSSM and wait for falling into L2 link state */
  701. err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0,
  702. value, PCIE_LINK_IS_L2(value), 20,
  703. jiffies_to_usecs(5 * HZ));
  704. if (err) {
  705. dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n");
  706. return err;
  707. }
  708. return 0;
  709. }
  710. static int __maybe_unused rockchip_pcie_suspend_noirq(struct device *dev)
  711. {
  712. struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
  713. int ret;
  714. /* disable core and cli int since we don't need to ack PME_ACK */
  715. rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) |
  716. PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK);
  717. rockchip_pcie_write(rockchip, (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK);
  718. ret = rockchip_pcie_wait_l2(rockchip);
  719. if (ret) {
  720. rockchip_pcie_enable_interrupts(rockchip);
  721. return ret;
  722. }
  723. rockchip_pcie_deinit_phys(rockchip);
  724. rockchip_pcie_disable_clocks(rockchip);
  725. regulator_disable(rockchip->vpcie0v9);
  726. return ret;
  727. }
  728. static int __maybe_unused rockchip_pcie_resume_noirq(struct device *dev)
  729. {
  730. struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
  731. int err;
  732. err = regulator_enable(rockchip->vpcie0v9);
  733. if (err) {
  734. dev_err(dev, "fail to enable vpcie0v9 regulator\n");
  735. return err;
  736. }
  737. err = rockchip_pcie_enable_clocks(rockchip);
  738. if (err)
  739. goto err_disable_0v9;
  740. err = rockchip_pcie_host_init_port(rockchip);
  741. if (err)
  742. goto err_pcie_resume;
  743. err = rockchip_pcie_cfg_atu(rockchip);
  744. if (err)
  745. goto err_err_deinit_port;
  746. /* Need this to enter L1 again */
  747. rockchip_pcie_update_txcredit_mui(rockchip);
  748. rockchip_pcie_enable_interrupts(rockchip);
  749. return 0;
  750. err_err_deinit_port:
  751. rockchip_pcie_deinit_phys(rockchip);
  752. err_pcie_resume:
  753. rockchip_pcie_disable_clocks(rockchip);
  754. err_disable_0v9:
  755. regulator_disable(rockchip->vpcie0v9);
  756. return err;
  757. }
  758. static int rockchip_pcie_probe(struct platform_device *pdev)
  759. {
  760. struct rockchip_pcie *rockchip;
  761. struct device *dev = &pdev->dev;
  762. struct pci_host_bridge *bridge;
  763. int err;
  764. if (!dev->of_node)
  765. return -ENODEV;
  766. bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rockchip));
  767. if (!bridge)
  768. return -ENOMEM;
  769. rockchip = pci_host_bridge_priv(bridge);
  770. platform_set_drvdata(pdev, rockchip);
  771. rockchip->dev = dev;
  772. rockchip->is_rc = true;
  773. err = rockchip_pcie_parse_host_dt(rockchip);
  774. if (err)
  775. return err;
  776. err = rockchip_pcie_enable_clocks(rockchip);
  777. if (err)
  778. return err;
  779. err = rockchip_pcie_set_vpcie(rockchip);
  780. if (err) {
  781. dev_err(dev, "failed to set vpcie regulator\n");
  782. goto err_set_vpcie;
  783. }
  784. err = rockchip_pcie_host_init_port(rockchip);
  785. if (err)
  786. goto err_vpcie;
  787. err = rockchip_pcie_init_irq_domain(rockchip);
  788. if (err < 0)
  789. goto err_deinit_port;
  790. err = rockchip_pcie_cfg_atu(rockchip);
  791. if (err)
  792. goto err_remove_irq_domain;
  793. rockchip->msg_region = devm_ioremap(dev, rockchip->msg_bus_addr, SZ_1M);
  794. if (!rockchip->msg_region) {
  795. err = -ENOMEM;
  796. goto err_remove_irq_domain;
  797. }
  798. bridge->sysdata = rockchip;
  799. bridge->ops = &rockchip_pcie_ops;
  800. err = rockchip_pcie_setup_irq(rockchip);
  801. if (err)
  802. goto err_remove_irq_domain;
  803. rockchip_pcie_enable_interrupts(rockchip);
  804. err = pci_host_probe(bridge);
  805. if (err < 0)
  806. goto err_remove_irq_domain;
  807. return 0;
  808. err_remove_irq_domain:
  809. irq_domain_remove(rockchip->irq_domain);
  810. err_deinit_port:
  811. rockchip_pcie_deinit_phys(rockchip);
  812. err_vpcie:
  813. if (!IS_ERR(rockchip->vpcie12v))
  814. regulator_disable(rockchip->vpcie12v);
  815. if (!IS_ERR(rockchip->vpcie3v3))
  816. regulator_disable(rockchip->vpcie3v3);
  817. regulator_disable(rockchip->vpcie1v8);
  818. regulator_disable(rockchip->vpcie0v9);
  819. err_set_vpcie:
  820. rockchip_pcie_disable_clocks(rockchip);
  821. return err;
  822. }
  823. static int rockchip_pcie_remove(struct platform_device *pdev)
  824. {
  825. struct device *dev = &pdev->dev;
  826. struct rockchip_pcie *rockchip = dev_get_drvdata(dev);
  827. struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip);
  828. pci_stop_root_bus(bridge->bus);
  829. pci_remove_root_bus(bridge->bus);
  830. irq_domain_remove(rockchip->irq_domain);
  831. rockchip_pcie_deinit_phys(rockchip);
  832. rockchip_pcie_disable_clocks(rockchip);
  833. if (!IS_ERR(rockchip->vpcie12v))
  834. regulator_disable(rockchip->vpcie12v);
  835. if (!IS_ERR(rockchip->vpcie3v3))
  836. regulator_disable(rockchip->vpcie3v3);
  837. regulator_disable(rockchip->vpcie1v8);
  838. regulator_disable(rockchip->vpcie0v9);
  839. return 0;
  840. }
  841. static const struct dev_pm_ops rockchip_pcie_pm_ops = {
  842. SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq,
  843. rockchip_pcie_resume_noirq)
  844. };
  845. static const struct of_device_id rockchip_pcie_of_match[] = {
  846. { .compatible = "rockchip,rk3399-pcie", },
  847. {}
  848. };
  849. MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match);
  850. static struct platform_driver rockchip_pcie_driver = {
  851. .driver = {
  852. .name = "rockchip-pcie",
  853. .of_match_table = rockchip_pcie_of_match,
  854. .pm = &rockchip_pcie_pm_ops,
  855. },
  856. .probe = rockchip_pcie_probe,
  857. .remove = rockchip_pcie_remove,
  858. };
  859. module_platform_driver(rockchip_pcie_driver);
  860. MODULE_AUTHOR("Rockchip Inc");
  861. MODULE_DESCRIPTION("Rockchip AXI PCIe driver");
  862. MODULE_LICENSE("GPL v2");