aspm.c 40 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Enable PCIe link L0s/L1 state and Clock Power Management
  4. *
  5. * Copyright (C) 2007 Intel
  6. * Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com)
  7. * Copyright (C) Shaohua Li (shaohua.li@intel.com)
  8. */
  9. #include <linux/kernel.h>
  10. #include <linux/module.h>
  11. #include <linux/moduleparam.h>
  12. #include <linux/pci.h>
  13. #include <linux/pci_regs.h>
  14. #include <linux/errno.h>
  15. #include <linux/pm.h>
  16. #include <linux/init.h>
  17. #include <linux/slab.h>
  18. #include <linux/jiffies.h>
  19. #include <linux/delay.h>
  20. #include "../pci.h"
  21. #ifdef MODULE_PARAM_PREFIX
  22. #undef MODULE_PARAM_PREFIX
  23. #endif
  24. #define MODULE_PARAM_PREFIX "pcie_aspm."
  25. /* Note: those are not register definitions */
  26. #define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */
  27. #define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */
  28. #define ASPM_STATE_L1 (4) /* L1 state */
  29. #define ASPM_STATE_L1_1 (8) /* ASPM L1.1 state */
  30. #define ASPM_STATE_L1_2 (0x10) /* ASPM L1.2 state */
  31. #define ASPM_STATE_L1_1_PCIPM (0x20) /* PCI PM L1.1 state */
  32. #define ASPM_STATE_L1_2_PCIPM (0x40) /* PCI PM L1.2 state */
  33. #define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM)
  34. #define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM)
  35. #define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\
  36. ASPM_STATE_L1_2_MASK)
  37. #define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW)
  38. #define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \
  39. ASPM_STATE_L1SS)
  40. struct aspm_latency {
  41. u32 l0s; /* L0s latency (nsec) */
  42. u32 l1; /* L1 latency (nsec) */
  43. };
  44. struct pcie_link_state {
  45. struct pci_dev *pdev; /* Upstream component of the Link */
  46. struct pci_dev *downstream; /* Downstream component, function 0 */
  47. struct pcie_link_state *root; /* pointer to the root port link */
  48. struct pcie_link_state *parent; /* pointer to the parent Link state */
  49. struct list_head sibling; /* node in link_list */
  50. /* ASPM state */
  51. u32 aspm_support:7; /* Supported ASPM state */
  52. u32 aspm_enabled:7; /* Enabled ASPM state */
  53. u32 aspm_capable:7; /* Capable ASPM state with latency */
  54. u32 aspm_default:7; /* Default ASPM state by BIOS */
  55. u32 aspm_disable:7; /* Disabled ASPM state */
  56. /* Clock PM state */
  57. u32 clkpm_capable:1; /* Clock PM capable? */
  58. u32 clkpm_enabled:1; /* Current Clock PM state */
  59. u32 clkpm_default:1; /* Default Clock PM state by BIOS */
  60. u32 clkpm_disable:1; /* Clock PM disabled */
  61. /* Exit latencies */
  62. struct aspm_latency latency_up; /* Upstream direction exit latency */
  63. struct aspm_latency latency_dw; /* Downstream direction exit latency */
  64. /*
  65. * Endpoint acceptable latencies. A pcie downstream port only
  66. * has one slot under it, so at most there are 8 functions.
  67. */
  68. struct aspm_latency acceptable[8];
  69. };
  70. static int aspm_disabled, aspm_force;
  71. static bool aspm_support_enabled = true;
  72. static DEFINE_MUTEX(aspm_lock);
  73. static LIST_HEAD(link_list);
  74. #define POLICY_DEFAULT 0 /* BIOS default setting */
  75. #define POLICY_PERFORMANCE 1 /* high performance */
  76. #define POLICY_POWERSAVE 2 /* high power saving */
  77. #define POLICY_POWER_SUPERSAVE 3 /* possibly even more power saving */
  78. #ifdef CONFIG_PCIEASPM_PERFORMANCE
  79. static int aspm_policy = POLICY_PERFORMANCE;
  80. #elif defined CONFIG_PCIEASPM_POWERSAVE
  81. static int aspm_policy = POLICY_POWERSAVE;
  82. #elif defined CONFIG_PCIEASPM_POWER_SUPERSAVE
  83. static int aspm_policy = POLICY_POWER_SUPERSAVE;
  84. #else
  85. static int aspm_policy;
  86. #endif
  87. static const char *policy_str[] = {
  88. [POLICY_DEFAULT] = "default",
  89. [POLICY_PERFORMANCE] = "performance",
  90. [POLICY_POWERSAVE] = "powersave",
  91. [POLICY_POWER_SUPERSAVE] = "powersupersave"
  92. };
  93. #define LINK_RETRAIN_TIMEOUT HZ
  94. static int policy_to_aspm_state(struct pcie_link_state *link)
  95. {
  96. switch (aspm_policy) {
  97. case POLICY_PERFORMANCE:
  98. /* Disable ASPM and Clock PM */
  99. return 0;
  100. case POLICY_POWERSAVE:
  101. /* Enable ASPM L0s/L1 */
  102. return (ASPM_STATE_L0S | ASPM_STATE_L1);
  103. case POLICY_POWER_SUPERSAVE:
  104. /* Enable Everything */
  105. return ASPM_STATE_ALL;
  106. case POLICY_DEFAULT:
  107. return link->aspm_default;
  108. }
  109. return 0;
  110. }
  111. static int policy_to_clkpm_state(struct pcie_link_state *link)
  112. {
  113. switch (aspm_policy) {
  114. case POLICY_PERFORMANCE:
  115. /* Disable ASPM and Clock PM */
  116. return 0;
  117. case POLICY_POWERSAVE:
  118. case POLICY_POWER_SUPERSAVE:
  119. /* Enable Clock PM */
  120. return 1;
  121. case POLICY_DEFAULT:
  122. return link->clkpm_default;
  123. }
  124. return 0;
  125. }
  126. static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)
  127. {
  128. struct pci_dev *child;
  129. struct pci_bus *linkbus = link->pdev->subordinate;
  130. u32 val = enable ? PCI_EXP_LNKCTL_CLKREQ_EN : 0;
  131. list_for_each_entry(child, &linkbus->devices, bus_list)
  132. pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL,
  133. PCI_EXP_LNKCTL_CLKREQ_EN,
  134. val);
  135. link->clkpm_enabled = !!enable;
  136. }
  137. static void pcie_set_clkpm(struct pcie_link_state *link, int enable)
  138. {
  139. /*
  140. * Don't enable Clock PM if the link is not Clock PM capable
  141. * or Clock PM is disabled
  142. */
  143. if (!link->clkpm_capable || link->clkpm_disable)
  144. enable = 0;
  145. /* Need nothing if the specified equals to current state */
  146. if (link->clkpm_enabled == enable)
  147. return;
  148. pcie_set_clkpm_nocheck(link, enable);
  149. }
  150. static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)
  151. {
  152. int capable = 1, enabled = 1;
  153. u32 reg32;
  154. u16 reg16;
  155. struct pci_dev *child;
  156. struct pci_bus *linkbus = link->pdev->subordinate;
  157. /* All functions should have the same cap and state, take the worst */
  158. list_for_each_entry(child, &linkbus->devices, bus_list) {
  159. pcie_capability_read_dword(child, PCI_EXP_LNKCAP, &reg32);
  160. if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
  161. capable = 0;
  162. enabled = 0;
  163. break;
  164. }
  165. pcie_capability_read_word(child, PCI_EXP_LNKCTL, &reg16);
  166. if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
  167. enabled = 0;
  168. }
  169. link->clkpm_enabled = enabled;
  170. link->clkpm_default = enabled;
  171. link->clkpm_capable = capable;
  172. link->clkpm_disable = blacklist ? 1 : 0;
  173. }
  174. static bool pcie_retrain_link(struct pcie_link_state *link)
  175. {
  176. struct pci_dev *parent = link->pdev;
  177. unsigned long end_jiffies;
  178. u16 reg16;
  179. pcie_capability_read_word(parent, PCI_EXP_LNKCTL, &reg16);
  180. reg16 |= PCI_EXP_LNKCTL_RL;
  181. pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16);
  182. if (parent->clear_retrain_link) {
  183. /*
  184. * Due to an erratum in some devices the Retrain Link bit
  185. * needs to be cleared again manually to allow the link
  186. * training to succeed.
  187. */
  188. reg16 &= ~PCI_EXP_LNKCTL_RL;
  189. pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16);
  190. }
  191. /* Wait for link training end. Break out after waiting for timeout */
  192. end_jiffies = jiffies + LINK_RETRAIN_TIMEOUT;
  193. do {
  194. pcie_capability_read_word(parent, PCI_EXP_LNKSTA, &reg16);
  195. if (!(reg16 & PCI_EXP_LNKSTA_LT))
  196. break;
  197. msleep(1);
  198. } while (time_before(jiffies, end_jiffies));
  199. return !(reg16 & PCI_EXP_LNKSTA_LT);
  200. }
  201. /*
  202. * pcie_aspm_configure_common_clock: check if the 2 ends of a link
  203. * could use common clock. If they are, configure them to use the
  204. * common clock. That will reduce the ASPM state exit latency.
  205. */
  206. static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)
  207. {
  208. int same_clock = 1;
  209. u16 reg16, parent_reg, child_reg[8];
  210. struct pci_dev *child, *parent = link->pdev;
  211. struct pci_bus *linkbus = parent->subordinate;
  212. /*
  213. * All functions of a slot should have the same Slot Clock
  214. * Configuration, so just check one function
  215. */
  216. child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
  217. BUG_ON(!pci_is_pcie(child));
  218. /* Check downstream component if bit Slot Clock Configuration is 1 */
  219. pcie_capability_read_word(child, PCI_EXP_LNKSTA, &reg16);
  220. if (!(reg16 & PCI_EXP_LNKSTA_SLC))
  221. same_clock = 0;
  222. /* Check upstream component if bit Slot Clock Configuration is 1 */
  223. pcie_capability_read_word(parent, PCI_EXP_LNKSTA, &reg16);
  224. if (!(reg16 & PCI_EXP_LNKSTA_SLC))
  225. same_clock = 0;
  226. /* Port might be already in common clock mode */
  227. pcie_capability_read_word(parent, PCI_EXP_LNKCTL, &reg16);
  228. if (same_clock && (reg16 & PCI_EXP_LNKCTL_CCC)) {
  229. bool consistent = true;
  230. list_for_each_entry(child, &linkbus->devices, bus_list) {
  231. pcie_capability_read_word(child, PCI_EXP_LNKCTL,
  232. &reg16);
  233. if (!(reg16 & PCI_EXP_LNKCTL_CCC)) {
  234. consistent = false;
  235. break;
  236. }
  237. }
  238. if (consistent)
  239. return;
  240. pci_info(parent, "ASPM: current common clock configuration is inconsistent, reconfiguring\n");
  241. }
  242. /* Configure downstream component, all functions */
  243. list_for_each_entry(child, &linkbus->devices, bus_list) {
  244. pcie_capability_read_word(child, PCI_EXP_LNKCTL, &reg16);
  245. child_reg[PCI_FUNC(child->devfn)] = reg16;
  246. if (same_clock)
  247. reg16 |= PCI_EXP_LNKCTL_CCC;
  248. else
  249. reg16 &= ~PCI_EXP_LNKCTL_CCC;
  250. pcie_capability_write_word(child, PCI_EXP_LNKCTL, reg16);
  251. }
  252. /* Configure upstream component */
  253. pcie_capability_read_word(parent, PCI_EXP_LNKCTL, &reg16);
  254. parent_reg = reg16;
  255. if (same_clock)
  256. reg16 |= PCI_EXP_LNKCTL_CCC;
  257. else
  258. reg16 &= ~PCI_EXP_LNKCTL_CCC;
  259. pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16);
  260. if (pcie_retrain_link(link))
  261. return;
  262. /* Training failed. Restore common clock configurations */
  263. pci_err(parent, "ASPM: Could not configure common clock\n");
  264. list_for_each_entry(child, &linkbus->devices, bus_list)
  265. pcie_capability_write_word(child, PCI_EXP_LNKCTL,
  266. child_reg[PCI_FUNC(child->devfn)]);
  267. pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_reg);
  268. }
  269. /* Convert L0s latency encoding to ns */
  270. static u32 calc_l0s_latency(u32 lnkcap)
  271. {
  272. u32 encoding = (lnkcap & PCI_EXP_LNKCAP_L0SEL) >> 12;
  273. if (encoding == 0x7)
  274. return (5 * 1000); /* > 4us */
  275. return (64 << encoding);
  276. }
  277. /* Convert L0s acceptable latency encoding to ns */
  278. static u32 calc_l0s_acceptable(u32 encoding)
  279. {
  280. if (encoding == 0x7)
  281. return -1U;
  282. return (64 << encoding);
  283. }
  284. /* Convert L1 latency encoding to ns */
  285. static u32 calc_l1_latency(u32 lnkcap)
  286. {
  287. u32 encoding = (lnkcap & PCI_EXP_LNKCAP_L1EL) >> 15;
  288. if (encoding == 0x7)
  289. return (65 * 1000); /* > 64us */
  290. return (1000 << encoding);
  291. }
  292. /* Convert L1 acceptable latency encoding to ns */
  293. static u32 calc_l1_acceptable(u32 encoding)
  294. {
  295. if (encoding == 0x7)
  296. return -1U;
  297. return (1000 << encoding);
  298. }
  299. /* Convert L1SS T_pwr encoding to usec */
  300. static u32 calc_l1ss_pwron(struct pci_dev *pdev, u32 scale, u32 val)
  301. {
  302. switch (scale) {
  303. case 0:
  304. return val * 2;
  305. case 1:
  306. return val * 10;
  307. case 2:
  308. return val * 100;
  309. }
  310. pci_err(pdev, "%s: Invalid T_PwrOn scale: %u\n", __func__, scale);
  311. return 0;
  312. }
  313. static void encode_l12_threshold(u32 threshold_us, u32 *scale, u32 *value)
  314. {
  315. u32 threshold_ns = threshold_us * 1000;
  316. /* See PCIe r3.1, sec 7.33.3 and sec 6.18 */
  317. if (threshold_ns < 32) {
  318. *scale = 0;
  319. *value = threshold_ns;
  320. } else if (threshold_ns < 1024) {
  321. *scale = 1;
  322. *value = threshold_ns >> 5;
  323. } else if (threshold_ns < 32768) {
  324. *scale = 2;
  325. *value = threshold_ns >> 10;
  326. } else if (threshold_ns < 1048576) {
  327. *scale = 3;
  328. *value = threshold_ns >> 15;
  329. } else if (threshold_ns < 33554432) {
  330. *scale = 4;
  331. *value = threshold_ns >> 20;
  332. } else {
  333. *scale = 5;
  334. *value = threshold_ns >> 25;
  335. }
  336. }
  337. static void pcie_aspm_check_latency(struct pci_dev *endpoint)
  338. {
  339. u32 latency, l1_switch_latency = 0;
  340. struct aspm_latency *acceptable;
  341. struct pcie_link_state *link;
  342. /* Device not in D0 doesn't need latency check */
  343. if ((endpoint->current_state != PCI_D0) &&
  344. (endpoint->current_state != PCI_UNKNOWN))
  345. return;
  346. link = endpoint->bus->self->link_state;
  347. acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)];
  348. while (link) {
  349. /* Check upstream direction L0s latency */
  350. if ((link->aspm_capable & ASPM_STATE_L0S_UP) &&
  351. (link->latency_up.l0s > acceptable->l0s))
  352. link->aspm_capable &= ~ASPM_STATE_L0S_UP;
  353. /* Check downstream direction L0s latency */
  354. if ((link->aspm_capable & ASPM_STATE_L0S_DW) &&
  355. (link->latency_dw.l0s > acceptable->l0s))
  356. link->aspm_capable &= ~ASPM_STATE_L0S_DW;
  357. /*
  358. * Check L1 latency.
  359. * Every switch on the path to root complex need 1
  360. * more microsecond for L1. Spec doesn't mention L0s.
  361. *
  362. * The exit latencies for L1 substates are not advertised
  363. * by a device. Since the spec also doesn't mention a way
  364. * to determine max latencies introduced by enabling L1
  365. * substates on the components, it is not clear how to do
  366. * a L1 substate exit latency check. We assume that the
  367. * L1 exit latencies advertised by a device include L1
  368. * substate latencies (and hence do not do any check).
  369. */
  370. latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1);
  371. if ((link->aspm_capable & ASPM_STATE_L1) &&
  372. (latency + l1_switch_latency > acceptable->l1))
  373. link->aspm_capable &= ~ASPM_STATE_L1;
  374. l1_switch_latency += 1000;
  375. link = link->parent;
  376. }
  377. }
  378. /*
  379. * The L1 PM substate capability is only implemented in function 0 in a
  380. * multi function device.
  381. */
  382. static struct pci_dev *pci_function_0(struct pci_bus *linkbus)
  383. {
  384. struct pci_dev *child;
  385. list_for_each_entry(child, &linkbus->devices, bus_list)
  386. if (PCI_FUNC(child->devfn) == 0)
  387. return child;
  388. return NULL;
  389. }
  390. static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos,
  391. u32 clear, u32 set)
  392. {
  393. u32 val;
  394. pci_read_config_dword(pdev, pos, &val);
  395. val &= ~clear;
  396. val |= set;
  397. pci_write_config_dword(pdev, pos, val);
  398. }
  399. /* Calculate L1.2 PM substate timing parameters */
  400. static void aspm_calc_l1ss_info(struct pcie_link_state *link,
  401. u32 parent_l1ss_cap, u32 child_l1ss_cap)
  402. {
  403. struct pci_dev *child = link->downstream, *parent = link->pdev;
  404. u32 val1, val2, scale1, scale2;
  405. u32 t_common_mode, t_power_on, l1_2_threshold, scale, value;
  406. u32 ctl1 = 0, ctl2 = 0;
  407. u32 pctl1, pctl2, cctl1, cctl2;
  408. u32 pl1_2_enables, cl1_2_enables;
  409. if (!(link->aspm_support & ASPM_STATE_L1_2_MASK))
  410. return;
  411. /* Choose the greater of the two Port Common_Mode_Restore_Times */
  412. val1 = (parent_l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8;
  413. val2 = (child_l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8;
  414. t_common_mode = max(val1, val2);
  415. /* Choose the greater of the two Port T_POWER_ON times */
  416. val1 = (parent_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19;
  417. scale1 = (parent_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16;
  418. val2 = (child_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19;
  419. scale2 = (child_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16;
  420. if (calc_l1ss_pwron(parent, scale1, val1) >
  421. calc_l1ss_pwron(child, scale2, val2)) {
  422. ctl2 |= scale1 | (val1 << 3);
  423. t_power_on = calc_l1ss_pwron(parent, scale1, val1);
  424. } else {
  425. ctl2 |= scale2 | (val2 << 3);
  426. t_power_on = calc_l1ss_pwron(child, scale2, val2);
  427. }
  428. /*
  429. * Set LTR_L1.2_THRESHOLD to the time required to transition the
  430. * Link from L0 to L1.2 and back to L0 so we enter L1.2 only if
  431. * downstream devices report (via LTR) that they can tolerate at
  432. * least that much latency.
  433. *
  434. * Based on PCIe r3.1, sec 5.5.3.3.1, Figures 5-16 and 5-17, and
  435. * Table 5-11. T(POWER_OFF) is at most 2us and T(L1.2) is at
  436. * least 4us.
  437. */
  438. l1_2_threshold = 2 + 4 + t_common_mode + t_power_on;
  439. encode_l12_threshold(l1_2_threshold, &scale, &value);
  440. ctl1 |= t_common_mode << 8 | scale << 29 | value << 16;
  441. pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, &pctl1);
  442. pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, &pctl2);
  443. pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, &cctl1);
  444. pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL2, &cctl2);
  445. if (ctl1 == pctl1 && ctl1 == cctl1 &&
  446. ctl2 == pctl2 && ctl2 == cctl2)
  447. return;
  448. /* Disable L1.2 while updating. See PCIe r5.0, sec 5.5.4, 7.8.3.3 */
  449. pl1_2_enables = pctl1 & PCI_L1SS_CTL1_L1_2_MASK;
  450. cl1_2_enables = cctl1 & PCI_L1SS_CTL1_L1_2_MASK;
  451. if (pl1_2_enables || cl1_2_enables) {
  452. pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
  453. PCI_L1SS_CTL1_L1_2_MASK, 0);
  454. pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
  455. PCI_L1SS_CTL1_L1_2_MASK, 0);
  456. }
  457. /* Program T_POWER_ON times in both ports */
  458. pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, ctl2);
  459. pci_write_config_dword(child, child->l1ss + PCI_L1SS_CTL2, ctl2);
  460. /* Program Common_Mode_Restore_Time in upstream device */
  461. pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
  462. PCI_L1SS_CTL1_CM_RESTORE_TIME, ctl1);
  463. /* Program LTR_L1.2_THRESHOLD time in both ports */
  464. pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
  465. PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
  466. PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1);
  467. pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
  468. PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
  469. PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1);
  470. if (pl1_2_enables || cl1_2_enables) {
  471. pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, 0,
  472. pl1_2_enables);
  473. pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, 0,
  474. cl1_2_enables);
  475. }
  476. }
  477. static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
  478. {
  479. struct pci_dev *child = link->downstream, *parent = link->pdev;
  480. u32 parent_lnkcap, child_lnkcap;
  481. u16 parent_lnkctl, child_lnkctl;
  482. u32 parent_l1ss_cap, child_l1ss_cap;
  483. u32 parent_l1ss_ctl1 = 0, child_l1ss_ctl1 = 0;
  484. struct pci_bus *linkbus = parent->subordinate;
  485. if (blacklist) {
  486. /* Set enabled/disable so that we will disable ASPM later */
  487. link->aspm_enabled = ASPM_STATE_ALL;
  488. link->aspm_disable = ASPM_STATE_ALL;
  489. return;
  490. }
  491. /*
  492. * If ASPM not supported, don't mess with the clocks and link,
  493. * bail out now.
  494. */
  495. pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &parent_lnkcap);
  496. pcie_capability_read_dword(child, PCI_EXP_LNKCAP, &child_lnkcap);
  497. if (!(parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPMS))
  498. return;
  499. /* Configure common clock before checking latencies */
  500. pcie_aspm_configure_common_clock(link);
  501. /*
  502. * Re-read upstream/downstream components' register state after
  503. * clock configuration. L0s & L1 exit latencies in the otherwise
  504. * read-only Link Capabilities may change depending on common clock
  505. * configuration (PCIe r5.0, sec 7.5.3.6).
  506. */
  507. pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &parent_lnkcap);
  508. pcie_capability_read_dword(child, PCI_EXP_LNKCAP, &child_lnkcap);
  509. pcie_capability_read_word(parent, PCI_EXP_LNKCTL, &parent_lnkctl);
  510. pcie_capability_read_word(child, PCI_EXP_LNKCTL, &child_lnkctl);
  511. /*
  512. * Setup L0s state
  513. *
  514. * Note that we must not enable L0s in either direction on a
  515. * given link unless components on both sides of the link each
  516. * support L0s.
  517. */
  518. if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L0S)
  519. link->aspm_support |= ASPM_STATE_L0S;
  520. if (child_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S)
  521. link->aspm_enabled |= ASPM_STATE_L0S_UP;
  522. if (parent_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S)
  523. link->aspm_enabled |= ASPM_STATE_L0S_DW;
  524. link->latency_up.l0s = calc_l0s_latency(parent_lnkcap);
  525. link->latency_dw.l0s = calc_l0s_latency(child_lnkcap);
  526. /* Setup L1 state */
  527. if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L1)
  528. link->aspm_support |= ASPM_STATE_L1;
  529. if (parent_lnkctl & child_lnkctl & PCI_EXP_LNKCTL_ASPM_L1)
  530. link->aspm_enabled |= ASPM_STATE_L1;
  531. link->latency_up.l1 = calc_l1_latency(parent_lnkcap);
  532. link->latency_dw.l1 = calc_l1_latency(child_lnkcap);
  533. /* Setup L1 substate */
  534. pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP,
  535. &parent_l1ss_cap);
  536. pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP,
  537. &child_l1ss_cap);
  538. if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))
  539. parent_l1ss_cap = 0;
  540. if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))
  541. child_l1ss_cap = 0;
  542. /*
  543. * If we don't have LTR for the entire path from the Root Complex
  544. * to this device, we can't use ASPM L1.2 because it relies on the
  545. * LTR_L1.2_THRESHOLD. See PCIe r4.0, secs 5.5.4, 6.18.
  546. */
  547. if (!child->ltr_path)
  548. child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2;
  549. if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1)
  550. link->aspm_support |= ASPM_STATE_L1_1;
  551. if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2)
  552. link->aspm_support |= ASPM_STATE_L1_2;
  553. if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1)
  554. link->aspm_support |= ASPM_STATE_L1_1_PCIPM;
  555. if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2)
  556. link->aspm_support |= ASPM_STATE_L1_2_PCIPM;
  557. if (parent_l1ss_cap)
  558. pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
  559. &parent_l1ss_ctl1);
  560. if (child_l1ss_cap)
  561. pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1,
  562. &child_l1ss_ctl1);
  563. if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1)
  564. link->aspm_enabled |= ASPM_STATE_L1_1;
  565. if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2)
  566. link->aspm_enabled |= ASPM_STATE_L1_2;
  567. if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1)
  568. link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM;
  569. if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2)
  570. link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM;
  571. if (link->aspm_support & ASPM_STATE_L1SS)
  572. aspm_calc_l1ss_info(link, parent_l1ss_cap, child_l1ss_cap);
  573. /* Save default state */
  574. link->aspm_default = link->aspm_enabled;
  575. /* Setup initial capable state. Will be updated later */
  576. link->aspm_capable = link->aspm_support;
  577. /* Get and check endpoint acceptable latencies */
  578. list_for_each_entry(child, &linkbus->devices, bus_list) {
  579. u32 reg32, encoding;
  580. struct aspm_latency *acceptable =
  581. &link->acceptable[PCI_FUNC(child->devfn)];
  582. if (pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT &&
  583. pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END)
  584. continue;
  585. pcie_capability_read_dword(child, PCI_EXP_DEVCAP, &reg32);
  586. /* Calculate endpoint L0s acceptable latency */
  587. encoding = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
  588. acceptable->l0s = calc_l0s_acceptable(encoding);
  589. /* Calculate endpoint L1 acceptable latency */
  590. encoding = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
  591. acceptable->l1 = calc_l1_acceptable(encoding);
  592. pcie_aspm_check_latency(child);
  593. }
  594. }
  595. /* Configure the ASPM L1 substates */
  596. static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state)
  597. {
  598. u32 val, enable_req;
  599. struct pci_dev *child = link->downstream, *parent = link->pdev;
  600. enable_req = (link->aspm_enabled ^ state) & state;
  601. /*
  602. * Here are the rules specified in the PCIe spec for enabling L1SS:
  603. * - When enabling L1.x, enable bit at parent first, then at child
  604. * - When disabling L1.x, disable bit at child first, then at parent
  605. * - When enabling ASPM L1.x, need to disable L1
  606. * (at child followed by parent).
  607. * - The ASPM/PCIPM L1.2 must be disabled while programming timing
  608. * parameters
  609. *
  610. * To keep it simple, disable all L1SS bits first, and later enable
  611. * what is needed.
  612. */
  613. /* Disable all L1 substates */
  614. pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
  615. PCI_L1SS_CTL1_L1SS_MASK, 0);
  616. pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
  617. PCI_L1SS_CTL1_L1SS_MASK, 0);
  618. /*
  619. * If needed, disable L1, and it gets enabled later
  620. * in pcie_config_aspm_link().
  621. */
  622. if (enable_req & (ASPM_STATE_L1_1 | ASPM_STATE_L1_2)) {
  623. pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL,
  624. PCI_EXP_LNKCTL_ASPM_L1, 0);
  625. pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
  626. PCI_EXP_LNKCTL_ASPM_L1, 0);
  627. }
  628. val = 0;
  629. if (state & ASPM_STATE_L1_1)
  630. val |= PCI_L1SS_CTL1_ASPM_L1_1;
  631. if (state & ASPM_STATE_L1_2)
  632. val |= PCI_L1SS_CTL1_ASPM_L1_2;
  633. if (state & ASPM_STATE_L1_1_PCIPM)
  634. val |= PCI_L1SS_CTL1_PCIPM_L1_1;
  635. if (state & ASPM_STATE_L1_2_PCIPM)
  636. val |= PCI_L1SS_CTL1_PCIPM_L1_2;
  637. /* Enable what we need to enable */
  638. pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
  639. PCI_L1SS_CTL1_L1SS_MASK, val);
  640. pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
  641. PCI_L1SS_CTL1_L1SS_MASK, val);
  642. }
  643. static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val)
  644. {
  645. pcie_capability_clear_and_set_word(pdev, PCI_EXP_LNKCTL,
  646. PCI_EXP_LNKCTL_ASPMC, val);
  647. }
  648. static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)
  649. {
  650. u32 upstream = 0, dwstream = 0;
  651. struct pci_dev *child = link->downstream, *parent = link->pdev;
  652. struct pci_bus *linkbus = parent->subordinate;
  653. /* Enable only the states that were not explicitly disabled */
  654. state &= (link->aspm_capable & ~link->aspm_disable);
  655. /* Can't enable any substates if L1 is not enabled */
  656. if (!(state & ASPM_STATE_L1))
  657. state &= ~ASPM_STATE_L1SS;
  658. /* Spec says both ports must be in D0 before enabling PCI PM substates*/
  659. if (parent->current_state != PCI_D0 || child->current_state != PCI_D0) {
  660. state &= ~ASPM_STATE_L1_SS_PCIPM;
  661. state |= (link->aspm_enabled & ASPM_STATE_L1_SS_PCIPM);
  662. }
  663. /* Nothing to do if the link is already in the requested state */
  664. if (link->aspm_enabled == state)
  665. return;
  666. /* Convert ASPM state to upstream/downstream ASPM register state */
  667. if (state & ASPM_STATE_L0S_UP)
  668. dwstream |= PCI_EXP_LNKCTL_ASPM_L0S;
  669. if (state & ASPM_STATE_L0S_DW)
  670. upstream |= PCI_EXP_LNKCTL_ASPM_L0S;
  671. if (state & ASPM_STATE_L1) {
  672. upstream |= PCI_EXP_LNKCTL_ASPM_L1;
  673. dwstream |= PCI_EXP_LNKCTL_ASPM_L1;
  674. }
  675. if (link->aspm_capable & ASPM_STATE_L1SS)
  676. pcie_config_aspm_l1ss(link, state);
  677. /*
  678. * Spec 2.0 suggests all functions should be configured the
  679. * same setting for ASPM. Enabling ASPM L1 should be done in
  680. * upstream component first and then downstream, and vice
  681. * versa for disabling ASPM L1. Spec doesn't mention L0S.
  682. */
  683. if (state & ASPM_STATE_L1)
  684. pcie_config_aspm_dev(parent, upstream);
  685. list_for_each_entry(child, &linkbus->devices, bus_list)
  686. pcie_config_aspm_dev(child, dwstream);
  687. if (!(state & ASPM_STATE_L1))
  688. pcie_config_aspm_dev(parent, upstream);
  689. link->aspm_enabled = state;
  690. }
  691. static void pcie_config_aspm_path(struct pcie_link_state *link)
  692. {
  693. while (link) {
  694. pcie_config_aspm_link(link, policy_to_aspm_state(link));
  695. link = link->parent;
  696. }
  697. }
  698. static void free_link_state(struct pcie_link_state *link)
  699. {
  700. link->pdev->link_state = NULL;
  701. kfree(link);
  702. }
  703. static int pcie_aspm_sanity_check(struct pci_dev *pdev)
  704. {
  705. struct pci_dev *child;
  706. u32 reg32;
  707. /*
  708. * Some functions in a slot might not all be PCIe functions,
  709. * very strange. Disable ASPM for the whole slot
  710. */
  711. list_for_each_entry(child, &pdev->subordinate->devices, bus_list) {
  712. if (!pci_is_pcie(child))
  713. return -EINVAL;
  714. /*
  715. * If ASPM is disabled then we're not going to change
  716. * the BIOS state. It's safe to continue even if it's a
  717. * pre-1.1 device
  718. */
  719. if (aspm_disabled)
  720. continue;
  721. /*
  722. * Disable ASPM for pre-1.1 PCIe device, we follow MS to use
  723. * RBER bit to determine if a function is 1.1 version device
  724. */
  725. pcie_capability_read_dword(child, PCI_EXP_DEVCAP, &reg32);
  726. if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
  727. pci_info(child, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\n");
  728. return -EINVAL;
  729. }
  730. }
  731. return 0;
  732. }
  733. static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev)
  734. {
  735. struct pcie_link_state *link;
  736. link = kzalloc(sizeof(*link), GFP_KERNEL);
  737. if (!link)
  738. return NULL;
  739. INIT_LIST_HEAD(&link->sibling);
  740. link->pdev = pdev;
  741. link->downstream = pci_function_0(pdev->subordinate);
  742. /*
  743. * Root Ports and PCI/PCI-X to PCIe Bridges are roots of PCIe
  744. * hierarchies. Note that some PCIe host implementations omit
  745. * the root ports entirely, in which case a downstream port on
  746. * a switch may become the root of the link state chain for all
  747. * its subordinate endpoints.
  748. */
  749. if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT ||
  750. pci_pcie_type(pdev) == PCI_EXP_TYPE_PCIE_BRIDGE ||
  751. !pdev->bus->parent->self) {
  752. link->root = link;
  753. } else {
  754. struct pcie_link_state *parent;
  755. parent = pdev->bus->parent->self->link_state;
  756. if (!parent) {
  757. kfree(link);
  758. return NULL;
  759. }
  760. link->parent = parent;
  761. link->root = link->parent->root;
  762. }
  763. list_add(&link->sibling, &link_list);
  764. pdev->link_state = link;
  765. return link;
  766. }
  767. static void pcie_aspm_update_sysfs_visibility(struct pci_dev *pdev)
  768. {
  769. struct pci_dev *child;
  770. list_for_each_entry(child, &pdev->subordinate->devices, bus_list)
  771. sysfs_update_group(&child->dev.kobj, &aspm_ctrl_attr_group);
  772. }
  773. /*
  774. * pcie_aspm_init_link_state: Initiate PCI express link state.
  775. * It is called after the pcie and its children devices are scanned.
  776. * @pdev: the root port or switch downstream port
  777. */
  778. void pcie_aspm_init_link_state(struct pci_dev *pdev)
  779. {
  780. struct pcie_link_state *link;
  781. int blacklist = !!pcie_aspm_sanity_check(pdev);
  782. if (!aspm_support_enabled)
  783. return;
  784. if (pdev->link_state)
  785. return;
  786. /*
  787. * We allocate pcie_link_state for the component on the upstream
  788. * end of a Link, so there's nothing to do unless this device is
  789. * downstream port.
  790. */
  791. if (!pcie_downstream_port(pdev))
  792. return;
  793. /* VIA has a strange chipset, root port is under a bridge */
  794. if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT &&
  795. pdev->bus->self)
  796. return;
  797. down_read(&pci_bus_sem);
  798. if (list_empty(&pdev->subordinate->devices))
  799. goto out;
  800. mutex_lock(&aspm_lock);
  801. link = alloc_pcie_link_state(pdev);
  802. if (!link)
  803. goto unlock;
  804. /*
  805. * Setup initial ASPM state. Note that we need to configure
  806. * upstream links also because capable state of them can be
  807. * update through pcie_aspm_cap_init().
  808. */
  809. pcie_aspm_cap_init(link, blacklist);
  810. /* Setup initial Clock PM state */
  811. pcie_clkpm_cap_init(link, blacklist);
  812. /*
  813. * At this stage drivers haven't had an opportunity to change the
  814. * link policy setting. Enabling ASPM on broken hardware can cripple
  815. * it even before the driver has had a chance to disable ASPM, so
  816. * default to a safe level right now. If we're enabling ASPM beyond
  817. * the BIOS's expectation, we'll do so once pci_enable_device() is
  818. * called.
  819. */
  820. if (aspm_policy != POLICY_POWERSAVE &&
  821. aspm_policy != POLICY_POWER_SUPERSAVE) {
  822. pcie_config_aspm_path(link);
  823. pcie_set_clkpm(link, policy_to_clkpm_state(link));
  824. }
  825. pcie_aspm_update_sysfs_visibility(pdev);
  826. unlock:
  827. mutex_unlock(&aspm_lock);
  828. out:
  829. up_read(&pci_bus_sem);
  830. }
  831. /* Recheck latencies and update aspm_capable for links under the root */
  832. static void pcie_update_aspm_capable(struct pcie_link_state *root)
  833. {
  834. struct pcie_link_state *link;
  835. BUG_ON(root->parent);
  836. list_for_each_entry(link, &link_list, sibling) {
  837. if (link->root != root)
  838. continue;
  839. link->aspm_capable = link->aspm_support;
  840. }
  841. list_for_each_entry(link, &link_list, sibling) {
  842. struct pci_dev *child;
  843. struct pci_bus *linkbus = link->pdev->subordinate;
  844. if (link->root != root)
  845. continue;
  846. list_for_each_entry(child, &linkbus->devices, bus_list) {
  847. if ((pci_pcie_type(child) != PCI_EXP_TYPE_ENDPOINT) &&
  848. (pci_pcie_type(child) != PCI_EXP_TYPE_LEG_END))
  849. continue;
  850. pcie_aspm_check_latency(child);
  851. }
  852. }
  853. }
  854. /* @pdev: the endpoint device */
  855. void pcie_aspm_exit_link_state(struct pci_dev *pdev)
  856. {
  857. struct pci_dev *parent = pdev->bus->self;
  858. struct pcie_link_state *link, *root, *parent_link;
  859. if (!parent || !parent->link_state)
  860. return;
  861. down_read(&pci_bus_sem);
  862. mutex_lock(&aspm_lock);
  863. /*
  864. * All PCIe functions are in one slot, remove one function will remove
  865. * the whole slot, so just wait until we are the last function left.
  866. */
  867. if (!list_empty(&parent->subordinate->devices))
  868. goto out;
  869. link = parent->link_state;
  870. root = link->root;
  871. parent_link = link->parent;
  872. /* All functions are removed, so just disable ASPM for the link */
  873. pcie_config_aspm_link(link, 0);
  874. list_del(&link->sibling);
  875. /* Clock PM is for endpoint device */
  876. free_link_state(link);
  877. /* Recheck latencies and configure upstream links */
  878. if (parent_link) {
  879. pcie_update_aspm_capable(root);
  880. pcie_config_aspm_path(parent_link);
  881. }
  882. out:
  883. mutex_unlock(&aspm_lock);
  884. up_read(&pci_bus_sem);
  885. }
  886. /* @pdev: the root port or switch downstream port */
  887. void pcie_aspm_pm_state_change(struct pci_dev *pdev)
  888. {
  889. struct pcie_link_state *link = pdev->link_state;
  890. if (aspm_disabled || !link)
  891. return;
  892. /*
  893. * Devices changed PM state, we should recheck if latency
  894. * meets all functions' requirement
  895. */
  896. down_read(&pci_bus_sem);
  897. mutex_lock(&aspm_lock);
  898. pcie_update_aspm_capable(link->root);
  899. pcie_config_aspm_path(link);
  900. mutex_unlock(&aspm_lock);
  901. up_read(&pci_bus_sem);
  902. }
  903. void pcie_aspm_powersave_config_link(struct pci_dev *pdev)
  904. {
  905. struct pcie_link_state *link = pdev->link_state;
  906. if (aspm_disabled || !link)
  907. return;
  908. if (aspm_policy != POLICY_POWERSAVE &&
  909. aspm_policy != POLICY_POWER_SUPERSAVE)
  910. return;
  911. down_read(&pci_bus_sem);
  912. mutex_lock(&aspm_lock);
  913. pcie_config_aspm_path(link);
  914. pcie_set_clkpm(link, policy_to_clkpm_state(link));
  915. mutex_unlock(&aspm_lock);
  916. up_read(&pci_bus_sem);
  917. }
  918. static struct pcie_link_state *pcie_aspm_get_link(struct pci_dev *pdev)
  919. {
  920. struct pci_dev *bridge;
  921. if (!pci_is_pcie(pdev))
  922. return NULL;
  923. bridge = pci_upstream_bridge(pdev);
  924. if (!bridge || !pci_is_pcie(bridge))
  925. return NULL;
  926. return bridge->link_state;
  927. }
  928. static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem)
  929. {
  930. struct pcie_link_state *link = pcie_aspm_get_link(pdev);
  931. if (!link)
  932. return -EINVAL;
  933. /*
  934. * A driver requested that ASPM be disabled on this device, but
  935. * if we don't have permission to manage ASPM (e.g., on ACPI
  936. * systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and
  937. * the _OSC method), we can't honor that request. Windows has
  938. * a similar mechanism using "PciASPMOptOut", which is also
  939. * ignored in this situation.
  940. */
  941. if (aspm_disabled) {
  942. pci_warn(pdev, "can't disable ASPM; OS doesn't have ASPM control\n");
  943. return -EPERM;
  944. }
  945. if (sem)
  946. down_read(&pci_bus_sem);
  947. mutex_lock(&aspm_lock);
  948. if (state & PCIE_LINK_STATE_L0S)
  949. link->aspm_disable |= ASPM_STATE_L0S;
  950. if (state & PCIE_LINK_STATE_L1)
  951. /* L1 PM substates require L1 */
  952. link->aspm_disable |= ASPM_STATE_L1 | ASPM_STATE_L1SS;
  953. if (state & PCIE_LINK_STATE_L1_1)
  954. link->aspm_disable |= ASPM_STATE_L1_1;
  955. if (state & PCIE_LINK_STATE_L1_2)
  956. link->aspm_disable |= ASPM_STATE_L1_2;
  957. if (state & PCIE_LINK_STATE_L1_1_PCIPM)
  958. link->aspm_disable |= ASPM_STATE_L1_1_PCIPM;
  959. if (state & PCIE_LINK_STATE_L1_2_PCIPM)
  960. link->aspm_disable |= ASPM_STATE_L1_2_PCIPM;
  961. pcie_config_aspm_link(link, policy_to_aspm_state(link));
  962. if (state & PCIE_LINK_STATE_CLKPM)
  963. link->clkpm_disable = 1;
  964. pcie_set_clkpm(link, policy_to_clkpm_state(link));
  965. mutex_unlock(&aspm_lock);
  966. if (sem)
  967. up_read(&pci_bus_sem);
  968. return 0;
  969. }
  970. int pci_disable_link_state_locked(struct pci_dev *pdev, int state)
  971. {
  972. return __pci_disable_link_state(pdev, state, false);
  973. }
  974. EXPORT_SYMBOL(pci_disable_link_state_locked);
  975. /**
  976. * pci_disable_link_state - Disable device's link state, so the link will
  977. * never enter specific states. Note that if the BIOS didn't grant ASPM
  978. * control to the OS, this does nothing because we can't touch the LNKCTL
  979. * register. Returns 0 or a negative errno.
  980. *
  981. * @pdev: PCI device
  982. * @state: ASPM link state to disable
  983. */
  984. int pci_disable_link_state(struct pci_dev *pdev, int state)
  985. {
  986. return __pci_disable_link_state(pdev, state, true);
  987. }
  988. EXPORT_SYMBOL(pci_disable_link_state);
  989. static int pcie_aspm_set_policy(const char *val,
  990. const struct kernel_param *kp)
  991. {
  992. int i;
  993. struct pcie_link_state *link;
  994. if (aspm_disabled)
  995. return -EPERM;
  996. i = sysfs_match_string(policy_str, val);
  997. if (i < 0)
  998. return i;
  999. if (i == aspm_policy)
  1000. return 0;
  1001. down_read(&pci_bus_sem);
  1002. mutex_lock(&aspm_lock);
  1003. aspm_policy = i;
  1004. list_for_each_entry(link, &link_list, sibling) {
  1005. pcie_config_aspm_link(link, policy_to_aspm_state(link));
  1006. pcie_set_clkpm(link, policy_to_clkpm_state(link));
  1007. }
  1008. mutex_unlock(&aspm_lock);
  1009. up_read(&pci_bus_sem);
  1010. return 0;
  1011. }
  1012. static int pcie_aspm_get_policy(char *buffer, const struct kernel_param *kp)
  1013. {
  1014. int i, cnt = 0;
  1015. for (i = 0; i < ARRAY_SIZE(policy_str); i++)
  1016. if (i == aspm_policy)
  1017. cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
  1018. else
  1019. cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
  1020. cnt += sprintf(buffer + cnt, "\n");
  1021. return cnt;
  1022. }
  1023. module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
  1024. NULL, 0644);
  1025. /**
  1026. * pcie_aspm_enabled - Check if PCIe ASPM has been enabled for a device.
  1027. * @pdev: Target device.
  1028. *
  1029. * Relies on the upstream bridge's link_state being valid. The link_state
  1030. * is deallocated only when the last child of the bridge (i.e., @pdev or a
  1031. * sibling) is removed, and the caller should be holding a reference to
  1032. * @pdev, so this should be safe.
  1033. */
  1034. bool pcie_aspm_enabled(struct pci_dev *pdev)
  1035. {
  1036. struct pcie_link_state *link = pcie_aspm_get_link(pdev);
  1037. if (!link)
  1038. return false;
  1039. return link->aspm_enabled;
  1040. }
  1041. EXPORT_SYMBOL_GPL(pcie_aspm_enabled);
  1042. static ssize_t aspm_attr_show_common(struct device *dev,
  1043. struct device_attribute *attr,
  1044. char *buf, u8 state)
  1045. {
  1046. struct pci_dev *pdev = to_pci_dev(dev);
  1047. struct pcie_link_state *link = pcie_aspm_get_link(pdev);
  1048. return sprintf(buf, "%d\n", (link->aspm_enabled & state) ? 1 : 0);
  1049. }
  1050. static ssize_t aspm_attr_store_common(struct device *dev,
  1051. struct device_attribute *attr,
  1052. const char *buf, size_t len, u8 state)
  1053. {
  1054. struct pci_dev *pdev = to_pci_dev(dev);
  1055. struct pcie_link_state *link = pcie_aspm_get_link(pdev);
  1056. bool state_enable;
  1057. if (strtobool(buf, &state_enable) < 0)
  1058. return -EINVAL;
  1059. down_read(&pci_bus_sem);
  1060. mutex_lock(&aspm_lock);
  1061. if (state_enable) {
  1062. link->aspm_disable &= ~state;
  1063. /* need to enable L1 for substates */
  1064. if (state & ASPM_STATE_L1SS)
  1065. link->aspm_disable &= ~ASPM_STATE_L1;
  1066. } else {
  1067. link->aspm_disable |= state;
  1068. }
  1069. pcie_config_aspm_link(link, policy_to_aspm_state(link));
  1070. mutex_unlock(&aspm_lock);
  1071. up_read(&pci_bus_sem);
  1072. return len;
  1073. }
  1074. #define ASPM_ATTR(_f, _s) \
  1075. static ssize_t _f##_show(struct device *dev, \
  1076. struct device_attribute *attr, char *buf) \
  1077. { return aspm_attr_show_common(dev, attr, buf, ASPM_STATE_##_s); } \
  1078. \
  1079. static ssize_t _f##_store(struct device *dev, \
  1080. struct device_attribute *attr, \
  1081. const char *buf, size_t len) \
  1082. { return aspm_attr_store_common(dev, attr, buf, len, ASPM_STATE_##_s); }
  1083. ASPM_ATTR(l0s_aspm, L0S)
  1084. ASPM_ATTR(l1_aspm, L1)
  1085. ASPM_ATTR(l1_1_aspm, L1_1)
  1086. ASPM_ATTR(l1_2_aspm, L1_2)
  1087. ASPM_ATTR(l1_1_pcipm, L1_1_PCIPM)
  1088. ASPM_ATTR(l1_2_pcipm, L1_2_PCIPM)
  1089. static ssize_t clkpm_show(struct device *dev,
  1090. struct device_attribute *attr, char *buf)
  1091. {
  1092. struct pci_dev *pdev = to_pci_dev(dev);
  1093. struct pcie_link_state *link = pcie_aspm_get_link(pdev);
  1094. return sprintf(buf, "%d\n", link->clkpm_enabled);
  1095. }
  1096. static ssize_t clkpm_store(struct device *dev,
  1097. struct device_attribute *attr,
  1098. const char *buf, size_t len)
  1099. {
  1100. struct pci_dev *pdev = to_pci_dev(dev);
  1101. struct pcie_link_state *link = pcie_aspm_get_link(pdev);
  1102. bool state_enable;
  1103. if (strtobool(buf, &state_enable) < 0)
  1104. return -EINVAL;
  1105. down_read(&pci_bus_sem);
  1106. mutex_lock(&aspm_lock);
  1107. link->clkpm_disable = !state_enable;
  1108. pcie_set_clkpm(link, policy_to_clkpm_state(link));
  1109. mutex_unlock(&aspm_lock);
  1110. up_read(&pci_bus_sem);
  1111. return len;
  1112. }
  1113. static DEVICE_ATTR_RW(clkpm);
  1114. static DEVICE_ATTR_RW(l0s_aspm);
  1115. static DEVICE_ATTR_RW(l1_aspm);
  1116. static DEVICE_ATTR_RW(l1_1_aspm);
  1117. static DEVICE_ATTR_RW(l1_2_aspm);
  1118. static DEVICE_ATTR_RW(l1_1_pcipm);
  1119. static DEVICE_ATTR_RW(l1_2_pcipm);
  1120. static struct attribute *aspm_ctrl_attrs[] = {
  1121. &dev_attr_clkpm.attr,
  1122. &dev_attr_l0s_aspm.attr,
  1123. &dev_attr_l1_aspm.attr,
  1124. &dev_attr_l1_1_aspm.attr,
  1125. &dev_attr_l1_2_aspm.attr,
  1126. &dev_attr_l1_1_pcipm.attr,
  1127. &dev_attr_l1_2_pcipm.attr,
  1128. NULL
  1129. };
  1130. static umode_t aspm_ctrl_attrs_are_visible(struct kobject *kobj,
  1131. struct attribute *a, int n)
  1132. {
  1133. struct device *dev = kobj_to_dev(kobj);
  1134. struct pci_dev *pdev = to_pci_dev(dev);
  1135. struct pcie_link_state *link = pcie_aspm_get_link(pdev);
  1136. static const u8 aspm_state_map[] = {
  1137. ASPM_STATE_L0S,
  1138. ASPM_STATE_L1,
  1139. ASPM_STATE_L1_1,
  1140. ASPM_STATE_L1_2,
  1141. ASPM_STATE_L1_1_PCIPM,
  1142. ASPM_STATE_L1_2_PCIPM,
  1143. };
  1144. if (aspm_disabled || !link)
  1145. return 0;
  1146. if (n == 0)
  1147. return link->clkpm_capable ? a->mode : 0;
  1148. return link->aspm_capable & aspm_state_map[n - 1] ? a->mode : 0;
  1149. }
  1150. const struct attribute_group aspm_ctrl_attr_group = {
  1151. .name = "link",
  1152. .attrs = aspm_ctrl_attrs,
  1153. .is_visible = aspm_ctrl_attrs_are_visible,
  1154. };
  1155. static int __init pcie_aspm_disable(char *str)
  1156. {
  1157. if (!strcmp(str, "off")) {
  1158. aspm_policy = POLICY_DEFAULT;
  1159. aspm_disabled = 1;
  1160. aspm_support_enabled = false;
  1161. printk(KERN_INFO "PCIe ASPM is disabled\n");
  1162. } else if (!strcmp(str, "force")) {
  1163. aspm_force = 1;
  1164. printk(KERN_INFO "PCIe ASPM is forcibly enabled\n");
  1165. }
  1166. return 1;
  1167. }
  1168. __setup("pcie_aspm=", pcie_aspm_disable);
  1169. void pcie_no_aspm(void)
  1170. {
  1171. /*
  1172. * Disabling ASPM is intended to prevent the kernel from modifying
  1173. * existing hardware state, not to clear existing state. To that end:
  1174. * (a) set policy to POLICY_DEFAULT in order to avoid changing state
  1175. * (b) prevent userspace from changing policy
  1176. */
  1177. if (!aspm_force) {
  1178. aspm_policy = POLICY_DEFAULT;
  1179. aspm_disabled = 1;
  1180. }
  1181. }
  1182. bool pcie_aspm_support_enabled(void)
  1183. {
  1184. return aspm_support_enabled;
  1185. }
  1186. EXPORT_SYMBOL(pcie_aspm_support_enabled);