intel_rapl_common.c 39 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Common code for Intel Running Average Power Limit (RAPL) support.
  4. * Copyright (c) 2019, Intel Corporation.
  5. */
  6. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  7. #include <linux/kernel.h>
  8. #include <linux/module.h>
  9. #include <linux/list.h>
  10. #include <linux/types.h>
  11. #include <linux/device.h>
  12. #include <linux/slab.h>
  13. #include <linux/log2.h>
  14. #include <linux/bitmap.h>
  15. #include <linux/delay.h>
  16. #include <linux/sysfs.h>
  17. #include <linux/cpu.h>
  18. #include <linux/powercap.h>
  19. #include <linux/suspend.h>
  20. #include <linux/intel_rapl.h>
  21. #include <linux/processor.h>
  22. #include <linux/platform_device.h>
  23. #include <asm/iosf_mbi.h>
  24. #include <asm/cpu_device_id.h>
  25. #include <asm/intel-family.h>
  26. /* bitmasks for RAPL MSRs, used by primitive access functions */
  27. #define ENERGY_STATUS_MASK 0xffffffff
  28. #define POWER_LIMIT1_MASK 0x7FFF
  29. #define POWER_LIMIT1_ENABLE BIT(15)
  30. #define POWER_LIMIT1_CLAMP BIT(16)
  31. #define POWER_LIMIT2_MASK (0x7FFFULL<<32)
  32. #define POWER_LIMIT2_ENABLE BIT_ULL(47)
  33. #define POWER_LIMIT2_CLAMP BIT_ULL(48)
  34. #define POWER_HIGH_LOCK BIT_ULL(63)
  35. #define POWER_LOW_LOCK BIT(31)
  36. #define POWER_LIMIT4_MASK 0x1FFF
  37. #define TIME_WINDOW1_MASK (0x7FULL<<17)
  38. #define TIME_WINDOW2_MASK (0x7FULL<<49)
  39. #define POWER_UNIT_OFFSET 0
  40. #define POWER_UNIT_MASK 0x0F
  41. #define ENERGY_UNIT_OFFSET 0x08
  42. #define ENERGY_UNIT_MASK 0x1F00
  43. #define TIME_UNIT_OFFSET 0x10
  44. #define TIME_UNIT_MASK 0xF0000
  45. #define POWER_INFO_MAX_MASK (0x7fffULL<<32)
  46. #define POWER_INFO_MIN_MASK (0x7fffULL<<16)
  47. #define POWER_INFO_MAX_TIME_WIN_MASK (0x3fULL<<48)
  48. #define POWER_INFO_THERMAL_SPEC_MASK 0x7fff
  49. #define PERF_STATUS_THROTTLE_TIME_MASK 0xffffffff
  50. #define PP_POLICY_MASK 0x1F
  51. /* Non HW constants */
  52. #define RAPL_PRIMITIVE_DERIVED BIT(1) /* not from raw data */
  53. #define RAPL_PRIMITIVE_DUMMY BIT(2)
  54. #define TIME_WINDOW_MAX_MSEC 40000
  55. #define TIME_WINDOW_MIN_MSEC 250
  56. #define ENERGY_UNIT_SCALE 1000 /* scale from driver unit to powercap unit */
  57. enum unit_type {
  58. ARBITRARY_UNIT, /* no translation */
  59. POWER_UNIT,
  60. ENERGY_UNIT,
  61. TIME_UNIT,
  62. };
  63. /* per domain data, some are optional */
  64. #define NR_RAW_PRIMITIVES (NR_RAPL_PRIMITIVES - 2)
  65. #define DOMAIN_STATE_INACTIVE BIT(0)
  66. #define DOMAIN_STATE_POWER_LIMIT_SET BIT(1)
  67. #define DOMAIN_STATE_BIOS_LOCKED BIT(2)
  68. static const char pl1_name[] = "long_term";
  69. static const char pl2_name[] = "short_term";
  70. static const char pl4_name[] = "peak_power";
  71. #define power_zone_to_rapl_domain(_zone) \
  72. container_of(_zone, struct rapl_domain, power_zone)
  73. struct rapl_defaults {
  74. u8 floor_freq_reg_addr;
  75. int (*check_unit)(struct rapl_package *rp, int cpu);
  76. void (*set_floor_freq)(struct rapl_domain *rd, bool mode);
  77. u64 (*compute_time_window)(struct rapl_package *rp, u64 val,
  78. bool to_raw);
  79. unsigned int dram_domain_energy_unit;
  80. unsigned int psys_domain_energy_unit;
  81. };
  82. static struct rapl_defaults *rapl_defaults;
  83. /* Sideband MBI registers */
  84. #define IOSF_CPU_POWER_BUDGET_CTL_BYT (0x2)
  85. #define IOSF_CPU_POWER_BUDGET_CTL_TNG (0xdf)
  86. #define PACKAGE_PLN_INT_SAVED BIT(0)
  87. #define MAX_PRIM_NAME (32)
  88. /* per domain data. used to describe individual knobs such that access function
  89. * can be consolidated into one instead of many inline functions.
  90. */
  91. struct rapl_primitive_info {
  92. const char *name;
  93. u64 mask;
  94. int shift;
  95. enum rapl_domain_reg_id id;
  96. enum unit_type unit;
  97. u32 flag;
  98. };
  99. #define PRIMITIVE_INFO_INIT(p, m, s, i, u, f) { \
  100. .name = #p, \
  101. .mask = m, \
  102. .shift = s, \
  103. .id = i, \
  104. .unit = u, \
  105. .flag = f \
  106. }
  107. static void rapl_init_domains(struct rapl_package *rp);
  108. static int rapl_read_data_raw(struct rapl_domain *rd,
  109. enum rapl_primitives prim,
  110. bool xlate, u64 *data);
  111. static int rapl_write_data_raw(struct rapl_domain *rd,
  112. enum rapl_primitives prim,
  113. unsigned long long value);
  114. static u64 rapl_unit_xlate(struct rapl_domain *rd,
  115. enum unit_type type, u64 value, int to_raw);
  116. static void package_power_limit_irq_save(struct rapl_package *rp);
  117. static LIST_HEAD(rapl_packages); /* guarded by CPU hotplug lock */
  118. static const char *const rapl_domain_names[] = {
  119. "package",
  120. "core",
  121. "uncore",
  122. "dram",
  123. "psys",
  124. };
  125. static int get_energy_counter(struct powercap_zone *power_zone,
  126. u64 *energy_raw)
  127. {
  128. struct rapl_domain *rd;
  129. u64 energy_now;
  130. /* prevent CPU hotplug, make sure the RAPL domain does not go
  131. * away while reading the counter.
  132. */
  133. get_online_cpus();
  134. rd = power_zone_to_rapl_domain(power_zone);
  135. if (!rapl_read_data_raw(rd, ENERGY_COUNTER, true, &energy_now)) {
  136. *energy_raw = energy_now;
  137. put_online_cpus();
  138. return 0;
  139. }
  140. put_online_cpus();
  141. return -EIO;
  142. }
  143. static int get_max_energy_counter(struct powercap_zone *pcd_dev, u64 *energy)
  144. {
  145. struct rapl_domain *rd = power_zone_to_rapl_domain(pcd_dev);
  146. *energy = rapl_unit_xlate(rd, ENERGY_UNIT, ENERGY_STATUS_MASK, 0);
  147. return 0;
  148. }
  149. static int release_zone(struct powercap_zone *power_zone)
  150. {
  151. struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);
  152. struct rapl_package *rp = rd->rp;
  153. /* package zone is the last zone of a package, we can free
  154. * memory here since all children has been unregistered.
  155. */
  156. if (rd->id == RAPL_DOMAIN_PACKAGE) {
  157. kfree(rd);
  158. rp->domains = NULL;
  159. }
  160. return 0;
  161. }
  162. static int find_nr_power_limit(struct rapl_domain *rd)
  163. {
  164. int i, nr_pl = 0;
  165. for (i = 0; i < NR_POWER_LIMITS; i++) {
  166. if (rd->rpl[i].name)
  167. nr_pl++;
  168. }
  169. return nr_pl;
  170. }
  171. static int set_domain_enable(struct powercap_zone *power_zone, bool mode)
  172. {
  173. struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);
  174. if (rd->state & DOMAIN_STATE_BIOS_LOCKED)
  175. return -EACCES;
  176. get_online_cpus();
  177. rapl_write_data_raw(rd, PL1_ENABLE, mode);
  178. if (rapl_defaults->set_floor_freq)
  179. rapl_defaults->set_floor_freq(rd, mode);
  180. put_online_cpus();
  181. return 0;
  182. }
  183. static int get_domain_enable(struct powercap_zone *power_zone, bool *mode)
  184. {
  185. struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);
  186. u64 val;
  187. if (rd->state & DOMAIN_STATE_BIOS_LOCKED) {
  188. *mode = false;
  189. return 0;
  190. }
  191. get_online_cpus();
  192. if (rapl_read_data_raw(rd, PL1_ENABLE, true, &val)) {
  193. put_online_cpus();
  194. return -EIO;
  195. }
  196. *mode = val;
  197. put_online_cpus();
  198. return 0;
  199. }
  200. /* per RAPL domain ops, in the order of rapl_domain_type */
  201. static const struct powercap_zone_ops zone_ops[] = {
  202. /* RAPL_DOMAIN_PACKAGE */
  203. {
  204. .get_energy_uj = get_energy_counter,
  205. .get_max_energy_range_uj = get_max_energy_counter,
  206. .release = release_zone,
  207. .set_enable = set_domain_enable,
  208. .get_enable = get_domain_enable,
  209. },
  210. /* RAPL_DOMAIN_PP0 */
  211. {
  212. .get_energy_uj = get_energy_counter,
  213. .get_max_energy_range_uj = get_max_energy_counter,
  214. .release = release_zone,
  215. .set_enable = set_domain_enable,
  216. .get_enable = get_domain_enable,
  217. },
  218. /* RAPL_DOMAIN_PP1 */
  219. {
  220. .get_energy_uj = get_energy_counter,
  221. .get_max_energy_range_uj = get_max_energy_counter,
  222. .release = release_zone,
  223. .set_enable = set_domain_enable,
  224. .get_enable = get_domain_enable,
  225. },
  226. /* RAPL_DOMAIN_DRAM */
  227. {
  228. .get_energy_uj = get_energy_counter,
  229. .get_max_energy_range_uj = get_max_energy_counter,
  230. .release = release_zone,
  231. .set_enable = set_domain_enable,
  232. .get_enable = get_domain_enable,
  233. },
  234. /* RAPL_DOMAIN_PLATFORM */
  235. {
  236. .get_energy_uj = get_energy_counter,
  237. .get_max_energy_range_uj = get_max_energy_counter,
  238. .release = release_zone,
  239. .set_enable = set_domain_enable,
  240. .get_enable = get_domain_enable,
  241. },
  242. };
  243. /*
  244. * Constraint index used by powercap can be different than power limit (PL)
  245. * index in that some PLs maybe missing due to non-existent MSRs. So we
  246. * need to convert here by finding the valid PLs only (name populated).
  247. */
  248. static int contraint_to_pl(struct rapl_domain *rd, int cid)
  249. {
  250. int i, j;
  251. for (i = 0, j = 0; i < NR_POWER_LIMITS; i++) {
  252. if ((rd->rpl[i].name) && j++ == cid) {
  253. pr_debug("%s: index %d\n", __func__, i);
  254. return i;
  255. }
  256. }
  257. pr_err("Cannot find matching power limit for constraint %d\n", cid);
  258. return -EINVAL;
  259. }
  260. static int set_power_limit(struct powercap_zone *power_zone, int cid,
  261. u64 power_limit)
  262. {
  263. struct rapl_domain *rd;
  264. struct rapl_package *rp;
  265. int ret = 0;
  266. int id;
  267. get_online_cpus();
  268. rd = power_zone_to_rapl_domain(power_zone);
  269. id = contraint_to_pl(rd, cid);
  270. if (id < 0) {
  271. ret = id;
  272. goto set_exit;
  273. }
  274. rp = rd->rp;
  275. if (rd->state & DOMAIN_STATE_BIOS_LOCKED) {
  276. dev_warn(&power_zone->dev,
  277. "%s locked by BIOS, monitoring only\n", rd->name);
  278. ret = -EACCES;
  279. goto set_exit;
  280. }
  281. switch (rd->rpl[id].prim_id) {
  282. case PL1_ENABLE:
  283. rapl_write_data_raw(rd, POWER_LIMIT1, power_limit);
  284. break;
  285. case PL2_ENABLE:
  286. rapl_write_data_raw(rd, POWER_LIMIT2, power_limit);
  287. break;
  288. case PL4_ENABLE:
  289. rapl_write_data_raw(rd, POWER_LIMIT4, power_limit);
  290. break;
  291. default:
  292. ret = -EINVAL;
  293. }
  294. if (!ret)
  295. package_power_limit_irq_save(rp);
  296. set_exit:
  297. put_online_cpus();
  298. return ret;
  299. }
  300. static int get_current_power_limit(struct powercap_zone *power_zone, int cid,
  301. u64 *data)
  302. {
  303. struct rapl_domain *rd;
  304. u64 val;
  305. int prim;
  306. int ret = 0;
  307. int id;
  308. get_online_cpus();
  309. rd = power_zone_to_rapl_domain(power_zone);
  310. id = contraint_to_pl(rd, cid);
  311. if (id < 0) {
  312. ret = id;
  313. goto get_exit;
  314. }
  315. switch (rd->rpl[id].prim_id) {
  316. case PL1_ENABLE:
  317. prim = POWER_LIMIT1;
  318. break;
  319. case PL2_ENABLE:
  320. prim = POWER_LIMIT2;
  321. break;
  322. case PL4_ENABLE:
  323. prim = POWER_LIMIT4;
  324. break;
  325. default:
  326. put_online_cpus();
  327. return -EINVAL;
  328. }
  329. if (rapl_read_data_raw(rd, prim, true, &val))
  330. ret = -EIO;
  331. else
  332. *data = val;
  333. get_exit:
  334. put_online_cpus();
  335. return ret;
  336. }
  337. static int set_time_window(struct powercap_zone *power_zone, int cid,
  338. u64 window)
  339. {
  340. struct rapl_domain *rd;
  341. int ret = 0;
  342. int id;
  343. get_online_cpus();
  344. rd = power_zone_to_rapl_domain(power_zone);
  345. id = contraint_to_pl(rd, cid);
  346. if (id < 0) {
  347. ret = id;
  348. goto set_time_exit;
  349. }
  350. switch (rd->rpl[id].prim_id) {
  351. case PL1_ENABLE:
  352. rapl_write_data_raw(rd, TIME_WINDOW1, window);
  353. break;
  354. case PL2_ENABLE:
  355. rapl_write_data_raw(rd, TIME_WINDOW2, window);
  356. break;
  357. default:
  358. ret = -EINVAL;
  359. }
  360. set_time_exit:
  361. put_online_cpus();
  362. return ret;
  363. }
  364. static int get_time_window(struct powercap_zone *power_zone, int cid,
  365. u64 *data)
  366. {
  367. struct rapl_domain *rd;
  368. u64 val;
  369. int ret = 0;
  370. int id;
  371. get_online_cpus();
  372. rd = power_zone_to_rapl_domain(power_zone);
  373. id = contraint_to_pl(rd, cid);
  374. if (id < 0) {
  375. ret = id;
  376. goto get_time_exit;
  377. }
  378. switch (rd->rpl[id].prim_id) {
  379. case PL1_ENABLE:
  380. ret = rapl_read_data_raw(rd, TIME_WINDOW1, true, &val);
  381. break;
  382. case PL2_ENABLE:
  383. ret = rapl_read_data_raw(rd, TIME_WINDOW2, true, &val);
  384. break;
  385. case PL4_ENABLE:
  386. /*
  387. * Time window parameter is not applicable for PL4 entry
  388. * so assigining '0' as default value.
  389. */
  390. val = 0;
  391. break;
  392. default:
  393. put_online_cpus();
  394. return -EINVAL;
  395. }
  396. if (!ret)
  397. *data = val;
  398. get_time_exit:
  399. put_online_cpus();
  400. return ret;
  401. }
  402. static const char *get_constraint_name(struct powercap_zone *power_zone,
  403. int cid)
  404. {
  405. struct rapl_domain *rd;
  406. int id;
  407. rd = power_zone_to_rapl_domain(power_zone);
  408. id = contraint_to_pl(rd, cid);
  409. if (id >= 0)
  410. return rd->rpl[id].name;
  411. return NULL;
  412. }
  413. static int get_max_power(struct powercap_zone *power_zone, int id, u64 *data)
  414. {
  415. struct rapl_domain *rd;
  416. u64 val;
  417. int prim;
  418. int ret = 0;
  419. get_online_cpus();
  420. rd = power_zone_to_rapl_domain(power_zone);
  421. switch (rd->rpl[id].prim_id) {
  422. case PL1_ENABLE:
  423. prim = THERMAL_SPEC_POWER;
  424. break;
  425. case PL2_ENABLE:
  426. prim = MAX_POWER;
  427. break;
  428. case PL4_ENABLE:
  429. prim = MAX_POWER;
  430. break;
  431. default:
  432. put_online_cpus();
  433. return -EINVAL;
  434. }
  435. if (rapl_read_data_raw(rd, prim, true, &val))
  436. ret = -EIO;
  437. else
  438. *data = val;
  439. /* As a generalization rule, PL4 would be around two times PL2. */
  440. if (rd->rpl[id].prim_id == PL4_ENABLE)
  441. *data = *data * 2;
  442. put_online_cpus();
  443. return ret;
  444. }
  445. static const struct powercap_zone_constraint_ops constraint_ops = {
  446. .set_power_limit_uw = set_power_limit,
  447. .get_power_limit_uw = get_current_power_limit,
  448. .set_time_window_us = set_time_window,
  449. .get_time_window_us = get_time_window,
  450. .get_max_power_uw = get_max_power,
  451. .get_name = get_constraint_name,
  452. };
  453. /* called after domain detection and package level data are set */
  454. static void rapl_init_domains(struct rapl_package *rp)
  455. {
  456. enum rapl_domain_type i;
  457. enum rapl_domain_reg_id j;
  458. struct rapl_domain *rd = rp->domains;
  459. for (i = 0; i < RAPL_DOMAIN_MAX; i++) {
  460. unsigned int mask = rp->domain_map & (1 << i);
  461. if (!mask)
  462. continue;
  463. rd->rp = rp;
  464. if (i == RAPL_DOMAIN_PLATFORM && rp->id > 0) {
  465. snprintf(rd->name, RAPL_DOMAIN_NAME_LENGTH, "psys-%d",
  466. cpu_data(rp->lead_cpu).phys_proc_id);
  467. } else
  468. snprintf(rd->name, RAPL_DOMAIN_NAME_LENGTH, "%s",
  469. rapl_domain_names[i]);
  470. rd->id = i;
  471. rd->rpl[0].prim_id = PL1_ENABLE;
  472. rd->rpl[0].name = pl1_name;
  473. /*
  474. * The PL2 power domain is applicable for limits two
  475. * and limits three
  476. */
  477. if (rp->priv->limits[i] >= 2) {
  478. rd->rpl[1].prim_id = PL2_ENABLE;
  479. rd->rpl[1].name = pl2_name;
  480. }
  481. /* Enable PL4 domain if the total power limits are three */
  482. if (rp->priv->limits[i] == 3) {
  483. rd->rpl[2].prim_id = PL4_ENABLE;
  484. rd->rpl[2].name = pl4_name;
  485. }
  486. for (j = 0; j < RAPL_DOMAIN_REG_MAX; j++)
  487. rd->regs[j] = rp->priv->regs[i][j];
  488. switch (i) {
  489. case RAPL_DOMAIN_DRAM:
  490. rd->domain_energy_unit =
  491. rapl_defaults->dram_domain_energy_unit;
  492. if (rd->domain_energy_unit)
  493. pr_info("DRAM domain energy unit %dpj\n",
  494. rd->domain_energy_unit);
  495. break;
  496. case RAPL_DOMAIN_PLATFORM:
  497. rd->domain_energy_unit =
  498. rapl_defaults->psys_domain_energy_unit;
  499. if (rd->domain_energy_unit)
  500. pr_info("Platform domain energy unit %dpj\n",
  501. rd->domain_energy_unit);
  502. break;
  503. default:
  504. break;
  505. }
  506. rd++;
  507. }
  508. }
  509. static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type,
  510. u64 value, int to_raw)
  511. {
  512. u64 units = 1;
  513. struct rapl_package *rp = rd->rp;
  514. u64 scale = 1;
  515. switch (type) {
  516. case POWER_UNIT:
  517. units = rp->power_unit;
  518. break;
  519. case ENERGY_UNIT:
  520. scale = ENERGY_UNIT_SCALE;
  521. /* per domain unit takes precedence */
  522. if (rd->domain_energy_unit)
  523. units = rd->domain_energy_unit;
  524. else
  525. units = rp->energy_unit;
  526. break;
  527. case TIME_UNIT:
  528. return rapl_defaults->compute_time_window(rp, value, to_raw);
  529. case ARBITRARY_UNIT:
  530. default:
  531. return value;
  532. }
  533. if (to_raw)
  534. return div64_u64(value, units) * scale;
  535. value *= units;
  536. return div64_u64(value, scale);
  537. }
  538. /* in the order of enum rapl_primitives */
  539. static struct rapl_primitive_info rpi[] = {
  540. /* name, mask, shift, msr index, unit divisor */
  541. PRIMITIVE_INFO_INIT(ENERGY_COUNTER, ENERGY_STATUS_MASK, 0,
  542. RAPL_DOMAIN_REG_STATUS, ENERGY_UNIT, 0),
  543. PRIMITIVE_INFO_INIT(POWER_LIMIT1, POWER_LIMIT1_MASK, 0,
  544. RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
  545. PRIMITIVE_INFO_INIT(POWER_LIMIT2, POWER_LIMIT2_MASK, 32,
  546. RAPL_DOMAIN_REG_LIMIT, POWER_UNIT, 0),
  547. PRIMITIVE_INFO_INIT(POWER_LIMIT4, POWER_LIMIT4_MASK, 0,
  548. RAPL_DOMAIN_REG_PL4, POWER_UNIT, 0),
  549. PRIMITIVE_INFO_INIT(FW_LOCK, POWER_LOW_LOCK, 31,
  550. RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
  551. PRIMITIVE_INFO_INIT(PL1_ENABLE, POWER_LIMIT1_ENABLE, 15,
  552. RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
  553. PRIMITIVE_INFO_INIT(PL1_CLAMP, POWER_LIMIT1_CLAMP, 16,
  554. RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
  555. PRIMITIVE_INFO_INIT(PL2_ENABLE, POWER_LIMIT2_ENABLE, 47,
  556. RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
  557. PRIMITIVE_INFO_INIT(PL2_CLAMP, POWER_LIMIT2_CLAMP, 48,
  558. RAPL_DOMAIN_REG_LIMIT, ARBITRARY_UNIT, 0),
  559. PRIMITIVE_INFO_INIT(PL4_ENABLE, POWER_LIMIT4_MASK, 0,
  560. RAPL_DOMAIN_REG_PL4, ARBITRARY_UNIT, 0),
  561. PRIMITIVE_INFO_INIT(TIME_WINDOW1, TIME_WINDOW1_MASK, 17,
  562. RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
  563. PRIMITIVE_INFO_INIT(TIME_WINDOW2, TIME_WINDOW2_MASK, 49,
  564. RAPL_DOMAIN_REG_LIMIT, TIME_UNIT, 0),
  565. PRIMITIVE_INFO_INIT(THERMAL_SPEC_POWER, POWER_INFO_THERMAL_SPEC_MASK,
  566. 0, RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
  567. PRIMITIVE_INFO_INIT(MAX_POWER, POWER_INFO_MAX_MASK, 32,
  568. RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
  569. PRIMITIVE_INFO_INIT(MIN_POWER, POWER_INFO_MIN_MASK, 16,
  570. RAPL_DOMAIN_REG_INFO, POWER_UNIT, 0),
  571. PRIMITIVE_INFO_INIT(MAX_TIME_WINDOW, POWER_INFO_MAX_TIME_WIN_MASK, 48,
  572. RAPL_DOMAIN_REG_INFO, TIME_UNIT, 0),
  573. PRIMITIVE_INFO_INIT(THROTTLED_TIME, PERF_STATUS_THROTTLE_TIME_MASK, 0,
  574. RAPL_DOMAIN_REG_PERF, TIME_UNIT, 0),
  575. PRIMITIVE_INFO_INIT(PRIORITY_LEVEL, PP_POLICY_MASK, 0,
  576. RAPL_DOMAIN_REG_POLICY, ARBITRARY_UNIT, 0),
  577. /* non-hardware */
  578. PRIMITIVE_INFO_INIT(AVERAGE_POWER, 0, 0, 0, POWER_UNIT,
  579. RAPL_PRIMITIVE_DERIVED),
  580. {NULL, 0, 0, 0},
  581. };
  582. /* Read primitive data based on its related struct rapl_primitive_info.
  583. * if xlate flag is set, return translated data based on data units, i.e.
  584. * time, energy, and power.
  585. * RAPL MSRs are non-architectual and are laid out not consistently across
  586. * domains. Here we use primitive info to allow writing consolidated access
  587. * functions.
  588. * For a given primitive, it is processed by MSR mask and shift. Unit conversion
  589. * is pre-assigned based on RAPL unit MSRs read at init time.
  590. * 63-------------------------- 31--------------------------- 0
  591. * | xxxxx (mask) |
  592. * | |<- shift ----------------|
  593. * 63-------------------------- 31--------------------------- 0
  594. */
  595. static int rapl_read_data_raw(struct rapl_domain *rd,
  596. enum rapl_primitives prim, bool xlate, u64 *data)
  597. {
  598. u64 value;
  599. struct rapl_primitive_info *rp = &rpi[prim];
  600. struct reg_action ra;
  601. int cpu;
  602. if (!rp->name || rp->flag & RAPL_PRIMITIVE_DUMMY)
  603. return -EINVAL;
  604. ra.reg = rd->regs[rp->id];
  605. if (!ra.reg)
  606. return -EINVAL;
  607. cpu = rd->rp->lead_cpu;
  608. /* domain with 2 limits has different bit */
  609. if (prim == FW_LOCK && rd->rp->priv->limits[rd->id] == 2) {
  610. rp->mask = POWER_HIGH_LOCK;
  611. rp->shift = 63;
  612. }
  613. /* non-hardware data are collected by the polling thread */
  614. if (rp->flag & RAPL_PRIMITIVE_DERIVED) {
  615. *data = rd->rdd.primitives[prim];
  616. return 0;
  617. }
  618. ra.mask = rp->mask;
  619. if (rd->rp->priv->read_raw(cpu, &ra)) {
  620. pr_debug("failed to read reg 0x%llx on cpu %d\n", ra.reg, cpu);
  621. return -EIO;
  622. }
  623. value = ra.value >> rp->shift;
  624. if (xlate)
  625. *data = rapl_unit_xlate(rd, rp->unit, value, 0);
  626. else
  627. *data = value;
  628. return 0;
  629. }
  630. /* Similar use of primitive info in the read counterpart */
  631. static int rapl_write_data_raw(struct rapl_domain *rd,
  632. enum rapl_primitives prim,
  633. unsigned long long value)
  634. {
  635. struct rapl_primitive_info *rp = &rpi[prim];
  636. int cpu;
  637. u64 bits;
  638. struct reg_action ra;
  639. int ret;
  640. cpu = rd->rp->lead_cpu;
  641. bits = rapl_unit_xlate(rd, rp->unit, value, 1);
  642. bits <<= rp->shift;
  643. bits &= rp->mask;
  644. memset(&ra, 0, sizeof(ra));
  645. ra.reg = rd->regs[rp->id];
  646. ra.mask = rp->mask;
  647. ra.value = bits;
  648. ret = rd->rp->priv->write_raw(cpu, &ra);
  649. return ret;
  650. }
  651. /*
  652. * Raw RAPL data stored in MSRs are in certain scales. We need to
  653. * convert them into standard units based on the units reported in
  654. * the RAPL unit MSRs. This is specific to CPUs as the method to
  655. * calculate units differ on different CPUs.
  656. * We convert the units to below format based on CPUs.
  657. * i.e.
  658. * energy unit: picoJoules : Represented in picoJoules by default
  659. * power unit : microWatts : Represented in milliWatts by default
  660. * time unit : microseconds: Represented in seconds by default
  661. */
  662. static int rapl_check_unit_core(struct rapl_package *rp, int cpu)
  663. {
  664. struct reg_action ra;
  665. u32 value;
  666. ra.reg = rp->priv->reg_unit;
  667. ra.mask = ~0;
  668. if (rp->priv->read_raw(cpu, &ra)) {
  669. pr_err("Failed to read power unit REG 0x%llx on CPU %d, exit.\n",
  670. rp->priv->reg_unit, cpu);
  671. return -ENODEV;
  672. }
  673. value = (ra.value & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET;
  674. rp->energy_unit = ENERGY_UNIT_SCALE * 1000000 / (1 << value);
  675. value = (ra.value & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET;
  676. rp->power_unit = 1000000 / (1 << value);
  677. value = (ra.value & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET;
  678. rp->time_unit = 1000000 / (1 << value);
  679. pr_debug("Core CPU %s energy=%dpJ, time=%dus, power=%duW\n",
  680. rp->name, rp->energy_unit, rp->time_unit, rp->power_unit);
  681. return 0;
  682. }
  683. static int rapl_check_unit_atom(struct rapl_package *rp, int cpu)
  684. {
  685. struct reg_action ra;
  686. u32 value;
  687. ra.reg = rp->priv->reg_unit;
  688. ra.mask = ~0;
  689. if (rp->priv->read_raw(cpu, &ra)) {
  690. pr_err("Failed to read power unit REG 0x%llx on CPU %d, exit.\n",
  691. rp->priv->reg_unit, cpu);
  692. return -ENODEV;
  693. }
  694. value = (ra.value & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET;
  695. rp->energy_unit = ENERGY_UNIT_SCALE * 1 << value;
  696. value = (ra.value & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET;
  697. rp->power_unit = (1 << value) * 1000;
  698. value = (ra.value & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET;
  699. rp->time_unit = 1000000 / (1 << value);
  700. pr_debug("Atom %s energy=%dpJ, time=%dus, power=%duW\n",
  701. rp->name, rp->energy_unit, rp->time_unit, rp->power_unit);
  702. return 0;
  703. }
  704. static void power_limit_irq_save_cpu(void *info)
  705. {
  706. u32 l, h = 0;
  707. struct rapl_package *rp = (struct rapl_package *)info;
  708. /* save the state of PLN irq mask bit before disabling it */
  709. rdmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &l, &h);
  710. if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) {
  711. rp->power_limit_irq = l & PACKAGE_THERM_INT_PLN_ENABLE;
  712. rp->power_limit_irq |= PACKAGE_PLN_INT_SAVED;
  713. }
  714. l &= ~PACKAGE_THERM_INT_PLN_ENABLE;
  715. wrmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
  716. }
  717. /* REVISIT:
  718. * When package power limit is set artificially low by RAPL, LVT
  719. * thermal interrupt for package power limit should be ignored
  720. * since we are not really exceeding the real limit. The intention
  721. * is to avoid excessive interrupts while we are trying to save power.
  722. * A useful feature might be routing the package_power_limit interrupt
  723. * to userspace via eventfd. once we have a usecase, this is simple
  724. * to do by adding an atomic notifier.
  725. */
  726. static void package_power_limit_irq_save(struct rapl_package *rp)
  727. {
  728. if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN))
  729. return;
  730. smp_call_function_single(rp->lead_cpu, power_limit_irq_save_cpu, rp, 1);
  731. }
  732. /*
  733. * Restore per package power limit interrupt enable state. Called from cpu
  734. * hotplug code on package removal.
  735. */
  736. static void package_power_limit_irq_restore(struct rapl_package *rp)
  737. {
  738. u32 l, h;
  739. if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN))
  740. return;
  741. /* irq enable state not saved, nothing to restore */
  742. if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED))
  743. return;
  744. rdmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &l, &h);
  745. if (rp->power_limit_irq & PACKAGE_THERM_INT_PLN_ENABLE)
  746. l |= PACKAGE_THERM_INT_PLN_ENABLE;
  747. else
  748. l &= ~PACKAGE_THERM_INT_PLN_ENABLE;
  749. wrmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
  750. }
  751. static void set_floor_freq_default(struct rapl_domain *rd, bool mode)
  752. {
  753. int nr_powerlimit = find_nr_power_limit(rd);
  754. /* always enable clamp such that p-state can go below OS requested
  755. * range. power capping priority over guranteed frequency.
  756. */
  757. rapl_write_data_raw(rd, PL1_CLAMP, mode);
  758. /* some domains have pl2 */
  759. if (nr_powerlimit > 1) {
  760. rapl_write_data_raw(rd, PL2_ENABLE, mode);
  761. rapl_write_data_raw(rd, PL2_CLAMP, mode);
  762. }
  763. }
  764. static void set_floor_freq_atom(struct rapl_domain *rd, bool enable)
  765. {
  766. static u32 power_ctrl_orig_val;
  767. u32 mdata;
  768. if (!rapl_defaults->floor_freq_reg_addr) {
  769. pr_err("Invalid floor frequency config register\n");
  770. return;
  771. }
  772. if (!power_ctrl_orig_val)
  773. iosf_mbi_read(BT_MBI_UNIT_PMC, MBI_CR_READ,
  774. rapl_defaults->floor_freq_reg_addr,
  775. &power_ctrl_orig_val);
  776. mdata = power_ctrl_orig_val;
  777. if (enable) {
  778. mdata &= ~(0x7f << 8);
  779. mdata |= 1 << 8;
  780. }
  781. iosf_mbi_write(BT_MBI_UNIT_PMC, MBI_CR_WRITE,
  782. rapl_defaults->floor_freq_reg_addr, mdata);
  783. }
  784. static u64 rapl_compute_time_window_core(struct rapl_package *rp, u64 value,
  785. bool to_raw)
  786. {
  787. u64 f, y; /* fraction and exp. used for time unit */
  788. /*
  789. * Special processing based on 2^Y*(1+F/4), refer
  790. * to Intel Software Developer's manual Vol.3B: CH 14.9.3.
  791. */
  792. if (!to_raw) {
  793. f = (value & 0x60) >> 5;
  794. y = value & 0x1f;
  795. value = (1 << y) * (4 + f) * rp->time_unit / 4;
  796. } else {
  797. do_div(value, rp->time_unit);
  798. y = ilog2(value);
  799. f = div64_u64(4 * (value - (1 << y)), 1 << y);
  800. value = (y & 0x1f) | ((f & 0x3) << 5);
  801. }
  802. return value;
  803. }
  804. static u64 rapl_compute_time_window_atom(struct rapl_package *rp, u64 value,
  805. bool to_raw)
  806. {
  807. /*
  808. * Atom time unit encoding is straight forward val * time_unit,
  809. * where time_unit is default to 1 sec. Never 0.
  810. */
  811. if (!to_raw)
  812. return (value) ? value *= rp->time_unit : rp->time_unit;
  813. value = div64_u64(value, rp->time_unit);
  814. return value;
  815. }
  816. static const struct rapl_defaults rapl_defaults_core = {
  817. .floor_freq_reg_addr = 0,
  818. .check_unit = rapl_check_unit_core,
  819. .set_floor_freq = set_floor_freq_default,
  820. .compute_time_window = rapl_compute_time_window_core,
  821. };
  822. static const struct rapl_defaults rapl_defaults_hsw_server = {
  823. .check_unit = rapl_check_unit_core,
  824. .set_floor_freq = set_floor_freq_default,
  825. .compute_time_window = rapl_compute_time_window_core,
  826. .dram_domain_energy_unit = 15300,
  827. };
  828. static const struct rapl_defaults rapl_defaults_spr_server = {
  829. .check_unit = rapl_check_unit_core,
  830. .set_floor_freq = set_floor_freq_default,
  831. .compute_time_window = rapl_compute_time_window_core,
  832. .dram_domain_energy_unit = 15300,
  833. .psys_domain_energy_unit = 1000000000,
  834. };
  835. static const struct rapl_defaults rapl_defaults_byt = {
  836. .floor_freq_reg_addr = IOSF_CPU_POWER_BUDGET_CTL_BYT,
  837. .check_unit = rapl_check_unit_atom,
  838. .set_floor_freq = set_floor_freq_atom,
  839. .compute_time_window = rapl_compute_time_window_atom,
  840. };
  841. static const struct rapl_defaults rapl_defaults_tng = {
  842. .floor_freq_reg_addr = IOSF_CPU_POWER_BUDGET_CTL_TNG,
  843. .check_unit = rapl_check_unit_atom,
  844. .set_floor_freq = set_floor_freq_atom,
  845. .compute_time_window = rapl_compute_time_window_atom,
  846. };
  847. static const struct rapl_defaults rapl_defaults_ann = {
  848. .floor_freq_reg_addr = 0,
  849. .check_unit = rapl_check_unit_atom,
  850. .set_floor_freq = NULL,
  851. .compute_time_window = rapl_compute_time_window_atom,
  852. };
  853. static const struct rapl_defaults rapl_defaults_cht = {
  854. .floor_freq_reg_addr = 0,
  855. .check_unit = rapl_check_unit_atom,
  856. .set_floor_freq = NULL,
  857. .compute_time_window = rapl_compute_time_window_atom,
  858. };
  859. static const struct x86_cpu_id rapl_ids[] __initconst = {
  860. X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE, &rapl_defaults_core),
  861. X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X, &rapl_defaults_core),
  862. X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE, &rapl_defaults_core),
  863. X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &rapl_defaults_core),
  864. X86_MATCH_INTEL_FAM6_MODEL(HASWELL, &rapl_defaults_core),
  865. X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L, &rapl_defaults_core),
  866. X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G, &rapl_defaults_core),
  867. X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &rapl_defaults_hsw_server),
  868. X86_MATCH_INTEL_FAM6_MODEL(BROADWELL, &rapl_defaults_core),
  869. X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, &rapl_defaults_core),
  870. X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &rapl_defaults_core),
  871. X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &rapl_defaults_hsw_server),
  872. X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE, &rapl_defaults_core),
  873. X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L, &rapl_defaults_core),
  874. X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &rapl_defaults_hsw_server),
  875. X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L, &rapl_defaults_core),
  876. X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE, &rapl_defaults_core),
  877. X86_MATCH_INTEL_FAM6_MODEL(CANNONLAKE_L, &rapl_defaults_core),
  878. X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_L, &rapl_defaults_core),
  879. X86_MATCH_INTEL_FAM6_MODEL(ICELAKE, &rapl_defaults_core),
  880. X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_NNPI, &rapl_defaults_core),
  881. X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &rapl_defaults_hsw_server),
  882. X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &rapl_defaults_hsw_server),
  883. X86_MATCH_INTEL_FAM6_MODEL(COMETLAKE_L, &rapl_defaults_core),
  884. X86_MATCH_INTEL_FAM6_MODEL(COMETLAKE, &rapl_defaults_core),
  885. X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE_L, &rapl_defaults_core),
  886. X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE, &rapl_defaults_core),
  887. X86_MATCH_INTEL_FAM6_MODEL(ROCKETLAKE, &rapl_defaults_core),
  888. X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, &rapl_defaults_core),
  889. X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &rapl_defaults_spr_server),
  890. X86_MATCH_INTEL_FAM6_MODEL(LAKEFIELD, &rapl_defaults_core),
  891. X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &rapl_defaults_byt),
  892. X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT, &rapl_defaults_cht),
  893. X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, &rapl_defaults_tng),
  894. X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT_MID, &rapl_defaults_ann),
  895. X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, &rapl_defaults_core),
  896. X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_PLUS, &rapl_defaults_core),
  897. X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D, &rapl_defaults_core),
  898. X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT, &rapl_defaults_core),
  899. X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &rapl_defaults_core),
  900. X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_L, &rapl_defaults_core),
  901. X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &rapl_defaults_hsw_server),
  902. X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &rapl_defaults_hsw_server),
  903. {}
  904. };
  905. MODULE_DEVICE_TABLE(x86cpu, rapl_ids);
  906. /* Read once for all raw primitive data for domains */
  907. static void rapl_update_domain_data(struct rapl_package *rp)
  908. {
  909. int dmn, prim;
  910. u64 val;
  911. for (dmn = 0; dmn < rp->nr_domains; dmn++) {
  912. pr_debug("update %s domain %s data\n", rp->name,
  913. rp->domains[dmn].name);
  914. /* exclude non-raw primitives */
  915. for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++) {
  916. if (!rapl_read_data_raw(&rp->domains[dmn], prim,
  917. rpi[prim].unit, &val))
  918. rp->domains[dmn].rdd.primitives[prim] = val;
  919. }
  920. }
  921. }
  922. static int rapl_package_register_powercap(struct rapl_package *rp)
  923. {
  924. struct rapl_domain *rd;
  925. struct powercap_zone *power_zone = NULL;
  926. int nr_pl, ret;
  927. /* Update the domain data of the new package */
  928. rapl_update_domain_data(rp);
  929. /* first we register package domain as the parent zone */
  930. for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
  931. if (rd->id == RAPL_DOMAIN_PACKAGE) {
  932. nr_pl = find_nr_power_limit(rd);
  933. pr_debug("register package domain %s\n", rp->name);
  934. power_zone = powercap_register_zone(&rd->power_zone,
  935. rp->priv->control_type, rp->name,
  936. NULL, &zone_ops[rd->id], nr_pl,
  937. &constraint_ops);
  938. if (IS_ERR(power_zone)) {
  939. pr_debug("failed to register power zone %s\n",
  940. rp->name);
  941. return PTR_ERR(power_zone);
  942. }
  943. /* track parent zone in per package/socket data */
  944. rp->power_zone = power_zone;
  945. /* done, only one package domain per socket */
  946. break;
  947. }
  948. }
  949. if (!power_zone) {
  950. pr_err("no package domain found, unknown topology!\n");
  951. return -ENODEV;
  952. }
  953. /* now register domains as children of the socket/package */
  954. for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
  955. struct powercap_zone *parent = rp->power_zone;
  956. if (rd->id == RAPL_DOMAIN_PACKAGE)
  957. continue;
  958. if (rd->id == RAPL_DOMAIN_PLATFORM)
  959. parent = NULL;
  960. /* number of power limits per domain varies */
  961. nr_pl = find_nr_power_limit(rd);
  962. power_zone = powercap_register_zone(&rd->power_zone,
  963. rp->priv->control_type,
  964. rd->name, parent,
  965. &zone_ops[rd->id], nr_pl,
  966. &constraint_ops);
  967. if (IS_ERR(power_zone)) {
  968. pr_debug("failed to register power_zone, %s:%s\n",
  969. rp->name, rd->name);
  970. ret = PTR_ERR(power_zone);
  971. goto err_cleanup;
  972. }
  973. }
  974. return 0;
  975. err_cleanup:
  976. /*
  977. * Clean up previously initialized domains within the package if we
  978. * failed after the first domain setup.
  979. */
  980. while (--rd >= rp->domains) {
  981. pr_debug("unregister %s domain %s\n", rp->name, rd->name);
  982. powercap_unregister_zone(rp->priv->control_type,
  983. &rd->power_zone);
  984. }
  985. return ret;
  986. }
  987. static int rapl_check_domain(int cpu, int domain, struct rapl_package *rp)
  988. {
  989. struct reg_action ra;
  990. switch (domain) {
  991. case RAPL_DOMAIN_PACKAGE:
  992. case RAPL_DOMAIN_PP0:
  993. case RAPL_DOMAIN_PP1:
  994. case RAPL_DOMAIN_DRAM:
  995. case RAPL_DOMAIN_PLATFORM:
  996. ra.reg = rp->priv->regs[domain][RAPL_DOMAIN_REG_STATUS];
  997. break;
  998. default:
  999. pr_err("invalid domain id %d\n", domain);
  1000. return -EINVAL;
  1001. }
  1002. /* make sure domain counters are available and contains non-zero
  1003. * values, otherwise skip it.
  1004. */
  1005. ra.mask = ENERGY_STATUS_MASK;
  1006. if (rp->priv->read_raw(cpu, &ra) || !ra.value)
  1007. return -ENODEV;
  1008. return 0;
  1009. }
  1010. /*
  1011. * Check if power limits are available. Two cases when they are not available:
  1012. * 1. Locked by BIOS, in this case we still provide read-only access so that
  1013. * users can see what limit is set by the BIOS.
  1014. * 2. Some CPUs make some domains monitoring only which means PLx MSRs may not
  1015. * exist at all. In this case, we do not show the constraints in powercap.
  1016. *
  1017. * Called after domains are detected and initialized.
  1018. */
  1019. static void rapl_detect_powerlimit(struct rapl_domain *rd)
  1020. {
  1021. u64 val64;
  1022. int i;
  1023. /* check if the domain is locked by BIOS, ignore if MSR doesn't exist */
  1024. if (!rapl_read_data_raw(rd, FW_LOCK, false, &val64)) {
  1025. if (val64) {
  1026. pr_info("RAPL %s domain %s locked by BIOS\n",
  1027. rd->rp->name, rd->name);
  1028. rd->state |= DOMAIN_STATE_BIOS_LOCKED;
  1029. }
  1030. }
  1031. /* check if power limit MSR exists, otherwise domain is monitoring only */
  1032. for (i = 0; i < NR_POWER_LIMITS; i++) {
  1033. int prim = rd->rpl[i].prim_id;
  1034. if (rapl_read_data_raw(rd, prim, false, &val64))
  1035. rd->rpl[i].name = NULL;
  1036. }
  1037. }
  1038. /* Detect active and valid domains for the given CPU, caller must
  1039. * ensure the CPU belongs to the targeted package and CPU hotlug is disabled.
  1040. */
  1041. static int rapl_detect_domains(struct rapl_package *rp, int cpu)
  1042. {
  1043. struct rapl_domain *rd;
  1044. int i;
  1045. for (i = 0; i < RAPL_DOMAIN_MAX; i++) {
  1046. /* use physical package id to read counters */
  1047. if (!rapl_check_domain(cpu, i, rp)) {
  1048. rp->domain_map |= 1 << i;
  1049. pr_info("Found RAPL domain %s\n", rapl_domain_names[i]);
  1050. }
  1051. }
  1052. rp->nr_domains = bitmap_weight(&rp->domain_map, RAPL_DOMAIN_MAX);
  1053. if (!rp->nr_domains) {
  1054. pr_debug("no valid rapl domains found in %s\n", rp->name);
  1055. return -ENODEV;
  1056. }
  1057. pr_debug("found %d domains on %s\n", rp->nr_domains, rp->name);
  1058. rp->domains = kcalloc(rp->nr_domains + 1, sizeof(struct rapl_domain),
  1059. GFP_KERNEL);
  1060. if (!rp->domains)
  1061. return -ENOMEM;
  1062. rapl_init_domains(rp);
  1063. for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++)
  1064. rapl_detect_powerlimit(rd);
  1065. return 0;
  1066. }
  1067. /* called from CPU hotplug notifier, hotplug lock held */
  1068. void rapl_remove_package(struct rapl_package *rp)
  1069. {
  1070. struct rapl_domain *rd, *rd_package = NULL;
  1071. package_power_limit_irq_restore(rp);
  1072. for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
  1073. rapl_write_data_raw(rd, PL1_ENABLE, 0);
  1074. rapl_write_data_raw(rd, PL1_CLAMP, 0);
  1075. if (find_nr_power_limit(rd) > 1) {
  1076. rapl_write_data_raw(rd, PL2_ENABLE, 0);
  1077. rapl_write_data_raw(rd, PL2_CLAMP, 0);
  1078. rapl_write_data_raw(rd, PL4_ENABLE, 0);
  1079. }
  1080. if (rd->id == RAPL_DOMAIN_PACKAGE) {
  1081. rd_package = rd;
  1082. continue;
  1083. }
  1084. pr_debug("remove package, undo power limit on %s: %s\n",
  1085. rp->name, rd->name);
  1086. powercap_unregister_zone(rp->priv->control_type,
  1087. &rd->power_zone);
  1088. }
  1089. /* do parent zone last */
  1090. powercap_unregister_zone(rp->priv->control_type,
  1091. &rd_package->power_zone);
  1092. list_del(&rp->plist);
  1093. kfree(rp);
  1094. }
  1095. EXPORT_SYMBOL_GPL(rapl_remove_package);
  1096. /* caller to ensure CPU hotplug lock is held */
  1097. struct rapl_package *rapl_find_package_domain(int cpu, struct rapl_if_priv *priv)
  1098. {
  1099. int id = topology_logical_die_id(cpu);
  1100. struct rapl_package *rp;
  1101. list_for_each_entry(rp, &rapl_packages, plist) {
  1102. if (rp->id == id
  1103. && rp->priv->control_type == priv->control_type)
  1104. return rp;
  1105. }
  1106. return NULL;
  1107. }
  1108. EXPORT_SYMBOL_GPL(rapl_find_package_domain);
  1109. /* called from CPU hotplug notifier, hotplug lock held */
  1110. struct rapl_package *rapl_add_package(int cpu, struct rapl_if_priv *priv)
  1111. {
  1112. int id = topology_logical_die_id(cpu);
  1113. struct rapl_package *rp;
  1114. struct cpuinfo_x86 *c = &cpu_data(cpu);
  1115. int ret;
  1116. if (!rapl_defaults)
  1117. return ERR_PTR(-ENODEV);
  1118. rp = kzalloc(sizeof(struct rapl_package), GFP_KERNEL);
  1119. if (!rp)
  1120. return ERR_PTR(-ENOMEM);
  1121. /* add the new package to the list */
  1122. rp->id = id;
  1123. rp->lead_cpu = cpu;
  1124. rp->priv = priv;
  1125. if (topology_max_die_per_package() > 1)
  1126. snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH,
  1127. "package-%d-die-%d", c->phys_proc_id, c->cpu_die_id);
  1128. else
  1129. snprintf(rp->name, PACKAGE_DOMAIN_NAME_LENGTH, "package-%d",
  1130. c->phys_proc_id);
  1131. /* check if the package contains valid domains */
  1132. if (rapl_detect_domains(rp, cpu) || rapl_defaults->check_unit(rp, cpu)) {
  1133. ret = -ENODEV;
  1134. goto err_free_package;
  1135. }
  1136. ret = rapl_package_register_powercap(rp);
  1137. if (!ret) {
  1138. INIT_LIST_HEAD(&rp->plist);
  1139. list_add(&rp->plist, &rapl_packages);
  1140. return rp;
  1141. }
  1142. err_free_package:
  1143. kfree(rp->domains);
  1144. kfree(rp);
  1145. return ERR_PTR(ret);
  1146. }
  1147. EXPORT_SYMBOL_GPL(rapl_add_package);
  1148. static void power_limit_state_save(void)
  1149. {
  1150. struct rapl_package *rp;
  1151. struct rapl_domain *rd;
  1152. int nr_pl, ret, i;
  1153. get_online_cpus();
  1154. list_for_each_entry(rp, &rapl_packages, plist) {
  1155. if (!rp->power_zone)
  1156. continue;
  1157. rd = power_zone_to_rapl_domain(rp->power_zone);
  1158. nr_pl = find_nr_power_limit(rd);
  1159. for (i = 0; i < nr_pl; i++) {
  1160. switch (rd->rpl[i].prim_id) {
  1161. case PL1_ENABLE:
  1162. ret = rapl_read_data_raw(rd,
  1163. POWER_LIMIT1, true,
  1164. &rd->rpl[i].last_power_limit);
  1165. if (ret)
  1166. rd->rpl[i].last_power_limit = 0;
  1167. break;
  1168. case PL2_ENABLE:
  1169. ret = rapl_read_data_raw(rd,
  1170. POWER_LIMIT2, true,
  1171. &rd->rpl[i].last_power_limit);
  1172. if (ret)
  1173. rd->rpl[i].last_power_limit = 0;
  1174. break;
  1175. case PL4_ENABLE:
  1176. ret = rapl_read_data_raw(rd,
  1177. POWER_LIMIT4, true,
  1178. &rd->rpl[i].last_power_limit);
  1179. if (ret)
  1180. rd->rpl[i].last_power_limit = 0;
  1181. break;
  1182. }
  1183. }
  1184. }
  1185. put_online_cpus();
  1186. }
  1187. static void power_limit_state_restore(void)
  1188. {
  1189. struct rapl_package *rp;
  1190. struct rapl_domain *rd;
  1191. int nr_pl, i;
  1192. get_online_cpus();
  1193. list_for_each_entry(rp, &rapl_packages, plist) {
  1194. if (!rp->power_zone)
  1195. continue;
  1196. rd = power_zone_to_rapl_domain(rp->power_zone);
  1197. nr_pl = find_nr_power_limit(rd);
  1198. for (i = 0; i < nr_pl; i++) {
  1199. switch (rd->rpl[i].prim_id) {
  1200. case PL1_ENABLE:
  1201. if (rd->rpl[i].last_power_limit)
  1202. rapl_write_data_raw(rd, POWER_LIMIT1,
  1203. rd->rpl[i].last_power_limit);
  1204. break;
  1205. case PL2_ENABLE:
  1206. if (rd->rpl[i].last_power_limit)
  1207. rapl_write_data_raw(rd, POWER_LIMIT2,
  1208. rd->rpl[i].last_power_limit);
  1209. break;
  1210. case PL4_ENABLE:
  1211. if (rd->rpl[i].last_power_limit)
  1212. rapl_write_data_raw(rd, POWER_LIMIT4,
  1213. rd->rpl[i].last_power_limit);
  1214. break;
  1215. }
  1216. }
  1217. }
  1218. put_online_cpus();
  1219. }
  1220. static int rapl_pm_callback(struct notifier_block *nb,
  1221. unsigned long mode, void *_unused)
  1222. {
  1223. switch (mode) {
  1224. case PM_SUSPEND_PREPARE:
  1225. power_limit_state_save();
  1226. break;
  1227. case PM_POST_SUSPEND:
  1228. power_limit_state_restore();
  1229. break;
  1230. }
  1231. return NOTIFY_OK;
  1232. }
  1233. static struct notifier_block rapl_pm_notifier = {
  1234. .notifier_call = rapl_pm_callback,
  1235. };
  1236. static struct platform_device *rapl_msr_platdev;
  1237. static int __init rapl_init(void)
  1238. {
  1239. const struct x86_cpu_id *id;
  1240. int ret;
  1241. id = x86_match_cpu(rapl_ids);
  1242. if (!id) {
  1243. pr_err("driver does not support CPU family %d model %d\n",
  1244. boot_cpu_data.x86, boot_cpu_data.x86_model);
  1245. return -ENODEV;
  1246. }
  1247. rapl_defaults = (struct rapl_defaults *)id->driver_data;
  1248. ret = register_pm_notifier(&rapl_pm_notifier);
  1249. if (ret)
  1250. return ret;
  1251. rapl_msr_platdev = platform_device_alloc("intel_rapl_msr", 0);
  1252. if (!rapl_msr_platdev) {
  1253. ret = -ENOMEM;
  1254. goto end;
  1255. }
  1256. ret = platform_device_add(rapl_msr_platdev);
  1257. if (ret)
  1258. platform_device_put(rapl_msr_platdev);
  1259. end:
  1260. if (ret)
  1261. unregister_pm_notifier(&rapl_pm_notifier);
  1262. return ret;
  1263. }
  1264. static void __exit rapl_exit(void)
  1265. {
  1266. platform_device_unregister(rapl_msr_platdev);
  1267. unregister_pm_notifier(&rapl_pm_notifier);
  1268. }
  1269. fs_initcall(rapl_init);
  1270. module_exit(rapl_exit);
  1271. MODULE_DESCRIPTION("Intel Runtime Average Power Limit (RAPL) common code");
  1272. MODULE_AUTHOR("Jacob Pan <jacob.jun.pan@intel.com>");
  1273. MODULE_LICENSE("GPL v2");