powernv-cpufreq.c 31 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * POWERNV cpufreq driver for the IBM POWER processors
  4. *
  5. * (C) Copyright IBM 2014
  6. *
  7. * Author: Vaidyanathan Srinivasan <svaidy at linux.vnet.ibm.com>
  8. */
  9. #define pr_fmt(fmt) "powernv-cpufreq: " fmt
  10. #include <linux/kernel.h>
  11. #include <linux/sysfs.h>
  12. #include <linux/cpumask.h>
  13. #include <linux/module.h>
  14. #include <linux/cpufreq.h>
  15. #include <linux/smp.h>
  16. #include <linux/of.h>
  17. #include <linux/reboot.h>
  18. #include <linux/slab.h>
  19. #include <linux/cpu.h>
  20. #include <linux/hashtable.h>
  21. #include <trace/events/power.h>
  22. #include <asm/cputhreads.h>
  23. #include <asm/firmware.h>
  24. #include <asm/reg.h>
  25. #include <asm/smp.h> /* Required for cpu_sibling_mask() in UP configs */
  26. #include <asm/opal.h>
  27. #include <linux/timer.h>
  28. #define POWERNV_MAX_PSTATES_ORDER 8
  29. #define POWERNV_MAX_PSTATES (1UL << (POWERNV_MAX_PSTATES_ORDER))
  30. #define PMSR_PSAFE_ENABLE (1UL << 30)
  31. #define PMSR_SPR_EM_DISABLE (1UL << 31)
  32. #define MAX_PSTATE_SHIFT 32
  33. #define LPSTATE_SHIFT 48
  34. #define GPSTATE_SHIFT 56
  35. #define MAX_NR_CHIPS 32
  36. #define MAX_RAMP_DOWN_TIME 5120
  37. /*
  38. * On an idle system we want the global pstate to ramp-down from max value to
  39. * min over a span of ~5 secs. Also we want it to initially ramp-down slowly and
  40. * then ramp-down rapidly later on.
  41. *
  42. * This gives a percentage rampdown for time elapsed in milliseconds.
  43. * ramp_down_percentage = ((ms * ms) >> 18)
  44. * ~= 3.8 * (sec * sec)
  45. *
  46. * At 0 ms ramp_down_percent = 0
  47. * At 5120 ms ramp_down_percent = 100
  48. */
  49. #define ramp_down_percent(time) ((time * time) >> 18)
  50. /* Interval after which the timer is queued to bring down global pstate */
  51. #define GPSTATE_TIMER_INTERVAL 2000
  52. /**
  53. * struct global_pstate_info - Per policy data structure to maintain history of
  54. * global pstates
  55. * @highest_lpstate_idx: The local pstate index from which we are
  56. * ramping down
  57. * @elapsed_time: Time in ms spent in ramping down from
  58. * highest_lpstate_idx
  59. * @last_sampled_time: Time from boot in ms when global pstates were
  60. * last set
  61. * @last_lpstate_idx: Last set value of local pstate and global
  62. * @last_gpstate_idx: pstate in terms of cpufreq table index
  63. * @timer: Is used for ramping down if cpu goes idle for
  64. * a long time with global pstate held high
  65. * @gpstate_lock: A spinlock to maintain synchronization between
  66. * routines called by the timer handler and
  67. * governer's target_index calls
  68. * @policy: Associated CPUFreq policy
  69. */
  70. struct global_pstate_info {
  71. int highest_lpstate_idx;
  72. unsigned int elapsed_time;
  73. unsigned int last_sampled_time;
  74. int last_lpstate_idx;
  75. int last_gpstate_idx;
  76. spinlock_t gpstate_lock;
  77. struct timer_list timer;
  78. struct cpufreq_policy *policy;
  79. };
  80. static struct cpufreq_frequency_table powernv_freqs[POWERNV_MAX_PSTATES+1];
  81. static DEFINE_HASHTABLE(pstate_revmap, POWERNV_MAX_PSTATES_ORDER);
  82. /**
  83. * struct pstate_idx_revmap_data: Entry in the hashmap pstate_revmap
  84. * indexed by a function of pstate id.
  85. *
  86. * @pstate_id: pstate id for this entry.
  87. *
  88. * @cpufreq_table_idx: Index into the powernv_freqs
  89. * cpufreq_frequency_table for frequency
  90. * corresponding to pstate_id.
  91. *
  92. * @hentry: hlist_node that hooks this entry into the pstate_revmap
  93. * hashtable
  94. */
  95. struct pstate_idx_revmap_data {
  96. u8 pstate_id;
  97. unsigned int cpufreq_table_idx;
  98. struct hlist_node hentry;
  99. };
  100. static bool rebooting, throttled, occ_reset;
  101. static const char * const throttle_reason[] = {
  102. "No throttling",
  103. "Power Cap",
  104. "Processor Over Temperature",
  105. "Power Supply Failure",
  106. "Over Current",
  107. "OCC Reset"
  108. };
  109. enum throttle_reason_type {
  110. NO_THROTTLE = 0,
  111. POWERCAP,
  112. CPU_OVERTEMP,
  113. POWER_SUPPLY_FAILURE,
  114. OVERCURRENT,
  115. OCC_RESET_THROTTLE,
  116. OCC_MAX_REASON
  117. };
  118. static struct chip {
  119. unsigned int id;
  120. bool throttled;
  121. bool restore;
  122. u8 throttle_reason;
  123. cpumask_t mask;
  124. struct work_struct throttle;
  125. int throttle_turbo;
  126. int throttle_sub_turbo;
  127. int reason[OCC_MAX_REASON];
  128. } *chips;
  129. static int nr_chips;
  130. static DEFINE_PER_CPU(struct chip *, chip_info);
  131. /*
  132. * Note:
  133. * The set of pstates consists of contiguous integers.
  134. * powernv_pstate_info stores the index of the frequency table for
  135. * max, min and nominal frequencies. It also stores number of
  136. * available frequencies.
  137. *
  138. * powernv_pstate_info.nominal indicates the index to the highest
  139. * non-turbo frequency.
  140. */
  141. static struct powernv_pstate_info {
  142. unsigned int min;
  143. unsigned int max;
  144. unsigned int nominal;
  145. unsigned int nr_pstates;
  146. bool wof_enabled;
  147. } powernv_pstate_info;
  148. static inline u8 extract_pstate(u64 pmsr_val, unsigned int shift)
  149. {
  150. return ((pmsr_val >> shift) & 0xFF);
  151. }
  152. #define extract_local_pstate(x) extract_pstate(x, LPSTATE_SHIFT)
  153. #define extract_global_pstate(x) extract_pstate(x, GPSTATE_SHIFT)
  154. #define extract_max_pstate(x) extract_pstate(x, MAX_PSTATE_SHIFT)
  155. /* Use following functions for conversions between pstate_id and index */
  156. /*
  157. * idx_to_pstate : Returns the pstate id corresponding to the
  158. * frequency in the cpufreq frequency table
  159. * powernv_freqs indexed by @i.
  160. *
  161. * If @i is out of bound, this will return the pstate
  162. * corresponding to the nominal frequency.
  163. */
  164. static inline u8 idx_to_pstate(unsigned int i)
  165. {
  166. if (unlikely(i >= powernv_pstate_info.nr_pstates)) {
  167. pr_warn_once("idx_to_pstate: index %u is out of bound\n", i);
  168. return powernv_freqs[powernv_pstate_info.nominal].driver_data;
  169. }
  170. return powernv_freqs[i].driver_data;
  171. }
  172. /*
  173. * pstate_to_idx : Returns the index in the cpufreq frequencytable
  174. * powernv_freqs for the frequency whose corresponding
  175. * pstate id is @pstate.
  176. *
  177. * If no frequency corresponding to @pstate is found,
  178. * this will return the index of the nominal
  179. * frequency.
  180. */
  181. static unsigned int pstate_to_idx(u8 pstate)
  182. {
  183. unsigned int key = pstate % POWERNV_MAX_PSTATES;
  184. struct pstate_idx_revmap_data *revmap_data;
  185. hash_for_each_possible(pstate_revmap, revmap_data, hentry, key) {
  186. if (revmap_data->pstate_id == pstate)
  187. return revmap_data->cpufreq_table_idx;
  188. }
  189. pr_warn_once("pstate_to_idx: pstate 0x%x not found\n", pstate);
  190. return powernv_pstate_info.nominal;
  191. }
  192. static inline void reset_gpstates(struct cpufreq_policy *policy)
  193. {
  194. struct global_pstate_info *gpstates = policy->driver_data;
  195. gpstates->highest_lpstate_idx = 0;
  196. gpstates->elapsed_time = 0;
  197. gpstates->last_sampled_time = 0;
  198. gpstates->last_lpstate_idx = 0;
  199. gpstates->last_gpstate_idx = 0;
  200. }
  201. /*
  202. * Initialize the freq table based on data obtained
  203. * from the firmware passed via device-tree
  204. */
  205. static int init_powernv_pstates(void)
  206. {
  207. struct device_node *power_mgt;
  208. int i, nr_pstates = 0;
  209. const __be32 *pstate_ids, *pstate_freqs;
  210. u32 len_ids, len_freqs;
  211. u32 pstate_min, pstate_max, pstate_nominal;
  212. u32 pstate_turbo, pstate_ultra_turbo;
  213. int rc = -ENODEV;
  214. power_mgt = of_find_node_by_path("/ibm,opal/power-mgt");
  215. if (!power_mgt) {
  216. pr_warn("power-mgt node not found\n");
  217. return -ENODEV;
  218. }
  219. if (of_property_read_u32(power_mgt, "ibm,pstate-min", &pstate_min)) {
  220. pr_warn("ibm,pstate-min node not found\n");
  221. goto out;
  222. }
  223. if (of_property_read_u32(power_mgt, "ibm,pstate-max", &pstate_max)) {
  224. pr_warn("ibm,pstate-max node not found\n");
  225. goto out;
  226. }
  227. if (of_property_read_u32(power_mgt, "ibm,pstate-nominal",
  228. &pstate_nominal)) {
  229. pr_warn("ibm,pstate-nominal not found\n");
  230. goto out;
  231. }
  232. if (of_property_read_u32(power_mgt, "ibm,pstate-ultra-turbo",
  233. &pstate_ultra_turbo)) {
  234. powernv_pstate_info.wof_enabled = false;
  235. goto next;
  236. }
  237. if (of_property_read_u32(power_mgt, "ibm,pstate-turbo",
  238. &pstate_turbo)) {
  239. powernv_pstate_info.wof_enabled = false;
  240. goto next;
  241. }
  242. if (pstate_turbo == pstate_ultra_turbo)
  243. powernv_pstate_info.wof_enabled = false;
  244. else
  245. powernv_pstate_info.wof_enabled = true;
  246. next:
  247. pr_info("cpufreq pstate min 0x%x nominal 0x%x max 0x%x\n", pstate_min,
  248. pstate_nominal, pstate_max);
  249. pr_info("Workload Optimized Frequency is %s in the platform\n",
  250. (powernv_pstate_info.wof_enabled) ? "enabled" : "disabled");
  251. pstate_ids = of_get_property(power_mgt, "ibm,pstate-ids", &len_ids);
  252. if (!pstate_ids) {
  253. pr_warn("ibm,pstate-ids not found\n");
  254. goto out;
  255. }
  256. pstate_freqs = of_get_property(power_mgt, "ibm,pstate-frequencies-mhz",
  257. &len_freqs);
  258. if (!pstate_freqs) {
  259. pr_warn("ibm,pstate-frequencies-mhz not found\n");
  260. goto out;
  261. }
  262. if (len_ids != len_freqs) {
  263. pr_warn("Entries in ibm,pstate-ids and "
  264. "ibm,pstate-frequencies-mhz does not match\n");
  265. }
  266. nr_pstates = min(len_ids, len_freqs) / sizeof(u32);
  267. if (!nr_pstates) {
  268. pr_warn("No PStates found\n");
  269. goto out;
  270. }
  271. powernv_pstate_info.nr_pstates = nr_pstates;
  272. pr_debug("NR PStates %d\n", nr_pstates);
  273. for (i = 0; i < nr_pstates; i++) {
  274. u32 id = be32_to_cpu(pstate_ids[i]);
  275. u32 freq = be32_to_cpu(pstate_freqs[i]);
  276. struct pstate_idx_revmap_data *revmap_data;
  277. unsigned int key;
  278. pr_debug("PState id %d freq %d MHz\n", id, freq);
  279. powernv_freqs[i].frequency = freq * 1000; /* kHz */
  280. powernv_freqs[i].driver_data = id & 0xFF;
  281. revmap_data = kmalloc(sizeof(*revmap_data), GFP_KERNEL);
  282. if (!revmap_data) {
  283. rc = -ENOMEM;
  284. goto out;
  285. }
  286. revmap_data->pstate_id = id & 0xFF;
  287. revmap_data->cpufreq_table_idx = i;
  288. key = (revmap_data->pstate_id) % POWERNV_MAX_PSTATES;
  289. hash_add(pstate_revmap, &revmap_data->hentry, key);
  290. if (id == pstate_max)
  291. powernv_pstate_info.max = i;
  292. if (id == pstate_nominal)
  293. powernv_pstate_info.nominal = i;
  294. if (id == pstate_min)
  295. powernv_pstate_info.min = i;
  296. if (powernv_pstate_info.wof_enabled && id == pstate_turbo) {
  297. int j;
  298. for (j = i - 1; j >= (int)powernv_pstate_info.max; j--)
  299. powernv_freqs[j].flags = CPUFREQ_BOOST_FREQ;
  300. }
  301. }
  302. /* End of list marker entry */
  303. powernv_freqs[i].frequency = CPUFREQ_TABLE_END;
  304. of_node_put(power_mgt);
  305. return 0;
  306. out:
  307. of_node_put(power_mgt);
  308. return rc;
  309. }
  310. /* Returns the CPU frequency corresponding to the pstate_id. */
  311. static unsigned int pstate_id_to_freq(u8 pstate_id)
  312. {
  313. int i;
  314. i = pstate_to_idx(pstate_id);
  315. if (i >= powernv_pstate_info.nr_pstates || i < 0) {
  316. pr_warn("PState id 0x%x outside of PState table, reporting nominal id 0x%x instead\n",
  317. pstate_id, idx_to_pstate(powernv_pstate_info.nominal));
  318. i = powernv_pstate_info.nominal;
  319. }
  320. return powernv_freqs[i].frequency;
  321. }
  322. /*
  323. * cpuinfo_nominal_freq_show - Show the nominal CPU frequency as indicated by
  324. * the firmware
  325. */
  326. static ssize_t cpuinfo_nominal_freq_show(struct cpufreq_policy *policy,
  327. char *buf)
  328. {
  329. return sprintf(buf, "%u\n",
  330. powernv_freqs[powernv_pstate_info.nominal].frequency);
  331. }
  332. static struct freq_attr cpufreq_freq_attr_cpuinfo_nominal_freq =
  333. __ATTR_RO(cpuinfo_nominal_freq);
  334. #define SCALING_BOOST_FREQS_ATTR_INDEX 2
  335. static struct freq_attr *powernv_cpu_freq_attr[] = {
  336. &cpufreq_freq_attr_scaling_available_freqs,
  337. &cpufreq_freq_attr_cpuinfo_nominal_freq,
  338. &cpufreq_freq_attr_scaling_boost_freqs,
  339. NULL,
  340. };
  341. #define throttle_attr(name, member) \
  342. static ssize_t name##_show(struct cpufreq_policy *policy, char *buf) \
  343. { \
  344. struct chip *chip = per_cpu(chip_info, policy->cpu); \
  345. \
  346. return sprintf(buf, "%u\n", chip->member); \
  347. } \
  348. \
  349. static struct freq_attr throttle_attr_##name = __ATTR_RO(name) \
  350. throttle_attr(unthrottle, reason[NO_THROTTLE]);
  351. throttle_attr(powercap, reason[POWERCAP]);
  352. throttle_attr(overtemp, reason[CPU_OVERTEMP]);
  353. throttle_attr(supply_fault, reason[POWER_SUPPLY_FAILURE]);
  354. throttle_attr(overcurrent, reason[OVERCURRENT]);
  355. throttle_attr(occ_reset, reason[OCC_RESET_THROTTLE]);
  356. throttle_attr(turbo_stat, throttle_turbo);
  357. throttle_attr(sub_turbo_stat, throttle_sub_turbo);
  358. static struct attribute *throttle_attrs[] = {
  359. &throttle_attr_unthrottle.attr,
  360. &throttle_attr_powercap.attr,
  361. &throttle_attr_overtemp.attr,
  362. &throttle_attr_supply_fault.attr,
  363. &throttle_attr_overcurrent.attr,
  364. &throttle_attr_occ_reset.attr,
  365. &throttle_attr_turbo_stat.attr,
  366. &throttle_attr_sub_turbo_stat.attr,
  367. NULL,
  368. };
  369. static const struct attribute_group throttle_attr_grp = {
  370. .name = "throttle_stats",
  371. .attrs = throttle_attrs,
  372. };
  373. /* Helper routines */
  374. /* Access helpers to power mgt SPR */
  375. static inline unsigned long get_pmspr(unsigned long sprn)
  376. {
  377. switch (sprn) {
  378. case SPRN_PMCR:
  379. return mfspr(SPRN_PMCR);
  380. case SPRN_PMICR:
  381. return mfspr(SPRN_PMICR);
  382. case SPRN_PMSR:
  383. return mfspr(SPRN_PMSR);
  384. }
  385. BUG();
  386. }
  387. static inline void set_pmspr(unsigned long sprn, unsigned long val)
  388. {
  389. switch (sprn) {
  390. case SPRN_PMCR:
  391. mtspr(SPRN_PMCR, val);
  392. return;
  393. case SPRN_PMICR:
  394. mtspr(SPRN_PMICR, val);
  395. return;
  396. }
  397. BUG();
  398. }
  399. /*
  400. * Use objects of this type to query/update
  401. * pstates on a remote CPU via smp_call_function.
  402. */
  403. struct powernv_smp_call_data {
  404. unsigned int freq;
  405. u8 pstate_id;
  406. u8 gpstate_id;
  407. };
  408. /*
  409. * powernv_read_cpu_freq: Reads the current frequency on this CPU.
  410. *
  411. * Called via smp_call_function.
  412. *
  413. * Note: The caller of the smp_call_function should pass an argument of
  414. * the type 'struct powernv_smp_call_data *' along with this function.
  415. *
  416. * The current frequency on this CPU will be returned via
  417. * ((struct powernv_smp_call_data *)arg)->freq;
  418. */
  419. static void powernv_read_cpu_freq(void *arg)
  420. {
  421. unsigned long pmspr_val;
  422. struct powernv_smp_call_data *freq_data = arg;
  423. pmspr_val = get_pmspr(SPRN_PMSR);
  424. freq_data->pstate_id = extract_local_pstate(pmspr_val);
  425. freq_data->freq = pstate_id_to_freq(freq_data->pstate_id);
  426. pr_debug("cpu %d pmsr %016lX pstate_id 0x%x frequency %d kHz\n",
  427. raw_smp_processor_id(), pmspr_val, freq_data->pstate_id,
  428. freq_data->freq);
  429. }
  430. /*
  431. * powernv_cpufreq_get: Returns the CPU frequency as reported by the
  432. * firmware for CPU 'cpu'. This value is reported through the sysfs
  433. * file cpuinfo_cur_freq.
  434. */
  435. static unsigned int powernv_cpufreq_get(unsigned int cpu)
  436. {
  437. struct powernv_smp_call_data freq_data;
  438. smp_call_function_any(cpu_sibling_mask(cpu), powernv_read_cpu_freq,
  439. &freq_data, 1);
  440. return freq_data.freq;
  441. }
  442. /*
  443. * set_pstate: Sets the pstate on this CPU.
  444. *
  445. * This is called via an smp_call_function.
  446. *
  447. * The caller must ensure that freq_data is of the type
  448. * (struct powernv_smp_call_data *) and the pstate_id which needs to be set
  449. * on this CPU should be present in freq_data->pstate_id.
  450. */
  451. static void set_pstate(void *data)
  452. {
  453. unsigned long val;
  454. struct powernv_smp_call_data *freq_data = data;
  455. unsigned long pstate_ul = freq_data->pstate_id;
  456. unsigned long gpstate_ul = freq_data->gpstate_id;
  457. val = get_pmspr(SPRN_PMCR);
  458. val = val & 0x0000FFFFFFFFFFFFULL;
  459. pstate_ul = pstate_ul & 0xFF;
  460. gpstate_ul = gpstate_ul & 0xFF;
  461. /* Set both global(bits 56..63) and local(bits 48..55) PStates */
  462. val = val | (gpstate_ul << 56) | (pstate_ul << 48);
  463. pr_debug("Setting cpu %d pmcr to %016lX\n",
  464. raw_smp_processor_id(), val);
  465. set_pmspr(SPRN_PMCR, val);
  466. }
  467. /*
  468. * get_nominal_index: Returns the index corresponding to the nominal
  469. * pstate in the cpufreq table
  470. */
  471. static inline unsigned int get_nominal_index(void)
  472. {
  473. return powernv_pstate_info.nominal;
  474. }
  475. static void powernv_cpufreq_throttle_check(void *data)
  476. {
  477. struct chip *chip;
  478. unsigned int cpu = smp_processor_id();
  479. unsigned long pmsr;
  480. u8 pmsr_pmax;
  481. unsigned int pmsr_pmax_idx;
  482. pmsr = get_pmspr(SPRN_PMSR);
  483. chip = this_cpu_read(chip_info);
  484. /* Check for Pmax Capping */
  485. pmsr_pmax = extract_max_pstate(pmsr);
  486. pmsr_pmax_idx = pstate_to_idx(pmsr_pmax);
  487. if (pmsr_pmax_idx != powernv_pstate_info.max) {
  488. if (chip->throttled)
  489. goto next;
  490. chip->throttled = true;
  491. if (pmsr_pmax_idx > powernv_pstate_info.nominal) {
  492. pr_warn_once("CPU %d on Chip %u has Pmax(0x%x) reduced below that of nominal frequency(0x%x)\n",
  493. cpu, chip->id, pmsr_pmax,
  494. idx_to_pstate(powernv_pstate_info.nominal));
  495. chip->throttle_sub_turbo++;
  496. } else {
  497. chip->throttle_turbo++;
  498. }
  499. trace_powernv_throttle(chip->id,
  500. throttle_reason[chip->throttle_reason],
  501. pmsr_pmax);
  502. } else if (chip->throttled) {
  503. chip->throttled = false;
  504. trace_powernv_throttle(chip->id,
  505. throttle_reason[chip->throttle_reason],
  506. pmsr_pmax);
  507. }
  508. /* Check if Psafe_mode_active is set in PMSR. */
  509. next:
  510. if (pmsr & PMSR_PSAFE_ENABLE) {
  511. throttled = true;
  512. pr_info("Pstate set to safe frequency\n");
  513. }
  514. /* Check if SPR_EM_DISABLE is set in PMSR */
  515. if (pmsr & PMSR_SPR_EM_DISABLE) {
  516. throttled = true;
  517. pr_info("Frequency Control disabled from OS\n");
  518. }
  519. if (throttled) {
  520. pr_info("PMSR = %16lx\n", pmsr);
  521. pr_warn("CPU Frequency could be throttled\n");
  522. }
  523. }
  524. /**
  525. * calc_global_pstate - Calculate global pstate
  526. * @elapsed_time: Elapsed time in milliseconds
  527. * @local_pstate_idx: New local pstate
  528. * @highest_lpstate_idx: pstate from which its ramping down
  529. *
  530. * Finds the appropriate global pstate based on the pstate from which its
  531. * ramping down and the time elapsed in ramping down. It follows a quadratic
  532. * equation which ensures that it reaches ramping down to pmin in 5sec.
  533. */
  534. static inline int calc_global_pstate(unsigned int elapsed_time,
  535. int highest_lpstate_idx,
  536. int local_pstate_idx)
  537. {
  538. int index_diff;
  539. /*
  540. * Using ramp_down_percent we get the percentage of rampdown
  541. * that we are expecting to be dropping. Difference between
  542. * highest_lpstate_idx and powernv_pstate_info.min will give a absolute
  543. * number of how many pstates we will drop eventually by the end of
  544. * 5 seconds, then just scale it get the number pstates to be dropped.
  545. */
  546. index_diff = ((int)ramp_down_percent(elapsed_time) *
  547. (powernv_pstate_info.min - highest_lpstate_idx)) / 100;
  548. /* Ensure that global pstate is >= to local pstate */
  549. if (highest_lpstate_idx + index_diff >= local_pstate_idx)
  550. return local_pstate_idx;
  551. else
  552. return highest_lpstate_idx + index_diff;
  553. }
  554. static inline void queue_gpstate_timer(struct global_pstate_info *gpstates)
  555. {
  556. unsigned int timer_interval;
  557. /*
  558. * Setting up timer to fire after GPSTATE_TIMER_INTERVAL ms, But
  559. * if it exceeds MAX_RAMP_DOWN_TIME ms for ramp down time.
  560. * Set timer such that it fires exactly at MAX_RAMP_DOWN_TIME
  561. * seconds of ramp down time.
  562. */
  563. if ((gpstates->elapsed_time + GPSTATE_TIMER_INTERVAL)
  564. > MAX_RAMP_DOWN_TIME)
  565. timer_interval = MAX_RAMP_DOWN_TIME - gpstates->elapsed_time;
  566. else
  567. timer_interval = GPSTATE_TIMER_INTERVAL;
  568. mod_timer(&gpstates->timer, jiffies + msecs_to_jiffies(timer_interval));
  569. }
  570. /**
  571. * gpstate_timer_handler
  572. *
  573. * @t: Timer context used to fetch global pstate info struct
  574. *
  575. * This handler brings down the global pstate closer to the local pstate
  576. * according quadratic equation. Queues a new timer if it is still not equal
  577. * to local pstate
  578. */
  579. static void gpstate_timer_handler(struct timer_list *t)
  580. {
  581. struct global_pstate_info *gpstates = from_timer(gpstates, t, timer);
  582. struct cpufreq_policy *policy = gpstates->policy;
  583. int gpstate_idx, lpstate_idx;
  584. unsigned long val;
  585. unsigned int time_diff = jiffies_to_msecs(jiffies)
  586. - gpstates->last_sampled_time;
  587. struct powernv_smp_call_data freq_data;
  588. if (!spin_trylock(&gpstates->gpstate_lock))
  589. return;
  590. /*
  591. * If the timer has migrated to the different cpu then bring
  592. * it back to one of the policy->cpus
  593. */
  594. if (!cpumask_test_cpu(raw_smp_processor_id(), policy->cpus)) {
  595. gpstates->timer.expires = jiffies + msecs_to_jiffies(1);
  596. add_timer_on(&gpstates->timer, cpumask_first(policy->cpus));
  597. spin_unlock(&gpstates->gpstate_lock);
  598. return;
  599. }
  600. /*
  601. * If PMCR was last updated was using fast_swtich then
  602. * We may have wrong in gpstate->last_lpstate_idx
  603. * value. Hence, read from PMCR to get correct data.
  604. */
  605. val = get_pmspr(SPRN_PMCR);
  606. freq_data.gpstate_id = extract_global_pstate(val);
  607. freq_data.pstate_id = extract_local_pstate(val);
  608. if (freq_data.gpstate_id == freq_data.pstate_id) {
  609. reset_gpstates(policy);
  610. spin_unlock(&gpstates->gpstate_lock);
  611. return;
  612. }
  613. gpstates->last_sampled_time += time_diff;
  614. gpstates->elapsed_time += time_diff;
  615. if (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME) {
  616. gpstate_idx = pstate_to_idx(freq_data.pstate_id);
  617. lpstate_idx = gpstate_idx;
  618. reset_gpstates(policy);
  619. gpstates->highest_lpstate_idx = gpstate_idx;
  620. } else {
  621. lpstate_idx = pstate_to_idx(freq_data.pstate_id);
  622. gpstate_idx = calc_global_pstate(gpstates->elapsed_time,
  623. gpstates->highest_lpstate_idx,
  624. lpstate_idx);
  625. }
  626. freq_data.gpstate_id = idx_to_pstate(gpstate_idx);
  627. gpstates->last_gpstate_idx = gpstate_idx;
  628. gpstates->last_lpstate_idx = lpstate_idx;
  629. /*
  630. * If local pstate is equal to global pstate, rampdown is over
  631. * So timer is not required to be queued.
  632. */
  633. if (gpstate_idx != gpstates->last_lpstate_idx)
  634. queue_gpstate_timer(gpstates);
  635. set_pstate(&freq_data);
  636. spin_unlock(&gpstates->gpstate_lock);
  637. }
  638. /*
  639. * powernv_cpufreq_target_index: Sets the frequency corresponding to
  640. * the cpufreq table entry indexed by new_index on the cpus in the
  641. * mask policy->cpus
  642. */
  643. static int powernv_cpufreq_target_index(struct cpufreq_policy *policy,
  644. unsigned int new_index)
  645. {
  646. struct powernv_smp_call_data freq_data;
  647. unsigned int cur_msec, gpstate_idx;
  648. struct global_pstate_info *gpstates = policy->driver_data;
  649. if (unlikely(rebooting) && new_index != get_nominal_index())
  650. return 0;
  651. if (!throttled) {
  652. /* we don't want to be preempted while
  653. * checking if the CPU frequency has been throttled
  654. */
  655. preempt_disable();
  656. powernv_cpufreq_throttle_check(NULL);
  657. preempt_enable();
  658. }
  659. cur_msec = jiffies_to_msecs(get_jiffies_64());
  660. freq_data.pstate_id = idx_to_pstate(new_index);
  661. if (!gpstates) {
  662. freq_data.gpstate_id = freq_data.pstate_id;
  663. goto no_gpstate;
  664. }
  665. spin_lock(&gpstates->gpstate_lock);
  666. if (!gpstates->last_sampled_time) {
  667. gpstate_idx = new_index;
  668. gpstates->highest_lpstate_idx = new_index;
  669. goto gpstates_done;
  670. }
  671. if (gpstates->last_gpstate_idx < new_index) {
  672. gpstates->elapsed_time += cur_msec -
  673. gpstates->last_sampled_time;
  674. /*
  675. * If its has been ramping down for more than MAX_RAMP_DOWN_TIME
  676. * we should be resetting all global pstate related data. Set it
  677. * equal to local pstate to start fresh.
  678. */
  679. if (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME) {
  680. reset_gpstates(policy);
  681. gpstates->highest_lpstate_idx = new_index;
  682. gpstate_idx = new_index;
  683. } else {
  684. /* Elaspsed_time is less than 5 seconds, continue to rampdown */
  685. gpstate_idx = calc_global_pstate(gpstates->elapsed_time,
  686. gpstates->highest_lpstate_idx,
  687. new_index);
  688. }
  689. } else {
  690. reset_gpstates(policy);
  691. gpstates->highest_lpstate_idx = new_index;
  692. gpstate_idx = new_index;
  693. }
  694. /*
  695. * If local pstate is equal to global pstate, rampdown is over
  696. * So timer is not required to be queued.
  697. */
  698. if (gpstate_idx != new_index)
  699. queue_gpstate_timer(gpstates);
  700. else
  701. del_timer_sync(&gpstates->timer);
  702. gpstates_done:
  703. freq_data.gpstate_id = idx_to_pstate(gpstate_idx);
  704. gpstates->last_sampled_time = cur_msec;
  705. gpstates->last_gpstate_idx = gpstate_idx;
  706. gpstates->last_lpstate_idx = new_index;
  707. spin_unlock(&gpstates->gpstate_lock);
  708. no_gpstate:
  709. /*
  710. * Use smp_call_function to send IPI and execute the
  711. * mtspr on target CPU. We could do that without IPI
  712. * if current CPU is within policy->cpus (core)
  713. */
  714. smp_call_function_any(policy->cpus, set_pstate, &freq_data, 1);
  715. return 0;
  716. }
  717. static int powernv_cpufreq_cpu_init(struct cpufreq_policy *policy)
  718. {
  719. int base, i;
  720. struct kernfs_node *kn;
  721. struct global_pstate_info *gpstates;
  722. base = cpu_first_thread_sibling(policy->cpu);
  723. for (i = 0; i < threads_per_core; i++)
  724. cpumask_set_cpu(base + i, policy->cpus);
  725. kn = kernfs_find_and_get(policy->kobj.sd, throttle_attr_grp.name);
  726. if (!kn) {
  727. int ret;
  728. ret = sysfs_create_group(&policy->kobj, &throttle_attr_grp);
  729. if (ret) {
  730. pr_info("Failed to create throttle stats directory for cpu %d\n",
  731. policy->cpu);
  732. return ret;
  733. }
  734. } else {
  735. kernfs_put(kn);
  736. }
  737. policy->freq_table = powernv_freqs;
  738. policy->fast_switch_possible = true;
  739. if (pvr_version_is(PVR_POWER9))
  740. return 0;
  741. /* Initialise Gpstate ramp-down timer only on POWER8 */
  742. gpstates = kzalloc(sizeof(*gpstates), GFP_KERNEL);
  743. if (!gpstates)
  744. return -ENOMEM;
  745. policy->driver_data = gpstates;
  746. /* initialize timer */
  747. gpstates->policy = policy;
  748. timer_setup(&gpstates->timer, gpstate_timer_handler,
  749. TIMER_PINNED | TIMER_DEFERRABLE);
  750. gpstates->timer.expires = jiffies +
  751. msecs_to_jiffies(GPSTATE_TIMER_INTERVAL);
  752. spin_lock_init(&gpstates->gpstate_lock);
  753. return 0;
  754. }
  755. static int powernv_cpufreq_cpu_exit(struct cpufreq_policy *policy)
  756. {
  757. /* timer is deleted in cpufreq_cpu_stop() */
  758. kfree(policy->driver_data);
  759. return 0;
  760. }
  761. static int powernv_cpufreq_reboot_notifier(struct notifier_block *nb,
  762. unsigned long action, void *unused)
  763. {
  764. int cpu;
  765. struct cpufreq_policy *cpu_policy;
  766. rebooting = true;
  767. for_each_online_cpu(cpu) {
  768. cpu_policy = cpufreq_cpu_get(cpu);
  769. if (!cpu_policy)
  770. continue;
  771. powernv_cpufreq_target_index(cpu_policy, get_nominal_index());
  772. cpufreq_cpu_put(cpu_policy);
  773. }
  774. return NOTIFY_DONE;
  775. }
  776. static struct notifier_block powernv_cpufreq_reboot_nb = {
  777. .notifier_call = powernv_cpufreq_reboot_notifier,
  778. };
  779. static void powernv_cpufreq_work_fn(struct work_struct *work)
  780. {
  781. struct chip *chip = container_of(work, struct chip, throttle);
  782. struct cpufreq_policy *policy;
  783. unsigned int cpu;
  784. cpumask_t mask;
  785. get_online_cpus();
  786. cpumask_and(&mask, &chip->mask, cpu_online_mask);
  787. smp_call_function_any(&mask,
  788. powernv_cpufreq_throttle_check, NULL, 0);
  789. if (!chip->restore)
  790. goto out;
  791. chip->restore = false;
  792. for_each_cpu(cpu, &mask) {
  793. int index;
  794. policy = cpufreq_cpu_get(cpu);
  795. if (!policy)
  796. continue;
  797. index = cpufreq_table_find_index_c(policy, policy->cur);
  798. powernv_cpufreq_target_index(policy, index);
  799. cpumask_andnot(&mask, &mask, policy->cpus);
  800. cpufreq_cpu_put(policy);
  801. }
  802. out:
  803. put_online_cpus();
  804. }
  805. static int powernv_cpufreq_occ_msg(struct notifier_block *nb,
  806. unsigned long msg_type, void *_msg)
  807. {
  808. struct opal_msg *msg = _msg;
  809. struct opal_occ_msg omsg;
  810. int i;
  811. if (msg_type != OPAL_MSG_OCC)
  812. return 0;
  813. omsg.type = be64_to_cpu(msg->params[0]);
  814. switch (omsg.type) {
  815. case OCC_RESET:
  816. occ_reset = true;
  817. pr_info("OCC (On Chip Controller - enforces hard thermal/power limits) Resetting\n");
  818. /*
  819. * powernv_cpufreq_throttle_check() is called in
  820. * target() callback which can detect the throttle state
  821. * for governors like ondemand.
  822. * But static governors will not call target() often thus
  823. * report throttling here.
  824. */
  825. if (!throttled) {
  826. throttled = true;
  827. pr_warn("CPU frequency is throttled for duration\n");
  828. }
  829. break;
  830. case OCC_LOAD:
  831. pr_info("OCC Loading, CPU frequency is throttled until OCC is started\n");
  832. break;
  833. case OCC_THROTTLE:
  834. omsg.chip = be64_to_cpu(msg->params[1]);
  835. omsg.throttle_status = be64_to_cpu(msg->params[2]);
  836. if (occ_reset) {
  837. occ_reset = false;
  838. throttled = false;
  839. pr_info("OCC Active, CPU frequency is no longer throttled\n");
  840. for (i = 0; i < nr_chips; i++) {
  841. chips[i].restore = true;
  842. schedule_work(&chips[i].throttle);
  843. }
  844. return 0;
  845. }
  846. for (i = 0; i < nr_chips; i++)
  847. if (chips[i].id == omsg.chip)
  848. break;
  849. if (omsg.throttle_status >= 0 &&
  850. omsg.throttle_status <= OCC_MAX_THROTTLE_STATUS) {
  851. chips[i].throttle_reason = omsg.throttle_status;
  852. chips[i].reason[omsg.throttle_status]++;
  853. }
  854. if (!omsg.throttle_status)
  855. chips[i].restore = true;
  856. schedule_work(&chips[i].throttle);
  857. }
  858. return 0;
  859. }
  860. static struct notifier_block powernv_cpufreq_opal_nb = {
  861. .notifier_call = powernv_cpufreq_occ_msg,
  862. .next = NULL,
  863. .priority = 0,
  864. };
  865. static void powernv_cpufreq_stop_cpu(struct cpufreq_policy *policy)
  866. {
  867. struct powernv_smp_call_data freq_data;
  868. struct global_pstate_info *gpstates = policy->driver_data;
  869. freq_data.pstate_id = idx_to_pstate(powernv_pstate_info.min);
  870. freq_data.gpstate_id = idx_to_pstate(powernv_pstate_info.min);
  871. smp_call_function_single(policy->cpu, set_pstate, &freq_data, 1);
  872. if (gpstates)
  873. del_timer_sync(&gpstates->timer);
  874. }
  875. static unsigned int powernv_fast_switch(struct cpufreq_policy *policy,
  876. unsigned int target_freq)
  877. {
  878. int index;
  879. struct powernv_smp_call_data freq_data;
  880. index = cpufreq_table_find_index_dl(policy, target_freq);
  881. freq_data.pstate_id = powernv_freqs[index].driver_data;
  882. freq_data.gpstate_id = powernv_freqs[index].driver_data;
  883. set_pstate(&freq_data);
  884. return powernv_freqs[index].frequency;
  885. }
  886. static struct cpufreq_driver powernv_cpufreq_driver = {
  887. .name = "powernv-cpufreq",
  888. .flags = CPUFREQ_CONST_LOOPS,
  889. .init = powernv_cpufreq_cpu_init,
  890. .exit = powernv_cpufreq_cpu_exit,
  891. .verify = cpufreq_generic_frequency_table_verify,
  892. .target_index = powernv_cpufreq_target_index,
  893. .fast_switch = powernv_fast_switch,
  894. .get = powernv_cpufreq_get,
  895. .stop_cpu = powernv_cpufreq_stop_cpu,
  896. .attr = powernv_cpu_freq_attr,
  897. };
  898. static int init_chip_info(void)
  899. {
  900. unsigned int *chip;
  901. unsigned int cpu, i;
  902. unsigned int prev_chip_id = UINT_MAX;
  903. cpumask_t *chip_cpu_mask;
  904. int ret = 0;
  905. chip = kcalloc(num_possible_cpus(), sizeof(*chip), GFP_KERNEL);
  906. if (!chip)
  907. return -ENOMEM;
  908. /* Allocate a chip cpu mask large enough to fit mask for all chips */
  909. chip_cpu_mask = kcalloc(MAX_NR_CHIPS, sizeof(cpumask_t), GFP_KERNEL);
  910. if (!chip_cpu_mask) {
  911. ret = -ENOMEM;
  912. goto free_and_return;
  913. }
  914. for_each_possible_cpu(cpu) {
  915. unsigned int id = cpu_to_chip_id(cpu);
  916. if (prev_chip_id != id) {
  917. prev_chip_id = id;
  918. chip[nr_chips++] = id;
  919. }
  920. cpumask_set_cpu(cpu, &chip_cpu_mask[nr_chips-1]);
  921. }
  922. chips = kcalloc(nr_chips, sizeof(struct chip), GFP_KERNEL);
  923. if (!chips) {
  924. ret = -ENOMEM;
  925. goto out_free_chip_cpu_mask;
  926. }
  927. for (i = 0; i < nr_chips; i++) {
  928. chips[i].id = chip[i];
  929. cpumask_copy(&chips[i].mask, &chip_cpu_mask[i]);
  930. INIT_WORK(&chips[i].throttle, powernv_cpufreq_work_fn);
  931. for_each_cpu(cpu, &chips[i].mask)
  932. per_cpu(chip_info, cpu) = &chips[i];
  933. }
  934. out_free_chip_cpu_mask:
  935. kfree(chip_cpu_mask);
  936. free_and_return:
  937. kfree(chip);
  938. return ret;
  939. }
  940. static inline void clean_chip_info(void)
  941. {
  942. int i;
  943. /* flush any pending work items */
  944. if (chips)
  945. for (i = 0; i < nr_chips; i++)
  946. cancel_work_sync(&chips[i].throttle);
  947. kfree(chips);
  948. }
  949. static inline void unregister_all_notifiers(void)
  950. {
  951. opal_message_notifier_unregister(OPAL_MSG_OCC,
  952. &powernv_cpufreq_opal_nb);
  953. unregister_reboot_notifier(&powernv_cpufreq_reboot_nb);
  954. }
  955. static int __init powernv_cpufreq_init(void)
  956. {
  957. int rc = 0;
  958. /* Don't probe on pseries (guest) platforms */
  959. if (!firmware_has_feature(FW_FEATURE_OPAL))
  960. return -ENODEV;
  961. /* Discover pstates from device tree and init */
  962. rc = init_powernv_pstates();
  963. if (rc)
  964. goto out;
  965. /* Populate chip info */
  966. rc = init_chip_info();
  967. if (rc)
  968. goto out;
  969. if (powernv_pstate_info.wof_enabled)
  970. powernv_cpufreq_driver.boost_enabled = true;
  971. else
  972. powernv_cpu_freq_attr[SCALING_BOOST_FREQS_ATTR_INDEX] = NULL;
  973. rc = cpufreq_register_driver(&powernv_cpufreq_driver);
  974. if (rc) {
  975. pr_info("Failed to register the cpufreq driver (%d)\n", rc);
  976. goto cleanup;
  977. }
  978. if (powernv_pstate_info.wof_enabled)
  979. cpufreq_enable_boost_support();
  980. register_reboot_notifier(&powernv_cpufreq_reboot_nb);
  981. opal_message_notifier_register(OPAL_MSG_OCC, &powernv_cpufreq_opal_nb);
  982. return 0;
  983. cleanup:
  984. clean_chip_info();
  985. out:
  986. pr_info("Platform driver disabled. System does not support PState control\n");
  987. return rc;
  988. }
  989. module_init(powernv_cpufreq_init);
  990. static void __exit powernv_cpufreq_exit(void)
  991. {
  992. cpufreq_unregister_driver(&powernv_cpufreq_driver);
  993. unregister_all_notifiers();
  994. clean_chip_info();
  995. }
  996. module_exit(powernv_cpufreq_exit);
  997. MODULE_LICENSE("GPL");
  998. MODULE_AUTHOR("Vaidyanathan Srinivasan <svaidy at linux.vnet.ibm.com>");