intel_pstate.c 78 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * intel_pstate.c: Native P state management for Intel processors
  4. *
  5. * (C) Copyright 2012 Intel Corporation
  6. * Author: Dirk Brandewie <dirk.j.brandewie@intel.com>
  7. */
  8. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  9. #include <linux/kernel.h>
  10. #include <linux/kernel_stat.h>
  11. #include <linux/module.h>
  12. #include <linux/ktime.h>
  13. #include <linux/hrtimer.h>
  14. #include <linux/tick.h>
  15. #include <linux/slab.h>
  16. #include <linux/sched/cpufreq.h>
  17. #include <linux/list.h>
  18. #include <linux/cpu.h>
  19. #include <linux/cpufreq.h>
  20. #include <linux/sysfs.h>
  21. #include <linux/types.h>
  22. #include <linux/fs.h>
  23. #include <linux/acpi.h>
  24. #include <linux/vmalloc.h>
  25. #include <linux/pm_qos.h>
  26. #include <trace/events/power.h>
  27. #include <asm/div64.h>
  28. #include <asm/msr.h>
  29. #include <asm/cpu_device_id.h>
  30. #include <asm/cpufeature.h>
  31. #include <asm/intel-family.h>
  32. #define INTEL_PSTATE_SAMPLING_INTERVAL (10 * NSEC_PER_MSEC)
  33. #define INTEL_CPUFREQ_TRANSITION_LATENCY 20000
  34. #define INTEL_CPUFREQ_TRANSITION_DELAY_HWP 5000
  35. #define INTEL_CPUFREQ_TRANSITION_DELAY 500
  36. #ifdef CONFIG_ACPI
  37. #include <acpi/processor.h>
  38. #include <acpi/cppc_acpi.h>
  39. #endif
  40. #define FRAC_BITS 8
  41. #define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
  42. #define fp_toint(X) ((X) >> FRAC_BITS)
  43. #define ONE_EIGHTH_FP ((int64_t)1 << (FRAC_BITS - 3))
  44. #define EXT_BITS 6
  45. #define EXT_FRAC_BITS (EXT_BITS + FRAC_BITS)
  46. #define fp_ext_toint(X) ((X) >> EXT_FRAC_BITS)
  47. #define int_ext_tofp(X) ((int64_t)(X) << EXT_FRAC_BITS)
  48. static inline int32_t mul_fp(int32_t x, int32_t y)
  49. {
  50. return ((int64_t)x * (int64_t)y) >> FRAC_BITS;
  51. }
  52. static inline int32_t div_fp(s64 x, s64 y)
  53. {
  54. return div64_s64((int64_t)x << FRAC_BITS, y);
  55. }
  56. static inline int ceiling_fp(int32_t x)
  57. {
  58. int mask, ret;
  59. ret = fp_toint(x);
  60. mask = (1 << FRAC_BITS) - 1;
  61. if (x & mask)
  62. ret += 1;
  63. return ret;
  64. }
  65. static inline int32_t percent_fp(int percent)
  66. {
  67. return div_fp(percent, 100);
  68. }
  69. static inline u64 mul_ext_fp(u64 x, u64 y)
  70. {
  71. return (x * y) >> EXT_FRAC_BITS;
  72. }
  73. static inline u64 div_ext_fp(u64 x, u64 y)
  74. {
  75. return div64_u64(x << EXT_FRAC_BITS, y);
  76. }
  77. static inline int32_t percent_ext_fp(int percent)
  78. {
  79. return div_ext_fp(percent, 100);
  80. }
  81. /**
  82. * struct sample - Store performance sample
  83. * @core_avg_perf: Ratio of APERF/MPERF which is the actual average
  84. * performance during last sample period
  85. * @busy_scaled: Scaled busy value which is used to calculate next
  86. * P state. This can be different than core_avg_perf
  87. * to account for cpu idle period
  88. * @aperf: Difference of actual performance frequency clock count
  89. * read from APERF MSR between last and current sample
  90. * @mperf: Difference of maximum performance frequency clock count
  91. * read from MPERF MSR between last and current sample
  92. * @tsc: Difference of time stamp counter between last and
  93. * current sample
  94. * @time: Current time from scheduler
  95. *
  96. * This structure is used in the cpudata structure to store performance sample
  97. * data for choosing next P State.
  98. */
  99. struct sample {
  100. int32_t core_avg_perf;
  101. int32_t busy_scaled;
  102. u64 aperf;
  103. u64 mperf;
  104. u64 tsc;
  105. u64 time;
  106. };
  107. /**
  108. * struct pstate_data - Store P state data
  109. * @current_pstate: Current requested P state
  110. * @min_pstate: Min P state possible for this platform
  111. * @max_pstate: Max P state possible for this platform
  112. * @max_pstate_physical:This is physical Max P state for a processor
  113. * This can be higher than the max_pstate which can
  114. * be limited by platform thermal design power limits
  115. * @scaling: Scaling factor to convert frequency to cpufreq
  116. * frequency units
  117. * @turbo_pstate: Max Turbo P state possible for this platform
  118. * @max_freq: @max_pstate frequency in cpufreq units
  119. * @turbo_freq: @turbo_pstate frequency in cpufreq units
  120. *
  121. * Stores the per cpu model P state limits and current P state.
  122. */
  123. struct pstate_data {
  124. int current_pstate;
  125. int min_pstate;
  126. int max_pstate;
  127. int max_pstate_physical;
  128. int scaling;
  129. int turbo_pstate;
  130. unsigned int max_freq;
  131. unsigned int turbo_freq;
  132. };
  133. /**
  134. * struct vid_data - Stores voltage information data
  135. * @min: VID data for this platform corresponding to
  136. * the lowest P state
  137. * @max: VID data corresponding to the highest P State.
  138. * @turbo: VID data for turbo P state
  139. * @ratio: Ratio of (vid max - vid min) /
  140. * (max P state - Min P State)
  141. *
  142. * Stores the voltage data for DVFS (Dynamic Voltage and Frequency Scaling)
  143. * This data is used in Atom platforms, where in addition to target P state,
  144. * the voltage data needs to be specified to select next P State.
  145. */
  146. struct vid_data {
  147. int min;
  148. int max;
  149. int turbo;
  150. int32_t ratio;
  151. };
  152. /**
  153. * struct global_params - Global parameters, mostly tunable via sysfs.
  154. * @no_turbo: Whether or not to use turbo P-states.
  155. * @turbo_disabled: Whether or not turbo P-states are available at all,
  156. * based on the MSR_IA32_MISC_ENABLE value and whether or
  157. * not the maximum reported turbo P-state is different from
  158. * the maximum reported non-turbo one.
  159. * @turbo_disabled_mf: The @turbo_disabled value reflected by cpuinfo.max_freq.
  160. * @min_perf_pct: Minimum capacity limit in percent of the maximum turbo
  161. * P-state capacity.
  162. * @max_perf_pct: Maximum capacity limit in percent of the maximum turbo
  163. * P-state capacity.
  164. */
  165. struct global_params {
  166. bool no_turbo;
  167. bool turbo_disabled;
  168. bool turbo_disabled_mf;
  169. int max_perf_pct;
  170. int min_perf_pct;
  171. };
  172. /**
  173. * struct cpudata - Per CPU instance data storage
  174. * @cpu: CPU number for this instance data
  175. * @policy: CPUFreq policy value
  176. * @update_util: CPUFreq utility callback information
  177. * @update_util_set: CPUFreq utility callback is set
  178. * @iowait_boost: iowait-related boost fraction
  179. * @last_update: Time of the last update.
  180. * @pstate: Stores P state limits for this CPU
  181. * @vid: Stores VID limits for this CPU
  182. * @last_sample_time: Last Sample time
  183. * @aperf_mperf_shift: APERF vs MPERF counting frequency difference
  184. * @prev_aperf: Last APERF value read from APERF MSR
  185. * @prev_mperf: Last MPERF value read from MPERF MSR
  186. * @prev_tsc: Last timestamp counter (TSC) value
  187. * @prev_cummulative_iowait: IO Wait time difference from last and
  188. * current sample
  189. * @sample: Storage for storing last Sample data
  190. * @min_perf_ratio: Minimum capacity in terms of PERF or HWP ratios
  191. * @max_perf_ratio: Maximum capacity in terms of PERF or HWP ratios
  192. * @acpi_perf_data: Stores ACPI perf information read from _PSS
  193. * @valid_pss_table: Set to true for valid ACPI _PSS entries found
  194. * @epp_powersave: Last saved HWP energy performance preference
  195. * (EPP) or energy performance bias (EPB),
  196. * when policy switched to performance
  197. * @epp_policy: Last saved policy used to set EPP/EPB
  198. * @epp_default: Power on default HWP energy performance
  199. * preference/bias
  200. * @epp_cached Cached HWP energy-performance preference value
  201. * @hwp_req_cached: Cached value of the last HWP Request MSR
  202. * @hwp_cap_cached: Cached value of the last HWP Capabilities MSR
  203. * @last_io_update: Last time when IO wake flag was set
  204. * @sched_flags: Store scheduler flags for possible cross CPU update
  205. * @hwp_boost_min: Last HWP boosted min performance
  206. * @suspended: Whether or not the driver has been suspended.
  207. *
  208. * This structure stores per CPU instance data for all CPUs.
  209. */
  210. struct cpudata {
  211. int cpu;
  212. unsigned int policy;
  213. struct update_util_data update_util;
  214. bool update_util_set;
  215. struct pstate_data pstate;
  216. struct vid_data vid;
  217. u64 last_update;
  218. u64 last_sample_time;
  219. u64 aperf_mperf_shift;
  220. u64 prev_aperf;
  221. u64 prev_mperf;
  222. u64 prev_tsc;
  223. u64 prev_cummulative_iowait;
  224. struct sample sample;
  225. int32_t min_perf_ratio;
  226. int32_t max_perf_ratio;
  227. #ifdef CONFIG_ACPI
  228. struct acpi_processor_performance acpi_perf_data;
  229. bool valid_pss_table;
  230. #endif
  231. unsigned int iowait_boost;
  232. s16 epp_powersave;
  233. s16 epp_policy;
  234. s16 epp_default;
  235. s16 epp_cached;
  236. u64 hwp_req_cached;
  237. u64 hwp_cap_cached;
  238. u64 last_io_update;
  239. unsigned int sched_flags;
  240. u32 hwp_boost_min;
  241. bool suspended;
  242. };
  243. static struct cpudata **all_cpu_data;
  244. /**
  245. * struct pstate_funcs - Per CPU model specific callbacks
  246. * @get_max: Callback to get maximum non turbo effective P state
  247. * @get_max_physical: Callback to get maximum non turbo physical P state
  248. * @get_min: Callback to get minimum P state
  249. * @get_turbo: Callback to get turbo P state
  250. * @get_scaling: Callback to get frequency scaling factor
  251. * @get_aperf_mperf_shift: Callback to get the APERF vs MPERF frequency difference
  252. * @get_val: Callback to convert P state to actual MSR write value
  253. * @get_vid: Callback to get VID data for Atom platforms
  254. *
  255. * Core and Atom CPU models have different way to get P State limits. This
  256. * structure is used to store those callbacks.
  257. */
  258. struct pstate_funcs {
  259. int (*get_max)(void);
  260. int (*get_max_physical)(void);
  261. int (*get_min)(void);
  262. int (*get_turbo)(void);
  263. int (*get_scaling)(void);
  264. int (*get_aperf_mperf_shift)(void);
  265. u64 (*get_val)(struct cpudata*, int pstate);
  266. void (*get_vid)(struct cpudata *);
  267. };
  268. static struct pstate_funcs pstate_funcs __read_mostly;
  269. static int hwp_active __read_mostly;
  270. static int hwp_mode_bdw __read_mostly;
  271. static bool per_cpu_limits __read_mostly;
  272. static bool hwp_boost __read_mostly;
  273. static struct cpufreq_driver *intel_pstate_driver __read_mostly;
  274. #ifdef CONFIG_ACPI
  275. static bool acpi_ppc;
  276. #endif
  277. static struct global_params global;
  278. static DEFINE_MUTEX(intel_pstate_driver_lock);
  279. static DEFINE_MUTEX(intel_pstate_limits_lock);
  280. #ifdef CONFIG_ACPI
  281. static bool intel_pstate_acpi_pm_profile_server(void)
  282. {
  283. if (acpi_gbl_FADT.preferred_profile == PM_ENTERPRISE_SERVER ||
  284. acpi_gbl_FADT.preferred_profile == PM_PERFORMANCE_SERVER)
  285. return true;
  286. return false;
  287. }
  288. static bool intel_pstate_get_ppc_enable_status(void)
  289. {
  290. if (intel_pstate_acpi_pm_profile_server())
  291. return true;
  292. return acpi_ppc;
  293. }
  294. #ifdef CONFIG_ACPI_CPPC_LIB
  295. /* The work item is needed to avoid CPU hotplug locking issues */
  296. static void intel_pstste_sched_itmt_work_fn(struct work_struct *work)
  297. {
  298. sched_set_itmt_support();
  299. }
  300. static DECLARE_WORK(sched_itmt_work, intel_pstste_sched_itmt_work_fn);
  301. static void intel_pstate_set_itmt_prio(int cpu)
  302. {
  303. struct cppc_perf_caps cppc_perf;
  304. static u32 max_highest_perf = 0, min_highest_perf = U32_MAX;
  305. int ret;
  306. ret = cppc_get_perf_caps(cpu, &cppc_perf);
  307. if (ret)
  308. return;
  309. /*
  310. * The priorities can be set regardless of whether or not
  311. * sched_set_itmt_support(true) has been called and it is valid to
  312. * update them at any time after it has been called.
  313. */
  314. sched_set_itmt_core_prio(cppc_perf.highest_perf, cpu);
  315. if (max_highest_perf <= min_highest_perf) {
  316. if (cppc_perf.highest_perf > max_highest_perf)
  317. max_highest_perf = cppc_perf.highest_perf;
  318. if (cppc_perf.highest_perf < min_highest_perf)
  319. min_highest_perf = cppc_perf.highest_perf;
  320. if (max_highest_perf > min_highest_perf) {
  321. /*
  322. * This code can be run during CPU online under the
  323. * CPU hotplug locks, so sched_set_itmt_support()
  324. * cannot be called from here. Queue up a work item
  325. * to invoke it.
  326. */
  327. schedule_work(&sched_itmt_work);
  328. }
  329. }
  330. }
  331. static int intel_pstate_get_cppc_guranteed(int cpu)
  332. {
  333. struct cppc_perf_caps cppc_perf;
  334. int ret;
  335. ret = cppc_get_perf_caps(cpu, &cppc_perf);
  336. if (ret)
  337. return ret;
  338. if (cppc_perf.guaranteed_perf)
  339. return cppc_perf.guaranteed_perf;
  340. return cppc_perf.nominal_perf;
  341. }
  342. #else /* CONFIG_ACPI_CPPC_LIB */
  343. static void intel_pstate_set_itmt_prio(int cpu)
  344. {
  345. }
  346. #endif /* CONFIG_ACPI_CPPC_LIB */
  347. static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
  348. {
  349. struct cpudata *cpu;
  350. int ret;
  351. int i;
  352. if (hwp_active) {
  353. intel_pstate_set_itmt_prio(policy->cpu);
  354. return;
  355. }
  356. if (!intel_pstate_get_ppc_enable_status())
  357. return;
  358. cpu = all_cpu_data[policy->cpu];
  359. ret = acpi_processor_register_performance(&cpu->acpi_perf_data,
  360. policy->cpu);
  361. if (ret)
  362. return;
  363. /*
  364. * Check if the control value in _PSS is for PERF_CTL MSR, which should
  365. * guarantee that the states returned by it map to the states in our
  366. * list directly.
  367. */
  368. if (cpu->acpi_perf_data.control_register.space_id !=
  369. ACPI_ADR_SPACE_FIXED_HARDWARE)
  370. goto err;
  371. /*
  372. * If there is only one entry _PSS, simply ignore _PSS and continue as
  373. * usual without taking _PSS into account
  374. */
  375. if (cpu->acpi_perf_data.state_count < 2)
  376. goto err;
  377. pr_debug("CPU%u - ACPI _PSS perf data\n", policy->cpu);
  378. for (i = 0; i < cpu->acpi_perf_data.state_count; i++) {
  379. pr_debug(" %cP%d: %u MHz, %u mW, 0x%x\n",
  380. (i == cpu->acpi_perf_data.state ? '*' : ' '), i,
  381. (u32) cpu->acpi_perf_data.states[i].core_frequency,
  382. (u32) cpu->acpi_perf_data.states[i].power,
  383. (u32) cpu->acpi_perf_data.states[i].control);
  384. }
  385. /*
  386. * The _PSS table doesn't contain whole turbo frequency range.
  387. * This just contains +1 MHZ above the max non turbo frequency,
  388. * with control value corresponding to max turbo ratio. But
  389. * when cpufreq set policy is called, it will call with this
  390. * max frequency, which will cause a reduced performance as
  391. * this driver uses real max turbo frequency as the max
  392. * frequency. So correct this frequency in _PSS table to
  393. * correct max turbo frequency based on the turbo state.
  394. * Also need to convert to MHz as _PSS freq is in MHz.
  395. */
  396. if (!global.turbo_disabled)
  397. cpu->acpi_perf_data.states[0].core_frequency =
  398. policy->cpuinfo.max_freq / 1000;
  399. cpu->valid_pss_table = true;
  400. pr_debug("_PPC limits will be enforced\n");
  401. return;
  402. err:
  403. cpu->valid_pss_table = false;
  404. acpi_processor_unregister_performance(policy->cpu);
  405. }
  406. static void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
  407. {
  408. struct cpudata *cpu;
  409. cpu = all_cpu_data[policy->cpu];
  410. if (!cpu->valid_pss_table)
  411. return;
  412. acpi_processor_unregister_performance(policy->cpu);
  413. }
  414. #else /* CONFIG_ACPI */
  415. static inline void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy)
  416. {
  417. }
  418. static inline void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy)
  419. {
  420. }
  421. static inline bool intel_pstate_acpi_pm_profile_server(void)
  422. {
  423. return false;
  424. }
  425. #endif /* CONFIG_ACPI */
  426. #ifndef CONFIG_ACPI_CPPC_LIB
  427. static int intel_pstate_get_cppc_guranteed(int cpu)
  428. {
  429. return -ENOTSUPP;
  430. }
  431. #endif /* CONFIG_ACPI_CPPC_LIB */
  432. static inline void update_turbo_state(void)
  433. {
  434. u64 misc_en;
  435. struct cpudata *cpu;
  436. cpu = all_cpu_data[0];
  437. rdmsrl(MSR_IA32_MISC_ENABLE, misc_en);
  438. global.turbo_disabled =
  439. (misc_en & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ||
  440. cpu->pstate.max_pstate == cpu->pstate.turbo_pstate);
  441. }
  442. static int min_perf_pct_min(void)
  443. {
  444. struct cpudata *cpu = all_cpu_data[0];
  445. int turbo_pstate = cpu->pstate.turbo_pstate;
  446. return turbo_pstate ?
  447. (cpu->pstate.min_pstate * 100 / turbo_pstate) : 0;
  448. }
  449. static s16 intel_pstate_get_epb(struct cpudata *cpu_data)
  450. {
  451. u64 epb;
  452. int ret;
  453. if (!boot_cpu_has(X86_FEATURE_EPB))
  454. return -ENXIO;
  455. ret = rdmsrl_on_cpu(cpu_data->cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb);
  456. if (ret)
  457. return (s16)ret;
  458. return (s16)(epb & 0x0f);
  459. }
  460. static s16 intel_pstate_get_epp(struct cpudata *cpu_data, u64 hwp_req_data)
  461. {
  462. s16 epp;
  463. if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
  464. /*
  465. * When hwp_req_data is 0, means that caller didn't read
  466. * MSR_HWP_REQUEST, so need to read and get EPP.
  467. */
  468. if (!hwp_req_data) {
  469. epp = rdmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST,
  470. &hwp_req_data);
  471. if (epp)
  472. return epp;
  473. }
  474. epp = (hwp_req_data >> 24) & 0xff;
  475. } else {
  476. /* When there is no EPP present, HWP uses EPB settings */
  477. epp = intel_pstate_get_epb(cpu_data);
  478. }
  479. return epp;
  480. }
  481. static int intel_pstate_set_epb(int cpu, s16 pref)
  482. {
  483. u64 epb;
  484. int ret;
  485. if (!boot_cpu_has(X86_FEATURE_EPB))
  486. return -ENXIO;
  487. ret = rdmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb);
  488. if (ret)
  489. return ret;
  490. epb = (epb & ~0x0f) | pref;
  491. wrmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, epb);
  492. return 0;
  493. }
  494. /*
  495. * EPP/EPB display strings corresponding to EPP index in the
  496. * energy_perf_strings[]
  497. * index String
  498. *-------------------------------------
  499. * 0 default
  500. * 1 performance
  501. * 2 balance_performance
  502. * 3 balance_power
  503. * 4 power
  504. */
  505. static const char * const energy_perf_strings[] = {
  506. "default",
  507. "performance",
  508. "balance_performance",
  509. "balance_power",
  510. "power",
  511. NULL
  512. };
  513. static const unsigned int epp_values[] = {
  514. HWP_EPP_PERFORMANCE,
  515. HWP_EPP_BALANCE_PERFORMANCE,
  516. HWP_EPP_BALANCE_POWERSAVE,
  517. HWP_EPP_POWERSAVE
  518. };
  519. static int intel_pstate_get_energy_pref_index(struct cpudata *cpu_data, int *raw_epp)
  520. {
  521. s16 epp;
  522. int index = -EINVAL;
  523. *raw_epp = 0;
  524. epp = intel_pstate_get_epp(cpu_data, 0);
  525. if (epp < 0)
  526. return epp;
  527. if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
  528. if (epp == HWP_EPP_PERFORMANCE)
  529. return 1;
  530. if (epp == HWP_EPP_BALANCE_PERFORMANCE)
  531. return 2;
  532. if (epp == HWP_EPP_BALANCE_POWERSAVE)
  533. return 3;
  534. if (epp == HWP_EPP_POWERSAVE)
  535. return 4;
  536. *raw_epp = epp;
  537. return 0;
  538. } else if (boot_cpu_has(X86_FEATURE_EPB)) {
  539. /*
  540. * Range:
  541. * 0x00-0x03 : Performance
  542. * 0x04-0x07 : Balance performance
  543. * 0x08-0x0B : Balance power
  544. * 0x0C-0x0F : Power
  545. * The EPB is a 4 bit value, but our ranges restrict the
  546. * value which can be set. Here only using top two bits
  547. * effectively.
  548. */
  549. index = (epp >> 2) + 1;
  550. }
  551. return index;
  552. }
  553. static int intel_pstate_set_epp(struct cpudata *cpu, u32 epp)
  554. {
  555. int ret;
  556. /*
  557. * Use the cached HWP Request MSR value, because in the active mode the
  558. * register itself may be updated by intel_pstate_hwp_boost_up() or
  559. * intel_pstate_hwp_boost_down() at any time.
  560. */
  561. u64 value = READ_ONCE(cpu->hwp_req_cached);
  562. value &= ~GENMASK_ULL(31, 24);
  563. value |= (u64)epp << 24;
  564. /*
  565. * The only other updater of hwp_req_cached in the active mode,
  566. * intel_pstate_hwp_set(), is called under the same lock as this
  567. * function, so it cannot run in parallel with the update below.
  568. */
  569. WRITE_ONCE(cpu->hwp_req_cached, value);
  570. ret = wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
  571. if (!ret)
  572. cpu->epp_cached = epp;
  573. return ret;
  574. }
  575. static int intel_pstate_set_energy_pref_index(struct cpudata *cpu_data,
  576. int pref_index, bool use_raw,
  577. u32 raw_epp)
  578. {
  579. int epp = -EINVAL;
  580. int ret;
  581. if (!pref_index)
  582. epp = cpu_data->epp_default;
  583. if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
  584. if (use_raw)
  585. epp = raw_epp;
  586. else if (epp == -EINVAL)
  587. epp = epp_values[pref_index - 1];
  588. /*
  589. * To avoid confusion, refuse to set EPP to any values different
  590. * from 0 (performance) if the current policy is "performance",
  591. * because those values would be overridden.
  592. */
  593. if (epp > 0 && cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE)
  594. return -EBUSY;
  595. ret = intel_pstate_set_epp(cpu_data, epp);
  596. } else {
  597. if (epp == -EINVAL)
  598. epp = (pref_index - 1) << 2;
  599. ret = intel_pstate_set_epb(cpu_data->cpu, epp);
  600. }
  601. return ret;
  602. }
  603. static ssize_t show_energy_performance_available_preferences(
  604. struct cpufreq_policy *policy, char *buf)
  605. {
  606. int i = 0;
  607. int ret = 0;
  608. while (energy_perf_strings[i] != NULL)
  609. ret += sprintf(&buf[ret], "%s ", energy_perf_strings[i++]);
  610. ret += sprintf(&buf[ret], "\n");
  611. return ret;
  612. }
  613. cpufreq_freq_attr_ro(energy_performance_available_preferences);
  614. static struct cpufreq_driver intel_pstate;
  615. static ssize_t store_energy_performance_preference(
  616. struct cpufreq_policy *policy, const char *buf, size_t count)
  617. {
  618. struct cpudata *cpu = all_cpu_data[policy->cpu];
  619. char str_preference[21];
  620. bool raw = false;
  621. ssize_t ret;
  622. u32 epp = 0;
  623. ret = sscanf(buf, "%20s", str_preference);
  624. if (ret != 1)
  625. return -EINVAL;
  626. ret = match_string(energy_perf_strings, -1, str_preference);
  627. if (ret < 0) {
  628. if (!boot_cpu_has(X86_FEATURE_HWP_EPP))
  629. return ret;
  630. ret = kstrtouint(buf, 10, &epp);
  631. if (ret)
  632. return ret;
  633. if (epp > 255)
  634. return -EINVAL;
  635. raw = true;
  636. }
  637. /*
  638. * This function runs with the policy R/W semaphore held, which
  639. * guarantees that the driver pointer will not change while it is
  640. * running.
  641. */
  642. if (!intel_pstate_driver)
  643. return -EAGAIN;
  644. mutex_lock(&intel_pstate_limits_lock);
  645. if (intel_pstate_driver == &intel_pstate) {
  646. ret = intel_pstate_set_energy_pref_index(cpu, ret, raw, epp);
  647. } else {
  648. /*
  649. * In the passive mode the governor needs to be stopped on the
  650. * target CPU before the EPP update and restarted after it,
  651. * which is super-heavy-weight, so make sure it is worth doing
  652. * upfront.
  653. */
  654. if (!raw)
  655. epp = ret ? epp_values[ret - 1] : cpu->epp_default;
  656. if (cpu->epp_cached != epp) {
  657. int err;
  658. cpufreq_stop_governor(policy);
  659. ret = intel_pstate_set_epp(cpu, epp);
  660. err = cpufreq_start_governor(policy);
  661. if (!ret)
  662. ret = err;
  663. }
  664. }
  665. mutex_unlock(&intel_pstate_limits_lock);
  666. return ret ?: count;
  667. }
  668. static ssize_t show_energy_performance_preference(
  669. struct cpufreq_policy *policy, char *buf)
  670. {
  671. struct cpudata *cpu_data = all_cpu_data[policy->cpu];
  672. int preference, raw_epp;
  673. preference = intel_pstate_get_energy_pref_index(cpu_data, &raw_epp);
  674. if (preference < 0)
  675. return preference;
  676. if (raw_epp)
  677. return sprintf(buf, "%d\n", raw_epp);
  678. else
  679. return sprintf(buf, "%s\n", energy_perf_strings[preference]);
  680. }
  681. cpufreq_freq_attr_rw(energy_performance_preference);
  682. static ssize_t show_base_frequency(struct cpufreq_policy *policy, char *buf)
  683. {
  684. struct cpudata *cpu;
  685. u64 cap;
  686. int ratio;
  687. ratio = intel_pstate_get_cppc_guranteed(policy->cpu);
  688. if (ratio <= 0) {
  689. rdmsrl_on_cpu(policy->cpu, MSR_HWP_CAPABILITIES, &cap);
  690. ratio = HWP_GUARANTEED_PERF(cap);
  691. }
  692. cpu = all_cpu_data[policy->cpu];
  693. return sprintf(buf, "%d\n", ratio * cpu->pstate.scaling);
  694. }
  695. cpufreq_freq_attr_ro(base_frequency);
  696. static struct freq_attr *hwp_cpufreq_attrs[] = {
  697. &energy_performance_preference,
  698. &energy_performance_available_preferences,
  699. &base_frequency,
  700. NULL,
  701. };
  702. static void intel_pstate_get_hwp_max(struct cpudata *cpu, int *phy_max,
  703. int *current_max)
  704. {
  705. u64 cap;
  706. rdmsrl_on_cpu(cpu->cpu, MSR_HWP_CAPABILITIES, &cap);
  707. WRITE_ONCE(cpu->hwp_cap_cached, cap);
  708. if (global.no_turbo || global.turbo_disabled)
  709. *current_max = HWP_GUARANTEED_PERF(cap);
  710. else
  711. *current_max = HWP_HIGHEST_PERF(cap);
  712. *phy_max = HWP_HIGHEST_PERF(cap);
  713. }
  714. static void intel_pstate_hwp_set(unsigned int cpu)
  715. {
  716. struct cpudata *cpu_data = all_cpu_data[cpu];
  717. int max, min;
  718. u64 value;
  719. s16 epp;
  720. max = cpu_data->max_perf_ratio;
  721. min = cpu_data->min_perf_ratio;
  722. if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE)
  723. min = max;
  724. rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value);
  725. value &= ~HWP_MIN_PERF(~0L);
  726. value |= HWP_MIN_PERF(min);
  727. value &= ~HWP_MAX_PERF(~0L);
  728. value |= HWP_MAX_PERF(max);
  729. if (cpu_data->epp_policy == cpu_data->policy)
  730. goto skip_epp;
  731. cpu_data->epp_policy = cpu_data->policy;
  732. if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) {
  733. epp = intel_pstate_get_epp(cpu_data, value);
  734. cpu_data->epp_powersave = epp;
  735. /* If EPP read was failed, then don't try to write */
  736. if (epp < 0)
  737. goto skip_epp;
  738. epp = 0;
  739. } else {
  740. /* skip setting EPP, when saved value is invalid */
  741. if (cpu_data->epp_powersave < 0)
  742. goto skip_epp;
  743. /*
  744. * No need to restore EPP when it is not zero. This
  745. * means:
  746. * - Policy is not changed
  747. * - user has manually changed
  748. * - Error reading EPB
  749. */
  750. epp = intel_pstate_get_epp(cpu_data, value);
  751. if (epp)
  752. goto skip_epp;
  753. epp = cpu_data->epp_powersave;
  754. }
  755. if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
  756. value &= ~GENMASK_ULL(31, 24);
  757. value |= (u64)epp << 24;
  758. } else {
  759. intel_pstate_set_epb(cpu, epp);
  760. }
  761. skip_epp:
  762. WRITE_ONCE(cpu_data->hwp_req_cached, value);
  763. wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value);
  764. }
  765. static void intel_pstate_hwp_offline(struct cpudata *cpu)
  766. {
  767. u64 value = READ_ONCE(cpu->hwp_req_cached);
  768. int min_perf;
  769. if (boot_cpu_has(X86_FEATURE_HWP_EPP)) {
  770. /*
  771. * In case the EPP has been set to "performance" by the
  772. * active mode "performance" scaling algorithm, replace that
  773. * temporary value with the cached EPP one.
  774. */
  775. value &= ~GENMASK_ULL(31, 24);
  776. value |= HWP_ENERGY_PERF_PREFERENCE(cpu->epp_cached);
  777. WRITE_ONCE(cpu->hwp_req_cached, value);
  778. }
  779. value &= ~GENMASK_ULL(31, 0);
  780. min_perf = HWP_LOWEST_PERF(cpu->hwp_cap_cached);
  781. /* Set hwp_max = hwp_min */
  782. value |= HWP_MAX_PERF(min_perf);
  783. value |= HWP_MIN_PERF(min_perf);
  784. /* Set EPP to min */
  785. if (boot_cpu_has(X86_FEATURE_HWP_EPP))
  786. value |= HWP_ENERGY_PERF_PREFERENCE(HWP_EPP_POWERSAVE);
  787. wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
  788. }
  789. #define POWER_CTL_EE_ENABLE 1
  790. #define POWER_CTL_EE_DISABLE 2
  791. static int power_ctl_ee_state;
  792. static void set_power_ctl_ee_state(bool input)
  793. {
  794. u64 power_ctl;
  795. mutex_lock(&intel_pstate_driver_lock);
  796. rdmsrl(MSR_IA32_POWER_CTL, power_ctl);
  797. if (input) {
  798. power_ctl &= ~BIT(MSR_IA32_POWER_CTL_BIT_EE);
  799. power_ctl_ee_state = POWER_CTL_EE_ENABLE;
  800. } else {
  801. power_ctl |= BIT(MSR_IA32_POWER_CTL_BIT_EE);
  802. power_ctl_ee_state = POWER_CTL_EE_DISABLE;
  803. }
  804. wrmsrl(MSR_IA32_POWER_CTL, power_ctl);
  805. mutex_unlock(&intel_pstate_driver_lock);
  806. }
  807. static void intel_pstate_hwp_enable(struct cpudata *cpudata);
  808. static void intel_pstate_hwp_reenable(struct cpudata *cpu)
  809. {
  810. intel_pstate_hwp_enable(cpu);
  811. wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, READ_ONCE(cpu->hwp_req_cached));
  812. }
  813. static int intel_pstate_suspend(struct cpufreq_policy *policy)
  814. {
  815. struct cpudata *cpu = all_cpu_data[policy->cpu];
  816. pr_debug("CPU %d suspending\n", cpu->cpu);
  817. cpu->suspended = true;
  818. return 0;
  819. }
  820. static int intel_pstate_resume(struct cpufreq_policy *policy)
  821. {
  822. struct cpudata *cpu = all_cpu_data[policy->cpu];
  823. pr_debug("CPU %d resuming\n", cpu->cpu);
  824. /* Only restore if the system default is changed */
  825. if (power_ctl_ee_state == POWER_CTL_EE_ENABLE)
  826. set_power_ctl_ee_state(true);
  827. else if (power_ctl_ee_state == POWER_CTL_EE_DISABLE)
  828. set_power_ctl_ee_state(false);
  829. if (cpu->suspended && hwp_active) {
  830. mutex_lock(&intel_pstate_limits_lock);
  831. /* Re-enable HWP, because "online" has not done that. */
  832. intel_pstate_hwp_reenable(cpu);
  833. mutex_unlock(&intel_pstate_limits_lock);
  834. }
  835. cpu->suspended = false;
  836. return 0;
  837. }
  838. static void intel_pstate_update_policies(void)
  839. {
  840. int cpu;
  841. for_each_possible_cpu(cpu)
  842. cpufreq_update_policy(cpu);
  843. }
  844. static void intel_pstate_update_max_freq(unsigned int cpu)
  845. {
  846. struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu);
  847. struct cpudata *cpudata;
  848. if (!policy)
  849. return;
  850. cpudata = all_cpu_data[cpu];
  851. policy->cpuinfo.max_freq = global.turbo_disabled_mf ?
  852. cpudata->pstate.max_freq : cpudata->pstate.turbo_freq;
  853. refresh_frequency_limits(policy);
  854. cpufreq_cpu_release(policy);
  855. }
  856. static void intel_pstate_update_limits(unsigned int cpu)
  857. {
  858. mutex_lock(&intel_pstate_driver_lock);
  859. update_turbo_state();
  860. /*
  861. * If turbo has been turned on or off globally, policy limits for
  862. * all CPUs need to be updated to reflect that.
  863. */
  864. if (global.turbo_disabled_mf != global.turbo_disabled) {
  865. global.turbo_disabled_mf = global.turbo_disabled;
  866. arch_set_max_freq_ratio(global.turbo_disabled);
  867. for_each_possible_cpu(cpu)
  868. intel_pstate_update_max_freq(cpu);
  869. } else {
  870. cpufreq_update_policy(cpu);
  871. }
  872. mutex_unlock(&intel_pstate_driver_lock);
  873. }
  874. /************************** sysfs begin ************************/
  875. #define show_one(file_name, object) \
  876. static ssize_t show_##file_name \
  877. (struct kobject *kobj, struct kobj_attribute *attr, char *buf) \
  878. { \
  879. return sprintf(buf, "%u\n", global.object); \
  880. }
  881. static ssize_t intel_pstate_show_status(char *buf);
  882. static int intel_pstate_update_status(const char *buf, size_t size);
  883. static ssize_t show_status(struct kobject *kobj,
  884. struct kobj_attribute *attr, char *buf)
  885. {
  886. ssize_t ret;
  887. mutex_lock(&intel_pstate_driver_lock);
  888. ret = intel_pstate_show_status(buf);
  889. mutex_unlock(&intel_pstate_driver_lock);
  890. return ret;
  891. }
  892. static ssize_t store_status(struct kobject *a, struct kobj_attribute *b,
  893. const char *buf, size_t count)
  894. {
  895. char *p = memchr(buf, '\n', count);
  896. int ret;
  897. mutex_lock(&intel_pstate_driver_lock);
  898. ret = intel_pstate_update_status(buf, p ? p - buf : count);
  899. mutex_unlock(&intel_pstate_driver_lock);
  900. return ret < 0 ? ret : count;
  901. }
  902. static ssize_t show_turbo_pct(struct kobject *kobj,
  903. struct kobj_attribute *attr, char *buf)
  904. {
  905. struct cpudata *cpu;
  906. int total, no_turbo, turbo_pct;
  907. uint32_t turbo_fp;
  908. mutex_lock(&intel_pstate_driver_lock);
  909. if (!intel_pstate_driver) {
  910. mutex_unlock(&intel_pstate_driver_lock);
  911. return -EAGAIN;
  912. }
  913. cpu = all_cpu_data[0];
  914. total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
  915. no_turbo = cpu->pstate.max_pstate - cpu->pstate.min_pstate + 1;
  916. turbo_fp = div_fp(no_turbo, total);
  917. turbo_pct = 100 - fp_toint(mul_fp(turbo_fp, int_tofp(100)));
  918. mutex_unlock(&intel_pstate_driver_lock);
  919. return sprintf(buf, "%u\n", turbo_pct);
  920. }
  921. static ssize_t show_num_pstates(struct kobject *kobj,
  922. struct kobj_attribute *attr, char *buf)
  923. {
  924. struct cpudata *cpu;
  925. int total;
  926. mutex_lock(&intel_pstate_driver_lock);
  927. if (!intel_pstate_driver) {
  928. mutex_unlock(&intel_pstate_driver_lock);
  929. return -EAGAIN;
  930. }
  931. cpu = all_cpu_data[0];
  932. total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1;
  933. mutex_unlock(&intel_pstate_driver_lock);
  934. return sprintf(buf, "%u\n", total);
  935. }
  936. static ssize_t show_no_turbo(struct kobject *kobj,
  937. struct kobj_attribute *attr, char *buf)
  938. {
  939. ssize_t ret;
  940. mutex_lock(&intel_pstate_driver_lock);
  941. if (!intel_pstate_driver) {
  942. mutex_unlock(&intel_pstate_driver_lock);
  943. return -EAGAIN;
  944. }
  945. update_turbo_state();
  946. if (global.turbo_disabled)
  947. ret = sprintf(buf, "%u\n", global.turbo_disabled);
  948. else
  949. ret = sprintf(buf, "%u\n", global.no_turbo);
  950. mutex_unlock(&intel_pstate_driver_lock);
  951. return ret;
  952. }
  953. static ssize_t store_no_turbo(struct kobject *a, struct kobj_attribute *b,
  954. const char *buf, size_t count)
  955. {
  956. unsigned int input;
  957. int ret;
  958. ret = sscanf(buf, "%u", &input);
  959. if (ret != 1)
  960. return -EINVAL;
  961. mutex_lock(&intel_pstate_driver_lock);
  962. if (!intel_pstate_driver) {
  963. mutex_unlock(&intel_pstate_driver_lock);
  964. return -EAGAIN;
  965. }
  966. mutex_lock(&intel_pstate_limits_lock);
  967. update_turbo_state();
  968. if (global.turbo_disabled) {
  969. pr_notice_once("Turbo disabled by BIOS or unavailable on processor\n");
  970. mutex_unlock(&intel_pstate_limits_lock);
  971. mutex_unlock(&intel_pstate_driver_lock);
  972. return -EPERM;
  973. }
  974. global.no_turbo = clamp_t(int, input, 0, 1);
  975. if (global.no_turbo) {
  976. struct cpudata *cpu = all_cpu_data[0];
  977. int pct = cpu->pstate.max_pstate * 100 / cpu->pstate.turbo_pstate;
  978. /* Squash the global minimum into the permitted range. */
  979. if (global.min_perf_pct > pct)
  980. global.min_perf_pct = pct;
  981. }
  982. mutex_unlock(&intel_pstate_limits_lock);
  983. intel_pstate_update_policies();
  984. mutex_unlock(&intel_pstate_driver_lock);
  985. return count;
  986. }
  987. static void update_qos_request(enum freq_qos_req_type type)
  988. {
  989. int max_state, turbo_max, freq, i, perf_pct;
  990. struct freq_qos_request *req;
  991. struct cpufreq_policy *policy;
  992. for_each_possible_cpu(i) {
  993. struct cpudata *cpu = all_cpu_data[i];
  994. policy = cpufreq_cpu_get(i);
  995. if (!policy)
  996. continue;
  997. req = policy->driver_data;
  998. cpufreq_cpu_put(policy);
  999. if (!req)
  1000. continue;
  1001. if (hwp_active)
  1002. intel_pstate_get_hwp_max(cpu, &turbo_max, &max_state);
  1003. else
  1004. turbo_max = cpu->pstate.turbo_pstate;
  1005. if (type == FREQ_QOS_MIN) {
  1006. perf_pct = global.min_perf_pct;
  1007. } else {
  1008. req++;
  1009. perf_pct = global.max_perf_pct;
  1010. }
  1011. freq = DIV_ROUND_UP(turbo_max * perf_pct, 100);
  1012. freq *= cpu->pstate.scaling;
  1013. if (freq_qos_update_request(req, freq) < 0)
  1014. pr_warn("Failed to update freq constraint: CPU%d\n", i);
  1015. }
  1016. }
  1017. static ssize_t store_max_perf_pct(struct kobject *a, struct kobj_attribute *b,
  1018. const char *buf, size_t count)
  1019. {
  1020. unsigned int input;
  1021. int ret;
  1022. ret = sscanf(buf, "%u", &input);
  1023. if (ret != 1)
  1024. return -EINVAL;
  1025. mutex_lock(&intel_pstate_driver_lock);
  1026. if (!intel_pstate_driver) {
  1027. mutex_unlock(&intel_pstate_driver_lock);
  1028. return -EAGAIN;
  1029. }
  1030. mutex_lock(&intel_pstate_limits_lock);
  1031. global.max_perf_pct = clamp_t(int, input, global.min_perf_pct, 100);
  1032. mutex_unlock(&intel_pstate_limits_lock);
  1033. if (intel_pstate_driver == &intel_pstate)
  1034. intel_pstate_update_policies();
  1035. else
  1036. update_qos_request(FREQ_QOS_MAX);
  1037. mutex_unlock(&intel_pstate_driver_lock);
  1038. return count;
  1039. }
  1040. static ssize_t store_min_perf_pct(struct kobject *a, struct kobj_attribute *b,
  1041. const char *buf, size_t count)
  1042. {
  1043. unsigned int input;
  1044. int ret;
  1045. ret = sscanf(buf, "%u", &input);
  1046. if (ret != 1)
  1047. return -EINVAL;
  1048. mutex_lock(&intel_pstate_driver_lock);
  1049. if (!intel_pstate_driver) {
  1050. mutex_unlock(&intel_pstate_driver_lock);
  1051. return -EAGAIN;
  1052. }
  1053. mutex_lock(&intel_pstate_limits_lock);
  1054. global.min_perf_pct = clamp_t(int, input,
  1055. min_perf_pct_min(), global.max_perf_pct);
  1056. mutex_unlock(&intel_pstate_limits_lock);
  1057. if (intel_pstate_driver == &intel_pstate)
  1058. intel_pstate_update_policies();
  1059. else
  1060. update_qos_request(FREQ_QOS_MIN);
  1061. mutex_unlock(&intel_pstate_driver_lock);
  1062. return count;
  1063. }
  1064. static ssize_t show_hwp_dynamic_boost(struct kobject *kobj,
  1065. struct kobj_attribute *attr, char *buf)
  1066. {
  1067. return sprintf(buf, "%u\n", hwp_boost);
  1068. }
  1069. static ssize_t store_hwp_dynamic_boost(struct kobject *a,
  1070. struct kobj_attribute *b,
  1071. const char *buf, size_t count)
  1072. {
  1073. unsigned int input;
  1074. int ret;
  1075. ret = kstrtouint(buf, 10, &input);
  1076. if (ret)
  1077. return ret;
  1078. mutex_lock(&intel_pstate_driver_lock);
  1079. hwp_boost = !!input;
  1080. intel_pstate_update_policies();
  1081. mutex_unlock(&intel_pstate_driver_lock);
  1082. return count;
  1083. }
  1084. static ssize_t show_energy_efficiency(struct kobject *kobj, struct kobj_attribute *attr,
  1085. char *buf)
  1086. {
  1087. u64 power_ctl;
  1088. int enable;
  1089. rdmsrl(MSR_IA32_POWER_CTL, power_ctl);
  1090. enable = !!(power_ctl & BIT(MSR_IA32_POWER_CTL_BIT_EE));
  1091. return sprintf(buf, "%d\n", !enable);
  1092. }
  1093. static ssize_t store_energy_efficiency(struct kobject *a, struct kobj_attribute *b,
  1094. const char *buf, size_t count)
  1095. {
  1096. bool input;
  1097. int ret;
  1098. ret = kstrtobool(buf, &input);
  1099. if (ret)
  1100. return ret;
  1101. set_power_ctl_ee_state(input);
  1102. return count;
  1103. }
  1104. show_one(max_perf_pct, max_perf_pct);
  1105. show_one(min_perf_pct, min_perf_pct);
  1106. define_one_global_rw(status);
  1107. define_one_global_rw(no_turbo);
  1108. define_one_global_rw(max_perf_pct);
  1109. define_one_global_rw(min_perf_pct);
  1110. define_one_global_ro(turbo_pct);
  1111. define_one_global_ro(num_pstates);
  1112. define_one_global_rw(hwp_dynamic_boost);
  1113. define_one_global_rw(energy_efficiency);
  1114. static struct attribute *intel_pstate_attributes[] = {
  1115. &status.attr,
  1116. &no_turbo.attr,
  1117. &turbo_pct.attr,
  1118. &num_pstates.attr,
  1119. NULL
  1120. };
  1121. static const struct attribute_group intel_pstate_attr_group = {
  1122. .attrs = intel_pstate_attributes,
  1123. };
  1124. static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[];
  1125. static struct kobject *intel_pstate_kobject;
  1126. static void __init intel_pstate_sysfs_expose_params(void)
  1127. {
  1128. int rc;
  1129. intel_pstate_kobject = kobject_create_and_add("intel_pstate",
  1130. &cpu_subsys.dev_root->kobj);
  1131. if (WARN_ON(!intel_pstate_kobject))
  1132. return;
  1133. rc = sysfs_create_group(intel_pstate_kobject, &intel_pstate_attr_group);
  1134. if (WARN_ON(rc))
  1135. return;
  1136. /*
  1137. * If per cpu limits are enforced there are no global limits, so
  1138. * return without creating max/min_perf_pct attributes
  1139. */
  1140. if (per_cpu_limits)
  1141. return;
  1142. rc = sysfs_create_file(intel_pstate_kobject, &max_perf_pct.attr);
  1143. WARN_ON(rc);
  1144. rc = sysfs_create_file(intel_pstate_kobject, &min_perf_pct.attr);
  1145. WARN_ON(rc);
  1146. if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids)) {
  1147. rc = sysfs_create_file(intel_pstate_kobject, &energy_efficiency.attr);
  1148. WARN_ON(rc);
  1149. }
  1150. }
  1151. static void __init intel_pstate_sysfs_remove(void)
  1152. {
  1153. if (!intel_pstate_kobject)
  1154. return;
  1155. sysfs_remove_group(intel_pstate_kobject, &intel_pstate_attr_group);
  1156. if (!per_cpu_limits) {
  1157. sysfs_remove_file(intel_pstate_kobject, &max_perf_pct.attr);
  1158. sysfs_remove_file(intel_pstate_kobject, &min_perf_pct.attr);
  1159. if (x86_match_cpu(intel_pstate_cpu_ee_disable_ids))
  1160. sysfs_remove_file(intel_pstate_kobject, &energy_efficiency.attr);
  1161. }
  1162. kobject_put(intel_pstate_kobject);
  1163. }
  1164. static void intel_pstate_sysfs_expose_hwp_dynamic_boost(void)
  1165. {
  1166. int rc;
  1167. if (!hwp_active)
  1168. return;
  1169. rc = sysfs_create_file(intel_pstate_kobject, &hwp_dynamic_boost.attr);
  1170. WARN_ON_ONCE(rc);
  1171. }
  1172. static void intel_pstate_sysfs_hide_hwp_dynamic_boost(void)
  1173. {
  1174. if (!hwp_active)
  1175. return;
  1176. sysfs_remove_file(intel_pstate_kobject, &hwp_dynamic_boost.attr);
  1177. }
  1178. /************************** sysfs end ************************/
  1179. static void intel_pstate_hwp_enable(struct cpudata *cpudata)
  1180. {
  1181. /* First disable HWP notification interrupt as we don't process them */
  1182. if (boot_cpu_has(X86_FEATURE_HWP_NOTIFY))
  1183. wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00);
  1184. wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1);
  1185. if (cpudata->epp_default == -EINVAL)
  1186. cpudata->epp_default = intel_pstate_get_epp(cpudata, 0);
  1187. }
  1188. static int atom_get_min_pstate(void)
  1189. {
  1190. u64 value;
  1191. rdmsrl(MSR_ATOM_CORE_RATIOS, value);
  1192. return (value >> 8) & 0x7F;
  1193. }
  1194. static int atom_get_max_pstate(void)
  1195. {
  1196. u64 value;
  1197. rdmsrl(MSR_ATOM_CORE_RATIOS, value);
  1198. return (value >> 16) & 0x7F;
  1199. }
  1200. static int atom_get_turbo_pstate(void)
  1201. {
  1202. u64 value;
  1203. rdmsrl(MSR_ATOM_CORE_TURBO_RATIOS, value);
  1204. return value & 0x7F;
  1205. }
  1206. static u64 atom_get_val(struct cpudata *cpudata, int pstate)
  1207. {
  1208. u64 val;
  1209. int32_t vid_fp;
  1210. u32 vid;
  1211. val = (u64)pstate << 8;
  1212. if (global.no_turbo && !global.turbo_disabled)
  1213. val |= (u64)1 << 32;
  1214. vid_fp = cpudata->vid.min + mul_fp(
  1215. int_tofp(pstate - cpudata->pstate.min_pstate),
  1216. cpudata->vid.ratio);
  1217. vid_fp = clamp_t(int32_t, vid_fp, cpudata->vid.min, cpudata->vid.max);
  1218. vid = ceiling_fp(vid_fp);
  1219. if (pstate > cpudata->pstate.max_pstate)
  1220. vid = cpudata->vid.turbo;
  1221. return val | vid;
  1222. }
  1223. static int silvermont_get_scaling(void)
  1224. {
  1225. u64 value;
  1226. int i;
  1227. /* Defined in Table 35-6 from SDM (Sept 2015) */
  1228. static int silvermont_freq_table[] = {
  1229. 83300, 100000, 133300, 116700, 80000};
  1230. rdmsrl(MSR_FSB_FREQ, value);
  1231. i = value & 0x7;
  1232. WARN_ON(i > 4);
  1233. return silvermont_freq_table[i];
  1234. }
  1235. static int airmont_get_scaling(void)
  1236. {
  1237. u64 value;
  1238. int i;
  1239. /* Defined in Table 35-10 from SDM (Sept 2015) */
  1240. static int airmont_freq_table[] = {
  1241. 83300, 100000, 133300, 116700, 80000,
  1242. 93300, 90000, 88900, 87500};
  1243. rdmsrl(MSR_FSB_FREQ, value);
  1244. i = value & 0xF;
  1245. WARN_ON(i > 8);
  1246. return airmont_freq_table[i];
  1247. }
  1248. static void atom_get_vid(struct cpudata *cpudata)
  1249. {
  1250. u64 value;
  1251. rdmsrl(MSR_ATOM_CORE_VIDS, value);
  1252. cpudata->vid.min = int_tofp((value >> 8) & 0x7f);
  1253. cpudata->vid.max = int_tofp((value >> 16) & 0x7f);
  1254. cpudata->vid.ratio = div_fp(
  1255. cpudata->vid.max - cpudata->vid.min,
  1256. int_tofp(cpudata->pstate.max_pstate -
  1257. cpudata->pstate.min_pstate));
  1258. rdmsrl(MSR_ATOM_CORE_TURBO_VIDS, value);
  1259. cpudata->vid.turbo = value & 0x7f;
  1260. }
  1261. static int core_get_min_pstate(void)
  1262. {
  1263. u64 value;
  1264. rdmsrl(MSR_PLATFORM_INFO, value);
  1265. return (value >> 40) & 0xFF;
  1266. }
  1267. static int core_get_max_pstate_physical(void)
  1268. {
  1269. u64 value;
  1270. rdmsrl(MSR_PLATFORM_INFO, value);
  1271. return (value >> 8) & 0xFF;
  1272. }
  1273. static int core_get_tdp_ratio(u64 plat_info)
  1274. {
  1275. /* Check how many TDP levels present */
  1276. if (plat_info & 0x600000000) {
  1277. u64 tdp_ctrl;
  1278. u64 tdp_ratio;
  1279. int tdp_msr;
  1280. int err;
  1281. /* Get the TDP level (0, 1, 2) to get ratios */
  1282. err = rdmsrl_safe(MSR_CONFIG_TDP_CONTROL, &tdp_ctrl);
  1283. if (err)
  1284. return err;
  1285. /* TDP MSR are continuous starting at 0x648 */
  1286. tdp_msr = MSR_CONFIG_TDP_NOMINAL + (tdp_ctrl & 0x03);
  1287. err = rdmsrl_safe(tdp_msr, &tdp_ratio);
  1288. if (err)
  1289. return err;
  1290. /* For level 1 and 2, bits[23:16] contain the ratio */
  1291. if (tdp_ctrl & 0x03)
  1292. tdp_ratio >>= 16;
  1293. tdp_ratio &= 0xff; /* ratios are only 8 bits long */
  1294. pr_debug("tdp_ratio %x\n", (int)tdp_ratio);
  1295. return (int)tdp_ratio;
  1296. }
  1297. return -ENXIO;
  1298. }
  1299. static int core_get_max_pstate(void)
  1300. {
  1301. u64 tar;
  1302. u64 plat_info;
  1303. int max_pstate;
  1304. int tdp_ratio;
  1305. int err;
  1306. rdmsrl(MSR_PLATFORM_INFO, plat_info);
  1307. max_pstate = (plat_info >> 8) & 0xFF;
  1308. tdp_ratio = core_get_tdp_ratio(plat_info);
  1309. if (tdp_ratio <= 0)
  1310. return max_pstate;
  1311. if (hwp_active) {
  1312. /* Turbo activation ratio is not used on HWP platforms */
  1313. return tdp_ratio;
  1314. }
  1315. err = rdmsrl_safe(MSR_TURBO_ACTIVATION_RATIO, &tar);
  1316. if (!err) {
  1317. int tar_levels;
  1318. /* Do some sanity checking for safety */
  1319. tar_levels = tar & 0xff;
  1320. if (tdp_ratio - 1 == tar_levels) {
  1321. max_pstate = tar_levels;
  1322. pr_debug("max_pstate=TAC %x\n", max_pstate);
  1323. }
  1324. }
  1325. return max_pstate;
  1326. }
  1327. static int core_get_turbo_pstate(void)
  1328. {
  1329. u64 value;
  1330. int nont, ret;
  1331. rdmsrl(MSR_TURBO_RATIO_LIMIT, value);
  1332. nont = core_get_max_pstate();
  1333. ret = (value) & 255;
  1334. if (ret <= nont)
  1335. ret = nont;
  1336. return ret;
  1337. }
  1338. static inline int core_get_scaling(void)
  1339. {
  1340. return 100000;
  1341. }
  1342. static u64 core_get_val(struct cpudata *cpudata, int pstate)
  1343. {
  1344. u64 val;
  1345. val = (u64)pstate << 8;
  1346. if (global.no_turbo && !global.turbo_disabled)
  1347. val |= (u64)1 << 32;
  1348. return val;
  1349. }
  1350. static int knl_get_aperf_mperf_shift(void)
  1351. {
  1352. return 10;
  1353. }
  1354. static int knl_get_turbo_pstate(void)
  1355. {
  1356. u64 value;
  1357. int nont, ret;
  1358. rdmsrl(MSR_TURBO_RATIO_LIMIT, value);
  1359. nont = core_get_max_pstate();
  1360. ret = (((value) >> 8) & 0xFF);
  1361. if (ret <= nont)
  1362. ret = nont;
  1363. return ret;
  1364. }
  1365. static void intel_pstate_set_pstate(struct cpudata *cpu, int pstate)
  1366. {
  1367. trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu);
  1368. cpu->pstate.current_pstate = pstate;
  1369. /*
  1370. * Generally, there is no guarantee that this code will always run on
  1371. * the CPU being updated, so force the register update to run on the
  1372. * right CPU.
  1373. */
  1374. wrmsrl_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL,
  1375. pstate_funcs.get_val(cpu, pstate));
  1376. }
  1377. static void intel_pstate_set_min_pstate(struct cpudata *cpu)
  1378. {
  1379. intel_pstate_set_pstate(cpu, cpu->pstate.min_pstate);
  1380. }
  1381. static void intel_pstate_max_within_limits(struct cpudata *cpu)
  1382. {
  1383. int pstate = max(cpu->pstate.min_pstate, cpu->max_perf_ratio);
  1384. update_turbo_state();
  1385. intel_pstate_set_pstate(cpu, pstate);
  1386. }
  1387. static void intel_pstate_get_cpu_pstates(struct cpudata *cpu)
  1388. {
  1389. cpu->pstate.min_pstate = pstate_funcs.get_min();
  1390. cpu->pstate.max_pstate_physical = pstate_funcs.get_max_physical();
  1391. cpu->pstate.turbo_pstate = pstate_funcs.get_turbo();
  1392. cpu->pstate.scaling = pstate_funcs.get_scaling();
  1393. if (hwp_active && !hwp_mode_bdw) {
  1394. unsigned int phy_max, current_max;
  1395. intel_pstate_get_hwp_max(cpu, &phy_max, &current_max);
  1396. cpu->pstate.turbo_freq = phy_max * cpu->pstate.scaling;
  1397. cpu->pstate.turbo_pstate = phy_max;
  1398. cpu->pstate.max_pstate = HWP_GUARANTEED_PERF(READ_ONCE(cpu->hwp_cap_cached));
  1399. } else {
  1400. cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * cpu->pstate.scaling;
  1401. cpu->pstate.max_pstate = pstate_funcs.get_max();
  1402. }
  1403. cpu->pstate.max_freq = cpu->pstate.max_pstate * cpu->pstate.scaling;
  1404. if (pstate_funcs.get_aperf_mperf_shift)
  1405. cpu->aperf_mperf_shift = pstate_funcs.get_aperf_mperf_shift();
  1406. if (pstate_funcs.get_vid)
  1407. pstate_funcs.get_vid(cpu);
  1408. intel_pstate_set_min_pstate(cpu);
  1409. }
  1410. /*
  1411. * Long hold time will keep high perf limits for long time,
  1412. * which negatively impacts perf/watt for some workloads,
  1413. * like specpower. 3ms is based on experiements on some
  1414. * workoads.
  1415. */
  1416. static int hwp_boost_hold_time_ns = 3 * NSEC_PER_MSEC;
  1417. static inline void intel_pstate_hwp_boost_up(struct cpudata *cpu)
  1418. {
  1419. u64 hwp_req = READ_ONCE(cpu->hwp_req_cached);
  1420. u32 max_limit = (hwp_req & 0xff00) >> 8;
  1421. u32 min_limit = (hwp_req & 0xff);
  1422. u32 boost_level1;
  1423. /*
  1424. * Cases to consider (User changes via sysfs or boot time):
  1425. * If, P0 (Turbo max) = P1 (Guaranteed max) = min:
  1426. * No boost, return.
  1427. * If, P0 (Turbo max) > P1 (Guaranteed max) = min:
  1428. * Should result in one level boost only for P0.
  1429. * If, P0 (Turbo max) = P1 (Guaranteed max) > min:
  1430. * Should result in two level boost:
  1431. * (min + p1)/2 and P1.
  1432. * If, P0 (Turbo max) > P1 (Guaranteed max) > min:
  1433. * Should result in three level boost:
  1434. * (min + p1)/2, P1 and P0.
  1435. */
  1436. /* If max and min are equal or already at max, nothing to boost */
  1437. if (max_limit == min_limit || cpu->hwp_boost_min >= max_limit)
  1438. return;
  1439. if (!cpu->hwp_boost_min)
  1440. cpu->hwp_boost_min = min_limit;
  1441. /* level at half way mark between min and guranteed */
  1442. boost_level1 = (HWP_GUARANTEED_PERF(cpu->hwp_cap_cached) + min_limit) >> 1;
  1443. if (cpu->hwp_boost_min < boost_level1)
  1444. cpu->hwp_boost_min = boost_level1;
  1445. else if (cpu->hwp_boost_min < HWP_GUARANTEED_PERF(cpu->hwp_cap_cached))
  1446. cpu->hwp_boost_min = HWP_GUARANTEED_PERF(cpu->hwp_cap_cached);
  1447. else if (cpu->hwp_boost_min == HWP_GUARANTEED_PERF(cpu->hwp_cap_cached) &&
  1448. max_limit != HWP_GUARANTEED_PERF(cpu->hwp_cap_cached))
  1449. cpu->hwp_boost_min = max_limit;
  1450. else
  1451. return;
  1452. hwp_req = (hwp_req & ~GENMASK_ULL(7, 0)) | cpu->hwp_boost_min;
  1453. wrmsrl(MSR_HWP_REQUEST, hwp_req);
  1454. cpu->last_update = cpu->sample.time;
  1455. }
  1456. static inline void intel_pstate_hwp_boost_down(struct cpudata *cpu)
  1457. {
  1458. if (cpu->hwp_boost_min) {
  1459. bool expired;
  1460. /* Check if we are idle for hold time to boost down */
  1461. expired = time_after64(cpu->sample.time, cpu->last_update +
  1462. hwp_boost_hold_time_ns);
  1463. if (expired) {
  1464. wrmsrl(MSR_HWP_REQUEST, cpu->hwp_req_cached);
  1465. cpu->hwp_boost_min = 0;
  1466. }
  1467. }
  1468. cpu->last_update = cpu->sample.time;
  1469. }
  1470. static inline void intel_pstate_update_util_hwp_local(struct cpudata *cpu,
  1471. u64 time)
  1472. {
  1473. cpu->sample.time = time;
  1474. if (cpu->sched_flags & SCHED_CPUFREQ_IOWAIT) {
  1475. bool do_io = false;
  1476. cpu->sched_flags = 0;
  1477. /*
  1478. * Set iowait_boost flag and update time. Since IO WAIT flag
  1479. * is set all the time, we can't just conclude that there is
  1480. * some IO bound activity is scheduled on this CPU with just
  1481. * one occurrence. If we receive at least two in two
  1482. * consecutive ticks, then we treat as boost candidate.
  1483. */
  1484. if (time_before64(time, cpu->last_io_update + 2 * TICK_NSEC))
  1485. do_io = true;
  1486. cpu->last_io_update = time;
  1487. if (do_io)
  1488. intel_pstate_hwp_boost_up(cpu);
  1489. } else {
  1490. intel_pstate_hwp_boost_down(cpu);
  1491. }
  1492. }
  1493. static inline void intel_pstate_update_util_hwp(struct update_util_data *data,
  1494. u64 time, unsigned int flags)
  1495. {
  1496. struct cpudata *cpu = container_of(data, struct cpudata, update_util);
  1497. cpu->sched_flags |= flags;
  1498. if (smp_processor_id() == cpu->cpu)
  1499. intel_pstate_update_util_hwp_local(cpu, time);
  1500. }
  1501. static inline void intel_pstate_calc_avg_perf(struct cpudata *cpu)
  1502. {
  1503. struct sample *sample = &cpu->sample;
  1504. sample->core_avg_perf = div_ext_fp(sample->aperf, sample->mperf);
  1505. }
  1506. static inline bool intel_pstate_sample(struct cpudata *cpu, u64 time)
  1507. {
  1508. u64 aperf, mperf;
  1509. unsigned long flags;
  1510. u64 tsc;
  1511. local_irq_save(flags);
  1512. rdmsrl(MSR_IA32_APERF, aperf);
  1513. rdmsrl(MSR_IA32_MPERF, mperf);
  1514. tsc = rdtsc();
  1515. if (cpu->prev_mperf == mperf || cpu->prev_tsc == tsc) {
  1516. local_irq_restore(flags);
  1517. return false;
  1518. }
  1519. local_irq_restore(flags);
  1520. cpu->last_sample_time = cpu->sample.time;
  1521. cpu->sample.time = time;
  1522. cpu->sample.aperf = aperf;
  1523. cpu->sample.mperf = mperf;
  1524. cpu->sample.tsc = tsc;
  1525. cpu->sample.aperf -= cpu->prev_aperf;
  1526. cpu->sample.mperf -= cpu->prev_mperf;
  1527. cpu->sample.tsc -= cpu->prev_tsc;
  1528. cpu->prev_aperf = aperf;
  1529. cpu->prev_mperf = mperf;
  1530. cpu->prev_tsc = tsc;
  1531. /*
  1532. * First time this function is invoked in a given cycle, all of the
  1533. * previous sample data fields are equal to zero or stale and they must
  1534. * be populated with meaningful numbers for things to work, so assume
  1535. * that sample.time will always be reset before setting the utilization
  1536. * update hook and make the caller skip the sample then.
  1537. */
  1538. if (cpu->last_sample_time) {
  1539. intel_pstate_calc_avg_perf(cpu);
  1540. return true;
  1541. }
  1542. return false;
  1543. }
  1544. static inline int32_t get_avg_frequency(struct cpudata *cpu)
  1545. {
  1546. return mul_ext_fp(cpu->sample.core_avg_perf, cpu_khz);
  1547. }
  1548. static inline int32_t get_avg_pstate(struct cpudata *cpu)
  1549. {
  1550. return mul_ext_fp(cpu->pstate.max_pstate_physical,
  1551. cpu->sample.core_avg_perf);
  1552. }
  1553. static inline int32_t get_target_pstate(struct cpudata *cpu)
  1554. {
  1555. struct sample *sample = &cpu->sample;
  1556. int32_t busy_frac;
  1557. int target, avg_pstate;
  1558. busy_frac = div_fp(sample->mperf << cpu->aperf_mperf_shift,
  1559. sample->tsc);
  1560. if (busy_frac < cpu->iowait_boost)
  1561. busy_frac = cpu->iowait_boost;
  1562. sample->busy_scaled = busy_frac * 100;
  1563. target = global.no_turbo || global.turbo_disabled ?
  1564. cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
  1565. target += target >> 2;
  1566. target = mul_fp(target, busy_frac);
  1567. if (target < cpu->pstate.min_pstate)
  1568. target = cpu->pstate.min_pstate;
  1569. /*
  1570. * If the average P-state during the previous cycle was higher than the
  1571. * current target, add 50% of the difference to the target to reduce
  1572. * possible performance oscillations and offset possible performance
  1573. * loss related to moving the workload from one CPU to another within
  1574. * a package/module.
  1575. */
  1576. avg_pstate = get_avg_pstate(cpu);
  1577. if (avg_pstate > target)
  1578. target += (avg_pstate - target) >> 1;
  1579. return target;
  1580. }
  1581. static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate)
  1582. {
  1583. int min_pstate = max(cpu->pstate.min_pstate, cpu->min_perf_ratio);
  1584. int max_pstate = max(min_pstate, cpu->max_perf_ratio);
  1585. return clamp_t(int, pstate, min_pstate, max_pstate);
  1586. }
  1587. static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate)
  1588. {
  1589. if (pstate == cpu->pstate.current_pstate)
  1590. return;
  1591. cpu->pstate.current_pstate = pstate;
  1592. wrmsrl(MSR_IA32_PERF_CTL, pstate_funcs.get_val(cpu, pstate));
  1593. }
  1594. static void intel_pstate_adjust_pstate(struct cpudata *cpu)
  1595. {
  1596. int from = cpu->pstate.current_pstate;
  1597. struct sample *sample;
  1598. int target_pstate;
  1599. update_turbo_state();
  1600. target_pstate = get_target_pstate(cpu);
  1601. target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
  1602. trace_cpu_frequency(target_pstate * cpu->pstate.scaling, cpu->cpu);
  1603. intel_pstate_update_pstate(cpu, target_pstate);
  1604. sample = &cpu->sample;
  1605. trace_pstate_sample(mul_ext_fp(100, sample->core_avg_perf),
  1606. fp_toint(sample->busy_scaled),
  1607. from,
  1608. cpu->pstate.current_pstate,
  1609. sample->mperf,
  1610. sample->aperf,
  1611. sample->tsc,
  1612. get_avg_frequency(cpu),
  1613. fp_toint(cpu->iowait_boost * 100));
  1614. }
  1615. static void intel_pstate_update_util(struct update_util_data *data, u64 time,
  1616. unsigned int flags)
  1617. {
  1618. struct cpudata *cpu = container_of(data, struct cpudata, update_util);
  1619. u64 delta_ns;
  1620. /* Don't allow remote callbacks */
  1621. if (smp_processor_id() != cpu->cpu)
  1622. return;
  1623. delta_ns = time - cpu->last_update;
  1624. if (flags & SCHED_CPUFREQ_IOWAIT) {
  1625. /* Start over if the CPU may have been idle. */
  1626. if (delta_ns > TICK_NSEC) {
  1627. cpu->iowait_boost = ONE_EIGHTH_FP;
  1628. } else if (cpu->iowait_boost >= ONE_EIGHTH_FP) {
  1629. cpu->iowait_boost <<= 1;
  1630. if (cpu->iowait_boost > int_tofp(1))
  1631. cpu->iowait_boost = int_tofp(1);
  1632. } else {
  1633. cpu->iowait_boost = ONE_EIGHTH_FP;
  1634. }
  1635. } else if (cpu->iowait_boost) {
  1636. /* Clear iowait_boost if the CPU may have been idle. */
  1637. if (delta_ns > TICK_NSEC)
  1638. cpu->iowait_boost = 0;
  1639. else
  1640. cpu->iowait_boost >>= 1;
  1641. }
  1642. cpu->last_update = time;
  1643. delta_ns = time - cpu->sample.time;
  1644. if ((s64)delta_ns < INTEL_PSTATE_SAMPLING_INTERVAL)
  1645. return;
  1646. if (intel_pstate_sample(cpu, time))
  1647. intel_pstate_adjust_pstate(cpu);
  1648. }
  1649. static struct pstate_funcs core_funcs = {
  1650. .get_max = core_get_max_pstate,
  1651. .get_max_physical = core_get_max_pstate_physical,
  1652. .get_min = core_get_min_pstate,
  1653. .get_turbo = core_get_turbo_pstate,
  1654. .get_scaling = core_get_scaling,
  1655. .get_val = core_get_val,
  1656. };
  1657. static const struct pstate_funcs silvermont_funcs = {
  1658. .get_max = atom_get_max_pstate,
  1659. .get_max_physical = atom_get_max_pstate,
  1660. .get_min = atom_get_min_pstate,
  1661. .get_turbo = atom_get_turbo_pstate,
  1662. .get_val = atom_get_val,
  1663. .get_scaling = silvermont_get_scaling,
  1664. .get_vid = atom_get_vid,
  1665. };
  1666. static const struct pstate_funcs airmont_funcs = {
  1667. .get_max = atom_get_max_pstate,
  1668. .get_max_physical = atom_get_max_pstate,
  1669. .get_min = atom_get_min_pstate,
  1670. .get_turbo = atom_get_turbo_pstate,
  1671. .get_val = atom_get_val,
  1672. .get_scaling = airmont_get_scaling,
  1673. .get_vid = atom_get_vid,
  1674. };
  1675. static const struct pstate_funcs knl_funcs = {
  1676. .get_max = core_get_max_pstate,
  1677. .get_max_physical = core_get_max_pstate_physical,
  1678. .get_min = core_get_min_pstate,
  1679. .get_turbo = knl_get_turbo_pstate,
  1680. .get_aperf_mperf_shift = knl_get_aperf_mperf_shift,
  1681. .get_scaling = core_get_scaling,
  1682. .get_val = core_get_val,
  1683. };
  1684. #define X86_MATCH(model, policy) \
  1685. X86_MATCH_VENDOR_FAM_MODEL_FEATURE(INTEL, 6, INTEL_FAM6_##model, \
  1686. X86_FEATURE_APERFMPERF, &policy)
  1687. static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
  1688. X86_MATCH(SANDYBRIDGE, core_funcs),
  1689. X86_MATCH(SANDYBRIDGE_X, core_funcs),
  1690. X86_MATCH(ATOM_SILVERMONT, silvermont_funcs),
  1691. X86_MATCH(IVYBRIDGE, core_funcs),
  1692. X86_MATCH(HASWELL, core_funcs),
  1693. X86_MATCH(BROADWELL, core_funcs),
  1694. X86_MATCH(IVYBRIDGE_X, core_funcs),
  1695. X86_MATCH(HASWELL_X, core_funcs),
  1696. X86_MATCH(HASWELL_L, core_funcs),
  1697. X86_MATCH(HASWELL_G, core_funcs),
  1698. X86_MATCH(BROADWELL_G, core_funcs),
  1699. X86_MATCH(ATOM_AIRMONT, airmont_funcs),
  1700. X86_MATCH(SKYLAKE_L, core_funcs),
  1701. X86_MATCH(BROADWELL_X, core_funcs),
  1702. X86_MATCH(SKYLAKE, core_funcs),
  1703. X86_MATCH(BROADWELL_D, core_funcs),
  1704. X86_MATCH(XEON_PHI_KNL, knl_funcs),
  1705. X86_MATCH(XEON_PHI_KNM, knl_funcs),
  1706. X86_MATCH(ATOM_GOLDMONT, core_funcs),
  1707. X86_MATCH(ATOM_GOLDMONT_PLUS, core_funcs),
  1708. X86_MATCH(SKYLAKE_X, core_funcs),
  1709. {}
  1710. };
  1711. MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
  1712. static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = {
  1713. X86_MATCH(BROADWELL_D, core_funcs),
  1714. X86_MATCH(BROADWELL_X, core_funcs),
  1715. X86_MATCH(SKYLAKE_X, core_funcs),
  1716. {}
  1717. };
  1718. static const struct x86_cpu_id intel_pstate_cpu_ee_disable_ids[] = {
  1719. X86_MATCH(KABYLAKE, core_funcs),
  1720. {}
  1721. };
  1722. static const struct x86_cpu_id intel_pstate_hwp_boost_ids[] = {
  1723. X86_MATCH(SKYLAKE_X, core_funcs),
  1724. X86_MATCH(SKYLAKE, core_funcs),
  1725. {}
  1726. };
  1727. static int intel_pstate_init_cpu(unsigned int cpunum)
  1728. {
  1729. struct cpudata *cpu;
  1730. cpu = all_cpu_data[cpunum];
  1731. if (!cpu) {
  1732. cpu = kzalloc(sizeof(*cpu), GFP_KERNEL);
  1733. if (!cpu)
  1734. return -ENOMEM;
  1735. all_cpu_data[cpunum] = cpu;
  1736. cpu->cpu = cpunum;
  1737. cpu->epp_default = -EINVAL;
  1738. if (hwp_active) {
  1739. const struct x86_cpu_id *id;
  1740. intel_pstate_hwp_enable(cpu);
  1741. id = x86_match_cpu(intel_pstate_hwp_boost_ids);
  1742. if (id && intel_pstate_acpi_pm_profile_server())
  1743. hwp_boost = true;
  1744. }
  1745. } else if (hwp_active) {
  1746. /*
  1747. * Re-enable HWP in case this happens after a resume from ACPI
  1748. * S3 if the CPU was offline during the whole system/resume
  1749. * cycle.
  1750. */
  1751. intel_pstate_hwp_reenable(cpu);
  1752. }
  1753. cpu->epp_powersave = -EINVAL;
  1754. cpu->epp_policy = 0;
  1755. intel_pstate_get_cpu_pstates(cpu);
  1756. pr_debug("controlling: cpu %d\n", cpunum);
  1757. return 0;
  1758. }
  1759. static void intel_pstate_set_update_util_hook(unsigned int cpu_num)
  1760. {
  1761. struct cpudata *cpu = all_cpu_data[cpu_num];
  1762. if (hwp_active && !hwp_boost)
  1763. return;
  1764. if (cpu->update_util_set)
  1765. return;
  1766. /* Prevent intel_pstate_update_util() from using stale data. */
  1767. cpu->sample.time = 0;
  1768. cpufreq_add_update_util_hook(cpu_num, &cpu->update_util,
  1769. (hwp_active ?
  1770. intel_pstate_update_util_hwp :
  1771. intel_pstate_update_util));
  1772. cpu->update_util_set = true;
  1773. }
  1774. static void intel_pstate_clear_update_util_hook(unsigned int cpu)
  1775. {
  1776. struct cpudata *cpu_data = all_cpu_data[cpu];
  1777. if (!cpu_data->update_util_set)
  1778. return;
  1779. cpufreq_remove_update_util_hook(cpu);
  1780. cpu_data->update_util_set = false;
  1781. synchronize_rcu();
  1782. }
  1783. static int intel_pstate_get_max_freq(struct cpudata *cpu)
  1784. {
  1785. return global.turbo_disabled || global.no_turbo ?
  1786. cpu->pstate.max_freq : cpu->pstate.turbo_freq;
  1787. }
  1788. static void intel_pstate_update_perf_limits(struct cpudata *cpu,
  1789. unsigned int policy_min,
  1790. unsigned int policy_max)
  1791. {
  1792. int32_t max_policy_perf, min_policy_perf;
  1793. int max_state, turbo_max;
  1794. int max_freq;
  1795. /*
  1796. * HWP needs some special consideration, because on BDX the
  1797. * HWP_REQUEST uses abstract value to represent performance
  1798. * rather than pure ratios.
  1799. */
  1800. if (hwp_active) {
  1801. intel_pstate_get_hwp_max(cpu, &turbo_max, &max_state);
  1802. } else {
  1803. max_state = global.no_turbo || global.turbo_disabled ?
  1804. cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
  1805. turbo_max = cpu->pstate.turbo_pstate;
  1806. }
  1807. max_freq = max_state * cpu->pstate.scaling;
  1808. max_policy_perf = max_state * policy_max / max_freq;
  1809. if (policy_max == policy_min) {
  1810. min_policy_perf = max_policy_perf;
  1811. } else {
  1812. min_policy_perf = max_state * policy_min / max_freq;
  1813. min_policy_perf = clamp_t(int32_t, min_policy_perf,
  1814. 0, max_policy_perf);
  1815. }
  1816. pr_debug("cpu:%d max_state %d min_policy_perf:%d max_policy_perf:%d\n",
  1817. cpu->cpu, max_state, min_policy_perf, max_policy_perf);
  1818. /* Normalize user input to [min_perf, max_perf] */
  1819. if (per_cpu_limits) {
  1820. cpu->min_perf_ratio = min_policy_perf;
  1821. cpu->max_perf_ratio = max_policy_perf;
  1822. } else {
  1823. int32_t global_min, global_max;
  1824. /* Global limits are in percent of the maximum turbo P-state. */
  1825. global_max = DIV_ROUND_UP(turbo_max * global.max_perf_pct, 100);
  1826. global_min = DIV_ROUND_UP(turbo_max * global.min_perf_pct, 100);
  1827. global_min = clamp_t(int32_t, global_min, 0, global_max);
  1828. pr_debug("cpu:%d global_min:%d global_max:%d\n", cpu->cpu,
  1829. global_min, global_max);
  1830. cpu->min_perf_ratio = max(min_policy_perf, global_min);
  1831. cpu->min_perf_ratio = min(cpu->min_perf_ratio, max_policy_perf);
  1832. cpu->max_perf_ratio = min(max_policy_perf, global_max);
  1833. cpu->max_perf_ratio = max(min_policy_perf, cpu->max_perf_ratio);
  1834. /* Make sure min_perf <= max_perf */
  1835. cpu->min_perf_ratio = min(cpu->min_perf_ratio,
  1836. cpu->max_perf_ratio);
  1837. }
  1838. pr_debug("cpu:%d max_perf_ratio:%d min_perf_ratio:%d\n", cpu->cpu,
  1839. cpu->max_perf_ratio,
  1840. cpu->min_perf_ratio);
  1841. }
  1842. static int intel_pstate_set_policy(struct cpufreq_policy *policy)
  1843. {
  1844. struct cpudata *cpu;
  1845. if (!policy->cpuinfo.max_freq)
  1846. return -ENODEV;
  1847. pr_debug("set_policy cpuinfo.max %u policy->max %u\n",
  1848. policy->cpuinfo.max_freq, policy->max);
  1849. cpu = all_cpu_data[policy->cpu];
  1850. cpu->policy = policy->policy;
  1851. mutex_lock(&intel_pstate_limits_lock);
  1852. intel_pstate_update_perf_limits(cpu, policy->min, policy->max);
  1853. if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) {
  1854. /*
  1855. * NOHZ_FULL CPUs need this as the governor callback may not
  1856. * be invoked on them.
  1857. */
  1858. intel_pstate_clear_update_util_hook(policy->cpu);
  1859. intel_pstate_max_within_limits(cpu);
  1860. } else {
  1861. intel_pstate_set_update_util_hook(policy->cpu);
  1862. }
  1863. if (hwp_active) {
  1864. /*
  1865. * When hwp_boost was active before and dynamically it
  1866. * was turned off, in that case we need to clear the
  1867. * update util hook.
  1868. */
  1869. if (!hwp_boost)
  1870. intel_pstate_clear_update_util_hook(policy->cpu);
  1871. intel_pstate_hwp_set(policy->cpu);
  1872. }
  1873. mutex_unlock(&intel_pstate_limits_lock);
  1874. return 0;
  1875. }
  1876. static void intel_pstate_adjust_policy_max(struct cpudata *cpu,
  1877. struct cpufreq_policy_data *policy)
  1878. {
  1879. if (!hwp_active &&
  1880. cpu->pstate.max_pstate_physical > cpu->pstate.max_pstate &&
  1881. policy->max < policy->cpuinfo.max_freq &&
  1882. policy->max > cpu->pstate.max_freq) {
  1883. pr_debug("policy->max > max non turbo frequency\n");
  1884. policy->max = policy->cpuinfo.max_freq;
  1885. }
  1886. }
  1887. static void intel_pstate_verify_cpu_policy(struct cpudata *cpu,
  1888. struct cpufreq_policy_data *policy)
  1889. {
  1890. int max_freq;
  1891. update_turbo_state();
  1892. if (hwp_active) {
  1893. int max_state, turbo_max;
  1894. intel_pstate_get_hwp_max(cpu, &turbo_max, &max_state);
  1895. max_freq = max_state * cpu->pstate.scaling;
  1896. } else {
  1897. max_freq = intel_pstate_get_max_freq(cpu);
  1898. }
  1899. cpufreq_verify_within_limits(policy, policy->cpuinfo.min_freq, max_freq);
  1900. intel_pstate_adjust_policy_max(cpu, policy);
  1901. }
  1902. static int intel_pstate_verify_policy(struct cpufreq_policy_data *policy)
  1903. {
  1904. intel_pstate_verify_cpu_policy(all_cpu_data[policy->cpu], policy);
  1905. return 0;
  1906. }
  1907. static int intel_pstate_cpu_offline(struct cpufreq_policy *policy)
  1908. {
  1909. struct cpudata *cpu = all_cpu_data[policy->cpu];
  1910. pr_debug("CPU %d going offline\n", cpu->cpu);
  1911. if (cpu->suspended)
  1912. return 0;
  1913. /*
  1914. * If the CPU is an SMT thread and it goes offline with the performance
  1915. * settings different from the minimum, it will prevent its sibling
  1916. * from getting to lower performance levels, so force the minimum
  1917. * performance on CPU offline to prevent that from happening.
  1918. */
  1919. if (hwp_active)
  1920. intel_pstate_hwp_offline(cpu);
  1921. else
  1922. intel_pstate_set_min_pstate(cpu);
  1923. intel_pstate_exit_perf_limits(policy);
  1924. return 0;
  1925. }
  1926. static int intel_pstate_cpu_online(struct cpufreq_policy *policy)
  1927. {
  1928. struct cpudata *cpu = all_cpu_data[policy->cpu];
  1929. pr_debug("CPU %d going online\n", cpu->cpu);
  1930. intel_pstate_init_acpi_perf_limits(policy);
  1931. if (hwp_active) {
  1932. /*
  1933. * Re-enable HWP and clear the "suspended" flag to let "resume"
  1934. * know that it need not do that.
  1935. */
  1936. intel_pstate_hwp_reenable(cpu);
  1937. cpu->suspended = false;
  1938. }
  1939. return 0;
  1940. }
  1941. static void intel_pstate_stop_cpu(struct cpufreq_policy *policy)
  1942. {
  1943. pr_debug("CPU %d stopping\n", policy->cpu);
  1944. intel_pstate_clear_update_util_hook(policy->cpu);
  1945. }
  1946. static int intel_pstate_cpu_exit(struct cpufreq_policy *policy)
  1947. {
  1948. pr_debug("CPU %d exiting\n", policy->cpu);
  1949. policy->fast_switch_possible = false;
  1950. return 0;
  1951. }
  1952. static int __intel_pstate_cpu_init(struct cpufreq_policy *policy)
  1953. {
  1954. struct cpudata *cpu;
  1955. int rc;
  1956. rc = intel_pstate_init_cpu(policy->cpu);
  1957. if (rc)
  1958. return rc;
  1959. cpu = all_cpu_data[policy->cpu];
  1960. cpu->max_perf_ratio = 0xFF;
  1961. cpu->min_perf_ratio = 0;
  1962. policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling;
  1963. policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling;
  1964. /* cpuinfo and default policy values */
  1965. policy->cpuinfo.min_freq = cpu->pstate.min_pstate * cpu->pstate.scaling;
  1966. update_turbo_state();
  1967. global.turbo_disabled_mf = global.turbo_disabled;
  1968. policy->cpuinfo.max_freq = global.turbo_disabled ?
  1969. cpu->pstate.max_pstate : cpu->pstate.turbo_pstate;
  1970. policy->cpuinfo.max_freq *= cpu->pstate.scaling;
  1971. if (hwp_active) {
  1972. unsigned int max_freq;
  1973. max_freq = global.turbo_disabled ?
  1974. cpu->pstate.max_freq : cpu->pstate.turbo_freq;
  1975. if (max_freq < policy->cpuinfo.max_freq)
  1976. policy->cpuinfo.max_freq = max_freq;
  1977. }
  1978. intel_pstate_init_acpi_perf_limits(policy);
  1979. policy->fast_switch_possible = true;
  1980. return 0;
  1981. }
  1982. static int intel_pstate_cpu_init(struct cpufreq_policy *policy)
  1983. {
  1984. int ret = __intel_pstate_cpu_init(policy);
  1985. if (ret)
  1986. return ret;
  1987. /*
  1988. * Set the policy to powersave to provide a valid fallback value in case
  1989. * the default cpufreq governor is neither powersave nor performance.
  1990. */
  1991. policy->policy = CPUFREQ_POLICY_POWERSAVE;
  1992. if (hwp_active) {
  1993. struct cpudata *cpu = all_cpu_data[policy->cpu];
  1994. cpu->epp_cached = intel_pstate_get_epp(cpu, 0);
  1995. }
  1996. return 0;
  1997. }
  1998. static struct cpufreq_driver intel_pstate = {
  1999. .flags = CPUFREQ_CONST_LOOPS,
  2000. .verify = intel_pstate_verify_policy,
  2001. .setpolicy = intel_pstate_set_policy,
  2002. .suspend = intel_pstate_suspend,
  2003. .resume = intel_pstate_resume,
  2004. .init = intel_pstate_cpu_init,
  2005. .exit = intel_pstate_cpu_exit,
  2006. .stop_cpu = intel_pstate_stop_cpu,
  2007. .offline = intel_pstate_cpu_offline,
  2008. .online = intel_pstate_cpu_online,
  2009. .update_limits = intel_pstate_update_limits,
  2010. .name = "intel_pstate",
  2011. };
  2012. static int intel_cpufreq_verify_policy(struct cpufreq_policy_data *policy)
  2013. {
  2014. struct cpudata *cpu = all_cpu_data[policy->cpu];
  2015. intel_pstate_verify_cpu_policy(cpu, policy);
  2016. intel_pstate_update_perf_limits(cpu, policy->min, policy->max);
  2017. return 0;
  2018. }
  2019. /* Use of trace in passive mode:
  2020. *
  2021. * In passive mode the trace core_busy field (also known as the
  2022. * performance field, and lablelled as such on the graphs; also known as
  2023. * core_avg_perf) is not needed and so is re-assigned to indicate if the
  2024. * driver call was via the normal or fast switch path. Various graphs
  2025. * output from the intel_pstate_tracer.py utility that include core_busy
  2026. * (or performance or core_avg_perf) have a fixed y-axis from 0 to 100%,
  2027. * so we use 10 to indicate the the normal path through the driver, and
  2028. * 90 to indicate the fast switch path through the driver.
  2029. * The scaled_busy field is not used, and is set to 0.
  2030. */
  2031. #define INTEL_PSTATE_TRACE_TARGET 10
  2032. #define INTEL_PSTATE_TRACE_FAST_SWITCH 90
  2033. static void intel_cpufreq_trace(struct cpudata *cpu, unsigned int trace_type, int old_pstate)
  2034. {
  2035. struct sample *sample;
  2036. if (!trace_pstate_sample_enabled())
  2037. return;
  2038. if (!intel_pstate_sample(cpu, ktime_get()))
  2039. return;
  2040. sample = &cpu->sample;
  2041. trace_pstate_sample(trace_type,
  2042. 0,
  2043. old_pstate,
  2044. cpu->pstate.current_pstate,
  2045. sample->mperf,
  2046. sample->aperf,
  2047. sample->tsc,
  2048. get_avg_frequency(cpu),
  2049. fp_toint(cpu->iowait_boost * 100));
  2050. }
  2051. static void intel_cpufreq_adjust_hwp(struct cpudata *cpu, u32 target_pstate,
  2052. bool strict, bool fast_switch)
  2053. {
  2054. u64 prev = READ_ONCE(cpu->hwp_req_cached), value = prev;
  2055. value &= ~HWP_MIN_PERF(~0L);
  2056. value |= HWP_MIN_PERF(target_pstate);
  2057. /*
  2058. * The entire MSR needs to be updated in order to update the HWP min
  2059. * field in it, so opportunistically update the max too if needed.
  2060. */
  2061. value &= ~HWP_MAX_PERF(~0L);
  2062. value |= HWP_MAX_PERF(strict ? target_pstate : cpu->max_perf_ratio);
  2063. if (value == prev)
  2064. return;
  2065. WRITE_ONCE(cpu->hwp_req_cached, value);
  2066. if (fast_switch)
  2067. wrmsrl(MSR_HWP_REQUEST, value);
  2068. else
  2069. wrmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, value);
  2070. }
  2071. static void intel_cpufreq_adjust_perf_ctl(struct cpudata *cpu,
  2072. u32 target_pstate, bool fast_switch)
  2073. {
  2074. if (fast_switch)
  2075. wrmsrl(MSR_IA32_PERF_CTL,
  2076. pstate_funcs.get_val(cpu, target_pstate));
  2077. else
  2078. wrmsrl_on_cpu(cpu->cpu, MSR_IA32_PERF_CTL,
  2079. pstate_funcs.get_val(cpu, target_pstate));
  2080. }
  2081. static int intel_cpufreq_update_pstate(struct cpufreq_policy *policy,
  2082. int target_pstate, bool fast_switch)
  2083. {
  2084. struct cpudata *cpu = all_cpu_data[policy->cpu];
  2085. int old_pstate = cpu->pstate.current_pstate;
  2086. target_pstate = intel_pstate_prepare_request(cpu, target_pstate);
  2087. if (hwp_active) {
  2088. intel_cpufreq_adjust_hwp(cpu, target_pstate,
  2089. policy->strict_target, fast_switch);
  2090. cpu->pstate.current_pstate = target_pstate;
  2091. } else if (target_pstate != old_pstate) {
  2092. intel_cpufreq_adjust_perf_ctl(cpu, target_pstate, fast_switch);
  2093. cpu->pstate.current_pstate = target_pstate;
  2094. }
  2095. intel_cpufreq_trace(cpu, fast_switch ? INTEL_PSTATE_TRACE_FAST_SWITCH :
  2096. INTEL_PSTATE_TRACE_TARGET, old_pstate);
  2097. return target_pstate;
  2098. }
  2099. static int intel_cpufreq_target(struct cpufreq_policy *policy,
  2100. unsigned int target_freq,
  2101. unsigned int relation)
  2102. {
  2103. struct cpudata *cpu = all_cpu_data[policy->cpu];
  2104. struct cpufreq_freqs freqs;
  2105. int target_pstate;
  2106. update_turbo_state();
  2107. freqs.old = policy->cur;
  2108. freqs.new = target_freq;
  2109. cpufreq_freq_transition_begin(policy, &freqs);
  2110. switch (relation) {
  2111. case CPUFREQ_RELATION_L:
  2112. target_pstate = DIV_ROUND_UP(freqs.new, cpu->pstate.scaling);
  2113. break;
  2114. case CPUFREQ_RELATION_H:
  2115. target_pstate = freqs.new / cpu->pstate.scaling;
  2116. break;
  2117. default:
  2118. target_pstate = DIV_ROUND_CLOSEST(freqs.new, cpu->pstate.scaling);
  2119. break;
  2120. }
  2121. target_pstate = intel_cpufreq_update_pstate(policy, target_pstate, false);
  2122. freqs.new = target_pstate * cpu->pstate.scaling;
  2123. cpufreq_freq_transition_end(policy, &freqs, false);
  2124. return 0;
  2125. }
  2126. static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy,
  2127. unsigned int target_freq)
  2128. {
  2129. struct cpudata *cpu = all_cpu_data[policy->cpu];
  2130. int target_pstate;
  2131. update_turbo_state();
  2132. target_pstate = DIV_ROUND_UP(target_freq, cpu->pstate.scaling);
  2133. target_pstate = intel_cpufreq_update_pstate(policy, target_pstate, true);
  2134. return target_pstate * cpu->pstate.scaling;
  2135. }
  2136. static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy)
  2137. {
  2138. int max_state, turbo_max, min_freq, max_freq, ret;
  2139. struct freq_qos_request *req;
  2140. struct cpudata *cpu;
  2141. struct device *dev;
  2142. dev = get_cpu_device(policy->cpu);
  2143. if (!dev)
  2144. return -ENODEV;
  2145. ret = __intel_pstate_cpu_init(policy);
  2146. if (ret)
  2147. return ret;
  2148. policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY;
  2149. /* This reflects the intel_pstate_get_cpu_pstates() setting. */
  2150. policy->cur = policy->cpuinfo.min_freq;
  2151. req = kcalloc(2, sizeof(*req), GFP_KERNEL);
  2152. if (!req) {
  2153. ret = -ENOMEM;
  2154. goto pstate_exit;
  2155. }
  2156. cpu = all_cpu_data[policy->cpu];
  2157. if (hwp_active) {
  2158. u64 value;
  2159. intel_pstate_get_hwp_max(cpu, &turbo_max, &max_state);
  2160. policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY_HWP;
  2161. rdmsrl_on_cpu(cpu->cpu, MSR_HWP_REQUEST, &value);
  2162. WRITE_ONCE(cpu->hwp_req_cached, value);
  2163. cpu->epp_cached = intel_pstate_get_epp(cpu, value);
  2164. } else {
  2165. turbo_max = cpu->pstate.turbo_pstate;
  2166. policy->transition_delay_us = INTEL_CPUFREQ_TRANSITION_DELAY;
  2167. }
  2168. min_freq = DIV_ROUND_UP(turbo_max * global.min_perf_pct, 100);
  2169. min_freq *= cpu->pstate.scaling;
  2170. max_freq = DIV_ROUND_UP(turbo_max * global.max_perf_pct, 100);
  2171. max_freq *= cpu->pstate.scaling;
  2172. ret = freq_qos_add_request(&policy->constraints, req, FREQ_QOS_MIN,
  2173. min_freq);
  2174. if (ret < 0) {
  2175. dev_err(dev, "Failed to add min-freq constraint (%d)\n", ret);
  2176. goto free_req;
  2177. }
  2178. ret = freq_qos_add_request(&policy->constraints, req + 1, FREQ_QOS_MAX,
  2179. max_freq);
  2180. if (ret < 0) {
  2181. dev_err(dev, "Failed to add max-freq constraint (%d)\n", ret);
  2182. goto remove_min_req;
  2183. }
  2184. policy->driver_data = req;
  2185. return 0;
  2186. remove_min_req:
  2187. freq_qos_remove_request(req);
  2188. free_req:
  2189. kfree(req);
  2190. pstate_exit:
  2191. intel_pstate_exit_perf_limits(policy);
  2192. return ret;
  2193. }
  2194. static int intel_cpufreq_cpu_exit(struct cpufreq_policy *policy)
  2195. {
  2196. struct freq_qos_request *req;
  2197. req = policy->driver_data;
  2198. freq_qos_remove_request(req + 1);
  2199. freq_qos_remove_request(req);
  2200. kfree(req);
  2201. return intel_pstate_cpu_exit(policy);
  2202. }
  2203. static struct cpufreq_driver intel_cpufreq = {
  2204. .flags = CPUFREQ_CONST_LOOPS,
  2205. .verify = intel_cpufreq_verify_policy,
  2206. .target = intel_cpufreq_target,
  2207. .fast_switch = intel_cpufreq_fast_switch,
  2208. .init = intel_cpufreq_cpu_init,
  2209. .exit = intel_cpufreq_cpu_exit,
  2210. .offline = intel_pstate_cpu_offline,
  2211. .online = intel_pstate_cpu_online,
  2212. .suspend = intel_pstate_suspend,
  2213. .resume = intel_pstate_resume,
  2214. .update_limits = intel_pstate_update_limits,
  2215. .name = "intel_cpufreq",
  2216. };
  2217. static struct cpufreq_driver *default_driver;
  2218. static void intel_pstate_driver_cleanup(void)
  2219. {
  2220. unsigned int cpu;
  2221. get_online_cpus();
  2222. for_each_online_cpu(cpu) {
  2223. if (all_cpu_data[cpu]) {
  2224. if (intel_pstate_driver == &intel_pstate)
  2225. intel_pstate_clear_update_util_hook(cpu);
  2226. kfree(all_cpu_data[cpu]);
  2227. all_cpu_data[cpu] = NULL;
  2228. }
  2229. }
  2230. put_online_cpus();
  2231. intel_pstate_driver = NULL;
  2232. }
  2233. static int intel_pstate_register_driver(struct cpufreq_driver *driver)
  2234. {
  2235. int ret;
  2236. if (driver == &intel_pstate)
  2237. intel_pstate_sysfs_expose_hwp_dynamic_boost();
  2238. memset(&global, 0, sizeof(global));
  2239. global.max_perf_pct = 100;
  2240. intel_pstate_driver = driver;
  2241. ret = cpufreq_register_driver(intel_pstate_driver);
  2242. if (ret) {
  2243. intel_pstate_driver_cleanup();
  2244. return ret;
  2245. }
  2246. global.min_perf_pct = min_perf_pct_min();
  2247. return 0;
  2248. }
  2249. static ssize_t intel_pstate_show_status(char *buf)
  2250. {
  2251. if (!intel_pstate_driver)
  2252. return sprintf(buf, "off\n");
  2253. return sprintf(buf, "%s\n", intel_pstate_driver == &intel_pstate ?
  2254. "active" : "passive");
  2255. }
  2256. static int intel_pstate_update_status(const char *buf, size_t size)
  2257. {
  2258. if (size == 3 && !strncmp(buf, "off", size)) {
  2259. if (!intel_pstate_driver)
  2260. return -EINVAL;
  2261. if (hwp_active)
  2262. return -EBUSY;
  2263. cpufreq_unregister_driver(intel_pstate_driver);
  2264. intel_pstate_driver_cleanup();
  2265. return 0;
  2266. }
  2267. if (size == 6 && !strncmp(buf, "active", size)) {
  2268. if (intel_pstate_driver) {
  2269. if (intel_pstate_driver == &intel_pstate)
  2270. return 0;
  2271. cpufreq_unregister_driver(intel_pstate_driver);
  2272. }
  2273. return intel_pstate_register_driver(&intel_pstate);
  2274. }
  2275. if (size == 7 && !strncmp(buf, "passive", size)) {
  2276. if (intel_pstate_driver) {
  2277. if (intel_pstate_driver == &intel_cpufreq)
  2278. return 0;
  2279. cpufreq_unregister_driver(intel_pstate_driver);
  2280. intel_pstate_sysfs_hide_hwp_dynamic_boost();
  2281. }
  2282. return intel_pstate_register_driver(&intel_cpufreq);
  2283. }
  2284. return -EINVAL;
  2285. }
  2286. static int no_load __initdata;
  2287. static int no_hwp __initdata;
  2288. static int hwp_only __initdata;
  2289. static unsigned int force_load __initdata;
  2290. static int __init intel_pstate_msrs_not_valid(void)
  2291. {
  2292. if (!pstate_funcs.get_max() ||
  2293. !pstate_funcs.get_min() ||
  2294. !pstate_funcs.get_turbo())
  2295. return -ENODEV;
  2296. return 0;
  2297. }
  2298. static void __init copy_cpu_funcs(struct pstate_funcs *funcs)
  2299. {
  2300. pstate_funcs.get_max = funcs->get_max;
  2301. pstate_funcs.get_max_physical = funcs->get_max_physical;
  2302. pstate_funcs.get_min = funcs->get_min;
  2303. pstate_funcs.get_turbo = funcs->get_turbo;
  2304. pstate_funcs.get_scaling = funcs->get_scaling;
  2305. pstate_funcs.get_val = funcs->get_val;
  2306. pstate_funcs.get_vid = funcs->get_vid;
  2307. pstate_funcs.get_aperf_mperf_shift = funcs->get_aperf_mperf_shift;
  2308. }
  2309. #ifdef CONFIG_ACPI
  2310. static bool __init intel_pstate_no_acpi_pss(void)
  2311. {
  2312. int i;
  2313. for_each_possible_cpu(i) {
  2314. acpi_status status;
  2315. union acpi_object *pss;
  2316. struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
  2317. struct acpi_processor *pr = per_cpu(processors, i);
  2318. if (!pr)
  2319. continue;
  2320. status = acpi_evaluate_object(pr->handle, "_PSS", NULL, &buffer);
  2321. if (ACPI_FAILURE(status))
  2322. continue;
  2323. pss = buffer.pointer;
  2324. if (pss && pss->type == ACPI_TYPE_PACKAGE) {
  2325. kfree(pss);
  2326. return false;
  2327. }
  2328. kfree(pss);
  2329. }
  2330. pr_debug("ACPI _PSS not found\n");
  2331. return true;
  2332. }
  2333. static bool __init intel_pstate_no_acpi_pcch(void)
  2334. {
  2335. acpi_status status;
  2336. acpi_handle handle;
  2337. status = acpi_get_handle(NULL, "\\_SB", &handle);
  2338. if (ACPI_FAILURE(status))
  2339. goto not_found;
  2340. if (acpi_has_method(handle, "PCCH"))
  2341. return false;
  2342. not_found:
  2343. pr_debug("ACPI PCCH not found\n");
  2344. return true;
  2345. }
  2346. static bool __init intel_pstate_has_acpi_ppc(void)
  2347. {
  2348. int i;
  2349. for_each_possible_cpu(i) {
  2350. struct acpi_processor *pr = per_cpu(processors, i);
  2351. if (!pr)
  2352. continue;
  2353. if (acpi_has_method(pr->handle, "_PPC"))
  2354. return true;
  2355. }
  2356. pr_debug("ACPI _PPC not found\n");
  2357. return false;
  2358. }
  2359. enum {
  2360. PSS,
  2361. PPC,
  2362. };
  2363. /* Hardware vendor-specific info that has its own power management modes */
  2364. static struct acpi_platform_list plat_info[] __initdata = {
  2365. {"HP ", "ProLiant", 0, ACPI_SIG_FADT, all_versions, NULL, PSS},
  2366. {"ORACLE", "X4-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
  2367. {"ORACLE", "X4-2L ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
  2368. {"ORACLE", "X4-2B ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
  2369. {"ORACLE", "X3-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
  2370. {"ORACLE", "X3-2L ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
  2371. {"ORACLE", "X3-2B ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
  2372. {"ORACLE", "X4470M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
  2373. {"ORACLE", "X4270M3 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
  2374. {"ORACLE", "X4270M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
  2375. {"ORACLE", "X4170M2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
  2376. {"ORACLE", "X4170 M3", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
  2377. {"ORACLE", "X4275 M3", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
  2378. {"ORACLE", "X6-2 ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
  2379. {"ORACLE", "Sudbury ", 0, ACPI_SIG_FADT, all_versions, NULL, PPC},
  2380. { } /* End */
  2381. };
  2382. #define BITMASK_OOB (BIT(8) | BIT(18))
  2383. static bool __init intel_pstate_platform_pwr_mgmt_exists(void)
  2384. {
  2385. const struct x86_cpu_id *id;
  2386. u64 misc_pwr;
  2387. int idx;
  2388. id = x86_match_cpu(intel_pstate_cpu_oob_ids);
  2389. if (id) {
  2390. rdmsrl(MSR_MISC_PWR_MGMT, misc_pwr);
  2391. if (misc_pwr & BITMASK_OOB) {
  2392. pr_debug("Bit 8 or 18 in the MISC_PWR_MGMT MSR set\n");
  2393. pr_debug("P states are controlled in Out of Band mode by the firmware/hardware\n");
  2394. return true;
  2395. }
  2396. }
  2397. idx = acpi_match_platform_list(plat_info);
  2398. if (idx < 0)
  2399. return false;
  2400. switch (plat_info[idx].data) {
  2401. case PSS:
  2402. if (!intel_pstate_no_acpi_pss())
  2403. return false;
  2404. return intel_pstate_no_acpi_pcch();
  2405. case PPC:
  2406. return intel_pstate_has_acpi_ppc() && !force_load;
  2407. }
  2408. return false;
  2409. }
  2410. static void intel_pstate_request_control_from_smm(void)
  2411. {
  2412. /*
  2413. * It may be unsafe to request P-states control from SMM if _PPC support
  2414. * has not been enabled.
  2415. */
  2416. if (acpi_ppc)
  2417. acpi_processor_pstate_control();
  2418. }
  2419. #else /* CONFIG_ACPI not enabled */
  2420. static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; }
  2421. static inline bool intel_pstate_has_acpi_ppc(void) { return false; }
  2422. static inline void intel_pstate_request_control_from_smm(void) {}
  2423. #endif /* CONFIG_ACPI */
  2424. #define INTEL_PSTATE_HWP_BROADWELL 0x01
  2425. #define X86_MATCH_HWP(model, hwp_mode) \
  2426. X86_MATCH_VENDOR_FAM_MODEL_FEATURE(INTEL, 6, INTEL_FAM6_##model, \
  2427. X86_FEATURE_HWP, hwp_mode)
  2428. static const struct x86_cpu_id hwp_support_ids[] __initconst = {
  2429. X86_MATCH_HWP(BROADWELL_X, INTEL_PSTATE_HWP_BROADWELL),
  2430. X86_MATCH_HWP(BROADWELL_D, INTEL_PSTATE_HWP_BROADWELL),
  2431. X86_MATCH_HWP(ANY, 0),
  2432. {}
  2433. };
  2434. static bool intel_pstate_hwp_is_enabled(void)
  2435. {
  2436. u64 value;
  2437. rdmsrl(MSR_PM_ENABLE, value);
  2438. return !!(value & 0x1);
  2439. }
  2440. static int __init intel_pstate_init(void)
  2441. {
  2442. const struct x86_cpu_id *id;
  2443. int rc;
  2444. if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
  2445. return -ENODEV;
  2446. id = x86_match_cpu(hwp_support_ids);
  2447. if (id) {
  2448. bool hwp_forced = intel_pstate_hwp_is_enabled();
  2449. if (hwp_forced)
  2450. pr_info("HWP enabled by BIOS\n");
  2451. else if (no_load)
  2452. return -ENODEV;
  2453. copy_cpu_funcs(&core_funcs);
  2454. /*
  2455. * Avoid enabling HWP for processors without EPP support,
  2456. * because that means incomplete HWP implementation which is a
  2457. * corner case and supporting it is generally problematic.
  2458. *
  2459. * If HWP is enabled already, though, there is no choice but to
  2460. * deal with it.
  2461. */
  2462. if ((!no_hwp && boot_cpu_has(X86_FEATURE_HWP_EPP)) || hwp_forced) {
  2463. hwp_active++;
  2464. hwp_mode_bdw = id->driver_data;
  2465. intel_pstate.attr = hwp_cpufreq_attrs;
  2466. intel_cpufreq.attr = hwp_cpufreq_attrs;
  2467. intel_cpufreq.flags |= CPUFREQ_NEED_UPDATE_LIMITS;
  2468. if (!default_driver)
  2469. default_driver = &intel_pstate;
  2470. goto hwp_cpu_matched;
  2471. }
  2472. pr_info("HWP not enabled\n");
  2473. } else {
  2474. if (no_load)
  2475. return -ENODEV;
  2476. id = x86_match_cpu(intel_pstate_cpu_ids);
  2477. if (!id) {
  2478. pr_info("CPU model not supported\n");
  2479. return -ENODEV;
  2480. }
  2481. copy_cpu_funcs((struct pstate_funcs *)id->driver_data);
  2482. }
  2483. if (intel_pstate_msrs_not_valid()) {
  2484. pr_info("Invalid MSRs\n");
  2485. return -ENODEV;
  2486. }
  2487. /* Without HWP start in the passive mode. */
  2488. if (!default_driver)
  2489. default_driver = &intel_cpufreq;
  2490. hwp_cpu_matched:
  2491. /*
  2492. * The Intel pstate driver will be ignored if the platform
  2493. * firmware has its own power management modes.
  2494. */
  2495. if (intel_pstate_platform_pwr_mgmt_exists()) {
  2496. pr_info("P-states controlled by the platform\n");
  2497. return -ENODEV;
  2498. }
  2499. if (!hwp_active && hwp_only)
  2500. return -ENOTSUPP;
  2501. pr_info("Intel P-state driver initializing\n");
  2502. all_cpu_data = vzalloc(array_size(sizeof(void *), num_possible_cpus()));
  2503. if (!all_cpu_data)
  2504. return -ENOMEM;
  2505. intel_pstate_request_control_from_smm();
  2506. intel_pstate_sysfs_expose_params();
  2507. mutex_lock(&intel_pstate_driver_lock);
  2508. rc = intel_pstate_register_driver(default_driver);
  2509. mutex_unlock(&intel_pstate_driver_lock);
  2510. if (rc) {
  2511. intel_pstate_sysfs_remove();
  2512. return rc;
  2513. }
  2514. if (hwp_active) {
  2515. const struct x86_cpu_id *id;
  2516. id = x86_match_cpu(intel_pstate_cpu_ee_disable_ids);
  2517. if (id) {
  2518. set_power_ctl_ee_state(false);
  2519. pr_info("Disabling energy efficiency optimization\n");
  2520. }
  2521. pr_info("HWP enabled\n");
  2522. }
  2523. return 0;
  2524. }
  2525. device_initcall(intel_pstate_init);
  2526. static int __init intel_pstate_setup(char *str)
  2527. {
  2528. if (!str)
  2529. return -EINVAL;
  2530. if (!strcmp(str, "disable"))
  2531. no_load = 1;
  2532. else if (!strcmp(str, "active"))
  2533. default_driver = &intel_pstate;
  2534. else if (!strcmp(str, "passive"))
  2535. default_driver = &intel_cpufreq;
  2536. if (!strcmp(str, "no_hwp"))
  2537. no_hwp = 1;
  2538. if (!strcmp(str, "force"))
  2539. force_load = 1;
  2540. if (!strcmp(str, "hwp_only"))
  2541. hwp_only = 1;
  2542. if (!strcmp(str, "per_cpu_perf_limits"))
  2543. per_cpu_limits = true;
  2544. #ifdef CONFIG_ACPI
  2545. if (!strcmp(str, "support_acpi_ppc"))
  2546. acpi_ppc = true;
  2547. #endif
  2548. return 0;
  2549. }
  2550. early_param("intel_pstate", intel_pstate_setup);
  2551. MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>");
  2552. MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors");
  2553. MODULE_LICENSE("GPL");