intel_ips.c 42 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (c) 2009-2010 Intel Corporation
  4. *
  5. * Authors:
  6. * Jesse Barnes <jbarnes@virtuousgeek.org>
  7. */
  8. /*
  9. * Some Intel Ibex Peak based platforms support so-called "intelligent
  10. * power sharing", which allows the CPU and GPU to cooperate to maximize
  11. * performance within a given TDP (thermal design point). This driver
  12. * performs the coordination between the CPU and GPU, monitors thermal and
  13. * power statistics in the platform, and initializes power monitoring
  14. * hardware. It also provides a few tunables to control behavior. Its
  15. * primary purpose is to safely allow CPU and GPU turbo modes to be enabled
  16. * by tracking power and thermal budget; secondarily it can boost turbo
  17. * performance by allocating more power or thermal budget to the CPU or GPU
  18. * based on available headroom and activity.
  19. *
  20. * The basic algorithm is driven by a 5s moving average of temperature. If
  21. * thermal headroom is available, the CPU and/or GPU power clamps may be
  22. * adjusted upwards. If we hit the thermal ceiling or a thermal trigger,
  23. * we scale back the clamp. Aside from trigger events (when we're critically
  24. * close or over our TDP) we don't adjust the clamps more than once every
  25. * five seconds.
  26. *
  27. * The thermal device (device 31, function 6) has a set of registers that
  28. * are updated by the ME firmware. The ME should also take the clamp values
  29. * written to those registers and write them to the CPU, but we currently
  30. * bypass that functionality and write the CPU MSR directly.
  31. *
  32. * UNSUPPORTED:
  33. * - dual MCP configs
  34. *
  35. * TODO:
  36. * - handle CPU hotplug
  37. * - provide turbo enable/disable api
  38. *
  39. * Related documents:
  40. * - CDI 403777, 403778 - Auburndale EDS vol 1 & 2
  41. * - CDI 401376 - Ibex Peak EDS
  42. * - ref 26037, 26641 - IPS BIOS spec
  43. * - ref 26489 - Nehalem BIOS writer's guide
  44. * - ref 26921 - Ibex Peak BIOS Specification
  45. */
  46. #include <linux/debugfs.h>
  47. #include <linux/delay.h>
  48. #include <linux/interrupt.h>
  49. #include <linux/kernel.h>
  50. #include <linux/kthread.h>
  51. #include <linux/module.h>
  52. #include <linux/pci.h>
  53. #include <linux/sched.h>
  54. #include <linux/sched/loadavg.h>
  55. #include <linux/seq_file.h>
  56. #include <linux/string.h>
  57. #include <linux/tick.h>
  58. #include <linux/timer.h>
  59. #include <linux/dmi.h>
  60. #include <drm/i915_drm.h>
  61. #include <asm/msr.h>
  62. #include <asm/processor.h>
  63. #include "intel_ips.h"
  64. #include <linux/io-64-nonatomic-lo-hi.h>
  65. #define PCI_DEVICE_ID_INTEL_THERMAL_SENSOR 0x3b32
  66. /*
  67. * Package level MSRs for monitor/control
  68. */
  69. #define PLATFORM_INFO 0xce
  70. #define PLATFORM_TDP (1<<29)
  71. #define PLATFORM_RATIO (1<<28)
  72. #define IA32_MISC_ENABLE 0x1a0
  73. #define IA32_MISC_TURBO_EN (1ULL<<38)
  74. #define TURBO_POWER_CURRENT_LIMIT 0x1ac
  75. #define TURBO_TDC_OVR_EN (1UL<<31)
  76. #define TURBO_TDC_MASK (0x000000007fff0000UL)
  77. #define TURBO_TDC_SHIFT (16)
  78. #define TURBO_TDP_OVR_EN (1UL<<15)
  79. #define TURBO_TDP_MASK (0x0000000000003fffUL)
  80. /*
  81. * Core/thread MSRs for monitoring
  82. */
  83. #define IA32_PERF_CTL 0x199
  84. #define IA32_PERF_TURBO_DIS (1ULL<<32)
  85. /*
  86. * Thermal PCI device regs
  87. */
  88. #define THM_CFG_TBAR 0x10
  89. #define THM_CFG_TBAR_HI 0x14
  90. #define THM_TSIU 0x00
  91. #define THM_TSE 0x01
  92. #define TSE_EN 0xb8
  93. #define THM_TSS 0x02
  94. #define THM_TSTR 0x03
  95. #define THM_TSTTP 0x04
  96. #define THM_TSCO 0x08
  97. #define THM_TSES 0x0c
  98. #define THM_TSGPEN 0x0d
  99. #define TSGPEN_HOT_LOHI (1<<1)
  100. #define TSGPEN_CRIT_LOHI (1<<2)
  101. #define THM_TSPC 0x0e
  102. #define THM_PPEC 0x10
  103. #define THM_CTA 0x12
  104. #define THM_PTA 0x14
  105. #define PTA_SLOPE_MASK (0xff00)
  106. #define PTA_SLOPE_SHIFT 8
  107. #define PTA_OFFSET_MASK (0x00ff)
  108. #define THM_MGTA 0x16
  109. #define MGTA_SLOPE_MASK (0xff00)
  110. #define MGTA_SLOPE_SHIFT 8
  111. #define MGTA_OFFSET_MASK (0x00ff)
  112. #define THM_TRC 0x1a
  113. #define TRC_CORE2_EN (1<<15)
  114. #define TRC_THM_EN (1<<12)
  115. #define TRC_C6_WAR (1<<8)
  116. #define TRC_CORE1_EN (1<<7)
  117. #define TRC_CORE_PWR (1<<6)
  118. #define TRC_PCH_EN (1<<5)
  119. #define TRC_MCH_EN (1<<4)
  120. #define TRC_DIMM4 (1<<3)
  121. #define TRC_DIMM3 (1<<2)
  122. #define TRC_DIMM2 (1<<1)
  123. #define TRC_DIMM1 (1<<0)
  124. #define THM_TES 0x20
  125. #define THM_TEN 0x21
  126. #define TEN_UPDATE_EN 1
  127. #define THM_PSC 0x24
  128. #define PSC_NTG (1<<0) /* No GFX turbo support */
  129. #define PSC_NTPC (1<<1) /* No CPU turbo support */
  130. #define PSC_PP_DEF (0<<2) /* Perf policy up to driver */
  131. #define PSP_PP_PC (1<<2) /* BIOS prefers CPU perf */
  132. #define PSP_PP_BAL (2<<2) /* BIOS wants balanced perf */
  133. #define PSP_PP_GFX (3<<2) /* BIOS prefers GFX perf */
  134. #define PSP_PBRT (1<<4) /* BIOS run time support */
  135. #define THM_CTV1 0x30
  136. #define CTV_TEMP_ERROR (1<<15)
  137. #define CTV_TEMP_MASK 0x3f
  138. #define CTV_
  139. #define THM_CTV2 0x32
  140. #define THM_CEC 0x34 /* undocumented power accumulator in joules */
  141. #define THM_AE 0x3f
  142. #define THM_HTS 0x50 /* 32 bits */
  143. #define HTS_PCPL_MASK (0x7fe00000)
  144. #define HTS_PCPL_SHIFT 21
  145. #define HTS_GPL_MASK (0x001ff000)
  146. #define HTS_GPL_SHIFT 12
  147. #define HTS_PP_MASK (0x00000c00)
  148. #define HTS_PP_SHIFT 10
  149. #define HTS_PP_DEF 0
  150. #define HTS_PP_PROC 1
  151. #define HTS_PP_BAL 2
  152. #define HTS_PP_GFX 3
  153. #define HTS_PCTD_DIS (1<<9)
  154. #define HTS_GTD_DIS (1<<8)
  155. #define HTS_PTL_MASK (0x000000fe)
  156. #define HTS_PTL_SHIFT 1
  157. #define HTS_NVV (1<<0)
  158. #define THM_HTSHI 0x54 /* 16 bits */
  159. #define HTS2_PPL_MASK (0x03ff)
  160. #define HTS2_PRST_MASK (0x3c00)
  161. #define HTS2_PRST_SHIFT 10
  162. #define HTS2_PRST_UNLOADED 0
  163. #define HTS2_PRST_RUNNING 1
  164. #define HTS2_PRST_TDISOP 2 /* turbo disabled due to power */
  165. #define HTS2_PRST_TDISHT 3 /* turbo disabled due to high temp */
  166. #define HTS2_PRST_TDISUSR 4 /* user disabled turbo */
  167. #define HTS2_PRST_TDISPLAT 5 /* platform disabled turbo */
  168. #define HTS2_PRST_TDISPM 6 /* power management disabled turbo */
  169. #define HTS2_PRST_TDISERR 7 /* some kind of error disabled turbo */
  170. #define THM_PTL 0x56
  171. #define THM_MGTV 0x58
  172. #define TV_MASK 0x000000000000ff00
  173. #define TV_SHIFT 8
  174. #define THM_PTV 0x60
  175. #define PTV_MASK 0x00ff
  176. #define THM_MMGPC 0x64
  177. #define THM_MPPC 0x66
  178. #define THM_MPCPC 0x68
  179. #define THM_TSPIEN 0x82
  180. #define TSPIEN_AUX_LOHI (1<<0)
  181. #define TSPIEN_HOT_LOHI (1<<1)
  182. #define TSPIEN_CRIT_LOHI (1<<2)
  183. #define TSPIEN_AUX2_LOHI (1<<3)
  184. #define THM_TSLOCK 0x83
  185. #define THM_ATR 0x84
  186. #define THM_TOF 0x87
  187. #define THM_STS 0x98
  188. #define STS_PCPL_MASK (0x7fe00000)
  189. #define STS_PCPL_SHIFT 21
  190. #define STS_GPL_MASK (0x001ff000)
  191. #define STS_GPL_SHIFT 12
  192. #define STS_PP_MASK (0x00000c00)
  193. #define STS_PP_SHIFT 10
  194. #define STS_PP_DEF 0
  195. #define STS_PP_PROC 1
  196. #define STS_PP_BAL 2
  197. #define STS_PP_GFX 3
  198. #define STS_PCTD_DIS (1<<9)
  199. #define STS_GTD_DIS (1<<8)
  200. #define STS_PTL_MASK (0x000000fe)
  201. #define STS_PTL_SHIFT 1
  202. #define STS_NVV (1<<0)
  203. #define THM_SEC 0x9c
  204. #define SEC_ACK (1<<0)
  205. #define THM_TC3 0xa4
  206. #define THM_TC1 0xa8
  207. #define STS_PPL_MASK (0x0003ff00)
  208. #define STS_PPL_SHIFT 16
  209. #define THM_TC2 0xac
  210. #define THM_DTV 0xb0
  211. #define THM_ITV 0xd8
  212. #define ITV_ME_SEQNO_MASK 0x00ff0000 /* ME should update every ~200ms */
  213. #define ITV_ME_SEQNO_SHIFT (16)
  214. #define ITV_MCH_TEMP_MASK 0x0000ff00
  215. #define ITV_MCH_TEMP_SHIFT (8)
  216. #define ITV_PCH_TEMP_MASK 0x000000ff
  217. #define thm_readb(off) readb(ips->regmap + (off))
  218. #define thm_readw(off) readw(ips->regmap + (off))
  219. #define thm_readl(off) readl(ips->regmap + (off))
  220. #define thm_readq(off) readq(ips->regmap + (off))
  221. #define thm_writeb(off, val) writeb((val), ips->regmap + (off))
  222. #define thm_writew(off, val) writew((val), ips->regmap + (off))
  223. #define thm_writel(off, val) writel((val), ips->regmap + (off))
  224. static const int IPS_ADJUST_PERIOD = 5000; /* ms */
  225. static bool late_i915_load = false;
  226. /* For initial average collection */
  227. static const int IPS_SAMPLE_PERIOD = 200; /* ms */
  228. static const int IPS_SAMPLE_WINDOW = 5000; /* 5s moving window of samples */
  229. #define IPS_SAMPLE_COUNT (IPS_SAMPLE_WINDOW / IPS_SAMPLE_PERIOD)
  230. /* Per-SKU limits */
  231. struct ips_mcp_limits {
  232. int mcp_power_limit; /* mW units */
  233. int core_power_limit;
  234. int mch_power_limit;
  235. int core_temp_limit; /* degrees C */
  236. int mch_temp_limit;
  237. };
  238. /* Max temps are -10 degrees C to avoid PROCHOT# */
  239. static struct ips_mcp_limits ips_sv_limits = {
  240. .mcp_power_limit = 35000,
  241. .core_power_limit = 29000,
  242. .mch_power_limit = 20000,
  243. .core_temp_limit = 95,
  244. .mch_temp_limit = 90
  245. };
  246. static struct ips_mcp_limits ips_lv_limits = {
  247. .mcp_power_limit = 25000,
  248. .core_power_limit = 21000,
  249. .mch_power_limit = 13000,
  250. .core_temp_limit = 95,
  251. .mch_temp_limit = 90
  252. };
  253. static struct ips_mcp_limits ips_ulv_limits = {
  254. .mcp_power_limit = 18000,
  255. .core_power_limit = 14000,
  256. .mch_power_limit = 11000,
  257. .core_temp_limit = 95,
  258. .mch_temp_limit = 90
  259. };
  260. struct ips_driver {
  261. struct device *dev;
  262. void __iomem *regmap;
  263. int irq;
  264. struct task_struct *monitor;
  265. struct task_struct *adjust;
  266. struct dentry *debug_root;
  267. struct timer_list timer;
  268. /* Average CPU core temps (all averages in .01 degrees C for precision) */
  269. u16 ctv1_avg_temp;
  270. u16 ctv2_avg_temp;
  271. /* GMCH average */
  272. u16 mch_avg_temp;
  273. /* Average for the CPU (both cores?) */
  274. u16 mcp_avg_temp;
  275. /* Average power consumption (in mW) */
  276. u32 cpu_avg_power;
  277. u32 mch_avg_power;
  278. /* Offset values */
  279. u16 cta_val;
  280. u16 pta_val;
  281. u16 mgta_val;
  282. /* Maximums & prefs, protected by turbo status lock */
  283. spinlock_t turbo_status_lock;
  284. u16 mcp_temp_limit;
  285. u16 mcp_power_limit;
  286. u16 core_power_limit;
  287. u16 mch_power_limit;
  288. bool cpu_turbo_enabled;
  289. bool __cpu_turbo_on;
  290. bool gpu_turbo_enabled;
  291. bool __gpu_turbo_on;
  292. bool gpu_preferred;
  293. bool poll_turbo_status;
  294. bool second_cpu;
  295. bool turbo_toggle_allowed;
  296. struct ips_mcp_limits *limits;
  297. /* Optional MCH interfaces for if i915 is in use */
  298. unsigned long (*read_mch_val)(void);
  299. bool (*gpu_raise)(void);
  300. bool (*gpu_lower)(void);
  301. bool (*gpu_busy)(void);
  302. bool (*gpu_turbo_disable)(void);
  303. /* For restoration at unload */
  304. u64 orig_turbo_limit;
  305. u64 orig_turbo_ratios;
  306. };
  307. static bool
  308. ips_gpu_turbo_enabled(struct ips_driver *ips);
  309. /**
  310. * ips_cpu_busy - is CPU busy?
  311. * @ips: IPS driver struct
  312. *
  313. * Check CPU for load to see whether we should increase its thermal budget.
  314. *
  315. * RETURNS:
  316. * True if the CPU could use more power, false otherwise.
  317. */
  318. static bool ips_cpu_busy(struct ips_driver *ips)
  319. {
  320. if ((avenrun[0] >> FSHIFT) > 1)
  321. return true;
  322. return false;
  323. }
  324. /**
  325. * ips_cpu_raise - raise CPU power clamp
  326. * @ips: IPS driver struct
  327. *
  328. * Raise the CPU power clamp by %IPS_CPU_STEP, in accordance with TDP for
  329. * this platform.
  330. *
  331. * We do this by adjusting the TURBO_POWER_CURRENT_LIMIT MSR upwards (as
  332. * long as we haven't hit the TDP limit for the SKU).
  333. */
  334. static void ips_cpu_raise(struct ips_driver *ips)
  335. {
  336. u64 turbo_override;
  337. u16 cur_tdp_limit, new_tdp_limit;
  338. if (!ips->cpu_turbo_enabled)
  339. return;
  340. rdmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_override);
  341. cur_tdp_limit = turbo_override & TURBO_TDP_MASK;
  342. new_tdp_limit = cur_tdp_limit + 8; /* 1W increase */
  343. /* Clamp to SKU TDP limit */
  344. if (((new_tdp_limit * 10) / 8) > ips->core_power_limit)
  345. new_tdp_limit = cur_tdp_limit;
  346. thm_writew(THM_MPCPC, (new_tdp_limit * 10) / 8);
  347. turbo_override |= TURBO_TDC_OVR_EN | TURBO_TDP_OVR_EN;
  348. wrmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_override);
  349. turbo_override &= ~TURBO_TDP_MASK;
  350. turbo_override |= new_tdp_limit;
  351. wrmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_override);
  352. }
  353. /**
  354. * ips_cpu_lower - lower CPU power clamp
  355. * @ips: IPS driver struct
  356. *
  357. * Lower CPU power clamp b %IPS_CPU_STEP if possible.
  358. *
  359. * We do this by adjusting the TURBO_POWER_CURRENT_LIMIT MSR down, going
  360. * as low as the platform limits will allow (though we could go lower there
  361. * wouldn't be much point).
  362. */
  363. static void ips_cpu_lower(struct ips_driver *ips)
  364. {
  365. u64 turbo_override;
  366. u16 cur_limit, new_limit;
  367. rdmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_override);
  368. cur_limit = turbo_override & TURBO_TDP_MASK;
  369. new_limit = cur_limit - 8; /* 1W decrease */
  370. /* Clamp to SKU TDP limit */
  371. if (new_limit < (ips->orig_turbo_limit & TURBO_TDP_MASK))
  372. new_limit = ips->orig_turbo_limit & TURBO_TDP_MASK;
  373. thm_writew(THM_MPCPC, (new_limit * 10) / 8);
  374. turbo_override |= TURBO_TDC_OVR_EN | TURBO_TDP_OVR_EN;
  375. wrmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_override);
  376. turbo_override &= ~TURBO_TDP_MASK;
  377. turbo_override |= new_limit;
  378. wrmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_override);
  379. }
  380. /**
  381. * do_enable_cpu_turbo - internal turbo enable function
  382. * @data: unused
  383. *
  384. * Internal function for actually updating MSRs. When we enable/disable
  385. * turbo, we need to do it on each CPU; this function is the one called
  386. * by on_each_cpu() when needed.
  387. */
  388. static void do_enable_cpu_turbo(void *data)
  389. {
  390. u64 perf_ctl;
  391. rdmsrl(IA32_PERF_CTL, perf_ctl);
  392. if (perf_ctl & IA32_PERF_TURBO_DIS) {
  393. perf_ctl &= ~IA32_PERF_TURBO_DIS;
  394. wrmsrl(IA32_PERF_CTL, perf_ctl);
  395. }
  396. }
  397. /**
  398. * ips_enable_cpu_turbo - enable turbo mode on all CPUs
  399. * @ips: IPS driver struct
  400. *
  401. * Enable turbo mode by clearing the disable bit in IA32_PERF_CTL on
  402. * all logical threads.
  403. */
  404. static void ips_enable_cpu_turbo(struct ips_driver *ips)
  405. {
  406. /* Already on, no need to mess with MSRs */
  407. if (ips->__cpu_turbo_on)
  408. return;
  409. if (ips->turbo_toggle_allowed)
  410. on_each_cpu(do_enable_cpu_turbo, ips, 1);
  411. ips->__cpu_turbo_on = true;
  412. }
  413. /**
  414. * do_disable_cpu_turbo - internal turbo disable function
  415. * @data: unused
  416. *
  417. * Internal function for actually updating MSRs. When we enable/disable
  418. * turbo, we need to do it on each CPU; this function is the one called
  419. * by on_each_cpu() when needed.
  420. */
  421. static void do_disable_cpu_turbo(void *data)
  422. {
  423. u64 perf_ctl;
  424. rdmsrl(IA32_PERF_CTL, perf_ctl);
  425. if (!(perf_ctl & IA32_PERF_TURBO_DIS)) {
  426. perf_ctl |= IA32_PERF_TURBO_DIS;
  427. wrmsrl(IA32_PERF_CTL, perf_ctl);
  428. }
  429. }
  430. /**
  431. * ips_disable_cpu_turbo - disable turbo mode on all CPUs
  432. * @ips: IPS driver struct
  433. *
  434. * Disable turbo mode by setting the disable bit in IA32_PERF_CTL on
  435. * all logical threads.
  436. */
  437. static void ips_disable_cpu_turbo(struct ips_driver *ips)
  438. {
  439. /* Already off, leave it */
  440. if (!ips->__cpu_turbo_on)
  441. return;
  442. if (ips->turbo_toggle_allowed)
  443. on_each_cpu(do_disable_cpu_turbo, ips, 1);
  444. ips->__cpu_turbo_on = false;
  445. }
  446. /**
  447. * ips_gpu_busy - is GPU busy?
  448. * @ips: IPS driver struct
  449. *
  450. * Check GPU for load to see whether we should increase its thermal budget.
  451. * We need to call into the i915 driver in this case.
  452. *
  453. * RETURNS:
  454. * True if the GPU could use more power, false otherwise.
  455. */
  456. static bool ips_gpu_busy(struct ips_driver *ips)
  457. {
  458. if (!ips_gpu_turbo_enabled(ips))
  459. return false;
  460. return ips->gpu_busy();
  461. }
  462. /**
  463. * ips_gpu_raise - raise GPU power clamp
  464. * @ips: IPS driver struct
  465. *
  466. * Raise the GPU frequency/power if possible. We need to call into the
  467. * i915 driver in this case.
  468. */
  469. static void ips_gpu_raise(struct ips_driver *ips)
  470. {
  471. if (!ips_gpu_turbo_enabled(ips))
  472. return;
  473. if (!ips->gpu_raise())
  474. ips->gpu_turbo_enabled = false;
  475. return;
  476. }
  477. /**
  478. * ips_gpu_lower - lower GPU power clamp
  479. * @ips: IPS driver struct
  480. *
  481. * Lower GPU frequency/power if possible. Need to call i915.
  482. */
  483. static void ips_gpu_lower(struct ips_driver *ips)
  484. {
  485. if (!ips_gpu_turbo_enabled(ips))
  486. return;
  487. if (!ips->gpu_lower())
  488. ips->gpu_turbo_enabled = false;
  489. return;
  490. }
  491. /**
  492. * ips_enable_gpu_turbo - notify the gfx driver turbo is available
  493. * @ips: IPS driver struct
  494. *
  495. * Call into the graphics driver indicating that it can safely use
  496. * turbo mode.
  497. */
  498. static void ips_enable_gpu_turbo(struct ips_driver *ips)
  499. {
  500. if (ips->__gpu_turbo_on)
  501. return;
  502. ips->__gpu_turbo_on = true;
  503. }
  504. /**
  505. * ips_disable_gpu_turbo - notify the gfx driver to disable turbo mode
  506. * @ips: IPS driver struct
  507. *
  508. * Request that the graphics driver disable turbo mode.
  509. */
  510. static void ips_disable_gpu_turbo(struct ips_driver *ips)
  511. {
  512. /* Avoid calling i915 if turbo is already disabled */
  513. if (!ips->__gpu_turbo_on)
  514. return;
  515. if (!ips->gpu_turbo_disable())
  516. dev_err(ips->dev, "failed to disable graphics turbo\n");
  517. else
  518. ips->__gpu_turbo_on = false;
  519. }
  520. /**
  521. * mcp_exceeded - check whether we're outside our thermal & power limits
  522. * @ips: IPS driver struct
  523. *
  524. * Check whether the MCP is over its thermal or power budget.
  525. */
  526. static bool mcp_exceeded(struct ips_driver *ips)
  527. {
  528. unsigned long flags;
  529. bool ret = false;
  530. u32 temp_limit;
  531. u32 avg_power;
  532. spin_lock_irqsave(&ips->turbo_status_lock, flags);
  533. temp_limit = ips->mcp_temp_limit * 100;
  534. if (ips->mcp_avg_temp > temp_limit)
  535. ret = true;
  536. avg_power = ips->cpu_avg_power + ips->mch_avg_power;
  537. if (avg_power > ips->mcp_power_limit)
  538. ret = true;
  539. spin_unlock_irqrestore(&ips->turbo_status_lock, flags);
  540. return ret;
  541. }
  542. /**
  543. * cpu_exceeded - check whether a CPU core is outside its limits
  544. * @ips: IPS driver struct
  545. * @cpu: CPU number to check
  546. *
  547. * Check a given CPU's average temp or power is over its limit.
  548. */
  549. static bool cpu_exceeded(struct ips_driver *ips, int cpu)
  550. {
  551. unsigned long flags;
  552. int avg;
  553. bool ret = false;
  554. spin_lock_irqsave(&ips->turbo_status_lock, flags);
  555. avg = cpu ? ips->ctv2_avg_temp : ips->ctv1_avg_temp;
  556. if (avg > (ips->limits->core_temp_limit * 100))
  557. ret = true;
  558. if (ips->cpu_avg_power > ips->core_power_limit * 100)
  559. ret = true;
  560. spin_unlock_irqrestore(&ips->turbo_status_lock, flags);
  561. if (ret)
  562. dev_info(ips->dev, "CPU power or thermal limit exceeded\n");
  563. return ret;
  564. }
  565. /**
  566. * mch_exceeded - check whether the GPU is over budget
  567. * @ips: IPS driver struct
  568. *
  569. * Check the MCH temp & power against their maximums.
  570. */
  571. static bool mch_exceeded(struct ips_driver *ips)
  572. {
  573. unsigned long flags;
  574. bool ret = false;
  575. spin_lock_irqsave(&ips->turbo_status_lock, flags);
  576. if (ips->mch_avg_temp > (ips->limits->mch_temp_limit * 100))
  577. ret = true;
  578. if (ips->mch_avg_power > ips->mch_power_limit)
  579. ret = true;
  580. spin_unlock_irqrestore(&ips->turbo_status_lock, flags);
  581. return ret;
  582. }
  583. /**
  584. * verify_limits - verify BIOS provided limits
  585. * @ips: IPS structure
  586. *
  587. * BIOS can optionally provide non-default limits for power and temp. Check
  588. * them here and use the defaults if the BIOS values are not provided or
  589. * are otherwise unusable.
  590. */
  591. static void verify_limits(struct ips_driver *ips)
  592. {
  593. if (ips->mcp_power_limit < ips->limits->mcp_power_limit ||
  594. ips->mcp_power_limit > 35000)
  595. ips->mcp_power_limit = ips->limits->mcp_power_limit;
  596. if (ips->mcp_temp_limit < ips->limits->core_temp_limit ||
  597. ips->mcp_temp_limit < ips->limits->mch_temp_limit ||
  598. ips->mcp_temp_limit > 150)
  599. ips->mcp_temp_limit = min(ips->limits->core_temp_limit,
  600. ips->limits->mch_temp_limit);
  601. }
  602. /**
  603. * update_turbo_limits - get various limits & settings from regs
  604. * @ips: IPS driver struct
  605. *
  606. * Update the IPS power & temp limits, along with turbo enable flags,
  607. * based on latest register contents.
  608. *
  609. * Used at init time and for runtime BIOS support, which requires polling
  610. * the regs for updates (as a result of AC->DC transition for example).
  611. *
  612. * LOCKING:
  613. * Caller must hold turbo_status_lock (outside of init)
  614. */
  615. static void update_turbo_limits(struct ips_driver *ips)
  616. {
  617. u32 hts = thm_readl(THM_HTS);
  618. ips->cpu_turbo_enabled = !(hts & HTS_PCTD_DIS);
  619. /*
  620. * Disable turbo for now, until we can figure out why the power figures
  621. * are wrong
  622. */
  623. ips->cpu_turbo_enabled = false;
  624. if (ips->gpu_busy)
  625. ips->gpu_turbo_enabled = !(hts & HTS_GTD_DIS);
  626. ips->core_power_limit = thm_readw(THM_MPCPC);
  627. ips->mch_power_limit = thm_readw(THM_MMGPC);
  628. ips->mcp_temp_limit = thm_readw(THM_PTL);
  629. ips->mcp_power_limit = thm_readw(THM_MPPC);
  630. verify_limits(ips);
  631. /* Ignore BIOS CPU vs GPU pref */
  632. }
  633. /**
  634. * ips_adjust - adjust power clamp based on thermal state
  635. * @data: ips driver structure
  636. *
  637. * Wake up every 5s or so and check whether we should adjust the power clamp.
  638. * Check CPU and GPU load to determine which needs adjustment. There are
  639. * several things to consider here:
  640. * - do we need to adjust up or down?
  641. * - is CPU busy?
  642. * - is GPU busy?
  643. * - is CPU in turbo?
  644. * - is GPU in turbo?
  645. * - is CPU or GPU preferred? (CPU is default)
  646. *
  647. * So, given the above, we do the following:
  648. * - up (TDP available)
  649. * - CPU not busy, GPU not busy - nothing
  650. * - CPU busy, GPU not busy - adjust CPU up
  651. * - CPU not busy, GPU busy - adjust GPU up
  652. * - CPU busy, GPU busy - adjust preferred unit up, taking headroom from
  653. * non-preferred unit if necessary
  654. * - down (at TDP limit)
  655. * - adjust both CPU and GPU down if possible
  656. *
  657. cpu+ gpu+ cpu+gpu- cpu-gpu+ cpu-gpu-
  658. cpu < gpu < cpu+gpu+ cpu+ gpu+ nothing
  659. cpu < gpu >= cpu+gpu-(mcp<) cpu+gpu-(mcp<) gpu- gpu-
  660. cpu >= gpu < cpu-gpu+(mcp<) cpu- cpu-gpu+(mcp<) cpu-
  661. cpu >= gpu >= cpu-gpu- cpu-gpu- cpu-gpu- cpu-gpu-
  662. *
  663. */
  664. static int ips_adjust(void *data)
  665. {
  666. struct ips_driver *ips = data;
  667. unsigned long flags;
  668. dev_dbg(ips->dev, "starting ips-adjust thread\n");
  669. /*
  670. * Adjust CPU and GPU clamps every 5s if needed. Doing it more
  671. * often isn't recommended due to ME interaction.
  672. */
  673. do {
  674. bool cpu_busy = ips_cpu_busy(ips);
  675. bool gpu_busy = ips_gpu_busy(ips);
  676. spin_lock_irqsave(&ips->turbo_status_lock, flags);
  677. if (ips->poll_turbo_status)
  678. update_turbo_limits(ips);
  679. spin_unlock_irqrestore(&ips->turbo_status_lock, flags);
  680. /* Update turbo status if necessary */
  681. if (ips->cpu_turbo_enabled)
  682. ips_enable_cpu_turbo(ips);
  683. else
  684. ips_disable_cpu_turbo(ips);
  685. if (ips->gpu_turbo_enabled)
  686. ips_enable_gpu_turbo(ips);
  687. else
  688. ips_disable_gpu_turbo(ips);
  689. /* We're outside our comfort zone, crank them down */
  690. if (mcp_exceeded(ips)) {
  691. ips_cpu_lower(ips);
  692. ips_gpu_lower(ips);
  693. goto sleep;
  694. }
  695. if (!cpu_exceeded(ips, 0) && cpu_busy)
  696. ips_cpu_raise(ips);
  697. else
  698. ips_cpu_lower(ips);
  699. if (!mch_exceeded(ips) && gpu_busy)
  700. ips_gpu_raise(ips);
  701. else
  702. ips_gpu_lower(ips);
  703. sleep:
  704. schedule_timeout_interruptible(msecs_to_jiffies(IPS_ADJUST_PERIOD));
  705. } while (!kthread_should_stop());
  706. dev_dbg(ips->dev, "ips-adjust thread stopped\n");
  707. return 0;
  708. }
  709. /*
  710. * Helpers for reading out temp/power values and calculating their
  711. * averages for the decision making and monitoring functions.
  712. */
  713. static u16 calc_avg_temp(struct ips_driver *ips, u16 *array)
  714. {
  715. u64 total = 0;
  716. int i;
  717. u16 avg;
  718. for (i = 0; i < IPS_SAMPLE_COUNT; i++)
  719. total += (u64)(array[i] * 100);
  720. do_div(total, IPS_SAMPLE_COUNT);
  721. avg = (u16)total;
  722. return avg;
  723. }
  724. static u16 read_mgtv(struct ips_driver *ips)
  725. {
  726. u16 ret;
  727. u64 slope, offset;
  728. u64 val;
  729. val = thm_readq(THM_MGTV);
  730. val = (val & TV_MASK) >> TV_SHIFT;
  731. slope = offset = thm_readw(THM_MGTA);
  732. slope = (slope & MGTA_SLOPE_MASK) >> MGTA_SLOPE_SHIFT;
  733. offset = offset & MGTA_OFFSET_MASK;
  734. ret = ((val * slope + 0x40) >> 7) + offset;
  735. return 0; /* MCH temp reporting buggy */
  736. }
  737. static u16 read_ptv(struct ips_driver *ips)
  738. {
  739. u16 val;
  740. val = thm_readw(THM_PTV) & PTV_MASK;
  741. return val;
  742. }
  743. static u16 read_ctv(struct ips_driver *ips, int cpu)
  744. {
  745. int reg = cpu ? THM_CTV2 : THM_CTV1;
  746. u16 val;
  747. val = thm_readw(reg);
  748. if (!(val & CTV_TEMP_ERROR))
  749. val = (val) >> 6; /* discard fractional component */
  750. else
  751. val = 0;
  752. return val;
  753. }
  754. static u32 get_cpu_power(struct ips_driver *ips, u32 *last, int period)
  755. {
  756. u32 val;
  757. u32 ret;
  758. /*
  759. * CEC is in joules/65535. Take difference over time to
  760. * get watts.
  761. */
  762. val = thm_readl(THM_CEC);
  763. /* period is in ms and we want mW */
  764. ret = (((val - *last) * 1000) / period);
  765. ret = (ret * 1000) / 65535;
  766. *last = val;
  767. return 0;
  768. }
  769. static const u16 temp_decay_factor = 2;
  770. static u16 update_average_temp(u16 avg, u16 val)
  771. {
  772. u16 ret;
  773. /* Multiply by 100 for extra precision */
  774. ret = (val * 100 / temp_decay_factor) +
  775. (((temp_decay_factor - 1) * avg) / temp_decay_factor);
  776. return ret;
  777. }
  778. static const u16 power_decay_factor = 2;
  779. static u16 update_average_power(u32 avg, u32 val)
  780. {
  781. u32 ret;
  782. ret = (val / power_decay_factor) +
  783. (((power_decay_factor - 1) * avg) / power_decay_factor);
  784. return ret;
  785. }
  786. static u32 calc_avg_power(struct ips_driver *ips, u32 *array)
  787. {
  788. u64 total = 0;
  789. u32 avg;
  790. int i;
  791. for (i = 0; i < IPS_SAMPLE_COUNT; i++)
  792. total += array[i];
  793. do_div(total, IPS_SAMPLE_COUNT);
  794. avg = (u32)total;
  795. return avg;
  796. }
  797. static void monitor_timeout(struct timer_list *t)
  798. {
  799. struct ips_driver *ips = from_timer(ips, t, timer);
  800. wake_up_process(ips->monitor);
  801. }
  802. /**
  803. * ips_monitor - temp/power monitoring thread
  804. * @data: ips driver structure
  805. *
  806. * This is the main function for the IPS driver. It monitors power and
  807. * tempurature in the MCP and adjusts CPU and GPU power clams accordingly.
  808. *
  809. * We keep a 5s moving average of power consumption and tempurature. Using
  810. * that data, along with CPU vs GPU preference, we adjust the power clamps
  811. * up or down.
  812. */
  813. static int ips_monitor(void *data)
  814. {
  815. struct ips_driver *ips = data;
  816. unsigned long seqno_timestamp, expire, last_msecs, last_sample_period;
  817. int i;
  818. u32 *cpu_samples, *mchp_samples, old_cpu_power;
  819. u16 *mcp_samples, *ctv1_samples, *ctv2_samples, *mch_samples;
  820. u8 cur_seqno, last_seqno;
  821. mcp_samples = kcalloc(IPS_SAMPLE_COUNT, sizeof(u16), GFP_KERNEL);
  822. ctv1_samples = kcalloc(IPS_SAMPLE_COUNT, sizeof(u16), GFP_KERNEL);
  823. ctv2_samples = kcalloc(IPS_SAMPLE_COUNT, sizeof(u16), GFP_KERNEL);
  824. mch_samples = kcalloc(IPS_SAMPLE_COUNT, sizeof(u16), GFP_KERNEL);
  825. cpu_samples = kcalloc(IPS_SAMPLE_COUNT, sizeof(u32), GFP_KERNEL);
  826. mchp_samples = kcalloc(IPS_SAMPLE_COUNT, sizeof(u32), GFP_KERNEL);
  827. if (!mcp_samples || !ctv1_samples || !ctv2_samples || !mch_samples ||
  828. !cpu_samples || !mchp_samples) {
  829. dev_err(ips->dev,
  830. "failed to allocate sample array, ips disabled\n");
  831. kfree(mcp_samples);
  832. kfree(ctv1_samples);
  833. kfree(ctv2_samples);
  834. kfree(mch_samples);
  835. kfree(cpu_samples);
  836. kfree(mchp_samples);
  837. return -ENOMEM;
  838. }
  839. last_seqno = (thm_readl(THM_ITV) & ITV_ME_SEQNO_MASK) >>
  840. ITV_ME_SEQNO_SHIFT;
  841. seqno_timestamp = get_jiffies_64();
  842. old_cpu_power = thm_readl(THM_CEC);
  843. schedule_timeout_interruptible(msecs_to_jiffies(IPS_SAMPLE_PERIOD));
  844. /* Collect an initial average */
  845. for (i = 0; i < IPS_SAMPLE_COUNT; i++) {
  846. u32 mchp, cpu_power;
  847. u16 val;
  848. mcp_samples[i] = read_ptv(ips);
  849. val = read_ctv(ips, 0);
  850. ctv1_samples[i] = val;
  851. val = read_ctv(ips, 1);
  852. ctv2_samples[i] = val;
  853. val = read_mgtv(ips);
  854. mch_samples[i] = val;
  855. cpu_power = get_cpu_power(ips, &old_cpu_power,
  856. IPS_SAMPLE_PERIOD);
  857. cpu_samples[i] = cpu_power;
  858. if (ips->read_mch_val) {
  859. mchp = ips->read_mch_val();
  860. mchp_samples[i] = mchp;
  861. }
  862. schedule_timeout_interruptible(msecs_to_jiffies(IPS_SAMPLE_PERIOD));
  863. if (kthread_should_stop())
  864. break;
  865. }
  866. ips->mcp_avg_temp = calc_avg_temp(ips, mcp_samples);
  867. ips->ctv1_avg_temp = calc_avg_temp(ips, ctv1_samples);
  868. ips->ctv2_avg_temp = calc_avg_temp(ips, ctv2_samples);
  869. ips->mch_avg_temp = calc_avg_temp(ips, mch_samples);
  870. ips->cpu_avg_power = calc_avg_power(ips, cpu_samples);
  871. ips->mch_avg_power = calc_avg_power(ips, mchp_samples);
  872. kfree(mcp_samples);
  873. kfree(ctv1_samples);
  874. kfree(ctv2_samples);
  875. kfree(mch_samples);
  876. kfree(cpu_samples);
  877. kfree(mchp_samples);
  878. /* Start the adjustment thread now that we have data */
  879. wake_up_process(ips->adjust);
  880. /*
  881. * Ok, now we have an initial avg. From here on out, we track the
  882. * running avg using a decaying average calculation. This allows
  883. * us to reduce the sample frequency if the CPU and GPU are idle.
  884. */
  885. old_cpu_power = thm_readl(THM_CEC);
  886. schedule_timeout_interruptible(msecs_to_jiffies(IPS_SAMPLE_PERIOD));
  887. last_sample_period = IPS_SAMPLE_PERIOD;
  888. timer_setup(&ips->timer, monitor_timeout, TIMER_DEFERRABLE);
  889. do {
  890. u32 cpu_val, mch_val;
  891. u16 val;
  892. /* MCP itself */
  893. val = read_ptv(ips);
  894. ips->mcp_avg_temp = update_average_temp(ips->mcp_avg_temp, val);
  895. /* Processor 0 */
  896. val = read_ctv(ips, 0);
  897. ips->ctv1_avg_temp =
  898. update_average_temp(ips->ctv1_avg_temp, val);
  899. /* Power */
  900. cpu_val = get_cpu_power(ips, &old_cpu_power,
  901. last_sample_period);
  902. ips->cpu_avg_power =
  903. update_average_power(ips->cpu_avg_power, cpu_val);
  904. if (ips->second_cpu) {
  905. /* Processor 1 */
  906. val = read_ctv(ips, 1);
  907. ips->ctv2_avg_temp =
  908. update_average_temp(ips->ctv2_avg_temp, val);
  909. }
  910. /* MCH */
  911. val = read_mgtv(ips);
  912. ips->mch_avg_temp = update_average_temp(ips->mch_avg_temp, val);
  913. /* Power */
  914. if (ips->read_mch_val) {
  915. mch_val = ips->read_mch_val();
  916. ips->mch_avg_power =
  917. update_average_power(ips->mch_avg_power,
  918. mch_val);
  919. }
  920. /*
  921. * Make sure ME is updating thermal regs.
  922. * Note:
  923. * If it's been more than a second since the last update,
  924. * the ME is probably hung.
  925. */
  926. cur_seqno = (thm_readl(THM_ITV) & ITV_ME_SEQNO_MASK) >>
  927. ITV_ME_SEQNO_SHIFT;
  928. if (cur_seqno == last_seqno &&
  929. time_after(jiffies, seqno_timestamp + HZ)) {
  930. dev_warn(ips->dev,
  931. "ME failed to update for more than 1s, likely hung\n");
  932. } else {
  933. seqno_timestamp = get_jiffies_64();
  934. last_seqno = cur_seqno;
  935. }
  936. last_msecs = jiffies_to_msecs(jiffies);
  937. expire = jiffies + msecs_to_jiffies(IPS_SAMPLE_PERIOD);
  938. __set_current_state(TASK_INTERRUPTIBLE);
  939. mod_timer(&ips->timer, expire);
  940. schedule();
  941. /* Calculate actual sample period for power averaging */
  942. last_sample_period = jiffies_to_msecs(jiffies) - last_msecs;
  943. if (!last_sample_period)
  944. last_sample_period = 1;
  945. } while (!kthread_should_stop());
  946. del_timer_sync(&ips->timer);
  947. dev_dbg(ips->dev, "ips-monitor thread stopped\n");
  948. return 0;
  949. }
  950. #if 0
  951. #define THM_DUMPW(reg) \
  952. { \
  953. u16 val = thm_readw(reg); \
  954. dev_dbg(ips->dev, #reg ": 0x%04x\n", val); \
  955. }
  956. #define THM_DUMPL(reg) \
  957. { \
  958. u32 val = thm_readl(reg); \
  959. dev_dbg(ips->dev, #reg ": 0x%08x\n", val); \
  960. }
  961. #define THM_DUMPQ(reg) \
  962. { \
  963. u64 val = thm_readq(reg); \
  964. dev_dbg(ips->dev, #reg ": 0x%016x\n", val); \
  965. }
  966. static void dump_thermal_info(struct ips_driver *ips)
  967. {
  968. u16 ptl;
  969. ptl = thm_readw(THM_PTL);
  970. dev_dbg(ips->dev, "Processor temp limit: %d\n", ptl);
  971. THM_DUMPW(THM_CTA);
  972. THM_DUMPW(THM_TRC);
  973. THM_DUMPW(THM_CTV1);
  974. THM_DUMPL(THM_STS);
  975. THM_DUMPW(THM_PTV);
  976. THM_DUMPQ(THM_MGTV);
  977. }
  978. #endif
  979. /**
  980. * ips_irq_handler - handle temperature triggers and other IPS events
  981. * @irq: irq number
  982. * @arg: unused
  983. *
  984. * Handle temperature limit trigger events, generally by lowering the clamps.
  985. * If we're at a critical limit, we clamp back to the lowest possible value
  986. * to prevent emergency shutdown.
  987. */
  988. static irqreturn_t ips_irq_handler(int irq, void *arg)
  989. {
  990. struct ips_driver *ips = arg;
  991. u8 tses = thm_readb(THM_TSES);
  992. u8 tes = thm_readb(THM_TES);
  993. if (!tses && !tes)
  994. return IRQ_NONE;
  995. dev_info(ips->dev, "TSES: 0x%02x\n", tses);
  996. dev_info(ips->dev, "TES: 0x%02x\n", tes);
  997. /* STS update from EC? */
  998. if (tes & 1) {
  999. u32 sts, tc1;
  1000. sts = thm_readl(THM_STS);
  1001. tc1 = thm_readl(THM_TC1);
  1002. if (sts & STS_NVV) {
  1003. spin_lock(&ips->turbo_status_lock);
  1004. ips->core_power_limit = (sts & STS_PCPL_MASK) >>
  1005. STS_PCPL_SHIFT;
  1006. ips->mch_power_limit = (sts & STS_GPL_MASK) >>
  1007. STS_GPL_SHIFT;
  1008. /* ignore EC CPU vs GPU pref */
  1009. ips->cpu_turbo_enabled = !(sts & STS_PCTD_DIS);
  1010. /*
  1011. * Disable turbo for now, until we can figure
  1012. * out why the power figures are wrong
  1013. */
  1014. ips->cpu_turbo_enabled = false;
  1015. if (ips->gpu_busy)
  1016. ips->gpu_turbo_enabled = !(sts & STS_GTD_DIS);
  1017. ips->mcp_temp_limit = (sts & STS_PTL_MASK) >>
  1018. STS_PTL_SHIFT;
  1019. ips->mcp_power_limit = (tc1 & STS_PPL_MASK) >>
  1020. STS_PPL_SHIFT;
  1021. verify_limits(ips);
  1022. spin_unlock(&ips->turbo_status_lock);
  1023. thm_writeb(THM_SEC, SEC_ACK);
  1024. }
  1025. thm_writeb(THM_TES, tes);
  1026. }
  1027. /* Thermal trip */
  1028. if (tses) {
  1029. dev_warn(ips->dev, "thermal trip occurred, tses: 0x%04x\n",
  1030. tses);
  1031. thm_writeb(THM_TSES, tses);
  1032. }
  1033. return IRQ_HANDLED;
  1034. }
  1035. #ifndef CONFIG_DEBUG_FS
  1036. static void ips_debugfs_init(struct ips_driver *ips) { return; }
  1037. static void ips_debugfs_cleanup(struct ips_driver *ips) { return; }
  1038. #else
  1039. /* Expose current state and limits in debugfs if possible */
  1040. static int cpu_temp_show(struct seq_file *m, void *data)
  1041. {
  1042. struct ips_driver *ips = m->private;
  1043. seq_printf(m, "%d.%02d\n", ips->ctv1_avg_temp / 100,
  1044. ips->ctv1_avg_temp % 100);
  1045. return 0;
  1046. }
  1047. DEFINE_SHOW_ATTRIBUTE(cpu_temp);
  1048. static int cpu_power_show(struct seq_file *m, void *data)
  1049. {
  1050. struct ips_driver *ips = m->private;
  1051. seq_printf(m, "%dmW\n", ips->cpu_avg_power);
  1052. return 0;
  1053. }
  1054. DEFINE_SHOW_ATTRIBUTE(cpu_power);
  1055. static int cpu_clamp_show(struct seq_file *m, void *data)
  1056. {
  1057. u64 turbo_override;
  1058. int tdp, tdc;
  1059. rdmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_override);
  1060. tdp = (int)(turbo_override & TURBO_TDP_MASK);
  1061. tdc = (int)((turbo_override & TURBO_TDC_MASK) >> TURBO_TDC_SHIFT);
  1062. /* Convert to .1W/A units */
  1063. tdp = tdp * 10 / 8;
  1064. tdc = tdc * 10 / 8;
  1065. /* Watts Amperes */
  1066. seq_printf(m, "%d.%dW %d.%dA\n", tdp / 10, tdp % 10,
  1067. tdc / 10, tdc % 10);
  1068. return 0;
  1069. }
  1070. DEFINE_SHOW_ATTRIBUTE(cpu_clamp);
  1071. static int mch_temp_show(struct seq_file *m, void *data)
  1072. {
  1073. struct ips_driver *ips = m->private;
  1074. seq_printf(m, "%d.%02d\n", ips->mch_avg_temp / 100,
  1075. ips->mch_avg_temp % 100);
  1076. return 0;
  1077. }
  1078. DEFINE_SHOW_ATTRIBUTE(mch_temp);
  1079. static int mch_power_show(struct seq_file *m, void *data)
  1080. {
  1081. struct ips_driver *ips = m->private;
  1082. seq_printf(m, "%dmW\n", ips->mch_avg_power);
  1083. return 0;
  1084. }
  1085. DEFINE_SHOW_ATTRIBUTE(mch_power);
  1086. static void ips_debugfs_cleanup(struct ips_driver *ips)
  1087. {
  1088. debugfs_remove_recursive(ips->debug_root);
  1089. }
  1090. static void ips_debugfs_init(struct ips_driver *ips)
  1091. {
  1092. ips->debug_root = debugfs_create_dir("ips", NULL);
  1093. debugfs_create_file("cpu_temp", 0444, ips->debug_root, ips, &cpu_temp_fops);
  1094. debugfs_create_file("cpu_power", 0444, ips->debug_root, ips, &cpu_power_fops);
  1095. debugfs_create_file("cpu_clamp", 0444, ips->debug_root, ips, &cpu_clamp_fops);
  1096. debugfs_create_file("mch_temp", 0444, ips->debug_root, ips, &mch_temp_fops);
  1097. debugfs_create_file("mch_power", 0444, ips->debug_root, ips, &mch_power_fops);
  1098. }
  1099. #endif /* CONFIG_DEBUG_FS */
  1100. /**
  1101. * ips_detect_cpu - detect whether CPU supports IPS
  1102. *
  1103. * Walk our list and see if we're on a supported CPU. If we find one,
  1104. * return the limits for it.
  1105. */
  1106. static struct ips_mcp_limits *ips_detect_cpu(struct ips_driver *ips)
  1107. {
  1108. u64 turbo_power, misc_en;
  1109. struct ips_mcp_limits *limits = NULL;
  1110. u16 tdp;
  1111. if (!(boot_cpu_data.x86 == 6 && boot_cpu_data.x86_model == 37)) {
  1112. dev_info(ips->dev, "Non-IPS CPU detected.\n");
  1113. return NULL;
  1114. }
  1115. rdmsrl(IA32_MISC_ENABLE, misc_en);
  1116. /*
  1117. * If the turbo enable bit isn't set, we shouldn't try to enable/disable
  1118. * turbo manually or we'll get an illegal MSR access, even though
  1119. * turbo will still be available.
  1120. */
  1121. if (misc_en & IA32_MISC_TURBO_EN)
  1122. ips->turbo_toggle_allowed = true;
  1123. else
  1124. ips->turbo_toggle_allowed = false;
  1125. if (strstr(boot_cpu_data.x86_model_id, "CPU M"))
  1126. limits = &ips_sv_limits;
  1127. else if (strstr(boot_cpu_data.x86_model_id, "CPU L"))
  1128. limits = &ips_lv_limits;
  1129. else if (strstr(boot_cpu_data.x86_model_id, "CPU U"))
  1130. limits = &ips_ulv_limits;
  1131. else {
  1132. dev_info(ips->dev, "No CPUID match found.\n");
  1133. return NULL;
  1134. }
  1135. rdmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_power);
  1136. tdp = turbo_power & TURBO_TDP_MASK;
  1137. /* Sanity check TDP against CPU */
  1138. if (limits->core_power_limit != (tdp / 8) * 1000) {
  1139. dev_info(ips->dev,
  1140. "CPU TDP doesn't match expected value (found %d, expected %d)\n",
  1141. tdp / 8, limits->core_power_limit / 1000);
  1142. limits->core_power_limit = (tdp / 8) * 1000;
  1143. }
  1144. return limits;
  1145. }
  1146. /**
  1147. * ips_get_i915_syms - try to get GPU control methods from i915 driver
  1148. * @ips: IPS driver
  1149. *
  1150. * The i915 driver exports several interfaces to allow the IPS driver to
  1151. * monitor and control graphics turbo mode. If we can find them, we can
  1152. * enable graphics turbo, otherwise we must disable it to avoid exceeding
  1153. * thermal and power limits in the MCP.
  1154. */
  1155. static bool ips_get_i915_syms(struct ips_driver *ips)
  1156. {
  1157. ips->read_mch_val = symbol_get(i915_read_mch_val);
  1158. if (!ips->read_mch_val)
  1159. goto out_err;
  1160. ips->gpu_raise = symbol_get(i915_gpu_raise);
  1161. if (!ips->gpu_raise)
  1162. goto out_put_mch;
  1163. ips->gpu_lower = symbol_get(i915_gpu_lower);
  1164. if (!ips->gpu_lower)
  1165. goto out_put_raise;
  1166. ips->gpu_busy = symbol_get(i915_gpu_busy);
  1167. if (!ips->gpu_busy)
  1168. goto out_put_lower;
  1169. ips->gpu_turbo_disable = symbol_get(i915_gpu_turbo_disable);
  1170. if (!ips->gpu_turbo_disable)
  1171. goto out_put_busy;
  1172. return true;
  1173. out_put_busy:
  1174. symbol_put(i915_gpu_busy);
  1175. out_put_lower:
  1176. symbol_put(i915_gpu_lower);
  1177. out_put_raise:
  1178. symbol_put(i915_gpu_raise);
  1179. out_put_mch:
  1180. symbol_put(i915_read_mch_val);
  1181. out_err:
  1182. return false;
  1183. }
  1184. static bool
  1185. ips_gpu_turbo_enabled(struct ips_driver *ips)
  1186. {
  1187. if (!ips->gpu_busy && late_i915_load) {
  1188. if (ips_get_i915_syms(ips)) {
  1189. dev_info(ips->dev,
  1190. "i915 driver attached, reenabling gpu turbo\n");
  1191. ips->gpu_turbo_enabled = !(thm_readl(THM_HTS) & HTS_GTD_DIS);
  1192. }
  1193. }
  1194. return ips->gpu_turbo_enabled;
  1195. }
  1196. void
  1197. ips_link_to_i915_driver(void)
  1198. {
  1199. /* We can't cleanly get at the various ips_driver structs from
  1200. * this caller (the i915 driver), so just set a flag saying
  1201. * that it's time to try getting the symbols again.
  1202. */
  1203. late_i915_load = true;
  1204. }
  1205. EXPORT_SYMBOL_GPL(ips_link_to_i915_driver);
  1206. static const struct pci_device_id ips_id_table[] = {
  1207. { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_THERMAL_SENSOR), },
  1208. { 0, }
  1209. };
  1210. MODULE_DEVICE_TABLE(pci, ips_id_table);
  1211. static int ips_blacklist_callback(const struct dmi_system_id *id)
  1212. {
  1213. pr_info("Blacklisted intel_ips for %s\n", id->ident);
  1214. return 1;
  1215. }
  1216. static const struct dmi_system_id ips_blacklist[] = {
  1217. {
  1218. .callback = ips_blacklist_callback,
  1219. .ident = "HP ProBook",
  1220. .matches = {
  1221. DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
  1222. DMI_MATCH(DMI_PRODUCT_NAME, "HP ProBook"),
  1223. },
  1224. },
  1225. { } /* terminating entry */
  1226. };
  1227. static int ips_probe(struct pci_dev *dev, const struct pci_device_id *id)
  1228. {
  1229. u64 platform_info;
  1230. struct ips_driver *ips;
  1231. u32 hts;
  1232. int ret = 0;
  1233. u16 htshi, trc, trc_required_mask;
  1234. u8 tse;
  1235. if (dmi_check_system(ips_blacklist))
  1236. return -ENODEV;
  1237. ips = devm_kzalloc(&dev->dev, sizeof(*ips), GFP_KERNEL);
  1238. if (!ips)
  1239. return -ENOMEM;
  1240. spin_lock_init(&ips->turbo_status_lock);
  1241. ips->dev = &dev->dev;
  1242. ips->limits = ips_detect_cpu(ips);
  1243. if (!ips->limits) {
  1244. dev_info(&dev->dev, "IPS not supported on this CPU\n");
  1245. return -ENXIO;
  1246. }
  1247. ret = pcim_enable_device(dev);
  1248. if (ret) {
  1249. dev_err(&dev->dev, "can't enable PCI device, aborting\n");
  1250. return ret;
  1251. }
  1252. ret = pcim_iomap_regions(dev, 1 << 0, pci_name(dev));
  1253. if (ret) {
  1254. dev_err(&dev->dev, "failed to map thermal regs, aborting\n");
  1255. return ret;
  1256. }
  1257. ips->regmap = pcim_iomap_table(dev)[0];
  1258. pci_set_drvdata(dev, ips);
  1259. tse = thm_readb(THM_TSE);
  1260. if (tse != TSE_EN) {
  1261. dev_err(&dev->dev, "thermal device not enabled (0x%02x), aborting\n", tse);
  1262. return -ENXIO;
  1263. }
  1264. trc = thm_readw(THM_TRC);
  1265. trc_required_mask = TRC_CORE1_EN | TRC_CORE_PWR | TRC_MCH_EN;
  1266. if ((trc & trc_required_mask) != trc_required_mask) {
  1267. dev_err(&dev->dev, "thermal reporting for required devices not enabled, aborting\n");
  1268. return -ENXIO;
  1269. }
  1270. if (trc & TRC_CORE2_EN)
  1271. ips->second_cpu = true;
  1272. update_turbo_limits(ips);
  1273. dev_dbg(&dev->dev, "max cpu power clamp: %dW\n",
  1274. ips->mcp_power_limit / 10);
  1275. dev_dbg(&dev->dev, "max core power clamp: %dW\n",
  1276. ips->core_power_limit / 10);
  1277. /* BIOS may update limits at runtime */
  1278. if (thm_readl(THM_PSC) & PSP_PBRT)
  1279. ips->poll_turbo_status = true;
  1280. if (!ips_get_i915_syms(ips)) {
  1281. dev_info(&dev->dev, "failed to get i915 symbols, graphics turbo disabled until i915 loads\n");
  1282. ips->gpu_turbo_enabled = false;
  1283. } else {
  1284. dev_dbg(&dev->dev, "graphics turbo enabled\n");
  1285. ips->gpu_turbo_enabled = true;
  1286. }
  1287. /*
  1288. * Check PLATFORM_INFO MSR to make sure this chip is
  1289. * turbo capable.
  1290. */
  1291. rdmsrl(PLATFORM_INFO, platform_info);
  1292. if (!(platform_info & PLATFORM_TDP)) {
  1293. dev_err(&dev->dev, "platform indicates TDP override unavailable, aborting\n");
  1294. return -ENODEV;
  1295. }
  1296. /*
  1297. * IRQ handler for ME interaction
  1298. * Note: don't use MSI here as the PCH has bugs.
  1299. */
  1300. ret = pci_alloc_irq_vectors(dev, 1, 1, PCI_IRQ_LEGACY);
  1301. if (ret < 0)
  1302. return ret;
  1303. ips->irq = pci_irq_vector(dev, 0);
  1304. ret = request_irq(ips->irq, ips_irq_handler, IRQF_SHARED, "ips", ips);
  1305. if (ret) {
  1306. dev_err(&dev->dev, "request irq failed, aborting\n");
  1307. return ret;
  1308. }
  1309. /* Enable aux, hot & critical interrupts */
  1310. thm_writeb(THM_TSPIEN, TSPIEN_AUX2_LOHI | TSPIEN_CRIT_LOHI |
  1311. TSPIEN_HOT_LOHI | TSPIEN_AUX_LOHI);
  1312. thm_writeb(THM_TEN, TEN_UPDATE_EN);
  1313. /* Collect adjustment values */
  1314. ips->cta_val = thm_readw(THM_CTA);
  1315. ips->pta_val = thm_readw(THM_PTA);
  1316. ips->mgta_val = thm_readw(THM_MGTA);
  1317. /* Save turbo limits & ratios */
  1318. rdmsrl(TURBO_POWER_CURRENT_LIMIT, ips->orig_turbo_limit);
  1319. ips_disable_cpu_turbo(ips);
  1320. ips->cpu_turbo_enabled = false;
  1321. /* Create thermal adjust thread */
  1322. ips->adjust = kthread_create(ips_adjust, ips, "ips-adjust");
  1323. if (IS_ERR(ips->adjust)) {
  1324. dev_err(&dev->dev,
  1325. "failed to create thermal adjust thread, aborting\n");
  1326. ret = -ENOMEM;
  1327. goto error_free_irq;
  1328. }
  1329. /*
  1330. * Set up the work queue and monitor thread. The monitor thread
  1331. * will wake up ips_adjust thread.
  1332. */
  1333. ips->monitor = kthread_run(ips_monitor, ips, "ips-monitor");
  1334. if (IS_ERR(ips->monitor)) {
  1335. dev_err(&dev->dev,
  1336. "failed to create thermal monitor thread, aborting\n");
  1337. ret = -ENOMEM;
  1338. goto error_thread_cleanup;
  1339. }
  1340. hts = (ips->core_power_limit << HTS_PCPL_SHIFT) |
  1341. (ips->mcp_temp_limit << HTS_PTL_SHIFT) | HTS_NVV;
  1342. htshi = HTS2_PRST_RUNNING << HTS2_PRST_SHIFT;
  1343. thm_writew(THM_HTSHI, htshi);
  1344. thm_writel(THM_HTS, hts);
  1345. ips_debugfs_init(ips);
  1346. dev_info(&dev->dev, "IPS driver initialized, MCP temp limit %d\n",
  1347. ips->mcp_temp_limit);
  1348. return ret;
  1349. error_thread_cleanup:
  1350. kthread_stop(ips->adjust);
  1351. error_free_irq:
  1352. free_irq(ips->irq, ips);
  1353. pci_free_irq_vectors(dev);
  1354. return ret;
  1355. }
  1356. static void ips_remove(struct pci_dev *dev)
  1357. {
  1358. struct ips_driver *ips = pci_get_drvdata(dev);
  1359. u64 turbo_override;
  1360. ips_debugfs_cleanup(ips);
  1361. /* Release i915 driver */
  1362. if (ips->read_mch_val)
  1363. symbol_put(i915_read_mch_val);
  1364. if (ips->gpu_raise)
  1365. symbol_put(i915_gpu_raise);
  1366. if (ips->gpu_lower)
  1367. symbol_put(i915_gpu_lower);
  1368. if (ips->gpu_busy)
  1369. symbol_put(i915_gpu_busy);
  1370. if (ips->gpu_turbo_disable)
  1371. symbol_put(i915_gpu_turbo_disable);
  1372. rdmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_override);
  1373. turbo_override &= ~(TURBO_TDC_OVR_EN | TURBO_TDP_OVR_EN);
  1374. wrmsrl(TURBO_POWER_CURRENT_LIMIT, turbo_override);
  1375. wrmsrl(TURBO_POWER_CURRENT_LIMIT, ips->orig_turbo_limit);
  1376. free_irq(ips->irq, ips);
  1377. pci_free_irq_vectors(dev);
  1378. if (ips->adjust)
  1379. kthread_stop(ips->adjust);
  1380. if (ips->monitor)
  1381. kthread_stop(ips->monitor);
  1382. dev_dbg(&dev->dev, "IPS driver removed\n");
  1383. }
  1384. static struct pci_driver ips_pci_driver = {
  1385. .name = "intel ips",
  1386. .id_table = ips_id_table,
  1387. .probe = ips_probe,
  1388. .remove = ips_remove,
  1389. };
  1390. module_pci_driver(ips_pci_driver);
  1391. MODULE_LICENSE("GPL v2");
  1392. MODULE_AUTHOR("Jesse Barnes <jbarnes@virtuousgeek.org>");
  1393. MODULE_DESCRIPTION("Intelligent Power Sharing Driver");