powernow-k8.c 30 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * (c) 2003-2012 Advanced Micro Devices, Inc.
  4. *
  5. * Maintainer:
  6. * Andreas Herrmann <herrmann.der.user@googlemail.com>
  7. *
  8. * Based on the powernow-k7.c module written by Dave Jones.
  9. * (C) 2003 Dave Jones on behalf of SuSE Labs
  10. * (C) 2004 Dominik Brodowski <linux@brodo.de>
  11. * (C) 2004 Pavel Machek <pavel@ucw.cz>
  12. * Based upon datasheets & sample CPUs kindly provided by AMD.
  13. *
  14. * Valuable input gratefully received from Dave Jones, Pavel Machek,
  15. * Dominik Brodowski, Jacob Shin, and others.
  16. * Originally developed by Paul Devriendt.
  17. *
  18. * Processor information obtained from Chapter 9 (Power and Thermal
  19. * Management) of the "BIOS and Kernel Developer's Guide (BKDG) for
  20. * the AMD Athlon 64 and AMD Opteron Processors" and section "2.x
  21. * Power Management" in BKDGs for newer AMD CPU families.
  22. *
  23. * Tables for specific CPUs can be inferred from AMD's processor
  24. * power and thermal data sheets, (e.g. 30417.pdf, 30430.pdf, 43375.pdf)
  25. */
  26. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  27. #include <linux/kernel.h>
  28. #include <linux/smp.h>
  29. #include <linux/module.h>
  30. #include <linux/init.h>
  31. #include <linux/cpufreq.h>
  32. #include <linux/slab.h>
  33. #include <linux/string.h>
  34. #include <linux/cpumask.h>
  35. #include <linux/io.h>
  36. #include <linux/delay.h>
  37. #include <asm/msr.h>
  38. #include <asm/cpu_device_id.h>
  39. #include <linux/acpi.h>
  40. #include <linux/mutex.h>
  41. #include <acpi/processor.h>
  42. #define VERSION "version 2.20.00"
  43. #include "powernow-k8.h"
  44. /* serialize freq changes */
  45. static DEFINE_MUTEX(fidvid_mutex);
  46. static DEFINE_PER_CPU(struct powernow_k8_data *, powernow_data);
  47. static struct cpufreq_driver cpufreq_amd64_driver;
  48. /* Return a frequency in MHz, given an input fid */
  49. static u32 find_freq_from_fid(u32 fid)
  50. {
  51. return 800 + (fid * 100);
  52. }
  53. /* Return a frequency in KHz, given an input fid */
  54. static u32 find_khz_freq_from_fid(u32 fid)
  55. {
  56. return 1000 * find_freq_from_fid(fid);
  57. }
  58. /* Return the vco fid for an input fid
  59. *
  60. * Each "low" fid has corresponding "high" fid, and you can get to "low" fids
  61. * only from corresponding high fids. This returns "high" fid corresponding to
  62. * "low" one.
  63. */
  64. static u32 convert_fid_to_vco_fid(u32 fid)
  65. {
  66. if (fid < HI_FID_TABLE_BOTTOM)
  67. return 8 + (2 * fid);
  68. else
  69. return fid;
  70. }
  71. /*
  72. * Return 1 if the pending bit is set. Unless we just instructed the processor
  73. * to transition to a new state, seeing this bit set is really bad news.
  74. */
  75. static int pending_bit_stuck(void)
  76. {
  77. u32 lo, hi __always_unused;
  78. rdmsr(MSR_FIDVID_STATUS, lo, hi);
  79. return lo & MSR_S_LO_CHANGE_PENDING ? 1 : 0;
  80. }
  81. /*
  82. * Update the global current fid / vid values from the status msr.
  83. * Returns 1 on error.
  84. */
  85. static int query_current_values_with_pending_wait(struct powernow_k8_data *data)
  86. {
  87. u32 lo, hi;
  88. u32 i = 0;
  89. do {
  90. if (i++ > 10000) {
  91. pr_debug("detected change pending stuck\n");
  92. return 1;
  93. }
  94. rdmsr(MSR_FIDVID_STATUS, lo, hi);
  95. } while (lo & MSR_S_LO_CHANGE_PENDING);
  96. data->currvid = hi & MSR_S_HI_CURRENT_VID;
  97. data->currfid = lo & MSR_S_LO_CURRENT_FID;
  98. return 0;
  99. }
  100. /* the isochronous relief time */
  101. static void count_off_irt(struct powernow_k8_data *data)
  102. {
  103. udelay((1 << data->irt) * 10);
  104. }
  105. /* the voltage stabilization time */
  106. static void count_off_vst(struct powernow_k8_data *data)
  107. {
  108. udelay(data->vstable * VST_UNITS_20US);
  109. }
  110. /* need to init the control msr to a safe value (for each cpu) */
  111. static void fidvid_msr_init(void)
  112. {
  113. u32 lo, hi;
  114. u8 fid, vid;
  115. rdmsr(MSR_FIDVID_STATUS, lo, hi);
  116. vid = hi & MSR_S_HI_CURRENT_VID;
  117. fid = lo & MSR_S_LO_CURRENT_FID;
  118. lo = fid | (vid << MSR_C_LO_VID_SHIFT);
  119. hi = MSR_C_HI_STP_GNT_BENIGN;
  120. pr_debug("cpu%d, init lo 0x%x, hi 0x%x\n", smp_processor_id(), lo, hi);
  121. wrmsr(MSR_FIDVID_CTL, lo, hi);
  122. }
  123. /* write the new fid value along with the other control fields to the msr */
  124. static int write_new_fid(struct powernow_k8_data *data, u32 fid)
  125. {
  126. u32 lo;
  127. u32 savevid = data->currvid;
  128. u32 i = 0;
  129. if ((fid & INVALID_FID_MASK) || (data->currvid & INVALID_VID_MASK)) {
  130. pr_err("internal error - overflow on fid write\n");
  131. return 1;
  132. }
  133. lo = fid;
  134. lo |= (data->currvid << MSR_C_LO_VID_SHIFT);
  135. lo |= MSR_C_LO_INIT_FID_VID;
  136. pr_debug("writing fid 0x%x, lo 0x%x, hi 0x%x\n",
  137. fid, lo, data->plllock * PLL_LOCK_CONVERSION);
  138. do {
  139. wrmsr(MSR_FIDVID_CTL, lo, data->plllock * PLL_LOCK_CONVERSION);
  140. if (i++ > 100) {
  141. pr_err("Hardware error - pending bit very stuck - no further pstate changes possible\n");
  142. return 1;
  143. }
  144. } while (query_current_values_with_pending_wait(data));
  145. count_off_irt(data);
  146. if (savevid != data->currvid) {
  147. pr_err("vid change on fid trans, old 0x%x, new 0x%x\n",
  148. savevid, data->currvid);
  149. return 1;
  150. }
  151. if (fid != data->currfid) {
  152. pr_err("fid trans failed, fid 0x%x, curr 0x%x\n", fid,
  153. data->currfid);
  154. return 1;
  155. }
  156. return 0;
  157. }
  158. /* Write a new vid to the hardware */
  159. static int write_new_vid(struct powernow_k8_data *data, u32 vid)
  160. {
  161. u32 lo;
  162. u32 savefid = data->currfid;
  163. int i = 0;
  164. if ((data->currfid & INVALID_FID_MASK) || (vid & INVALID_VID_MASK)) {
  165. pr_err("internal error - overflow on vid write\n");
  166. return 1;
  167. }
  168. lo = data->currfid;
  169. lo |= (vid << MSR_C_LO_VID_SHIFT);
  170. lo |= MSR_C_LO_INIT_FID_VID;
  171. pr_debug("writing vid 0x%x, lo 0x%x, hi 0x%x\n",
  172. vid, lo, STOP_GRANT_5NS);
  173. do {
  174. wrmsr(MSR_FIDVID_CTL, lo, STOP_GRANT_5NS);
  175. if (i++ > 100) {
  176. pr_err("internal error - pending bit very stuck - no further pstate changes possible\n");
  177. return 1;
  178. }
  179. } while (query_current_values_with_pending_wait(data));
  180. if (savefid != data->currfid) {
  181. pr_err("fid changed on vid trans, old 0x%x new 0x%x\n",
  182. savefid, data->currfid);
  183. return 1;
  184. }
  185. if (vid != data->currvid) {
  186. pr_err("vid trans failed, vid 0x%x, curr 0x%x\n",
  187. vid, data->currvid);
  188. return 1;
  189. }
  190. return 0;
  191. }
  192. /*
  193. * Reduce the vid by the max of step or reqvid.
  194. * Decreasing vid codes represent increasing voltages:
  195. * vid of 0 is 1.550V, vid of 0x1e is 0.800V, vid of VID_OFF is off.
  196. */
  197. static int decrease_vid_code_by_step(struct powernow_k8_data *data,
  198. u32 reqvid, u32 step)
  199. {
  200. if ((data->currvid - reqvid) > step)
  201. reqvid = data->currvid - step;
  202. if (write_new_vid(data, reqvid))
  203. return 1;
  204. count_off_vst(data);
  205. return 0;
  206. }
  207. /* Change Opteron/Athlon64 fid and vid, by the 3 phases. */
  208. static int transition_fid_vid(struct powernow_k8_data *data,
  209. u32 reqfid, u32 reqvid)
  210. {
  211. if (core_voltage_pre_transition(data, reqvid, reqfid))
  212. return 1;
  213. if (core_frequency_transition(data, reqfid))
  214. return 1;
  215. if (core_voltage_post_transition(data, reqvid))
  216. return 1;
  217. if (query_current_values_with_pending_wait(data))
  218. return 1;
  219. if ((reqfid != data->currfid) || (reqvid != data->currvid)) {
  220. pr_err("failed (cpu%d): req 0x%x 0x%x, curr 0x%x 0x%x\n",
  221. smp_processor_id(),
  222. reqfid, reqvid, data->currfid, data->currvid);
  223. return 1;
  224. }
  225. pr_debug("transitioned (cpu%d): new fid 0x%x, vid 0x%x\n",
  226. smp_processor_id(), data->currfid, data->currvid);
  227. return 0;
  228. }
  229. /* Phase 1 - core voltage transition ... setup voltage */
  230. static int core_voltage_pre_transition(struct powernow_k8_data *data,
  231. u32 reqvid, u32 reqfid)
  232. {
  233. u32 rvosteps = data->rvo;
  234. u32 savefid = data->currfid;
  235. u32 maxvid, lo __always_unused, rvomult = 1;
  236. pr_debug("ph1 (cpu%d): start, currfid 0x%x, currvid 0x%x, reqvid 0x%x, rvo 0x%x\n",
  237. smp_processor_id(),
  238. data->currfid, data->currvid, reqvid, data->rvo);
  239. if ((savefid < LO_FID_TABLE_TOP) && (reqfid < LO_FID_TABLE_TOP))
  240. rvomult = 2;
  241. rvosteps *= rvomult;
  242. rdmsr(MSR_FIDVID_STATUS, lo, maxvid);
  243. maxvid = 0x1f & (maxvid >> 16);
  244. pr_debug("ph1 maxvid=0x%x\n", maxvid);
  245. if (reqvid < maxvid) /* lower numbers are higher voltages */
  246. reqvid = maxvid;
  247. while (data->currvid > reqvid) {
  248. pr_debug("ph1: curr 0x%x, req vid 0x%x\n",
  249. data->currvid, reqvid);
  250. if (decrease_vid_code_by_step(data, reqvid, data->vidmvs))
  251. return 1;
  252. }
  253. while ((rvosteps > 0) &&
  254. ((rvomult * data->rvo + data->currvid) > reqvid)) {
  255. if (data->currvid == maxvid) {
  256. rvosteps = 0;
  257. } else {
  258. pr_debug("ph1: changing vid for rvo, req 0x%x\n",
  259. data->currvid - 1);
  260. if (decrease_vid_code_by_step(data, data->currvid-1, 1))
  261. return 1;
  262. rvosteps--;
  263. }
  264. }
  265. if (query_current_values_with_pending_wait(data))
  266. return 1;
  267. if (savefid != data->currfid) {
  268. pr_err("ph1 err, currfid changed 0x%x\n", data->currfid);
  269. return 1;
  270. }
  271. pr_debug("ph1 complete, currfid 0x%x, currvid 0x%x\n",
  272. data->currfid, data->currvid);
  273. return 0;
  274. }
  275. /* Phase 2 - core frequency transition */
  276. static int core_frequency_transition(struct powernow_k8_data *data, u32 reqfid)
  277. {
  278. u32 vcoreqfid, vcocurrfid, vcofiddiff;
  279. u32 fid_interval, savevid = data->currvid;
  280. if (data->currfid == reqfid) {
  281. pr_err("ph2 null fid transition 0x%x\n", data->currfid);
  282. return 0;
  283. }
  284. pr_debug("ph2 (cpu%d): starting, currfid 0x%x, currvid 0x%x, reqfid 0x%x\n",
  285. smp_processor_id(),
  286. data->currfid, data->currvid, reqfid);
  287. vcoreqfid = convert_fid_to_vco_fid(reqfid);
  288. vcocurrfid = convert_fid_to_vco_fid(data->currfid);
  289. vcofiddiff = vcocurrfid > vcoreqfid ? vcocurrfid - vcoreqfid
  290. : vcoreqfid - vcocurrfid;
  291. if ((reqfid <= LO_FID_TABLE_TOP) && (data->currfid <= LO_FID_TABLE_TOP))
  292. vcofiddiff = 0;
  293. while (vcofiddiff > 2) {
  294. (data->currfid & 1) ? (fid_interval = 1) : (fid_interval = 2);
  295. if (reqfid > data->currfid) {
  296. if (data->currfid > LO_FID_TABLE_TOP) {
  297. if (write_new_fid(data,
  298. data->currfid + fid_interval))
  299. return 1;
  300. } else {
  301. if (write_new_fid
  302. (data,
  303. 2 + convert_fid_to_vco_fid(data->currfid)))
  304. return 1;
  305. }
  306. } else {
  307. if (write_new_fid(data, data->currfid - fid_interval))
  308. return 1;
  309. }
  310. vcocurrfid = convert_fid_to_vco_fid(data->currfid);
  311. vcofiddiff = vcocurrfid > vcoreqfid ? vcocurrfid - vcoreqfid
  312. : vcoreqfid - vcocurrfid;
  313. }
  314. if (write_new_fid(data, reqfid))
  315. return 1;
  316. if (query_current_values_with_pending_wait(data))
  317. return 1;
  318. if (data->currfid != reqfid) {
  319. pr_err("ph2: mismatch, failed fid transition, curr 0x%x, req 0x%x\n",
  320. data->currfid, reqfid);
  321. return 1;
  322. }
  323. if (savevid != data->currvid) {
  324. pr_err("ph2: vid changed, save 0x%x, curr 0x%x\n",
  325. savevid, data->currvid);
  326. return 1;
  327. }
  328. pr_debug("ph2 complete, currfid 0x%x, currvid 0x%x\n",
  329. data->currfid, data->currvid);
  330. return 0;
  331. }
  332. /* Phase 3 - core voltage transition flow ... jump to the final vid. */
  333. static int core_voltage_post_transition(struct powernow_k8_data *data,
  334. u32 reqvid)
  335. {
  336. u32 savefid = data->currfid;
  337. u32 savereqvid = reqvid;
  338. pr_debug("ph3 (cpu%d): starting, currfid 0x%x, currvid 0x%x\n",
  339. smp_processor_id(),
  340. data->currfid, data->currvid);
  341. if (reqvid != data->currvid) {
  342. if (write_new_vid(data, reqvid))
  343. return 1;
  344. if (savefid != data->currfid) {
  345. pr_err("ph3: bad fid change, save 0x%x, curr 0x%x\n",
  346. savefid, data->currfid);
  347. return 1;
  348. }
  349. if (data->currvid != reqvid) {
  350. pr_err("ph3: failed vid transition\n, req 0x%x, curr 0x%x",
  351. reqvid, data->currvid);
  352. return 1;
  353. }
  354. }
  355. if (query_current_values_with_pending_wait(data))
  356. return 1;
  357. if (savereqvid != data->currvid) {
  358. pr_debug("ph3 failed, currvid 0x%x\n", data->currvid);
  359. return 1;
  360. }
  361. if (savefid != data->currfid) {
  362. pr_debug("ph3 failed, currfid changed 0x%x\n",
  363. data->currfid);
  364. return 1;
  365. }
  366. pr_debug("ph3 complete, currfid 0x%x, currvid 0x%x\n",
  367. data->currfid, data->currvid);
  368. return 0;
  369. }
  370. static const struct x86_cpu_id powernow_k8_ids[] = {
  371. /* IO based frequency switching */
  372. X86_MATCH_VENDOR_FAM(AMD, 0xf, NULL),
  373. {}
  374. };
  375. MODULE_DEVICE_TABLE(x86cpu, powernow_k8_ids);
  376. static void check_supported_cpu(void *_rc)
  377. {
  378. u32 eax, ebx, ecx, edx;
  379. int *rc = _rc;
  380. *rc = -ENODEV;
  381. eax = cpuid_eax(CPUID_PROCESSOR_SIGNATURE);
  382. if ((eax & CPUID_XFAM) == CPUID_XFAM_K8) {
  383. if (((eax & CPUID_USE_XFAM_XMOD) != CPUID_USE_XFAM_XMOD) ||
  384. ((eax & CPUID_XMOD) > CPUID_XMOD_REV_MASK)) {
  385. pr_info("Processor cpuid %x not supported\n", eax);
  386. return;
  387. }
  388. eax = cpuid_eax(CPUID_GET_MAX_CAPABILITIES);
  389. if (eax < CPUID_FREQ_VOLT_CAPABILITIES) {
  390. pr_info("No frequency change capabilities detected\n");
  391. return;
  392. }
  393. cpuid(CPUID_FREQ_VOLT_CAPABILITIES, &eax, &ebx, &ecx, &edx);
  394. if ((edx & P_STATE_TRANSITION_CAPABLE)
  395. != P_STATE_TRANSITION_CAPABLE) {
  396. pr_info("Power state transitions not supported\n");
  397. return;
  398. }
  399. *rc = 0;
  400. }
  401. }
  402. static int check_pst_table(struct powernow_k8_data *data, struct pst_s *pst,
  403. u8 maxvid)
  404. {
  405. unsigned int j;
  406. u8 lastfid = 0xff;
  407. for (j = 0; j < data->numps; j++) {
  408. if (pst[j].vid > LEAST_VID) {
  409. pr_err(FW_BUG "vid %d invalid : 0x%x\n", j,
  410. pst[j].vid);
  411. return -EINVAL;
  412. }
  413. if (pst[j].vid < data->rvo) {
  414. /* vid + rvo >= 0 */
  415. pr_err(FW_BUG "0 vid exceeded with pstate %d\n", j);
  416. return -ENODEV;
  417. }
  418. if (pst[j].vid < maxvid + data->rvo) {
  419. /* vid + rvo >= maxvid */
  420. pr_err(FW_BUG "maxvid exceeded with pstate %d\n", j);
  421. return -ENODEV;
  422. }
  423. if (pst[j].fid > MAX_FID) {
  424. pr_err(FW_BUG "maxfid exceeded with pstate %d\n", j);
  425. return -ENODEV;
  426. }
  427. if (j && (pst[j].fid < HI_FID_TABLE_BOTTOM)) {
  428. /* Only first fid is allowed to be in "low" range */
  429. pr_err(FW_BUG "two low fids - %d : 0x%x\n", j,
  430. pst[j].fid);
  431. return -EINVAL;
  432. }
  433. if (pst[j].fid < lastfid)
  434. lastfid = pst[j].fid;
  435. }
  436. if (lastfid & 1) {
  437. pr_err(FW_BUG "lastfid invalid\n");
  438. return -EINVAL;
  439. }
  440. if (lastfid > LO_FID_TABLE_TOP)
  441. pr_info(FW_BUG "first fid not from lo freq table\n");
  442. return 0;
  443. }
  444. static void invalidate_entry(struct cpufreq_frequency_table *powernow_table,
  445. unsigned int entry)
  446. {
  447. powernow_table[entry].frequency = CPUFREQ_ENTRY_INVALID;
  448. }
  449. static void print_basics(struct powernow_k8_data *data)
  450. {
  451. int j;
  452. for (j = 0; j < data->numps; j++) {
  453. if (data->powernow_table[j].frequency !=
  454. CPUFREQ_ENTRY_INVALID) {
  455. pr_info("fid 0x%x (%d MHz), vid 0x%x\n",
  456. data->powernow_table[j].driver_data & 0xff,
  457. data->powernow_table[j].frequency/1000,
  458. data->powernow_table[j].driver_data >> 8);
  459. }
  460. }
  461. if (data->batps)
  462. pr_info("Only %d pstates on battery\n", data->batps);
  463. }
  464. static int fill_powernow_table(struct powernow_k8_data *data,
  465. struct pst_s *pst, u8 maxvid)
  466. {
  467. struct cpufreq_frequency_table *powernow_table;
  468. unsigned int j;
  469. if (data->batps) {
  470. /* use ACPI support to get full speed on mains power */
  471. pr_warn("Only %d pstates usable (use ACPI driver for full range\n",
  472. data->batps);
  473. data->numps = data->batps;
  474. }
  475. for (j = 1; j < data->numps; j++) {
  476. if (pst[j-1].fid >= pst[j].fid) {
  477. pr_err("PST out of sequence\n");
  478. return -EINVAL;
  479. }
  480. }
  481. if (data->numps < 2) {
  482. pr_err("no p states to transition\n");
  483. return -ENODEV;
  484. }
  485. if (check_pst_table(data, pst, maxvid))
  486. return -EINVAL;
  487. powernow_table = kzalloc((sizeof(*powernow_table)
  488. * (data->numps + 1)), GFP_KERNEL);
  489. if (!powernow_table)
  490. return -ENOMEM;
  491. for (j = 0; j < data->numps; j++) {
  492. int freq;
  493. powernow_table[j].driver_data = pst[j].fid; /* lower 8 bits */
  494. powernow_table[j].driver_data |= (pst[j].vid << 8); /* upper 8 bits */
  495. freq = find_khz_freq_from_fid(pst[j].fid);
  496. powernow_table[j].frequency = freq;
  497. }
  498. powernow_table[data->numps].frequency = CPUFREQ_TABLE_END;
  499. powernow_table[data->numps].driver_data = 0;
  500. if (query_current_values_with_pending_wait(data)) {
  501. kfree(powernow_table);
  502. return -EIO;
  503. }
  504. pr_debug("cfid 0x%x, cvid 0x%x\n", data->currfid, data->currvid);
  505. data->powernow_table = powernow_table;
  506. if (cpumask_first(topology_core_cpumask(data->cpu)) == data->cpu)
  507. print_basics(data);
  508. for (j = 0; j < data->numps; j++)
  509. if ((pst[j].fid == data->currfid) &&
  510. (pst[j].vid == data->currvid))
  511. return 0;
  512. pr_debug("currfid/vid do not match PST, ignoring\n");
  513. return 0;
  514. }
  515. /* Find and validate the PSB/PST table in BIOS. */
  516. static int find_psb_table(struct powernow_k8_data *data)
  517. {
  518. struct psb_s *psb;
  519. unsigned int i;
  520. u32 mvs;
  521. u8 maxvid;
  522. u32 cpst = 0;
  523. u32 thiscpuid;
  524. for (i = 0xc0000; i < 0xffff0; i += 0x10) {
  525. /* Scan BIOS looking for the signature. */
  526. /* It can not be at ffff0 - it is too big. */
  527. psb = phys_to_virt(i);
  528. if (memcmp(psb, PSB_ID_STRING, PSB_ID_STRING_LEN) != 0)
  529. continue;
  530. pr_debug("found PSB header at 0x%p\n", psb);
  531. pr_debug("table vers: 0x%x\n", psb->tableversion);
  532. if (psb->tableversion != PSB_VERSION_1_4) {
  533. pr_err(FW_BUG "PSB table is not v1.4\n");
  534. return -ENODEV;
  535. }
  536. pr_debug("flags: 0x%x\n", psb->flags1);
  537. if (psb->flags1) {
  538. pr_err(FW_BUG "unknown flags\n");
  539. return -ENODEV;
  540. }
  541. data->vstable = psb->vstable;
  542. pr_debug("voltage stabilization time: %d(*20us)\n",
  543. data->vstable);
  544. pr_debug("flags2: 0x%x\n", psb->flags2);
  545. data->rvo = psb->flags2 & 3;
  546. data->irt = ((psb->flags2) >> 2) & 3;
  547. mvs = ((psb->flags2) >> 4) & 3;
  548. data->vidmvs = 1 << mvs;
  549. data->batps = ((psb->flags2) >> 6) & 3;
  550. pr_debug("ramp voltage offset: %d\n", data->rvo);
  551. pr_debug("isochronous relief time: %d\n", data->irt);
  552. pr_debug("maximum voltage step: %d - 0x%x\n", mvs, data->vidmvs);
  553. pr_debug("numpst: 0x%x\n", psb->num_tables);
  554. cpst = psb->num_tables;
  555. if ((psb->cpuid == 0x00000fc0) ||
  556. (psb->cpuid == 0x00000fe0)) {
  557. thiscpuid = cpuid_eax(CPUID_PROCESSOR_SIGNATURE);
  558. if ((thiscpuid == 0x00000fc0) ||
  559. (thiscpuid == 0x00000fe0))
  560. cpst = 1;
  561. }
  562. if (cpst != 1) {
  563. pr_err(FW_BUG "numpst must be 1\n");
  564. return -ENODEV;
  565. }
  566. data->plllock = psb->plllocktime;
  567. pr_debug("plllocktime: 0x%x (units 1us)\n", psb->plllocktime);
  568. pr_debug("maxfid: 0x%x\n", psb->maxfid);
  569. pr_debug("maxvid: 0x%x\n", psb->maxvid);
  570. maxvid = psb->maxvid;
  571. data->numps = psb->numps;
  572. pr_debug("numpstates: 0x%x\n", data->numps);
  573. return fill_powernow_table(data,
  574. (struct pst_s *)(psb+1), maxvid);
  575. }
  576. /*
  577. * If you see this message, complain to BIOS manufacturer. If
  578. * he tells you "we do not support Linux" or some similar
  579. * nonsense, remember that Windows 2000 uses the same legacy
  580. * mechanism that the old Linux PSB driver uses. Tell them it
  581. * is broken with Windows 2000.
  582. *
  583. * The reference to the AMD documentation is chapter 9 in the
  584. * BIOS and Kernel Developer's Guide, which is available on
  585. * www.amd.com
  586. */
  587. pr_err(FW_BUG "No PSB or ACPI _PSS objects\n");
  588. pr_err("Make sure that your BIOS is up to date and Cool'N'Quiet support is enabled in BIOS setup\n");
  589. return -ENODEV;
  590. }
  591. static void powernow_k8_acpi_pst_values(struct powernow_k8_data *data,
  592. unsigned int index)
  593. {
  594. u64 control;
  595. if (!data->acpi_data.state_count)
  596. return;
  597. control = data->acpi_data.states[index].control;
  598. data->irt = (control >> IRT_SHIFT) & IRT_MASK;
  599. data->rvo = (control >> RVO_SHIFT) & RVO_MASK;
  600. data->exttype = (control >> EXT_TYPE_SHIFT) & EXT_TYPE_MASK;
  601. data->plllock = (control >> PLL_L_SHIFT) & PLL_L_MASK;
  602. data->vidmvs = 1 << ((control >> MVS_SHIFT) & MVS_MASK);
  603. data->vstable = (control >> VST_SHIFT) & VST_MASK;
  604. }
  605. static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data)
  606. {
  607. struct cpufreq_frequency_table *powernow_table;
  608. int ret_val = -ENODEV;
  609. u64 control, status;
  610. if (acpi_processor_register_performance(&data->acpi_data, data->cpu)) {
  611. pr_debug("register performance failed: bad ACPI data\n");
  612. return -EIO;
  613. }
  614. /* verify the data contained in the ACPI structures */
  615. if (data->acpi_data.state_count <= 1) {
  616. pr_debug("No ACPI P-States\n");
  617. goto err_out;
  618. }
  619. control = data->acpi_data.control_register.space_id;
  620. status = data->acpi_data.status_register.space_id;
  621. if ((control != ACPI_ADR_SPACE_FIXED_HARDWARE) ||
  622. (status != ACPI_ADR_SPACE_FIXED_HARDWARE)) {
  623. pr_debug("Invalid control/status registers (%llx - %llx)\n",
  624. control, status);
  625. goto err_out;
  626. }
  627. /* fill in data->powernow_table */
  628. powernow_table = kzalloc((sizeof(*powernow_table)
  629. * (data->acpi_data.state_count + 1)), GFP_KERNEL);
  630. if (!powernow_table)
  631. goto err_out;
  632. /* fill in data */
  633. data->numps = data->acpi_data.state_count;
  634. powernow_k8_acpi_pst_values(data, 0);
  635. ret_val = fill_powernow_table_fidvid(data, powernow_table);
  636. if (ret_val)
  637. goto err_out_mem;
  638. powernow_table[data->acpi_data.state_count].frequency =
  639. CPUFREQ_TABLE_END;
  640. data->powernow_table = powernow_table;
  641. if (cpumask_first(topology_core_cpumask(data->cpu)) == data->cpu)
  642. print_basics(data);
  643. /* notify BIOS that we exist */
  644. acpi_processor_notify_smm(THIS_MODULE);
  645. if (!zalloc_cpumask_var(&data->acpi_data.shared_cpu_map, GFP_KERNEL)) {
  646. pr_err("unable to alloc powernow_k8_data cpumask\n");
  647. ret_val = -ENOMEM;
  648. goto err_out_mem;
  649. }
  650. return 0;
  651. err_out_mem:
  652. kfree(powernow_table);
  653. err_out:
  654. acpi_processor_unregister_performance(data->cpu);
  655. /* data->acpi_data.state_count informs us at ->exit()
  656. * whether ACPI was used */
  657. data->acpi_data.state_count = 0;
  658. return ret_val;
  659. }
  660. static int fill_powernow_table_fidvid(struct powernow_k8_data *data,
  661. struct cpufreq_frequency_table *powernow_table)
  662. {
  663. int i;
  664. for (i = 0; i < data->acpi_data.state_count; i++) {
  665. u32 fid;
  666. u32 vid;
  667. u32 freq, index;
  668. u64 status, control;
  669. if (data->exttype) {
  670. status = data->acpi_data.states[i].status;
  671. fid = status & EXT_FID_MASK;
  672. vid = (status >> VID_SHIFT) & EXT_VID_MASK;
  673. } else {
  674. control = data->acpi_data.states[i].control;
  675. fid = control & FID_MASK;
  676. vid = (control >> VID_SHIFT) & VID_MASK;
  677. }
  678. pr_debug(" %d : fid 0x%x, vid 0x%x\n", i, fid, vid);
  679. index = fid | (vid<<8);
  680. powernow_table[i].driver_data = index;
  681. freq = find_khz_freq_from_fid(fid);
  682. powernow_table[i].frequency = freq;
  683. /* verify frequency is OK */
  684. if ((freq > (MAX_FREQ * 1000)) || (freq < (MIN_FREQ * 1000))) {
  685. pr_debug("invalid freq %u kHz, ignoring\n", freq);
  686. invalidate_entry(powernow_table, i);
  687. continue;
  688. }
  689. /* verify voltage is OK -
  690. * BIOSs are using "off" to indicate invalid */
  691. if (vid == VID_OFF) {
  692. pr_debug("invalid vid %u, ignoring\n", vid);
  693. invalidate_entry(powernow_table, i);
  694. continue;
  695. }
  696. if (freq != (data->acpi_data.states[i].core_frequency * 1000)) {
  697. pr_info("invalid freq entries %u kHz vs. %u kHz\n",
  698. freq, (unsigned int)
  699. (data->acpi_data.states[i].core_frequency
  700. * 1000));
  701. invalidate_entry(powernow_table, i);
  702. continue;
  703. }
  704. }
  705. return 0;
  706. }
  707. static void powernow_k8_cpu_exit_acpi(struct powernow_k8_data *data)
  708. {
  709. if (data->acpi_data.state_count)
  710. acpi_processor_unregister_performance(data->cpu);
  711. free_cpumask_var(data->acpi_data.shared_cpu_map);
  712. }
  713. static int get_transition_latency(struct powernow_k8_data *data)
  714. {
  715. int max_latency = 0;
  716. int i;
  717. for (i = 0; i < data->acpi_data.state_count; i++) {
  718. int cur_latency = data->acpi_data.states[i].transition_latency
  719. + data->acpi_data.states[i].bus_master_latency;
  720. if (cur_latency > max_latency)
  721. max_latency = cur_latency;
  722. }
  723. if (max_latency == 0) {
  724. pr_err(FW_WARN "Invalid zero transition latency\n");
  725. max_latency = 1;
  726. }
  727. /* value in usecs, needs to be in nanoseconds */
  728. return 1000 * max_latency;
  729. }
  730. /* Take a frequency, and issue the fid/vid transition command */
  731. static int transition_frequency_fidvid(struct powernow_k8_data *data,
  732. unsigned int index,
  733. struct cpufreq_policy *policy)
  734. {
  735. u32 fid = 0;
  736. u32 vid = 0;
  737. int res;
  738. struct cpufreq_freqs freqs;
  739. pr_debug("cpu %d transition to index %u\n", smp_processor_id(), index);
  740. /* fid/vid correctness check for k8 */
  741. /* fid are the lower 8 bits of the index we stored into
  742. * the cpufreq frequency table in find_psb_table, vid
  743. * are the upper 8 bits.
  744. */
  745. fid = data->powernow_table[index].driver_data & 0xFF;
  746. vid = (data->powernow_table[index].driver_data & 0xFF00) >> 8;
  747. pr_debug("table matched fid 0x%x, giving vid 0x%x\n", fid, vid);
  748. if (query_current_values_with_pending_wait(data))
  749. return 1;
  750. if ((data->currvid == vid) && (data->currfid == fid)) {
  751. pr_debug("target matches current values (fid 0x%x, vid 0x%x)\n",
  752. fid, vid);
  753. return 0;
  754. }
  755. pr_debug("cpu %d, changing to fid 0x%x, vid 0x%x\n",
  756. smp_processor_id(), fid, vid);
  757. freqs.old = find_khz_freq_from_fid(data->currfid);
  758. freqs.new = find_khz_freq_from_fid(fid);
  759. cpufreq_freq_transition_begin(policy, &freqs);
  760. res = transition_fid_vid(data, fid, vid);
  761. cpufreq_freq_transition_end(policy, &freqs, res);
  762. return res;
  763. }
  764. struct powernowk8_target_arg {
  765. struct cpufreq_policy *pol;
  766. unsigned newstate;
  767. };
  768. static long powernowk8_target_fn(void *arg)
  769. {
  770. struct powernowk8_target_arg *pta = arg;
  771. struct cpufreq_policy *pol = pta->pol;
  772. unsigned newstate = pta->newstate;
  773. struct powernow_k8_data *data = per_cpu(powernow_data, pol->cpu);
  774. u32 checkfid;
  775. u32 checkvid;
  776. int ret;
  777. if (!data)
  778. return -EINVAL;
  779. checkfid = data->currfid;
  780. checkvid = data->currvid;
  781. if (pending_bit_stuck()) {
  782. pr_err("failing targ, change pending bit set\n");
  783. return -EIO;
  784. }
  785. pr_debug("targ: cpu %d, %d kHz, min %d, max %d\n",
  786. pol->cpu, data->powernow_table[newstate].frequency, pol->min,
  787. pol->max);
  788. if (query_current_values_with_pending_wait(data))
  789. return -EIO;
  790. pr_debug("targ: curr fid 0x%x, vid 0x%x\n",
  791. data->currfid, data->currvid);
  792. if ((checkvid != data->currvid) ||
  793. (checkfid != data->currfid)) {
  794. pr_info("error - out of sync, fix 0x%x 0x%x, vid 0x%x 0x%x\n",
  795. checkfid, data->currfid,
  796. checkvid, data->currvid);
  797. }
  798. mutex_lock(&fidvid_mutex);
  799. powernow_k8_acpi_pst_values(data, newstate);
  800. ret = transition_frequency_fidvid(data, newstate, pol);
  801. if (ret) {
  802. pr_err("transition frequency failed\n");
  803. mutex_unlock(&fidvid_mutex);
  804. return 1;
  805. }
  806. mutex_unlock(&fidvid_mutex);
  807. pol->cur = find_khz_freq_from_fid(data->currfid);
  808. return 0;
  809. }
  810. /* Driver entry point to switch to the target frequency */
  811. static int powernowk8_target(struct cpufreq_policy *pol, unsigned index)
  812. {
  813. struct powernowk8_target_arg pta = { .pol = pol, .newstate = index };
  814. return work_on_cpu(pol->cpu, powernowk8_target_fn, &pta);
  815. }
  816. struct init_on_cpu {
  817. struct powernow_k8_data *data;
  818. int rc;
  819. };
  820. static void powernowk8_cpu_init_on_cpu(void *_init_on_cpu)
  821. {
  822. struct init_on_cpu *init_on_cpu = _init_on_cpu;
  823. if (pending_bit_stuck()) {
  824. pr_err("failing init, change pending bit set\n");
  825. init_on_cpu->rc = -ENODEV;
  826. return;
  827. }
  828. if (query_current_values_with_pending_wait(init_on_cpu->data)) {
  829. init_on_cpu->rc = -ENODEV;
  830. return;
  831. }
  832. fidvid_msr_init();
  833. init_on_cpu->rc = 0;
  834. }
  835. #define MISSING_PSS_MSG \
  836. FW_BUG "No compatible ACPI _PSS objects found.\n" \
  837. FW_BUG "First, make sure Cool'N'Quiet is enabled in the BIOS.\n" \
  838. FW_BUG "If that doesn't help, try upgrading your BIOS.\n"
  839. /* per CPU init entry point to the driver */
  840. static int powernowk8_cpu_init(struct cpufreq_policy *pol)
  841. {
  842. struct powernow_k8_data *data;
  843. struct init_on_cpu init_on_cpu;
  844. int rc, cpu;
  845. smp_call_function_single(pol->cpu, check_supported_cpu, &rc, 1);
  846. if (rc)
  847. return -ENODEV;
  848. data = kzalloc(sizeof(*data), GFP_KERNEL);
  849. if (!data)
  850. return -ENOMEM;
  851. data->cpu = pol->cpu;
  852. if (powernow_k8_cpu_init_acpi(data)) {
  853. /*
  854. * Use the PSB BIOS structure. This is only available on
  855. * an UP version, and is deprecated by AMD.
  856. */
  857. if (num_online_cpus() != 1) {
  858. pr_err_once(MISSING_PSS_MSG);
  859. goto err_out;
  860. }
  861. if (pol->cpu != 0) {
  862. pr_err(FW_BUG "No ACPI _PSS objects for CPU other than CPU0. Complain to your BIOS vendor.\n");
  863. goto err_out;
  864. }
  865. rc = find_psb_table(data);
  866. if (rc)
  867. goto err_out;
  868. /* Take a crude guess here.
  869. * That guess was in microseconds, so multiply with 1000 */
  870. pol->cpuinfo.transition_latency = (
  871. ((data->rvo + 8) * data->vstable * VST_UNITS_20US) +
  872. ((1 << data->irt) * 30)) * 1000;
  873. } else /* ACPI _PSS objects available */
  874. pol->cpuinfo.transition_latency = get_transition_latency(data);
  875. /* only run on specific CPU from here on */
  876. init_on_cpu.data = data;
  877. smp_call_function_single(data->cpu, powernowk8_cpu_init_on_cpu,
  878. &init_on_cpu, 1);
  879. rc = init_on_cpu.rc;
  880. if (rc != 0)
  881. goto err_out_exit_acpi;
  882. cpumask_copy(pol->cpus, topology_core_cpumask(pol->cpu));
  883. data->available_cores = pol->cpus;
  884. pol->freq_table = data->powernow_table;
  885. pr_debug("cpu_init done, current fid 0x%x, vid 0x%x\n",
  886. data->currfid, data->currvid);
  887. /* Point all the CPUs in this policy to the same data */
  888. for_each_cpu(cpu, pol->cpus)
  889. per_cpu(powernow_data, cpu) = data;
  890. return 0;
  891. err_out_exit_acpi:
  892. powernow_k8_cpu_exit_acpi(data);
  893. err_out:
  894. kfree(data);
  895. return -ENODEV;
  896. }
  897. static int powernowk8_cpu_exit(struct cpufreq_policy *pol)
  898. {
  899. struct powernow_k8_data *data = per_cpu(powernow_data, pol->cpu);
  900. int cpu;
  901. if (!data)
  902. return -EINVAL;
  903. powernow_k8_cpu_exit_acpi(data);
  904. kfree(data->powernow_table);
  905. kfree(data);
  906. for_each_cpu(cpu, pol->cpus)
  907. per_cpu(powernow_data, cpu) = NULL;
  908. return 0;
  909. }
  910. static void query_values_on_cpu(void *_err)
  911. {
  912. int *err = _err;
  913. struct powernow_k8_data *data = __this_cpu_read(powernow_data);
  914. *err = query_current_values_with_pending_wait(data);
  915. }
  916. static unsigned int powernowk8_get(unsigned int cpu)
  917. {
  918. struct powernow_k8_data *data = per_cpu(powernow_data, cpu);
  919. unsigned int khz = 0;
  920. int err;
  921. if (!data)
  922. return 0;
  923. smp_call_function_single(cpu, query_values_on_cpu, &err, true);
  924. if (err)
  925. goto out;
  926. khz = find_khz_freq_from_fid(data->currfid);
  927. out:
  928. return khz;
  929. }
  930. static struct cpufreq_driver cpufreq_amd64_driver = {
  931. .flags = CPUFREQ_ASYNC_NOTIFICATION,
  932. .verify = cpufreq_generic_frequency_table_verify,
  933. .target_index = powernowk8_target,
  934. .bios_limit = acpi_processor_get_bios_limit,
  935. .init = powernowk8_cpu_init,
  936. .exit = powernowk8_cpu_exit,
  937. .get = powernowk8_get,
  938. .name = "powernow-k8",
  939. .attr = cpufreq_generic_attr,
  940. };
  941. static void __request_acpi_cpufreq(void)
  942. {
  943. const char drv[] = "acpi-cpufreq";
  944. const char *cur_drv;
  945. cur_drv = cpufreq_get_current_driver();
  946. if (!cur_drv)
  947. goto request;
  948. if (strncmp(cur_drv, drv, min_t(size_t, strlen(cur_drv), strlen(drv))))
  949. pr_warn("WTF driver: %s\n", cur_drv);
  950. return;
  951. request:
  952. pr_warn("This CPU is not supported anymore, using acpi-cpufreq instead.\n");
  953. request_module(drv);
  954. }
  955. /* driver entry point for init */
  956. static int powernowk8_init(void)
  957. {
  958. unsigned int i, supported_cpus = 0;
  959. int ret;
  960. if (boot_cpu_has(X86_FEATURE_HW_PSTATE)) {
  961. __request_acpi_cpufreq();
  962. return -ENODEV;
  963. }
  964. if (!x86_match_cpu(powernow_k8_ids))
  965. return -ENODEV;
  966. get_online_cpus();
  967. for_each_online_cpu(i) {
  968. smp_call_function_single(i, check_supported_cpu, &ret, 1);
  969. if (!ret)
  970. supported_cpus++;
  971. }
  972. if (supported_cpus != num_online_cpus()) {
  973. put_online_cpus();
  974. return -ENODEV;
  975. }
  976. put_online_cpus();
  977. ret = cpufreq_register_driver(&cpufreq_amd64_driver);
  978. if (ret)
  979. return ret;
  980. pr_info("Found %d %s (%d cpu cores) (" VERSION ")\n",
  981. num_online_nodes(), boot_cpu_data.x86_model_id, supported_cpus);
  982. return ret;
  983. }
  984. /* driver entry point for term */
  985. static void __exit powernowk8_exit(void)
  986. {
  987. pr_debug("exit\n");
  988. cpufreq_unregister_driver(&cpufreq_amd64_driver);
  989. }
  990. MODULE_AUTHOR("Paul Devriendt <paul.devriendt@amd.com>");
  991. MODULE_AUTHOR("Mark Langsdorf <mark.langsdorf@amd.com>");
  992. MODULE_DESCRIPTION("AMD Athlon 64 and Opteron processor frequency driver.");
  993. MODULE_LICENSE("GPL");
  994. late_initcall(powernowk8_init);
  995. module_exit(powernowk8_exit);