calibrate.c 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316
  1. // SPDX-License-Identifier: GPL-2.0
  2. /* calibrate.c: default delay calibration
  3. *
  4. * Excised from init/main.c
  5. * Copyright (C) 1991, 1992 Linus Torvalds
  6. */
  7. #include <linux/jiffies.h>
  8. #include <linux/delay.h>
  9. #include <linux/init.h>
  10. #include <linux/timex.h>
  11. #include <linux/smp.h>
  12. #include <linux/percpu.h>
  13. unsigned long lpj_fine;
  14. unsigned long preset_lpj;
  15. static int __init lpj_setup(char *str)
  16. {
  17. preset_lpj = simple_strtoul(str,NULL,0);
  18. return 1;
  19. }
  20. __setup("lpj=", lpj_setup);
  21. #ifdef ARCH_HAS_READ_CURRENT_TIMER
  22. /* This routine uses the read_current_timer() routine and gets the
  23. * loops per jiffy directly, instead of guessing it using delay().
  24. * Also, this code tries to handle non-maskable asynchronous events
  25. * (like SMIs)
  26. */
  27. #define DELAY_CALIBRATION_TICKS ((HZ < 100) ? 1 : (HZ/100))
  28. #define MAX_DIRECT_CALIBRATION_RETRIES 5
  29. static unsigned long calibrate_delay_direct(void)
  30. {
  31. unsigned long pre_start, start, post_start;
  32. unsigned long pre_end, end, post_end;
  33. unsigned long start_jiffies;
  34. unsigned long timer_rate_min, timer_rate_max;
  35. unsigned long good_timer_sum = 0;
  36. unsigned long good_timer_count = 0;
  37. unsigned long measured_times[MAX_DIRECT_CALIBRATION_RETRIES];
  38. int max = -1; /* index of measured_times with max/min values or not set */
  39. int min = -1;
  40. int i;
  41. if (read_current_timer(&pre_start) < 0 )
  42. return 0;
  43. /*
  44. * A simple loop like
  45. * while ( jiffies < start_jiffies+1)
  46. * start = read_current_timer();
  47. * will not do. As we don't really know whether jiffy switch
  48. * happened first or timer_value was read first. And some asynchronous
  49. * event can happen between these two events introducing errors in lpj.
  50. *
  51. * So, we do
  52. * 1. pre_start <- When we are sure that jiffy switch hasn't happened
  53. * 2. check jiffy switch
  54. * 3. start <- timer value before or after jiffy switch
  55. * 4. post_start <- When we are sure that jiffy switch has happened
  56. *
  57. * Note, we don't know anything about order of 2 and 3.
  58. * Now, by looking at post_start and pre_start difference, we can
  59. * check whether any asynchronous event happened or not
  60. */
  61. for (i = 0; i < MAX_DIRECT_CALIBRATION_RETRIES; i++) {
  62. pre_start = 0;
  63. read_current_timer(&start);
  64. start_jiffies = jiffies;
  65. while (time_before_eq(jiffies, start_jiffies + 1)) {
  66. pre_start = start;
  67. read_current_timer(&start);
  68. }
  69. read_current_timer(&post_start);
  70. pre_end = 0;
  71. end = post_start;
  72. while (time_before_eq(jiffies, start_jiffies + 1 +
  73. DELAY_CALIBRATION_TICKS)) {
  74. pre_end = end;
  75. read_current_timer(&end);
  76. }
  77. read_current_timer(&post_end);
  78. timer_rate_max = (post_end - pre_start) /
  79. DELAY_CALIBRATION_TICKS;
  80. timer_rate_min = (pre_end - post_start) /
  81. DELAY_CALIBRATION_TICKS;
  82. /*
  83. * If the upper limit and lower limit of the timer_rate is
  84. * >= 12.5% apart, redo calibration.
  85. */
  86. if (start >= post_end)
  87. printk(KERN_NOTICE "calibrate_delay_direct() ignoring "
  88. "timer_rate as we had a TSC wrap around"
  89. " start=%lu >=post_end=%lu\n",
  90. start, post_end);
  91. if (start < post_end && pre_start != 0 && pre_end != 0 &&
  92. (timer_rate_max - timer_rate_min) < (timer_rate_max >> 3)) {
  93. good_timer_count++;
  94. good_timer_sum += timer_rate_max;
  95. measured_times[i] = timer_rate_max;
  96. if (max < 0 || timer_rate_max > measured_times[max])
  97. max = i;
  98. if (min < 0 || timer_rate_max < measured_times[min])
  99. min = i;
  100. } else
  101. measured_times[i] = 0;
  102. }
  103. /*
  104. * Find the maximum & minimum - if they differ too much throw out the
  105. * one with the largest difference from the mean and try again...
  106. */
  107. while (good_timer_count > 1) {
  108. unsigned long estimate;
  109. unsigned long maxdiff;
  110. /* compute the estimate */
  111. estimate = (good_timer_sum/good_timer_count);
  112. maxdiff = estimate >> 3;
  113. /* if range is within 12% let's take it */
  114. if ((measured_times[max] - measured_times[min]) < maxdiff)
  115. return estimate;
  116. /* ok - drop the worse value and try again... */
  117. good_timer_sum = 0;
  118. good_timer_count = 0;
  119. if ((measured_times[max] - estimate) <
  120. (estimate - measured_times[min])) {
  121. printk(KERN_NOTICE "calibrate_delay_direct() dropping "
  122. "min bogoMips estimate %d = %lu\n",
  123. min, measured_times[min]);
  124. measured_times[min] = 0;
  125. min = max;
  126. } else {
  127. printk(KERN_NOTICE "calibrate_delay_direct() dropping "
  128. "max bogoMips estimate %d = %lu\n",
  129. max, measured_times[max]);
  130. measured_times[max] = 0;
  131. max = min;
  132. }
  133. for (i = 0; i < MAX_DIRECT_CALIBRATION_RETRIES; i++) {
  134. if (measured_times[i] == 0)
  135. continue;
  136. good_timer_count++;
  137. good_timer_sum += measured_times[i];
  138. if (measured_times[i] < measured_times[min])
  139. min = i;
  140. if (measured_times[i] > measured_times[max])
  141. max = i;
  142. }
  143. }
  144. printk(KERN_NOTICE "calibrate_delay_direct() failed to get a good "
  145. "estimate for loops_per_jiffy.\nProbably due to long platform "
  146. "interrupts. Consider using \"lpj=\" boot option.\n");
  147. return 0;
  148. }
  149. #else
  150. static unsigned long calibrate_delay_direct(void)
  151. {
  152. return 0;
  153. }
  154. #endif
  155. /*
  156. * This is the number of bits of precision for the loops_per_jiffy. Each
  157. * time we refine our estimate after the first takes 1.5/HZ seconds, so try
  158. * to start with a good estimate.
  159. * For the boot cpu we can skip the delay calibration and assign it a value
  160. * calculated based on the timer frequency.
  161. * For the rest of the CPUs we cannot assume that the timer frequency is same as
  162. * the cpu frequency, hence do the calibration for those.
  163. */
  164. #define LPS_PREC 8
  165. static unsigned long calibrate_delay_converge(void)
  166. {
  167. /* First stage - slowly accelerate to find initial bounds */
  168. unsigned long lpj, lpj_base, ticks, loopadd, loopadd_base, chop_limit;
  169. int trials = 0, band = 0, trial_in_band = 0;
  170. lpj = (1<<12);
  171. /* wait for "start of" clock tick */
  172. ticks = jiffies;
  173. while (ticks == jiffies)
  174. ; /* nothing */
  175. /* Go .. */
  176. ticks = jiffies;
  177. do {
  178. if (++trial_in_band == (1<<band)) {
  179. ++band;
  180. trial_in_band = 0;
  181. }
  182. __delay(lpj * band);
  183. trials += band;
  184. } while (ticks == jiffies);
  185. /*
  186. * We overshot, so retreat to a clear underestimate. Then estimate
  187. * the largest likely undershoot. This defines our chop bounds.
  188. */
  189. trials -= band;
  190. loopadd_base = lpj * band;
  191. lpj_base = lpj * trials;
  192. recalibrate:
  193. lpj = lpj_base;
  194. loopadd = loopadd_base;
  195. /*
  196. * Do a binary approximation to get lpj set to
  197. * equal one clock (up to LPS_PREC bits)
  198. */
  199. chop_limit = lpj >> LPS_PREC;
  200. while (loopadd > chop_limit) {
  201. lpj += loopadd;
  202. ticks = jiffies;
  203. while (ticks == jiffies)
  204. ; /* nothing */
  205. ticks = jiffies;
  206. __delay(lpj);
  207. if (jiffies != ticks) /* longer than 1 tick */
  208. lpj -= loopadd;
  209. loopadd >>= 1;
  210. }
  211. /*
  212. * If we incremented every single time possible, presume we've
  213. * massively underestimated initially, and retry with a higher
  214. * start, and larger range. (Only seen on x86_64, due to SMIs)
  215. */
  216. if (lpj + loopadd * 2 == lpj_base + loopadd_base * 2) {
  217. lpj_base = lpj;
  218. loopadd_base <<= 2;
  219. goto recalibrate;
  220. }
  221. return lpj;
  222. }
  223. static DEFINE_PER_CPU(unsigned long, cpu_loops_per_jiffy) = { 0 };
  224. /*
  225. * Check if cpu calibration delay is already known. For example,
  226. * some processors with multi-core sockets may have all cores
  227. * with the same calibration delay.
  228. *
  229. * Architectures should override this function if a faster calibration
  230. * method is available.
  231. */
  232. unsigned long __attribute__((weak)) calibrate_delay_is_known(void)
  233. {
  234. return 0;
  235. }
  236. /*
  237. * Indicate the cpu delay calibration is done. This can be used by
  238. * architectures to stop accepting delay timer registrations after this point.
  239. */
  240. void __attribute__((weak)) calibration_delay_done(void)
  241. {
  242. }
  243. void calibrate_delay(void)
  244. {
  245. unsigned long lpj;
  246. static bool printed;
  247. int this_cpu = smp_processor_id();
  248. if (per_cpu(cpu_loops_per_jiffy, this_cpu)) {
  249. lpj = per_cpu(cpu_loops_per_jiffy, this_cpu);
  250. if (!printed)
  251. pr_info("Calibrating delay loop (skipped) "
  252. "already calibrated this CPU");
  253. } else if (preset_lpj) {
  254. lpj = preset_lpj;
  255. if (!printed)
  256. pr_info("Calibrating delay loop (skipped) "
  257. "preset value.. ");
  258. } else if ((!printed) && lpj_fine) {
  259. lpj = lpj_fine;
  260. pr_info("Calibrating delay loop (skipped), "
  261. "value calculated using timer frequency.. ");
  262. } else if ((lpj = calibrate_delay_is_known())) {
  263. ;
  264. } else if ((lpj = calibrate_delay_direct()) != 0) {
  265. if (!printed)
  266. pr_info("Calibrating delay using timer "
  267. "specific routine.. ");
  268. } else {
  269. if (!printed)
  270. pr_info("Calibrating delay loop... ");
  271. lpj = calibrate_delay_converge();
  272. }
  273. per_cpu(cpu_loops_per_jiffy, this_cpu) = lpj;
  274. if (!printed)
  275. pr_cont("%lu.%02lu BogoMIPS (lpj=%lu)\n",
  276. lpj/(500000/HZ),
  277. (lpj/(5000/HZ)) % 100, lpj);
  278. loops_per_jiffy = lpj;
  279. printed = true;
  280. calibration_delay_done();
  281. }