cpufreq_ondemand.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * drivers/cpufreq/cpufreq_ondemand.c
  4. *
  5. * Copyright (C) 2001 Russell King
  6. * (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
  7. * Jun Nakajima <jun.nakajima@intel.com>
  8. */
  9. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  10. #include <linux/cpu.h>
  11. #include <linux/percpu-defs.h>
  12. #include <linux/slab.h>
  13. #include <linux/tick.h>
  14. #include <linux/sched/cpufreq.h>
  15. #include "cpufreq_ondemand.h"
  16. /* On-demand governor macros */
  17. #define DEF_FREQUENCY_UP_THRESHOLD (80)
  18. #define DEF_SAMPLING_DOWN_FACTOR (1)
  19. #define MAX_SAMPLING_DOWN_FACTOR (100000)
  20. #define MICRO_FREQUENCY_UP_THRESHOLD (95)
  21. #define MICRO_FREQUENCY_MIN_SAMPLE_RATE (10000)
  22. #define MIN_FREQUENCY_UP_THRESHOLD (1)
  23. #define MAX_FREQUENCY_UP_THRESHOLD (100)
  24. static struct od_ops od_ops;
  25. static unsigned int default_powersave_bias;
  26. /*
  27. * Not all CPUs want IO time to be accounted as busy; this depends on how
  28. * efficient idling at a higher frequency/voltage is.
  29. * Pavel Machek says this is not so for various generations of AMD and old
  30. * Intel systems.
  31. * Mike Chan (android.com) claims this is also not true for ARM.
  32. * Because of this, whitelist specific known (series) of CPUs by default, and
  33. * leave all others up to the user.
  34. */
  35. static int should_io_be_busy(void)
  36. {
  37. #if defined(CONFIG_X86)
  38. /*
  39. * For Intel, Core 2 (model 15) and later have an efficient idle.
  40. */
  41. if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL &&
  42. boot_cpu_data.x86 == 6 &&
  43. boot_cpu_data.x86_model >= 15)
  44. return 1;
  45. #endif
  46. return 0;
  47. }
  48. /*
  49. * Find right freq to be set now with powersave_bias on.
  50. * Returns the freq_hi to be used right now and will set freq_hi_delay_us,
  51. * freq_lo, and freq_lo_delay_us in percpu area for averaging freqs.
  52. */
  53. static unsigned int generic_powersave_bias_target(struct cpufreq_policy *policy,
  54. unsigned int freq_next, unsigned int relation)
  55. {
  56. unsigned int freq_req, freq_reduc, freq_avg;
  57. unsigned int freq_hi, freq_lo;
  58. unsigned int index;
  59. unsigned int delay_hi_us;
  60. struct policy_dbs_info *policy_dbs = policy->governor_data;
  61. struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs);
  62. struct dbs_data *dbs_data = policy_dbs->dbs_data;
  63. struct od_dbs_tuners *od_tuners = dbs_data->tuners;
  64. struct cpufreq_frequency_table *freq_table = policy->freq_table;
  65. if (!freq_table) {
  66. dbs_info->freq_lo = 0;
  67. dbs_info->freq_lo_delay_us = 0;
  68. return freq_next;
  69. }
  70. index = cpufreq_frequency_table_target(policy, freq_next, relation);
  71. freq_req = freq_table[index].frequency;
  72. freq_reduc = freq_req * od_tuners->powersave_bias / 1000;
  73. freq_avg = freq_req - freq_reduc;
  74. /* Find freq bounds for freq_avg in freq_table */
  75. index = cpufreq_table_find_index_h(policy, freq_avg);
  76. freq_lo = freq_table[index].frequency;
  77. index = cpufreq_table_find_index_l(policy, freq_avg);
  78. freq_hi = freq_table[index].frequency;
  79. /* Find out how long we have to be in hi and lo freqs */
  80. if (freq_hi == freq_lo) {
  81. dbs_info->freq_lo = 0;
  82. dbs_info->freq_lo_delay_us = 0;
  83. return freq_lo;
  84. }
  85. delay_hi_us = (freq_avg - freq_lo) * dbs_data->sampling_rate;
  86. delay_hi_us += (freq_hi - freq_lo) / 2;
  87. delay_hi_us /= freq_hi - freq_lo;
  88. dbs_info->freq_hi_delay_us = delay_hi_us;
  89. dbs_info->freq_lo = freq_lo;
  90. dbs_info->freq_lo_delay_us = dbs_data->sampling_rate - delay_hi_us;
  91. return freq_hi;
  92. }
  93. static void ondemand_powersave_bias_init(struct cpufreq_policy *policy)
  94. {
  95. struct od_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data);
  96. dbs_info->freq_lo = 0;
  97. }
  98. static void dbs_freq_increase(struct cpufreq_policy *policy, unsigned int freq)
  99. {
  100. struct policy_dbs_info *policy_dbs = policy->governor_data;
  101. struct dbs_data *dbs_data = policy_dbs->dbs_data;
  102. struct od_dbs_tuners *od_tuners = dbs_data->tuners;
  103. if (od_tuners->powersave_bias)
  104. freq = od_ops.powersave_bias_target(policy, freq,
  105. CPUFREQ_RELATION_H);
  106. else if (policy->cur == policy->max)
  107. return;
  108. __cpufreq_driver_target(policy, freq, od_tuners->powersave_bias ?
  109. CPUFREQ_RELATION_L : CPUFREQ_RELATION_H);
  110. }
  111. /*
  112. * Every sampling_rate, we check, if current idle time is less than 20%
  113. * (default), then we try to increase frequency. Else, we adjust the frequency
  114. * proportional to load.
  115. */
  116. static void od_update(struct cpufreq_policy *policy)
  117. {
  118. struct policy_dbs_info *policy_dbs = policy->governor_data;
  119. struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs);
  120. struct dbs_data *dbs_data = policy_dbs->dbs_data;
  121. struct od_dbs_tuners *od_tuners = dbs_data->tuners;
  122. unsigned int load = dbs_update(policy);
  123. dbs_info->freq_lo = 0;
  124. /* Check for frequency increase */
  125. if (load > dbs_data->up_threshold) {
  126. /* If switching to max speed, apply sampling_down_factor */
  127. if (policy->cur < policy->max)
  128. policy_dbs->rate_mult = dbs_data->sampling_down_factor;
  129. dbs_freq_increase(policy, policy->max);
  130. } else {
  131. /* Calculate the next frequency proportional to load */
  132. unsigned int freq_next, min_f, max_f;
  133. min_f = policy->cpuinfo.min_freq;
  134. max_f = policy->cpuinfo.max_freq;
  135. freq_next = min_f + load * (max_f - min_f) / 100;
  136. /* No longer fully busy, reset rate_mult */
  137. policy_dbs->rate_mult = 1;
  138. if (od_tuners->powersave_bias)
  139. freq_next = od_ops.powersave_bias_target(policy,
  140. freq_next,
  141. CPUFREQ_RELATION_L);
  142. __cpufreq_driver_target(policy, freq_next, CPUFREQ_RELATION_C);
  143. }
  144. }
  145. static unsigned int od_dbs_update(struct cpufreq_policy *policy)
  146. {
  147. struct policy_dbs_info *policy_dbs = policy->governor_data;
  148. struct dbs_data *dbs_data = policy_dbs->dbs_data;
  149. struct od_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs);
  150. int sample_type = dbs_info->sample_type;
  151. /* Common NORMAL_SAMPLE setup */
  152. dbs_info->sample_type = OD_NORMAL_SAMPLE;
  153. /*
  154. * OD_SUB_SAMPLE doesn't make sense if sample_delay_ns is 0, so ignore
  155. * it then.
  156. */
  157. if (sample_type == OD_SUB_SAMPLE && policy_dbs->sample_delay_ns > 0) {
  158. __cpufreq_driver_target(policy, dbs_info->freq_lo,
  159. CPUFREQ_RELATION_H);
  160. return dbs_info->freq_lo_delay_us;
  161. }
  162. od_update(policy);
  163. if (dbs_info->freq_lo) {
  164. /* Setup SUB_SAMPLE */
  165. dbs_info->sample_type = OD_SUB_SAMPLE;
  166. return dbs_info->freq_hi_delay_us;
  167. }
  168. return dbs_data->sampling_rate * policy_dbs->rate_mult;
  169. }
  170. /************************** sysfs interface ************************/
  171. static struct dbs_governor od_dbs_gov;
  172. static ssize_t store_io_is_busy(struct gov_attr_set *attr_set, const char *buf,
  173. size_t count)
  174. {
  175. struct dbs_data *dbs_data = to_dbs_data(attr_set);
  176. unsigned int input;
  177. int ret;
  178. ret = sscanf(buf, "%u", &input);
  179. if (ret != 1)
  180. return -EINVAL;
  181. dbs_data->io_is_busy = !!input;
  182. /* we need to re-evaluate prev_cpu_idle */
  183. gov_update_cpu_data(dbs_data);
  184. return count;
  185. }
  186. static ssize_t store_up_threshold(struct gov_attr_set *attr_set,
  187. const char *buf, size_t count)
  188. {
  189. struct dbs_data *dbs_data = to_dbs_data(attr_set);
  190. unsigned int input;
  191. int ret;
  192. ret = sscanf(buf, "%u", &input);
  193. if (ret != 1 || input > MAX_FREQUENCY_UP_THRESHOLD ||
  194. input < MIN_FREQUENCY_UP_THRESHOLD) {
  195. return -EINVAL;
  196. }
  197. dbs_data->up_threshold = input;
  198. return count;
  199. }
  200. static ssize_t store_sampling_down_factor(struct gov_attr_set *attr_set,
  201. const char *buf, size_t count)
  202. {
  203. struct dbs_data *dbs_data = to_dbs_data(attr_set);
  204. struct policy_dbs_info *policy_dbs;
  205. unsigned int input;
  206. int ret;
  207. ret = sscanf(buf, "%u", &input);
  208. if (ret != 1 || input > MAX_SAMPLING_DOWN_FACTOR || input < 1)
  209. return -EINVAL;
  210. dbs_data->sampling_down_factor = input;
  211. /* Reset down sampling multiplier in case it was active */
  212. list_for_each_entry(policy_dbs, &attr_set->policy_list, list) {
  213. /*
  214. * Doing this without locking might lead to using different
  215. * rate_mult values in od_update() and od_dbs_update().
  216. */
  217. mutex_lock(&policy_dbs->update_mutex);
  218. policy_dbs->rate_mult = 1;
  219. mutex_unlock(&policy_dbs->update_mutex);
  220. }
  221. return count;
  222. }
  223. static ssize_t store_ignore_nice_load(struct gov_attr_set *attr_set,
  224. const char *buf, size_t count)
  225. {
  226. struct dbs_data *dbs_data = to_dbs_data(attr_set);
  227. unsigned int input;
  228. int ret;
  229. ret = sscanf(buf, "%u", &input);
  230. if (ret != 1)
  231. return -EINVAL;
  232. if (input > 1)
  233. input = 1;
  234. if (input == dbs_data->ignore_nice_load) { /* nothing to do */
  235. return count;
  236. }
  237. dbs_data->ignore_nice_load = input;
  238. /* we need to re-evaluate prev_cpu_idle */
  239. gov_update_cpu_data(dbs_data);
  240. return count;
  241. }
  242. static ssize_t store_powersave_bias(struct gov_attr_set *attr_set,
  243. const char *buf, size_t count)
  244. {
  245. struct dbs_data *dbs_data = to_dbs_data(attr_set);
  246. struct od_dbs_tuners *od_tuners = dbs_data->tuners;
  247. struct policy_dbs_info *policy_dbs;
  248. unsigned int input;
  249. int ret;
  250. ret = sscanf(buf, "%u", &input);
  251. if (ret != 1)
  252. return -EINVAL;
  253. if (input > 1000)
  254. input = 1000;
  255. od_tuners->powersave_bias = input;
  256. list_for_each_entry(policy_dbs, &attr_set->policy_list, list)
  257. ondemand_powersave_bias_init(policy_dbs->policy);
  258. return count;
  259. }
  260. gov_show_one_common(sampling_rate);
  261. gov_show_one_common(up_threshold);
  262. gov_show_one_common(sampling_down_factor);
  263. gov_show_one_common(ignore_nice_load);
  264. gov_show_one_common(io_is_busy);
  265. gov_show_one(od, powersave_bias);
  266. gov_attr_rw(sampling_rate);
  267. gov_attr_rw(io_is_busy);
  268. gov_attr_rw(up_threshold);
  269. gov_attr_rw(sampling_down_factor);
  270. gov_attr_rw(ignore_nice_load);
  271. gov_attr_rw(powersave_bias);
  272. static struct attribute *od_attributes[] = {
  273. &sampling_rate.attr,
  274. &up_threshold.attr,
  275. &sampling_down_factor.attr,
  276. &ignore_nice_load.attr,
  277. &powersave_bias.attr,
  278. &io_is_busy.attr,
  279. NULL
  280. };
  281. /************************** sysfs end ************************/
  282. static struct policy_dbs_info *od_alloc(void)
  283. {
  284. struct od_policy_dbs_info *dbs_info;
  285. dbs_info = kzalloc(sizeof(*dbs_info), GFP_KERNEL);
  286. return dbs_info ? &dbs_info->policy_dbs : NULL;
  287. }
  288. static void od_free(struct policy_dbs_info *policy_dbs)
  289. {
  290. kfree(to_dbs_info(policy_dbs));
  291. }
  292. static int od_init(struct dbs_data *dbs_data)
  293. {
  294. struct od_dbs_tuners *tuners;
  295. u64 idle_time;
  296. int cpu;
  297. tuners = kzalloc(sizeof(*tuners), GFP_KERNEL);
  298. if (!tuners)
  299. return -ENOMEM;
  300. cpu = get_cpu();
  301. idle_time = get_cpu_idle_time_us(cpu, NULL);
  302. put_cpu();
  303. if (idle_time != -1ULL) {
  304. /* Idle micro accounting is supported. Use finer thresholds */
  305. dbs_data->up_threshold = MICRO_FREQUENCY_UP_THRESHOLD;
  306. } else {
  307. dbs_data->up_threshold = DEF_FREQUENCY_UP_THRESHOLD;
  308. }
  309. dbs_data->sampling_down_factor = DEF_SAMPLING_DOWN_FACTOR;
  310. dbs_data->ignore_nice_load = 0;
  311. tuners->powersave_bias = default_powersave_bias;
  312. dbs_data->io_is_busy = should_io_be_busy();
  313. dbs_data->tuners = tuners;
  314. return 0;
  315. }
  316. static void od_exit(struct dbs_data *dbs_data)
  317. {
  318. kfree(dbs_data->tuners);
  319. }
  320. static void od_start(struct cpufreq_policy *policy)
  321. {
  322. struct od_policy_dbs_info *dbs_info = to_dbs_info(policy->governor_data);
  323. dbs_info->sample_type = OD_NORMAL_SAMPLE;
  324. ondemand_powersave_bias_init(policy);
  325. }
  326. static struct od_ops od_ops = {
  327. .powersave_bias_target = generic_powersave_bias_target,
  328. };
  329. static struct dbs_governor od_dbs_gov = {
  330. .gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("ondemand"),
  331. .kobj_type = { .default_attrs = od_attributes },
  332. .gov_dbs_update = od_dbs_update,
  333. .alloc = od_alloc,
  334. .free = od_free,
  335. .init = od_init,
  336. .exit = od_exit,
  337. .start = od_start,
  338. };
  339. #define CPU_FREQ_GOV_ONDEMAND (od_dbs_gov.gov)
  340. static void od_set_powersave_bias(unsigned int powersave_bias)
  341. {
  342. unsigned int cpu;
  343. cpumask_t done;
  344. default_powersave_bias = powersave_bias;
  345. cpumask_clear(&done);
  346. get_online_cpus();
  347. for_each_online_cpu(cpu) {
  348. struct cpufreq_policy *policy;
  349. struct policy_dbs_info *policy_dbs;
  350. struct dbs_data *dbs_data;
  351. struct od_dbs_tuners *od_tuners;
  352. if (cpumask_test_cpu(cpu, &done))
  353. continue;
  354. policy = cpufreq_cpu_get_raw(cpu);
  355. if (!policy || policy->governor != &CPU_FREQ_GOV_ONDEMAND)
  356. continue;
  357. policy_dbs = policy->governor_data;
  358. if (!policy_dbs)
  359. continue;
  360. cpumask_or(&done, &done, policy->cpus);
  361. dbs_data = policy_dbs->dbs_data;
  362. od_tuners = dbs_data->tuners;
  363. od_tuners->powersave_bias = default_powersave_bias;
  364. }
  365. put_online_cpus();
  366. }
  367. void od_register_powersave_bias_handler(unsigned int (*f)
  368. (struct cpufreq_policy *, unsigned int, unsigned int),
  369. unsigned int powersave_bias)
  370. {
  371. od_ops.powersave_bias_target = f;
  372. od_set_powersave_bias(powersave_bias);
  373. }
  374. EXPORT_SYMBOL_GPL(od_register_powersave_bias_handler);
  375. void od_unregister_powersave_bias_handler(void)
  376. {
  377. od_ops.powersave_bias_target = generic_powersave_bias_target;
  378. od_set_powersave_bias(0);
  379. }
  380. EXPORT_SYMBOL_GPL(od_unregister_powersave_bias_handler);
  381. MODULE_AUTHOR("Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>");
  382. MODULE_AUTHOR("Alexey Starikovskiy <alexey.y.starikovskiy@intel.com>");
  383. MODULE_DESCRIPTION("'cpufreq_ondemand' - A dynamic cpufreq governor for "
  384. "Low Latency Frequency Transition capable processors");
  385. MODULE_LICENSE("GPL");
  386. #ifdef CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND
  387. struct cpufreq_governor *cpufreq_default_governor(void)
  388. {
  389. return &CPU_FREQ_GOV_ONDEMAND;
  390. }
  391. #endif
  392. cpufreq_governor_init(CPU_FREQ_GOV_ONDEMAND);
  393. cpufreq_governor_exit(CPU_FREQ_GOV_ONDEMAND);