itmt.c 5.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * itmt.c: Support Intel Turbo Boost Max Technology 3.0
  4. *
  5. * (C) Copyright 2016 Intel Corporation
  6. * Author: Tim Chen <tim.c.chen@linux.intel.com>
  7. *
  8. * On platforms supporting Intel Turbo Boost Max Technology 3.0, (ITMT),
  9. * the maximum turbo frequencies of some cores in a CPU package may be
  10. * higher than for the other cores in the same package. In that case,
  11. * better performance can be achieved by making the scheduler prefer
  12. * to run tasks on the CPUs with higher max turbo frequencies.
  13. *
  14. * This file provides functions and data structures for enabling the
  15. * scheduler to favor scheduling on cores can be boosted to a higher
  16. * frequency under ITMT.
  17. */
  18. #include <linux/sched.h>
  19. #include <linux/cpumask.h>
  20. #include <linux/cpuset.h>
  21. #include <linux/mutex.h>
  22. #include <linux/sysctl.h>
  23. #include <linux/nodemask.h>
  24. static DEFINE_MUTEX(itmt_update_mutex);
  25. DEFINE_PER_CPU_READ_MOSTLY(int, sched_core_priority);
  26. /* Boolean to track if system has ITMT capabilities */
  27. static bool __read_mostly sched_itmt_capable;
  28. /*
  29. * Boolean to control whether we want to move processes to cpu capable
  30. * of higher turbo frequency for cpus supporting Intel Turbo Boost Max
  31. * Technology 3.0.
  32. *
  33. * It can be set via /proc/sys/kernel/sched_itmt_enabled
  34. */
  35. unsigned int __read_mostly sysctl_sched_itmt_enabled;
  36. static int sched_itmt_update_handler(struct ctl_table *table, int write,
  37. void *buffer, size_t *lenp, loff_t *ppos)
  38. {
  39. unsigned int old_sysctl;
  40. int ret;
  41. mutex_lock(&itmt_update_mutex);
  42. if (!sched_itmt_capable) {
  43. mutex_unlock(&itmt_update_mutex);
  44. return -EINVAL;
  45. }
  46. old_sysctl = sysctl_sched_itmt_enabled;
  47. ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
  48. if (!ret && write && old_sysctl != sysctl_sched_itmt_enabled) {
  49. x86_topology_update = true;
  50. rebuild_sched_domains();
  51. }
  52. mutex_unlock(&itmt_update_mutex);
  53. return ret;
  54. }
  55. static struct ctl_table itmt_kern_table[] = {
  56. {
  57. .procname = "sched_itmt_enabled",
  58. .data = &sysctl_sched_itmt_enabled,
  59. .maxlen = sizeof(unsigned int),
  60. .mode = 0644,
  61. .proc_handler = sched_itmt_update_handler,
  62. .extra1 = SYSCTL_ZERO,
  63. .extra2 = SYSCTL_ONE,
  64. },
  65. {}
  66. };
  67. static struct ctl_table itmt_root_table[] = {
  68. {
  69. .procname = "kernel",
  70. .mode = 0555,
  71. .child = itmt_kern_table,
  72. },
  73. {}
  74. };
  75. static struct ctl_table_header *itmt_sysctl_header;
  76. /**
  77. * sched_set_itmt_support() - Indicate platform supports ITMT
  78. *
  79. * This function is used by the OS to indicate to scheduler that the platform
  80. * is capable of supporting the ITMT feature.
  81. *
  82. * The current scheme has the pstate driver detects if the system
  83. * is ITMT capable and call sched_set_itmt_support.
  84. *
  85. * This must be done only after sched_set_itmt_core_prio
  86. * has been called to set the cpus' priorities.
  87. * It must not be called with cpu hot plug lock
  88. * held as we need to acquire the lock to rebuild sched domains
  89. * later.
  90. *
  91. * Return: 0 on success
  92. */
  93. int sched_set_itmt_support(void)
  94. {
  95. mutex_lock(&itmt_update_mutex);
  96. if (sched_itmt_capable) {
  97. mutex_unlock(&itmt_update_mutex);
  98. return 0;
  99. }
  100. itmt_sysctl_header = register_sysctl_table(itmt_root_table);
  101. if (!itmt_sysctl_header) {
  102. mutex_unlock(&itmt_update_mutex);
  103. return -ENOMEM;
  104. }
  105. sched_itmt_capable = true;
  106. sysctl_sched_itmt_enabled = 1;
  107. x86_topology_update = true;
  108. rebuild_sched_domains();
  109. mutex_unlock(&itmt_update_mutex);
  110. return 0;
  111. }
  112. /**
  113. * sched_clear_itmt_support() - Revoke platform's support of ITMT
  114. *
  115. * This function is used by the OS to indicate that it has
  116. * revoked the platform's support of ITMT feature.
  117. *
  118. * It must not be called with cpu hot plug lock
  119. * held as we need to acquire the lock to rebuild sched domains
  120. * later.
  121. */
  122. void sched_clear_itmt_support(void)
  123. {
  124. mutex_lock(&itmt_update_mutex);
  125. if (!sched_itmt_capable) {
  126. mutex_unlock(&itmt_update_mutex);
  127. return;
  128. }
  129. sched_itmt_capable = false;
  130. if (itmt_sysctl_header) {
  131. unregister_sysctl_table(itmt_sysctl_header);
  132. itmt_sysctl_header = NULL;
  133. }
  134. if (sysctl_sched_itmt_enabled) {
  135. /* disable sched_itmt if we are no longer ITMT capable */
  136. sysctl_sched_itmt_enabled = 0;
  137. x86_topology_update = true;
  138. rebuild_sched_domains();
  139. }
  140. mutex_unlock(&itmt_update_mutex);
  141. }
  142. int arch_asym_cpu_priority(int cpu)
  143. {
  144. return per_cpu(sched_core_priority, cpu);
  145. }
  146. /**
  147. * sched_set_itmt_core_prio() - Set CPU priority based on ITMT
  148. * @prio: Priority of cpu core
  149. * @core_cpu: The cpu number associated with the core
  150. *
  151. * The pstate driver will find out the max boost frequency
  152. * and call this function to set a priority proportional
  153. * to the max boost frequency. CPU with higher boost
  154. * frequency will receive higher priority.
  155. *
  156. * No need to rebuild sched domain after updating
  157. * the CPU priorities. The sched domains have no
  158. * dependency on CPU priorities.
  159. */
  160. void sched_set_itmt_core_prio(int prio, int core_cpu)
  161. {
  162. int cpu, i = 1;
  163. for_each_cpu(cpu, topology_sibling_cpumask(core_cpu)) {
  164. int smt_prio;
  165. /*
  166. * Ensure that the siblings are moved to the end
  167. * of the priority chain and only used when
  168. * all other high priority cpus are out of capacity.
  169. */
  170. smt_prio = prio * smp_num_siblings / i;
  171. per_cpu(sched_core_priority, cpu) = smt_prio;
  172. i++;
  173. }
  174. }