isolation.c 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Housekeeping management. Manage the targets for routine code that can run on
  4. * any CPU: unbound workqueues, timers, kthreads and any offloadable work.
  5. *
  6. * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
  7. * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
  8. *
  9. */
  10. #include "sched.h"
  11. DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
  12. EXPORT_SYMBOL_GPL(housekeeping_overridden);
  13. static cpumask_var_t housekeeping_mask;
  14. static unsigned int housekeeping_flags;
  15. bool housekeeping_enabled(enum hk_flags flags)
  16. {
  17. return !!(housekeeping_flags & flags);
  18. }
  19. EXPORT_SYMBOL_GPL(housekeeping_enabled);
  20. int housekeeping_any_cpu(enum hk_flags flags)
  21. {
  22. int cpu;
  23. if (static_branch_unlikely(&housekeeping_overridden)) {
  24. if (housekeeping_flags & flags) {
  25. cpu = sched_numa_find_closest(housekeeping_mask, smp_processor_id());
  26. if (cpu < nr_cpu_ids)
  27. return cpu;
  28. return cpumask_any_and(housekeeping_mask, cpu_online_mask);
  29. }
  30. }
  31. return smp_processor_id();
  32. }
  33. EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
  34. const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
  35. {
  36. if (static_branch_unlikely(&housekeeping_overridden))
  37. if (housekeeping_flags & flags)
  38. return housekeeping_mask;
  39. return cpu_possible_mask;
  40. }
  41. EXPORT_SYMBOL_GPL(housekeeping_cpumask);
  42. void housekeeping_affine(struct task_struct *t, enum hk_flags flags)
  43. {
  44. if (static_branch_unlikely(&housekeeping_overridden))
  45. if (housekeeping_flags & flags)
  46. set_cpus_allowed_ptr(t, housekeeping_mask);
  47. }
  48. EXPORT_SYMBOL_GPL(housekeeping_affine);
  49. bool housekeeping_test_cpu(int cpu, enum hk_flags flags)
  50. {
  51. if (static_branch_unlikely(&housekeeping_overridden))
  52. if (housekeeping_flags & flags)
  53. return cpumask_test_cpu(cpu, housekeeping_mask);
  54. return true;
  55. }
  56. EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
  57. void __init housekeeping_init(void)
  58. {
  59. if (!housekeeping_flags)
  60. return;
  61. static_branch_enable(&housekeeping_overridden);
  62. if (housekeeping_flags & HK_FLAG_TICK)
  63. sched_tick_offload_init();
  64. /* We need at least one CPU to handle housekeeping work */
  65. WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
  66. }
  67. static int __init housekeeping_setup(char *str, enum hk_flags flags)
  68. {
  69. cpumask_var_t non_housekeeping_mask;
  70. cpumask_var_t tmp;
  71. int err;
  72. alloc_bootmem_cpumask_var(&non_housekeeping_mask);
  73. err = cpulist_parse(str, non_housekeeping_mask);
  74. if (err < 0 || cpumask_last(non_housekeeping_mask) >= nr_cpu_ids) {
  75. pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
  76. free_bootmem_cpumask_var(non_housekeeping_mask);
  77. return 0;
  78. }
  79. alloc_bootmem_cpumask_var(&tmp);
  80. if (!housekeeping_flags) {
  81. alloc_bootmem_cpumask_var(&housekeeping_mask);
  82. cpumask_andnot(housekeeping_mask,
  83. cpu_possible_mask, non_housekeeping_mask);
  84. cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
  85. if (cpumask_empty(tmp)) {
  86. pr_warn("Housekeeping: must include one present CPU, "
  87. "using boot CPU:%d\n", smp_processor_id());
  88. __cpumask_set_cpu(smp_processor_id(), housekeeping_mask);
  89. __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
  90. }
  91. } else {
  92. cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
  93. if (cpumask_empty(tmp))
  94. __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
  95. cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask);
  96. if (!cpumask_equal(tmp, housekeeping_mask)) {
  97. pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
  98. free_bootmem_cpumask_var(tmp);
  99. free_bootmem_cpumask_var(non_housekeeping_mask);
  100. return 0;
  101. }
  102. }
  103. free_bootmem_cpumask_var(tmp);
  104. if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) {
  105. if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
  106. tick_nohz_full_setup(non_housekeeping_mask);
  107. } else {
  108. pr_warn("Housekeeping: nohz unsupported."
  109. " Build with CONFIG_NO_HZ_FULL\n");
  110. free_bootmem_cpumask_var(non_housekeeping_mask);
  111. return 0;
  112. }
  113. }
  114. housekeeping_flags |= flags;
  115. free_bootmem_cpumask_var(non_housekeeping_mask);
  116. return 1;
  117. }
  118. static int __init housekeeping_nohz_full_setup(char *str)
  119. {
  120. unsigned int flags;
  121. flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU |
  122. HK_FLAG_MISC | HK_FLAG_KTHREAD;
  123. return housekeeping_setup(str, flags);
  124. }
  125. __setup("nohz_full=", housekeeping_nohz_full_setup);
  126. static int __init housekeeping_isolcpus_setup(char *str)
  127. {
  128. unsigned int flags = 0;
  129. bool illegal = false;
  130. char *par;
  131. int len;
  132. while (isalpha(*str)) {
  133. if (!strncmp(str, "nohz,", 5)) {
  134. str += 5;
  135. flags |= HK_FLAG_TICK;
  136. continue;
  137. }
  138. if (!strncmp(str, "domain,", 7)) {
  139. str += 7;
  140. flags |= HK_FLAG_DOMAIN;
  141. continue;
  142. }
  143. if (!strncmp(str, "managed_irq,", 12)) {
  144. str += 12;
  145. flags |= HK_FLAG_MANAGED_IRQ;
  146. continue;
  147. }
  148. /*
  149. * Skip unknown sub-parameter and validate that it is not
  150. * containing an invalid character.
  151. */
  152. for (par = str, len = 0; *str && *str != ','; str++, len++) {
  153. if (!isalpha(*str) && *str != '_')
  154. illegal = true;
  155. }
  156. if (illegal) {
  157. pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
  158. return 0;
  159. }
  160. pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
  161. str++;
  162. }
  163. /* Default behaviour for isolcpus without flags */
  164. if (!flags)
  165. flags |= HK_FLAG_DOMAIN;
  166. return housekeeping_setup(str, flags);
  167. }
  168. __setup("isolcpus=", housekeeping_isolcpus_setup);