coupled.c 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * coupled.c - helper functions to enter the same idle state on multiple cpus
  4. *
  5. * Copyright (c) 2011 Google, Inc.
  6. *
  7. * Author: Colin Cross <ccross@android.com>
  8. */
  9. #include <linux/kernel.h>
  10. #include <linux/cpu.h>
  11. #include <linux/cpuidle.h>
  12. #include <linux/mutex.h>
  13. #include <linux/sched.h>
  14. #include <linux/slab.h>
  15. #include <linux/spinlock.h>
  16. #include "cpuidle.h"
  17. /**
  18. * DOC: Coupled cpuidle states
  19. *
  20. * On some ARM SMP SoCs (OMAP4460, Tegra 2, and probably more), the
  21. * cpus cannot be independently powered down, either due to
  22. * sequencing restrictions (on Tegra 2, cpu 0 must be the last to
  23. * power down), or due to HW bugs (on OMAP4460, a cpu powering up
  24. * will corrupt the gic state unless the other cpu runs a work
  25. * around). Each cpu has a power state that it can enter without
  26. * coordinating with the other cpu (usually Wait For Interrupt, or
  27. * WFI), and one or more "coupled" power states that affect blocks
  28. * shared between the cpus (L2 cache, interrupt controller, and
  29. * sometimes the whole SoC). Entering a coupled power state must
  30. * be tightly controlled on both cpus.
  31. *
  32. * This file implements a solution, where each cpu will wait in the
  33. * WFI state until all cpus are ready to enter a coupled state, at
  34. * which point the coupled state function will be called on all
  35. * cpus at approximately the same time.
  36. *
  37. * Once all cpus are ready to enter idle, they are woken by an smp
  38. * cross call. At this point, there is a chance that one of the
  39. * cpus will find work to do, and choose not to enter idle. A
  40. * final pass is needed to guarantee that all cpus will call the
  41. * power state enter function at the same time. During this pass,
  42. * each cpu will increment the ready counter, and continue once the
  43. * ready counter matches the number of online coupled cpus. If any
  44. * cpu exits idle, the other cpus will decrement their counter and
  45. * retry.
  46. *
  47. * requested_state stores the deepest coupled idle state each cpu
  48. * is ready for. It is assumed that the states are indexed from
  49. * shallowest (highest power, lowest exit latency) to deepest
  50. * (lowest power, highest exit latency). The requested_state
  51. * variable is not locked. It is only written from the cpu that
  52. * it stores (or by the on/offlining cpu if that cpu is offline),
  53. * and only read after all the cpus are ready for the coupled idle
  54. * state are are no longer updating it.
  55. *
  56. * Three atomic counters are used. alive_count tracks the number
  57. * of cpus in the coupled set that are currently or soon will be
  58. * online. waiting_count tracks the number of cpus that are in
  59. * the waiting loop, in the ready loop, or in the coupled idle state.
  60. * ready_count tracks the number of cpus that are in the ready loop
  61. * or in the coupled idle state.
  62. *
  63. * To use coupled cpuidle states, a cpuidle driver must:
  64. *
  65. * Set struct cpuidle_device.coupled_cpus to the mask of all
  66. * coupled cpus, usually the same as cpu_possible_mask if all cpus
  67. * are part of the same cluster. The coupled_cpus mask must be
  68. * set in the struct cpuidle_device for each cpu.
  69. *
  70. * Set struct cpuidle_device.safe_state to a state that is not a
  71. * coupled state. This is usually WFI.
  72. *
  73. * Set CPUIDLE_FLAG_COUPLED in struct cpuidle_state.flags for each
  74. * state that affects multiple cpus.
  75. *
  76. * Provide a struct cpuidle_state.enter function for each state
  77. * that affects multiple cpus. This function is guaranteed to be
  78. * called on all cpus at approximately the same time. The driver
  79. * should ensure that the cpus all abort together if any cpu tries
  80. * to abort once the function is called. The function should return
  81. * with interrupts still disabled.
  82. */
  83. /**
  84. * struct cpuidle_coupled - data for set of cpus that share a coupled idle state
  85. * @coupled_cpus: mask of cpus that are part of the coupled set
  86. * @requested_state: array of requested states for cpus in the coupled set
  87. * @ready_waiting_counts: combined count of cpus in ready or waiting loops
  88. * @abort_barrier: synchronisation point for abort cases
  89. * @online_count: count of cpus that are online
  90. * @refcnt: reference count of cpuidle devices that are using this struct
  91. * @prevent: flag to prevent coupled idle while a cpu is hotplugging
  92. */
  93. struct cpuidle_coupled {
  94. cpumask_t coupled_cpus;
  95. int requested_state[NR_CPUS];
  96. atomic_t ready_waiting_counts;
  97. atomic_t abort_barrier;
  98. int online_count;
  99. int refcnt;
  100. int prevent;
  101. };
  102. #define WAITING_BITS 16
  103. #define MAX_WAITING_CPUS (1 << WAITING_BITS)
  104. #define WAITING_MASK (MAX_WAITING_CPUS - 1)
  105. #define READY_MASK (~WAITING_MASK)
  106. #define CPUIDLE_COUPLED_NOT_IDLE (-1)
  107. static DEFINE_PER_CPU(call_single_data_t, cpuidle_coupled_poke_cb);
  108. /*
  109. * The cpuidle_coupled_poke_pending mask is used to avoid calling
  110. * __smp_call_function_single with the per cpu call_single_data_t struct already
  111. * in use. This prevents a deadlock where two cpus are waiting for each others
  112. * call_single_data_t struct to be available
  113. */
  114. static cpumask_t cpuidle_coupled_poke_pending;
  115. /*
  116. * The cpuidle_coupled_poked mask is used to ensure that each cpu has been poked
  117. * once to minimize entering the ready loop with a poke pending, which would
  118. * require aborting and retrying.
  119. */
  120. static cpumask_t cpuidle_coupled_poked;
  121. /**
  122. * cpuidle_coupled_parallel_barrier - synchronize all online coupled cpus
  123. * @dev: cpuidle_device of the calling cpu
  124. * @a: atomic variable to hold the barrier
  125. *
  126. * No caller to this function will return from this function until all online
  127. * cpus in the same coupled group have called this function. Once any caller
  128. * has returned from this function, the barrier is immediately available for
  129. * reuse.
  130. *
  131. * The atomic variable must be initialized to 0 before any cpu calls
  132. * this function, will be reset to 0 before any cpu returns from this function.
  133. *
  134. * Must only be called from within a coupled idle state handler
  135. * (state.enter when state.flags has CPUIDLE_FLAG_COUPLED set).
  136. *
  137. * Provides full smp barrier semantics before and after calling.
  138. */
  139. void cpuidle_coupled_parallel_barrier(struct cpuidle_device *dev, atomic_t *a)
  140. {
  141. int n = dev->coupled->online_count;
  142. smp_mb__before_atomic();
  143. atomic_inc(a);
  144. while (atomic_read(a) < n)
  145. cpu_relax();
  146. if (atomic_inc_return(a) == n * 2) {
  147. atomic_set(a, 0);
  148. return;
  149. }
  150. while (atomic_read(a) > n)
  151. cpu_relax();
  152. }
  153. /**
  154. * cpuidle_state_is_coupled - check if a state is part of a coupled set
  155. * @drv: struct cpuidle_driver for the platform
  156. * @state: index of the target state in drv->states
  157. *
  158. * Returns true if the target state is coupled with cpus besides this one
  159. */
  160. bool cpuidle_state_is_coupled(struct cpuidle_driver *drv, int state)
  161. {
  162. return drv->states[state].flags & CPUIDLE_FLAG_COUPLED;
  163. }
  164. /**
  165. * cpuidle_coupled_state_verify - check if the coupled states are correctly set.
  166. * @drv: struct cpuidle_driver for the platform
  167. *
  168. * Returns 0 for valid state values, a negative error code otherwise:
  169. * * -EINVAL if any coupled state(safe_state_index) is wrongly set.
  170. */
  171. int cpuidle_coupled_state_verify(struct cpuidle_driver *drv)
  172. {
  173. int i;
  174. for (i = drv->state_count - 1; i >= 0; i--) {
  175. if (cpuidle_state_is_coupled(drv, i) &&
  176. (drv->safe_state_index == i ||
  177. drv->safe_state_index < 0 ||
  178. drv->safe_state_index >= drv->state_count))
  179. return -EINVAL;
  180. }
  181. return 0;
  182. }
  183. /**
  184. * cpuidle_coupled_set_ready - mark a cpu as ready
  185. * @coupled: the struct coupled that contains the current cpu
  186. */
  187. static inline void cpuidle_coupled_set_ready(struct cpuidle_coupled *coupled)
  188. {
  189. atomic_add(MAX_WAITING_CPUS, &coupled->ready_waiting_counts);
  190. }
  191. /**
  192. * cpuidle_coupled_set_not_ready - mark a cpu as not ready
  193. * @coupled: the struct coupled that contains the current cpu
  194. *
  195. * Decrements the ready counter, unless the ready (and thus the waiting) counter
  196. * is equal to the number of online cpus. Prevents a race where one cpu
  197. * decrements the waiting counter and then re-increments it just before another
  198. * cpu has decremented its ready counter, leading to the ready counter going
  199. * down from the number of online cpus without going through the coupled idle
  200. * state.
  201. *
  202. * Returns 0 if the counter was decremented successfully, -EINVAL if the ready
  203. * counter was equal to the number of online cpus.
  204. */
  205. static
  206. inline int cpuidle_coupled_set_not_ready(struct cpuidle_coupled *coupled)
  207. {
  208. int all;
  209. int ret;
  210. all = coupled->online_count | (coupled->online_count << WAITING_BITS);
  211. ret = atomic_add_unless(&coupled->ready_waiting_counts,
  212. -MAX_WAITING_CPUS, all);
  213. return ret ? 0 : -EINVAL;
  214. }
  215. /**
  216. * cpuidle_coupled_no_cpus_ready - check if no cpus in a coupled set are ready
  217. * @coupled: the struct coupled that contains the current cpu
  218. *
  219. * Returns true if all of the cpus in a coupled set are out of the ready loop.
  220. */
  221. static inline int cpuidle_coupled_no_cpus_ready(struct cpuidle_coupled *coupled)
  222. {
  223. int r = atomic_read(&coupled->ready_waiting_counts) >> WAITING_BITS;
  224. return r == 0;
  225. }
  226. /**
  227. * cpuidle_coupled_cpus_ready - check if all cpus in a coupled set are ready
  228. * @coupled: the struct coupled that contains the current cpu
  229. *
  230. * Returns true if all cpus coupled to this target state are in the ready loop
  231. */
  232. static inline bool cpuidle_coupled_cpus_ready(struct cpuidle_coupled *coupled)
  233. {
  234. int r = atomic_read(&coupled->ready_waiting_counts) >> WAITING_BITS;
  235. return r == coupled->online_count;
  236. }
  237. /**
  238. * cpuidle_coupled_cpus_waiting - check if all cpus in a coupled set are waiting
  239. * @coupled: the struct coupled that contains the current cpu
  240. *
  241. * Returns true if all cpus coupled to this target state are in the wait loop
  242. */
  243. static inline bool cpuidle_coupled_cpus_waiting(struct cpuidle_coupled *coupled)
  244. {
  245. int w = atomic_read(&coupled->ready_waiting_counts) & WAITING_MASK;
  246. return w == coupled->online_count;
  247. }
  248. /**
  249. * cpuidle_coupled_no_cpus_waiting - check if no cpus in coupled set are waiting
  250. * @coupled: the struct coupled that contains the current cpu
  251. *
  252. * Returns true if all of the cpus in a coupled set are out of the waiting loop.
  253. */
  254. static inline int cpuidle_coupled_no_cpus_waiting(struct cpuidle_coupled *coupled)
  255. {
  256. int w = atomic_read(&coupled->ready_waiting_counts) & WAITING_MASK;
  257. return w == 0;
  258. }
  259. /**
  260. * cpuidle_coupled_get_state - determine the deepest idle state
  261. * @dev: struct cpuidle_device for this cpu
  262. * @coupled: the struct coupled that contains the current cpu
  263. *
  264. * Returns the deepest idle state that all coupled cpus can enter
  265. */
  266. static inline int cpuidle_coupled_get_state(struct cpuidle_device *dev,
  267. struct cpuidle_coupled *coupled)
  268. {
  269. int i;
  270. int state = INT_MAX;
  271. /*
  272. * Read barrier ensures that read of requested_state is ordered after
  273. * reads of ready_count. Matches the write barriers
  274. * cpuidle_set_state_waiting.
  275. */
  276. smp_rmb();
  277. for_each_cpu(i, &coupled->coupled_cpus)
  278. if (cpu_online(i) && coupled->requested_state[i] < state)
  279. state = coupled->requested_state[i];
  280. return state;
  281. }
  282. static void cpuidle_coupled_handle_poke(void *info)
  283. {
  284. int cpu = (unsigned long)info;
  285. cpumask_set_cpu(cpu, &cpuidle_coupled_poked);
  286. cpumask_clear_cpu(cpu, &cpuidle_coupled_poke_pending);
  287. }
  288. /**
  289. * cpuidle_coupled_poke - wake up a cpu that may be waiting
  290. * @cpu: target cpu
  291. *
  292. * Ensures that the target cpu exits it's waiting idle state (if it is in it)
  293. * and will see updates to waiting_count before it re-enters it's waiting idle
  294. * state.
  295. *
  296. * If cpuidle_coupled_poked_mask is already set for the target cpu, that cpu
  297. * either has or will soon have a pending IPI that will wake it out of idle,
  298. * or it is currently processing the IPI and is not in idle.
  299. */
  300. static void cpuidle_coupled_poke(int cpu)
  301. {
  302. call_single_data_t *csd = &per_cpu(cpuidle_coupled_poke_cb, cpu);
  303. if (!cpumask_test_and_set_cpu(cpu, &cpuidle_coupled_poke_pending))
  304. smp_call_function_single_async(cpu, csd);
  305. }
  306. /**
  307. * cpuidle_coupled_poke_others - wake up all other cpus that may be waiting
  308. * @this_cpu: target cpu
  309. * @coupled: the struct coupled that contains the current cpu
  310. *
  311. * Calls cpuidle_coupled_poke on all other online cpus.
  312. */
  313. static void cpuidle_coupled_poke_others(int this_cpu,
  314. struct cpuidle_coupled *coupled)
  315. {
  316. int cpu;
  317. for_each_cpu(cpu, &coupled->coupled_cpus)
  318. if (cpu != this_cpu && cpu_online(cpu))
  319. cpuidle_coupled_poke(cpu);
  320. }
  321. /**
  322. * cpuidle_coupled_set_waiting - mark this cpu as in the wait loop
  323. * @cpu: target cpu
  324. * @coupled: the struct coupled that contains the current cpu
  325. * @next_state: the index in drv->states of the requested state for this cpu
  326. *
  327. * Updates the requested idle state for the specified cpuidle device.
  328. * Returns the number of waiting cpus.
  329. */
  330. static int cpuidle_coupled_set_waiting(int cpu,
  331. struct cpuidle_coupled *coupled, int next_state)
  332. {
  333. coupled->requested_state[cpu] = next_state;
  334. /*
  335. * The atomic_inc_return provides a write barrier to order the write
  336. * to requested_state with the later write that increments ready_count.
  337. */
  338. return atomic_inc_return(&coupled->ready_waiting_counts) & WAITING_MASK;
  339. }
  340. /**
  341. * cpuidle_coupled_set_not_waiting - mark this cpu as leaving the wait loop
  342. * @cpu: target cpu
  343. * @coupled: the struct coupled that contains the current cpu
  344. *
  345. * Removes the requested idle state for the specified cpuidle device.
  346. */
  347. static void cpuidle_coupled_set_not_waiting(int cpu,
  348. struct cpuidle_coupled *coupled)
  349. {
  350. /*
  351. * Decrementing waiting count can race with incrementing it in
  352. * cpuidle_coupled_set_waiting, but that's OK. Worst case, some
  353. * cpus will increment ready_count and then spin until they
  354. * notice that this cpu has cleared it's requested_state.
  355. */
  356. atomic_dec(&coupled->ready_waiting_counts);
  357. coupled->requested_state[cpu] = CPUIDLE_COUPLED_NOT_IDLE;
  358. }
  359. /**
  360. * cpuidle_coupled_set_done - mark this cpu as leaving the ready loop
  361. * @cpu: the current cpu
  362. * @coupled: the struct coupled that contains the current cpu
  363. *
  364. * Marks this cpu as no longer in the ready and waiting loops. Decrements
  365. * the waiting count first to prevent another cpu looping back in and seeing
  366. * this cpu as waiting just before it exits idle.
  367. */
  368. static void cpuidle_coupled_set_done(int cpu, struct cpuidle_coupled *coupled)
  369. {
  370. cpuidle_coupled_set_not_waiting(cpu, coupled);
  371. atomic_sub(MAX_WAITING_CPUS, &coupled->ready_waiting_counts);
  372. }
  373. /**
  374. * cpuidle_coupled_clear_pokes - spin until the poke interrupt is processed
  375. * @cpu: this cpu
  376. *
  377. * Turns on interrupts and spins until any outstanding poke interrupts have
  378. * been processed and the poke bit has been cleared.
  379. *
  380. * Other interrupts may also be processed while interrupts are enabled, so
  381. * need_resched() must be tested after this function returns to make sure
  382. * the interrupt didn't schedule work that should take the cpu out of idle.
  383. *
  384. * Returns 0 if no poke was pending, 1 if a poke was cleared.
  385. */
  386. static int cpuidle_coupled_clear_pokes(int cpu)
  387. {
  388. if (!cpumask_test_cpu(cpu, &cpuidle_coupled_poke_pending))
  389. return 0;
  390. local_irq_enable();
  391. while (cpumask_test_cpu(cpu, &cpuidle_coupled_poke_pending))
  392. cpu_relax();
  393. local_irq_disable();
  394. return 1;
  395. }
  396. static bool cpuidle_coupled_any_pokes_pending(struct cpuidle_coupled *coupled)
  397. {
  398. cpumask_t cpus;
  399. int ret;
  400. cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus);
  401. ret = cpumask_and(&cpus, &cpuidle_coupled_poke_pending, &cpus);
  402. return ret;
  403. }
  404. /**
  405. * cpuidle_enter_state_coupled - attempt to enter a state with coupled cpus
  406. * @dev: struct cpuidle_device for the current cpu
  407. * @drv: struct cpuidle_driver for the platform
  408. * @next_state: index of the requested state in drv->states
  409. *
  410. * Coordinate with coupled cpus to enter the target state. This is a two
  411. * stage process. In the first stage, the cpus are operating independently,
  412. * and may call into cpuidle_enter_state_coupled at completely different times.
  413. * To save as much power as possible, the first cpus to call this function will
  414. * go to an intermediate state (the cpuidle_device's safe state), and wait for
  415. * all the other cpus to call this function. Once all coupled cpus are idle,
  416. * the second stage will start. Each coupled cpu will spin until all cpus have
  417. * guaranteed that they will call the target_state.
  418. *
  419. * This function must be called with interrupts disabled. It may enable
  420. * interrupts while preparing for idle, and it will always return with
  421. * interrupts enabled.
  422. */
  423. int cpuidle_enter_state_coupled(struct cpuidle_device *dev,
  424. struct cpuidle_driver *drv, int next_state)
  425. {
  426. int entered_state = -1;
  427. struct cpuidle_coupled *coupled = dev->coupled;
  428. int w;
  429. if (!coupled)
  430. return -EINVAL;
  431. while (coupled->prevent) {
  432. cpuidle_coupled_clear_pokes(dev->cpu);
  433. if (need_resched()) {
  434. local_irq_enable();
  435. return entered_state;
  436. }
  437. entered_state = cpuidle_enter_state(dev, drv,
  438. drv->safe_state_index);
  439. local_irq_disable();
  440. }
  441. /* Read barrier ensures online_count is read after prevent is cleared */
  442. smp_rmb();
  443. reset:
  444. cpumask_clear_cpu(dev->cpu, &cpuidle_coupled_poked);
  445. w = cpuidle_coupled_set_waiting(dev->cpu, coupled, next_state);
  446. /*
  447. * If this is the last cpu to enter the waiting state, poke
  448. * all the other cpus out of their waiting state so they can
  449. * enter a deeper state. This can race with one of the cpus
  450. * exiting the waiting state due to an interrupt and
  451. * decrementing waiting_count, see comment below.
  452. */
  453. if (w == coupled->online_count) {
  454. cpumask_set_cpu(dev->cpu, &cpuidle_coupled_poked);
  455. cpuidle_coupled_poke_others(dev->cpu, coupled);
  456. }
  457. retry:
  458. /*
  459. * Wait for all coupled cpus to be idle, using the deepest state
  460. * allowed for a single cpu. If this was not the poking cpu, wait
  461. * for at least one poke before leaving to avoid a race where
  462. * two cpus could arrive at the waiting loop at the same time,
  463. * but the first of the two to arrive could skip the loop without
  464. * processing the pokes from the last to arrive.
  465. */
  466. while (!cpuidle_coupled_cpus_waiting(coupled) ||
  467. !cpumask_test_cpu(dev->cpu, &cpuidle_coupled_poked)) {
  468. if (cpuidle_coupled_clear_pokes(dev->cpu))
  469. continue;
  470. if (need_resched()) {
  471. cpuidle_coupled_set_not_waiting(dev->cpu, coupled);
  472. goto out;
  473. }
  474. if (coupled->prevent) {
  475. cpuidle_coupled_set_not_waiting(dev->cpu, coupled);
  476. goto out;
  477. }
  478. entered_state = cpuidle_enter_state(dev, drv,
  479. drv->safe_state_index);
  480. local_irq_disable();
  481. }
  482. cpuidle_coupled_clear_pokes(dev->cpu);
  483. if (need_resched()) {
  484. cpuidle_coupled_set_not_waiting(dev->cpu, coupled);
  485. goto out;
  486. }
  487. /*
  488. * Make sure final poke status for this cpu is visible before setting
  489. * cpu as ready.
  490. */
  491. smp_wmb();
  492. /*
  493. * All coupled cpus are probably idle. There is a small chance that
  494. * one of the other cpus just became active. Increment the ready count,
  495. * and spin until all coupled cpus have incremented the counter. Once a
  496. * cpu has incremented the ready counter, it cannot abort idle and must
  497. * spin until either all cpus have incremented the ready counter, or
  498. * another cpu leaves idle and decrements the waiting counter.
  499. */
  500. cpuidle_coupled_set_ready(coupled);
  501. while (!cpuidle_coupled_cpus_ready(coupled)) {
  502. /* Check if any other cpus bailed out of idle. */
  503. if (!cpuidle_coupled_cpus_waiting(coupled))
  504. if (!cpuidle_coupled_set_not_ready(coupled))
  505. goto retry;
  506. cpu_relax();
  507. }
  508. /*
  509. * Make sure read of all cpus ready is done before reading pending pokes
  510. */
  511. smp_rmb();
  512. /*
  513. * There is a small chance that a cpu left and reentered idle after this
  514. * cpu saw that all cpus were waiting. The cpu that reentered idle will
  515. * have sent this cpu a poke, which will still be pending after the
  516. * ready loop. The pending interrupt may be lost by the interrupt
  517. * controller when entering the deep idle state. It's not possible to
  518. * clear a pending interrupt without turning interrupts on and handling
  519. * it, and it's too late to turn on interrupts here, so reset the
  520. * coupled idle state of all cpus and retry.
  521. */
  522. if (cpuidle_coupled_any_pokes_pending(coupled)) {
  523. cpuidle_coupled_set_done(dev->cpu, coupled);
  524. /* Wait for all cpus to see the pending pokes */
  525. cpuidle_coupled_parallel_barrier(dev, &coupled->abort_barrier);
  526. goto reset;
  527. }
  528. /* all cpus have acked the coupled state */
  529. next_state = cpuidle_coupled_get_state(dev, coupled);
  530. entered_state = cpuidle_enter_state(dev, drv, next_state);
  531. cpuidle_coupled_set_done(dev->cpu, coupled);
  532. out:
  533. /*
  534. * Normal cpuidle states are expected to return with irqs enabled.
  535. * That leads to an inefficiency where a cpu receiving an interrupt
  536. * that brings it out of idle will process that interrupt before
  537. * exiting the idle enter function and decrementing ready_count. All
  538. * other cpus will need to spin waiting for the cpu that is processing
  539. * the interrupt. If the driver returns with interrupts disabled,
  540. * all other cpus will loop back into the safe idle state instead of
  541. * spinning, saving power.
  542. *
  543. * Calling local_irq_enable here allows coupled states to return with
  544. * interrupts disabled, but won't cause problems for drivers that
  545. * exit with interrupts enabled.
  546. */
  547. local_irq_enable();
  548. /*
  549. * Wait until all coupled cpus have exited idle. There is no risk that
  550. * a cpu exits and re-enters the ready state because this cpu has
  551. * already decremented its waiting_count.
  552. */
  553. while (!cpuidle_coupled_no_cpus_ready(coupled))
  554. cpu_relax();
  555. return entered_state;
  556. }
  557. static void cpuidle_coupled_update_online_cpus(struct cpuidle_coupled *coupled)
  558. {
  559. cpumask_t cpus;
  560. cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus);
  561. coupled->online_count = cpumask_weight(&cpus);
  562. }
  563. /**
  564. * cpuidle_coupled_register_device - register a coupled cpuidle device
  565. * @dev: struct cpuidle_device for the current cpu
  566. *
  567. * Called from cpuidle_register_device to handle coupled idle init. Finds the
  568. * cpuidle_coupled struct for this set of coupled cpus, or creates one if none
  569. * exists yet.
  570. */
  571. int cpuidle_coupled_register_device(struct cpuidle_device *dev)
  572. {
  573. int cpu;
  574. struct cpuidle_device *other_dev;
  575. call_single_data_t *csd;
  576. struct cpuidle_coupled *coupled;
  577. if (cpumask_empty(&dev->coupled_cpus))
  578. return 0;
  579. for_each_cpu(cpu, &dev->coupled_cpus) {
  580. other_dev = per_cpu(cpuidle_devices, cpu);
  581. if (other_dev && other_dev->coupled) {
  582. coupled = other_dev->coupled;
  583. goto have_coupled;
  584. }
  585. }
  586. /* No existing coupled info found, create a new one */
  587. coupled = kzalloc(sizeof(struct cpuidle_coupled), GFP_KERNEL);
  588. if (!coupled)
  589. return -ENOMEM;
  590. coupled->coupled_cpus = dev->coupled_cpus;
  591. have_coupled:
  592. dev->coupled = coupled;
  593. if (WARN_ON(!cpumask_equal(&dev->coupled_cpus, &coupled->coupled_cpus)))
  594. coupled->prevent++;
  595. cpuidle_coupled_update_online_cpus(coupled);
  596. coupled->refcnt++;
  597. csd = &per_cpu(cpuidle_coupled_poke_cb, dev->cpu);
  598. csd->func = cpuidle_coupled_handle_poke;
  599. csd->info = (void *)(unsigned long)dev->cpu;
  600. return 0;
  601. }
  602. /**
  603. * cpuidle_coupled_unregister_device - unregister a coupled cpuidle device
  604. * @dev: struct cpuidle_device for the current cpu
  605. *
  606. * Called from cpuidle_unregister_device to tear down coupled idle. Removes the
  607. * cpu from the coupled idle set, and frees the cpuidle_coupled_info struct if
  608. * this was the last cpu in the set.
  609. */
  610. void cpuidle_coupled_unregister_device(struct cpuidle_device *dev)
  611. {
  612. struct cpuidle_coupled *coupled = dev->coupled;
  613. if (cpumask_empty(&dev->coupled_cpus))
  614. return;
  615. if (--coupled->refcnt)
  616. kfree(coupled);
  617. dev->coupled = NULL;
  618. }
  619. /**
  620. * cpuidle_coupled_prevent_idle - prevent cpus from entering a coupled state
  621. * @coupled: the struct coupled that contains the cpu that is changing state
  622. *
  623. * Disables coupled cpuidle on a coupled set of cpus. Used to ensure that
  624. * cpu_online_mask doesn't change while cpus are coordinating coupled idle.
  625. */
  626. static void cpuidle_coupled_prevent_idle(struct cpuidle_coupled *coupled)
  627. {
  628. int cpu = get_cpu();
  629. /* Force all cpus out of the waiting loop. */
  630. coupled->prevent++;
  631. cpuidle_coupled_poke_others(cpu, coupled);
  632. put_cpu();
  633. while (!cpuidle_coupled_no_cpus_waiting(coupled))
  634. cpu_relax();
  635. }
  636. /**
  637. * cpuidle_coupled_allow_idle - allows cpus to enter a coupled state
  638. * @coupled: the struct coupled that contains the cpu that is changing state
  639. *
  640. * Enables coupled cpuidle on a coupled set of cpus. Used to ensure that
  641. * cpu_online_mask doesn't change while cpus are coordinating coupled idle.
  642. */
  643. static void cpuidle_coupled_allow_idle(struct cpuidle_coupled *coupled)
  644. {
  645. int cpu = get_cpu();
  646. /*
  647. * Write barrier ensures readers see the new online_count when they
  648. * see prevent == 0.
  649. */
  650. smp_wmb();
  651. coupled->prevent--;
  652. /* Force cpus out of the prevent loop. */
  653. cpuidle_coupled_poke_others(cpu, coupled);
  654. put_cpu();
  655. }
  656. static int coupled_cpu_online(unsigned int cpu)
  657. {
  658. struct cpuidle_device *dev;
  659. mutex_lock(&cpuidle_lock);
  660. dev = per_cpu(cpuidle_devices, cpu);
  661. if (dev && dev->coupled) {
  662. cpuidle_coupled_update_online_cpus(dev->coupled);
  663. cpuidle_coupled_allow_idle(dev->coupled);
  664. }
  665. mutex_unlock(&cpuidle_lock);
  666. return 0;
  667. }
  668. static int coupled_cpu_up_prepare(unsigned int cpu)
  669. {
  670. struct cpuidle_device *dev;
  671. mutex_lock(&cpuidle_lock);
  672. dev = per_cpu(cpuidle_devices, cpu);
  673. if (dev && dev->coupled)
  674. cpuidle_coupled_prevent_idle(dev->coupled);
  675. mutex_unlock(&cpuidle_lock);
  676. return 0;
  677. }
  678. static int __init cpuidle_coupled_init(void)
  679. {
  680. int ret;
  681. ret = cpuhp_setup_state_nocalls(CPUHP_CPUIDLE_COUPLED_PREPARE,
  682. "cpuidle/coupled:prepare",
  683. coupled_cpu_up_prepare,
  684. coupled_cpu_online);
  685. if (ret)
  686. return ret;
  687. ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
  688. "cpuidle/coupled:online",
  689. coupled_cpu_online,
  690. coupled_cpu_up_prepare);
  691. if (ret < 0)
  692. cpuhp_remove_state_nocalls(CPUHP_CPUIDLE_COUPLED_PREPARE);
  693. return ret;
  694. }
  695. core_initcall(cpuidle_coupled_init);