rtmutex.c 51 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * RT-Mutexes: simple blocking mutual exclusion locks with PI support
  4. *
  5. * started by Ingo Molnar and Thomas Gleixner.
  6. *
  7. * Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
  8. * Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner <tglx@timesys.com>
  9. * Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt
  10. * Copyright (C) 2006 Esben Nielsen
  11. *
  12. * See Documentation/locking/rt-mutex-design.rst for details.
  13. */
  14. #include <linux/spinlock.h>
  15. #include <linux/export.h>
  16. #include <linux/sched/signal.h>
  17. #include <linux/sched/rt.h>
  18. #include <linux/sched/deadline.h>
  19. #include <linux/sched/wake_q.h>
  20. #include <linux/sched/debug.h>
  21. #include <linux/timer.h>
  22. #include <trace/hooks/dtask.h>
  23. #include "rtmutex_common.h"
  24. /*
  25. * lock->owner state tracking:
  26. *
  27. * lock->owner holds the task_struct pointer of the owner. Bit 0
  28. * is used to keep track of the "lock has waiters" state.
  29. *
  30. * owner bit0
  31. * NULL 0 lock is free (fast acquire possible)
  32. * NULL 1 lock is free and has waiters and the top waiter
  33. * is going to take the lock*
  34. * taskpointer 0 lock is held (fast release possible)
  35. * taskpointer 1 lock is held and has waiters**
  36. *
  37. * The fast atomic compare exchange based acquire and release is only
  38. * possible when bit 0 of lock->owner is 0.
  39. *
  40. * (*) It also can be a transitional state when grabbing the lock
  41. * with ->wait_lock is held. To prevent any fast path cmpxchg to the lock,
  42. * we need to set the bit0 before looking at the lock, and the owner may be
  43. * NULL in this small time, hence this can be a transitional state.
  44. *
  45. * (**) There is a small time when bit 0 is set but there are no
  46. * waiters. This can happen when grabbing the lock in the slow path.
  47. * To prevent a cmpxchg of the owner releasing the lock, we need to
  48. * set this bit before looking at the lock.
  49. */
  50. static void
  51. rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner)
  52. {
  53. unsigned long val = (unsigned long)owner;
  54. if (rt_mutex_has_waiters(lock))
  55. val |= RT_MUTEX_HAS_WAITERS;
  56. WRITE_ONCE(lock->owner, (struct task_struct *)val);
  57. }
  58. static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)
  59. {
  60. lock->owner = (struct task_struct *)
  61. ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
  62. }
  63. static void fixup_rt_mutex_waiters(struct rt_mutex *lock)
  64. {
  65. unsigned long owner, *p = (unsigned long *) &lock->owner;
  66. if (rt_mutex_has_waiters(lock))
  67. return;
  68. /*
  69. * The rbtree has no waiters enqueued, now make sure that the
  70. * lock->owner still has the waiters bit set, otherwise the
  71. * following can happen:
  72. *
  73. * CPU 0 CPU 1 CPU2
  74. * l->owner=T1
  75. * rt_mutex_lock(l)
  76. * lock(l->lock)
  77. * l->owner = T1 | HAS_WAITERS;
  78. * enqueue(T2)
  79. * boost()
  80. * unlock(l->lock)
  81. * block()
  82. *
  83. * rt_mutex_lock(l)
  84. * lock(l->lock)
  85. * l->owner = T1 | HAS_WAITERS;
  86. * enqueue(T3)
  87. * boost()
  88. * unlock(l->lock)
  89. * block()
  90. * signal(->T2) signal(->T3)
  91. * lock(l->lock)
  92. * dequeue(T2)
  93. * deboost()
  94. * unlock(l->lock)
  95. * lock(l->lock)
  96. * dequeue(T3)
  97. * ==> wait list is empty
  98. * deboost()
  99. * unlock(l->lock)
  100. * lock(l->lock)
  101. * fixup_rt_mutex_waiters()
  102. * if (wait_list_empty(l) {
  103. * l->owner = owner
  104. * owner = l->owner & ~HAS_WAITERS;
  105. * ==> l->owner = T1
  106. * }
  107. * lock(l->lock)
  108. * rt_mutex_unlock(l) fixup_rt_mutex_waiters()
  109. * if (wait_list_empty(l) {
  110. * owner = l->owner & ~HAS_WAITERS;
  111. * cmpxchg(l->owner, T1, NULL)
  112. * ===> Success (l->owner = NULL)
  113. *
  114. * l->owner = owner
  115. * ==> l->owner = T1
  116. * }
  117. *
  118. * With the check for the waiter bit in place T3 on CPU2 will not
  119. * overwrite. All tasks fiddling with the waiters bit are
  120. * serialized by l->lock, so nothing else can modify the waiters
  121. * bit. If the bit is set then nothing can change l->owner either
  122. * so the simple RMW is safe. The cmpxchg() will simply fail if it
  123. * happens in the middle of the RMW because the waiters bit is
  124. * still set.
  125. */
  126. owner = READ_ONCE(*p);
  127. if (owner & RT_MUTEX_HAS_WAITERS)
  128. WRITE_ONCE(*p, owner & ~RT_MUTEX_HAS_WAITERS);
  129. }
  130. /*
  131. * We can speed up the acquire/release, if there's no debugging state to be
  132. * set up.
  133. */
  134. #ifndef CONFIG_DEBUG_RT_MUTEXES
  135. # define rt_mutex_cmpxchg_acquire(l,c,n) (cmpxchg_acquire(&l->owner, c, n) == c)
  136. # define rt_mutex_cmpxchg_release(l,c,n) (cmpxchg_release(&l->owner, c, n) == c)
  137. /*
  138. * Callers must hold the ->wait_lock -- which is the whole purpose as we force
  139. * all future threads that attempt to [Rmw] the lock to the slowpath. As such
  140. * relaxed semantics suffice.
  141. */
  142. static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
  143. {
  144. unsigned long owner, *p = (unsigned long *) &lock->owner;
  145. do {
  146. owner = *p;
  147. } while (cmpxchg_relaxed(p, owner,
  148. owner | RT_MUTEX_HAS_WAITERS) != owner);
  149. }
  150. /*
  151. * Safe fastpath aware unlock:
  152. * 1) Clear the waiters bit
  153. * 2) Drop lock->wait_lock
  154. * 3) Try to unlock the lock with cmpxchg
  155. */
  156. static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
  157. unsigned long flags)
  158. __releases(lock->wait_lock)
  159. {
  160. struct task_struct *owner = rt_mutex_owner(lock);
  161. clear_rt_mutex_waiters(lock);
  162. raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  163. /*
  164. * If a new waiter comes in between the unlock and the cmpxchg
  165. * we have two situations:
  166. *
  167. * unlock(wait_lock);
  168. * lock(wait_lock);
  169. * cmpxchg(p, owner, 0) == owner
  170. * mark_rt_mutex_waiters(lock);
  171. * acquire(lock);
  172. * or:
  173. *
  174. * unlock(wait_lock);
  175. * lock(wait_lock);
  176. * mark_rt_mutex_waiters(lock);
  177. *
  178. * cmpxchg(p, owner, 0) != owner
  179. * enqueue_waiter();
  180. * unlock(wait_lock);
  181. * lock(wait_lock);
  182. * wake waiter();
  183. * unlock(wait_lock);
  184. * lock(wait_lock);
  185. * acquire(lock);
  186. */
  187. return rt_mutex_cmpxchg_release(lock, owner, NULL);
  188. }
  189. #else
  190. # define rt_mutex_cmpxchg_acquire(l,c,n) (0)
  191. # define rt_mutex_cmpxchg_release(l,c,n) (0)
  192. static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
  193. {
  194. lock->owner = (struct task_struct *)
  195. ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);
  196. }
  197. /*
  198. * Simple slow path only version: lock->owner is protected by lock->wait_lock.
  199. */
  200. static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
  201. unsigned long flags)
  202. __releases(lock->wait_lock)
  203. {
  204. lock->owner = NULL;
  205. raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  206. return true;
  207. }
  208. #endif
  209. /*
  210. * Only use with rt_mutex_waiter_{less,equal}()
  211. */
  212. #define task_to_waiter(p) \
  213. &(struct rt_mutex_waiter){ .prio = (p)->prio, .deadline = (p)->dl.deadline }
  214. static inline int
  215. rt_mutex_waiter_less(struct rt_mutex_waiter *left,
  216. struct rt_mutex_waiter *right)
  217. {
  218. if (left->prio < right->prio)
  219. return 1;
  220. /*
  221. * If both waiters have dl_prio(), we check the deadlines of the
  222. * associated tasks.
  223. * If left waiter has a dl_prio(), and we didn't return 1 above,
  224. * then right waiter has a dl_prio() too.
  225. */
  226. if (dl_prio(left->prio))
  227. return dl_time_before(left->deadline, right->deadline);
  228. return 0;
  229. }
  230. static inline int
  231. rt_mutex_waiter_equal(struct rt_mutex_waiter *left,
  232. struct rt_mutex_waiter *right)
  233. {
  234. if (left->prio != right->prio)
  235. return 0;
  236. /*
  237. * If both waiters have dl_prio(), we check the deadlines of the
  238. * associated tasks.
  239. * If left waiter has a dl_prio(), and we didn't return 0 above,
  240. * then right waiter has a dl_prio() too.
  241. */
  242. if (dl_prio(left->prio))
  243. return left->deadline == right->deadline;
  244. return 1;
  245. }
  246. static void
  247. rt_mutex_enqueue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)
  248. {
  249. struct rb_node **link = &lock->waiters.rb_root.rb_node;
  250. struct rb_node *parent = NULL;
  251. struct rt_mutex_waiter *entry;
  252. bool leftmost = true;
  253. while (*link) {
  254. parent = *link;
  255. entry = rb_entry(parent, struct rt_mutex_waiter, tree_entry);
  256. if (rt_mutex_waiter_less(waiter, entry)) {
  257. link = &parent->rb_left;
  258. } else {
  259. link = &parent->rb_right;
  260. leftmost = false;
  261. }
  262. }
  263. rb_link_node(&waiter->tree_entry, parent, link);
  264. rb_insert_color_cached(&waiter->tree_entry, &lock->waiters, leftmost);
  265. }
  266. static void
  267. rt_mutex_dequeue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)
  268. {
  269. if (RB_EMPTY_NODE(&waiter->tree_entry))
  270. return;
  271. rb_erase_cached(&waiter->tree_entry, &lock->waiters);
  272. RB_CLEAR_NODE(&waiter->tree_entry);
  273. }
  274. static void
  275. rt_mutex_enqueue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)
  276. {
  277. struct rb_node **link = &task->pi_waiters.rb_root.rb_node;
  278. struct rb_node *parent = NULL;
  279. struct rt_mutex_waiter *entry;
  280. bool leftmost = true;
  281. while (*link) {
  282. parent = *link;
  283. entry = rb_entry(parent, struct rt_mutex_waiter, pi_tree_entry);
  284. if (rt_mutex_waiter_less(waiter, entry)) {
  285. link = &parent->rb_left;
  286. } else {
  287. link = &parent->rb_right;
  288. leftmost = false;
  289. }
  290. }
  291. rb_link_node(&waiter->pi_tree_entry, parent, link);
  292. rb_insert_color_cached(&waiter->pi_tree_entry, &task->pi_waiters, leftmost);
  293. }
  294. static void
  295. rt_mutex_dequeue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)
  296. {
  297. if (RB_EMPTY_NODE(&waiter->pi_tree_entry))
  298. return;
  299. rb_erase_cached(&waiter->pi_tree_entry, &task->pi_waiters);
  300. RB_CLEAR_NODE(&waiter->pi_tree_entry);
  301. }
  302. static void rt_mutex_adjust_prio(struct task_struct *p)
  303. {
  304. struct task_struct *pi_task = NULL;
  305. lockdep_assert_held(&p->pi_lock);
  306. if (task_has_pi_waiters(p))
  307. pi_task = task_top_pi_waiter(p)->task;
  308. rt_mutex_setprio(p, pi_task);
  309. }
  310. /*
  311. * Deadlock detection is conditional:
  312. *
  313. * If CONFIG_DEBUG_RT_MUTEXES=n, deadlock detection is only conducted
  314. * if the detect argument is == RT_MUTEX_FULL_CHAINWALK.
  315. *
  316. * If CONFIG_DEBUG_RT_MUTEXES=y, deadlock detection is always
  317. * conducted independent of the detect argument.
  318. *
  319. * If the waiter argument is NULL this indicates the deboost path and
  320. * deadlock detection is disabled independent of the detect argument
  321. * and the config settings.
  322. */
  323. static bool rt_mutex_cond_detect_deadlock(struct rt_mutex_waiter *waiter,
  324. enum rtmutex_chainwalk chwalk)
  325. {
  326. /*
  327. * This is just a wrapper function for the following call,
  328. * because debug_rt_mutex_detect_deadlock() smells like a magic
  329. * debug feature and I wanted to keep the cond function in the
  330. * main source file along with the comments instead of having
  331. * two of the same in the headers.
  332. */
  333. return debug_rt_mutex_detect_deadlock(waiter, chwalk);
  334. }
  335. /*
  336. * Max number of times we'll walk the boosting chain:
  337. */
  338. int max_lock_depth = 1024;
  339. static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p)
  340. {
  341. return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL;
  342. }
  343. /*
  344. * Adjust the priority chain. Also used for deadlock detection.
  345. * Decreases task's usage by one - may thus free the task.
  346. *
  347. * @task: the task owning the mutex (owner) for which a chain walk is
  348. * probably needed
  349. * @chwalk: do we have to carry out deadlock detection?
  350. * @orig_lock: the mutex (can be NULL if we are walking the chain to recheck
  351. * things for a task that has just got its priority adjusted, and
  352. * is waiting on a mutex)
  353. * @next_lock: the mutex on which the owner of @orig_lock was blocked before
  354. * we dropped its pi_lock. Is never dereferenced, only used for
  355. * comparison to detect lock chain changes.
  356. * @orig_waiter: rt_mutex_waiter struct for the task that has just donated
  357. * its priority to the mutex owner (can be NULL in the case
  358. * depicted above or if the top waiter is gone away and we are
  359. * actually deboosting the owner)
  360. * @top_task: the current top waiter
  361. *
  362. * Returns 0 or -EDEADLK.
  363. *
  364. * Chain walk basics and protection scope
  365. *
  366. * [R] refcount on task
  367. * [P] task->pi_lock held
  368. * [L] rtmutex->wait_lock held
  369. *
  370. * Step Description Protected by
  371. * function arguments:
  372. * @task [R]
  373. * @orig_lock if != NULL @top_task is blocked on it
  374. * @next_lock Unprotected. Cannot be
  375. * dereferenced. Only used for
  376. * comparison.
  377. * @orig_waiter if != NULL @top_task is blocked on it
  378. * @top_task current, or in case of proxy
  379. * locking protected by calling
  380. * code
  381. * again:
  382. * loop_sanity_check();
  383. * retry:
  384. * [1] lock(task->pi_lock); [R] acquire [P]
  385. * [2] waiter = task->pi_blocked_on; [P]
  386. * [3] check_exit_conditions_1(); [P]
  387. * [4] lock = waiter->lock; [P]
  388. * [5] if (!try_lock(lock->wait_lock)) { [P] try to acquire [L]
  389. * unlock(task->pi_lock); release [P]
  390. * goto retry;
  391. * }
  392. * [6] check_exit_conditions_2(); [P] + [L]
  393. * [7] requeue_lock_waiter(lock, waiter); [P] + [L]
  394. * [8] unlock(task->pi_lock); release [P]
  395. * put_task_struct(task); release [R]
  396. * [9] check_exit_conditions_3(); [L]
  397. * [10] task = owner(lock); [L]
  398. * get_task_struct(task); [L] acquire [R]
  399. * lock(task->pi_lock); [L] acquire [P]
  400. * [11] requeue_pi_waiter(tsk, waiters(lock));[P] + [L]
  401. * [12] check_exit_conditions_4(); [P] + [L]
  402. * [13] unlock(task->pi_lock); release [P]
  403. * unlock(lock->wait_lock); release [L]
  404. * goto again;
  405. */
  406. static int rt_mutex_adjust_prio_chain(struct task_struct *task,
  407. enum rtmutex_chainwalk chwalk,
  408. struct rt_mutex *orig_lock,
  409. struct rt_mutex *next_lock,
  410. struct rt_mutex_waiter *orig_waiter,
  411. struct task_struct *top_task)
  412. {
  413. struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;
  414. struct rt_mutex_waiter *prerequeue_top_waiter;
  415. int ret = 0, depth = 0;
  416. struct rt_mutex *lock;
  417. bool detect_deadlock;
  418. bool requeue = true;
  419. detect_deadlock = rt_mutex_cond_detect_deadlock(orig_waiter, chwalk);
  420. /*
  421. * The (de)boosting is a step by step approach with a lot of
  422. * pitfalls. We want this to be preemptible and we want hold a
  423. * maximum of two locks per step. So we have to check
  424. * carefully whether things change under us.
  425. */
  426. again:
  427. /*
  428. * We limit the lock chain length for each invocation.
  429. */
  430. if (++depth > max_lock_depth) {
  431. static int prev_max;
  432. /*
  433. * Print this only once. If the admin changes the limit,
  434. * print a new message when reaching the limit again.
  435. */
  436. if (prev_max != max_lock_depth) {
  437. prev_max = max_lock_depth;
  438. printk(KERN_WARNING "Maximum lock depth %d reached "
  439. "task: %s (%d)\n", max_lock_depth,
  440. top_task->comm, task_pid_nr(top_task));
  441. }
  442. put_task_struct(task);
  443. return -EDEADLK;
  444. }
  445. /*
  446. * We are fully preemptible here and only hold the refcount on
  447. * @task. So everything can have changed under us since the
  448. * caller or our own code below (goto retry/again) dropped all
  449. * locks.
  450. */
  451. retry:
  452. /*
  453. * [1] Task cannot go away as we did a get_task() before !
  454. */
  455. raw_spin_lock_irq(&task->pi_lock);
  456. /*
  457. * [2] Get the waiter on which @task is blocked on.
  458. */
  459. waiter = task->pi_blocked_on;
  460. /*
  461. * [3] check_exit_conditions_1() protected by task->pi_lock.
  462. */
  463. /*
  464. * Check whether the end of the boosting chain has been
  465. * reached or the state of the chain has changed while we
  466. * dropped the locks.
  467. */
  468. if (!waiter)
  469. goto out_unlock_pi;
  470. /*
  471. * Check the orig_waiter state. After we dropped the locks,
  472. * the previous owner of the lock might have released the lock.
  473. */
  474. if (orig_waiter && !rt_mutex_owner(orig_lock))
  475. goto out_unlock_pi;
  476. /*
  477. * We dropped all locks after taking a refcount on @task, so
  478. * the task might have moved on in the lock chain or even left
  479. * the chain completely and blocks now on an unrelated lock or
  480. * on @orig_lock.
  481. *
  482. * We stored the lock on which @task was blocked in @next_lock,
  483. * so we can detect the chain change.
  484. */
  485. if (next_lock != waiter->lock)
  486. goto out_unlock_pi;
  487. /*
  488. * Drop out, when the task has no waiters. Note,
  489. * top_waiter can be NULL, when we are in the deboosting
  490. * mode!
  491. */
  492. if (top_waiter) {
  493. if (!task_has_pi_waiters(task))
  494. goto out_unlock_pi;
  495. /*
  496. * If deadlock detection is off, we stop here if we
  497. * are not the top pi waiter of the task. If deadlock
  498. * detection is enabled we continue, but stop the
  499. * requeueing in the chain walk.
  500. */
  501. if (top_waiter != task_top_pi_waiter(task)) {
  502. if (!detect_deadlock)
  503. goto out_unlock_pi;
  504. else
  505. requeue = false;
  506. }
  507. }
  508. /*
  509. * If the waiter priority is the same as the task priority
  510. * then there is no further priority adjustment necessary. If
  511. * deadlock detection is off, we stop the chain walk. If its
  512. * enabled we continue, but stop the requeueing in the chain
  513. * walk.
  514. */
  515. if (rt_mutex_waiter_equal(waiter, task_to_waiter(task))) {
  516. if (!detect_deadlock)
  517. goto out_unlock_pi;
  518. else
  519. requeue = false;
  520. }
  521. /*
  522. * [4] Get the next lock
  523. */
  524. lock = waiter->lock;
  525. /*
  526. * [5] We need to trylock here as we are holding task->pi_lock,
  527. * which is the reverse lock order versus the other rtmutex
  528. * operations.
  529. */
  530. if (!raw_spin_trylock(&lock->wait_lock)) {
  531. raw_spin_unlock_irq(&task->pi_lock);
  532. cpu_relax();
  533. goto retry;
  534. }
  535. /*
  536. * [6] check_exit_conditions_2() protected by task->pi_lock and
  537. * lock->wait_lock.
  538. *
  539. * Deadlock detection. If the lock is the same as the original
  540. * lock which caused us to walk the lock chain or if the
  541. * current lock is owned by the task which initiated the chain
  542. * walk, we detected a deadlock.
  543. */
  544. if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {
  545. debug_rt_mutex_deadlock(chwalk, orig_waiter, lock);
  546. raw_spin_unlock(&lock->wait_lock);
  547. ret = -EDEADLK;
  548. goto out_unlock_pi;
  549. }
  550. /*
  551. * If we just follow the lock chain for deadlock detection, no
  552. * need to do all the requeue operations. To avoid a truckload
  553. * of conditionals around the various places below, just do the
  554. * minimum chain walk checks.
  555. */
  556. if (!requeue) {
  557. /*
  558. * No requeue[7] here. Just release @task [8]
  559. */
  560. raw_spin_unlock(&task->pi_lock);
  561. put_task_struct(task);
  562. /*
  563. * [9] check_exit_conditions_3 protected by lock->wait_lock.
  564. * If there is no owner of the lock, end of chain.
  565. */
  566. if (!rt_mutex_owner(lock)) {
  567. raw_spin_unlock_irq(&lock->wait_lock);
  568. return 0;
  569. }
  570. /* [10] Grab the next task, i.e. owner of @lock */
  571. task = get_task_struct(rt_mutex_owner(lock));
  572. raw_spin_lock(&task->pi_lock);
  573. /*
  574. * No requeue [11] here. We just do deadlock detection.
  575. *
  576. * [12] Store whether owner is blocked
  577. * itself. Decision is made after dropping the locks
  578. */
  579. next_lock = task_blocked_on_lock(task);
  580. /*
  581. * Get the top waiter for the next iteration
  582. */
  583. top_waiter = rt_mutex_top_waiter(lock);
  584. /* [13] Drop locks */
  585. raw_spin_unlock(&task->pi_lock);
  586. raw_spin_unlock_irq(&lock->wait_lock);
  587. /* If owner is not blocked, end of chain. */
  588. if (!next_lock)
  589. goto out_put_task;
  590. goto again;
  591. }
  592. /*
  593. * Store the current top waiter before doing the requeue
  594. * operation on @lock. We need it for the boost/deboost
  595. * decision below.
  596. */
  597. prerequeue_top_waiter = rt_mutex_top_waiter(lock);
  598. /* [7] Requeue the waiter in the lock waiter tree. */
  599. rt_mutex_dequeue(lock, waiter);
  600. /*
  601. * Update the waiter prio fields now that we're dequeued.
  602. *
  603. * These values can have changed through either:
  604. *
  605. * sys_sched_set_scheduler() / sys_sched_setattr()
  606. *
  607. * or
  608. *
  609. * DL CBS enforcement advancing the effective deadline.
  610. *
  611. * Even though pi_waiters also uses these fields, and that tree is only
  612. * updated in [11], we can do this here, since we hold [L], which
  613. * serializes all pi_waiters access and rb_erase() does not care about
  614. * the values of the node being removed.
  615. */
  616. waiter->prio = task->prio;
  617. waiter->deadline = task->dl.deadline;
  618. rt_mutex_enqueue(lock, waiter);
  619. /* [8] Release the task */
  620. raw_spin_unlock(&task->pi_lock);
  621. put_task_struct(task);
  622. /*
  623. * [9] check_exit_conditions_3 protected by lock->wait_lock.
  624. *
  625. * We must abort the chain walk if there is no lock owner even
  626. * in the dead lock detection case, as we have nothing to
  627. * follow here. This is the end of the chain we are walking.
  628. */
  629. if (!rt_mutex_owner(lock)) {
  630. /*
  631. * If the requeue [7] above changed the top waiter,
  632. * then we need to wake the new top waiter up to try
  633. * to get the lock.
  634. */
  635. if (prerequeue_top_waiter != rt_mutex_top_waiter(lock))
  636. wake_up_process(rt_mutex_top_waiter(lock)->task);
  637. raw_spin_unlock_irq(&lock->wait_lock);
  638. return 0;
  639. }
  640. /* [10] Grab the next task, i.e. the owner of @lock */
  641. task = get_task_struct(rt_mutex_owner(lock));
  642. raw_spin_lock(&task->pi_lock);
  643. /* [11] requeue the pi waiters if necessary */
  644. if (waiter == rt_mutex_top_waiter(lock)) {
  645. /*
  646. * The waiter became the new top (highest priority)
  647. * waiter on the lock. Replace the previous top waiter
  648. * in the owner tasks pi waiters tree with this waiter
  649. * and adjust the priority of the owner.
  650. */
  651. rt_mutex_dequeue_pi(task, prerequeue_top_waiter);
  652. rt_mutex_enqueue_pi(task, waiter);
  653. rt_mutex_adjust_prio(task);
  654. } else if (prerequeue_top_waiter == waiter) {
  655. /*
  656. * The waiter was the top waiter on the lock, but is
  657. * no longer the top prority waiter. Replace waiter in
  658. * the owner tasks pi waiters tree with the new top
  659. * (highest priority) waiter and adjust the priority
  660. * of the owner.
  661. * The new top waiter is stored in @waiter so that
  662. * @waiter == @top_waiter evaluates to true below and
  663. * we continue to deboost the rest of the chain.
  664. */
  665. rt_mutex_dequeue_pi(task, waiter);
  666. waiter = rt_mutex_top_waiter(lock);
  667. rt_mutex_enqueue_pi(task, waiter);
  668. rt_mutex_adjust_prio(task);
  669. } else {
  670. /*
  671. * Nothing changed. No need to do any priority
  672. * adjustment.
  673. */
  674. }
  675. /*
  676. * [12] check_exit_conditions_4() protected by task->pi_lock
  677. * and lock->wait_lock. The actual decisions are made after we
  678. * dropped the locks.
  679. *
  680. * Check whether the task which owns the current lock is pi
  681. * blocked itself. If yes we store a pointer to the lock for
  682. * the lock chain change detection above. After we dropped
  683. * task->pi_lock next_lock cannot be dereferenced anymore.
  684. */
  685. next_lock = task_blocked_on_lock(task);
  686. /*
  687. * Store the top waiter of @lock for the end of chain walk
  688. * decision below.
  689. */
  690. top_waiter = rt_mutex_top_waiter(lock);
  691. /* [13] Drop the locks */
  692. raw_spin_unlock(&task->pi_lock);
  693. raw_spin_unlock_irq(&lock->wait_lock);
  694. /*
  695. * Make the actual exit decisions [12], based on the stored
  696. * values.
  697. *
  698. * We reached the end of the lock chain. Stop right here. No
  699. * point to go back just to figure that out.
  700. */
  701. if (!next_lock)
  702. goto out_put_task;
  703. /*
  704. * If the current waiter is not the top waiter on the lock,
  705. * then we can stop the chain walk here if we are not in full
  706. * deadlock detection mode.
  707. */
  708. if (!detect_deadlock && waiter != top_waiter)
  709. goto out_put_task;
  710. goto again;
  711. out_unlock_pi:
  712. raw_spin_unlock_irq(&task->pi_lock);
  713. out_put_task:
  714. put_task_struct(task);
  715. return ret;
  716. }
  717. /*
  718. * Try to take an rt-mutex
  719. *
  720. * Must be called with lock->wait_lock held and interrupts disabled
  721. *
  722. * @lock: The lock to be acquired.
  723. * @task: The task which wants to acquire the lock
  724. * @waiter: The waiter that is queued to the lock's wait tree if the
  725. * callsite called task_blocked_on_lock(), otherwise NULL
  726. */
  727. static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
  728. struct rt_mutex_waiter *waiter)
  729. {
  730. lockdep_assert_held(&lock->wait_lock);
  731. /*
  732. * Before testing whether we can acquire @lock, we set the
  733. * RT_MUTEX_HAS_WAITERS bit in @lock->owner. This forces all
  734. * other tasks which try to modify @lock into the slow path
  735. * and they serialize on @lock->wait_lock.
  736. *
  737. * The RT_MUTEX_HAS_WAITERS bit can have a transitional state
  738. * as explained at the top of this file if and only if:
  739. *
  740. * - There is a lock owner. The caller must fixup the
  741. * transient state if it does a trylock or leaves the lock
  742. * function due to a signal or timeout.
  743. *
  744. * - @task acquires the lock and there are no other
  745. * waiters. This is undone in rt_mutex_set_owner(@task) at
  746. * the end of this function.
  747. */
  748. mark_rt_mutex_waiters(lock);
  749. /*
  750. * If @lock has an owner, give up.
  751. */
  752. if (rt_mutex_owner(lock))
  753. return 0;
  754. /*
  755. * If @waiter != NULL, @task has already enqueued the waiter
  756. * into @lock waiter tree. If @waiter == NULL then this is a
  757. * trylock attempt.
  758. */
  759. if (waiter) {
  760. /*
  761. * If waiter is not the highest priority waiter of
  762. * @lock, give up.
  763. */
  764. if (waiter != rt_mutex_top_waiter(lock))
  765. return 0;
  766. /*
  767. * We can acquire the lock. Remove the waiter from the
  768. * lock waiters tree.
  769. */
  770. rt_mutex_dequeue(lock, waiter);
  771. } else {
  772. /*
  773. * If the lock has waiters already we check whether @task is
  774. * eligible to take over the lock.
  775. *
  776. * If there are no other waiters, @task can acquire
  777. * the lock. @task->pi_blocked_on is NULL, so it does
  778. * not need to be dequeued.
  779. */
  780. if (rt_mutex_has_waiters(lock)) {
  781. /*
  782. * If @task->prio is greater than or equal to
  783. * the top waiter priority (kernel view),
  784. * @task lost.
  785. */
  786. if (!rt_mutex_waiter_less(task_to_waiter(task),
  787. rt_mutex_top_waiter(lock)))
  788. return 0;
  789. /*
  790. * The current top waiter stays enqueued. We
  791. * don't have to change anything in the lock
  792. * waiters order.
  793. */
  794. } else {
  795. /*
  796. * No waiters. Take the lock without the
  797. * pi_lock dance.@task->pi_blocked_on is NULL
  798. * and we have no waiters to enqueue in @task
  799. * pi waiters tree.
  800. */
  801. goto takeit;
  802. }
  803. }
  804. /*
  805. * Clear @task->pi_blocked_on. Requires protection by
  806. * @task->pi_lock. Redundant operation for the @waiter == NULL
  807. * case, but conditionals are more expensive than a redundant
  808. * store.
  809. */
  810. raw_spin_lock(&task->pi_lock);
  811. task->pi_blocked_on = NULL;
  812. /*
  813. * Finish the lock acquisition. @task is the new owner. If
  814. * other waiters exist we have to insert the highest priority
  815. * waiter into @task->pi_waiters tree.
  816. */
  817. if (rt_mutex_has_waiters(lock))
  818. rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock));
  819. raw_spin_unlock(&task->pi_lock);
  820. takeit:
  821. /* We got the lock. */
  822. debug_rt_mutex_lock(lock);
  823. /*
  824. * This either preserves the RT_MUTEX_HAS_WAITERS bit if there
  825. * are still waiters or clears it.
  826. */
  827. rt_mutex_set_owner(lock, task);
  828. return 1;
  829. }
  830. /*
  831. * Task blocks on lock.
  832. *
  833. * Prepare waiter and propagate pi chain
  834. *
  835. * This must be called with lock->wait_lock held and interrupts disabled
  836. */
  837. static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
  838. struct rt_mutex_waiter *waiter,
  839. struct task_struct *task,
  840. enum rtmutex_chainwalk chwalk)
  841. {
  842. struct task_struct *owner = rt_mutex_owner(lock);
  843. struct rt_mutex_waiter *top_waiter = waiter;
  844. struct rt_mutex *next_lock;
  845. int chain_walk = 0, res;
  846. lockdep_assert_held(&lock->wait_lock);
  847. /*
  848. * Early deadlock detection. We really don't want the task to
  849. * enqueue on itself just to untangle the mess later. It's not
  850. * only an optimization. We drop the locks, so another waiter
  851. * can come in before the chain walk detects the deadlock. So
  852. * the other will detect the deadlock and return -EDEADLOCK,
  853. * which is wrong, as the other waiter is not in a deadlock
  854. * situation.
  855. */
  856. if (owner == task)
  857. return -EDEADLK;
  858. raw_spin_lock(&task->pi_lock);
  859. waiter->task = task;
  860. waiter->lock = lock;
  861. waiter->prio = task->prio;
  862. waiter->deadline = task->dl.deadline;
  863. /* Get the top priority waiter on the lock */
  864. if (rt_mutex_has_waiters(lock))
  865. top_waiter = rt_mutex_top_waiter(lock);
  866. rt_mutex_enqueue(lock, waiter);
  867. task->pi_blocked_on = waiter;
  868. raw_spin_unlock(&task->pi_lock);
  869. if (!owner)
  870. return 0;
  871. raw_spin_lock(&owner->pi_lock);
  872. if (waiter == rt_mutex_top_waiter(lock)) {
  873. rt_mutex_dequeue_pi(owner, top_waiter);
  874. rt_mutex_enqueue_pi(owner, waiter);
  875. rt_mutex_adjust_prio(owner);
  876. if (owner->pi_blocked_on)
  877. chain_walk = 1;
  878. } else if (rt_mutex_cond_detect_deadlock(waiter, chwalk)) {
  879. chain_walk = 1;
  880. }
  881. /* Store the lock on which owner is blocked or NULL */
  882. next_lock = task_blocked_on_lock(owner);
  883. raw_spin_unlock(&owner->pi_lock);
  884. /*
  885. * Even if full deadlock detection is on, if the owner is not
  886. * blocked itself, we can avoid finding this out in the chain
  887. * walk.
  888. */
  889. if (!chain_walk || !next_lock)
  890. return 0;
  891. /*
  892. * The owner can't disappear while holding a lock,
  893. * so the owner struct is protected by wait_lock.
  894. * Gets dropped in rt_mutex_adjust_prio_chain()!
  895. */
  896. get_task_struct(owner);
  897. raw_spin_unlock_irq(&lock->wait_lock);
  898. res = rt_mutex_adjust_prio_chain(owner, chwalk, lock,
  899. next_lock, waiter, task);
  900. raw_spin_lock_irq(&lock->wait_lock);
  901. return res;
  902. }
  903. /*
  904. * Remove the top waiter from the current tasks pi waiter tree and
  905. * queue it up.
  906. *
  907. * Called with lock->wait_lock held and interrupts disabled.
  908. */
  909. static void mark_wakeup_next_waiter(struct wake_q_head *wake_q,
  910. struct rt_mutex *lock)
  911. {
  912. struct rt_mutex_waiter *waiter;
  913. raw_spin_lock(&current->pi_lock);
  914. waiter = rt_mutex_top_waiter(lock);
  915. /*
  916. * Remove it from current->pi_waiters and deboost.
  917. *
  918. * We must in fact deboost here in order to ensure we call
  919. * rt_mutex_setprio() to update p->pi_top_task before the
  920. * task unblocks.
  921. */
  922. rt_mutex_dequeue_pi(current, waiter);
  923. rt_mutex_adjust_prio(current);
  924. /*
  925. * As we are waking up the top waiter, and the waiter stays
  926. * queued on the lock until it gets the lock, this lock
  927. * obviously has waiters. Just set the bit here and this has
  928. * the added benefit of forcing all new tasks into the
  929. * slow path making sure no task of lower priority than
  930. * the top waiter can steal this lock.
  931. */
  932. lock->owner = (void *) RT_MUTEX_HAS_WAITERS;
  933. /*
  934. * We deboosted before waking the top waiter task such that we don't
  935. * run two tasks with the 'same' priority (and ensure the
  936. * p->pi_top_task pointer points to a blocked task). This however can
  937. * lead to priority inversion if we would get preempted after the
  938. * deboost but before waking our donor task, hence the preempt_disable()
  939. * before unlock.
  940. *
  941. * Pairs with preempt_enable() in rt_mutex_postunlock();
  942. */
  943. preempt_disable();
  944. wake_q_add(wake_q, waiter->task);
  945. raw_spin_unlock(&current->pi_lock);
  946. }
  947. /*
  948. * Remove a waiter from a lock and give up
  949. *
  950. * Must be called with lock->wait_lock held and interrupts disabled. I must
  951. * have just failed to try_to_take_rt_mutex().
  952. */
  953. static void remove_waiter(struct rt_mutex *lock,
  954. struct rt_mutex_waiter *waiter)
  955. {
  956. bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));
  957. struct task_struct *owner = rt_mutex_owner(lock);
  958. struct rt_mutex *next_lock;
  959. lockdep_assert_held(&lock->wait_lock);
  960. raw_spin_lock(&current->pi_lock);
  961. rt_mutex_dequeue(lock, waiter);
  962. current->pi_blocked_on = NULL;
  963. raw_spin_unlock(&current->pi_lock);
  964. /*
  965. * Only update priority if the waiter was the highest priority
  966. * waiter of the lock and there is an owner to update.
  967. */
  968. if (!owner || !is_top_waiter)
  969. return;
  970. raw_spin_lock(&owner->pi_lock);
  971. rt_mutex_dequeue_pi(owner, waiter);
  972. if (rt_mutex_has_waiters(lock))
  973. rt_mutex_enqueue_pi(owner, rt_mutex_top_waiter(lock));
  974. rt_mutex_adjust_prio(owner);
  975. /* Store the lock on which owner is blocked or NULL */
  976. next_lock = task_blocked_on_lock(owner);
  977. raw_spin_unlock(&owner->pi_lock);
  978. /*
  979. * Don't walk the chain, if the owner task is not blocked
  980. * itself.
  981. */
  982. if (!next_lock)
  983. return;
  984. /* gets dropped in rt_mutex_adjust_prio_chain()! */
  985. get_task_struct(owner);
  986. raw_spin_unlock_irq(&lock->wait_lock);
  987. rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock,
  988. next_lock, NULL, current);
  989. raw_spin_lock_irq(&lock->wait_lock);
  990. }
  991. /*
  992. * Recheck the pi chain, in case we got a priority setting
  993. *
  994. * Called from sched_setscheduler
  995. */
  996. void rt_mutex_adjust_pi(struct task_struct *task)
  997. {
  998. struct rt_mutex_waiter *waiter;
  999. struct rt_mutex *next_lock;
  1000. unsigned long flags;
  1001. raw_spin_lock_irqsave(&task->pi_lock, flags);
  1002. waiter = task->pi_blocked_on;
  1003. if (!waiter || rt_mutex_waiter_equal(waiter, task_to_waiter(task))) {
  1004. raw_spin_unlock_irqrestore(&task->pi_lock, flags);
  1005. return;
  1006. }
  1007. next_lock = waiter->lock;
  1008. raw_spin_unlock_irqrestore(&task->pi_lock, flags);
  1009. /* gets dropped in rt_mutex_adjust_prio_chain()! */
  1010. get_task_struct(task);
  1011. rt_mutex_adjust_prio_chain(task, RT_MUTEX_MIN_CHAINWALK, NULL,
  1012. next_lock, NULL, task);
  1013. }
  1014. void rt_mutex_init_waiter(struct rt_mutex_waiter *waiter)
  1015. {
  1016. debug_rt_mutex_init_waiter(waiter);
  1017. RB_CLEAR_NODE(&waiter->pi_tree_entry);
  1018. RB_CLEAR_NODE(&waiter->tree_entry);
  1019. waiter->task = NULL;
  1020. }
  1021. /**
  1022. * __rt_mutex_slowlock() - Perform the wait-wake-try-to-take loop
  1023. * @lock: the rt_mutex to take
  1024. * @state: the state the task should block in (TASK_INTERRUPTIBLE
  1025. * or TASK_UNINTERRUPTIBLE)
  1026. * @timeout: the pre-initialized and started timer, or NULL for none
  1027. * @waiter: the pre-initialized rt_mutex_waiter
  1028. *
  1029. * Must be called with lock->wait_lock held and interrupts disabled
  1030. */
  1031. static int __sched
  1032. __rt_mutex_slowlock(struct rt_mutex *lock, int state,
  1033. struct hrtimer_sleeper *timeout,
  1034. struct rt_mutex_waiter *waiter)
  1035. {
  1036. int ret = 0;
  1037. trace_android_vh_rtmutex_wait_start(lock);
  1038. for (;;) {
  1039. /* Try to acquire the lock: */
  1040. if (try_to_take_rt_mutex(lock, current, waiter))
  1041. break;
  1042. /*
  1043. * TASK_INTERRUPTIBLE checks for signals and
  1044. * timeout. Ignored otherwise.
  1045. */
  1046. if (likely(state == TASK_INTERRUPTIBLE)) {
  1047. /* Signal pending? */
  1048. if (signal_pending(current))
  1049. ret = -EINTR;
  1050. if (timeout && !timeout->task)
  1051. ret = -ETIMEDOUT;
  1052. if (ret)
  1053. break;
  1054. }
  1055. raw_spin_unlock_irq(&lock->wait_lock);
  1056. debug_rt_mutex_print_deadlock(waiter);
  1057. schedule();
  1058. raw_spin_lock_irq(&lock->wait_lock);
  1059. set_current_state(state);
  1060. }
  1061. trace_android_vh_rtmutex_wait_finish(lock);
  1062. __set_current_state(TASK_RUNNING);
  1063. return ret;
  1064. }
  1065. static void rt_mutex_handle_deadlock(int res, int detect_deadlock,
  1066. struct rt_mutex_waiter *w)
  1067. {
  1068. /*
  1069. * If the result is not -EDEADLOCK or the caller requested
  1070. * deadlock detection, nothing to do here.
  1071. */
  1072. if (res != -EDEADLOCK || detect_deadlock)
  1073. return;
  1074. /*
  1075. * Yell lowdly and stop the task right here.
  1076. */
  1077. rt_mutex_print_deadlock(w);
  1078. while (1) {
  1079. set_current_state(TASK_INTERRUPTIBLE);
  1080. schedule();
  1081. }
  1082. }
  1083. /*
  1084. * Slow path lock function:
  1085. */
  1086. static int __sched
  1087. rt_mutex_slowlock(struct rt_mutex *lock, int state,
  1088. struct hrtimer_sleeper *timeout,
  1089. enum rtmutex_chainwalk chwalk)
  1090. {
  1091. struct rt_mutex_waiter waiter;
  1092. unsigned long flags;
  1093. int ret = 0;
  1094. rt_mutex_init_waiter(&waiter);
  1095. /*
  1096. * Technically we could use raw_spin_[un]lock_irq() here, but this can
  1097. * be called in early boot if the cmpxchg() fast path is disabled
  1098. * (debug, no architecture support). In this case we will acquire the
  1099. * rtmutex with lock->wait_lock held. But we cannot unconditionally
  1100. * enable interrupts in that early boot case. So we need to use the
  1101. * irqsave/restore variants.
  1102. */
  1103. raw_spin_lock_irqsave(&lock->wait_lock, flags);
  1104. /* Try to acquire the lock again: */
  1105. if (try_to_take_rt_mutex(lock, current, NULL)) {
  1106. raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  1107. return 0;
  1108. }
  1109. set_current_state(state);
  1110. /* Setup the timer, when timeout != NULL */
  1111. if (unlikely(timeout))
  1112. hrtimer_start_expires(&timeout->timer, HRTIMER_MODE_ABS);
  1113. ret = task_blocks_on_rt_mutex(lock, &waiter, current, chwalk);
  1114. if (likely(!ret))
  1115. /* sleep on the mutex */
  1116. ret = __rt_mutex_slowlock(lock, state, timeout, &waiter);
  1117. if (unlikely(ret)) {
  1118. __set_current_state(TASK_RUNNING);
  1119. remove_waiter(lock, &waiter);
  1120. rt_mutex_handle_deadlock(ret, chwalk, &waiter);
  1121. }
  1122. /*
  1123. * try_to_take_rt_mutex() sets the waiter bit
  1124. * unconditionally. We might have to fix that up.
  1125. */
  1126. fixup_rt_mutex_waiters(lock);
  1127. raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  1128. /* Remove pending timer: */
  1129. if (unlikely(timeout))
  1130. hrtimer_cancel(&timeout->timer);
  1131. debug_rt_mutex_free_waiter(&waiter);
  1132. return ret;
  1133. }
  1134. static inline int __rt_mutex_slowtrylock(struct rt_mutex *lock)
  1135. {
  1136. int ret = try_to_take_rt_mutex(lock, current, NULL);
  1137. /*
  1138. * try_to_take_rt_mutex() sets the lock waiters bit
  1139. * unconditionally. Clean this up.
  1140. */
  1141. fixup_rt_mutex_waiters(lock);
  1142. return ret;
  1143. }
  1144. /*
  1145. * Slow path try-lock function:
  1146. */
  1147. static inline int rt_mutex_slowtrylock(struct rt_mutex *lock)
  1148. {
  1149. unsigned long flags;
  1150. int ret;
  1151. /*
  1152. * If the lock already has an owner we fail to get the lock.
  1153. * This can be done without taking the @lock->wait_lock as
  1154. * it is only being read, and this is a trylock anyway.
  1155. */
  1156. if (rt_mutex_owner(lock))
  1157. return 0;
  1158. /*
  1159. * The mutex has currently no owner. Lock the wait lock and try to
  1160. * acquire the lock. We use irqsave here to support early boot calls.
  1161. */
  1162. raw_spin_lock_irqsave(&lock->wait_lock, flags);
  1163. ret = __rt_mutex_slowtrylock(lock);
  1164. raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  1165. return ret;
  1166. }
  1167. /*
  1168. * Slow path to release a rt-mutex.
  1169. *
  1170. * Return whether the current task needs to call rt_mutex_postunlock().
  1171. */
  1172. static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock,
  1173. struct wake_q_head *wake_q)
  1174. {
  1175. unsigned long flags;
  1176. /* irqsave required to support early boot calls */
  1177. raw_spin_lock_irqsave(&lock->wait_lock, flags);
  1178. debug_rt_mutex_unlock(lock);
  1179. /*
  1180. * We must be careful here if the fast path is enabled. If we
  1181. * have no waiters queued we cannot set owner to NULL here
  1182. * because of:
  1183. *
  1184. * foo->lock->owner = NULL;
  1185. * rtmutex_lock(foo->lock); <- fast path
  1186. * free = atomic_dec_and_test(foo->refcnt);
  1187. * rtmutex_unlock(foo->lock); <- fast path
  1188. * if (free)
  1189. * kfree(foo);
  1190. * raw_spin_unlock(foo->lock->wait_lock);
  1191. *
  1192. * So for the fastpath enabled kernel:
  1193. *
  1194. * Nothing can set the waiters bit as long as we hold
  1195. * lock->wait_lock. So we do the following sequence:
  1196. *
  1197. * owner = rt_mutex_owner(lock);
  1198. * clear_rt_mutex_waiters(lock);
  1199. * raw_spin_unlock(&lock->wait_lock);
  1200. * if (cmpxchg(&lock->owner, owner, 0) == owner)
  1201. * return;
  1202. * goto retry;
  1203. *
  1204. * The fastpath disabled variant is simple as all access to
  1205. * lock->owner is serialized by lock->wait_lock:
  1206. *
  1207. * lock->owner = NULL;
  1208. * raw_spin_unlock(&lock->wait_lock);
  1209. */
  1210. while (!rt_mutex_has_waiters(lock)) {
  1211. /* Drops lock->wait_lock ! */
  1212. if (unlock_rt_mutex_safe(lock, flags) == true)
  1213. return false;
  1214. /* Relock the rtmutex and try again */
  1215. raw_spin_lock_irqsave(&lock->wait_lock, flags);
  1216. }
  1217. /*
  1218. * The wakeup next waiter path does not suffer from the above
  1219. * race. See the comments there.
  1220. *
  1221. * Queue the next waiter for wakeup once we release the wait_lock.
  1222. */
  1223. mark_wakeup_next_waiter(wake_q, lock);
  1224. raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  1225. return true; /* call rt_mutex_postunlock() */
  1226. }
  1227. /*
  1228. * debug aware fast / slowpath lock,trylock,unlock
  1229. *
  1230. * The atomic acquire/release ops are compiled away, when either the
  1231. * architecture does not support cmpxchg or when debugging is enabled.
  1232. */
  1233. static inline int
  1234. rt_mutex_fastlock(struct rt_mutex *lock, int state,
  1235. int (*slowfn)(struct rt_mutex *lock, int state,
  1236. struct hrtimer_sleeper *timeout,
  1237. enum rtmutex_chainwalk chwalk))
  1238. {
  1239. if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
  1240. return 0;
  1241. return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK);
  1242. }
  1243. static inline int
  1244. rt_mutex_timed_fastlock(struct rt_mutex *lock, int state,
  1245. struct hrtimer_sleeper *timeout,
  1246. enum rtmutex_chainwalk chwalk,
  1247. int (*slowfn)(struct rt_mutex *lock, int state,
  1248. struct hrtimer_sleeper *timeout,
  1249. enum rtmutex_chainwalk chwalk))
  1250. {
  1251. if (chwalk == RT_MUTEX_MIN_CHAINWALK &&
  1252. likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
  1253. return 0;
  1254. return slowfn(lock, state, timeout, chwalk);
  1255. }
  1256. static inline int
  1257. rt_mutex_fasttrylock(struct rt_mutex *lock,
  1258. int (*slowfn)(struct rt_mutex *lock))
  1259. {
  1260. if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
  1261. return 1;
  1262. return slowfn(lock);
  1263. }
  1264. /*
  1265. * Performs the wakeup of the top-waiter and re-enables preemption.
  1266. */
  1267. void rt_mutex_postunlock(struct wake_q_head *wake_q)
  1268. {
  1269. wake_up_q(wake_q);
  1270. /* Pairs with preempt_disable() in rt_mutex_slowunlock() */
  1271. preempt_enable();
  1272. }
  1273. static inline void
  1274. rt_mutex_fastunlock(struct rt_mutex *lock,
  1275. bool (*slowfn)(struct rt_mutex *lock,
  1276. struct wake_q_head *wqh))
  1277. {
  1278. DEFINE_WAKE_Q(wake_q);
  1279. if (likely(rt_mutex_cmpxchg_release(lock, current, NULL)))
  1280. return;
  1281. if (slowfn(lock, &wake_q))
  1282. rt_mutex_postunlock(&wake_q);
  1283. }
  1284. static inline void __rt_mutex_lock(struct rt_mutex *lock, unsigned int subclass)
  1285. {
  1286. might_sleep();
  1287. mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
  1288. rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, rt_mutex_slowlock);
  1289. }
  1290. #ifdef CONFIG_DEBUG_LOCK_ALLOC
  1291. /**
  1292. * rt_mutex_lock_nested - lock a rt_mutex
  1293. *
  1294. * @lock: the rt_mutex to be locked
  1295. * @subclass: the lockdep subclass
  1296. */
  1297. void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass)
  1298. {
  1299. __rt_mutex_lock(lock, subclass);
  1300. }
  1301. EXPORT_SYMBOL_GPL(rt_mutex_lock_nested);
  1302. #else /* !CONFIG_DEBUG_LOCK_ALLOC */
  1303. /**
  1304. * rt_mutex_lock - lock a rt_mutex
  1305. *
  1306. * @lock: the rt_mutex to be locked
  1307. */
  1308. void __sched rt_mutex_lock(struct rt_mutex *lock)
  1309. {
  1310. __rt_mutex_lock(lock, 0);
  1311. }
  1312. EXPORT_SYMBOL_GPL(rt_mutex_lock);
  1313. #endif
  1314. /**
  1315. * rt_mutex_lock_interruptible - lock a rt_mutex interruptible
  1316. *
  1317. * @lock: the rt_mutex to be locked
  1318. *
  1319. * Returns:
  1320. * 0 on success
  1321. * -EINTR when interrupted by a signal
  1322. */
  1323. int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock)
  1324. {
  1325. int ret;
  1326. might_sleep();
  1327. mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_);
  1328. ret = rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, rt_mutex_slowlock);
  1329. if (ret)
  1330. mutex_release(&lock->dep_map, _RET_IP_);
  1331. return ret;
  1332. }
  1333. EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
  1334. /*
  1335. * Futex variant, must not use fastpath.
  1336. */
  1337. int __sched rt_mutex_futex_trylock(struct rt_mutex *lock)
  1338. {
  1339. return rt_mutex_slowtrylock(lock);
  1340. }
  1341. int __sched __rt_mutex_futex_trylock(struct rt_mutex *lock)
  1342. {
  1343. return __rt_mutex_slowtrylock(lock);
  1344. }
  1345. /**
  1346. * rt_mutex_timed_lock - lock a rt_mutex interruptible
  1347. * the timeout structure is provided
  1348. * by the caller
  1349. *
  1350. * @lock: the rt_mutex to be locked
  1351. * @timeout: timeout structure or NULL (no timeout)
  1352. *
  1353. * Returns:
  1354. * 0 on success
  1355. * -EINTR when interrupted by a signal
  1356. * -ETIMEDOUT when the timeout expired
  1357. */
  1358. int
  1359. rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout)
  1360. {
  1361. int ret;
  1362. might_sleep();
  1363. mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_);
  1364. ret = rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
  1365. RT_MUTEX_MIN_CHAINWALK,
  1366. rt_mutex_slowlock);
  1367. if (ret)
  1368. mutex_release(&lock->dep_map, _RET_IP_);
  1369. return ret;
  1370. }
  1371. EXPORT_SYMBOL_GPL(rt_mutex_timed_lock);
  1372. /**
  1373. * rt_mutex_trylock - try to lock a rt_mutex
  1374. *
  1375. * @lock: the rt_mutex to be locked
  1376. *
  1377. * This function can only be called in thread context. It's safe to
  1378. * call it from atomic regions, but not from hard interrupt or soft
  1379. * interrupt context.
  1380. *
  1381. * Returns 1 on success and 0 on contention
  1382. */
  1383. int __sched rt_mutex_trylock(struct rt_mutex *lock)
  1384. {
  1385. int ret;
  1386. if (WARN_ON_ONCE(in_irq() || in_nmi() || in_serving_softirq()))
  1387. return 0;
  1388. ret = rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock);
  1389. if (ret)
  1390. mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  1391. return ret;
  1392. }
  1393. EXPORT_SYMBOL_GPL(rt_mutex_trylock);
  1394. /**
  1395. * rt_mutex_unlock - unlock a rt_mutex
  1396. *
  1397. * @lock: the rt_mutex to be unlocked
  1398. */
  1399. void __sched rt_mutex_unlock(struct rt_mutex *lock)
  1400. {
  1401. mutex_release(&lock->dep_map, _RET_IP_);
  1402. rt_mutex_fastunlock(lock, rt_mutex_slowunlock);
  1403. }
  1404. EXPORT_SYMBOL_GPL(rt_mutex_unlock);
  1405. /**
  1406. * Futex variant, that since futex variants do not use the fast-path, can be
  1407. * simple and will not need to retry.
  1408. */
  1409. bool __sched __rt_mutex_futex_unlock(struct rt_mutex *lock,
  1410. struct wake_q_head *wake_q)
  1411. {
  1412. lockdep_assert_held(&lock->wait_lock);
  1413. debug_rt_mutex_unlock(lock);
  1414. if (!rt_mutex_has_waiters(lock)) {
  1415. lock->owner = NULL;
  1416. return false; /* done */
  1417. }
  1418. /*
  1419. * We've already deboosted, mark_wakeup_next_waiter() will
  1420. * retain preempt_disabled when we drop the wait_lock, to
  1421. * avoid inversion prior to the wakeup. preempt_disable()
  1422. * therein pairs with rt_mutex_postunlock().
  1423. */
  1424. mark_wakeup_next_waiter(wake_q, lock);
  1425. return true; /* call postunlock() */
  1426. }
  1427. void __sched rt_mutex_futex_unlock(struct rt_mutex *lock)
  1428. {
  1429. DEFINE_WAKE_Q(wake_q);
  1430. unsigned long flags;
  1431. bool postunlock;
  1432. raw_spin_lock_irqsave(&lock->wait_lock, flags);
  1433. postunlock = __rt_mutex_futex_unlock(lock, &wake_q);
  1434. raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
  1435. if (postunlock)
  1436. rt_mutex_postunlock(&wake_q);
  1437. }
  1438. /**
  1439. * rt_mutex_destroy - mark a mutex unusable
  1440. * @lock: the mutex to be destroyed
  1441. *
  1442. * This function marks the mutex uninitialized, and any subsequent
  1443. * use of the mutex is forbidden. The mutex must not be locked when
  1444. * this function is called.
  1445. */
  1446. void rt_mutex_destroy(struct rt_mutex *lock)
  1447. {
  1448. WARN_ON(rt_mutex_is_locked(lock));
  1449. #ifdef CONFIG_DEBUG_RT_MUTEXES
  1450. lock->magic = NULL;
  1451. #endif
  1452. }
  1453. EXPORT_SYMBOL_GPL(rt_mutex_destroy);
  1454. /**
  1455. * __rt_mutex_init - initialize the rt lock
  1456. *
  1457. * @lock: the rt lock to be initialized
  1458. *
  1459. * Initialize the rt lock to unlocked state.
  1460. *
  1461. * Initializing of a locked rt lock is not allowed
  1462. */
  1463. void __rt_mutex_init(struct rt_mutex *lock, const char *name,
  1464. struct lock_class_key *key)
  1465. {
  1466. lock->owner = NULL;
  1467. raw_spin_lock_init(&lock->wait_lock);
  1468. lock->waiters = RB_ROOT_CACHED;
  1469. if (name && key)
  1470. debug_rt_mutex_init(lock, name, key);
  1471. }
  1472. EXPORT_SYMBOL_GPL(__rt_mutex_init);
  1473. /**
  1474. * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a
  1475. * proxy owner
  1476. *
  1477. * @lock: the rt_mutex to be locked
  1478. * @proxy_owner:the task to set as owner
  1479. *
  1480. * No locking. Caller has to do serializing itself
  1481. *
  1482. * Special API call for PI-futex support. This initializes the rtmutex and
  1483. * assigns it to @proxy_owner. Concurrent operations on the rtmutex are not
  1484. * possible at this point because the pi_state which contains the rtmutex
  1485. * is not yet visible to other tasks.
  1486. */
  1487. void rt_mutex_init_proxy_locked(struct rt_mutex *lock,
  1488. struct task_struct *proxy_owner)
  1489. {
  1490. __rt_mutex_init(lock, NULL, NULL);
  1491. debug_rt_mutex_proxy_lock(lock, proxy_owner);
  1492. rt_mutex_set_owner(lock, proxy_owner);
  1493. }
  1494. /**
  1495. * rt_mutex_proxy_unlock - release a lock on behalf of owner
  1496. *
  1497. * @lock: the rt_mutex to be locked
  1498. *
  1499. * No locking. Caller has to do serializing itself
  1500. *
  1501. * Special API call for PI-futex support. This merrily cleans up the rtmutex
  1502. * (debugging) state. Concurrent operations on this rt_mutex are not
  1503. * possible because it belongs to the pi_state which is about to be freed
  1504. * and it is not longer visible to other tasks.
  1505. */
  1506. void rt_mutex_proxy_unlock(struct rt_mutex *lock)
  1507. {
  1508. debug_rt_mutex_proxy_unlock(lock);
  1509. rt_mutex_set_owner(lock, NULL);
  1510. }
  1511. /**
  1512. * __rt_mutex_start_proxy_lock() - Start lock acquisition for another task
  1513. * @lock: the rt_mutex to take
  1514. * @waiter: the pre-initialized rt_mutex_waiter
  1515. * @task: the task to prepare
  1516. *
  1517. * Starts the rt_mutex acquire; it enqueues the @waiter and does deadlock
  1518. * detection. It does not wait, see rt_mutex_wait_proxy_lock() for that.
  1519. *
  1520. * NOTE: does _NOT_ remove the @waiter on failure; must either call
  1521. * rt_mutex_wait_proxy_lock() or rt_mutex_cleanup_proxy_lock() after this.
  1522. *
  1523. * Returns:
  1524. * 0 - task blocked on lock
  1525. * 1 - acquired the lock for task, caller should wake it up
  1526. * <0 - error
  1527. *
  1528. * Special API call for PI-futex support.
  1529. */
  1530. int __rt_mutex_start_proxy_lock(struct rt_mutex *lock,
  1531. struct rt_mutex_waiter *waiter,
  1532. struct task_struct *task)
  1533. {
  1534. int ret;
  1535. lockdep_assert_held(&lock->wait_lock);
  1536. if (try_to_take_rt_mutex(lock, task, NULL))
  1537. return 1;
  1538. /* We enforce deadlock detection for futexes */
  1539. ret = task_blocks_on_rt_mutex(lock, waiter, task,
  1540. RT_MUTEX_FULL_CHAINWALK);
  1541. if (ret && !rt_mutex_owner(lock)) {
  1542. /*
  1543. * Reset the return value. We might have
  1544. * returned with -EDEADLK and the owner
  1545. * released the lock while we were walking the
  1546. * pi chain. Let the waiter sort it out.
  1547. */
  1548. ret = 0;
  1549. }
  1550. debug_rt_mutex_print_deadlock(waiter);
  1551. return ret;
  1552. }
  1553. /**
  1554. * rt_mutex_start_proxy_lock() - Start lock acquisition for another task
  1555. * @lock: the rt_mutex to take
  1556. * @waiter: the pre-initialized rt_mutex_waiter
  1557. * @task: the task to prepare
  1558. *
  1559. * Starts the rt_mutex acquire; it enqueues the @waiter and does deadlock
  1560. * detection. It does not wait, see rt_mutex_wait_proxy_lock() for that.
  1561. *
  1562. * NOTE: unlike __rt_mutex_start_proxy_lock this _DOES_ remove the @waiter
  1563. * on failure.
  1564. *
  1565. * Returns:
  1566. * 0 - task blocked on lock
  1567. * 1 - acquired the lock for task, caller should wake it up
  1568. * <0 - error
  1569. *
  1570. * Special API call for PI-futex support.
  1571. */
  1572. int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
  1573. struct rt_mutex_waiter *waiter,
  1574. struct task_struct *task)
  1575. {
  1576. int ret;
  1577. raw_spin_lock_irq(&lock->wait_lock);
  1578. ret = __rt_mutex_start_proxy_lock(lock, waiter, task);
  1579. if (unlikely(ret))
  1580. remove_waiter(lock, waiter);
  1581. raw_spin_unlock_irq(&lock->wait_lock);
  1582. return ret;
  1583. }
  1584. /**
  1585. * rt_mutex_next_owner - return the next owner of the lock
  1586. *
  1587. * @lock: the rt lock query
  1588. *
  1589. * Returns the next owner of the lock or NULL
  1590. *
  1591. * Caller has to serialize against other accessors to the lock
  1592. * itself.
  1593. *
  1594. * Special API call for PI-futex support
  1595. */
  1596. struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock)
  1597. {
  1598. if (!rt_mutex_has_waiters(lock))
  1599. return NULL;
  1600. return rt_mutex_top_waiter(lock)->task;
  1601. }
  1602. /**
  1603. * rt_mutex_wait_proxy_lock() - Wait for lock acquisition
  1604. * @lock: the rt_mutex we were woken on
  1605. * @to: the timeout, null if none. hrtimer should already have
  1606. * been started.
  1607. * @waiter: the pre-initialized rt_mutex_waiter
  1608. *
  1609. * Wait for the lock acquisition started on our behalf by
  1610. * rt_mutex_start_proxy_lock(). Upon failure, the caller must call
  1611. * rt_mutex_cleanup_proxy_lock().
  1612. *
  1613. * Returns:
  1614. * 0 - success
  1615. * <0 - error, one of -EINTR, -ETIMEDOUT
  1616. *
  1617. * Special API call for PI-futex support
  1618. */
  1619. int rt_mutex_wait_proxy_lock(struct rt_mutex *lock,
  1620. struct hrtimer_sleeper *to,
  1621. struct rt_mutex_waiter *waiter)
  1622. {
  1623. int ret;
  1624. raw_spin_lock_irq(&lock->wait_lock);
  1625. /* sleep on the mutex */
  1626. set_current_state(TASK_INTERRUPTIBLE);
  1627. ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter);
  1628. /*
  1629. * try_to_take_rt_mutex() sets the waiter bit unconditionally. We might
  1630. * have to fix that up.
  1631. */
  1632. fixup_rt_mutex_waiters(lock);
  1633. raw_spin_unlock_irq(&lock->wait_lock);
  1634. return ret;
  1635. }
  1636. /**
  1637. * rt_mutex_cleanup_proxy_lock() - Cleanup failed lock acquisition
  1638. * @lock: the rt_mutex we were woken on
  1639. * @waiter: the pre-initialized rt_mutex_waiter
  1640. *
  1641. * Attempt to clean up after a failed __rt_mutex_start_proxy_lock() or
  1642. * rt_mutex_wait_proxy_lock().
  1643. *
  1644. * Unless we acquired the lock; we're still enqueued on the wait-list and can
  1645. * in fact still be granted ownership until we're removed. Therefore we can
  1646. * find we are in fact the owner and must disregard the
  1647. * rt_mutex_wait_proxy_lock() failure.
  1648. *
  1649. * Returns:
  1650. * true - did the cleanup, we done.
  1651. * false - we acquired the lock after rt_mutex_wait_proxy_lock() returned,
  1652. * caller should disregards its return value.
  1653. *
  1654. * Special API call for PI-futex support
  1655. */
  1656. bool rt_mutex_cleanup_proxy_lock(struct rt_mutex *lock,
  1657. struct rt_mutex_waiter *waiter)
  1658. {
  1659. bool cleanup = false;
  1660. raw_spin_lock_irq(&lock->wait_lock);
  1661. /*
  1662. * Do an unconditional try-lock, this deals with the lock stealing
  1663. * state where __rt_mutex_futex_unlock() -> mark_wakeup_next_waiter()
  1664. * sets a NULL owner.
  1665. *
  1666. * We're not interested in the return value, because the subsequent
  1667. * test on rt_mutex_owner() will infer that. If the trylock succeeded,
  1668. * we will own the lock and it will have removed the waiter. If we
  1669. * failed the trylock, we're still not owner and we need to remove
  1670. * ourselves.
  1671. */
  1672. try_to_take_rt_mutex(lock, current, waiter);
  1673. /*
  1674. * Unless we're the owner; we're still enqueued on the wait_list.
  1675. * So check if we became owner, if not, take us off the wait_list.
  1676. */
  1677. if (rt_mutex_owner(lock) != current) {
  1678. remove_waiter(lock, waiter);
  1679. cleanup = true;
  1680. }
  1681. /*
  1682. * try_to_take_rt_mutex() sets the waiter bit unconditionally. We might
  1683. * have to fix that up.
  1684. */
  1685. fixup_rt_mutex_waiters(lock);
  1686. raw_spin_unlock_irq(&lock->wait_lock);
  1687. return cleanup;
  1688. }