rtmutex.c 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990
  1. /*
  2. * RT-Mutexes: simple blocking mutual exclusion locks with PI support
  3. *
  4. * started by Ingo Molnar and Thomas Gleixner.
  5. *
  6. * Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
  7. * Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner <tglx@timesys.com>
  8. * Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt
  9. * Copyright (C) 2006 Esben Nielsen
  10. *
  11. * See Documentation/rt-mutex-design.txt for details.
  12. */
  13. #include <linux/spinlock.h>
  14. #include <linux/module.h>
  15. #include <linux/sched.h>
  16. #include <linux/timer.h>
  17. #include "rtmutex_common.h"
  18. #ifdef CONFIG_DEBUG_RT_MUTEXES
  19. # include "rtmutex-debug.h"
  20. #else
  21. # include "rtmutex.h"
  22. #endif
  23. /*
  24. * lock->owner state tracking:
  25. *
  26. * lock->owner holds the task_struct pointer of the owner. Bit 0 and 1
  27. * are used to keep track of the "owner is pending" and "lock has
  28. * waiters" state.
  29. *
  30. * owner bit1 bit0
  31. * NULL 0 0 lock is free (fast acquire possible)
  32. * NULL 0 1 invalid state
  33. * NULL 1 0 Transitional State*
  34. * NULL 1 1 invalid state
  35. * taskpointer 0 0 lock is held (fast release possible)
  36. * taskpointer 0 1 task is pending owner
  37. * taskpointer 1 0 lock is held and has waiters
  38. * taskpointer 1 1 task is pending owner and lock has more waiters
  39. *
  40. * Pending ownership is assigned to the top (highest priority)
  41. * waiter of the lock, when the lock is released. The thread is woken
  42. * up and can now take the lock. Until the lock is taken (bit 0
  43. * cleared) a competing higher priority thread can steal the lock
  44. * which puts the woken up thread back on the waiters list.
  45. *
  46. * The fast atomic compare exchange based acquire and release is only
  47. * possible when bit 0 and 1 of lock->owner are 0.
  48. *
  49. * (*) There's a small time where the owner can be NULL and the
  50. * "lock has waiters" bit is set. This can happen when grabbing the lock.
  51. * To prevent a cmpxchg of the owner releasing the lock, we need to set this
  52. * bit before looking at the lock, hence the reason this is a transitional
  53. * state.
  54. */
  55. static void
  56. rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner,
  57. unsigned long mask)
  58. {
  59. unsigned long val = (unsigned long)owner | mask;
  60. if (rt_mutex_has_waiters(lock))
  61. val |= RT_MUTEX_HAS_WAITERS;
  62. lock->owner = (struct task_struct *)val;
  63. }
  64. static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)
  65. {
  66. lock->owner = (struct task_struct *)
  67. ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
  68. }
  69. static void fixup_rt_mutex_waiters(struct rt_mutex *lock)
  70. {
  71. if (!rt_mutex_has_waiters(lock))
  72. clear_rt_mutex_waiters(lock);
  73. }
  74. /*
  75. * We can speed up the acquire/release, if the architecture
  76. * supports cmpxchg and if there's no debugging state to be set up
  77. */
  78. #if defined(__HAVE_ARCH_CMPXCHG) && !defined(CONFIG_DEBUG_RT_MUTEXES)
  79. # define rt_mutex_cmpxchg(l,c,n) (cmpxchg(&l->owner, c, n) == c)
  80. static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
  81. {
  82. unsigned long owner, *p = (unsigned long *) &lock->owner;
  83. do {
  84. owner = *p;
  85. } while (cmpxchg(p, owner, owner | RT_MUTEX_HAS_WAITERS) != owner);
  86. }
  87. #else
  88. # define rt_mutex_cmpxchg(l,c,n) (0)
  89. static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
  90. {
  91. lock->owner = (struct task_struct *)
  92. ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);
  93. }
  94. #endif
  95. /*
  96. * Calculate task priority from the waiter list priority
  97. *
  98. * Return task->normal_prio when the waiter list is empty or when
  99. * the waiter is not allowed to do priority boosting
  100. */
  101. int rt_mutex_getprio(struct task_struct *task)
  102. {
  103. if (likely(!task_has_pi_waiters(task)))
  104. return task->normal_prio;
  105. return min(task_top_pi_waiter(task)->pi_list_entry.prio,
  106. task->normal_prio);
  107. }
  108. /*
  109. * Adjust the priority of a task, after its pi_waiters got modified.
  110. *
  111. * This can be both boosting and unboosting. task->pi_lock must be held.
  112. */
  113. static void __rt_mutex_adjust_prio(struct task_struct *task)
  114. {
  115. int prio = rt_mutex_getprio(task);
  116. if (task->prio != prio)
  117. rt_mutex_setprio(task, prio);
  118. }
  119. /*
  120. * Adjust task priority (undo boosting). Called from the exit path of
  121. * rt_mutex_slowunlock() and rt_mutex_slowlock().
  122. *
  123. * (Note: We do this outside of the protection of lock->wait_lock to
  124. * allow the lock to be taken while or before we readjust the priority
  125. * of task. We do not use the spin_xx_mutex() variants here as we are
  126. * outside of the debug path.)
  127. */
  128. static void rt_mutex_adjust_prio(struct task_struct *task)
  129. {
  130. unsigned long flags;
  131. spin_lock_irqsave(&task->pi_lock, flags);
  132. __rt_mutex_adjust_prio(task);
  133. spin_unlock_irqrestore(&task->pi_lock, flags);
  134. }
  135. /*
  136. * Max number of times we'll walk the boosting chain:
  137. */
  138. int max_lock_depth = 1024;
  139. /*
  140. * Adjust the priority chain. Also used for deadlock detection.
  141. * Decreases task's usage by one - may thus free the task.
  142. * Returns 0 or -EDEADLK.
  143. */
  144. static int rt_mutex_adjust_prio_chain(struct task_struct *task,
  145. int deadlock_detect,
  146. struct rt_mutex *orig_lock,
  147. struct rt_mutex_waiter *orig_waiter,
  148. struct task_struct *top_task)
  149. {
  150. struct rt_mutex *lock;
  151. struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;
  152. int detect_deadlock, ret = 0, depth = 0;
  153. unsigned long flags;
  154. detect_deadlock = debug_rt_mutex_detect_deadlock(orig_waiter,
  155. deadlock_detect);
  156. /*
  157. * The (de)boosting is a step by step approach with a lot of
  158. * pitfalls. We want this to be preemptible and we want hold a
  159. * maximum of two locks per step. So we have to check
  160. * carefully whether things change under us.
  161. */
  162. again:
  163. if (++depth > max_lock_depth) {
  164. static int prev_max;
  165. /*
  166. * Print this only once. If the admin changes the limit,
  167. * print a new message when reaching the limit again.
  168. */
  169. if (prev_max != max_lock_depth) {
  170. prev_max = max_lock_depth;
  171. printk(KERN_WARNING "Maximum lock depth %d reached "
  172. "task: %s (%d)\n", max_lock_depth,
  173. top_task->comm, top_task->pid);
  174. }
  175. put_task_struct(task);
  176. return deadlock_detect ? -EDEADLK : 0;
  177. }
  178. retry:
  179. /*
  180. * Task can not go away as we did a get_task() before !
  181. */
  182. spin_lock_irqsave(&task->pi_lock, flags);
  183. waiter = task->pi_blocked_on;
  184. /*
  185. * Check whether the end of the boosting chain has been
  186. * reached or the state of the chain has changed while we
  187. * dropped the locks.
  188. */
  189. if (!waiter || !waiter->task)
  190. goto out_unlock_pi;
  191. if (top_waiter && (!task_has_pi_waiters(task) ||
  192. top_waiter != task_top_pi_waiter(task)))
  193. goto out_unlock_pi;
  194. /*
  195. * When deadlock detection is off then we check, if further
  196. * priority adjustment is necessary.
  197. */
  198. if (!detect_deadlock && waiter->list_entry.prio == task->prio)
  199. goto out_unlock_pi;
  200. lock = waiter->lock;
  201. if (!spin_trylock(&lock->wait_lock)) {
  202. spin_unlock_irqrestore(&task->pi_lock, flags);
  203. cpu_relax();
  204. goto retry;
  205. }
  206. /* Deadlock detection */
  207. if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {
  208. debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock);
  209. spin_unlock(&lock->wait_lock);
  210. ret = deadlock_detect ? -EDEADLK : 0;
  211. goto out_unlock_pi;
  212. }
  213. top_waiter = rt_mutex_top_waiter(lock);
  214. /* Requeue the waiter */
  215. plist_del(&waiter->list_entry, &lock->wait_list);
  216. waiter->list_entry.prio = task->prio;
  217. plist_add(&waiter->list_entry, &lock->wait_list);
  218. /* Release the task */
  219. spin_unlock_irqrestore(&task->pi_lock, flags);
  220. put_task_struct(task);
  221. /* Grab the next task */
  222. task = rt_mutex_owner(lock);
  223. get_task_struct(task);
  224. spin_lock_irqsave(&task->pi_lock, flags);
  225. if (waiter == rt_mutex_top_waiter(lock)) {
  226. /* Boost the owner */
  227. plist_del(&top_waiter->pi_list_entry, &task->pi_waiters);
  228. waiter->pi_list_entry.prio = waiter->list_entry.prio;
  229. plist_add(&waiter->pi_list_entry, &task->pi_waiters);
  230. __rt_mutex_adjust_prio(task);
  231. } else if (top_waiter == waiter) {
  232. /* Deboost the owner */
  233. plist_del(&waiter->pi_list_entry, &task->pi_waiters);
  234. waiter = rt_mutex_top_waiter(lock);
  235. waiter->pi_list_entry.prio = waiter->list_entry.prio;
  236. plist_add(&waiter->pi_list_entry, &task->pi_waiters);
  237. __rt_mutex_adjust_prio(task);
  238. }
  239. spin_unlock_irqrestore(&task->pi_lock, flags);
  240. top_waiter = rt_mutex_top_waiter(lock);
  241. spin_unlock(&lock->wait_lock);
  242. if (!detect_deadlock && waiter != top_waiter)
  243. goto out_put_task;
  244. goto again;
  245. out_unlock_pi:
  246. spin_unlock_irqrestore(&task->pi_lock, flags);
  247. out_put_task:
  248. put_task_struct(task);
  249. return ret;
  250. }
  251. /*
  252. * Optimization: check if we can steal the lock from the
  253. * assigned pending owner [which might not have taken the
  254. * lock yet]:
  255. */
  256. static inline int try_to_steal_lock(struct rt_mutex *lock)
  257. {
  258. struct task_struct *pendowner = rt_mutex_owner(lock);
  259. struct rt_mutex_waiter *next;
  260. unsigned long flags;
  261. if (!rt_mutex_owner_pending(lock))
  262. return 0;
  263. if (pendowner == current)
  264. return 1;
  265. spin_lock_irqsave(&pendowner->pi_lock, flags);
  266. if (current->prio >= pendowner->prio) {
  267. spin_unlock_irqrestore(&pendowner->pi_lock, flags);
  268. return 0;
  269. }
  270. /*
  271. * Check if a waiter is enqueued on the pending owners
  272. * pi_waiters list. Remove it and readjust pending owners
  273. * priority.
  274. */
  275. if (likely(!rt_mutex_has_waiters(lock))) {
  276. spin_unlock_irqrestore(&pendowner->pi_lock, flags);
  277. return 1;
  278. }
  279. /* No chain handling, pending owner is not blocked on anything: */
  280. next = rt_mutex_top_waiter(lock);
  281. plist_del(&next->pi_list_entry, &pendowner->pi_waiters);
  282. __rt_mutex_adjust_prio(pendowner);
  283. spin_unlock_irqrestore(&pendowner->pi_lock, flags);
  284. /*
  285. * We are going to steal the lock and a waiter was
  286. * enqueued on the pending owners pi_waiters queue. So
  287. * we have to enqueue this waiter into
  288. * current->pi_waiters list. This covers the case,
  289. * where current is boosted because it holds another
  290. * lock and gets unboosted because the booster is
  291. * interrupted, so we would delay a waiter with higher
  292. * priority as current->normal_prio.
  293. *
  294. * Note: in the rare case of a SCHED_OTHER task changing
  295. * its priority and thus stealing the lock, next->task
  296. * might be current:
  297. */
  298. if (likely(next->task != current)) {
  299. spin_lock_irqsave(&current->pi_lock, flags);
  300. plist_add(&next->pi_list_entry, &current->pi_waiters);
  301. __rt_mutex_adjust_prio(current);
  302. spin_unlock_irqrestore(&current->pi_lock, flags);
  303. }
  304. return 1;
  305. }
  306. /*
  307. * Try to take an rt-mutex
  308. *
  309. * This fails
  310. * - when the lock has a real owner
  311. * - when a different pending owner exists and has higher priority than current
  312. *
  313. * Must be called with lock->wait_lock held.
  314. */
  315. static int try_to_take_rt_mutex(struct rt_mutex *lock)
  316. {
  317. /*
  318. * We have to be careful here if the atomic speedups are
  319. * enabled, such that, when
  320. * - no other waiter is on the lock
  321. * - the lock has been released since we did the cmpxchg
  322. * the lock can be released or taken while we are doing the
  323. * checks and marking the lock with RT_MUTEX_HAS_WAITERS.
  324. *
  325. * The atomic acquire/release aware variant of
  326. * mark_rt_mutex_waiters uses a cmpxchg loop. After setting
  327. * the WAITERS bit, the atomic release / acquire can not
  328. * happen anymore and lock->wait_lock protects us from the
  329. * non-atomic case.
  330. *
  331. * Note, that this might set lock->owner =
  332. * RT_MUTEX_HAS_WAITERS in the case the lock is not contended
  333. * any more. This is fixed up when we take the ownership.
  334. * This is the transitional state explained at the top of this file.
  335. */
  336. mark_rt_mutex_waiters(lock);
  337. if (rt_mutex_owner(lock) && !try_to_steal_lock(lock))
  338. return 0;
  339. /* We got the lock. */
  340. debug_rt_mutex_lock(lock);
  341. rt_mutex_set_owner(lock, current, 0);
  342. rt_mutex_deadlock_account_lock(lock, current);
  343. return 1;
  344. }
  345. /*
  346. * Task blocks on lock.
  347. *
  348. * Prepare waiter and propagate pi chain
  349. *
  350. * This must be called with lock->wait_lock held.
  351. */
  352. static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
  353. struct rt_mutex_waiter *waiter,
  354. int detect_deadlock)
  355. {
  356. struct task_struct *owner = rt_mutex_owner(lock);
  357. struct rt_mutex_waiter *top_waiter = waiter;
  358. unsigned long flags;
  359. int chain_walk = 0, res;
  360. spin_lock_irqsave(&current->pi_lock, flags);
  361. __rt_mutex_adjust_prio(current);
  362. waiter->task = current;
  363. waiter->lock = lock;
  364. plist_node_init(&waiter->list_entry, current->prio);
  365. plist_node_init(&waiter->pi_list_entry, current->prio);
  366. /* Get the top priority waiter on the lock */
  367. if (rt_mutex_has_waiters(lock))
  368. top_waiter = rt_mutex_top_waiter(lock);
  369. plist_add(&waiter->list_entry, &lock->wait_list);
  370. current->pi_blocked_on = waiter;
  371. spin_unlock_irqrestore(&current->pi_lock, flags);
  372. if (waiter == rt_mutex_top_waiter(lock)) {
  373. spin_lock_irqsave(&owner->pi_lock, flags);
  374. plist_del(&top_waiter->pi_list_entry, &owner->pi_waiters);
  375. plist_add(&waiter->pi_list_entry, &owner->pi_waiters);
  376. __rt_mutex_adjust_prio(owner);
  377. if (owner->pi_blocked_on)
  378. chain_walk = 1;
  379. spin_unlock_irqrestore(&owner->pi_lock, flags);
  380. }
  381. else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock))
  382. chain_walk = 1;
  383. if (!chain_walk)
  384. return 0;
  385. /*
  386. * The owner can't disappear while holding a lock,
  387. * so the owner struct is protected by wait_lock.
  388. * Gets dropped in rt_mutex_adjust_prio_chain()!
  389. */
  390. get_task_struct(owner);
  391. spin_unlock(&lock->wait_lock);
  392. res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter,
  393. current);
  394. spin_lock(&lock->wait_lock);
  395. return res;
  396. }
  397. /*
  398. * Wake up the next waiter on the lock.
  399. *
  400. * Remove the top waiter from the current tasks waiter list and from
  401. * the lock waiter list. Set it as pending owner. Then wake it up.
  402. *
  403. * Called with lock->wait_lock held.
  404. */
  405. static void wakeup_next_waiter(struct rt_mutex *lock)
  406. {
  407. struct rt_mutex_waiter *waiter;
  408. struct task_struct *pendowner;
  409. unsigned long flags;
  410. spin_lock_irqsave(&current->pi_lock, flags);
  411. waiter = rt_mutex_top_waiter(lock);
  412. plist_del(&waiter->list_entry, &lock->wait_list);
  413. /*
  414. * Remove it from current->pi_waiters. We do not adjust a
  415. * possible priority boost right now. We execute wakeup in the
  416. * boosted mode and go back to normal after releasing
  417. * lock->wait_lock.
  418. */
  419. plist_del(&waiter->pi_list_entry, &current->pi_waiters);
  420. pendowner = waiter->task;
  421. waiter->task = NULL;
  422. rt_mutex_set_owner(lock, pendowner, RT_MUTEX_OWNER_PENDING);
  423. spin_unlock_irqrestore(&current->pi_lock, flags);
  424. /*
  425. * Clear the pi_blocked_on variable and enqueue a possible
  426. * waiter into the pi_waiters list of the pending owner. This
  427. * prevents that in case the pending owner gets unboosted a
  428. * waiter with higher priority than pending-owner->normal_prio
  429. * is blocked on the unboosted (pending) owner.
  430. */
  431. spin_lock_irqsave(&pendowner->pi_lock, flags);
  432. WARN_ON(!pendowner->pi_blocked_on);
  433. WARN_ON(pendowner->pi_blocked_on != waiter);
  434. WARN_ON(pendowner->pi_blocked_on->lock != lock);
  435. pendowner->pi_blocked_on = NULL;
  436. if (rt_mutex_has_waiters(lock)) {
  437. struct rt_mutex_waiter *next;
  438. next = rt_mutex_top_waiter(lock);
  439. plist_add(&next->pi_list_entry, &pendowner->pi_waiters);
  440. }
  441. spin_unlock_irqrestore(&pendowner->pi_lock, flags);
  442. wake_up_process(pendowner);
  443. }
  444. /*
  445. * Remove a waiter from a lock
  446. *
  447. * Must be called with lock->wait_lock held
  448. */
  449. static void remove_waiter(struct rt_mutex *lock,
  450. struct rt_mutex_waiter *waiter)
  451. {
  452. int first = (waiter == rt_mutex_top_waiter(lock));
  453. struct task_struct *owner = rt_mutex_owner(lock);
  454. unsigned long flags;
  455. int chain_walk = 0;
  456. spin_lock_irqsave(&current->pi_lock, flags);
  457. plist_del(&waiter->list_entry, &lock->wait_list);
  458. waiter->task = NULL;
  459. current->pi_blocked_on = NULL;
  460. spin_unlock_irqrestore(&current->pi_lock, flags);
  461. if (first && owner != current) {
  462. spin_lock_irqsave(&owner->pi_lock, flags);
  463. plist_del(&waiter->pi_list_entry, &owner->pi_waiters);
  464. if (rt_mutex_has_waiters(lock)) {
  465. struct rt_mutex_waiter *next;
  466. next = rt_mutex_top_waiter(lock);
  467. plist_add(&next->pi_list_entry, &owner->pi_waiters);
  468. }
  469. __rt_mutex_adjust_prio(owner);
  470. if (owner->pi_blocked_on)
  471. chain_walk = 1;
  472. spin_unlock_irqrestore(&owner->pi_lock, flags);
  473. }
  474. WARN_ON(!plist_node_empty(&waiter->pi_list_entry));
  475. if (!chain_walk)
  476. return;
  477. /* gets dropped in rt_mutex_adjust_prio_chain()! */
  478. get_task_struct(owner);
  479. spin_unlock(&lock->wait_lock);
  480. rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current);
  481. spin_lock(&lock->wait_lock);
  482. }
  483. /*
  484. * Recheck the pi chain, in case we got a priority setting
  485. *
  486. * Called from sched_setscheduler
  487. */
  488. void rt_mutex_adjust_pi(struct task_struct *task)
  489. {
  490. struct rt_mutex_waiter *waiter;
  491. unsigned long flags;
  492. spin_lock_irqsave(&task->pi_lock, flags);
  493. waiter = task->pi_blocked_on;
  494. if (!waiter || waiter->list_entry.prio == task->prio) {
  495. spin_unlock_irqrestore(&task->pi_lock, flags);
  496. return;
  497. }
  498. spin_unlock_irqrestore(&task->pi_lock, flags);
  499. /* gets dropped in rt_mutex_adjust_prio_chain()! */
  500. get_task_struct(task);
  501. rt_mutex_adjust_prio_chain(task, 0, NULL, NULL, task);
  502. }
  503. /*
  504. * Slow path lock function:
  505. */
  506. static int __sched
  507. rt_mutex_slowlock(struct rt_mutex *lock, int state,
  508. struct hrtimer_sleeper *timeout,
  509. int detect_deadlock)
  510. {
  511. struct rt_mutex_waiter waiter;
  512. int ret = 0;
  513. debug_rt_mutex_init_waiter(&waiter);
  514. waiter.task = NULL;
  515. spin_lock(&lock->wait_lock);
  516. /* Try to acquire the lock again: */
  517. if (try_to_take_rt_mutex(lock)) {
  518. spin_unlock(&lock->wait_lock);
  519. return 0;
  520. }
  521. set_current_state(state);
  522. /* Setup the timer, when timeout != NULL */
  523. if (unlikely(timeout))
  524. hrtimer_start(&timeout->timer, timeout->timer.expires,
  525. HRTIMER_MODE_ABS);
  526. for (;;) {
  527. /* Try to acquire the lock: */
  528. if (try_to_take_rt_mutex(lock))
  529. break;
  530. /*
  531. * TASK_INTERRUPTIBLE checks for signals and
  532. * timeout. Ignored otherwise.
  533. */
  534. if (unlikely(state == TASK_INTERRUPTIBLE)) {
  535. /* Signal pending? */
  536. if (signal_pending(current))
  537. ret = -EINTR;
  538. if (timeout && !timeout->task)
  539. ret = -ETIMEDOUT;
  540. if (ret)
  541. break;
  542. }
  543. /*
  544. * waiter.task is NULL the first time we come here and
  545. * when we have been woken up by the previous owner
  546. * but the lock got stolen by a higher prio task.
  547. */
  548. if (!waiter.task) {
  549. ret = task_blocks_on_rt_mutex(lock, &waiter,
  550. detect_deadlock);
  551. /*
  552. * If we got woken up by the owner then start loop
  553. * all over without going into schedule to try
  554. * to get the lock now:
  555. */
  556. if (unlikely(!waiter.task))
  557. continue;
  558. if (unlikely(ret))
  559. break;
  560. }
  561. spin_unlock(&lock->wait_lock);
  562. debug_rt_mutex_print_deadlock(&waiter);
  563. if (waiter.task)
  564. schedule_rt_mutex(lock);
  565. spin_lock(&lock->wait_lock);
  566. set_current_state(state);
  567. }
  568. set_current_state(TASK_RUNNING);
  569. if (unlikely(waiter.task))
  570. remove_waiter(lock, &waiter);
  571. /*
  572. * try_to_take_rt_mutex() sets the waiter bit
  573. * unconditionally. We might have to fix that up.
  574. */
  575. fixup_rt_mutex_waiters(lock);
  576. spin_unlock(&lock->wait_lock);
  577. /* Remove pending timer: */
  578. if (unlikely(timeout))
  579. hrtimer_cancel(&timeout->timer);
  580. /*
  581. * Readjust priority, when we did not get the lock. We might
  582. * have been the pending owner and boosted. Since we did not
  583. * take the lock, the PI boost has to go.
  584. */
  585. if (unlikely(ret))
  586. rt_mutex_adjust_prio(current);
  587. debug_rt_mutex_free_waiter(&waiter);
  588. return ret;
  589. }
  590. /*
  591. * Slow path try-lock function:
  592. */
  593. static inline int
  594. rt_mutex_slowtrylock(struct rt_mutex *lock)
  595. {
  596. int ret = 0;
  597. spin_lock(&lock->wait_lock);
  598. if (likely(rt_mutex_owner(lock) != current)) {
  599. ret = try_to_take_rt_mutex(lock);
  600. /*
  601. * try_to_take_rt_mutex() sets the lock waiters
  602. * bit unconditionally. Clean this up.
  603. */
  604. fixup_rt_mutex_waiters(lock);
  605. }
  606. spin_unlock(&lock->wait_lock);
  607. return ret;
  608. }
  609. /*
  610. * Slow path to release a rt-mutex:
  611. */
  612. static void __sched
  613. rt_mutex_slowunlock(struct rt_mutex *lock)
  614. {
  615. spin_lock(&lock->wait_lock);
  616. debug_rt_mutex_unlock(lock);
  617. rt_mutex_deadlock_account_unlock(current);
  618. if (!rt_mutex_has_waiters(lock)) {
  619. lock->owner = NULL;
  620. spin_unlock(&lock->wait_lock);
  621. return;
  622. }
  623. wakeup_next_waiter(lock);
  624. spin_unlock(&lock->wait_lock);
  625. /* Undo pi boosting if necessary: */
  626. rt_mutex_adjust_prio(current);
  627. }
  628. /*
  629. * debug aware fast / slowpath lock,trylock,unlock
  630. *
  631. * The atomic acquire/release ops are compiled away, when either the
  632. * architecture does not support cmpxchg or when debugging is enabled.
  633. */
  634. static inline int
  635. rt_mutex_fastlock(struct rt_mutex *lock, int state,
  636. int detect_deadlock,
  637. int (*slowfn)(struct rt_mutex *lock, int state,
  638. struct hrtimer_sleeper *timeout,
  639. int detect_deadlock))
  640. {
  641. if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
  642. rt_mutex_deadlock_account_lock(lock, current);
  643. return 0;
  644. } else
  645. return slowfn(lock, state, NULL, detect_deadlock);
  646. }
  647. static inline int
  648. rt_mutex_timed_fastlock(struct rt_mutex *lock, int state,
  649. struct hrtimer_sleeper *timeout, int detect_deadlock,
  650. int (*slowfn)(struct rt_mutex *lock, int state,
  651. struct hrtimer_sleeper *timeout,
  652. int detect_deadlock))
  653. {
  654. if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
  655. rt_mutex_deadlock_account_lock(lock, current);
  656. return 0;
  657. } else
  658. return slowfn(lock, state, timeout, detect_deadlock);
  659. }
  660. static inline int
  661. rt_mutex_fasttrylock(struct rt_mutex *lock,
  662. int (*slowfn)(struct rt_mutex *lock))
  663. {
  664. if (likely(rt_mutex_cmpxchg(lock, NULL, current))) {
  665. rt_mutex_deadlock_account_lock(lock, current);
  666. return 1;
  667. }
  668. return slowfn(lock);
  669. }
  670. static inline void
  671. rt_mutex_fastunlock(struct rt_mutex *lock,
  672. void (*slowfn)(struct rt_mutex *lock))
  673. {
  674. if (likely(rt_mutex_cmpxchg(lock, current, NULL)))
  675. rt_mutex_deadlock_account_unlock(current);
  676. else
  677. slowfn(lock);
  678. }
  679. /**
  680. * rt_mutex_lock - lock a rt_mutex
  681. *
  682. * @lock: the rt_mutex to be locked
  683. */
  684. void __sched rt_mutex_lock(struct rt_mutex *lock)
  685. {
  686. might_sleep();
  687. rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, 0, rt_mutex_slowlock);
  688. }
  689. EXPORT_SYMBOL_GPL(rt_mutex_lock);
  690. /**
  691. * rt_mutex_lock_interruptible - lock a rt_mutex interruptible
  692. *
  693. * @lock: the rt_mutex to be locked
  694. * @detect_deadlock: deadlock detection on/off
  695. *
  696. * Returns:
  697. * 0 on success
  698. * -EINTR when interrupted by a signal
  699. * -EDEADLK when the lock would deadlock (when deadlock detection is on)
  700. */
  701. int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock,
  702. int detect_deadlock)
  703. {
  704. might_sleep();
  705. return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE,
  706. detect_deadlock, rt_mutex_slowlock);
  707. }
  708. EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
  709. /**
  710. * rt_mutex_lock_interruptible_ktime - lock a rt_mutex interruptible
  711. * the timeout structure is provided
  712. * by the caller
  713. *
  714. * @lock: the rt_mutex to be locked
  715. * @timeout: timeout structure or NULL (no timeout)
  716. * @detect_deadlock: deadlock detection on/off
  717. *
  718. * Returns:
  719. * 0 on success
  720. * -EINTR when interrupted by a signal
  721. * -ETIMEOUT when the timeout expired
  722. * -EDEADLK when the lock would deadlock (when deadlock detection is on)
  723. */
  724. int
  725. rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout,
  726. int detect_deadlock)
  727. {
  728. might_sleep();
  729. return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
  730. detect_deadlock, rt_mutex_slowlock);
  731. }
  732. EXPORT_SYMBOL_GPL(rt_mutex_timed_lock);
  733. /**
  734. * rt_mutex_trylock - try to lock a rt_mutex
  735. *
  736. * @lock: the rt_mutex to be locked
  737. *
  738. * Returns 1 on success and 0 on contention
  739. */
  740. int __sched rt_mutex_trylock(struct rt_mutex *lock)
  741. {
  742. return rt_mutex_fasttrylock(lock, rt_mutex_slowtrylock);
  743. }
  744. EXPORT_SYMBOL_GPL(rt_mutex_trylock);
  745. /**
  746. * rt_mutex_unlock - unlock a rt_mutex
  747. *
  748. * @lock: the rt_mutex to be unlocked
  749. */
  750. void __sched rt_mutex_unlock(struct rt_mutex *lock)
  751. {
  752. rt_mutex_fastunlock(lock, rt_mutex_slowunlock);
  753. }
  754. EXPORT_SYMBOL_GPL(rt_mutex_unlock);
  755. /***
  756. * rt_mutex_destroy - mark a mutex unusable
  757. * @lock: the mutex to be destroyed
  758. *
  759. * This function marks the mutex uninitialized, and any subsequent
  760. * use of the mutex is forbidden. The mutex must not be locked when
  761. * this function is called.
  762. */
  763. void rt_mutex_destroy(struct rt_mutex *lock)
  764. {
  765. WARN_ON(rt_mutex_is_locked(lock));
  766. #ifdef CONFIG_DEBUG_RT_MUTEXES
  767. lock->magic = NULL;
  768. #endif
  769. }
  770. EXPORT_SYMBOL_GPL(rt_mutex_destroy);
  771. /**
  772. * __rt_mutex_init - initialize the rt lock
  773. *
  774. * @lock: the rt lock to be initialized
  775. *
  776. * Initialize the rt lock to unlocked state.
  777. *
  778. * Initializing of a locked rt lock is not allowed
  779. */
  780. void __rt_mutex_init(struct rt_mutex *lock, const char *name)
  781. {
  782. lock->owner = NULL;
  783. spin_lock_init(&lock->wait_lock);
  784. plist_head_init(&lock->wait_list, &lock->wait_lock);
  785. debug_rt_mutex_init(lock, name);
  786. }
  787. EXPORT_SYMBOL_GPL(__rt_mutex_init);
  788. /**
  789. * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a
  790. * proxy owner
  791. *
  792. * @lock: the rt_mutex to be locked
  793. * @proxy_owner:the task to set as owner
  794. *
  795. * No locking. Caller has to do serializing itself
  796. * Special API call for PI-futex support
  797. */
  798. void rt_mutex_init_proxy_locked(struct rt_mutex *lock,
  799. struct task_struct *proxy_owner)
  800. {
  801. __rt_mutex_init(lock, NULL);
  802. debug_rt_mutex_proxy_lock(lock, proxy_owner);
  803. rt_mutex_set_owner(lock, proxy_owner, 0);
  804. rt_mutex_deadlock_account_lock(lock, proxy_owner);
  805. }
  806. /**
  807. * rt_mutex_proxy_unlock - release a lock on behalf of owner
  808. *
  809. * @lock: the rt_mutex to be locked
  810. *
  811. * No locking. Caller has to do serializing itself
  812. * Special API call for PI-futex support
  813. */
  814. void rt_mutex_proxy_unlock(struct rt_mutex *lock,
  815. struct task_struct *proxy_owner)
  816. {
  817. debug_rt_mutex_proxy_unlock(lock);
  818. rt_mutex_set_owner(lock, NULL, 0);
  819. rt_mutex_deadlock_account_unlock(proxy_owner);
  820. }
  821. /**
  822. * rt_mutex_next_owner - return the next owner of the lock
  823. *
  824. * @lock: the rt lock query
  825. *
  826. * Returns the next owner of the lock or NULL
  827. *
  828. * Caller has to serialize against other accessors to the lock
  829. * itself.
  830. *
  831. * Special API call for PI-futex support
  832. */
  833. struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock)
  834. {
  835. if (!rt_mutex_has_waiters(lock))
  836. return NULL;
  837. return rt_mutex_top_waiter(lock)->task;
  838. }