mutex.c 38 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * kernel/locking/mutex.c
  4. *
  5. * Mutexes: blocking mutual exclusion locks
  6. *
  7. * Started by Ingo Molnar:
  8. *
  9. * Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
  10. *
  11. * Many thanks to Arjan van de Ven, Thomas Gleixner, Steven Rostedt and
  12. * David Howells for suggestions and improvements.
  13. *
  14. * - Adaptive spinning for mutexes by Peter Zijlstra. (Ported to mainline
  15. * from the -rt tree, where it was originally implemented for rtmutexes
  16. * by Steven Rostedt, based on work by Gregory Haskins, Peter Morreale
  17. * and Sven Dietrich.
  18. *
  19. * Also see Documentation/locking/mutex-design.rst.
  20. */
  21. #include <linux/mutex.h>
  22. #include <linux/ww_mutex.h>
  23. #include <linux/sched/signal.h>
  24. #include <linux/sched/rt.h>
  25. #include <linux/sched/wake_q.h>
  26. #include <linux/sched/debug.h>
  27. #include <linux/export.h>
  28. #include <linux/spinlock.h>
  29. #include <linux/interrupt.h>
  30. #include <linux/debug_locks.h>
  31. #include <linux/osq_lock.h>
  32. #ifdef CONFIG_DEBUG_MUTEXES
  33. # include "mutex-debug.h"
  34. #else
  35. # include "mutex.h"
  36. #endif
  37. #include <trace/hooks/dtask.h>
  38. void
  39. __mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key)
  40. {
  41. atomic_long_set(&lock->owner, 0);
  42. spin_lock_init(&lock->wait_lock);
  43. INIT_LIST_HEAD(&lock->wait_list);
  44. #ifdef CONFIG_MUTEX_SPIN_ON_OWNER
  45. osq_lock_init(&lock->osq);
  46. #endif
  47. debug_mutex_init(lock, name, key);
  48. }
  49. EXPORT_SYMBOL(__mutex_init);
  50. /*
  51. * @owner: contains: 'struct task_struct *' to the current lock owner,
  52. * NULL means not owned. Since task_struct pointers are aligned at
  53. * at least L1_CACHE_BYTES, we have low bits to store extra state.
  54. *
  55. * Bit0 indicates a non-empty waiter list; unlock must issue a wakeup.
  56. * Bit1 indicates unlock needs to hand the lock to the top-waiter
  57. * Bit2 indicates handoff has been done and we're waiting for pickup.
  58. */
  59. #define MUTEX_FLAG_WAITERS 0x01
  60. #define MUTEX_FLAG_HANDOFF 0x02
  61. #define MUTEX_FLAG_PICKUP 0x04
  62. #define MUTEX_FLAGS 0x07
  63. /*
  64. * Internal helper function; C doesn't allow us to hide it :/
  65. *
  66. * DO NOT USE (outside of mutex code).
  67. */
  68. static inline struct task_struct *__mutex_owner(struct mutex *lock)
  69. {
  70. return (struct task_struct *)(atomic_long_read(&lock->owner) & ~MUTEX_FLAGS);
  71. }
  72. static inline struct task_struct *__owner_task(unsigned long owner)
  73. {
  74. return (struct task_struct *)(owner & ~MUTEX_FLAGS);
  75. }
  76. bool mutex_is_locked(struct mutex *lock)
  77. {
  78. return __mutex_owner(lock) != NULL;
  79. }
  80. EXPORT_SYMBOL(mutex_is_locked);
  81. __must_check enum mutex_trylock_recursive_enum
  82. mutex_trylock_recursive(struct mutex *lock)
  83. {
  84. if (unlikely(__mutex_owner(lock) == current))
  85. return MUTEX_TRYLOCK_RECURSIVE;
  86. return mutex_trylock(lock);
  87. }
  88. EXPORT_SYMBOL(mutex_trylock_recursive);
  89. static inline unsigned long __owner_flags(unsigned long owner)
  90. {
  91. return owner & MUTEX_FLAGS;
  92. }
  93. /*
  94. * Trylock variant that retuns the owning task on failure.
  95. */
  96. static inline struct task_struct *__mutex_trylock_or_owner(struct mutex *lock)
  97. {
  98. unsigned long owner, curr = (unsigned long)current;
  99. owner = atomic_long_read(&lock->owner);
  100. for (;;) { /* must loop, can race against a flag */
  101. unsigned long old, flags = __owner_flags(owner);
  102. unsigned long task = owner & ~MUTEX_FLAGS;
  103. if (task) {
  104. if (likely(task != curr))
  105. break;
  106. if (likely(!(flags & MUTEX_FLAG_PICKUP)))
  107. break;
  108. flags &= ~MUTEX_FLAG_PICKUP;
  109. } else {
  110. #ifdef CONFIG_DEBUG_MUTEXES
  111. DEBUG_LOCKS_WARN_ON(flags & MUTEX_FLAG_PICKUP);
  112. #endif
  113. }
  114. /*
  115. * We set the HANDOFF bit, we must make sure it doesn't live
  116. * past the point where we acquire it. This would be possible
  117. * if we (accidentally) set the bit on an unlocked mutex.
  118. */
  119. flags &= ~MUTEX_FLAG_HANDOFF;
  120. old = atomic_long_cmpxchg_acquire(&lock->owner, owner, curr | flags);
  121. if (old == owner)
  122. return NULL;
  123. owner = old;
  124. }
  125. return __owner_task(owner);
  126. }
  127. /*
  128. * Actual trylock that will work on any unlocked state.
  129. */
  130. static inline bool __mutex_trylock(struct mutex *lock)
  131. {
  132. return !__mutex_trylock_or_owner(lock);
  133. }
  134. #ifndef CONFIG_DEBUG_LOCK_ALLOC
  135. /*
  136. * Lockdep annotations are contained to the slow paths for simplicity.
  137. * There is nothing that would stop spreading the lockdep annotations outwards
  138. * except more code.
  139. */
  140. /*
  141. * Optimistic trylock that only works in the uncontended case. Make sure to
  142. * follow with a __mutex_trylock() before failing.
  143. */
  144. static __always_inline bool __mutex_trylock_fast(struct mutex *lock)
  145. {
  146. unsigned long curr = (unsigned long)current;
  147. unsigned long zero = 0UL;
  148. if (atomic_long_try_cmpxchg_acquire(&lock->owner, &zero, curr))
  149. return true;
  150. return false;
  151. }
  152. static __always_inline bool __mutex_unlock_fast(struct mutex *lock)
  153. {
  154. unsigned long curr = (unsigned long)current;
  155. if (atomic_long_cmpxchg_release(&lock->owner, curr, 0UL) == curr)
  156. return true;
  157. return false;
  158. }
  159. #endif
  160. static inline void __mutex_set_flag(struct mutex *lock, unsigned long flag)
  161. {
  162. atomic_long_or(flag, &lock->owner);
  163. }
  164. static inline void __mutex_clear_flag(struct mutex *lock, unsigned long flag)
  165. {
  166. atomic_long_andnot(flag, &lock->owner);
  167. }
  168. static inline bool __mutex_waiter_is_first(struct mutex *lock, struct mutex_waiter *waiter)
  169. {
  170. return list_first_entry(&lock->wait_list, struct mutex_waiter, list) == waiter;
  171. }
  172. /*
  173. * Add @waiter to a given location in the lock wait_list and set the
  174. * FLAG_WAITERS flag if it's the first waiter.
  175. */
  176. static void
  177. __mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,
  178. struct list_head *list)
  179. {
  180. bool already_on_list = false;
  181. debug_mutex_add_waiter(lock, waiter, current);
  182. trace_android_vh_alter_mutex_list_add(lock, waiter, list, &already_on_list);
  183. if (!already_on_list)
  184. list_add_tail(&waiter->list, list);
  185. if (__mutex_waiter_is_first(lock, waiter))
  186. __mutex_set_flag(lock, MUTEX_FLAG_WAITERS);
  187. }
  188. static void
  189. __mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter)
  190. {
  191. list_del(&waiter->list);
  192. if (likely(list_empty(&lock->wait_list)))
  193. __mutex_clear_flag(lock, MUTEX_FLAGS);
  194. debug_mutex_remove_waiter(lock, waiter, current);
  195. }
  196. /*
  197. * Give up ownership to a specific task, when @task = NULL, this is equivalent
  198. * to a regular unlock. Sets PICKUP on a handoff, clears HANDOF, preserves
  199. * WAITERS. Provides RELEASE semantics like a regular unlock, the
  200. * __mutex_trylock() provides a matching ACQUIRE semantics for the handoff.
  201. */
  202. static void __mutex_handoff(struct mutex *lock, struct task_struct *task)
  203. {
  204. unsigned long owner = atomic_long_read(&lock->owner);
  205. for (;;) {
  206. unsigned long old, new;
  207. #ifdef CONFIG_DEBUG_MUTEXES
  208. DEBUG_LOCKS_WARN_ON(__owner_task(owner) != current);
  209. DEBUG_LOCKS_WARN_ON(owner & MUTEX_FLAG_PICKUP);
  210. #endif
  211. new = (owner & MUTEX_FLAG_WAITERS);
  212. new |= (unsigned long)task;
  213. if (task)
  214. new |= MUTEX_FLAG_PICKUP;
  215. old = atomic_long_cmpxchg_release(&lock->owner, owner, new);
  216. if (old == owner)
  217. break;
  218. owner = old;
  219. }
  220. }
  221. #ifndef CONFIG_DEBUG_LOCK_ALLOC
  222. /*
  223. * We split the mutex lock/unlock logic into separate fastpath and
  224. * slowpath functions, to reduce the register pressure on the fastpath.
  225. * We also put the fastpath first in the kernel image, to make sure the
  226. * branch is predicted by the CPU as default-untaken.
  227. */
  228. static void __sched __mutex_lock_slowpath(struct mutex *lock);
  229. /**
  230. * mutex_lock - acquire the mutex
  231. * @lock: the mutex to be acquired
  232. *
  233. * Lock the mutex exclusively for this task. If the mutex is not
  234. * available right now, it will sleep until it can get it.
  235. *
  236. * The mutex must later on be released by the same task that
  237. * acquired it. Recursive locking is not allowed. The task
  238. * may not exit without first unlocking the mutex. Also, kernel
  239. * memory where the mutex resides must not be freed with
  240. * the mutex still locked. The mutex must first be initialized
  241. * (or statically defined) before it can be locked. memset()-ing
  242. * the mutex to 0 is not allowed.
  243. *
  244. * (The CONFIG_DEBUG_MUTEXES .config option turns on debugging
  245. * checks that will enforce the restrictions and will also do
  246. * deadlock debugging)
  247. *
  248. * This function is similar to (but not equivalent to) down().
  249. */
  250. void __sched mutex_lock(struct mutex *lock)
  251. {
  252. might_sleep();
  253. if (!__mutex_trylock_fast(lock))
  254. __mutex_lock_slowpath(lock);
  255. }
  256. EXPORT_SYMBOL(mutex_lock);
  257. #endif
  258. /*
  259. * Wait-Die:
  260. * The newer transactions are killed when:
  261. * It (the new transaction) makes a request for a lock being held
  262. * by an older transaction.
  263. *
  264. * Wound-Wait:
  265. * The newer transactions are wounded when:
  266. * An older transaction makes a request for a lock being held by
  267. * the newer transaction.
  268. */
  269. /*
  270. * Associate the ww_mutex @ww with the context @ww_ctx under which we acquired
  271. * it.
  272. */
  273. static __always_inline void
  274. ww_mutex_lock_acquired(struct ww_mutex *ww, struct ww_acquire_ctx *ww_ctx)
  275. {
  276. #ifdef CONFIG_DEBUG_MUTEXES
  277. /*
  278. * If this WARN_ON triggers, you used ww_mutex_lock to acquire,
  279. * but released with a normal mutex_unlock in this call.
  280. *
  281. * This should never happen, always use ww_mutex_unlock.
  282. */
  283. DEBUG_LOCKS_WARN_ON(ww->ctx);
  284. /*
  285. * Not quite done after calling ww_acquire_done() ?
  286. */
  287. DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire);
  288. if (ww_ctx->contending_lock) {
  289. /*
  290. * After -EDEADLK you tried to
  291. * acquire a different ww_mutex? Bad!
  292. */
  293. DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww);
  294. /*
  295. * You called ww_mutex_lock after receiving -EDEADLK,
  296. * but 'forgot' to unlock everything else first?
  297. */
  298. DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0);
  299. ww_ctx->contending_lock = NULL;
  300. }
  301. /*
  302. * Naughty, using a different class will lead to undefined behavior!
  303. */
  304. DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class);
  305. #endif
  306. ww_ctx->acquired++;
  307. ww->ctx = ww_ctx;
  308. }
  309. /*
  310. * Determine if context @a is 'after' context @b. IOW, @a is a younger
  311. * transaction than @b and depending on algorithm either needs to wait for
  312. * @b or die.
  313. */
  314. static inline bool __sched
  315. __ww_ctx_stamp_after(struct ww_acquire_ctx *a, struct ww_acquire_ctx *b)
  316. {
  317. return (signed long)(a->stamp - b->stamp) > 0;
  318. }
  319. /*
  320. * Wait-Die; wake a younger waiter context (when locks held) such that it can
  321. * die.
  322. *
  323. * Among waiters with context, only the first one can have other locks acquired
  324. * already (ctx->acquired > 0), because __ww_mutex_add_waiter() and
  325. * __ww_mutex_check_kill() wake any but the earliest context.
  326. */
  327. static bool __sched
  328. __ww_mutex_die(struct mutex *lock, struct mutex_waiter *waiter,
  329. struct ww_acquire_ctx *ww_ctx)
  330. {
  331. if (!ww_ctx->is_wait_die)
  332. return false;
  333. if (waiter->ww_ctx->acquired > 0 &&
  334. __ww_ctx_stamp_after(waiter->ww_ctx, ww_ctx)) {
  335. debug_mutex_wake_waiter(lock, waiter);
  336. wake_up_process(waiter->task);
  337. }
  338. return true;
  339. }
  340. /*
  341. * Wound-Wait; wound a younger @hold_ctx if it holds the lock.
  342. *
  343. * Wound the lock holder if there are waiters with older transactions than
  344. * the lock holders. Even if multiple waiters may wound the lock holder,
  345. * it's sufficient that only one does.
  346. */
  347. static bool __ww_mutex_wound(struct mutex *lock,
  348. struct ww_acquire_ctx *ww_ctx,
  349. struct ww_acquire_ctx *hold_ctx)
  350. {
  351. struct task_struct *owner = __mutex_owner(lock);
  352. lockdep_assert_held(&lock->wait_lock);
  353. /*
  354. * Possible through __ww_mutex_add_waiter() when we race with
  355. * ww_mutex_set_context_fastpath(). In that case we'll get here again
  356. * through __ww_mutex_check_waiters().
  357. */
  358. if (!hold_ctx)
  359. return false;
  360. /*
  361. * Can have !owner because of __mutex_unlock_slowpath(), but if owner,
  362. * it cannot go away because we'll have FLAG_WAITERS set and hold
  363. * wait_lock.
  364. */
  365. if (!owner)
  366. return false;
  367. if (ww_ctx->acquired > 0 && __ww_ctx_stamp_after(hold_ctx, ww_ctx)) {
  368. hold_ctx->wounded = 1;
  369. /*
  370. * wake_up_process() paired with set_current_state()
  371. * inserts sufficient barriers to make sure @owner either sees
  372. * it's wounded in __ww_mutex_check_kill() or has a
  373. * wakeup pending to re-read the wounded state.
  374. */
  375. if (owner != current)
  376. wake_up_process(owner);
  377. return true;
  378. }
  379. return false;
  380. }
  381. /*
  382. * We just acquired @lock under @ww_ctx, if there are later contexts waiting
  383. * behind us on the wait-list, check if they need to die, or wound us.
  384. *
  385. * See __ww_mutex_add_waiter() for the list-order construction; basically the
  386. * list is ordered by stamp, smallest (oldest) first.
  387. *
  388. * This relies on never mixing wait-die/wound-wait on the same wait-list;
  389. * which is currently ensured by that being a ww_class property.
  390. *
  391. * The current task must not be on the wait list.
  392. */
  393. static void __sched
  394. __ww_mutex_check_waiters(struct mutex *lock, struct ww_acquire_ctx *ww_ctx)
  395. {
  396. struct mutex_waiter *cur;
  397. lockdep_assert_held(&lock->wait_lock);
  398. list_for_each_entry(cur, &lock->wait_list, list) {
  399. if (!cur->ww_ctx)
  400. continue;
  401. if (__ww_mutex_die(lock, cur, ww_ctx) ||
  402. __ww_mutex_wound(lock, cur->ww_ctx, ww_ctx))
  403. break;
  404. }
  405. }
  406. /*
  407. * After acquiring lock with fastpath, where we do not hold wait_lock, set ctx
  408. * and wake up any waiters so they can recheck.
  409. */
  410. static __always_inline void
  411. ww_mutex_set_context_fastpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
  412. {
  413. ww_mutex_lock_acquired(lock, ctx);
  414. /*
  415. * The lock->ctx update should be visible on all cores before
  416. * the WAITERS check is done, otherwise contended waiters might be
  417. * missed. The contended waiters will either see ww_ctx == NULL
  418. * and keep spinning, or it will acquire wait_lock, add itself
  419. * to waiter list and sleep.
  420. */
  421. smp_mb(); /* See comments above and below. */
  422. /*
  423. * [W] ww->ctx = ctx [W] MUTEX_FLAG_WAITERS
  424. * MB MB
  425. * [R] MUTEX_FLAG_WAITERS [R] ww->ctx
  426. *
  427. * The memory barrier above pairs with the memory barrier in
  428. * __ww_mutex_add_waiter() and makes sure we either observe ww->ctx
  429. * and/or !empty list.
  430. */
  431. if (likely(!(atomic_long_read(&lock->base.owner) & MUTEX_FLAG_WAITERS)))
  432. return;
  433. /*
  434. * Uh oh, we raced in fastpath, check if any of the waiters need to
  435. * die or wound us.
  436. */
  437. spin_lock(&lock->base.wait_lock);
  438. __ww_mutex_check_waiters(&lock->base, ctx);
  439. spin_unlock(&lock->base.wait_lock);
  440. }
  441. #ifdef CONFIG_MUTEX_SPIN_ON_OWNER
  442. static inline
  443. bool ww_mutex_spin_on_owner(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
  444. struct mutex_waiter *waiter)
  445. {
  446. struct ww_mutex *ww;
  447. ww = container_of(lock, struct ww_mutex, base);
  448. /*
  449. * If ww->ctx is set the contents are undefined, only
  450. * by acquiring wait_lock there is a guarantee that
  451. * they are not invalid when reading.
  452. *
  453. * As such, when deadlock detection needs to be
  454. * performed the optimistic spinning cannot be done.
  455. *
  456. * Check this in every inner iteration because we may
  457. * be racing against another thread's ww_mutex_lock.
  458. */
  459. if (ww_ctx->acquired > 0 && READ_ONCE(ww->ctx))
  460. return false;
  461. /*
  462. * If we aren't on the wait list yet, cancel the spin
  463. * if there are waiters. We want to avoid stealing the
  464. * lock from a waiter with an earlier stamp, since the
  465. * other thread may already own a lock that we also
  466. * need.
  467. */
  468. if (!waiter && (atomic_long_read(&lock->owner) & MUTEX_FLAG_WAITERS))
  469. return false;
  470. /*
  471. * Similarly, stop spinning if we are no longer the
  472. * first waiter.
  473. */
  474. if (waiter && !__mutex_waiter_is_first(lock, waiter))
  475. return false;
  476. return true;
  477. }
  478. /*
  479. * Look out! "owner" is an entirely speculative pointer access and not
  480. * reliable.
  481. *
  482. * "noinline" so that this function shows up on perf profiles.
  483. */
  484. static noinline
  485. bool mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner,
  486. struct ww_acquire_ctx *ww_ctx, struct mutex_waiter *waiter)
  487. {
  488. bool ret = true;
  489. rcu_read_lock();
  490. while (__mutex_owner(lock) == owner) {
  491. /*
  492. * Ensure we emit the owner->on_cpu, dereference _after_
  493. * checking lock->owner still matches owner. If that fails,
  494. * owner might point to freed memory. If it still matches,
  495. * the rcu_read_lock() ensures the memory stays valid.
  496. */
  497. barrier();
  498. /*
  499. * Use vcpu_is_preempted to detect lock holder preemption issue.
  500. */
  501. if (!owner->on_cpu || need_resched() ||
  502. vcpu_is_preempted(task_cpu(owner))) {
  503. ret = false;
  504. break;
  505. }
  506. if (ww_ctx && !ww_mutex_spin_on_owner(lock, ww_ctx, waiter)) {
  507. ret = false;
  508. break;
  509. }
  510. cpu_relax();
  511. }
  512. rcu_read_unlock();
  513. return ret;
  514. }
  515. /*
  516. * Initial check for entering the mutex spinning loop
  517. */
  518. static inline int mutex_can_spin_on_owner(struct mutex *lock)
  519. {
  520. struct task_struct *owner;
  521. int retval = 1;
  522. if (need_resched())
  523. return 0;
  524. rcu_read_lock();
  525. owner = __mutex_owner(lock);
  526. /*
  527. * As lock holder preemption issue, we both skip spinning if task is not
  528. * on cpu or its cpu is preempted
  529. */
  530. if (owner)
  531. retval = owner->on_cpu && !vcpu_is_preempted(task_cpu(owner));
  532. rcu_read_unlock();
  533. /*
  534. * If lock->owner is not set, the mutex has been released. Return true
  535. * such that we'll trylock in the spin path, which is a faster option
  536. * than the blocking slow path.
  537. */
  538. return retval;
  539. }
  540. /*
  541. * Optimistic spinning.
  542. *
  543. * We try to spin for acquisition when we find that the lock owner
  544. * is currently running on a (different) CPU and while we don't
  545. * need to reschedule. The rationale is that if the lock owner is
  546. * running, it is likely to release the lock soon.
  547. *
  548. * The mutex spinners are queued up using MCS lock so that only one
  549. * spinner can compete for the mutex. However, if mutex spinning isn't
  550. * going to happen, there is no point in going through the lock/unlock
  551. * overhead.
  552. *
  553. * Returns true when the lock was taken, otherwise false, indicating
  554. * that we need to jump to the slowpath and sleep.
  555. *
  556. * The waiter flag is set to true if the spinner is a waiter in the wait
  557. * queue. The waiter-spinner will spin on the lock directly and concurrently
  558. * with the spinner at the head of the OSQ, if present, until the owner is
  559. * changed to itself.
  560. */
  561. static __always_inline bool
  562. mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
  563. struct mutex_waiter *waiter)
  564. {
  565. if (!waiter) {
  566. /*
  567. * The purpose of the mutex_can_spin_on_owner() function is
  568. * to eliminate the overhead of osq_lock() and osq_unlock()
  569. * in case spinning isn't possible. As a waiter-spinner
  570. * is not going to take OSQ lock anyway, there is no need
  571. * to call mutex_can_spin_on_owner().
  572. */
  573. if (!mutex_can_spin_on_owner(lock))
  574. goto fail;
  575. /*
  576. * In order to avoid a stampede of mutex spinners trying to
  577. * acquire the mutex all at once, the spinners need to take a
  578. * MCS (queued) lock first before spinning on the owner field.
  579. */
  580. if (!osq_lock(&lock->osq))
  581. goto fail;
  582. }
  583. for (;;) {
  584. struct task_struct *owner;
  585. /* Try to acquire the mutex... */
  586. owner = __mutex_trylock_or_owner(lock);
  587. if (!owner)
  588. break;
  589. /*
  590. * There's an owner, wait for it to either
  591. * release the lock or go to sleep.
  592. */
  593. if (!mutex_spin_on_owner(lock, owner, ww_ctx, waiter))
  594. goto fail_unlock;
  595. /*
  596. * The cpu_relax() call is a compiler barrier which forces
  597. * everything in this loop to be re-loaded. We don't need
  598. * memory barriers as we'll eventually observe the right
  599. * values at the cost of a few extra spins.
  600. */
  601. cpu_relax();
  602. }
  603. if (!waiter)
  604. osq_unlock(&lock->osq);
  605. return true;
  606. fail_unlock:
  607. if (!waiter)
  608. osq_unlock(&lock->osq);
  609. fail:
  610. /*
  611. * If we fell out of the spin path because of need_resched(),
  612. * reschedule now, before we try-lock the mutex. This avoids getting
  613. * scheduled out right after we obtained the mutex.
  614. */
  615. if (need_resched()) {
  616. /*
  617. * We _should_ have TASK_RUNNING here, but just in case
  618. * we do not, make it so, otherwise we might get stuck.
  619. */
  620. __set_current_state(TASK_RUNNING);
  621. schedule_preempt_disabled();
  622. }
  623. return false;
  624. }
  625. #else
  626. static __always_inline bool
  627. mutex_optimistic_spin(struct mutex *lock, struct ww_acquire_ctx *ww_ctx,
  628. struct mutex_waiter *waiter)
  629. {
  630. return false;
  631. }
  632. #endif
  633. static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip);
  634. /**
  635. * mutex_unlock - release the mutex
  636. * @lock: the mutex to be released
  637. *
  638. * Unlock a mutex that has been locked by this task previously.
  639. *
  640. * This function must not be used in interrupt context. Unlocking
  641. * of a not locked mutex is not allowed.
  642. *
  643. * This function is similar to (but not equivalent to) up().
  644. */
  645. void __sched mutex_unlock(struct mutex *lock)
  646. {
  647. #ifndef CONFIG_DEBUG_LOCK_ALLOC
  648. if (__mutex_unlock_fast(lock))
  649. return;
  650. #endif
  651. __mutex_unlock_slowpath(lock, _RET_IP_);
  652. }
  653. EXPORT_SYMBOL(mutex_unlock);
  654. /**
  655. * ww_mutex_unlock - release the w/w mutex
  656. * @lock: the mutex to be released
  657. *
  658. * Unlock a mutex that has been locked by this task previously with any of the
  659. * ww_mutex_lock* functions (with or without an acquire context). It is
  660. * forbidden to release the locks after releasing the acquire context.
  661. *
  662. * This function must not be used in interrupt context. Unlocking
  663. * of a unlocked mutex is not allowed.
  664. */
  665. void __sched ww_mutex_unlock(struct ww_mutex *lock)
  666. {
  667. /*
  668. * The unlocking fastpath is the 0->1 transition from 'locked'
  669. * into 'unlocked' state:
  670. */
  671. if (lock->ctx) {
  672. #ifdef CONFIG_DEBUG_MUTEXES
  673. DEBUG_LOCKS_WARN_ON(!lock->ctx->acquired);
  674. #endif
  675. if (lock->ctx->acquired > 0)
  676. lock->ctx->acquired--;
  677. lock->ctx = NULL;
  678. }
  679. mutex_unlock(&lock->base);
  680. }
  681. EXPORT_SYMBOL(ww_mutex_unlock);
  682. static __always_inline int __sched
  683. __ww_mutex_kill(struct mutex *lock, struct ww_acquire_ctx *ww_ctx)
  684. {
  685. if (ww_ctx->acquired > 0) {
  686. #ifdef CONFIG_DEBUG_MUTEXES
  687. struct ww_mutex *ww;
  688. ww = container_of(lock, struct ww_mutex, base);
  689. DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock);
  690. ww_ctx->contending_lock = ww;
  691. #endif
  692. return -EDEADLK;
  693. }
  694. return 0;
  695. }
  696. /*
  697. * Check the wound condition for the current lock acquire.
  698. *
  699. * Wound-Wait: If we're wounded, kill ourself.
  700. *
  701. * Wait-Die: If we're trying to acquire a lock already held by an older
  702. * context, kill ourselves.
  703. *
  704. * Since __ww_mutex_add_waiter() orders the wait-list on stamp, we only have to
  705. * look at waiters before us in the wait-list.
  706. */
  707. static inline int __sched
  708. __ww_mutex_check_kill(struct mutex *lock, struct mutex_waiter *waiter,
  709. struct ww_acquire_ctx *ctx)
  710. {
  711. struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
  712. struct ww_acquire_ctx *hold_ctx = READ_ONCE(ww->ctx);
  713. struct mutex_waiter *cur;
  714. if (ctx->acquired == 0)
  715. return 0;
  716. if (!ctx->is_wait_die) {
  717. if (ctx->wounded)
  718. return __ww_mutex_kill(lock, ctx);
  719. return 0;
  720. }
  721. if (hold_ctx && __ww_ctx_stamp_after(ctx, hold_ctx))
  722. return __ww_mutex_kill(lock, ctx);
  723. /*
  724. * If there is a waiter in front of us that has a context, then its
  725. * stamp is earlier than ours and we must kill ourself.
  726. */
  727. cur = waiter;
  728. list_for_each_entry_continue_reverse(cur, &lock->wait_list, list) {
  729. if (!cur->ww_ctx)
  730. continue;
  731. return __ww_mutex_kill(lock, ctx);
  732. }
  733. return 0;
  734. }
  735. /*
  736. * Add @waiter to the wait-list, keep the wait-list ordered by stamp, smallest
  737. * first. Such that older contexts are preferred to acquire the lock over
  738. * younger contexts.
  739. *
  740. * Waiters without context are interspersed in FIFO order.
  741. *
  742. * Furthermore, for Wait-Die kill ourself immediately when possible (there are
  743. * older contexts already waiting) to avoid unnecessary waiting and for
  744. * Wound-Wait ensure we wound the owning context when it is younger.
  745. */
  746. static inline int __sched
  747. __ww_mutex_add_waiter(struct mutex_waiter *waiter,
  748. struct mutex *lock,
  749. struct ww_acquire_ctx *ww_ctx)
  750. {
  751. struct mutex_waiter *cur;
  752. struct list_head *pos;
  753. bool is_wait_die;
  754. if (!ww_ctx) {
  755. __mutex_add_waiter(lock, waiter, &lock->wait_list);
  756. return 0;
  757. }
  758. is_wait_die = ww_ctx->is_wait_die;
  759. /*
  760. * Add the waiter before the first waiter with a higher stamp.
  761. * Waiters without a context are skipped to avoid starving
  762. * them. Wait-Die waiters may die here. Wound-Wait waiters
  763. * never die here, but they are sorted in stamp order and
  764. * may wound the lock holder.
  765. */
  766. pos = &lock->wait_list;
  767. list_for_each_entry_reverse(cur, &lock->wait_list, list) {
  768. if (!cur->ww_ctx)
  769. continue;
  770. if (__ww_ctx_stamp_after(ww_ctx, cur->ww_ctx)) {
  771. /*
  772. * Wait-Die: if we find an older context waiting, there
  773. * is no point in queueing behind it, as we'd have to
  774. * die the moment it would acquire the lock.
  775. */
  776. if (is_wait_die) {
  777. int ret = __ww_mutex_kill(lock, ww_ctx);
  778. if (ret)
  779. return ret;
  780. }
  781. break;
  782. }
  783. pos = &cur->list;
  784. /* Wait-Die: ensure younger waiters die. */
  785. __ww_mutex_die(lock, cur, ww_ctx);
  786. }
  787. __mutex_add_waiter(lock, waiter, pos);
  788. /*
  789. * Wound-Wait: if we're blocking on a mutex owned by a younger context,
  790. * wound that such that we might proceed.
  791. */
  792. if (!is_wait_die) {
  793. struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
  794. /*
  795. * See ww_mutex_set_context_fastpath(). Orders setting
  796. * MUTEX_FLAG_WAITERS vs the ww->ctx load,
  797. * such that either we or the fastpath will wound @ww->ctx.
  798. */
  799. smp_mb();
  800. __ww_mutex_wound(lock, ww_ctx, ww->ctx);
  801. }
  802. return 0;
  803. }
  804. /*
  805. * Lock a mutex (possibly interruptible), slowpath:
  806. */
  807. static __always_inline int __sched
  808. __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
  809. struct lockdep_map *nest_lock, unsigned long ip,
  810. struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)
  811. {
  812. struct mutex_waiter waiter;
  813. struct ww_mutex *ww;
  814. int ret;
  815. if (!use_ww_ctx)
  816. ww_ctx = NULL;
  817. might_sleep();
  818. #ifdef CONFIG_DEBUG_MUTEXES
  819. DEBUG_LOCKS_WARN_ON(lock->magic != lock);
  820. #endif
  821. ww = container_of(lock, struct ww_mutex, base);
  822. if (ww_ctx) {
  823. if (unlikely(ww_ctx == READ_ONCE(ww->ctx)))
  824. return -EALREADY;
  825. /*
  826. * Reset the wounded flag after a kill. No other process can
  827. * race and wound us here since they can't have a valid owner
  828. * pointer if we don't have any locks held.
  829. */
  830. if (ww_ctx->acquired == 0)
  831. ww_ctx->wounded = 0;
  832. }
  833. preempt_disable();
  834. mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
  835. if (__mutex_trylock(lock) ||
  836. mutex_optimistic_spin(lock, ww_ctx, NULL)) {
  837. /* got the lock, yay! */
  838. lock_acquired(&lock->dep_map, ip);
  839. if (ww_ctx)
  840. ww_mutex_set_context_fastpath(ww, ww_ctx);
  841. preempt_enable();
  842. return 0;
  843. }
  844. spin_lock(&lock->wait_lock);
  845. /*
  846. * After waiting to acquire the wait_lock, try again.
  847. */
  848. if (__mutex_trylock(lock)) {
  849. if (ww_ctx)
  850. __ww_mutex_check_waiters(lock, ww_ctx);
  851. goto skip_wait;
  852. }
  853. debug_mutex_lock_common(lock, &waiter);
  854. lock_contended(&lock->dep_map, ip);
  855. if (!use_ww_ctx) {
  856. /* add waiting tasks to the end of the waitqueue (FIFO): */
  857. __mutex_add_waiter(lock, &waiter, &lock->wait_list);
  858. #ifdef CONFIG_DEBUG_MUTEXES
  859. waiter.ww_ctx = MUTEX_POISON_WW_CTX;
  860. #endif
  861. } else {
  862. /*
  863. * Add in stamp order, waking up waiters that must kill
  864. * themselves.
  865. */
  866. ret = __ww_mutex_add_waiter(&waiter, lock, ww_ctx);
  867. if (ret)
  868. goto err_early_kill;
  869. waiter.ww_ctx = ww_ctx;
  870. }
  871. waiter.task = current;
  872. trace_android_vh_mutex_wait_start(lock);
  873. set_current_state(state);
  874. for (;;) {
  875. bool first;
  876. /*
  877. * Once we hold wait_lock, we're serialized against
  878. * mutex_unlock() handing the lock off to us, do a trylock
  879. * before testing the error conditions to make sure we pick up
  880. * the handoff.
  881. */
  882. if (__mutex_trylock(lock))
  883. goto acquired;
  884. /*
  885. * Check for signals and kill conditions while holding
  886. * wait_lock. This ensures the lock cancellation is ordered
  887. * against mutex_unlock() and wake-ups do not go missing.
  888. */
  889. if (signal_pending_state(state, current)) {
  890. ret = -EINTR;
  891. goto err;
  892. }
  893. if (ww_ctx) {
  894. ret = __ww_mutex_check_kill(lock, &waiter, ww_ctx);
  895. if (ret)
  896. goto err;
  897. }
  898. spin_unlock(&lock->wait_lock);
  899. schedule_preempt_disabled();
  900. first = __mutex_waiter_is_first(lock, &waiter);
  901. if (first)
  902. __mutex_set_flag(lock, MUTEX_FLAG_HANDOFF);
  903. set_current_state(state);
  904. /*
  905. * Here we order against unlock; we must either see it change
  906. * state back to RUNNING and fall through the next schedule(),
  907. * or we must see its unlock and acquire.
  908. */
  909. if (__mutex_trylock(lock) ||
  910. (first && mutex_optimistic_spin(lock, ww_ctx, &waiter)))
  911. break;
  912. spin_lock(&lock->wait_lock);
  913. }
  914. spin_lock(&lock->wait_lock);
  915. acquired:
  916. __set_current_state(TASK_RUNNING);
  917. trace_android_vh_mutex_wait_finish(lock);
  918. if (ww_ctx) {
  919. /*
  920. * Wound-Wait; we stole the lock (!first_waiter), check the
  921. * waiters as anyone might want to wound us.
  922. */
  923. if (!ww_ctx->is_wait_die &&
  924. !__mutex_waiter_is_first(lock, &waiter))
  925. __ww_mutex_check_waiters(lock, ww_ctx);
  926. }
  927. __mutex_remove_waiter(lock, &waiter);
  928. debug_mutex_free_waiter(&waiter);
  929. skip_wait:
  930. /* got the lock - cleanup and rejoice! */
  931. lock_acquired(&lock->dep_map, ip);
  932. if (ww_ctx)
  933. ww_mutex_lock_acquired(ww, ww_ctx);
  934. spin_unlock(&lock->wait_lock);
  935. preempt_enable();
  936. return 0;
  937. err:
  938. __set_current_state(TASK_RUNNING);
  939. trace_android_vh_mutex_wait_finish(lock);
  940. __mutex_remove_waiter(lock, &waiter);
  941. err_early_kill:
  942. spin_unlock(&lock->wait_lock);
  943. debug_mutex_free_waiter(&waiter);
  944. mutex_release(&lock->dep_map, ip);
  945. preempt_enable();
  946. return ret;
  947. }
  948. static int __sched
  949. __mutex_lock(struct mutex *lock, long state, unsigned int subclass,
  950. struct lockdep_map *nest_lock, unsigned long ip)
  951. {
  952. return __mutex_lock_common(lock, state, subclass, nest_lock, ip, NULL, false);
  953. }
  954. static int __sched
  955. __ww_mutex_lock(struct mutex *lock, long state, unsigned int subclass,
  956. struct lockdep_map *nest_lock, unsigned long ip,
  957. struct ww_acquire_ctx *ww_ctx)
  958. {
  959. return __mutex_lock_common(lock, state, subclass, nest_lock, ip, ww_ctx, true);
  960. }
  961. #ifdef CONFIG_DEBUG_LOCK_ALLOC
  962. void __sched
  963. mutex_lock_nested(struct mutex *lock, unsigned int subclass)
  964. {
  965. __mutex_lock(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_);
  966. }
  967. EXPORT_SYMBOL_GPL(mutex_lock_nested);
  968. void __sched
  969. _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest)
  970. {
  971. __mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, nest, _RET_IP_);
  972. }
  973. EXPORT_SYMBOL_GPL(_mutex_lock_nest_lock);
  974. int __sched
  975. mutex_lock_killable_nested(struct mutex *lock, unsigned int subclass)
  976. {
  977. return __mutex_lock(lock, TASK_KILLABLE, subclass, NULL, _RET_IP_);
  978. }
  979. EXPORT_SYMBOL_GPL(mutex_lock_killable_nested);
  980. int __sched
  981. mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass)
  982. {
  983. return __mutex_lock(lock, TASK_INTERRUPTIBLE, subclass, NULL, _RET_IP_);
  984. }
  985. EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested);
  986. void __sched
  987. mutex_lock_io_nested(struct mutex *lock, unsigned int subclass)
  988. {
  989. int token;
  990. might_sleep();
  991. token = io_schedule_prepare();
  992. __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,
  993. subclass, NULL, _RET_IP_, NULL, 0);
  994. io_schedule_finish(token);
  995. }
  996. EXPORT_SYMBOL_GPL(mutex_lock_io_nested);
  997. static inline int
  998. ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
  999. {
  1000. #ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
  1001. unsigned tmp;
  1002. if (ctx->deadlock_inject_countdown-- == 0) {
  1003. tmp = ctx->deadlock_inject_interval;
  1004. if (tmp > UINT_MAX/4)
  1005. tmp = UINT_MAX;
  1006. else
  1007. tmp = tmp*2 + tmp + tmp/2;
  1008. ctx->deadlock_inject_interval = tmp;
  1009. ctx->deadlock_inject_countdown = tmp;
  1010. ctx->contending_lock = lock;
  1011. ww_mutex_unlock(lock);
  1012. return -EDEADLK;
  1013. }
  1014. #endif
  1015. return 0;
  1016. }
  1017. int __sched
  1018. ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
  1019. {
  1020. int ret;
  1021. might_sleep();
  1022. ret = __ww_mutex_lock(&lock->base, TASK_UNINTERRUPTIBLE,
  1023. 0, ctx ? &ctx->dep_map : NULL, _RET_IP_,
  1024. ctx);
  1025. if (!ret && ctx && ctx->acquired > 1)
  1026. return ww_mutex_deadlock_injection(lock, ctx);
  1027. return ret;
  1028. }
  1029. EXPORT_SYMBOL_GPL(ww_mutex_lock);
  1030. int __sched
  1031. ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
  1032. {
  1033. int ret;
  1034. might_sleep();
  1035. ret = __ww_mutex_lock(&lock->base, TASK_INTERRUPTIBLE,
  1036. 0, ctx ? &ctx->dep_map : NULL, _RET_IP_,
  1037. ctx);
  1038. if (!ret && ctx && ctx->acquired > 1)
  1039. return ww_mutex_deadlock_injection(lock, ctx);
  1040. return ret;
  1041. }
  1042. EXPORT_SYMBOL_GPL(ww_mutex_lock_interruptible);
  1043. #endif
  1044. /*
  1045. * Release the lock, slowpath:
  1046. */
  1047. static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)
  1048. {
  1049. struct task_struct *next = NULL;
  1050. DEFINE_WAKE_Q(wake_q);
  1051. unsigned long owner;
  1052. mutex_release(&lock->dep_map, ip);
  1053. /*
  1054. * Release the lock before (potentially) taking the spinlock such that
  1055. * other contenders can get on with things ASAP.
  1056. *
  1057. * Except when HANDOFF, in that case we must not clear the owner field,
  1058. * but instead set it to the top waiter.
  1059. */
  1060. owner = atomic_long_read(&lock->owner);
  1061. for (;;) {
  1062. unsigned long old;
  1063. #ifdef CONFIG_DEBUG_MUTEXES
  1064. DEBUG_LOCKS_WARN_ON(__owner_task(owner) != current);
  1065. DEBUG_LOCKS_WARN_ON(owner & MUTEX_FLAG_PICKUP);
  1066. #endif
  1067. if (owner & MUTEX_FLAG_HANDOFF)
  1068. break;
  1069. old = atomic_long_cmpxchg_release(&lock->owner, owner,
  1070. __owner_flags(owner));
  1071. if (old == owner) {
  1072. if (owner & MUTEX_FLAG_WAITERS)
  1073. break;
  1074. return;
  1075. }
  1076. owner = old;
  1077. }
  1078. spin_lock(&lock->wait_lock);
  1079. debug_mutex_unlock(lock);
  1080. if (!list_empty(&lock->wait_list)) {
  1081. /* get the first entry from the wait-list: */
  1082. struct mutex_waiter *waiter =
  1083. list_first_entry(&lock->wait_list,
  1084. struct mutex_waiter, list);
  1085. next = waiter->task;
  1086. debug_mutex_wake_waiter(lock, waiter);
  1087. wake_q_add(&wake_q, next);
  1088. }
  1089. if (owner & MUTEX_FLAG_HANDOFF)
  1090. __mutex_handoff(lock, next);
  1091. trace_android_vh_mutex_unlock_slowpath(lock);
  1092. spin_unlock(&lock->wait_lock);
  1093. wake_up_q(&wake_q);
  1094. trace_android_vh_mutex_unlock_slowpath_end(lock, next);
  1095. }
  1096. #ifndef CONFIG_DEBUG_LOCK_ALLOC
  1097. /*
  1098. * Here come the less common (and hence less performance-critical) APIs:
  1099. * mutex_lock_interruptible() and mutex_trylock().
  1100. */
  1101. static noinline int __sched
  1102. __mutex_lock_killable_slowpath(struct mutex *lock);
  1103. static noinline int __sched
  1104. __mutex_lock_interruptible_slowpath(struct mutex *lock);
  1105. /**
  1106. * mutex_lock_interruptible() - Acquire the mutex, interruptible by signals.
  1107. * @lock: The mutex to be acquired.
  1108. *
  1109. * Lock the mutex like mutex_lock(). If a signal is delivered while the
  1110. * process is sleeping, this function will return without acquiring the
  1111. * mutex.
  1112. *
  1113. * Context: Process context.
  1114. * Return: 0 if the lock was successfully acquired or %-EINTR if a
  1115. * signal arrived.
  1116. */
  1117. int __sched mutex_lock_interruptible(struct mutex *lock)
  1118. {
  1119. might_sleep();
  1120. if (__mutex_trylock_fast(lock))
  1121. return 0;
  1122. return __mutex_lock_interruptible_slowpath(lock);
  1123. }
  1124. EXPORT_SYMBOL(mutex_lock_interruptible);
  1125. /**
  1126. * mutex_lock_killable() - Acquire the mutex, interruptible by fatal signals.
  1127. * @lock: The mutex to be acquired.
  1128. *
  1129. * Lock the mutex like mutex_lock(). If a signal which will be fatal to
  1130. * the current process is delivered while the process is sleeping, this
  1131. * function will return without acquiring the mutex.
  1132. *
  1133. * Context: Process context.
  1134. * Return: 0 if the lock was successfully acquired or %-EINTR if a
  1135. * fatal signal arrived.
  1136. */
  1137. int __sched mutex_lock_killable(struct mutex *lock)
  1138. {
  1139. might_sleep();
  1140. if (__mutex_trylock_fast(lock))
  1141. return 0;
  1142. return __mutex_lock_killable_slowpath(lock);
  1143. }
  1144. EXPORT_SYMBOL(mutex_lock_killable);
  1145. /**
  1146. * mutex_lock_io() - Acquire the mutex and mark the process as waiting for I/O
  1147. * @lock: The mutex to be acquired.
  1148. *
  1149. * Lock the mutex like mutex_lock(). While the task is waiting for this
  1150. * mutex, it will be accounted as being in the IO wait state by the
  1151. * scheduler.
  1152. *
  1153. * Context: Process context.
  1154. */
  1155. void __sched mutex_lock_io(struct mutex *lock)
  1156. {
  1157. int token;
  1158. token = io_schedule_prepare();
  1159. mutex_lock(lock);
  1160. io_schedule_finish(token);
  1161. }
  1162. EXPORT_SYMBOL_GPL(mutex_lock_io);
  1163. static noinline void __sched
  1164. __mutex_lock_slowpath(struct mutex *lock)
  1165. {
  1166. __mutex_lock(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_);
  1167. }
  1168. static noinline int __sched
  1169. __mutex_lock_killable_slowpath(struct mutex *lock)
  1170. {
  1171. return __mutex_lock(lock, TASK_KILLABLE, 0, NULL, _RET_IP_);
  1172. }
  1173. static noinline int __sched
  1174. __mutex_lock_interruptible_slowpath(struct mutex *lock)
  1175. {
  1176. return __mutex_lock(lock, TASK_INTERRUPTIBLE, 0, NULL, _RET_IP_);
  1177. }
  1178. static noinline int __sched
  1179. __ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
  1180. {
  1181. return __ww_mutex_lock(&lock->base, TASK_UNINTERRUPTIBLE, 0, NULL,
  1182. _RET_IP_, ctx);
  1183. }
  1184. static noinline int __sched
  1185. __ww_mutex_lock_interruptible_slowpath(struct ww_mutex *lock,
  1186. struct ww_acquire_ctx *ctx)
  1187. {
  1188. return __ww_mutex_lock(&lock->base, TASK_INTERRUPTIBLE, 0, NULL,
  1189. _RET_IP_, ctx);
  1190. }
  1191. #endif
  1192. /**
  1193. * mutex_trylock - try to acquire the mutex, without waiting
  1194. * @lock: the mutex to be acquired
  1195. *
  1196. * Try to acquire the mutex atomically. Returns 1 if the mutex
  1197. * has been acquired successfully, and 0 on contention.
  1198. *
  1199. * NOTE: this function follows the spin_trylock() convention, so
  1200. * it is negated from the down_trylock() return values! Be careful
  1201. * about this when converting semaphore users to mutexes.
  1202. *
  1203. * This function must not be used in interrupt context. The
  1204. * mutex must be released by the same task that acquired it.
  1205. */
  1206. int __sched mutex_trylock(struct mutex *lock)
  1207. {
  1208. bool locked;
  1209. #ifdef CONFIG_DEBUG_MUTEXES
  1210. DEBUG_LOCKS_WARN_ON(lock->magic != lock);
  1211. #endif
  1212. locked = __mutex_trylock(lock);
  1213. if (locked)
  1214. mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  1215. return locked;
  1216. }
  1217. EXPORT_SYMBOL(mutex_trylock);
  1218. #ifndef CONFIG_DEBUG_LOCK_ALLOC
  1219. int __sched
  1220. ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
  1221. {
  1222. might_sleep();
  1223. if (__mutex_trylock_fast(&lock->base)) {
  1224. if (ctx)
  1225. ww_mutex_set_context_fastpath(lock, ctx);
  1226. return 0;
  1227. }
  1228. return __ww_mutex_lock_slowpath(lock, ctx);
  1229. }
  1230. EXPORT_SYMBOL(ww_mutex_lock);
  1231. int __sched
  1232. ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
  1233. {
  1234. might_sleep();
  1235. if (__mutex_trylock_fast(&lock->base)) {
  1236. if (ctx)
  1237. ww_mutex_set_context_fastpath(lock, ctx);
  1238. return 0;
  1239. }
  1240. return __ww_mutex_lock_interruptible_slowpath(lock, ctx);
  1241. }
  1242. EXPORT_SYMBOL(ww_mutex_lock_interruptible);
  1243. #endif
  1244. /**
  1245. * atomic_dec_and_mutex_lock - return holding mutex if we dec to 0
  1246. * @cnt: the atomic which we are to dec
  1247. * @lock: the mutex to return holding if we dec to 0
  1248. *
  1249. * return true and hold lock if we dec to 0, return false otherwise
  1250. */
  1251. int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock)
  1252. {
  1253. /* dec if we can't possibly hit 0 */
  1254. if (atomic_add_unless(cnt, -1, 1))
  1255. return 0;
  1256. /* we might hit 0, so take the lock */
  1257. mutex_lock(lock);
  1258. if (!atomic_dec_and_test(cnt)) {
  1259. /* when we actually did the dec, we didn't hit 0 */
  1260. mutex_unlock(lock);
  1261. return 0;
  1262. }
  1263. /* we hit 0, and we hold the lock */
  1264. return 1;
  1265. }
  1266. EXPORT_SYMBOL(atomic_dec_and_mutex_lock);