mutex.c 9.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347
  1. /*
  2. * kernel/mutex.c
  3. *
  4. * Mutexes: blocking mutual exclusion locks
  5. *
  6. * Started by Ingo Molnar:
  7. *
  8. * Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
  9. *
  10. * Many thanks to Arjan van de Ven, Thomas Gleixner, Steven Rostedt and
  11. * David Howells for suggestions and improvements.
  12. *
  13. * Also see Documentation/mutex-design.txt.
  14. */
  15. #include <linux/mutex.h>
  16. #include <linux/sched.h>
  17. #include <linux/module.h>
  18. #include <linux/spinlock.h>
  19. #include <linux/interrupt.h>
  20. #include <linux/debug_locks.h>
  21. /*
  22. * In the DEBUG case we are using the "NULL fastpath" for mutexes,
  23. * which forces all calls into the slowpath:
  24. */
  25. #ifdef CONFIG_DEBUG_MUTEXES
  26. # include "mutex-debug.h"
  27. # include <asm-generic/mutex-null.h>
  28. #else
  29. # include "mutex.h"
  30. # include <asm/mutex.h>
  31. #endif
  32. /***
  33. * mutex_init - initialize the mutex
  34. * @lock: the mutex to be initialized
  35. *
  36. * Initialize the mutex to unlocked state.
  37. *
  38. * It is not allowed to initialize an already locked mutex.
  39. */
  40. void
  41. __mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key)
  42. {
  43. atomic_set(&lock->count, 1);
  44. spin_lock_init(&lock->wait_lock);
  45. INIT_LIST_HEAD(&lock->wait_list);
  46. debug_mutex_init(lock, name, key);
  47. }
  48. EXPORT_SYMBOL(__mutex_init);
  49. /*
  50. * We split the mutex lock/unlock logic into separate fastpath and
  51. * slowpath functions, to reduce the register pressure on the fastpath.
  52. * We also put the fastpath first in the kernel image, to make sure the
  53. * branch is predicted by the CPU as default-untaken.
  54. */
  55. static void fastcall noinline __sched
  56. __mutex_lock_slowpath(atomic_t *lock_count);
  57. /***
  58. * mutex_lock - acquire the mutex
  59. * @lock: the mutex to be acquired
  60. *
  61. * Lock the mutex exclusively for this task. If the mutex is not
  62. * available right now, it will sleep until it can get it.
  63. *
  64. * The mutex must later on be released by the same task that
  65. * acquired it. Recursive locking is not allowed. The task
  66. * may not exit without first unlocking the mutex. Also, kernel
  67. * memory where the mutex resides mutex must not be freed with
  68. * the mutex still locked. The mutex must first be initialized
  69. * (or statically defined) before it can be locked. memset()-ing
  70. * the mutex to 0 is not allowed.
  71. *
  72. * ( The CONFIG_DEBUG_MUTEXES .config option turns on debugging
  73. * checks that will enforce the restrictions and will also do
  74. * deadlock debugging. )
  75. *
  76. * This function is similar to (but not equivalent to) down().
  77. */
  78. void inline fastcall __sched mutex_lock(struct mutex *lock)
  79. {
  80. might_sleep();
  81. /*
  82. * The locking fastpath is the 1->0 transition from
  83. * 'unlocked' into 'locked' state.
  84. */
  85. __mutex_fastpath_lock(&lock->count, __mutex_lock_slowpath);
  86. }
  87. EXPORT_SYMBOL(mutex_lock);
  88. static void fastcall noinline __sched
  89. __mutex_unlock_slowpath(atomic_t *lock_count);
  90. /***
  91. * mutex_unlock - release the mutex
  92. * @lock: the mutex to be released
  93. *
  94. * Unlock a mutex that has been locked by this task previously.
  95. *
  96. * This function must not be used in interrupt context. Unlocking
  97. * of a not locked mutex is not allowed.
  98. *
  99. * This function is similar to (but not equivalent to) up().
  100. */
  101. void fastcall __sched mutex_unlock(struct mutex *lock)
  102. {
  103. /*
  104. * The unlocking fastpath is the 0->1 transition from 'locked'
  105. * into 'unlocked' state:
  106. */
  107. __mutex_fastpath_unlock(&lock->count, __mutex_unlock_slowpath);
  108. }
  109. EXPORT_SYMBOL(mutex_unlock);
  110. /*
  111. * Lock a mutex (possibly interruptible), slowpath:
  112. */
  113. static inline int __sched
  114. __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass)
  115. {
  116. struct task_struct *task = current;
  117. struct mutex_waiter waiter;
  118. unsigned int old_val;
  119. unsigned long flags;
  120. spin_lock_mutex(&lock->wait_lock, flags);
  121. debug_mutex_lock_common(lock, &waiter);
  122. mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
  123. debug_mutex_add_waiter(lock, &waiter, task->thread_info);
  124. /* add waiting tasks to the end of the waitqueue (FIFO): */
  125. list_add_tail(&waiter.list, &lock->wait_list);
  126. waiter.task = task;
  127. for (;;) {
  128. /*
  129. * Lets try to take the lock again - this is needed even if
  130. * we get here for the first time (shortly after failing to
  131. * acquire the lock), to make sure that we get a wakeup once
  132. * it's unlocked. Later on, if we sleep, this is the
  133. * operation that gives us the lock. We xchg it to -1, so
  134. * that when we release the lock, we properly wake up the
  135. * other waiters:
  136. */
  137. old_val = atomic_xchg(&lock->count, -1);
  138. if (old_val == 1)
  139. break;
  140. /*
  141. * got a signal? (This code gets eliminated in the
  142. * TASK_UNINTERRUPTIBLE case.)
  143. */
  144. if (unlikely(state == TASK_INTERRUPTIBLE &&
  145. signal_pending(task))) {
  146. mutex_remove_waiter(lock, &waiter, task->thread_info);
  147. mutex_release(&lock->dep_map, 1, _RET_IP_);
  148. spin_unlock_mutex(&lock->wait_lock, flags);
  149. debug_mutex_free_waiter(&waiter);
  150. return -EINTR;
  151. }
  152. __set_task_state(task, state);
  153. /* didnt get the lock, go to sleep: */
  154. spin_unlock_mutex(&lock->wait_lock, flags);
  155. schedule();
  156. spin_lock_mutex(&lock->wait_lock, flags);
  157. }
  158. /* got the lock - rejoice! */
  159. mutex_remove_waiter(lock, &waiter, task->thread_info);
  160. debug_mutex_set_owner(lock, task->thread_info);
  161. /* set it to 0 if there are no waiters left: */
  162. if (likely(list_empty(&lock->wait_list)))
  163. atomic_set(&lock->count, 0);
  164. spin_unlock_mutex(&lock->wait_lock, flags);
  165. debug_mutex_free_waiter(&waiter);
  166. return 0;
  167. }
  168. static void fastcall noinline __sched
  169. __mutex_lock_slowpath(atomic_t *lock_count)
  170. {
  171. struct mutex *lock = container_of(lock_count, struct mutex, count);
  172. __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0);
  173. }
  174. #ifdef CONFIG_DEBUG_LOCK_ALLOC
  175. void __sched
  176. mutex_lock_nested(struct mutex *lock, unsigned int subclass)
  177. {
  178. might_sleep();
  179. __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass);
  180. }
  181. EXPORT_SYMBOL_GPL(mutex_lock_nested);
  182. int __sched
  183. mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass)
  184. {
  185. might_sleep();
  186. return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, subclass);
  187. }
  188. EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested);
  189. #endif
  190. /*
  191. * Release the lock, slowpath:
  192. */
  193. static fastcall inline void
  194. __mutex_unlock_common_slowpath(atomic_t *lock_count, int nested)
  195. {
  196. struct mutex *lock = container_of(lock_count, struct mutex, count);
  197. unsigned long flags;
  198. spin_lock_mutex(&lock->wait_lock, flags);
  199. mutex_release(&lock->dep_map, nested, _RET_IP_);
  200. debug_mutex_unlock(lock);
  201. /*
  202. * some architectures leave the lock unlocked in the fastpath failure
  203. * case, others need to leave it locked. In the later case we have to
  204. * unlock it here
  205. */
  206. if (__mutex_slowpath_needs_to_unlock())
  207. atomic_set(&lock->count, 1);
  208. if (!list_empty(&lock->wait_list)) {
  209. /* get the first entry from the wait-list: */
  210. struct mutex_waiter *waiter =
  211. list_entry(lock->wait_list.next,
  212. struct mutex_waiter, list);
  213. debug_mutex_wake_waiter(lock, waiter);
  214. wake_up_process(waiter->task);
  215. }
  216. debug_mutex_clear_owner(lock);
  217. spin_unlock_mutex(&lock->wait_lock, flags);
  218. }
  219. /*
  220. * Release the lock, slowpath:
  221. */
  222. static fastcall noinline void
  223. __mutex_unlock_slowpath(atomic_t *lock_count)
  224. {
  225. __mutex_unlock_common_slowpath(lock_count, 1);
  226. }
  227. /*
  228. * Here come the less common (and hence less performance-critical) APIs:
  229. * mutex_lock_interruptible() and mutex_trylock().
  230. */
  231. static int fastcall noinline __sched
  232. __mutex_lock_interruptible_slowpath(atomic_t *lock_count);
  233. /***
  234. * mutex_lock_interruptible - acquire the mutex, interruptable
  235. * @lock: the mutex to be acquired
  236. *
  237. * Lock the mutex like mutex_lock(), and return 0 if the mutex has
  238. * been acquired or sleep until the mutex becomes available. If a
  239. * signal arrives while waiting for the lock then this function
  240. * returns -EINTR.
  241. *
  242. * This function is similar to (but not equivalent to) down_interruptible().
  243. */
  244. int fastcall __sched mutex_lock_interruptible(struct mutex *lock)
  245. {
  246. might_sleep();
  247. return __mutex_fastpath_lock_retval
  248. (&lock->count, __mutex_lock_interruptible_slowpath);
  249. }
  250. EXPORT_SYMBOL(mutex_lock_interruptible);
  251. static int fastcall noinline __sched
  252. __mutex_lock_interruptible_slowpath(atomic_t *lock_count)
  253. {
  254. struct mutex *lock = container_of(lock_count, struct mutex, count);
  255. return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0);
  256. }
  257. /*
  258. * Spinlock based trylock, we take the spinlock and check whether we
  259. * can get the lock:
  260. */
  261. static inline int __mutex_trylock_slowpath(atomic_t *lock_count)
  262. {
  263. struct mutex *lock = container_of(lock_count, struct mutex, count);
  264. unsigned long flags;
  265. int prev;
  266. spin_lock_mutex(&lock->wait_lock, flags);
  267. prev = atomic_xchg(&lock->count, -1);
  268. if (likely(prev == 1)) {
  269. debug_mutex_set_owner(lock, current_thread_info());
  270. mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  271. }
  272. /* Set it back to 0 if there are no waiters: */
  273. if (likely(list_empty(&lock->wait_list)))
  274. atomic_set(&lock->count, 0);
  275. spin_unlock_mutex(&lock->wait_lock, flags);
  276. return prev == 1;
  277. }
  278. /***
  279. * mutex_trylock - try acquire the mutex, without waiting
  280. * @lock: the mutex to be acquired
  281. *
  282. * Try to acquire the mutex atomically. Returns 1 if the mutex
  283. * has been acquired successfully, and 0 on contention.
  284. *
  285. * NOTE: this function follows the spin_trylock() convention, so
  286. * it is negated to the down_trylock() return values! Be careful
  287. * about this when converting semaphore users to mutexes.
  288. *
  289. * This function must not be used in interrupt context. The
  290. * mutex must be released by the same task that acquired it.
  291. */
  292. int fastcall __sched mutex_trylock(struct mutex *lock)
  293. {
  294. return __mutex_fastpath_trylock(&lock->count,
  295. __mutex_trylock_slowpath);
  296. }
  297. EXPORT_SYMBOL(mutex_trylock);