rwsem-spinlock.c 6.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316
  1. /* rwsem-spinlock.c: R/W semaphores: contention handling functions for
  2. * generic spinlock implementation
  3. *
  4. * Copyright (c) 2001 David Howells (dhowells@redhat.com).
  5. * - Derived partially from idea by Andrea Arcangeli <andrea@suse.de>
  6. * - Derived also from comments by Linus
  7. */
  8. #include <linux/rwsem.h>
  9. #include <linux/sched.h>
  10. #include <linux/module.h>
  11. struct rwsem_waiter {
  12. struct list_head list;
  13. struct task_struct *task;
  14. unsigned int flags;
  15. #define RWSEM_WAITING_FOR_READ 0x00000001
  16. #define RWSEM_WAITING_FOR_WRITE 0x00000002
  17. };
  18. /*
  19. * initialise the semaphore
  20. */
  21. void __init_rwsem(struct rw_semaphore *sem, const char *name,
  22. struct lock_class_key *key)
  23. {
  24. #ifdef CONFIG_DEBUG_LOCK_ALLOC
  25. /*
  26. * Make sure we are not reinitializing a held semaphore:
  27. */
  28. debug_check_no_locks_freed((void *)sem, sizeof(*sem));
  29. lockdep_init_map(&sem->dep_map, name, key, 0);
  30. #endif
  31. sem->activity = 0;
  32. spin_lock_init(&sem->wait_lock);
  33. INIT_LIST_HEAD(&sem->wait_list);
  34. }
  35. /*
  36. * handle the lock release when processes blocked on it that can now run
  37. * - if we come here, then:
  38. * - the 'active count' _reached_ zero
  39. * - the 'waiting count' is non-zero
  40. * - the spinlock must be held by the caller
  41. * - woken process blocks are discarded from the list after having task zeroed
  42. * - writers are only woken if wakewrite is non-zero
  43. */
  44. static inline struct rw_semaphore *
  45. __rwsem_do_wake(struct rw_semaphore *sem, int wakewrite)
  46. {
  47. struct rwsem_waiter *waiter;
  48. struct task_struct *tsk;
  49. int woken;
  50. waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list);
  51. if (!wakewrite) {
  52. if (waiter->flags & RWSEM_WAITING_FOR_WRITE)
  53. goto out;
  54. goto dont_wake_writers;
  55. }
  56. /* if we are allowed to wake writers try to grant a single write lock
  57. * if there's a writer at the front of the queue
  58. * - we leave the 'waiting count' incremented to signify potential
  59. * contention
  60. */
  61. if (waiter->flags & RWSEM_WAITING_FOR_WRITE) {
  62. sem->activity = -1;
  63. list_del(&waiter->list);
  64. tsk = waiter->task;
  65. /* Don't touch waiter after ->task has been NULLed */
  66. smp_mb();
  67. waiter->task = NULL;
  68. wake_up_process(tsk);
  69. put_task_struct(tsk);
  70. goto out;
  71. }
  72. /* grant an infinite number of read locks to the front of the queue */
  73. dont_wake_writers:
  74. woken = 0;
  75. while (waiter->flags & RWSEM_WAITING_FOR_READ) {
  76. struct list_head *next = waiter->list.next;
  77. list_del(&waiter->list);
  78. tsk = waiter->task;
  79. smp_mb();
  80. waiter->task = NULL;
  81. wake_up_process(tsk);
  82. put_task_struct(tsk);
  83. woken++;
  84. if (list_empty(&sem->wait_list))
  85. break;
  86. waiter = list_entry(next, struct rwsem_waiter, list);
  87. }
  88. sem->activity += woken;
  89. out:
  90. return sem;
  91. }
  92. /*
  93. * wake a single writer
  94. */
  95. static inline struct rw_semaphore *
  96. __rwsem_wake_one_writer(struct rw_semaphore *sem)
  97. {
  98. struct rwsem_waiter *waiter;
  99. struct task_struct *tsk;
  100. sem->activity = -1;
  101. waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list);
  102. list_del(&waiter->list);
  103. tsk = waiter->task;
  104. smp_mb();
  105. waiter->task = NULL;
  106. wake_up_process(tsk);
  107. put_task_struct(tsk);
  108. return sem;
  109. }
  110. /*
  111. * get a read lock on the semaphore
  112. */
  113. void fastcall __sched __down_read(struct rw_semaphore *sem)
  114. {
  115. struct rwsem_waiter waiter;
  116. struct task_struct *tsk;
  117. spin_lock_irq(&sem->wait_lock);
  118. if (sem->activity >= 0 && list_empty(&sem->wait_list)) {
  119. /* granted */
  120. sem->activity++;
  121. spin_unlock_irq(&sem->wait_lock);
  122. goto out;
  123. }
  124. tsk = current;
  125. set_task_state(tsk, TASK_UNINTERRUPTIBLE);
  126. /* set up my own style of waitqueue */
  127. waiter.task = tsk;
  128. waiter.flags = RWSEM_WAITING_FOR_READ;
  129. get_task_struct(tsk);
  130. list_add_tail(&waiter.list, &sem->wait_list);
  131. /* we don't need to touch the semaphore struct anymore */
  132. spin_unlock_irq(&sem->wait_lock);
  133. /* wait to be given the lock */
  134. for (;;) {
  135. if (!waiter.task)
  136. break;
  137. schedule();
  138. set_task_state(tsk, TASK_UNINTERRUPTIBLE);
  139. }
  140. tsk->state = TASK_RUNNING;
  141. out:
  142. ;
  143. }
  144. /*
  145. * trylock for reading -- returns 1 if successful, 0 if contention
  146. */
  147. int fastcall __down_read_trylock(struct rw_semaphore *sem)
  148. {
  149. unsigned long flags;
  150. int ret = 0;
  151. spin_lock_irqsave(&sem->wait_lock, flags);
  152. if (sem->activity >= 0 && list_empty(&sem->wait_list)) {
  153. /* granted */
  154. sem->activity++;
  155. ret = 1;
  156. }
  157. spin_unlock_irqrestore(&sem->wait_lock, flags);
  158. return ret;
  159. }
  160. /*
  161. * get a write lock on the semaphore
  162. * - we increment the waiting count anyway to indicate an exclusive lock
  163. */
  164. void fastcall __sched __down_write_nested(struct rw_semaphore *sem, int subclass)
  165. {
  166. struct rwsem_waiter waiter;
  167. struct task_struct *tsk;
  168. spin_lock_irq(&sem->wait_lock);
  169. if (sem->activity == 0 && list_empty(&sem->wait_list)) {
  170. /* granted */
  171. sem->activity = -1;
  172. spin_unlock_irq(&sem->wait_lock);
  173. goto out;
  174. }
  175. tsk = current;
  176. set_task_state(tsk, TASK_UNINTERRUPTIBLE);
  177. /* set up my own style of waitqueue */
  178. waiter.task = tsk;
  179. waiter.flags = RWSEM_WAITING_FOR_WRITE;
  180. get_task_struct(tsk);
  181. list_add_tail(&waiter.list, &sem->wait_list);
  182. /* we don't need to touch the semaphore struct anymore */
  183. spin_unlock_irq(&sem->wait_lock);
  184. /* wait to be given the lock */
  185. for (;;) {
  186. if (!waiter.task)
  187. break;
  188. schedule();
  189. set_task_state(tsk, TASK_UNINTERRUPTIBLE);
  190. }
  191. tsk->state = TASK_RUNNING;
  192. out:
  193. ;
  194. }
  195. void fastcall __sched __down_write(struct rw_semaphore *sem)
  196. {
  197. __down_write_nested(sem, 0);
  198. }
  199. /*
  200. * trylock for writing -- returns 1 if successful, 0 if contention
  201. */
  202. int fastcall __down_write_trylock(struct rw_semaphore *sem)
  203. {
  204. unsigned long flags;
  205. int ret = 0;
  206. spin_lock_irqsave(&sem->wait_lock, flags);
  207. if (sem->activity == 0 && list_empty(&sem->wait_list)) {
  208. /* granted */
  209. sem->activity = -1;
  210. ret = 1;
  211. }
  212. spin_unlock_irqrestore(&sem->wait_lock, flags);
  213. return ret;
  214. }
  215. /*
  216. * release a read lock on the semaphore
  217. */
  218. void fastcall __up_read(struct rw_semaphore *sem)
  219. {
  220. unsigned long flags;
  221. spin_lock_irqsave(&sem->wait_lock, flags);
  222. if (--sem->activity == 0 && !list_empty(&sem->wait_list))
  223. sem = __rwsem_wake_one_writer(sem);
  224. spin_unlock_irqrestore(&sem->wait_lock, flags);
  225. }
  226. /*
  227. * release a write lock on the semaphore
  228. */
  229. void fastcall __up_write(struct rw_semaphore *sem)
  230. {
  231. unsigned long flags;
  232. spin_lock_irqsave(&sem->wait_lock, flags);
  233. sem->activity = 0;
  234. if (!list_empty(&sem->wait_list))
  235. sem = __rwsem_do_wake(sem, 1);
  236. spin_unlock_irqrestore(&sem->wait_lock, flags);
  237. }
  238. /*
  239. * downgrade a write lock into a read lock
  240. * - just wake up any readers at the front of the queue
  241. */
  242. void fastcall __downgrade_write(struct rw_semaphore *sem)
  243. {
  244. unsigned long flags;
  245. spin_lock_irqsave(&sem->wait_lock, flags);
  246. sem->activity = 1;
  247. if (!list_empty(&sem->wait_list))
  248. sem = __rwsem_do_wake(sem, 0);
  249. spin_unlock_irqrestore(&sem->wait_lock, flags);
  250. }
  251. EXPORT_SYMBOL(__init_rwsem);
  252. EXPORT_SYMBOL(__down_read);
  253. EXPORT_SYMBOL(__down_read_trylock);
  254. EXPORT_SYMBOL(__down_write_nested);
  255. EXPORT_SYMBOL(__down_write);
  256. EXPORT_SYMBOL(__down_write_trylock);
  257. EXPORT_SYMBOL(__up_read);
  258. EXPORT_SYMBOL(__up_write);
  259. EXPORT_SYMBOL(__downgrade_write);