qrwlock.c 2.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Queued read/write locks
  4. *
  5. * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
  6. *
  7. * Authors: Waiman Long <waiman.long@hp.com>
  8. */
  9. #include <linux/smp.h>
  10. #include <linux/bug.h>
  11. #include <linux/cpumask.h>
  12. #include <linux/percpu.h>
  13. #include <linux/hardirq.h>
  14. #include <linux/spinlock.h>
  15. #include <asm/qrwlock.h>
  16. /**
  17. * queued_read_lock_slowpath - acquire read lock of a queue rwlock
  18. * @lock: Pointer to queue rwlock structure
  19. */
  20. void queued_read_lock_slowpath(struct qrwlock *lock)
  21. {
  22. /*
  23. * Readers come here when they cannot get the lock without waiting
  24. */
  25. if (unlikely(in_interrupt())) {
  26. /*
  27. * Readers in interrupt context will get the lock immediately
  28. * if the writer is just waiting (not holding the lock yet),
  29. * so spin with ACQUIRE semantics until the lock is available
  30. * without waiting in the queue.
  31. */
  32. atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));
  33. return;
  34. }
  35. atomic_sub(_QR_BIAS, &lock->cnts);
  36. /*
  37. * Put the reader into the wait queue
  38. */
  39. arch_spin_lock(&lock->wait_lock);
  40. atomic_add(_QR_BIAS, &lock->cnts);
  41. /*
  42. * The ACQUIRE semantics of the following spinning code ensure
  43. * that accesses can't leak upwards out of our subsequent critical
  44. * section in the case that the lock is currently held for write.
  45. */
  46. atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));
  47. /*
  48. * Signal the next one in queue to become queue head
  49. */
  50. arch_spin_unlock(&lock->wait_lock);
  51. }
  52. EXPORT_SYMBOL(queued_read_lock_slowpath);
  53. /**
  54. * queued_write_lock_slowpath - acquire write lock of a queue rwlock
  55. * @lock : Pointer to queue rwlock structure
  56. */
  57. void queued_write_lock_slowpath(struct qrwlock *lock)
  58. {
  59. int cnts;
  60. /* Put the writer into the wait queue */
  61. arch_spin_lock(&lock->wait_lock);
  62. /* Try to acquire the lock directly if no reader is present */
  63. if (!atomic_read(&lock->cnts) &&
  64. (atomic_cmpxchg_acquire(&lock->cnts, 0, _QW_LOCKED) == 0))
  65. goto unlock;
  66. /* Set the waiting flag to notify readers that a writer is pending */
  67. atomic_add(_QW_WAITING, &lock->cnts);
  68. /* When no more readers or writers, set the locked flag */
  69. do {
  70. cnts = atomic_cond_read_relaxed(&lock->cnts, VAL == _QW_WAITING);
  71. } while (!atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED));
  72. unlock:
  73. arch_spin_unlock(&lock->wait_lock);
  74. }
  75. EXPORT_SYMBOL(queued_write_lock_slowpath);