seqlock.h 4.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185
  1. #ifndef __LINUX_SEQLOCK_H
  2. #define __LINUX_SEQLOCK_H
  3. /*
  4. * Reader/writer consistent mechanism without starving writers. This type of
  5. * lock for data where the reader wants a consistent set of information
  6. * and is willing to retry if the information changes. Readers never
  7. * block but they may have to retry if a writer is in
  8. * progress. Writers do not wait for readers.
  9. *
  10. * This is not as cache friendly as brlock. Also, this will not work
  11. * for data that contains pointers, because any writer could
  12. * invalidate a pointer that a reader was following.
  13. *
  14. * Expected reader usage:
  15. * do {
  16. * seq = read_seqbegin(&foo);
  17. * ...
  18. * } while (read_seqretry(&foo, seq));
  19. *
  20. *
  21. * On non-SMP the spin locks disappear but the writer still needs
  22. * to increment the sequence variables because an interrupt routine could
  23. * change the state of the data.
  24. *
  25. * Based on x86_64 vsyscall gettimeofday
  26. * by Keith Owens and Andrea Arcangeli
  27. */
  28. #include <linux/spinlock.h>
  29. #include <linux/preempt.h>
  30. typedef struct {
  31. unsigned sequence;
  32. spinlock_t lock;
  33. } seqlock_t;
  34. /*
  35. * These macros triggered gcc-3.x compile-time problems. We think these are
  36. * OK now. Be cautious.
  37. */
  38. #define __SEQLOCK_UNLOCKED(lockname) \
  39. { 0, __SPIN_LOCK_UNLOCKED(lockname) }
  40. #define SEQLOCK_UNLOCKED \
  41. __SEQLOCK_UNLOCKED(old_style_seqlock_init)
  42. #define seqlock_init(x) \
  43. do { \
  44. (x)->sequence = 0; \
  45. spin_lock_init(&(x)->lock); \
  46. } while (0)
  47. #define DEFINE_SEQLOCK(x) \
  48. seqlock_t x = __SEQLOCK_UNLOCKED(x)
  49. /* Lock out other writers and update the count.
  50. * Acts like a normal spin_lock/unlock.
  51. * Don't need preempt_disable() because that is in the spin_lock already.
  52. */
  53. static inline void write_seqlock(seqlock_t *sl)
  54. {
  55. spin_lock(&sl->lock);
  56. ++sl->sequence;
  57. smp_wmb();
  58. }
  59. static inline void write_sequnlock(seqlock_t *sl)
  60. {
  61. smp_wmb();
  62. sl->sequence++;
  63. spin_unlock(&sl->lock);
  64. }
  65. static inline int write_tryseqlock(seqlock_t *sl)
  66. {
  67. int ret = spin_trylock(&sl->lock);
  68. if (ret) {
  69. ++sl->sequence;
  70. smp_wmb();
  71. }
  72. return ret;
  73. }
  74. /* Start of read calculation -- fetch last complete writer token */
  75. static __always_inline unsigned read_seqbegin(const seqlock_t *sl)
  76. {
  77. unsigned ret = sl->sequence;
  78. smp_rmb();
  79. return ret;
  80. }
  81. /* Test if reader processed invalid data.
  82. * If initial values is odd,
  83. * then writer had already started when section was entered
  84. * If sequence value changed
  85. * then writer changed data while in section
  86. *
  87. * Using xor saves one conditional branch.
  88. */
  89. static __always_inline int read_seqretry(const seqlock_t *sl, unsigned iv)
  90. {
  91. smp_rmb();
  92. return (iv & 1) | (sl->sequence ^ iv);
  93. }
  94. /*
  95. * Version using sequence counter only.
  96. * This can be used when code has its own mutex protecting the
  97. * updating starting before the write_seqcountbeqin() and ending
  98. * after the write_seqcount_end().
  99. */
  100. typedef struct seqcount {
  101. unsigned sequence;
  102. } seqcount_t;
  103. #define SEQCNT_ZERO { 0 }
  104. #define seqcount_init(x) do { *(x) = (seqcount_t) SEQCNT_ZERO; } while (0)
  105. /* Start of read using pointer to a sequence counter only. */
  106. static inline unsigned read_seqcount_begin(const seqcount_t *s)
  107. {
  108. unsigned ret = s->sequence;
  109. smp_rmb();
  110. return ret;
  111. }
  112. /* Test if reader processed invalid data.
  113. * Equivalent to: iv is odd or sequence number has changed.
  114. * (iv & 1) || (*s != iv)
  115. * Using xor saves one conditional branch.
  116. */
  117. static inline int read_seqcount_retry(const seqcount_t *s, unsigned iv)
  118. {
  119. smp_rmb();
  120. return (iv & 1) | (s->sequence ^ iv);
  121. }
  122. /*
  123. * Sequence counter only version assumes that callers are using their
  124. * own mutexing.
  125. */
  126. static inline void write_seqcount_begin(seqcount_t *s)
  127. {
  128. s->sequence++;
  129. smp_wmb();
  130. }
  131. static inline void write_seqcount_end(seqcount_t *s)
  132. {
  133. smp_wmb();
  134. s->sequence++;
  135. }
  136. /*
  137. * Possible sw/hw IRQ protected versions of the interfaces.
  138. */
  139. #define write_seqlock_irqsave(lock, flags) \
  140. do { local_irq_save(flags); write_seqlock(lock); } while (0)
  141. #define write_seqlock_irq(lock) \
  142. do { local_irq_disable(); write_seqlock(lock); } while (0)
  143. #define write_seqlock_bh(lock) \
  144. do { local_bh_disable(); write_seqlock(lock); } while (0)
  145. #define write_sequnlock_irqrestore(lock, flags) \
  146. do { write_sequnlock(lock); local_irq_restore(flags); } while(0)
  147. #define write_sequnlock_irq(lock) \
  148. do { write_sequnlock(lock); local_irq_enable(); } while(0)
  149. #define write_sequnlock_bh(lock) \
  150. do { write_sequnlock(lock); local_bh_enable(); } while(0)
  151. #define read_seqbegin_irqsave(lock, flags) \
  152. ({ local_irq_save(flags); read_seqbegin(lock); })
  153. #define read_seqretry_irqrestore(lock, iv, flags) \
  154. ({ \
  155. int ret = read_seqretry(lock, iv); \
  156. local_irq_restore(flags); \
  157. ret; \
  158. })
  159. #endif /* __LINUX_SEQLOCK_H */