srcutiny.h 2.7 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091
  1. /* SPDX-License-Identifier: GPL-2.0+ */
  2. /*
  3. * Sleepable Read-Copy Update mechanism for mutual exclusion,
  4. * tiny variant.
  5. *
  6. * Copyright (C) IBM Corporation, 2017
  7. *
  8. * Author: Paul McKenney <paulmck@linux.ibm.com>
  9. */
  10. #ifndef _LINUX_SRCU_TINY_H
  11. #define _LINUX_SRCU_TINY_H
  12. #include <linux/swait.h>
  13. struct srcu_struct {
  14. short srcu_lock_nesting[2]; /* srcu_read_lock() nesting depth. */
  15. unsigned short srcu_idx; /* Current reader array element in bit 0x2. */
  16. unsigned short srcu_idx_max; /* Furthest future srcu_idx request. */
  17. u8 srcu_gp_running; /* GP workqueue running? */
  18. u8 srcu_gp_waiting; /* GP waiting for readers? */
  19. struct swait_queue_head srcu_wq;
  20. /* Last srcu_read_unlock() wakes GP. */
  21. struct rcu_head *srcu_cb_head; /* Pending callbacks: Head. */
  22. struct rcu_head **srcu_cb_tail; /* Pending callbacks: Tail. */
  23. struct work_struct srcu_work; /* For driving grace periods. */
  24. #ifdef CONFIG_DEBUG_LOCK_ALLOC
  25. struct lockdep_map dep_map;
  26. #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
  27. };
  28. void srcu_drive_gp(struct work_struct *wp);
  29. #define __SRCU_STRUCT_INIT(name, __ignored) \
  30. { \
  31. .srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq), \
  32. .srcu_cb_tail = &name.srcu_cb_head, \
  33. .srcu_work = __WORK_INITIALIZER(name.srcu_work, srcu_drive_gp), \
  34. __SRCU_DEP_MAP_INIT(name) \
  35. }
  36. /*
  37. * This odd _STATIC_ arrangement is needed for API compatibility with
  38. * Tree SRCU, which needs some per-CPU data.
  39. */
  40. #define DEFINE_SRCU(name) \
  41. struct srcu_struct name = __SRCU_STRUCT_INIT(name, name)
  42. #define DEFINE_STATIC_SRCU(name) \
  43. static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name)
  44. void synchronize_srcu(struct srcu_struct *ssp);
  45. /*
  46. * Counts the new reader in the appropriate per-CPU element of the
  47. * srcu_struct. Can be invoked from irq/bh handlers, but the matching
  48. * __srcu_read_unlock() must be in the same handler instance. Returns an
  49. * index that must be passed to the matching srcu_read_unlock().
  50. */
  51. static inline int __srcu_read_lock(struct srcu_struct *ssp)
  52. {
  53. int idx;
  54. idx = ((READ_ONCE(ssp->srcu_idx) + 1) & 0x2) >> 1;
  55. WRITE_ONCE(ssp->srcu_lock_nesting[idx], ssp->srcu_lock_nesting[idx] + 1);
  56. return idx;
  57. }
  58. static inline void synchronize_srcu_expedited(struct srcu_struct *ssp)
  59. {
  60. synchronize_srcu(ssp);
  61. }
  62. static inline void srcu_barrier(struct srcu_struct *ssp)
  63. {
  64. synchronize_srcu(ssp);
  65. }
  66. /* Defined here to avoid size increase for non-torture kernels. */
  67. static inline void srcu_torture_stats_print(struct srcu_struct *ssp,
  68. char *tt, char *tf)
  69. {
  70. int idx;
  71. idx = ((READ_ONCE(ssp->srcu_idx) + 1) & 0x2) >> 1;
  72. pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd)\n",
  73. tt, tf, idx,
  74. READ_ONCE(ssp->srcu_lock_nesting[!idx]),
  75. READ_ONCE(ssp->srcu_lock_nesting[idx]));
  76. }
  77. #endif