percpu.h 3.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106
  1. #ifndef __LINUX_PERCPU_H
  2. #define __LINUX_PERCPU_H
  3. #include <linux/spinlock.h> /* For preempt_disable() */
  4. #include <linux/slab.h> /* For kmalloc() */
  5. #include <linux/smp.h>
  6. #include <linux/string.h> /* For memset() */
  7. #include <linux/cpumask.h>
  8. #include <asm/percpu.h>
  9. /* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */
  10. #ifndef PERCPU_ENOUGH_ROOM
  11. #define PERCPU_ENOUGH_ROOM 32768
  12. #endif
  13. /*
  14. * Must be an lvalue. Since @var must be a simple identifier,
  15. * we force a syntax error here if it isn't.
  16. */
  17. #define get_cpu_var(var) (*({ \
  18. extern int simple_identifier_##var(void); \
  19. preempt_disable(); \
  20. &__get_cpu_var(var); }))
  21. #define put_cpu_var(var) preempt_enable()
  22. #ifdef CONFIG_SMP
  23. struct percpu_data {
  24. void *ptrs[NR_CPUS];
  25. };
  26. #define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata)
  27. /*
  28. * Use this to get to a cpu's version of the per-cpu object dynamically
  29. * allocated. Non-atomic access to the current CPU's version should
  30. * probably be combined with get_cpu()/put_cpu().
  31. */
  32. #define percpu_ptr(ptr, cpu) \
  33. ({ \
  34. struct percpu_data *__p = __percpu_disguise(ptr); \
  35. (__typeof__(ptr))__p->ptrs[(cpu)]; \
  36. })
  37. extern void *percpu_populate(void *__pdata, size_t size, gfp_t gfp, int cpu);
  38. extern void percpu_depopulate(void *__pdata, int cpu);
  39. extern int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp,
  40. cpumask_t *mask);
  41. extern void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask);
  42. extern void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask);
  43. extern void percpu_free(void *__pdata);
  44. #else /* CONFIG_SMP */
  45. #define percpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
  46. static inline void percpu_depopulate(void *__pdata, int cpu)
  47. {
  48. }
  49. static inline void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask)
  50. {
  51. }
  52. static inline void *percpu_populate(void *__pdata, size_t size, gfp_t gfp,
  53. int cpu)
  54. {
  55. return percpu_ptr(__pdata, cpu);
  56. }
  57. static inline int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp,
  58. cpumask_t *mask)
  59. {
  60. return 0;
  61. }
  62. static __always_inline void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask)
  63. {
  64. return kzalloc(size, gfp);
  65. }
  66. static inline void percpu_free(void *__pdata)
  67. {
  68. kfree(__pdata);
  69. }
  70. #endif /* CONFIG_SMP */
  71. #define percpu_populate_mask(__pdata, size, gfp, mask) \
  72. __percpu_populate_mask((__pdata), (size), (gfp), &(mask))
  73. #define percpu_depopulate_mask(__pdata, mask) \
  74. __percpu_depopulate_mask((__pdata), &(mask))
  75. #define percpu_alloc_mask(size, gfp, mask) \
  76. __percpu_alloc_mask((size), (gfp), &(mask))
  77. #define percpu_alloc(size, gfp) percpu_alloc_mask((size), (gfp), cpu_online_map)
  78. /* (legacy) interface for use without CPU hotplug handling */
  79. #define __alloc_percpu(size) percpu_alloc_mask((size), GFP_KERNEL, \
  80. cpu_possible_map)
  81. #define alloc_percpu(type) (type *)__alloc_percpu(sizeof(type))
  82. #define free_percpu(ptr) percpu_free((ptr))
  83. #define per_cpu_ptr(ptr, cpu) percpu_ptr((ptr), (cpu))
  84. #endif /* __LINUX_PERCPU_H */