percpu-defs.h 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527
  1. /* SPDX-License-Identifier: GPL-2.0-only */
  2. /*
  3. * linux/percpu-defs.h - basic definitions for percpu areas
  4. *
  5. * DO NOT INCLUDE DIRECTLY OUTSIDE PERCPU IMPLEMENTATION PROPER.
  6. *
  7. * This file is separate from linux/percpu.h to avoid cyclic inclusion
  8. * dependency from arch header files. Only to be included from
  9. * asm/percpu.h.
  10. *
  11. * This file includes macros necessary to declare percpu sections and
  12. * variables, and definitions of percpu accessors and operations. It
  13. * should provide enough percpu features to arch header files even when
  14. * they can only include asm/percpu.h to avoid cyclic inclusion dependency.
  15. */
  16. #ifndef _LINUX_PERCPU_DEFS_H
  17. #define _LINUX_PERCPU_DEFS_H
  18. #ifdef CONFIG_SMP
  19. #ifdef MODULE
  20. #define PER_CPU_SHARED_ALIGNED_SECTION ""
  21. #define PER_CPU_ALIGNED_SECTION ""
  22. #else
  23. #define PER_CPU_SHARED_ALIGNED_SECTION "..shared_aligned"
  24. #define PER_CPU_ALIGNED_SECTION "..shared_aligned"
  25. #endif
  26. #define PER_CPU_FIRST_SECTION "..first"
  27. #else
  28. #define PER_CPU_SHARED_ALIGNED_SECTION ""
  29. #define PER_CPU_ALIGNED_SECTION "..shared_aligned"
  30. #define PER_CPU_FIRST_SECTION ""
  31. #endif
  32. /*
  33. * Base implementations of per-CPU variable declarations and definitions, where
  34. * the section in which the variable is to be placed is provided by the
  35. * 'sec' argument. This may be used to affect the parameters governing the
  36. * variable's storage.
  37. *
  38. * NOTE! The sections for the DECLARE and for the DEFINE must match, lest
  39. * linkage errors occur due the compiler generating the wrong code to access
  40. * that section.
  41. */
  42. #define __PCPU_ATTRS(sec) \
  43. __percpu __attribute__((section(PER_CPU_BASE_SECTION sec))) \
  44. PER_CPU_ATTRIBUTES
  45. #define __PCPU_DUMMY_ATTRS \
  46. __section(".discard") __attribute__((unused))
  47. /*
  48. * s390 and alpha modules require percpu variables to be defined as
  49. * weak to force the compiler to generate GOT based external
  50. * references for them. This is necessary because percpu sections
  51. * will be located outside of the usually addressable area.
  52. *
  53. * This definition puts the following two extra restrictions when
  54. * defining percpu variables.
  55. *
  56. * 1. The symbol must be globally unique, even the static ones.
  57. * 2. Static percpu variables cannot be defined inside a function.
  58. *
  59. * Archs which need weak percpu definitions should define
  60. * ARCH_NEEDS_WEAK_PER_CPU in asm/percpu.h when necessary.
  61. *
  62. * To ensure that the generic code observes the above two
  63. * restrictions, if CONFIG_DEBUG_FORCE_WEAK_PER_CPU is set weak
  64. * definition is used for all cases.
  65. */
  66. #if defined(ARCH_NEEDS_WEAK_PER_CPU) || defined(CONFIG_DEBUG_FORCE_WEAK_PER_CPU)
  67. /*
  68. * __pcpu_scope_* dummy variable is used to enforce scope. It
  69. * receives the static modifier when it's used in front of
  70. * DEFINE_PER_CPU() and will trigger build failure if
  71. * DECLARE_PER_CPU() is used for the same variable.
  72. *
  73. * __pcpu_unique_* dummy variable is used to enforce symbol uniqueness
  74. * such that hidden weak symbol collision, which will cause unrelated
  75. * variables to share the same address, can be detected during build.
  76. */
  77. #define DECLARE_PER_CPU_SECTION(type, name, sec) \
  78. extern __PCPU_DUMMY_ATTRS char __pcpu_scope_##name; \
  79. extern __PCPU_ATTRS(sec) __typeof__(type) name
  80. #define DEFINE_PER_CPU_SECTION(type, name, sec) \
  81. __PCPU_DUMMY_ATTRS char __pcpu_scope_##name; \
  82. extern __PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \
  83. __PCPU_DUMMY_ATTRS char __pcpu_unique_##name; \
  84. extern __PCPU_ATTRS(sec) __typeof__(type) name; \
  85. __PCPU_ATTRS(sec) __weak __typeof__(type) name
  86. #else
  87. /*
  88. * Normal declaration and definition macros.
  89. */
  90. #define DECLARE_PER_CPU_SECTION(type, name, sec) \
  91. extern __PCPU_ATTRS(sec) __typeof__(type) name
  92. #define DEFINE_PER_CPU_SECTION(type, name, sec) \
  93. __PCPU_ATTRS(sec) __typeof__(type) name
  94. #endif
  95. /*
  96. * Variant on the per-CPU variable declaration/definition theme used for
  97. * ordinary per-CPU variables.
  98. */
  99. #define DECLARE_PER_CPU(type, name) \
  100. DECLARE_PER_CPU_SECTION(type, name, "")
  101. #define DEFINE_PER_CPU(type, name) \
  102. DEFINE_PER_CPU_SECTION(type, name, "")
  103. /*
  104. * Declaration/definition used for per-CPU variables that must come first in
  105. * the set of variables.
  106. */
  107. #define DECLARE_PER_CPU_FIRST(type, name) \
  108. DECLARE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION)
  109. #define DEFINE_PER_CPU_FIRST(type, name) \
  110. DEFINE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION)
  111. /*
  112. * Declaration/definition used for per-CPU variables that must be cacheline
  113. * aligned under SMP conditions so that, whilst a particular instance of the
  114. * data corresponds to a particular CPU, inefficiencies due to direct access by
  115. * other CPUs are reduced by preventing the data from unnecessarily spanning
  116. * cachelines.
  117. *
  118. * An example of this would be statistical data, where each CPU's set of data
  119. * is updated by that CPU alone, but the data from across all CPUs is collated
  120. * by a CPU processing a read from a proc file.
  121. */
  122. #define DECLARE_PER_CPU_SHARED_ALIGNED(type, name) \
  123. DECLARE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \
  124. ____cacheline_aligned_in_smp
  125. #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \
  126. DEFINE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \
  127. ____cacheline_aligned_in_smp
  128. #define DECLARE_PER_CPU_ALIGNED(type, name) \
  129. DECLARE_PER_CPU_SECTION(type, name, PER_CPU_ALIGNED_SECTION) \
  130. ____cacheline_aligned
  131. #define DEFINE_PER_CPU_ALIGNED(type, name) \
  132. DEFINE_PER_CPU_SECTION(type, name, PER_CPU_ALIGNED_SECTION) \
  133. ____cacheline_aligned
  134. /*
  135. * Declaration/definition used for per-CPU variables that must be page aligned.
  136. */
  137. #define DECLARE_PER_CPU_PAGE_ALIGNED(type, name) \
  138. DECLARE_PER_CPU_SECTION(type, name, "..page_aligned") \
  139. __aligned(PAGE_SIZE)
  140. #define DEFINE_PER_CPU_PAGE_ALIGNED(type, name) \
  141. DEFINE_PER_CPU_SECTION(type, name, "..page_aligned") \
  142. __aligned(PAGE_SIZE)
  143. /*
  144. * Declaration/definition used for per-CPU variables that must be read mostly.
  145. */
  146. #define DECLARE_PER_CPU_READ_MOSTLY(type, name) \
  147. DECLARE_PER_CPU_SECTION(type, name, "..read_mostly")
  148. #define DEFINE_PER_CPU_READ_MOSTLY(type, name) \
  149. DEFINE_PER_CPU_SECTION(type, name, "..read_mostly")
  150. /*
  151. * Declaration/definition used for per-CPU variables that should be accessed
  152. * as decrypted when memory encryption is enabled in the guest.
  153. */
  154. #ifdef CONFIG_AMD_MEM_ENCRYPT
  155. #define DECLARE_PER_CPU_DECRYPTED(type, name) \
  156. DECLARE_PER_CPU_SECTION(type, name, "..decrypted")
  157. #define DEFINE_PER_CPU_DECRYPTED(type, name) \
  158. DEFINE_PER_CPU_SECTION(type, name, "..decrypted")
  159. #else
  160. #define DEFINE_PER_CPU_DECRYPTED(type, name) DEFINE_PER_CPU(type, name)
  161. #endif
  162. /*
  163. * Intermodule exports for per-CPU variables. sparse forgets about
  164. * address space across EXPORT_SYMBOL(), change EXPORT_SYMBOL() to
  165. * noop if __CHECKER__.
  166. */
  167. #ifndef __CHECKER__
  168. #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(var)
  169. #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(var)
  170. #else
  171. #define EXPORT_PER_CPU_SYMBOL(var)
  172. #define EXPORT_PER_CPU_SYMBOL_GPL(var)
  173. #endif
  174. /*
  175. * Accessors and operations.
  176. */
  177. #ifndef __ASSEMBLY__
  178. /*
  179. * __verify_pcpu_ptr() verifies @ptr is a percpu pointer without evaluating
  180. * @ptr and is invoked once before a percpu area is accessed by all
  181. * accessors and operations. This is performed in the generic part of
  182. * percpu and arch overrides don't need to worry about it; however, if an
  183. * arch wants to implement an arch-specific percpu accessor or operation,
  184. * it may use __verify_pcpu_ptr() to verify the parameters.
  185. *
  186. * + 0 is required in order to convert the pointer type from a
  187. * potential array type to a pointer to a single item of the array.
  188. */
  189. #define __verify_pcpu_ptr(ptr) \
  190. do { \
  191. const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL; \
  192. (void)__vpp_verify; \
  193. } while (0)
  194. #ifdef CONFIG_SMP
  195. /*
  196. * Add an offset to a pointer but keep the pointer as-is. Use RELOC_HIDE()
  197. * to prevent the compiler from making incorrect assumptions about the
  198. * pointer value. The weird cast keeps both GCC and sparse happy.
  199. */
  200. #define SHIFT_PERCPU_PTR(__p, __offset) \
  201. RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset))
  202. #define per_cpu_ptr(ptr, cpu) \
  203. ({ \
  204. __verify_pcpu_ptr(ptr); \
  205. SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))); \
  206. })
  207. #define raw_cpu_ptr(ptr) \
  208. ({ \
  209. __verify_pcpu_ptr(ptr); \
  210. arch_raw_cpu_ptr(ptr); \
  211. })
  212. #ifdef CONFIG_DEBUG_PREEMPT
  213. #define this_cpu_ptr(ptr) \
  214. ({ \
  215. __verify_pcpu_ptr(ptr); \
  216. SHIFT_PERCPU_PTR(ptr, my_cpu_offset); \
  217. })
  218. #else
  219. #define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
  220. #endif
  221. #else /* CONFIG_SMP */
  222. #define VERIFY_PERCPU_PTR(__p) \
  223. ({ \
  224. __verify_pcpu_ptr(__p); \
  225. (typeof(*(__p)) __kernel __force *)(__p); \
  226. })
  227. #define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR(ptr); })
  228. #define raw_cpu_ptr(ptr) per_cpu_ptr(ptr, 0)
  229. #define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
  230. #endif /* CONFIG_SMP */
  231. #define per_cpu(var, cpu) (*per_cpu_ptr(&(var), cpu))
  232. /*
  233. * Must be an lvalue. Since @var must be a simple identifier,
  234. * we force a syntax error here if it isn't.
  235. */
  236. #define get_cpu_var(var) \
  237. (*({ \
  238. preempt_disable(); \
  239. this_cpu_ptr(&var); \
  240. }))
  241. /*
  242. * The weird & is necessary because sparse considers (void)(var) to be
  243. * a direct dereference of percpu variable (var).
  244. */
  245. #define put_cpu_var(var) \
  246. do { \
  247. (void)&(var); \
  248. preempt_enable(); \
  249. } while (0)
  250. #define get_cpu_ptr(var) \
  251. ({ \
  252. preempt_disable(); \
  253. this_cpu_ptr(var); \
  254. })
  255. #define put_cpu_ptr(var) \
  256. do { \
  257. (void)(var); \
  258. preempt_enable(); \
  259. } while (0)
  260. /*
  261. * Branching function to split up a function into a set of functions that
  262. * are called for different scalar sizes of the objects handled.
  263. */
  264. extern void __bad_size_call_parameter(void);
  265. #ifdef CONFIG_DEBUG_PREEMPT
  266. extern void __this_cpu_preempt_check(const char *op);
  267. #else
  268. static inline void __this_cpu_preempt_check(const char *op) { }
  269. #endif
  270. #define __pcpu_size_call_return(stem, variable) \
  271. ({ \
  272. typeof(variable) pscr_ret__; \
  273. __verify_pcpu_ptr(&(variable)); \
  274. switch(sizeof(variable)) { \
  275. case 1: pscr_ret__ = stem##1(variable); break; \
  276. case 2: pscr_ret__ = stem##2(variable); break; \
  277. case 4: pscr_ret__ = stem##4(variable); break; \
  278. case 8: pscr_ret__ = stem##8(variable); break; \
  279. default: \
  280. __bad_size_call_parameter(); break; \
  281. } \
  282. pscr_ret__; \
  283. })
  284. #define __pcpu_size_call_return2(stem, variable, ...) \
  285. ({ \
  286. typeof(variable) pscr2_ret__; \
  287. __verify_pcpu_ptr(&(variable)); \
  288. switch(sizeof(variable)) { \
  289. case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \
  290. case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \
  291. case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \
  292. case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \
  293. default: \
  294. __bad_size_call_parameter(); break; \
  295. } \
  296. pscr2_ret__; \
  297. })
  298. /*
  299. * Special handling for cmpxchg_double. cmpxchg_double is passed two
  300. * percpu variables. The first has to be aligned to a double word
  301. * boundary and the second has to follow directly thereafter.
  302. * We enforce this on all architectures even if they don't support
  303. * a double cmpxchg instruction, since it's a cheap requirement, and it
  304. * avoids breaking the requirement for architectures with the instruction.
  305. */
  306. #define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \
  307. ({ \
  308. bool pdcrb_ret__; \
  309. __verify_pcpu_ptr(&(pcp1)); \
  310. BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \
  311. VM_BUG_ON((unsigned long)(&(pcp1)) % (2 * sizeof(pcp1))); \
  312. VM_BUG_ON((unsigned long)(&(pcp2)) != \
  313. (unsigned long)(&(pcp1)) + sizeof(pcp1)); \
  314. switch(sizeof(pcp1)) { \
  315. case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \
  316. case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \
  317. case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \
  318. case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \
  319. default: \
  320. __bad_size_call_parameter(); break; \
  321. } \
  322. pdcrb_ret__; \
  323. })
  324. #define __pcpu_size_call(stem, variable, ...) \
  325. do { \
  326. __verify_pcpu_ptr(&(variable)); \
  327. switch(sizeof(variable)) { \
  328. case 1: stem##1(variable, __VA_ARGS__);break; \
  329. case 2: stem##2(variable, __VA_ARGS__);break; \
  330. case 4: stem##4(variable, __VA_ARGS__);break; \
  331. case 8: stem##8(variable, __VA_ARGS__);break; \
  332. default: \
  333. __bad_size_call_parameter();break; \
  334. } \
  335. } while (0)
  336. /*
  337. * this_cpu operations (C) 2008-2013 Christoph Lameter <cl@linux.com>
  338. *
  339. * Optimized manipulation for memory allocated through the per cpu
  340. * allocator or for addresses of per cpu variables.
  341. *
  342. * These operation guarantee exclusivity of access for other operations
  343. * on the *same* processor. The assumption is that per cpu data is only
  344. * accessed by a single processor instance (the current one).
  345. *
  346. * The arch code can provide optimized implementation by defining macros
  347. * for certain scalar sizes. F.e. provide this_cpu_add_2() to provide per
  348. * cpu atomic operations for 2 byte sized RMW actions. If arch code does
  349. * not provide operations for a scalar size then the fallback in the
  350. * generic code will be used.
  351. *
  352. * cmpxchg_double replaces two adjacent scalars at once. The first two
  353. * parameters are per cpu variables which have to be of the same size. A
  354. * truth value is returned to indicate success or failure (since a double
  355. * register result is difficult to handle). There is very limited hardware
  356. * support for these operations, so only certain sizes may work.
  357. */
  358. /*
  359. * Operations for contexts where we do not want to do any checks for
  360. * preemptions. Unless strictly necessary, always use [__]this_cpu_*()
  361. * instead.
  362. *
  363. * If there is no other protection through preempt disable and/or disabling
  364. * interupts then one of these RMW operations can show unexpected behavior
  365. * because the execution thread was rescheduled on another processor or an
  366. * interrupt occurred and the same percpu variable was modified from the
  367. * interrupt context.
  368. */
  369. #define raw_cpu_read(pcp) __pcpu_size_call_return(raw_cpu_read_, pcp)
  370. #define raw_cpu_write(pcp, val) __pcpu_size_call(raw_cpu_write_, pcp, val)
  371. #define raw_cpu_add(pcp, val) __pcpu_size_call(raw_cpu_add_, pcp, val)
  372. #define raw_cpu_and(pcp, val) __pcpu_size_call(raw_cpu_and_, pcp, val)
  373. #define raw_cpu_or(pcp, val) __pcpu_size_call(raw_cpu_or_, pcp, val)
  374. #define raw_cpu_add_return(pcp, val) __pcpu_size_call_return2(raw_cpu_add_return_, pcp, val)
  375. #define raw_cpu_xchg(pcp, nval) __pcpu_size_call_return2(raw_cpu_xchg_, pcp, nval)
  376. #define raw_cpu_cmpxchg(pcp, oval, nval) \
  377. __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)
  378. #define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
  379. __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
  380. #define raw_cpu_sub(pcp, val) raw_cpu_add(pcp, -(val))
  381. #define raw_cpu_inc(pcp) raw_cpu_add(pcp, 1)
  382. #define raw_cpu_dec(pcp) raw_cpu_sub(pcp, 1)
  383. #define raw_cpu_sub_return(pcp, val) raw_cpu_add_return(pcp, -(typeof(pcp))(val))
  384. #define raw_cpu_inc_return(pcp) raw_cpu_add_return(pcp, 1)
  385. #define raw_cpu_dec_return(pcp) raw_cpu_add_return(pcp, -1)
  386. /*
  387. * Operations for contexts that are safe from preemption/interrupts. These
  388. * operations verify that preemption is disabled.
  389. */
  390. #define __this_cpu_read(pcp) \
  391. ({ \
  392. __this_cpu_preempt_check("read"); \
  393. raw_cpu_read(pcp); \
  394. })
  395. #define __this_cpu_write(pcp, val) \
  396. ({ \
  397. __this_cpu_preempt_check("write"); \
  398. raw_cpu_write(pcp, val); \
  399. })
  400. #define __this_cpu_add(pcp, val) \
  401. ({ \
  402. __this_cpu_preempt_check("add"); \
  403. raw_cpu_add(pcp, val); \
  404. })
  405. #define __this_cpu_and(pcp, val) \
  406. ({ \
  407. __this_cpu_preempt_check("and"); \
  408. raw_cpu_and(pcp, val); \
  409. })
  410. #define __this_cpu_or(pcp, val) \
  411. ({ \
  412. __this_cpu_preempt_check("or"); \
  413. raw_cpu_or(pcp, val); \
  414. })
  415. #define __this_cpu_add_return(pcp, val) \
  416. ({ \
  417. __this_cpu_preempt_check("add_return"); \
  418. raw_cpu_add_return(pcp, val); \
  419. })
  420. #define __this_cpu_xchg(pcp, nval) \
  421. ({ \
  422. __this_cpu_preempt_check("xchg"); \
  423. raw_cpu_xchg(pcp, nval); \
  424. })
  425. #define __this_cpu_cmpxchg(pcp, oval, nval) \
  426. ({ \
  427. __this_cpu_preempt_check("cmpxchg"); \
  428. raw_cpu_cmpxchg(pcp, oval, nval); \
  429. })
  430. #define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
  431. ({ __this_cpu_preempt_check("cmpxchg_double"); \
  432. raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2); \
  433. })
  434. #define __this_cpu_sub(pcp, val) __this_cpu_add(pcp, -(typeof(pcp))(val))
  435. #define __this_cpu_inc(pcp) __this_cpu_add(pcp, 1)
  436. #define __this_cpu_dec(pcp) __this_cpu_sub(pcp, 1)
  437. #define __this_cpu_sub_return(pcp, val) __this_cpu_add_return(pcp, -(typeof(pcp))(val))
  438. #define __this_cpu_inc_return(pcp) __this_cpu_add_return(pcp, 1)
  439. #define __this_cpu_dec_return(pcp) __this_cpu_add_return(pcp, -1)
  440. /*
  441. * Operations with implied preemption/interrupt protection. These
  442. * operations can be used without worrying about preemption or interrupt.
  443. */
  444. #define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, pcp)
  445. #define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, pcp, val)
  446. #define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, pcp, val)
  447. #define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, pcp, val)
  448. #define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, pcp, val)
  449. #define this_cpu_add_return(pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
  450. #define this_cpu_xchg(pcp, nval) __pcpu_size_call_return2(this_cpu_xchg_, pcp, nval)
  451. #define this_cpu_cmpxchg(pcp, oval, nval) \
  452. __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
  453. #define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
  454. __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
  455. #define this_cpu_sub(pcp, val) this_cpu_add(pcp, -(typeof(pcp))(val))
  456. #define this_cpu_inc(pcp) this_cpu_add(pcp, 1)
  457. #define this_cpu_dec(pcp) this_cpu_sub(pcp, 1)
  458. #define this_cpu_sub_return(pcp, val) this_cpu_add_return(pcp, -(typeof(pcp))(val))
  459. #define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1)
  460. #define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1)
  461. #endif /* __ASSEMBLY__ */
  462. #endif /* _LINUX_PERCPU_DEFS_H */