barrier.h 2.4 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576
  1. /* SPDX-License-Identifier: GPL-2.0-only */
  2. /*
  3. * Based on arch/arm/include/asm/barrier.h
  4. *
  5. * Copyright (C) 2012 ARM Ltd.
  6. * Copyright (C) 2013 Regents of the University of California
  7. * Copyright (C) 2017 SiFive
  8. */
  9. #ifndef _ASM_RISCV_BARRIER_H
  10. #define _ASM_RISCV_BARRIER_H
  11. #ifndef __ASSEMBLY__
  12. #define nop() __asm__ __volatile__ ("nop")
  13. #define RISCV_FENCE(p, s) \
  14. __asm__ __volatile__ ("fence " #p "," #s : : : "memory")
  15. /* These barriers need to enforce ordering on both devices or memory. */
  16. #define mb() RISCV_FENCE(iorw,iorw)
  17. #define rmb() RISCV_FENCE(ir,ir)
  18. #define wmb() RISCV_FENCE(ow,ow)
  19. /* These barriers do not need to enforce ordering on devices, just memory. */
  20. #define __smp_mb() RISCV_FENCE(rw,rw)
  21. #define __smp_rmb() RISCV_FENCE(r,r)
  22. #define __smp_wmb() RISCV_FENCE(w,w)
  23. #define __smp_store_release(p, v) \
  24. do { \
  25. compiletime_assert_atomic_type(*p); \
  26. RISCV_FENCE(rw,w); \
  27. WRITE_ONCE(*p, v); \
  28. } while (0)
  29. #define __smp_load_acquire(p) \
  30. ({ \
  31. typeof(*p) ___p1 = READ_ONCE(*p); \
  32. compiletime_assert_atomic_type(*p); \
  33. RISCV_FENCE(r,rw); \
  34. ___p1; \
  35. })
  36. /*
  37. * This is a very specific barrier: it's currently only used in two places in
  38. * the kernel, both in the scheduler. See include/linux/spinlock.h for the two
  39. * orderings it guarantees, but the "critical section is RCsc" guarantee
  40. * mandates a barrier on RISC-V. The sequence looks like:
  41. *
  42. * lr.aq lock
  43. * sc lock <= LOCKED
  44. * smp_mb__after_spinlock()
  45. * // critical section
  46. * lr lock
  47. * sc.rl lock <= UNLOCKED
  48. *
  49. * The AQ/RL pair provides a RCpc critical section, but there's not really any
  50. * way we can take advantage of that here because the ordering is only enforced
  51. * on that one lock. Thus, we're just doing a full fence.
  52. *
  53. * Since we allow writeX to be called from preemptive regions we need at least
  54. * an "o" in the predecessor set to ensure device writes are visible before the
  55. * task is marked as available for scheduling on a new hart. While I don't see
  56. * any concrete reason we need a full IO fence, it seems safer to just upgrade
  57. * this in order to avoid any IO crossing a scheduling boundary. In both
  58. * instances the scheduler pairs this with an mb(), so nothing is necessary on
  59. * the new hart.
  60. */
  61. #define smp_mb__after_spinlock() RISCV_FENCE(iorw,iorw)
  62. #include <asm-generic/barrier.h>
  63. #endif /* __ASSEMBLY__ */
  64. #endif /* _ASM_RISCV_BARRIER_H */