sbi_ipi.c 6.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307
  1. /*
  2. * SPDX-License-Identifier: BSD-2-Clause
  3. *
  4. * Copyright (c) 2019 Western Digital Corporation or its affiliates.
  5. *
  6. * Authors:
  7. * Anup Patel <anup.patel@wdc.com>
  8. * Nick Kossifidis <mick@ics.forth.gr>
  9. */
  10. #include <sbi/riscv_asm.h>
  11. #include <sbi/riscv_atomic.h>
  12. #include <sbi/riscv_barrier.h>
  13. #include <sbi/sbi_console.h>
  14. #include <sbi/sbi_bitops.h>
  15. #include <sbi/sbi_domain.h>
  16. #include <sbi/sbi_error.h>
  17. #include <sbi/sbi_hart.h>
  18. #include <sbi/sbi_hsm.h>
  19. #include <sbi/sbi_init.h>
  20. #include <sbi/sbi_ipi.h>
  21. #include <sbi/sbi_platform.h>
  22. #include <sbi/sbi_pmu.h>
  23. #include <sbi/sbi_string.h>
  24. #include <sbi/sbi_tlb.h>
  25. struct sbi_ipi_data {
  26. unsigned long ipi_type;
  27. };
  28. static unsigned long ipi_data_off;
  29. static const struct sbi_ipi_device *ipi_dev = NULL;
  30. static const struct sbi_ipi_event_ops *ipi_ops_array[SBI_IPI_EVENT_MAX];
  31. static int sbi_ipi_send(struct sbi_scratch *scratch, u32 remote_hartid,
  32. u32 event, void *data)
  33. {
  34. int ret;
  35. struct sbi_scratch *remote_scratch = NULL;
  36. struct sbi_ipi_data *ipi_data;
  37. const struct sbi_ipi_event_ops *ipi_ops;
  38. if ((SBI_IPI_EVENT_MAX <= event) ||
  39. !ipi_ops_array[event])
  40. return SBI_EINVAL;
  41. ipi_ops = ipi_ops_array[event];
  42. remote_scratch = sbi_hartid_to_scratch(remote_hartid);
  43. if (!remote_scratch)
  44. return SBI_EINVAL;
  45. ipi_data = sbi_scratch_offset_ptr(remote_scratch, ipi_data_off);
  46. if (ipi_ops->update) {
  47. ret = ipi_ops->update(scratch, remote_scratch,
  48. remote_hartid, data);
  49. if (ret < 0)
  50. return ret;
  51. }
  52. /*
  53. * Set IPI type on remote hart's scratch area and
  54. * trigger the interrupt
  55. */
  56. atomic_raw_set_bit(event, &ipi_data->ipi_type);
  57. smp_wmb();
  58. if (ipi_dev && ipi_dev->ipi_send)
  59. ipi_dev->ipi_send(remote_hartid);
  60. sbi_pmu_ctr_incr_fw(SBI_PMU_FW_IPI_SENT);
  61. if (ipi_ops->sync)
  62. ipi_ops->sync(scratch);
  63. return 0;
  64. }
  65. /**
  66. * As this this function only handlers scalar values of hart mask, it must be
  67. * set to all online harts if the intention is to send IPIs to all the harts.
  68. * If hmask is zero, no IPIs will be sent.
  69. */
  70. int sbi_ipi_send_many(ulong hmask, ulong hbase, u32 event, void *data)
  71. {
  72. int rc;
  73. ulong i, m;
  74. struct sbi_domain *dom = sbi_domain_thishart_ptr();
  75. struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
  76. if (hbase != -1UL) {
  77. rc = sbi_hsm_hart_interruptible_mask(dom, hbase, &m);
  78. if (rc)
  79. return rc;
  80. m &= hmask;
  81. /* Send IPIs */
  82. for (i = hbase; m; i++, m >>= 1) {
  83. if (m & 1UL)
  84. sbi_ipi_send(scratch, i, event, data);
  85. }
  86. } else {
  87. hbase = 0;
  88. while (!sbi_hsm_hart_interruptible_mask(dom, hbase, &m)) {
  89. /* Send IPIs */
  90. for (i = hbase; m; i++, m >>= 1) {
  91. if (m & 1UL)
  92. sbi_ipi_send(scratch, i, event, data);
  93. }
  94. hbase += BITS_PER_LONG;
  95. }
  96. }
  97. return 0;
  98. }
  99. int sbi_ipi_event_create(const struct sbi_ipi_event_ops *ops)
  100. {
  101. int i, ret = SBI_ENOSPC;
  102. if (!ops || !ops->process)
  103. return SBI_EINVAL;
  104. for (i = 0; i < SBI_IPI_EVENT_MAX; i++) {
  105. if (!ipi_ops_array[i]) {
  106. ret = i;
  107. ipi_ops_array[i] = ops;
  108. break;
  109. }
  110. }
  111. return ret;
  112. }
  113. void sbi_ipi_event_destroy(u32 event)
  114. {
  115. if (SBI_IPI_EVENT_MAX <= event)
  116. return;
  117. ipi_ops_array[event] = NULL;
  118. }
  119. static void sbi_ipi_process_smode(struct sbi_scratch *scratch)
  120. {
  121. csr_set(CSR_MIP, MIP_SSIP);
  122. }
  123. static struct sbi_ipi_event_ops ipi_smode_ops = {
  124. .name = "IPI_SMODE",
  125. .process = sbi_ipi_process_smode,
  126. };
  127. static u32 ipi_smode_event = SBI_IPI_EVENT_MAX;
  128. static unsigned long* amp_data_addr;
  129. int sbi_ipi_send_smode(ulong hmask, ulong hbase)
  130. {
  131. return sbi_ipi_send_many(hmask, hbase, ipi_smode_event, NULL);
  132. }
  133. int sbi_ipi_send_ext(u32 hartid, void *data, u32 msg_bits)
  134. {
  135. if (!amp_data_addr)
  136. return SBI_EINVAL;
  137. atomic_raw_set_bit((1 << msg_bits), (void *)(amp_data_addr + hartid));
  138. return sbi_ipi_send(sbi_scratch_thishart_ptr(), hartid, ipi_smode_event, NULL);
  139. }
  140. void sbi_ipi_set_amp_data_addr(unsigned long addr)
  141. {
  142. amp_data_addr = (void *)addr;
  143. }
  144. void sbi_ipi_clear_smode(void)
  145. {
  146. csr_clear(CSR_MIP, MIP_SSIP);
  147. }
  148. static void sbi_ipi_process_halt(struct sbi_scratch *scratch)
  149. {
  150. sbi_hsm_hart_stop(scratch, true);
  151. }
  152. static struct sbi_ipi_event_ops ipi_halt_ops = {
  153. .name = "IPI_HALT",
  154. .process = sbi_ipi_process_halt,
  155. };
  156. static u32 ipi_halt_event = SBI_IPI_EVENT_MAX;
  157. int sbi_ipi_send_halt(ulong hmask, ulong hbase)
  158. {
  159. return sbi_ipi_send_many(hmask, hbase, ipi_halt_event, NULL);
  160. }
  161. void sbi_ipi_process(void)
  162. {
  163. unsigned long ipi_type;
  164. unsigned int ipi_event;
  165. const struct sbi_ipi_event_ops *ipi_ops;
  166. struct sbi_scratch *scratch = sbi_scratch_thishart_ptr();
  167. struct sbi_ipi_data *ipi_data =
  168. sbi_scratch_offset_ptr(scratch, ipi_data_off);
  169. u32 hartid = current_hartid();
  170. sbi_pmu_ctr_incr_fw(SBI_PMU_FW_IPI_RECVD);
  171. if (ipi_dev && ipi_dev->ipi_clear)
  172. ipi_dev->ipi_clear(hartid);
  173. ipi_type = atomic_raw_xchg_ulong(&ipi_data->ipi_type, 0);
  174. ipi_event = 0;
  175. while (ipi_type) {
  176. if (!(ipi_type & 1UL))
  177. goto skip;
  178. ipi_ops = ipi_ops_array[ipi_event];
  179. if (ipi_ops && ipi_ops->process)
  180. ipi_ops->process(scratch);
  181. skip:
  182. ipi_type = ipi_type >> 1;
  183. ipi_event++;
  184. };
  185. }
  186. int sbi_ipi_raw_send(u32 target_hart)
  187. {
  188. if (!ipi_dev || !ipi_dev->ipi_send)
  189. return SBI_EINVAL;
  190. ipi_dev->ipi_send(target_hart);
  191. return 0;
  192. }
  193. void sbi_ipi_raw_clear(u32 target_hart)
  194. {
  195. if (ipi_dev && ipi_dev->ipi_clear)
  196. ipi_dev->ipi_clear(target_hart);
  197. }
  198. const struct sbi_ipi_device *sbi_ipi_get_device(void)
  199. {
  200. return ipi_dev;
  201. }
  202. void sbi_ipi_set_device(const struct sbi_ipi_device *dev)
  203. {
  204. if (!dev || ipi_dev)
  205. return;
  206. ipi_dev = dev;
  207. }
  208. int sbi_ipi_init(struct sbi_scratch *scratch, bool cold_boot)
  209. {
  210. int ret;
  211. struct sbi_ipi_data *ipi_data;
  212. if (cold_boot) {
  213. ipi_data_off = sbi_scratch_alloc_offset(sizeof(*ipi_data));
  214. if (!ipi_data_off)
  215. return SBI_ENOMEM;
  216. ret = sbi_ipi_event_create(&ipi_smode_ops);
  217. if (ret < 0)
  218. return ret;
  219. ipi_smode_event = ret;
  220. ret = sbi_ipi_event_create(&ipi_halt_ops);
  221. if (ret < 0)
  222. return ret;
  223. ipi_halt_event = ret;
  224. } else {
  225. if (!ipi_data_off)
  226. return SBI_ENOMEM;
  227. if (SBI_IPI_EVENT_MAX <= ipi_smode_event ||
  228. SBI_IPI_EVENT_MAX <= ipi_halt_event)
  229. return SBI_ENOSPC;
  230. }
  231. ipi_data = sbi_scratch_offset_ptr(scratch, ipi_data_off);
  232. ipi_data->ipi_type = 0x00;
  233. /*
  234. * Initialize platform IPI support. This will also clear any
  235. * pending IPIs for current/calling HART.
  236. */
  237. ret = sbi_platform_ipi_init(sbi_platform_ptr(scratch), cold_boot);
  238. if (ret)
  239. return ret;
  240. /* Enable software interrupts */
  241. csr_set(CSR_MIE, MIP_MSIP);
  242. return 0;
  243. }
  244. void sbi_ipi_exit(struct sbi_scratch *scratch)
  245. {
  246. /* Disable software interrupts */
  247. csr_clear(CSR_MIE, MIP_MSIP);
  248. /* Process pending IPIs */
  249. sbi_ipi_process();
  250. /* Platform exit */
  251. sbi_platform_ipi_exit(sbi_platform_ptr(scratch));
  252. }