sbi_tlb.c 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429
  1. /*
  2. * SPDX-License-Identifier: BSD-2-Clause
  3. *
  4. * Copyright (c) 2019 Western Digital Corporation or its affiliates.
  5. *
  6. * Authors:
  7. * Atish Patra <atish.patra@wdc.com>
  8. * Anup Patel <anup.patel@wdc.com>
  9. */
  10. #include <sbi/riscv_asm.h>
  11. #include <sbi/riscv_atomic.h>
  12. #include <sbi/riscv_barrier.h>
  13. #include <sbi/sbi_error.h>
  14. #include <sbi/sbi_fifo.h>
  15. #include <sbi/sbi_hart.h>
  16. #include <sbi/sbi_ipi.h>
  17. #include <sbi/sbi_scratch.h>
  18. #include <sbi/sbi_tlb.h>
  19. #include <sbi/sbi_hfence.h>
  20. #include <sbi/sbi_string.h>
  21. #include <sbi/sbi_console.h>
  22. #include <sbi/sbi_platform.h>
  23. static unsigned long tlb_sync_off;
  24. static unsigned long tlb_fifo_off;
  25. static unsigned long tlb_fifo_mem_off;
  26. static unsigned long tlb_range_flush_limit;
  27. static void sbi_tlb_flush_all(void)
  28. {
  29. __asm__ __volatile("sfence.vma");
  30. }
  31. void sbi_tlb_local_hfence_vvma(struct sbi_tlb_info *tinfo)
  32. {
  33. unsigned long start = tinfo->start;
  34. unsigned long size = tinfo->size;
  35. unsigned long vmid = tinfo->vmid;
  36. unsigned long i, hgatp;
  37. hgatp = csr_swap(CSR_HGATP,
  38. (vmid << HGATP_VMID_SHIFT) & HGATP_VMID_MASK);
  39. if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) {
  40. __sbi_hfence_vvma_all();
  41. goto done;
  42. }
  43. for (i = 0; i < size; i += PAGE_SIZE) {
  44. __sbi_hfence_vvma_va(start+i);
  45. }
  46. done:
  47. csr_write(CSR_HGATP, hgatp);
  48. }
  49. void sbi_tlb_local_hfence_gvma(struct sbi_tlb_info *tinfo)
  50. {
  51. unsigned long start = tinfo->start;
  52. unsigned long size = tinfo->size;
  53. unsigned long i;
  54. if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) {
  55. __sbi_hfence_gvma_all();
  56. return;
  57. }
  58. for (i = 0; i < size; i += PAGE_SIZE) {
  59. __sbi_hfence_gvma_gpa(start+i);
  60. }
  61. }
  62. void sbi_tlb_local_sfence_vma(struct sbi_tlb_info *tinfo)
  63. {
  64. unsigned long start = tinfo->start;
  65. unsigned long size = tinfo->size;
  66. unsigned long i;
  67. if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) {
  68. sbi_tlb_flush_all();
  69. return;
  70. }
  71. for (i = 0; i < size; i += PAGE_SIZE) {
  72. __asm__ __volatile__("sfence.vma %0"
  73. :
  74. : "r"(start + i)
  75. : "memory");
  76. }
  77. }
  78. void sbi_tlb_local_hfence_vvma_asid(struct sbi_tlb_info *tinfo)
  79. {
  80. unsigned long start = tinfo->start;
  81. unsigned long size = tinfo->size;
  82. unsigned long asid = tinfo->asid;
  83. unsigned long vmid = tinfo->vmid;
  84. unsigned long i, hgatp;
  85. hgatp = csr_swap(CSR_HGATP,
  86. (vmid << HGATP_VMID_SHIFT) & HGATP_VMID_MASK);
  87. if (start == 0 && size == 0) {
  88. __sbi_hfence_vvma_all();
  89. goto done;
  90. }
  91. if (size == SBI_TLB_FLUSH_ALL) {
  92. __sbi_hfence_vvma_asid(asid);
  93. goto done;
  94. }
  95. for (i = 0; i < size; i += PAGE_SIZE) {
  96. __sbi_hfence_vvma_asid_va(start + i, asid);
  97. }
  98. done:
  99. csr_write(CSR_HGATP, hgatp);
  100. }
  101. void sbi_tlb_local_hfence_gvma_vmid(struct sbi_tlb_info *tinfo)
  102. {
  103. unsigned long start = tinfo->start;
  104. unsigned long size = tinfo->size;
  105. unsigned long vmid = tinfo->vmid;
  106. unsigned long i;
  107. if (start == 0 && size == 0) {
  108. __sbi_hfence_gvma_all();
  109. return;
  110. }
  111. if (size == SBI_TLB_FLUSH_ALL) {
  112. __sbi_hfence_gvma_vmid(vmid);
  113. return;
  114. }
  115. for (i = 0; i < size; i += PAGE_SIZE) {
  116. __sbi_hfence_gvma_vmid_gpa(start + i, vmid);
  117. }
  118. }
  119. void sbi_tlb_local_sfence_vma_asid(struct sbi_tlb_info *tinfo)
  120. {
  121. unsigned long start = tinfo->start;
  122. unsigned long size = tinfo->size;
  123. unsigned long asid = tinfo->asid;
  124. unsigned long i;
  125. if (start == 0 && size == 0) {
  126. sbi_tlb_flush_all();
  127. return;
  128. }
  129. /* Flush entire MM context for a given ASID */
  130. if (size == SBI_TLB_FLUSH_ALL) {
  131. __asm__ __volatile__("sfence.vma x0, %0"
  132. :
  133. : "r"(asid)
  134. : "memory");
  135. return;
  136. }
  137. for (i = 0; i < size; i += PAGE_SIZE) {
  138. __asm__ __volatile__("sfence.vma %0, %1"
  139. :
  140. : "r"(start + i), "r"(asid)
  141. : "memory");
  142. }
  143. }
  144. void sbi_tlb_local_fence_i(struct sbi_tlb_info *tinfo)
  145. {
  146. __asm__ __volatile("fence.i");
  147. }
  148. static void sbi_tlb_entry_process(struct sbi_tlb_info *tinfo)
  149. {
  150. u32 rhartid;
  151. struct sbi_scratch *rscratch = NULL;
  152. unsigned long *rtlb_sync = NULL;
  153. tinfo->local_fn(tinfo);
  154. sbi_hartmask_for_each_hart(rhartid, &tinfo->smask) {
  155. rscratch = sbi_hartid_to_scratch(rhartid);
  156. if (!rscratch)
  157. continue;
  158. rtlb_sync = sbi_scratch_offset_ptr(rscratch, tlb_sync_off);
  159. while (atomic_raw_xchg_ulong(rtlb_sync, 1)) ;
  160. }
  161. }
  162. static void sbi_tlb_process_count(struct sbi_scratch *scratch, int count)
  163. {
  164. struct sbi_tlb_info tinfo;
  165. u32 deq_count = 0;
  166. struct sbi_fifo *tlb_fifo =
  167. sbi_scratch_offset_ptr(scratch, tlb_fifo_off);
  168. while (!sbi_fifo_dequeue(tlb_fifo, &tinfo)) {
  169. sbi_tlb_entry_process(&tinfo);
  170. deq_count++;
  171. if (deq_count > count)
  172. break;
  173. }
  174. }
  175. static void sbi_tlb_process(struct sbi_scratch *scratch)
  176. {
  177. struct sbi_tlb_info tinfo;
  178. struct sbi_fifo *tlb_fifo =
  179. sbi_scratch_offset_ptr(scratch, tlb_fifo_off);
  180. while (!sbi_fifo_dequeue(tlb_fifo, &tinfo))
  181. sbi_tlb_entry_process(&tinfo);
  182. }
  183. static void sbi_tlb_sync(struct sbi_scratch *scratch)
  184. {
  185. unsigned long *tlb_sync =
  186. sbi_scratch_offset_ptr(scratch, tlb_sync_off);
  187. while (!atomic_raw_xchg_ulong(tlb_sync, 0)) {
  188. /*
  189. * While we are waiting for remote hart to set the sync,
  190. * consume fifo requests to avoid deadlock.
  191. */
  192. sbi_tlb_process_count(scratch, 1);
  193. }
  194. return;
  195. }
  196. static inline int __sbi_tlb_range_check(struct sbi_tlb_info *curr,
  197. struct sbi_tlb_info *next)
  198. {
  199. unsigned long curr_end;
  200. unsigned long next_end;
  201. int ret = SBI_FIFO_UNCHANGED;
  202. if (!curr || !next)
  203. return ret;
  204. next_end = next->start + next->size;
  205. curr_end = curr->start + curr->size;
  206. if (next->start <= curr->start && next_end > curr_end) {
  207. curr->start = next->start;
  208. curr->size = next->size;
  209. sbi_hartmask_or(&curr->smask, &curr->smask, &next->smask);
  210. ret = SBI_FIFO_UPDATED;
  211. } else if (next->start >= curr->start && next_end <= curr_end) {
  212. sbi_hartmask_or(&curr->smask, &curr->smask, &next->smask);
  213. ret = SBI_FIFO_SKIP;
  214. }
  215. return ret;
  216. }
  217. /**
  218. * Call back to decide if an inplace fifo update is required or next entry can
  219. * can be skipped. Here are the different cases that are being handled.
  220. *
  221. * Case1:
  222. * if next flush request range lies within one of the existing entry, skip
  223. * the next entry.
  224. * Case2:
  225. * if flush request range in current fifo entry lies within next flush
  226. * request, update the current entry.
  227. *
  228. * Note:
  229. * We can not issue a fifo reset anymore if a complete vma flush is requested.
  230. * This is because we are queueing FENCE.I requests as well now.
  231. * To ease up the pressure in enqueue/fifo sync path, try to dequeue 1 element
  232. * before continuing the while loop. This method is preferred over wfi/ipi because
  233. * of MMIO cost involved in later method.
  234. */
  235. static int sbi_tlb_update_cb(void *in, void *data)
  236. {
  237. struct sbi_tlb_info *curr;
  238. struct sbi_tlb_info *next;
  239. int ret = SBI_FIFO_UNCHANGED;
  240. if (!in || !data)
  241. return ret;
  242. curr = (struct sbi_tlb_info *)data;
  243. next = (struct sbi_tlb_info *)in;
  244. if (next->local_fn == sbi_tlb_local_sfence_vma_asid &&
  245. curr->local_fn == sbi_tlb_local_sfence_vma_asid) {
  246. if (next->asid == curr->asid)
  247. ret = __sbi_tlb_range_check(curr, next);
  248. } else if (next->local_fn == sbi_tlb_local_sfence_vma &&
  249. curr->local_fn == sbi_tlb_local_sfence_vma) {
  250. ret = __sbi_tlb_range_check(curr, next);
  251. }
  252. return ret;
  253. }
  254. static int sbi_tlb_update(struct sbi_scratch *scratch,
  255. struct sbi_scratch *remote_scratch,
  256. u32 remote_hartid, void *data)
  257. {
  258. int ret;
  259. struct sbi_fifo *tlb_fifo_r;
  260. struct sbi_tlb_info *tinfo = data;
  261. u32 curr_hartid = current_hartid();
  262. /*
  263. * If address range to flush is too big then simply
  264. * upgrade it to flush all because we can only flush
  265. * 4KB at a time.
  266. */
  267. if (tinfo->size > tlb_range_flush_limit) {
  268. tinfo->start = 0;
  269. tinfo->size = SBI_TLB_FLUSH_ALL;
  270. }
  271. /*
  272. * If the request is to queue a tlb flush entry for itself
  273. * then just do a local flush and return;
  274. */
  275. if (remote_hartid == curr_hartid) {
  276. tinfo->local_fn(tinfo);
  277. return -1;
  278. }
  279. tlb_fifo_r = sbi_scratch_offset_ptr(remote_scratch, tlb_fifo_off);
  280. ret = sbi_fifo_inplace_update(tlb_fifo_r, data, sbi_tlb_update_cb);
  281. if (ret != SBI_FIFO_UNCHANGED) {
  282. return 1;
  283. }
  284. while (sbi_fifo_enqueue(tlb_fifo_r, data) < 0) {
  285. /**
  286. * For now, Busy loop until there is space in the fifo.
  287. * There may be case where target hart is also
  288. * enqueue in source hart's fifo. Both hart may busy
  289. * loop leading to a deadlock.
  290. * TODO: Introduce a wait/wakeup event mechanism to handle
  291. * this properly.
  292. */
  293. sbi_tlb_process_count(scratch, 1);
  294. sbi_dprintf("hart%d: hart%d tlb fifo full\n",
  295. curr_hartid, remote_hartid);
  296. }
  297. return 0;
  298. }
  299. static struct sbi_ipi_event_ops tlb_ops = {
  300. .name = "IPI_TLB",
  301. .update = sbi_tlb_update,
  302. .sync = sbi_tlb_sync,
  303. .process = sbi_tlb_process,
  304. };
  305. static u32 tlb_event = SBI_IPI_EVENT_MAX;
  306. int sbi_tlb_request(ulong hmask, ulong hbase, struct sbi_tlb_info *tinfo)
  307. {
  308. if (!tinfo->local_fn)
  309. return SBI_EINVAL;
  310. return sbi_ipi_send_many(hmask, hbase, tlb_event, tinfo);
  311. }
  312. int sbi_tlb_init(struct sbi_scratch *scratch, bool cold_boot)
  313. {
  314. int ret;
  315. void *tlb_mem;
  316. unsigned long *tlb_sync;
  317. struct sbi_fifo *tlb_q;
  318. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  319. if (cold_boot) {
  320. tlb_sync_off = sbi_scratch_alloc_offset(sizeof(*tlb_sync),
  321. "IPI_TLB_SYNC");
  322. if (!tlb_sync_off)
  323. return SBI_ENOMEM;
  324. tlb_fifo_off = sbi_scratch_alloc_offset(sizeof(*tlb_q),
  325. "IPI_TLB_FIFO");
  326. if (!tlb_fifo_off) {
  327. sbi_scratch_free_offset(tlb_sync_off);
  328. return SBI_ENOMEM;
  329. }
  330. tlb_fifo_mem_off = sbi_scratch_alloc_offset(
  331. SBI_TLB_FIFO_NUM_ENTRIES * SBI_TLB_INFO_SIZE,
  332. "IPI_TLB_FIFO_MEM");
  333. if (!tlb_fifo_mem_off) {
  334. sbi_scratch_free_offset(tlb_fifo_off);
  335. sbi_scratch_free_offset(tlb_sync_off);
  336. return SBI_ENOMEM;
  337. }
  338. ret = sbi_ipi_event_create(&tlb_ops);
  339. if (ret < 0) {
  340. sbi_scratch_free_offset(tlb_fifo_mem_off);
  341. sbi_scratch_free_offset(tlb_fifo_off);
  342. sbi_scratch_free_offset(tlb_sync_off);
  343. return ret;
  344. }
  345. tlb_event = ret;
  346. tlb_range_flush_limit = sbi_platform_tlbr_flush_limit(plat);
  347. } else {
  348. if (!tlb_sync_off ||
  349. !tlb_fifo_off ||
  350. !tlb_fifo_mem_off)
  351. return SBI_ENOMEM;
  352. if (SBI_IPI_EVENT_MAX <= tlb_event)
  353. return SBI_ENOSPC;
  354. }
  355. tlb_sync = sbi_scratch_offset_ptr(scratch, tlb_sync_off);
  356. tlb_q = sbi_scratch_offset_ptr(scratch, tlb_fifo_off);
  357. tlb_mem = sbi_scratch_offset_ptr(scratch, tlb_fifo_mem_off);
  358. *tlb_sync = 0;
  359. sbi_fifo_init(tlb_q, tlb_mem,
  360. SBI_TLB_FIFO_NUM_ENTRIES, SBI_TLB_INFO_SIZE);
  361. return 0;
  362. }