sbi_tlb.c 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441
  1. /*
  2. * SPDX-License-Identifier: BSD-2-Clause
  3. *
  4. * Copyright (c) 2019 Western Digital Corporation or its affiliates.
  5. *
  6. * Authors:
  7. * Atish Patra <atish.patra@wdc.com>
  8. * Anup Patel <anup.patel@wdc.com>
  9. */
  10. #include <sbi/riscv_asm.h>
  11. #include <sbi/riscv_atomic.h>
  12. #include <sbi/riscv_barrier.h>
  13. #include <sbi/sbi_error.h>
  14. #include <sbi/sbi_fifo.h>
  15. #include <sbi/sbi_hart.h>
  16. #include <sbi/sbi_ipi.h>
  17. #include <sbi/sbi_scratch.h>
  18. #include <sbi/sbi_tlb.h>
  19. #include <sbi/sbi_hfence.h>
  20. #include <sbi/sbi_string.h>
  21. #include <sbi/sbi_console.h>
  22. #include <sbi/sbi_platform.h>
  23. static unsigned long tlb_sync_off;
  24. static unsigned long tlb_fifo_off;
  25. static unsigned long tlb_fifo_mem_off;
  26. static unsigned long tlb_range_flush_limit;
  27. static void sbi_tlb_flush_all(void)
  28. {
  29. #ifdef FW_NO_SFENCE_VMA
  30. csr_write(0x9c3, 1 << 26);
  31. #else
  32. __asm__ __volatile("sfence.vma");
  33. #endif
  34. }
  35. void sbi_tlb_local_hfence_vvma(struct sbi_tlb_info *tinfo)
  36. {
  37. unsigned long start = tinfo->start;
  38. unsigned long size = tinfo->size;
  39. unsigned long vmid = tinfo->vmid;
  40. unsigned long i, hgatp;
  41. hgatp = csr_swap(CSR_HGATP,
  42. (vmid << HGATP_VMID_SHIFT) & HGATP_VMID_MASK);
  43. if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) {
  44. __sbi_hfence_vvma_all();
  45. goto done;
  46. }
  47. for (i = 0; i < size; i += PAGE_SIZE) {
  48. __sbi_hfence_vvma_va(start+i);
  49. }
  50. done:
  51. csr_write(CSR_HGATP, hgatp);
  52. }
  53. void sbi_tlb_local_hfence_gvma(struct sbi_tlb_info *tinfo)
  54. {
  55. unsigned long start = tinfo->start;
  56. unsigned long size = tinfo->size;
  57. unsigned long i;
  58. if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) {
  59. __sbi_hfence_gvma_all();
  60. return;
  61. }
  62. for (i = 0; i < size; i += PAGE_SIZE) {
  63. __sbi_hfence_gvma_gpa(start+i);
  64. }
  65. }
  66. void sbi_tlb_local_sfence_vma(struct sbi_tlb_info *tinfo)
  67. {
  68. #ifdef FW_NO_SFENCE_VMA
  69. csr_write(0x9c3, 1 << 26);
  70. #else
  71. unsigned long start = tinfo->start;
  72. unsigned long size = tinfo->size;
  73. unsigned long i;
  74. if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) {
  75. sbi_tlb_flush_all();
  76. return;
  77. }
  78. for (i = 0; i < size; i += PAGE_SIZE) {
  79. __asm__ __volatile__("sfence.vma %0"
  80. :
  81. : "r"(start + i)
  82. : "memory");
  83. }
  84. #endif
  85. }
  86. void sbi_tlb_local_hfence_vvma_asid(struct sbi_tlb_info *tinfo)
  87. {
  88. unsigned long start = tinfo->start;
  89. unsigned long size = tinfo->size;
  90. unsigned long asid = tinfo->asid;
  91. unsigned long vmid = tinfo->vmid;
  92. unsigned long i, hgatp;
  93. hgatp = csr_swap(CSR_HGATP,
  94. (vmid << HGATP_VMID_SHIFT) & HGATP_VMID_MASK);
  95. if (start == 0 && size == 0) {
  96. __sbi_hfence_vvma_all();
  97. goto done;
  98. }
  99. if (size == SBI_TLB_FLUSH_ALL) {
  100. __sbi_hfence_vvma_asid(asid);
  101. goto done;
  102. }
  103. for (i = 0; i < size; i += PAGE_SIZE) {
  104. __sbi_hfence_vvma_asid_va(start + i, asid);
  105. }
  106. done:
  107. csr_write(CSR_HGATP, hgatp);
  108. }
  109. void sbi_tlb_local_hfence_gvma_vmid(struct sbi_tlb_info *tinfo)
  110. {
  111. unsigned long start = tinfo->start;
  112. unsigned long size = tinfo->size;
  113. unsigned long vmid = tinfo->vmid;
  114. unsigned long i;
  115. if (start == 0 && size == 0) {
  116. __sbi_hfence_gvma_all();
  117. return;
  118. }
  119. if (size == SBI_TLB_FLUSH_ALL) {
  120. __sbi_hfence_gvma_vmid(vmid);
  121. return;
  122. }
  123. for (i = 0; i < size; i += PAGE_SIZE) {
  124. __sbi_hfence_gvma_vmid_gpa(start + i, vmid);
  125. }
  126. }
  127. void sbi_tlb_local_sfence_vma_asid(struct sbi_tlb_info *tinfo)
  128. {
  129. #ifdef FW_NO_SFENCE_VMA
  130. csr_write(0x9c3, 1 << 26);
  131. #else
  132. unsigned long start = tinfo->start;
  133. unsigned long size = tinfo->size;
  134. unsigned long asid = tinfo->asid;
  135. unsigned long i;
  136. if (start == 0 && size == 0) {
  137. sbi_tlb_flush_all();
  138. return;
  139. }
  140. /* Flush entire MM context for a given ASID */
  141. if (size == SBI_TLB_FLUSH_ALL) {
  142. __asm__ __volatile__("sfence.vma x0, %0"
  143. :
  144. : "r"(asid)
  145. : "memory");
  146. return;
  147. }
  148. for (i = 0; i < size; i += PAGE_SIZE) {
  149. __asm__ __volatile__("sfence.vma %0, %1"
  150. :
  151. : "r"(start + i), "r"(asid)
  152. : "memory");
  153. }
  154. #endif
  155. }
  156. void sbi_tlb_local_fence_i(struct sbi_tlb_info *tinfo)
  157. {
  158. __asm__ __volatile("fence.i");
  159. }
  160. static void sbi_tlb_entry_process(struct sbi_tlb_info *tinfo)
  161. {
  162. u32 rhartid;
  163. struct sbi_scratch *rscratch = NULL;
  164. unsigned long *rtlb_sync = NULL;
  165. tinfo->local_fn(tinfo);
  166. sbi_hartmask_for_each_hart(rhartid, &tinfo->smask) {
  167. rscratch = sbi_hartid_to_scratch(rhartid);
  168. if (!rscratch)
  169. continue;
  170. rtlb_sync = sbi_scratch_offset_ptr(rscratch, tlb_sync_off);
  171. while (atomic_raw_xchg_ulong(rtlb_sync, 1)) ;
  172. }
  173. }
  174. static void sbi_tlb_process_count(struct sbi_scratch *scratch, int count)
  175. {
  176. struct sbi_tlb_info tinfo;
  177. u32 deq_count = 0;
  178. struct sbi_fifo *tlb_fifo =
  179. sbi_scratch_offset_ptr(scratch, tlb_fifo_off);
  180. while (!sbi_fifo_dequeue(tlb_fifo, &tinfo)) {
  181. sbi_tlb_entry_process(&tinfo);
  182. deq_count++;
  183. if (deq_count > count)
  184. break;
  185. }
  186. }
  187. static void sbi_tlb_process(struct sbi_scratch *scratch)
  188. {
  189. struct sbi_tlb_info tinfo;
  190. struct sbi_fifo *tlb_fifo =
  191. sbi_scratch_offset_ptr(scratch, tlb_fifo_off);
  192. while (!sbi_fifo_dequeue(tlb_fifo, &tinfo))
  193. sbi_tlb_entry_process(&tinfo);
  194. }
  195. static void sbi_tlb_sync(struct sbi_scratch *scratch)
  196. {
  197. unsigned long *tlb_sync =
  198. sbi_scratch_offset_ptr(scratch, tlb_sync_off);
  199. while (!atomic_raw_xchg_ulong(tlb_sync, 0)) {
  200. /*
  201. * While we are waiting for remote hart to set the sync,
  202. * consume fifo requests to avoid deadlock.
  203. */
  204. sbi_tlb_process_count(scratch, 1);
  205. }
  206. return;
  207. }
  208. static inline int __sbi_tlb_range_check(struct sbi_tlb_info *curr,
  209. struct sbi_tlb_info *next)
  210. {
  211. unsigned long curr_end;
  212. unsigned long next_end;
  213. int ret = SBI_FIFO_UNCHANGED;
  214. if (!curr || !next)
  215. return ret;
  216. next_end = next->start + next->size;
  217. curr_end = curr->start + curr->size;
  218. if (next->start <= curr->start && next_end > curr_end) {
  219. curr->start = next->start;
  220. curr->size = next->size;
  221. sbi_hartmask_or(&curr->smask, &curr->smask, &next->smask);
  222. ret = SBI_FIFO_UPDATED;
  223. } else if (next->start >= curr->start && next_end <= curr_end) {
  224. sbi_hartmask_or(&curr->smask, &curr->smask, &next->smask);
  225. ret = SBI_FIFO_SKIP;
  226. }
  227. return ret;
  228. }
  229. /**
  230. * Call back to decide if an inplace fifo update is required or next entry can
  231. * can be skipped. Here are the different cases that are being handled.
  232. *
  233. * Case1:
  234. * if next flush request range lies within one of the existing entry, skip
  235. * the next entry.
  236. * Case2:
  237. * if flush request range in current fifo entry lies within next flush
  238. * request, update the current entry.
  239. *
  240. * Note:
  241. * We can not issue a fifo reset anymore if a complete vma flush is requested.
  242. * This is because we are queueing FENCE.I requests as well now.
  243. * To ease up the pressure in enqueue/fifo sync path, try to dequeue 1 element
  244. * before continuing the while loop. This method is preferred over wfi/ipi because
  245. * of MMIO cost involved in later method.
  246. */
  247. static int sbi_tlb_update_cb(void *in, void *data)
  248. {
  249. struct sbi_tlb_info *curr;
  250. struct sbi_tlb_info *next;
  251. int ret = SBI_FIFO_UNCHANGED;
  252. if (!in || !data)
  253. return ret;
  254. curr = (struct sbi_tlb_info *)data;
  255. next = (struct sbi_tlb_info *)in;
  256. if (next->local_fn == sbi_tlb_local_sfence_vma_asid &&
  257. curr->local_fn == sbi_tlb_local_sfence_vma_asid) {
  258. if (next->asid == curr->asid)
  259. ret = __sbi_tlb_range_check(curr, next);
  260. } else if (next->local_fn == sbi_tlb_local_sfence_vma &&
  261. curr->local_fn == sbi_tlb_local_sfence_vma) {
  262. ret = __sbi_tlb_range_check(curr, next);
  263. }
  264. return ret;
  265. }
  266. static int sbi_tlb_update(struct sbi_scratch *scratch,
  267. struct sbi_scratch *remote_scratch,
  268. u32 remote_hartid, void *data)
  269. {
  270. int ret;
  271. struct sbi_fifo *tlb_fifo_r;
  272. struct sbi_tlb_info *tinfo = data;
  273. u32 curr_hartid = current_hartid();
  274. /*
  275. * If address range to flush is too big then simply
  276. * upgrade it to flush all because we can only flush
  277. * 4KB at a time.
  278. */
  279. if (tinfo->size > tlb_range_flush_limit) {
  280. tinfo->start = 0;
  281. tinfo->size = SBI_TLB_FLUSH_ALL;
  282. }
  283. /*
  284. * If the request is to queue a tlb flush entry for itself
  285. * then just do a local flush and return;
  286. */
  287. if (remote_hartid == curr_hartid) {
  288. tinfo->local_fn(tinfo);
  289. return -1;
  290. }
  291. tlb_fifo_r = sbi_scratch_offset_ptr(remote_scratch, tlb_fifo_off);
  292. ret = sbi_fifo_inplace_update(tlb_fifo_r, data, sbi_tlb_update_cb);
  293. if (ret != SBI_FIFO_UNCHANGED) {
  294. return 1;
  295. }
  296. while (sbi_fifo_enqueue(tlb_fifo_r, data) < 0) {
  297. /**
  298. * For now, Busy loop until there is space in the fifo.
  299. * There may be case where target hart is also
  300. * enqueue in source hart's fifo. Both hart may busy
  301. * loop leading to a deadlock.
  302. * TODO: Introduce a wait/wakeup event mechanism to handle
  303. * this properly.
  304. */
  305. sbi_tlb_process_count(scratch, 1);
  306. sbi_dprintf("hart%d: hart%d tlb fifo full\n",
  307. curr_hartid, remote_hartid);
  308. }
  309. return 0;
  310. }
  311. static struct sbi_ipi_event_ops tlb_ops = {
  312. .name = "IPI_TLB",
  313. .update = sbi_tlb_update,
  314. .sync = sbi_tlb_sync,
  315. .process = sbi_tlb_process,
  316. };
  317. static u32 tlb_event = SBI_IPI_EVENT_MAX;
  318. int sbi_tlb_request(ulong hmask, ulong hbase, struct sbi_tlb_info *tinfo)
  319. {
  320. if (!tinfo->local_fn)
  321. return SBI_EINVAL;
  322. return sbi_ipi_send_many(hmask, hbase, tlb_event, tinfo);
  323. }
  324. int sbi_tlb_init(struct sbi_scratch *scratch, bool cold_boot)
  325. {
  326. int ret;
  327. void *tlb_mem;
  328. unsigned long *tlb_sync;
  329. struct sbi_fifo *tlb_q;
  330. const struct sbi_platform *plat = sbi_platform_ptr(scratch);
  331. if (cold_boot) {
  332. tlb_sync_off = sbi_scratch_alloc_offset(sizeof(*tlb_sync),
  333. "IPI_TLB_SYNC");
  334. if (!tlb_sync_off)
  335. return SBI_ENOMEM;
  336. tlb_fifo_off = sbi_scratch_alloc_offset(sizeof(*tlb_q),
  337. "IPI_TLB_FIFO");
  338. if (!tlb_fifo_off) {
  339. sbi_scratch_free_offset(tlb_sync_off);
  340. return SBI_ENOMEM;
  341. }
  342. tlb_fifo_mem_off = sbi_scratch_alloc_offset(
  343. SBI_TLB_FIFO_NUM_ENTRIES * SBI_TLB_INFO_SIZE,
  344. "IPI_TLB_FIFO_MEM");
  345. if (!tlb_fifo_mem_off) {
  346. sbi_scratch_free_offset(tlb_fifo_off);
  347. sbi_scratch_free_offset(tlb_sync_off);
  348. return SBI_ENOMEM;
  349. }
  350. ret = sbi_ipi_event_create(&tlb_ops);
  351. if (ret < 0) {
  352. sbi_scratch_free_offset(tlb_fifo_mem_off);
  353. sbi_scratch_free_offset(tlb_fifo_off);
  354. sbi_scratch_free_offset(tlb_sync_off);
  355. return ret;
  356. }
  357. tlb_event = ret;
  358. tlb_range_flush_limit = sbi_platform_tlbr_flush_limit(plat);
  359. } else {
  360. if (!tlb_sync_off ||
  361. !tlb_fifo_off ||
  362. !tlb_fifo_mem_off)
  363. return SBI_ENOMEM;
  364. if (SBI_IPI_EVENT_MAX <= tlb_event)
  365. return SBI_ENOSPC;
  366. }
  367. tlb_sync = sbi_scratch_offset_ptr(scratch, tlb_sync_off);
  368. tlb_q = sbi_scratch_offset_ptr(scratch, tlb_fifo_off);
  369. tlb_mem = sbi_scratch_offset_ptr(scratch, tlb_fifo_mem_off);
  370. *tlb_sync = 0;
  371. sbi_fifo_init(tlb_q, tlb_mem,
  372. SBI_TLB_FIFO_NUM_ENTRIES, SBI_TLB_INFO_SIZE);
  373. return 0;
  374. }