grufault.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * SN Platform GRU Driver
  4. *
  5. * FAULT HANDLER FOR GRU DETECTED TLB MISSES
  6. *
  7. * This file contains code that handles TLB misses within the GRU.
  8. * These misses are reported either via interrupts or user polling of
  9. * the user CB.
  10. *
  11. * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.
  12. */
  13. #include <linux/kernel.h>
  14. #include <linux/errno.h>
  15. #include <linux/spinlock.h>
  16. #include <linux/mm.h>
  17. #include <linux/hugetlb.h>
  18. #include <linux/device.h>
  19. #include <linux/io.h>
  20. #include <linux/uaccess.h>
  21. #include <linux/security.h>
  22. #include <linux/sync_core.h>
  23. #include <linux/prefetch.h>
  24. #include "gru.h"
  25. #include "grutables.h"
  26. #include "grulib.h"
  27. #include "gru_instructions.h"
  28. #include <asm/uv/uv_hub.h>
  29. /* Return codes for vtop functions */
  30. #define VTOP_SUCCESS 0
  31. #define VTOP_INVALID -1
  32. #define VTOP_RETRY -2
  33. /*
  34. * Test if a physical address is a valid GRU GSEG address
  35. */
  36. static inline int is_gru_paddr(unsigned long paddr)
  37. {
  38. return paddr >= gru_start_paddr && paddr < gru_end_paddr;
  39. }
  40. /*
  41. * Find the vma of a GRU segment. Caller must hold mmap_lock.
  42. */
  43. struct vm_area_struct *gru_find_vma(unsigned long vaddr)
  44. {
  45. struct vm_area_struct *vma;
  46. vma = find_vma(current->mm, vaddr);
  47. if (vma && vma->vm_start <= vaddr && vma->vm_ops == &gru_vm_ops)
  48. return vma;
  49. return NULL;
  50. }
  51. /*
  52. * Find and lock the gts that contains the specified user vaddr.
  53. *
  54. * Returns:
  55. * - *gts with the mmap_lock locked for read and the GTS locked.
  56. * - NULL if vaddr invalid OR is not a valid GSEG vaddr.
  57. */
  58. static struct gru_thread_state *gru_find_lock_gts(unsigned long vaddr)
  59. {
  60. struct mm_struct *mm = current->mm;
  61. struct vm_area_struct *vma;
  62. struct gru_thread_state *gts = NULL;
  63. mmap_read_lock(mm);
  64. vma = gru_find_vma(vaddr);
  65. if (vma)
  66. gts = gru_find_thread_state(vma, TSID(vaddr, vma));
  67. if (gts)
  68. mutex_lock(&gts->ts_ctxlock);
  69. else
  70. mmap_read_unlock(mm);
  71. return gts;
  72. }
  73. static struct gru_thread_state *gru_alloc_locked_gts(unsigned long vaddr)
  74. {
  75. struct mm_struct *mm = current->mm;
  76. struct vm_area_struct *vma;
  77. struct gru_thread_state *gts = ERR_PTR(-EINVAL);
  78. mmap_write_lock(mm);
  79. vma = gru_find_vma(vaddr);
  80. if (!vma)
  81. goto err;
  82. gts = gru_alloc_thread_state(vma, TSID(vaddr, vma));
  83. if (IS_ERR(gts))
  84. goto err;
  85. mutex_lock(&gts->ts_ctxlock);
  86. mmap_write_downgrade(mm);
  87. return gts;
  88. err:
  89. mmap_write_unlock(mm);
  90. return gts;
  91. }
  92. /*
  93. * Unlock a GTS that was previously locked with gru_find_lock_gts().
  94. */
  95. static void gru_unlock_gts(struct gru_thread_state *gts)
  96. {
  97. mutex_unlock(&gts->ts_ctxlock);
  98. mmap_read_unlock(current->mm);
  99. }
  100. /*
  101. * Set a CB.istatus to active using a user virtual address. This must be done
  102. * just prior to a TFH RESTART. The new cb.istatus is an in-cache status ONLY.
  103. * If the line is evicted, the status may be lost. The in-cache update
  104. * is necessary to prevent the user from seeing a stale cb.istatus that will
  105. * change as soon as the TFH restart is complete. Races may cause an
  106. * occasional failure to clear the cb.istatus, but that is ok.
  107. */
  108. static void gru_cb_set_istatus_active(struct gru_instruction_bits *cbk)
  109. {
  110. if (cbk) {
  111. cbk->istatus = CBS_ACTIVE;
  112. }
  113. }
  114. /*
  115. * Read & clear a TFM
  116. *
  117. * The GRU has an array of fault maps. A map is private to a cpu
  118. * Only one cpu will be accessing a cpu's fault map.
  119. *
  120. * This function scans the cpu-private fault map & clears all bits that
  121. * are set. The function returns a bitmap that indicates the bits that
  122. * were cleared. Note that sense the maps may be updated asynchronously by
  123. * the GRU, atomic operations must be used to clear bits.
  124. */
  125. static void get_clear_fault_map(struct gru_state *gru,
  126. struct gru_tlb_fault_map *imap,
  127. struct gru_tlb_fault_map *dmap)
  128. {
  129. unsigned long i, k;
  130. struct gru_tlb_fault_map *tfm;
  131. tfm = get_tfm_for_cpu(gru, gru_cpu_fault_map_id());
  132. prefetchw(tfm); /* Helps on hardware, required for emulator */
  133. for (i = 0; i < BITS_TO_LONGS(GRU_NUM_CBE); i++) {
  134. k = tfm->fault_bits[i];
  135. if (k)
  136. k = xchg(&tfm->fault_bits[i], 0UL);
  137. imap->fault_bits[i] = k;
  138. k = tfm->done_bits[i];
  139. if (k)
  140. k = xchg(&tfm->done_bits[i], 0UL);
  141. dmap->fault_bits[i] = k;
  142. }
  143. /*
  144. * Not functionally required but helps performance. (Required
  145. * on emulator)
  146. */
  147. gru_flush_cache(tfm);
  148. }
  149. /*
  150. * Atomic (interrupt context) & non-atomic (user context) functions to
  151. * convert a vaddr into a physical address. The size of the page
  152. * is returned in pageshift.
  153. * returns:
  154. * 0 - successful
  155. * < 0 - error code
  156. * 1 - (atomic only) try again in non-atomic context
  157. */
  158. static int non_atomic_pte_lookup(struct vm_area_struct *vma,
  159. unsigned long vaddr, int write,
  160. unsigned long *paddr, int *pageshift)
  161. {
  162. struct page *page;
  163. #ifdef CONFIG_HUGETLB_PAGE
  164. *pageshift = is_vm_hugetlb_page(vma) ? HPAGE_SHIFT : PAGE_SHIFT;
  165. #else
  166. *pageshift = PAGE_SHIFT;
  167. #endif
  168. if (get_user_pages(vaddr, 1, write ? FOLL_WRITE : 0, &page, NULL) <= 0)
  169. return -EFAULT;
  170. *paddr = page_to_phys(page);
  171. put_page(page);
  172. return 0;
  173. }
  174. /*
  175. * atomic_pte_lookup
  176. *
  177. * Convert a user virtual address to a physical address
  178. * Only supports Intel large pages (2MB only) on x86_64.
  179. * ZZZ - hugepage support is incomplete
  180. *
  181. * NOTE: mmap_lock is already held on entry to this function. This
  182. * guarantees existence of the page tables.
  183. */
  184. static int atomic_pte_lookup(struct vm_area_struct *vma, unsigned long vaddr,
  185. int write, unsigned long *paddr, int *pageshift)
  186. {
  187. pgd_t *pgdp;
  188. p4d_t *p4dp;
  189. pud_t *pudp;
  190. pmd_t *pmdp;
  191. pte_t pte;
  192. pgdp = pgd_offset(vma->vm_mm, vaddr);
  193. if (unlikely(pgd_none(*pgdp)))
  194. goto err;
  195. p4dp = p4d_offset(pgdp, vaddr);
  196. if (unlikely(p4d_none(*p4dp)))
  197. goto err;
  198. pudp = pud_offset(p4dp, vaddr);
  199. if (unlikely(pud_none(*pudp)))
  200. goto err;
  201. pmdp = pmd_offset(pudp, vaddr);
  202. if (unlikely(pmd_none(*pmdp)))
  203. goto err;
  204. #ifdef CONFIG_X86_64
  205. if (unlikely(pmd_large(*pmdp)))
  206. pte = *(pte_t *) pmdp;
  207. else
  208. #endif
  209. pte = *pte_offset_kernel(pmdp, vaddr);
  210. if (unlikely(!pte_present(pte) ||
  211. (write && (!pte_write(pte) || !pte_dirty(pte)))))
  212. return 1;
  213. *paddr = pte_pfn(pte) << PAGE_SHIFT;
  214. #ifdef CONFIG_HUGETLB_PAGE
  215. *pageshift = is_vm_hugetlb_page(vma) ? HPAGE_SHIFT : PAGE_SHIFT;
  216. #else
  217. *pageshift = PAGE_SHIFT;
  218. #endif
  219. return 0;
  220. err:
  221. return 1;
  222. }
  223. static int gru_vtop(struct gru_thread_state *gts, unsigned long vaddr,
  224. int write, int atomic, unsigned long *gpa, int *pageshift)
  225. {
  226. struct mm_struct *mm = gts->ts_mm;
  227. struct vm_area_struct *vma;
  228. unsigned long paddr;
  229. int ret, ps;
  230. vma = find_vma(mm, vaddr);
  231. if (!vma)
  232. goto inval;
  233. /*
  234. * Atomic lookup is faster & usually works even if called in non-atomic
  235. * context.
  236. */
  237. rmb(); /* Must/check ms_range_active before loading PTEs */
  238. ret = atomic_pte_lookup(vma, vaddr, write, &paddr, &ps);
  239. if (ret) {
  240. if (atomic)
  241. goto upm;
  242. if (non_atomic_pte_lookup(vma, vaddr, write, &paddr, &ps))
  243. goto inval;
  244. }
  245. if (is_gru_paddr(paddr))
  246. goto inval;
  247. paddr = paddr & ~((1UL << ps) - 1);
  248. *gpa = uv_soc_phys_ram_to_gpa(paddr);
  249. *pageshift = ps;
  250. return VTOP_SUCCESS;
  251. inval:
  252. return VTOP_INVALID;
  253. upm:
  254. return VTOP_RETRY;
  255. }
  256. /*
  257. * Flush a CBE from cache. The CBE is clean in the cache. Dirty the
  258. * CBE cacheline so that the line will be written back to home agent.
  259. * Otherwise the line may be silently dropped. This has no impact
  260. * except on performance.
  261. */
  262. static void gru_flush_cache_cbe(struct gru_control_block_extended *cbe)
  263. {
  264. if (unlikely(cbe)) {
  265. cbe->cbrexecstatus = 0; /* make CL dirty */
  266. gru_flush_cache(cbe);
  267. }
  268. }
  269. /*
  270. * Preload the TLB with entries that may be required. Currently, preloading
  271. * is implemented only for BCOPY. Preload <tlb_preload_count> pages OR to
  272. * the end of the bcopy tranfer, whichever is smaller.
  273. */
  274. static void gru_preload_tlb(struct gru_state *gru,
  275. struct gru_thread_state *gts, int atomic,
  276. unsigned long fault_vaddr, int asid, int write,
  277. unsigned char tlb_preload_count,
  278. struct gru_tlb_fault_handle *tfh,
  279. struct gru_control_block_extended *cbe)
  280. {
  281. unsigned long vaddr = 0, gpa;
  282. int ret, pageshift;
  283. if (cbe->opccpy != OP_BCOPY)
  284. return;
  285. if (fault_vaddr == cbe->cbe_baddr0)
  286. vaddr = fault_vaddr + GRU_CACHE_LINE_BYTES * cbe->cbe_src_cl - 1;
  287. else if (fault_vaddr == cbe->cbe_baddr1)
  288. vaddr = fault_vaddr + (1 << cbe->xtypecpy) * cbe->cbe_nelemcur - 1;
  289. fault_vaddr &= PAGE_MASK;
  290. vaddr &= PAGE_MASK;
  291. vaddr = min(vaddr, fault_vaddr + tlb_preload_count * PAGE_SIZE);
  292. while (vaddr > fault_vaddr) {
  293. ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift);
  294. if (ret || tfh_write_only(tfh, gpa, GAA_RAM, vaddr, asid, write,
  295. GRU_PAGESIZE(pageshift)))
  296. return;
  297. gru_dbg(grudev,
  298. "%s: gid %d, gts 0x%p, tfh 0x%p, vaddr 0x%lx, asid 0x%x, rw %d, ps %d, gpa 0x%lx\n",
  299. atomic ? "atomic" : "non-atomic", gru->gs_gid, gts, tfh,
  300. vaddr, asid, write, pageshift, gpa);
  301. vaddr -= PAGE_SIZE;
  302. STAT(tlb_preload_page);
  303. }
  304. }
  305. /*
  306. * Drop a TLB entry into the GRU. The fault is described by info in an TFH.
  307. * Input:
  308. * cb Address of user CBR. Null if not running in user context
  309. * Return:
  310. * 0 = dropin, exception, or switch to UPM successful
  311. * 1 = range invalidate active
  312. * < 0 = error code
  313. *
  314. */
  315. static int gru_try_dropin(struct gru_state *gru,
  316. struct gru_thread_state *gts,
  317. struct gru_tlb_fault_handle *tfh,
  318. struct gru_instruction_bits *cbk)
  319. {
  320. struct gru_control_block_extended *cbe = NULL;
  321. unsigned char tlb_preload_count = gts->ts_tlb_preload_count;
  322. int pageshift = 0, asid, write, ret, atomic = !cbk, indexway;
  323. unsigned long gpa = 0, vaddr = 0;
  324. /*
  325. * NOTE: The GRU contains magic hardware that eliminates races between
  326. * TLB invalidates and TLB dropins. If an invalidate occurs
  327. * in the window between reading the TFH and the subsequent TLB dropin,
  328. * the dropin is ignored. This eliminates the need for additional locks.
  329. */
  330. /*
  331. * Prefetch the CBE if doing TLB preloading
  332. */
  333. if (unlikely(tlb_preload_count)) {
  334. cbe = gru_tfh_to_cbe(tfh);
  335. prefetchw(cbe);
  336. }
  337. /*
  338. * Error if TFH state is IDLE or FMM mode & the user issuing a UPM call.
  339. * Might be a hardware race OR a stupid user. Ignore FMM because FMM
  340. * is a transient state.
  341. */
  342. if (tfh->status != TFHSTATUS_EXCEPTION) {
  343. gru_flush_cache(tfh);
  344. sync_core();
  345. if (tfh->status != TFHSTATUS_EXCEPTION)
  346. goto failnoexception;
  347. STAT(tfh_stale_on_fault);
  348. }
  349. if (tfh->state == TFHSTATE_IDLE)
  350. goto failidle;
  351. if (tfh->state == TFHSTATE_MISS_FMM && cbk)
  352. goto failfmm;
  353. write = (tfh->cause & TFHCAUSE_TLB_MOD) != 0;
  354. vaddr = tfh->missvaddr;
  355. asid = tfh->missasid;
  356. indexway = tfh->indexway;
  357. if (asid == 0)
  358. goto failnoasid;
  359. rmb(); /* TFH must be cache resident before reading ms_range_active */
  360. /*
  361. * TFH is cache resident - at least briefly. Fail the dropin
  362. * if a range invalidate is active.
  363. */
  364. if (atomic_read(&gts->ts_gms->ms_range_active))
  365. goto failactive;
  366. ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift);
  367. if (ret == VTOP_INVALID)
  368. goto failinval;
  369. if (ret == VTOP_RETRY)
  370. goto failupm;
  371. if (!(gts->ts_sizeavail & GRU_SIZEAVAIL(pageshift))) {
  372. gts->ts_sizeavail |= GRU_SIZEAVAIL(pageshift);
  373. if (atomic || !gru_update_cch(gts)) {
  374. gts->ts_force_cch_reload = 1;
  375. goto failupm;
  376. }
  377. }
  378. if (unlikely(cbe) && pageshift == PAGE_SHIFT) {
  379. gru_preload_tlb(gru, gts, atomic, vaddr, asid, write, tlb_preload_count, tfh, cbe);
  380. gru_flush_cache_cbe(cbe);
  381. }
  382. gru_cb_set_istatus_active(cbk);
  383. gts->ustats.tlbdropin++;
  384. tfh_write_restart(tfh, gpa, GAA_RAM, vaddr, asid, write,
  385. GRU_PAGESIZE(pageshift));
  386. gru_dbg(grudev,
  387. "%s: gid %d, gts 0x%p, tfh 0x%p, vaddr 0x%lx, asid 0x%x, indexway 0x%x,"
  388. " rw %d, ps %d, gpa 0x%lx\n",
  389. atomic ? "atomic" : "non-atomic", gru->gs_gid, gts, tfh, vaddr, asid,
  390. indexway, write, pageshift, gpa);
  391. STAT(tlb_dropin);
  392. return 0;
  393. failnoasid:
  394. /* No asid (delayed unload). */
  395. STAT(tlb_dropin_fail_no_asid);
  396. gru_dbg(grudev, "FAILED no_asid tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr);
  397. if (!cbk)
  398. tfh_user_polling_mode(tfh);
  399. else
  400. gru_flush_cache(tfh);
  401. gru_flush_cache_cbe(cbe);
  402. return -EAGAIN;
  403. failupm:
  404. /* Atomic failure switch CBR to UPM */
  405. tfh_user_polling_mode(tfh);
  406. gru_flush_cache_cbe(cbe);
  407. STAT(tlb_dropin_fail_upm);
  408. gru_dbg(grudev, "FAILED upm tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr);
  409. return 1;
  410. failfmm:
  411. /* FMM state on UPM call */
  412. gru_flush_cache(tfh);
  413. gru_flush_cache_cbe(cbe);
  414. STAT(tlb_dropin_fail_fmm);
  415. gru_dbg(grudev, "FAILED fmm tfh: 0x%p, state %d\n", tfh, tfh->state);
  416. return 0;
  417. failnoexception:
  418. /* TFH status did not show exception pending */
  419. gru_flush_cache(tfh);
  420. gru_flush_cache_cbe(cbe);
  421. if (cbk)
  422. gru_flush_cache(cbk);
  423. STAT(tlb_dropin_fail_no_exception);
  424. gru_dbg(grudev, "FAILED non-exception tfh: 0x%p, status %d, state %d\n",
  425. tfh, tfh->status, tfh->state);
  426. return 0;
  427. failidle:
  428. /* TFH state was idle - no miss pending */
  429. gru_flush_cache(tfh);
  430. gru_flush_cache_cbe(cbe);
  431. if (cbk)
  432. gru_flush_cache(cbk);
  433. STAT(tlb_dropin_fail_idle);
  434. gru_dbg(grudev, "FAILED idle tfh: 0x%p, state %d\n", tfh, tfh->state);
  435. return 0;
  436. failinval:
  437. /* All errors (atomic & non-atomic) switch CBR to EXCEPTION state */
  438. tfh_exception(tfh);
  439. gru_flush_cache_cbe(cbe);
  440. STAT(tlb_dropin_fail_invalid);
  441. gru_dbg(grudev, "FAILED inval tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr);
  442. return -EFAULT;
  443. failactive:
  444. /* Range invalidate active. Switch to UPM iff atomic */
  445. if (!cbk)
  446. tfh_user_polling_mode(tfh);
  447. else
  448. gru_flush_cache(tfh);
  449. gru_flush_cache_cbe(cbe);
  450. STAT(tlb_dropin_fail_range_active);
  451. gru_dbg(grudev, "FAILED range active: tfh 0x%p, vaddr 0x%lx\n",
  452. tfh, vaddr);
  453. return 1;
  454. }
  455. /*
  456. * Process an external interrupt from the GRU. This interrupt is
  457. * caused by a TLB miss.
  458. * Note that this is the interrupt handler that is registered with linux
  459. * interrupt handlers.
  460. */
  461. static irqreturn_t gru_intr(int chiplet, int blade)
  462. {
  463. struct gru_state *gru;
  464. struct gru_tlb_fault_map imap, dmap;
  465. struct gru_thread_state *gts;
  466. struct gru_tlb_fault_handle *tfh = NULL;
  467. struct completion *cmp;
  468. int cbrnum, ctxnum;
  469. STAT(intr);
  470. gru = &gru_base[blade]->bs_grus[chiplet];
  471. if (!gru) {
  472. dev_err(grudev, "GRU: invalid interrupt: cpu %d, chiplet %d\n",
  473. raw_smp_processor_id(), chiplet);
  474. return IRQ_NONE;
  475. }
  476. get_clear_fault_map(gru, &imap, &dmap);
  477. gru_dbg(grudev,
  478. "cpu %d, chiplet %d, gid %d, imap %016lx %016lx, dmap %016lx %016lx\n",
  479. smp_processor_id(), chiplet, gru->gs_gid,
  480. imap.fault_bits[0], imap.fault_bits[1],
  481. dmap.fault_bits[0], dmap.fault_bits[1]);
  482. for_each_cbr_in_tfm(cbrnum, dmap.fault_bits) {
  483. STAT(intr_cbr);
  484. cmp = gru->gs_blade->bs_async_wq;
  485. if (cmp)
  486. complete(cmp);
  487. gru_dbg(grudev, "gid %d, cbr_done %d, done %d\n",
  488. gru->gs_gid, cbrnum, cmp ? cmp->done : -1);
  489. }
  490. for_each_cbr_in_tfm(cbrnum, imap.fault_bits) {
  491. STAT(intr_tfh);
  492. tfh = get_tfh_by_index(gru, cbrnum);
  493. prefetchw(tfh); /* Helps on hdw, required for emulator */
  494. /*
  495. * When hardware sets a bit in the faultmap, it implicitly
  496. * locks the GRU context so that it cannot be unloaded.
  497. * The gts cannot change until a TFH start/writestart command
  498. * is issued.
  499. */
  500. ctxnum = tfh->ctxnum;
  501. gts = gru->gs_gts[ctxnum];
  502. /* Spurious interrupts can cause this. Ignore. */
  503. if (!gts) {
  504. STAT(intr_spurious);
  505. continue;
  506. }
  507. /*
  508. * This is running in interrupt context. Trylock the mmap_lock.
  509. * If it fails, retry the fault in user context.
  510. */
  511. gts->ustats.fmm_tlbmiss++;
  512. if (!gts->ts_force_cch_reload &&
  513. mmap_read_trylock(gts->ts_mm)) {
  514. gru_try_dropin(gru, gts, tfh, NULL);
  515. mmap_read_unlock(gts->ts_mm);
  516. } else {
  517. tfh_user_polling_mode(tfh);
  518. STAT(intr_mm_lock_failed);
  519. }
  520. }
  521. return IRQ_HANDLED;
  522. }
  523. irqreturn_t gru0_intr(int irq, void *dev_id)
  524. {
  525. return gru_intr(0, uv_numa_blade_id());
  526. }
  527. irqreturn_t gru1_intr(int irq, void *dev_id)
  528. {
  529. return gru_intr(1, uv_numa_blade_id());
  530. }
  531. irqreturn_t gru_intr_mblade(int irq, void *dev_id)
  532. {
  533. int blade;
  534. for_each_possible_blade(blade) {
  535. if (uv_blade_nr_possible_cpus(blade))
  536. continue;
  537. gru_intr(0, blade);
  538. gru_intr(1, blade);
  539. }
  540. return IRQ_HANDLED;
  541. }
  542. static int gru_user_dropin(struct gru_thread_state *gts,
  543. struct gru_tlb_fault_handle *tfh,
  544. void *cb)
  545. {
  546. struct gru_mm_struct *gms = gts->ts_gms;
  547. int ret;
  548. gts->ustats.upm_tlbmiss++;
  549. while (1) {
  550. wait_event(gms->ms_wait_queue,
  551. atomic_read(&gms->ms_range_active) == 0);
  552. prefetchw(tfh); /* Helps on hdw, required for emulator */
  553. ret = gru_try_dropin(gts->ts_gru, gts, tfh, cb);
  554. if (ret <= 0)
  555. return ret;
  556. STAT(call_os_wait_queue);
  557. }
  558. }
  559. /*
  560. * This interface is called as a result of a user detecting a "call OS" bit
  561. * in a user CB. Normally means that a TLB fault has occurred.
  562. * cb - user virtual address of the CB
  563. */
  564. int gru_handle_user_call_os(unsigned long cb)
  565. {
  566. struct gru_tlb_fault_handle *tfh;
  567. struct gru_thread_state *gts;
  568. void *cbk;
  569. int ucbnum, cbrnum, ret = -EINVAL;
  570. STAT(call_os);
  571. /* sanity check the cb pointer */
  572. ucbnum = get_cb_number((void *)cb);
  573. if ((cb & (GRU_HANDLE_STRIDE - 1)) || ucbnum >= GRU_NUM_CB)
  574. return -EINVAL;
  575. gts = gru_find_lock_gts(cb);
  576. if (!gts)
  577. return -EINVAL;
  578. gru_dbg(grudev, "address 0x%lx, gid %d, gts 0x%p\n", cb, gts->ts_gru ? gts->ts_gru->gs_gid : -1, gts);
  579. if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE)
  580. goto exit;
  581. gru_check_context_placement(gts);
  582. /*
  583. * CCH may contain stale data if ts_force_cch_reload is set.
  584. */
  585. if (gts->ts_gru && gts->ts_force_cch_reload) {
  586. gts->ts_force_cch_reload = 0;
  587. gru_update_cch(gts);
  588. }
  589. ret = -EAGAIN;
  590. cbrnum = thread_cbr_number(gts, ucbnum);
  591. if (gts->ts_gru) {
  592. tfh = get_tfh_by_index(gts->ts_gru, cbrnum);
  593. cbk = get_gseg_base_address_cb(gts->ts_gru->gs_gru_base_vaddr,
  594. gts->ts_ctxnum, ucbnum);
  595. ret = gru_user_dropin(gts, tfh, cbk);
  596. }
  597. exit:
  598. gru_unlock_gts(gts);
  599. return ret;
  600. }
  601. /*
  602. * Fetch the exception detail information for a CB that terminated with
  603. * an exception.
  604. */
  605. int gru_get_exception_detail(unsigned long arg)
  606. {
  607. struct control_block_extended_exc_detail excdet;
  608. struct gru_control_block_extended *cbe;
  609. struct gru_thread_state *gts;
  610. int ucbnum, cbrnum, ret;
  611. STAT(user_exception);
  612. if (copy_from_user(&excdet, (void __user *)arg, sizeof(excdet)))
  613. return -EFAULT;
  614. gts = gru_find_lock_gts(excdet.cb);
  615. if (!gts)
  616. return -EINVAL;
  617. gru_dbg(grudev, "address 0x%lx, gid %d, gts 0x%p\n", excdet.cb, gts->ts_gru ? gts->ts_gru->gs_gid : -1, gts);
  618. ucbnum = get_cb_number((void *)excdet.cb);
  619. if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE) {
  620. ret = -EINVAL;
  621. } else if (gts->ts_gru) {
  622. cbrnum = thread_cbr_number(gts, ucbnum);
  623. cbe = get_cbe_by_index(gts->ts_gru, cbrnum);
  624. gru_flush_cache(cbe); /* CBE not coherent */
  625. sync_core(); /* make sure we are have current data */
  626. excdet.opc = cbe->opccpy;
  627. excdet.exopc = cbe->exopccpy;
  628. excdet.ecause = cbe->ecause;
  629. excdet.exceptdet0 = cbe->idef1upd;
  630. excdet.exceptdet1 = cbe->idef3upd;
  631. excdet.cbrstate = cbe->cbrstate;
  632. excdet.cbrexecstatus = cbe->cbrexecstatus;
  633. gru_flush_cache_cbe(cbe);
  634. ret = 0;
  635. } else {
  636. ret = -EAGAIN;
  637. }
  638. gru_unlock_gts(gts);
  639. gru_dbg(grudev,
  640. "cb 0x%lx, op %d, exopc %d, cbrstate %d, cbrexecstatus 0x%x, ecause 0x%x, "
  641. "exdet0 0x%lx, exdet1 0x%x\n",
  642. excdet.cb, excdet.opc, excdet.exopc, excdet.cbrstate, excdet.cbrexecstatus,
  643. excdet.ecause, excdet.exceptdet0, excdet.exceptdet1);
  644. if (!ret && copy_to_user((void __user *)arg, &excdet, sizeof(excdet)))
  645. ret = -EFAULT;
  646. return ret;
  647. }
  648. /*
  649. * User request to unload a context. Content is saved for possible reload.
  650. */
  651. static int gru_unload_all_contexts(void)
  652. {
  653. struct gru_thread_state *gts;
  654. struct gru_state *gru;
  655. int gid, ctxnum;
  656. if (!capable(CAP_SYS_ADMIN))
  657. return -EPERM;
  658. foreach_gid(gid) {
  659. gru = GID_TO_GRU(gid);
  660. spin_lock(&gru->gs_lock);
  661. for (ctxnum = 0; ctxnum < GRU_NUM_CCH; ctxnum++) {
  662. gts = gru->gs_gts[ctxnum];
  663. if (gts && mutex_trylock(&gts->ts_ctxlock)) {
  664. spin_unlock(&gru->gs_lock);
  665. gru_unload_context(gts, 1);
  666. mutex_unlock(&gts->ts_ctxlock);
  667. spin_lock(&gru->gs_lock);
  668. }
  669. }
  670. spin_unlock(&gru->gs_lock);
  671. }
  672. return 0;
  673. }
  674. int gru_user_unload_context(unsigned long arg)
  675. {
  676. struct gru_thread_state *gts;
  677. struct gru_unload_context_req req;
  678. STAT(user_unload_context);
  679. if (copy_from_user(&req, (void __user *)arg, sizeof(req)))
  680. return -EFAULT;
  681. gru_dbg(grudev, "gseg 0x%lx\n", req.gseg);
  682. if (!req.gseg)
  683. return gru_unload_all_contexts();
  684. gts = gru_find_lock_gts(req.gseg);
  685. if (!gts)
  686. return -EINVAL;
  687. if (gts->ts_gru)
  688. gru_unload_context(gts, 1);
  689. gru_unlock_gts(gts);
  690. return 0;
  691. }
  692. /*
  693. * User request to flush a range of virtual addresses from the GRU TLB
  694. * (Mainly for testing).
  695. */
  696. int gru_user_flush_tlb(unsigned long arg)
  697. {
  698. struct gru_thread_state *gts;
  699. struct gru_flush_tlb_req req;
  700. struct gru_mm_struct *gms;
  701. STAT(user_flush_tlb);
  702. if (copy_from_user(&req, (void __user *)arg, sizeof(req)))
  703. return -EFAULT;
  704. gru_dbg(grudev, "gseg 0x%lx, vaddr 0x%lx, len 0x%lx\n", req.gseg,
  705. req.vaddr, req.len);
  706. gts = gru_find_lock_gts(req.gseg);
  707. if (!gts)
  708. return -EINVAL;
  709. gms = gts->ts_gms;
  710. gru_unlock_gts(gts);
  711. gru_flush_tlb_range(gms, req.vaddr, req.len);
  712. return 0;
  713. }
  714. /*
  715. * Fetch GSEG statisticss
  716. */
  717. long gru_get_gseg_statistics(unsigned long arg)
  718. {
  719. struct gru_thread_state *gts;
  720. struct gru_get_gseg_statistics_req req;
  721. if (copy_from_user(&req, (void __user *)arg, sizeof(req)))
  722. return -EFAULT;
  723. /*
  724. * The library creates arrays of contexts for threaded programs.
  725. * If no gts exists in the array, the context has never been used & all
  726. * statistics are implicitly 0.
  727. */
  728. gts = gru_find_lock_gts(req.gseg);
  729. if (gts) {
  730. memcpy(&req.stats, &gts->ustats, sizeof(gts->ustats));
  731. gru_unlock_gts(gts);
  732. } else {
  733. memset(&req.stats, 0, sizeof(gts->ustats));
  734. }
  735. if (copy_to_user((void __user *)arg, &req, sizeof(req)))
  736. return -EFAULT;
  737. return 0;
  738. }
  739. /*
  740. * Register the current task as the user of the GSEG slice.
  741. * Needed for TLB fault interrupt targeting.
  742. */
  743. int gru_set_context_option(unsigned long arg)
  744. {
  745. struct gru_thread_state *gts;
  746. struct gru_set_context_option_req req;
  747. int ret = 0;
  748. STAT(set_context_option);
  749. if (copy_from_user(&req, (void __user *)arg, sizeof(req)))
  750. return -EFAULT;
  751. gru_dbg(grudev, "op %d, gseg 0x%lx, value1 0x%lx\n", req.op, req.gseg, req.val1);
  752. gts = gru_find_lock_gts(req.gseg);
  753. if (!gts) {
  754. gts = gru_alloc_locked_gts(req.gseg);
  755. if (IS_ERR(gts))
  756. return PTR_ERR(gts);
  757. }
  758. switch (req.op) {
  759. case sco_blade_chiplet:
  760. /* Select blade/chiplet for GRU context */
  761. if (req.val0 < -1 || req.val0 >= GRU_CHIPLETS_PER_HUB ||
  762. req.val1 < -1 || req.val1 >= GRU_MAX_BLADES ||
  763. (req.val1 >= 0 && !gru_base[req.val1])) {
  764. ret = -EINVAL;
  765. } else {
  766. gts->ts_user_blade_id = req.val1;
  767. gts->ts_user_chiplet_id = req.val0;
  768. gru_check_context_placement(gts);
  769. }
  770. break;
  771. case sco_gseg_owner:
  772. /* Register the current task as the GSEG owner */
  773. gts->ts_tgid_owner = current->tgid;
  774. break;
  775. case sco_cch_req_slice:
  776. /* Set the CCH slice option */
  777. gts->ts_cch_req_slice = req.val1 & 3;
  778. break;
  779. default:
  780. ret = -EINVAL;
  781. }
  782. gru_unlock_gts(gts);
  783. return ret;
  784. }