grumain.c 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * SN Platform GRU Driver
  4. *
  5. * DRIVER TABLE MANAGER + GRU CONTEXT LOAD/UNLOAD
  6. *
  7. * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.
  8. */
  9. #include <linux/kernel.h>
  10. #include <linux/slab.h>
  11. #include <linux/mm.h>
  12. #include <linux/spinlock.h>
  13. #include <linux/sched.h>
  14. #include <linux/device.h>
  15. #include <linux/list.h>
  16. #include <linux/err.h>
  17. #include <linux/prefetch.h>
  18. #include <asm/uv/uv_hub.h>
  19. #include "gru.h"
  20. #include "grutables.h"
  21. #include "gruhandles.h"
  22. unsigned long gru_options __read_mostly;
  23. static struct device_driver gru_driver = {
  24. .name = "gru"
  25. };
  26. static struct device gru_device = {
  27. .init_name = "",
  28. .driver = &gru_driver,
  29. };
  30. struct device *grudev = &gru_device;
  31. /*
  32. * Select a gru fault map to be used by the current cpu. Note that
  33. * multiple cpus may be using the same map.
  34. * ZZZ should be inline but did not work on emulator
  35. */
  36. int gru_cpu_fault_map_id(void)
  37. {
  38. #ifdef CONFIG_IA64
  39. return uv_blade_processor_id() % GRU_NUM_TFM;
  40. #else
  41. int cpu = smp_processor_id();
  42. int id, core;
  43. core = uv_cpu_core_number(cpu);
  44. id = core + UV_MAX_INT_CORES * uv_cpu_socket_number(cpu);
  45. return id;
  46. #endif
  47. }
  48. /*--------- ASID Management -------------------------------------------
  49. *
  50. * Initially, assign asids sequentially from MIN_ASID .. MAX_ASID.
  51. * Once MAX is reached, flush the TLB & start over. However,
  52. * some asids may still be in use. There won't be many (percentage wise) still
  53. * in use. Search active contexts & determine the value of the first
  54. * asid in use ("x"s below). Set "limit" to this value.
  55. * This defines a block of assignable asids.
  56. *
  57. * When "limit" is reached, search forward from limit+1 and determine the
  58. * next block of assignable asids.
  59. *
  60. * Repeat until MAX_ASID is reached, then start over again.
  61. *
  62. * Each time MAX_ASID is reached, increment the asid generation. Since
  63. * the search for in-use asids only checks contexts with GRUs currently
  64. * assigned, asids in some contexts will be missed. Prior to loading
  65. * a context, the asid generation of the GTS asid is rechecked. If it
  66. * doesn't match the current generation, a new asid will be assigned.
  67. *
  68. * 0---------------x------------x---------------------x----|
  69. * ^-next ^-limit ^-MAX_ASID
  70. *
  71. * All asid manipulation & context loading/unloading is protected by the
  72. * gs_lock.
  73. */
  74. /* Hit the asid limit. Start over */
  75. static int gru_wrap_asid(struct gru_state *gru)
  76. {
  77. gru_dbg(grudev, "gid %d\n", gru->gs_gid);
  78. STAT(asid_wrap);
  79. gru->gs_asid_gen++;
  80. return MIN_ASID;
  81. }
  82. /* Find the next chunk of unused asids */
  83. static int gru_reset_asid_limit(struct gru_state *gru, int asid)
  84. {
  85. int i, gid, inuse_asid, limit;
  86. gru_dbg(grudev, "gid %d, asid 0x%x\n", gru->gs_gid, asid);
  87. STAT(asid_next);
  88. limit = MAX_ASID;
  89. if (asid >= limit)
  90. asid = gru_wrap_asid(gru);
  91. gru_flush_all_tlb(gru);
  92. gid = gru->gs_gid;
  93. again:
  94. for (i = 0; i < GRU_NUM_CCH; i++) {
  95. if (!gru->gs_gts[i] || is_kernel_context(gru->gs_gts[i]))
  96. continue;
  97. inuse_asid = gru->gs_gts[i]->ts_gms->ms_asids[gid].mt_asid;
  98. gru_dbg(grudev, "gid %d, gts %p, gms %p, inuse 0x%x, cxt %d\n",
  99. gru->gs_gid, gru->gs_gts[i], gru->gs_gts[i]->ts_gms,
  100. inuse_asid, i);
  101. if (inuse_asid == asid) {
  102. asid += ASID_INC;
  103. if (asid >= limit) {
  104. /*
  105. * empty range: reset the range limit and
  106. * start over
  107. */
  108. limit = MAX_ASID;
  109. if (asid >= MAX_ASID)
  110. asid = gru_wrap_asid(gru);
  111. goto again;
  112. }
  113. }
  114. if ((inuse_asid > asid) && (inuse_asid < limit))
  115. limit = inuse_asid;
  116. }
  117. gru->gs_asid_limit = limit;
  118. gru->gs_asid = asid;
  119. gru_dbg(grudev, "gid %d, new asid 0x%x, new_limit 0x%x\n", gru->gs_gid,
  120. asid, limit);
  121. return asid;
  122. }
  123. /* Assign a new ASID to a thread context. */
  124. static int gru_assign_asid(struct gru_state *gru)
  125. {
  126. int asid;
  127. gru->gs_asid += ASID_INC;
  128. asid = gru->gs_asid;
  129. if (asid >= gru->gs_asid_limit)
  130. asid = gru_reset_asid_limit(gru, asid);
  131. gru_dbg(grudev, "gid %d, asid 0x%x\n", gru->gs_gid, asid);
  132. return asid;
  133. }
  134. /*
  135. * Clear n bits in a word. Return a word indicating the bits that were cleared.
  136. * Optionally, build an array of chars that contain the bit numbers allocated.
  137. */
  138. static unsigned long reserve_resources(unsigned long *p, int n, int mmax,
  139. char *idx)
  140. {
  141. unsigned long bits = 0;
  142. int i;
  143. while (n--) {
  144. i = find_first_bit(p, mmax);
  145. if (i == mmax)
  146. BUG();
  147. __clear_bit(i, p);
  148. __set_bit(i, &bits);
  149. if (idx)
  150. *idx++ = i;
  151. }
  152. return bits;
  153. }
  154. unsigned long gru_reserve_cb_resources(struct gru_state *gru, int cbr_au_count,
  155. char *cbmap)
  156. {
  157. return reserve_resources(&gru->gs_cbr_map, cbr_au_count, GRU_CBR_AU,
  158. cbmap);
  159. }
  160. unsigned long gru_reserve_ds_resources(struct gru_state *gru, int dsr_au_count,
  161. char *dsmap)
  162. {
  163. return reserve_resources(&gru->gs_dsr_map, dsr_au_count, GRU_DSR_AU,
  164. dsmap);
  165. }
  166. static void reserve_gru_resources(struct gru_state *gru,
  167. struct gru_thread_state *gts)
  168. {
  169. gru->gs_active_contexts++;
  170. gts->ts_cbr_map =
  171. gru_reserve_cb_resources(gru, gts->ts_cbr_au_count,
  172. gts->ts_cbr_idx);
  173. gts->ts_dsr_map =
  174. gru_reserve_ds_resources(gru, gts->ts_dsr_au_count, NULL);
  175. }
  176. static void free_gru_resources(struct gru_state *gru,
  177. struct gru_thread_state *gts)
  178. {
  179. gru->gs_active_contexts--;
  180. gru->gs_cbr_map |= gts->ts_cbr_map;
  181. gru->gs_dsr_map |= gts->ts_dsr_map;
  182. }
  183. /*
  184. * Check if a GRU has sufficient free resources to satisfy an allocation
  185. * request. Note: GRU locks may or may not be held when this is called. If
  186. * not held, recheck after acquiring the appropriate locks.
  187. *
  188. * Returns 1 if sufficient resources, 0 if not
  189. */
  190. static int check_gru_resources(struct gru_state *gru, int cbr_au_count,
  191. int dsr_au_count, int max_active_contexts)
  192. {
  193. return hweight64(gru->gs_cbr_map) >= cbr_au_count
  194. && hweight64(gru->gs_dsr_map) >= dsr_au_count
  195. && gru->gs_active_contexts < max_active_contexts;
  196. }
  197. /*
  198. * TLB manangment requires tracking all GRU chiplets that have loaded a GSEG
  199. * context.
  200. */
  201. static int gru_load_mm_tracker(struct gru_state *gru,
  202. struct gru_thread_state *gts)
  203. {
  204. struct gru_mm_struct *gms = gts->ts_gms;
  205. struct gru_mm_tracker *asids = &gms->ms_asids[gru->gs_gid];
  206. unsigned short ctxbitmap = (1 << gts->ts_ctxnum);
  207. int asid;
  208. spin_lock(&gms->ms_asid_lock);
  209. asid = asids->mt_asid;
  210. spin_lock(&gru->gs_asid_lock);
  211. if (asid == 0 || (asids->mt_ctxbitmap == 0 && asids->mt_asid_gen !=
  212. gru->gs_asid_gen)) {
  213. asid = gru_assign_asid(gru);
  214. asids->mt_asid = asid;
  215. asids->mt_asid_gen = gru->gs_asid_gen;
  216. STAT(asid_new);
  217. } else {
  218. STAT(asid_reuse);
  219. }
  220. spin_unlock(&gru->gs_asid_lock);
  221. BUG_ON(asids->mt_ctxbitmap & ctxbitmap);
  222. asids->mt_ctxbitmap |= ctxbitmap;
  223. if (!test_bit(gru->gs_gid, gms->ms_asidmap))
  224. __set_bit(gru->gs_gid, gms->ms_asidmap);
  225. spin_unlock(&gms->ms_asid_lock);
  226. gru_dbg(grudev,
  227. "gid %d, gts %p, gms %p, ctxnum %d, asid 0x%x, asidmap 0x%lx\n",
  228. gru->gs_gid, gts, gms, gts->ts_ctxnum, asid,
  229. gms->ms_asidmap[0]);
  230. return asid;
  231. }
  232. static void gru_unload_mm_tracker(struct gru_state *gru,
  233. struct gru_thread_state *gts)
  234. {
  235. struct gru_mm_struct *gms = gts->ts_gms;
  236. struct gru_mm_tracker *asids;
  237. unsigned short ctxbitmap;
  238. asids = &gms->ms_asids[gru->gs_gid];
  239. ctxbitmap = (1 << gts->ts_ctxnum);
  240. spin_lock(&gms->ms_asid_lock);
  241. spin_lock(&gru->gs_asid_lock);
  242. BUG_ON((asids->mt_ctxbitmap & ctxbitmap) != ctxbitmap);
  243. asids->mt_ctxbitmap ^= ctxbitmap;
  244. gru_dbg(grudev, "gid %d, gts %p, gms %p, ctxnum %d, asidmap 0x%lx\n",
  245. gru->gs_gid, gts, gms, gts->ts_ctxnum, gms->ms_asidmap[0]);
  246. spin_unlock(&gru->gs_asid_lock);
  247. spin_unlock(&gms->ms_asid_lock);
  248. }
  249. /*
  250. * Decrement the reference count on a GTS structure. Free the structure
  251. * if the reference count goes to zero.
  252. */
  253. void gts_drop(struct gru_thread_state *gts)
  254. {
  255. if (gts && atomic_dec_return(&gts->ts_refcnt) == 0) {
  256. if (gts->ts_gms)
  257. gru_drop_mmu_notifier(gts->ts_gms);
  258. kfree(gts);
  259. STAT(gts_free);
  260. }
  261. }
  262. /*
  263. * Locate the GTS structure for the current thread.
  264. */
  265. static struct gru_thread_state *gru_find_current_gts_nolock(struct gru_vma_data
  266. *vdata, int tsid)
  267. {
  268. struct gru_thread_state *gts;
  269. list_for_each_entry(gts, &vdata->vd_head, ts_next)
  270. if (gts->ts_tsid == tsid)
  271. return gts;
  272. return NULL;
  273. }
  274. /*
  275. * Allocate a thread state structure.
  276. */
  277. struct gru_thread_state *gru_alloc_gts(struct vm_area_struct *vma,
  278. int cbr_au_count, int dsr_au_count,
  279. unsigned char tlb_preload_count, int options, int tsid)
  280. {
  281. struct gru_thread_state *gts;
  282. struct gru_mm_struct *gms;
  283. int bytes;
  284. bytes = DSR_BYTES(dsr_au_count) + CBR_BYTES(cbr_au_count);
  285. bytes += sizeof(struct gru_thread_state);
  286. gts = kmalloc(bytes, GFP_KERNEL);
  287. if (!gts)
  288. return ERR_PTR(-ENOMEM);
  289. STAT(gts_alloc);
  290. memset(gts, 0, sizeof(struct gru_thread_state)); /* zero out header */
  291. atomic_set(&gts->ts_refcnt, 1);
  292. mutex_init(&gts->ts_ctxlock);
  293. gts->ts_cbr_au_count = cbr_au_count;
  294. gts->ts_dsr_au_count = dsr_au_count;
  295. gts->ts_tlb_preload_count = tlb_preload_count;
  296. gts->ts_user_options = options;
  297. gts->ts_user_blade_id = -1;
  298. gts->ts_user_chiplet_id = -1;
  299. gts->ts_tsid = tsid;
  300. gts->ts_ctxnum = NULLCTX;
  301. gts->ts_tlb_int_select = -1;
  302. gts->ts_cch_req_slice = -1;
  303. gts->ts_sizeavail = GRU_SIZEAVAIL(PAGE_SHIFT);
  304. if (vma) {
  305. gts->ts_mm = current->mm;
  306. gts->ts_vma = vma;
  307. gms = gru_register_mmu_notifier();
  308. if (IS_ERR(gms))
  309. goto err;
  310. gts->ts_gms = gms;
  311. }
  312. gru_dbg(grudev, "alloc gts %p\n", gts);
  313. return gts;
  314. err:
  315. gts_drop(gts);
  316. return ERR_CAST(gms);
  317. }
  318. /*
  319. * Allocate a vma private data structure.
  320. */
  321. struct gru_vma_data *gru_alloc_vma_data(struct vm_area_struct *vma, int tsid)
  322. {
  323. struct gru_vma_data *vdata = NULL;
  324. vdata = kmalloc(sizeof(*vdata), GFP_KERNEL);
  325. if (!vdata)
  326. return NULL;
  327. STAT(vdata_alloc);
  328. INIT_LIST_HEAD(&vdata->vd_head);
  329. spin_lock_init(&vdata->vd_lock);
  330. gru_dbg(grudev, "alloc vdata %p\n", vdata);
  331. return vdata;
  332. }
  333. /*
  334. * Find the thread state structure for the current thread.
  335. */
  336. struct gru_thread_state *gru_find_thread_state(struct vm_area_struct *vma,
  337. int tsid)
  338. {
  339. struct gru_vma_data *vdata = vma->vm_private_data;
  340. struct gru_thread_state *gts;
  341. spin_lock(&vdata->vd_lock);
  342. gts = gru_find_current_gts_nolock(vdata, tsid);
  343. spin_unlock(&vdata->vd_lock);
  344. gru_dbg(grudev, "vma %p, gts %p\n", vma, gts);
  345. return gts;
  346. }
  347. /*
  348. * Allocate a new thread state for a GSEG. Note that races may allow
  349. * another thread to race to create a gts.
  350. */
  351. struct gru_thread_state *gru_alloc_thread_state(struct vm_area_struct *vma,
  352. int tsid)
  353. {
  354. struct gru_vma_data *vdata = vma->vm_private_data;
  355. struct gru_thread_state *gts, *ngts;
  356. gts = gru_alloc_gts(vma, vdata->vd_cbr_au_count,
  357. vdata->vd_dsr_au_count,
  358. vdata->vd_tlb_preload_count,
  359. vdata->vd_user_options, tsid);
  360. if (IS_ERR(gts))
  361. return gts;
  362. spin_lock(&vdata->vd_lock);
  363. ngts = gru_find_current_gts_nolock(vdata, tsid);
  364. if (ngts) {
  365. gts_drop(gts);
  366. gts = ngts;
  367. STAT(gts_double_allocate);
  368. } else {
  369. list_add(&gts->ts_next, &vdata->vd_head);
  370. }
  371. spin_unlock(&vdata->vd_lock);
  372. gru_dbg(grudev, "vma %p, gts %p\n", vma, gts);
  373. return gts;
  374. }
  375. /*
  376. * Free the GRU context assigned to the thread state.
  377. */
  378. static void gru_free_gru_context(struct gru_thread_state *gts)
  379. {
  380. struct gru_state *gru;
  381. gru = gts->ts_gru;
  382. gru_dbg(grudev, "gts %p, gid %d\n", gts, gru->gs_gid);
  383. spin_lock(&gru->gs_lock);
  384. gru->gs_gts[gts->ts_ctxnum] = NULL;
  385. free_gru_resources(gru, gts);
  386. BUG_ON(test_bit(gts->ts_ctxnum, &gru->gs_context_map) == 0);
  387. __clear_bit(gts->ts_ctxnum, &gru->gs_context_map);
  388. gts->ts_ctxnum = NULLCTX;
  389. gts->ts_gru = NULL;
  390. gts->ts_blade = -1;
  391. spin_unlock(&gru->gs_lock);
  392. gts_drop(gts);
  393. STAT(free_context);
  394. }
  395. /*
  396. * Prefetching cachelines help hardware performance.
  397. * (Strictly a performance enhancement. Not functionally required).
  398. */
  399. static void prefetch_data(void *p, int num, int stride)
  400. {
  401. while (num-- > 0) {
  402. prefetchw(p);
  403. p += stride;
  404. }
  405. }
  406. static inline long gru_copy_handle(void *d, void *s)
  407. {
  408. memcpy(d, s, GRU_HANDLE_BYTES);
  409. return GRU_HANDLE_BYTES;
  410. }
  411. static void gru_prefetch_context(void *gseg, void *cb, void *cbe,
  412. unsigned long cbrmap, unsigned long length)
  413. {
  414. int i, scr;
  415. prefetch_data(gseg + GRU_DS_BASE, length / GRU_CACHE_LINE_BYTES,
  416. GRU_CACHE_LINE_BYTES);
  417. for_each_cbr_in_allocation_map(i, &cbrmap, scr) {
  418. prefetch_data(cb, 1, GRU_CACHE_LINE_BYTES);
  419. prefetch_data(cbe + i * GRU_HANDLE_STRIDE, 1,
  420. GRU_CACHE_LINE_BYTES);
  421. cb += GRU_HANDLE_STRIDE;
  422. }
  423. }
  424. static void gru_load_context_data(void *save, void *grubase, int ctxnum,
  425. unsigned long cbrmap, unsigned long dsrmap,
  426. int data_valid)
  427. {
  428. void *gseg, *cb, *cbe;
  429. unsigned long length;
  430. int i, scr;
  431. gseg = grubase + ctxnum * GRU_GSEG_STRIDE;
  432. cb = gseg + GRU_CB_BASE;
  433. cbe = grubase + GRU_CBE_BASE;
  434. length = hweight64(dsrmap) * GRU_DSR_AU_BYTES;
  435. gru_prefetch_context(gseg, cb, cbe, cbrmap, length);
  436. for_each_cbr_in_allocation_map(i, &cbrmap, scr) {
  437. if (data_valid) {
  438. save += gru_copy_handle(cb, save);
  439. save += gru_copy_handle(cbe + i * GRU_HANDLE_STRIDE,
  440. save);
  441. } else {
  442. memset(cb, 0, GRU_CACHE_LINE_BYTES);
  443. memset(cbe + i * GRU_HANDLE_STRIDE, 0,
  444. GRU_CACHE_LINE_BYTES);
  445. }
  446. /* Flush CBE to hide race in context restart */
  447. mb();
  448. gru_flush_cache(cbe + i * GRU_HANDLE_STRIDE);
  449. cb += GRU_HANDLE_STRIDE;
  450. }
  451. if (data_valid)
  452. memcpy(gseg + GRU_DS_BASE, save, length);
  453. else
  454. memset(gseg + GRU_DS_BASE, 0, length);
  455. }
  456. static void gru_unload_context_data(void *save, void *grubase, int ctxnum,
  457. unsigned long cbrmap, unsigned long dsrmap)
  458. {
  459. void *gseg, *cb, *cbe;
  460. unsigned long length;
  461. int i, scr;
  462. gseg = grubase + ctxnum * GRU_GSEG_STRIDE;
  463. cb = gseg + GRU_CB_BASE;
  464. cbe = grubase + GRU_CBE_BASE;
  465. length = hweight64(dsrmap) * GRU_DSR_AU_BYTES;
  466. /* CBEs may not be coherent. Flush them from cache */
  467. for_each_cbr_in_allocation_map(i, &cbrmap, scr)
  468. gru_flush_cache(cbe + i * GRU_HANDLE_STRIDE);
  469. mb(); /* Let the CL flush complete */
  470. gru_prefetch_context(gseg, cb, cbe, cbrmap, length);
  471. for_each_cbr_in_allocation_map(i, &cbrmap, scr) {
  472. save += gru_copy_handle(save, cb);
  473. save += gru_copy_handle(save, cbe + i * GRU_HANDLE_STRIDE);
  474. cb += GRU_HANDLE_STRIDE;
  475. }
  476. memcpy(save, gseg + GRU_DS_BASE, length);
  477. }
  478. void gru_unload_context(struct gru_thread_state *gts, int savestate)
  479. {
  480. struct gru_state *gru = gts->ts_gru;
  481. struct gru_context_configuration_handle *cch;
  482. int ctxnum = gts->ts_ctxnum;
  483. if (!is_kernel_context(gts))
  484. zap_vma_ptes(gts->ts_vma, UGRUADDR(gts), GRU_GSEG_PAGESIZE);
  485. cch = get_cch(gru->gs_gru_base_vaddr, ctxnum);
  486. gru_dbg(grudev, "gts %p, cbrmap 0x%lx, dsrmap 0x%lx\n",
  487. gts, gts->ts_cbr_map, gts->ts_dsr_map);
  488. lock_cch_handle(cch);
  489. if (cch_interrupt_sync(cch))
  490. BUG();
  491. if (!is_kernel_context(gts))
  492. gru_unload_mm_tracker(gru, gts);
  493. if (savestate) {
  494. gru_unload_context_data(gts->ts_gdata, gru->gs_gru_base_vaddr,
  495. ctxnum, gts->ts_cbr_map,
  496. gts->ts_dsr_map);
  497. gts->ts_data_valid = 1;
  498. }
  499. if (cch_deallocate(cch))
  500. BUG();
  501. unlock_cch_handle(cch);
  502. gru_free_gru_context(gts);
  503. }
  504. /*
  505. * Load a GRU context by copying it from the thread data structure in memory
  506. * to the GRU.
  507. */
  508. void gru_load_context(struct gru_thread_state *gts)
  509. {
  510. struct gru_state *gru = gts->ts_gru;
  511. struct gru_context_configuration_handle *cch;
  512. int i, err, asid, ctxnum = gts->ts_ctxnum;
  513. cch = get_cch(gru->gs_gru_base_vaddr, ctxnum);
  514. lock_cch_handle(cch);
  515. cch->tfm_fault_bit_enable =
  516. (gts->ts_user_options == GRU_OPT_MISS_FMM_POLL
  517. || gts->ts_user_options == GRU_OPT_MISS_FMM_INTR);
  518. cch->tlb_int_enable = (gts->ts_user_options == GRU_OPT_MISS_FMM_INTR);
  519. if (cch->tlb_int_enable) {
  520. gts->ts_tlb_int_select = gru_cpu_fault_map_id();
  521. cch->tlb_int_select = gts->ts_tlb_int_select;
  522. }
  523. if (gts->ts_cch_req_slice >= 0) {
  524. cch->req_slice_set_enable = 1;
  525. cch->req_slice = gts->ts_cch_req_slice;
  526. } else {
  527. cch->req_slice_set_enable =0;
  528. }
  529. cch->tfm_done_bit_enable = 0;
  530. cch->dsr_allocation_map = gts->ts_dsr_map;
  531. cch->cbr_allocation_map = gts->ts_cbr_map;
  532. if (is_kernel_context(gts)) {
  533. cch->unmap_enable = 1;
  534. cch->tfm_done_bit_enable = 1;
  535. cch->cb_int_enable = 1;
  536. cch->tlb_int_select = 0; /* For now, ints go to cpu 0 */
  537. } else {
  538. cch->unmap_enable = 0;
  539. cch->tfm_done_bit_enable = 0;
  540. cch->cb_int_enable = 0;
  541. asid = gru_load_mm_tracker(gru, gts);
  542. for (i = 0; i < 8; i++) {
  543. cch->asid[i] = asid + i;
  544. cch->sizeavail[i] = gts->ts_sizeavail;
  545. }
  546. }
  547. err = cch_allocate(cch);
  548. if (err) {
  549. gru_dbg(grudev,
  550. "err %d: cch %p, gts %p, cbr 0x%lx, dsr 0x%lx\n",
  551. err, cch, gts, gts->ts_cbr_map, gts->ts_dsr_map);
  552. BUG();
  553. }
  554. gru_load_context_data(gts->ts_gdata, gru->gs_gru_base_vaddr, ctxnum,
  555. gts->ts_cbr_map, gts->ts_dsr_map, gts->ts_data_valid);
  556. if (cch_start(cch))
  557. BUG();
  558. unlock_cch_handle(cch);
  559. gru_dbg(grudev, "gid %d, gts %p, cbrmap 0x%lx, dsrmap 0x%lx, tie %d, tis %d\n",
  560. gts->ts_gru->gs_gid, gts, gts->ts_cbr_map, gts->ts_dsr_map,
  561. (gts->ts_user_options == GRU_OPT_MISS_FMM_INTR), gts->ts_tlb_int_select);
  562. }
  563. /*
  564. * Update fields in an active CCH:
  565. * - retarget interrupts on local blade
  566. * - update sizeavail mask
  567. */
  568. int gru_update_cch(struct gru_thread_state *gts)
  569. {
  570. struct gru_context_configuration_handle *cch;
  571. struct gru_state *gru = gts->ts_gru;
  572. int i, ctxnum = gts->ts_ctxnum, ret = 0;
  573. cch = get_cch(gru->gs_gru_base_vaddr, ctxnum);
  574. lock_cch_handle(cch);
  575. if (cch->state == CCHSTATE_ACTIVE) {
  576. if (gru->gs_gts[gts->ts_ctxnum] != gts)
  577. goto exit;
  578. if (cch_interrupt(cch))
  579. BUG();
  580. for (i = 0; i < 8; i++)
  581. cch->sizeavail[i] = gts->ts_sizeavail;
  582. gts->ts_tlb_int_select = gru_cpu_fault_map_id();
  583. cch->tlb_int_select = gru_cpu_fault_map_id();
  584. cch->tfm_fault_bit_enable =
  585. (gts->ts_user_options == GRU_OPT_MISS_FMM_POLL
  586. || gts->ts_user_options == GRU_OPT_MISS_FMM_INTR);
  587. if (cch_start(cch))
  588. BUG();
  589. ret = 1;
  590. }
  591. exit:
  592. unlock_cch_handle(cch);
  593. return ret;
  594. }
  595. /*
  596. * Update CCH tlb interrupt select. Required when all the following is true:
  597. * - task's GRU context is loaded into a GRU
  598. * - task is using interrupt notification for TLB faults
  599. * - task has migrated to a different cpu on the same blade where
  600. * it was previously running.
  601. */
  602. static int gru_retarget_intr(struct gru_thread_state *gts)
  603. {
  604. if (gts->ts_tlb_int_select < 0
  605. || gts->ts_tlb_int_select == gru_cpu_fault_map_id())
  606. return 0;
  607. gru_dbg(grudev, "retarget from %d to %d\n", gts->ts_tlb_int_select,
  608. gru_cpu_fault_map_id());
  609. return gru_update_cch(gts);
  610. }
  611. /*
  612. * Check if a GRU context is allowed to use a specific chiplet. By default
  613. * a context is assigned to any blade-local chiplet. However, users can
  614. * override this.
  615. * Returns 1 if assignment allowed, 0 otherwise
  616. */
  617. static int gru_check_chiplet_assignment(struct gru_state *gru,
  618. struct gru_thread_state *gts)
  619. {
  620. int blade_id;
  621. int chiplet_id;
  622. blade_id = gts->ts_user_blade_id;
  623. if (blade_id < 0)
  624. blade_id = uv_numa_blade_id();
  625. chiplet_id = gts->ts_user_chiplet_id;
  626. return gru->gs_blade_id == blade_id &&
  627. (chiplet_id < 0 || chiplet_id == gru->gs_chiplet_id);
  628. }
  629. /*
  630. * Unload the gru context if it is not assigned to the correct blade or
  631. * chiplet. Misassignment can occur if the process migrates to a different
  632. * blade or if the user changes the selected blade/chiplet.
  633. */
  634. void gru_check_context_placement(struct gru_thread_state *gts)
  635. {
  636. struct gru_state *gru;
  637. /*
  638. * If the current task is the context owner, verify that the
  639. * context is correctly placed. This test is skipped for non-owner
  640. * references. Pthread apps use non-owner references to the CBRs.
  641. */
  642. gru = gts->ts_gru;
  643. if (!gru || gts->ts_tgid_owner != current->tgid)
  644. return;
  645. if (!gru_check_chiplet_assignment(gru, gts)) {
  646. STAT(check_context_unload);
  647. gru_unload_context(gts, 1);
  648. } else if (gru_retarget_intr(gts)) {
  649. STAT(check_context_retarget_intr);
  650. }
  651. }
  652. /*
  653. * Insufficient GRU resources available on the local blade. Steal a context from
  654. * a process. This is a hack until a _real_ resource scheduler is written....
  655. */
  656. #define next_ctxnum(n) ((n) < GRU_NUM_CCH - 2 ? (n) + 1 : 0)
  657. #define next_gru(b, g) (((g) < &(b)->bs_grus[GRU_CHIPLETS_PER_BLADE - 1]) ? \
  658. ((g)+1) : &(b)->bs_grus[0])
  659. static int is_gts_stealable(struct gru_thread_state *gts,
  660. struct gru_blade_state *bs)
  661. {
  662. if (is_kernel_context(gts))
  663. return down_write_trylock(&bs->bs_kgts_sema);
  664. else
  665. return mutex_trylock(&gts->ts_ctxlock);
  666. }
  667. static void gts_stolen(struct gru_thread_state *gts,
  668. struct gru_blade_state *bs)
  669. {
  670. if (is_kernel_context(gts)) {
  671. up_write(&bs->bs_kgts_sema);
  672. STAT(steal_kernel_context);
  673. } else {
  674. mutex_unlock(&gts->ts_ctxlock);
  675. STAT(steal_user_context);
  676. }
  677. }
  678. void gru_steal_context(struct gru_thread_state *gts)
  679. {
  680. struct gru_blade_state *blade;
  681. struct gru_state *gru, *gru0;
  682. struct gru_thread_state *ngts = NULL;
  683. int ctxnum, ctxnum0, flag = 0, cbr, dsr;
  684. int blade_id;
  685. blade_id = gts->ts_user_blade_id;
  686. if (blade_id < 0)
  687. blade_id = uv_numa_blade_id();
  688. cbr = gts->ts_cbr_au_count;
  689. dsr = gts->ts_dsr_au_count;
  690. blade = gru_base[blade_id];
  691. spin_lock(&blade->bs_lock);
  692. ctxnum = next_ctxnum(blade->bs_lru_ctxnum);
  693. gru = blade->bs_lru_gru;
  694. if (ctxnum == 0)
  695. gru = next_gru(blade, gru);
  696. blade->bs_lru_gru = gru;
  697. blade->bs_lru_ctxnum = ctxnum;
  698. ctxnum0 = ctxnum;
  699. gru0 = gru;
  700. while (1) {
  701. if (gru_check_chiplet_assignment(gru, gts)) {
  702. if (check_gru_resources(gru, cbr, dsr, GRU_NUM_CCH))
  703. break;
  704. spin_lock(&gru->gs_lock);
  705. for (; ctxnum < GRU_NUM_CCH; ctxnum++) {
  706. if (flag && gru == gru0 && ctxnum == ctxnum0)
  707. break;
  708. ngts = gru->gs_gts[ctxnum];
  709. /*
  710. * We are grabbing locks out of order, so trylock is
  711. * needed. GTSs are usually not locked, so the odds of
  712. * success are high. If trylock fails, try to steal a
  713. * different GSEG.
  714. */
  715. if (ngts && is_gts_stealable(ngts, blade))
  716. break;
  717. ngts = NULL;
  718. }
  719. spin_unlock(&gru->gs_lock);
  720. if (ngts || (flag && gru == gru0 && ctxnum == ctxnum0))
  721. break;
  722. }
  723. if (flag && gru == gru0)
  724. break;
  725. flag = 1;
  726. ctxnum = 0;
  727. gru = next_gru(blade, gru);
  728. }
  729. spin_unlock(&blade->bs_lock);
  730. if (ngts) {
  731. gts->ustats.context_stolen++;
  732. ngts->ts_steal_jiffies = jiffies;
  733. gru_unload_context(ngts, is_kernel_context(ngts) ? 0 : 1);
  734. gts_stolen(ngts, blade);
  735. } else {
  736. STAT(steal_context_failed);
  737. }
  738. gru_dbg(grudev,
  739. "stole gid %d, ctxnum %d from gts %p. Need cb %d, ds %d;"
  740. " avail cb %ld, ds %ld\n",
  741. gru->gs_gid, ctxnum, ngts, cbr, dsr, hweight64(gru->gs_cbr_map),
  742. hweight64(gru->gs_dsr_map));
  743. }
  744. /*
  745. * Assign a gru context.
  746. */
  747. static int gru_assign_context_number(struct gru_state *gru)
  748. {
  749. int ctxnum;
  750. ctxnum = find_first_zero_bit(&gru->gs_context_map, GRU_NUM_CCH);
  751. __set_bit(ctxnum, &gru->gs_context_map);
  752. return ctxnum;
  753. }
  754. /*
  755. * Scan the GRUs on the local blade & assign a GRU context.
  756. */
  757. struct gru_state *gru_assign_gru_context(struct gru_thread_state *gts)
  758. {
  759. struct gru_state *gru, *grux;
  760. int i, max_active_contexts;
  761. int blade_id = gts->ts_user_blade_id;
  762. if (blade_id < 0)
  763. blade_id = uv_numa_blade_id();
  764. again:
  765. gru = NULL;
  766. max_active_contexts = GRU_NUM_CCH;
  767. for_each_gru_on_blade(grux, blade_id, i) {
  768. if (!gru_check_chiplet_assignment(grux, gts))
  769. continue;
  770. if (check_gru_resources(grux, gts->ts_cbr_au_count,
  771. gts->ts_dsr_au_count,
  772. max_active_contexts)) {
  773. gru = grux;
  774. max_active_contexts = grux->gs_active_contexts;
  775. if (max_active_contexts == 0)
  776. break;
  777. }
  778. }
  779. if (gru) {
  780. spin_lock(&gru->gs_lock);
  781. if (!check_gru_resources(gru, gts->ts_cbr_au_count,
  782. gts->ts_dsr_au_count, GRU_NUM_CCH)) {
  783. spin_unlock(&gru->gs_lock);
  784. goto again;
  785. }
  786. reserve_gru_resources(gru, gts);
  787. gts->ts_gru = gru;
  788. gts->ts_blade = gru->gs_blade_id;
  789. gts->ts_ctxnum = gru_assign_context_number(gru);
  790. atomic_inc(&gts->ts_refcnt);
  791. gru->gs_gts[gts->ts_ctxnum] = gts;
  792. spin_unlock(&gru->gs_lock);
  793. STAT(assign_context);
  794. gru_dbg(grudev,
  795. "gseg %p, gts %p, gid %d, ctx %d, cbr %d, dsr %d\n",
  796. gseg_virtual_address(gts->ts_gru, gts->ts_ctxnum), gts,
  797. gts->ts_gru->gs_gid, gts->ts_ctxnum,
  798. gts->ts_cbr_au_count, gts->ts_dsr_au_count);
  799. } else {
  800. gru_dbg(grudev, "failed to allocate a GTS %s\n", "");
  801. STAT(assign_context_failed);
  802. }
  803. return gru;
  804. }
  805. /*
  806. * gru_nopage
  807. *
  808. * Map the user's GRU segment
  809. *
  810. * Note: gru segments alway mmaped on GRU_GSEG_PAGESIZE boundaries.
  811. */
  812. vm_fault_t gru_fault(struct vm_fault *vmf)
  813. {
  814. struct vm_area_struct *vma = vmf->vma;
  815. struct gru_thread_state *gts;
  816. unsigned long paddr, vaddr;
  817. unsigned long expires;
  818. vaddr = vmf->address;
  819. gru_dbg(grudev, "vma %p, vaddr 0x%lx (0x%lx)\n",
  820. vma, vaddr, GSEG_BASE(vaddr));
  821. STAT(nopfn);
  822. /* The following check ensures vaddr is a valid address in the VMA */
  823. gts = gru_find_thread_state(vma, TSID(vaddr, vma));
  824. if (!gts)
  825. return VM_FAULT_SIGBUS;
  826. again:
  827. mutex_lock(&gts->ts_ctxlock);
  828. preempt_disable();
  829. gru_check_context_placement(gts);
  830. if (!gts->ts_gru) {
  831. STAT(load_user_context);
  832. if (!gru_assign_gru_context(gts)) {
  833. preempt_enable();
  834. mutex_unlock(&gts->ts_ctxlock);
  835. set_current_state(TASK_INTERRUPTIBLE);
  836. schedule_timeout(GRU_ASSIGN_DELAY); /* true hack ZZZ */
  837. expires = gts->ts_steal_jiffies + GRU_STEAL_DELAY;
  838. if (time_before(expires, jiffies))
  839. gru_steal_context(gts);
  840. goto again;
  841. }
  842. gru_load_context(gts);
  843. paddr = gseg_physical_address(gts->ts_gru, gts->ts_ctxnum);
  844. remap_pfn_range(vma, vaddr & ~(GRU_GSEG_PAGESIZE - 1),
  845. paddr >> PAGE_SHIFT, GRU_GSEG_PAGESIZE,
  846. vma->vm_page_prot);
  847. }
  848. preempt_enable();
  849. mutex_unlock(&gts->ts_ctxlock);
  850. return VM_FAULT_NOPAGE;
  851. }