grukservices.c 29 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * SN Platform GRU Driver
  4. *
  5. * KERNEL SERVICES THAT USE THE GRU
  6. *
  7. * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.
  8. */
  9. #include <linux/kernel.h>
  10. #include <linux/errno.h>
  11. #include <linux/slab.h>
  12. #include <linux/mm.h>
  13. #include <linux/spinlock.h>
  14. #include <linux/device.h>
  15. #include <linux/miscdevice.h>
  16. #include <linux/proc_fs.h>
  17. #include <linux/interrupt.h>
  18. #include <linux/sync_core.h>
  19. #include <linux/uaccess.h>
  20. #include <linux/delay.h>
  21. #include <linux/export.h>
  22. #include <asm/io_apic.h>
  23. #include "gru.h"
  24. #include "grulib.h"
  25. #include "grutables.h"
  26. #include "grukservices.h"
  27. #include "gru_instructions.h"
  28. #include <asm/uv/uv_hub.h>
  29. /*
  30. * Kernel GRU Usage
  31. *
  32. * The following is an interim algorithm for management of kernel GRU
  33. * resources. This will likely be replaced when we better understand the
  34. * kernel/user requirements.
  35. *
  36. * Blade percpu resources reserved for kernel use. These resources are
  37. * reserved whenever the the kernel context for the blade is loaded. Note
  38. * that the kernel context is not guaranteed to be always available. It is
  39. * loaded on demand & can be stolen by a user if the user demand exceeds the
  40. * kernel demand. The kernel can always reload the kernel context but
  41. * a SLEEP may be required!!!.
  42. *
  43. * Async Overview:
  44. *
  45. * Each blade has one "kernel context" that owns GRU kernel resources
  46. * located on the blade. Kernel drivers use GRU resources in this context
  47. * for sending messages, zeroing memory, etc.
  48. *
  49. * The kernel context is dynamically loaded on demand. If it is not in
  50. * use by the kernel, the kernel context can be unloaded & given to a user.
  51. * The kernel context will be reloaded when needed. This may require that
  52. * a context be stolen from a user.
  53. * NOTE: frequent unloading/reloading of the kernel context is
  54. * expensive. We are depending on batch schedulers, cpusets, sane
  55. * drivers or some other mechanism to prevent the need for frequent
  56. * stealing/reloading.
  57. *
  58. * The kernel context consists of two parts:
  59. * - 1 CB & a few DSRs that are reserved for each cpu on the blade.
  60. * Each cpu has it's own private resources & does not share them
  61. * with other cpus. These resources are used serially, ie,
  62. * locked, used & unlocked on each call to a function in
  63. * grukservices.
  64. * (Now that we have dynamic loading of kernel contexts, I
  65. * may rethink this & allow sharing between cpus....)
  66. *
  67. * - Additional resources can be reserved long term & used directly
  68. * by UV drivers located in the kernel. Drivers using these GRU
  69. * resources can use asynchronous GRU instructions that send
  70. * interrupts on completion.
  71. * - these resources must be explicitly locked/unlocked
  72. * - locked resources prevent (obviously) the kernel
  73. * context from being unloaded.
  74. * - drivers using these resource directly issue their own
  75. * GRU instruction and must wait/check completion.
  76. *
  77. * When these resources are reserved, the caller can optionally
  78. * associate a wait_queue with the resources and use asynchronous
  79. * GRU instructions. When an async GRU instruction completes, the
  80. * driver will do a wakeup on the event.
  81. *
  82. */
  83. #define ASYNC_HAN_TO_BID(h) ((h) - 1)
  84. #define ASYNC_BID_TO_HAN(b) ((b) + 1)
  85. #define ASYNC_HAN_TO_BS(h) gru_base[ASYNC_HAN_TO_BID(h)]
  86. #define GRU_NUM_KERNEL_CBR 1
  87. #define GRU_NUM_KERNEL_DSR_BYTES 256
  88. #define GRU_NUM_KERNEL_DSR_CL (GRU_NUM_KERNEL_DSR_BYTES / \
  89. GRU_CACHE_LINE_BYTES)
  90. /* GRU instruction attributes for all instructions */
  91. #define IMA IMA_CB_DELAY
  92. /* GRU cacheline size is always 64 bytes - even on arches with 128 byte lines */
  93. #define __gru_cacheline_aligned__ \
  94. __attribute__((__aligned__(GRU_CACHE_LINE_BYTES)))
  95. #define MAGIC 0x1234567887654321UL
  96. /* Default retry count for GRU errors on kernel instructions */
  97. #define EXCEPTION_RETRY_LIMIT 3
  98. /* Status of message queue sections */
  99. #define MQS_EMPTY 0
  100. #define MQS_FULL 1
  101. #define MQS_NOOP 2
  102. /*----------------- RESOURCE MANAGEMENT -------------------------------------*/
  103. /* optimized for x86_64 */
  104. struct message_queue {
  105. union gru_mesqhead head __gru_cacheline_aligned__; /* CL 0 */
  106. int qlines; /* DW 1 */
  107. long hstatus[2];
  108. void *next __gru_cacheline_aligned__;/* CL 1 */
  109. void *limit;
  110. void *start;
  111. void *start2;
  112. char data ____cacheline_aligned; /* CL 2 */
  113. };
  114. /* First word in every message - used by mesq interface */
  115. struct message_header {
  116. char present;
  117. char present2;
  118. char lines;
  119. char fill;
  120. };
  121. #define HSTATUS(mq, h) ((mq) + offsetof(struct message_queue, hstatus[h]))
  122. /*
  123. * Reload the blade's kernel context into a GRU chiplet. Called holding
  124. * the bs_kgts_sema for READ. Will steal user contexts if necessary.
  125. */
  126. static void gru_load_kernel_context(struct gru_blade_state *bs, int blade_id)
  127. {
  128. struct gru_state *gru;
  129. struct gru_thread_state *kgts;
  130. void *vaddr;
  131. int ctxnum, ncpus;
  132. up_read(&bs->bs_kgts_sema);
  133. down_write(&bs->bs_kgts_sema);
  134. if (!bs->bs_kgts) {
  135. do {
  136. bs->bs_kgts = gru_alloc_gts(NULL, 0, 0, 0, 0, 0);
  137. if (!IS_ERR(bs->bs_kgts))
  138. break;
  139. msleep(1);
  140. } while (true);
  141. bs->bs_kgts->ts_user_blade_id = blade_id;
  142. }
  143. kgts = bs->bs_kgts;
  144. if (!kgts->ts_gru) {
  145. STAT(load_kernel_context);
  146. ncpus = uv_blade_nr_possible_cpus(blade_id);
  147. kgts->ts_cbr_au_count = GRU_CB_COUNT_TO_AU(
  148. GRU_NUM_KERNEL_CBR * ncpus + bs->bs_async_cbrs);
  149. kgts->ts_dsr_au_count = GRU_DS_BYTES_TO_AU(
  150. GRU_NUM_KERNEL_DSR_BYTES * ncpus +
  151. bs->bs_async_dsr_bytes);
  152. while (!gru_assign_gru_context(kgts)) {
  153. msleep(1);
  154. gru_steal_context(kgts);
  155. }
  156. gru_load_context(kgts);
  157. gru = bs->bs_kgts->ts_gru;
  158. vaddr = gru->gs_gru_base_vaddr;
  159. ctxnum = kgts->ts_ctxnum;
  160. bs->kernel_cb = get_gseg_base_address_cb(vaddr, ctxnum, 0);
  161. bs->kernel_dsr = get_gseg_base_address_ds(vaddr, ctxnum, 0);
  162. }
  163. downgrade_write(&bs->bs_kgts_sema);
  164. }
  165. /*
  166. * Free all kernel contexts that are not currently in use.
  167. * Returns 0 if all freed, else number of inuse context.
  168. */
  169. static int gru_free_kernel_contexts(void)
  170. {
  171. struct gru_blade_state *bs;
  172. struct gru_thread_state *kgts;
  173. int bid, ret = 0;
  174. for (bid = 0; bid < GRU_MAX_BLADES; bid++) {
  175. bs = gru_base[bid];
  176. if (!bs)
  177. continue;
  178. /* Ignore busy contexts. Don't want to block here. */
  179. if (down_write_trylock(&bs->bs_kgts_sema)) {
  180. kgts = bs->bs_kgts;
  181. if (kgts && kgts->ts_gru)
  182. gru_unload_context(kgts, 0);
  183. bs->bs_kgts = NULL;
  184. up_write(&bs->bs_kgts_sema);
  185. kfree(kgts);
  186. } else {
  187. ret++;
  188. }
  189. }
  190. return ret;
  191. }
  192. /*
  193. * Lock & load the kernel context for the specified blade.
  194. */
  195. static struct gru_blade_state *gru_lock_kernel_context(int blade_id)
  196. {
  197. struct gru_blade_state *bs;
  198. int bid;
  199. STAT(lock_kernel_context);
  200. again:
  201. bid = blade_id < 0 ? uv_numa_blade_id() : blade_id;
  202. bs = gru_base[bid];
  203. /* Handle the case where migration occurred while waiting for the sema */
  204. down_read(&bs->bs_kgts_sema);
  205. if (blade_id < 0 && bid != uv_numa_blade_id()) {
  206. up_read(&bs->bs_kgts_sema);
  207. goto again;
  208. }
  209. if (!bs->bs_kgts || !bs->bs_kgts->ts_gru)
  210. gru_load_kernel_context(bs, bid);
  211. return bs;
  212. }
  213. /*
  214. * Unlock the kernel context for the specified blade. Context is not
  215. * unloaded but may be stolen before next use.
  216. */
  217. static void gru_unlock_kernel_context(int blade_id)
  218. {
  219. struct gru_blade_state *bs;
  220. bs = gru_base[blade_id];
  221. up_read(&bs->bs_kgts_sema);
  222. STAT(unlock_kernel_context);
  223. }
  224. /*
  225. * Reserve & get pointers to the DSR/CBRs reserved for the current cpu.
  226. * - returns with preemption disabled
  227. */
  228. static int gru_get_cpu_resources(int dsr_bytes, void **cb, void **dsr)
  229. {
  230. struct gru_blade_state *bs;
  231. int lcpu;
  232. BUG_ON(dsr_bytes > GRU_NUM_KERNEL_DSR_BYTES);
  233. preempt_disable();
  234. bs = gru_lock_kernel_context(-1);
  235. lcpu = uv_blade_processor_id();
  236. *cb = bs->kernel_cb + lcpu * GRU_HANDLE_STRIDE;
  237. *dsr = bs->kernel_dsr + lcpu * GRU_NUM_KERNEL_DSR_BYTES;
  238. return 0;
  239. }
  240. /*
  241. * Free the current cpus reserved DSR/CBR resources.
  242. */
  243. static void gru_free_cpu_resources(void *cb, void *dsr)
  244. {
  245. gru_unlock_kernel_context(uv_numa_blade_id());
  246. preempt_enable();
  247. }
  248. /*
  249. * Reserve GRU resources to be used asynchronously.
  250. * Note: currently supports only 1 reservation per blade.
  251. *
  252. * input:
  253. * blade_id - blade on which resources should be reserved
  254. * cbrs - number of CBRs
  255. * dsr_bytes - number of DSR bytes needed
  256. * output:
  257. * handle to identify resource
  258. * (0 = async resources already reserved)
  259. */
  260. unsigned long gru_reserve_async_resources(int blade_id, int cbrs, int dsr_bytes,
  261. struct completion *cmp)
  262. {
  263. struct gru_blade_state *bs;
  264. struct gru_thread_state *kgts;
  265. int ret = 0;
  266. bs = gru_base[blade_id];
  267. down_write(&bs->bs_kgts_sema);
  268. /* Verify no resources already reserved */
  269. if (bs->bs_async_dsr_bytes + bs->bs_async_cbrs)
  270. goto done;
  271. bs->bs_async_dsr_bytes = dsr_bytes;
  272. bs->bs_async_cbrs = cbrs;
  273. bs->bs_async_wq = cmp;
  274. kgts = bs->bs_kgts;
  275. /* Resources changed. Unload context if already loaded */
  276. if (kgts && kgts->ts_gru)
  277. gru_unload_context(kgts, 0);
  278. ret = ASYNC_BID_TO_HAN(blade_id);
  279. done:
  280. up_write(&bs->bs_kgts_sema);
  281. return ret;
  282. }
  283. /*
  284. * Release async resources previously reserved.
  285. *
  286. * input:
  287. * han - handle to identify resources
  288. */
  289. void gru_release_async_resources(unsigned long han)
  290. {
  291. struct gru_blade_state *bs = ASYNC_HAN_TO_BS(han);
  292. down_write(&bs->bs_kgts_sema);
  293. bs->bs_async_dsr_bytes = 0;
  294. bs->bs_async_cbrs = 0;
  295. bs->bs_async_wq = NULL;
  296. up_write(&bs->bs_kgts_sema);
  297. }
  298. /*
  299. * Wait for async GRU instructions to complete.
  300. *
  301. * input:
  302. * han - handle to identify resources
  303. */
  304. void gru_wait_async_cbr(unsigned long han)
  305. {
  306. struct gru_blade_state *bs = ASYNC_HAN_TO_BS(han);
  307. wait_for_completion(bs->bs_async_wq);
  308. mb();
  309. }
  310. /*
  311. * Lock previous reserved async GRU resources
  312. *
  313. * input:
  314. * han - handle to identify resources
  315. * output:
  316. * cb - pointer to first CBR
  317. * dsr - pointer to first DSR
  318. */
  319. void gru_lock_async_resource(unsigned long han, void **cb, void **dsr)
  320. {
  321. struct gru_blade_state *bs = ASYNC_HAN_TO_BS(han);
  322. int blade_id = ASYNC_HAN_TO_BID(han);
  323. int ncpus;
  324. gru_lock_kernel_context(blade_id);
  325. ncpus = uv_blade_nr_possible_cpus(blade_id);
  326. if (cb)
  327. *cb = bs->kernel_cb + ncpus * GRU_HANDLE_STRIDE;
  328. if (dsr)
  329. *dsr = bs->kernel_dsr + ncpus * GRU_NUM_KERNEL_DSR_BYTES;
  330. }
  331. /*
  332. * Unlock previous reserved async GRU resources
  333. *
  334. * input:
  335. * han - handle to identify resources
  336. */
  337. void gru_unlock_async_resource(unsigned long han)
  338. {
  339. int blade_id = ASYNC_HAN_TO_BID(han);
  340. gru_unlock_kernel_context(blade_id);
  341. }
  342. /*----------------------------------------------------------------------*/
  343. int gru_get_cb_exception_detail(void *cb,
  344. struct control_block_extended_exc_detail *excdet)
  345. {
  346. struct gru_control_block_extended *cbe;
  347. struct gru_thread_state *kgts = NULL;
  348. unsigned long off;
  349. int cbrnum, bid;
  350. /*
  351. * Locate kgts for cb. This algorithm is SLOW but
  352. * this function is rarely called (ie., almost never).
  353. * Performance does not matter.
  354. */
  355. for_each_possible_blade(bid) {
  356. if (!gru_base[bid])
  357. break;
  358. kgts = gru_base[bid]->bs_kgts;
  359. if (!kgts || !kgts->ts_gru)
  360. continue;
  361. off = cb - kgts->ts_gru->gs_gru_base_vaddr;
  362. if (off < GRU_SIZE)
  363. break;
  364. kgts = NULL;
  365. }
  366. BUG_ON(!kgts);
  367. cbrnum = thread_cbr_number(kgts, get_cb_number(cb));
  368. cbe = get_cbe(GRUBASE(cb), cbrnum);
  369. gru_flush_cache(cbe); /* CBE not coherent */
  370. sync_core();
  371. excdet->opc = cbe->opccpy;
  372. excdet->exopc = cbe->exopccpy;
  373. excdet->ecause = cbe->ecause;
  374. excdet->exceptdet0 = cbe->idef1upd;
  375. excdet->exceptdet1 = cbe->idef3upd;
  376. gru_flush_cache(cbe);
  377. return 0;
  378. }
  379. static char *gru_get_cb_exception_detail_str(int ret, void *cb,
  380. char *buf, int size)
  381. {
  382. struct gru_control_block_status *gen = (void *)cb;
  383. struct control_block_extended_exc_detail excdet;
  384. if (ret > 0 && gen->istatus == CBS_EXCEPTION) {
  385. gru_get_cb_exception_detail(cb, &excdet);
  386. snprintf(buf, size,
  387. "GRU:%d exception: cb %p, opc %d, exopc %d, ecause 0x%x,"
  388. "excdet0 0x%lx, excdet1 0x%x", smp_processor_id(),
  389. gen, excdet.opc, excdet.exopc, excdet.ecause,
  390. excdet.exceptdet0, excdet.exceptdet1);
  391. } else {
  392. snprintf(buf, size, "No exception");
  393. }
  394. return buf;
  395. }
  396. static int gru_wait_idle_or_exception(struct gru_control_block_status *gen)
  397. {
  398. while (gen->istatus >= CBS_ACTIVE) {
  399. cpu_relax();
  400. barrier();
  401. }
  402. return gen->istatus;
  403. }
  404. static int gru_retry_exception(void *cb)
  405. {
  406. struct gru_control_block_status *gen = (void *)cb;
  407. struct control_block_extended_exc_detail excdet;
  408. int retry = EXCEPTION_RETRY_LIMIT;
  409. while (1) {
  410. if (gru_wait_idle_or_exception(gen) == CBS_IDLE)
  411. return CBS_IDLE;
  412. if (gru_get_cb_message_queue_substatus(cb))
  413. return CBS_EXCEPTION;
  414. gru_get_cb_exception_detail(cb, &excdet);
  415. if ((excdet.ecause & ~EXCEPTION_RETRY_BITS) ||
  416. (excdet.cbrexecstatus & CBR_EXS_ABORT_OCC))
  417. break;
  418. if (retry-- == 0)
  419. break;
  420. gen->icmd = 1;
  421. gru_flush_cache(gen);
  422. }
  423. return CBS_EXCEPTION;
  424. }
  425. int gru_check_status_proc(void *cb)
  426. {
  427. struct gru_control_block_status *gen = (void *)cb;
  428. int ret;
  429. ret = gen->istatus;
  430. if (ret == CBS_EXCEPTION)
  431. ret = gru_retry_exception(cb);
  432. rmb();
  433. return ret;
  434. }
  435. int gru_wait_proc(void *cb)
  436. {
  437. struct gru_control_block_status *gen = (void *)cb;
  438. int ret;
  439. ret = gru_wait_idle_or_exception(gen);
  440. if (ret == CBS_EXCEPTION)
  441. ret = gru_retry_exception(cb);
  442. rmb();
  443. return ret;
  444. }
  445. static void gru_abort(int ret, void *cb, char *str)
  446. {
  447. char buf[GRU_EXC_STR_SIZE];
  448. panic("GRU FATAL ERROR: %s - %s\n", str,
  449. gru_get_cb_exception_detail_str(ret, cb, buf, sizeof(buf)));
  450. }
  451. void gru_wait_abort_proc(void *cb)
  452. {
  453. int ret;
  454. ret = gru_wait_proc(cb);
  455. if (ret)
  456. gru_abort(ret, cb, "gru_wait_abort");
  457. }
  458. /*------------------------------ MESSAGE QUEUES -----------------------------*/
  459. /* Internal status . These are NOT returned to the user. */
  460. #define MQIE_AGAIN -1 /* try again */
  461. /*
  462. * Save/restore the "present" flag that is in the second line of 2-line
  463. * messages
  464. */
  465. static inline int get_present2(void *p)
  466. {
  467. struct message_header *mhdr = p + GRU_CACHE_LINE_BYTES;
  468. return mhdr->present;
  469. }
  470. static inline void restore_present2(void *p, int val)
  471. {
  472. struct message_header *mhdr = p + GRU_CACHE_LINE_BYTES;
  473. mhdr->present = val;
  474. }
  475. /*
  476. * Create a message queue.
  477. * qlines - message queue size in cache lines. Includes 2-line header.
  478. */
  479. int gru_create_message_queue(struct gru_message_queue_desc *mqd,
  480. void *p, unsigned int bytes, int nasid, int vector, int apicid)
  481. {
  482. struct message_queue *mq = p;
  483. unsigned int qlines;
  484. qlines = bytes / GRU_CACHE_LINE_BYTES - 2;
  485. memset(mq, 0, bytes);
  486. mq->start = &mq->data;
  487. mq->start2 = &mq->data + (qlines / 2 - 1) * GRU_CACHE_LINE_BYTES;
  488. mq->next = &mq->data;
  489. mq->limit = &mq->data + (qlines - 2) * GRU_CACHE_LINE_BYTES;
  490. mq->qlines = qlines;
  491. mq->hstatus[0] = 0;
  492. mq->hstatus[1] = 1;
  493. mq->head = gru_mesq_head(2, qlines / 2 + 1);
  494. mqd->mq = mq;
  495. mqd->mq_gpa = uv_gpa(mq);
  496. mqd->qlines = qlines;
  497. mqd->interrupt_pnode = nasid >> 1;
  498. mqd->interrupt_vector = vector;
  499. mqd->interrupt_apicid = apicid;
  500. return 0;
  501. }
  502. EXPORT_SYMBOL_GPL(gru_create_message_queue);
  503. /*
  504. * Send a NOOP message to a message queue
  505. * Returns:
  506. * 0 - if queue is full after the send. This is the normal case
  507. * but various races can change this.
  508. * -1 - if mesq sent successfully but queue not full
  509. * >0 - unexpected error. MQE_xxx returned
  510. */
  511. static int send_noop_message(void *cb, struct gru_message_queue_desc *mqd,
  512. void *mesg)
  513. {
  514. const struct message_header noop_header = {
  515. .present = MQS_NOOP, .lines = 1};
  516. unsigned long m;
  517. int substatus, ret;
  518. struct message_header save_mhdr, *mhdr = mesg;
  519. STAT(mesq_noop);
  520. save_mhdr = *mhdr;
  521. *mhdr = noop_header;
  522. gru_mesq(cb, mqd->mq_gpa, gru_get_tri(mhdr), 1, IMA);
  523. ret = gru_wait(cb);
  524. if (ret) {
  525. substatus = gru_get_cb_message_queue_substatus(cb);
  526. switch (substatus) {
  527. case CBSS_NO_ERROR:
  528. STAT(mesq_noop_unexpected_error);
  529. ret = MQE_UNEXPECTED_CB_ERR;
  530. break;
  531. case CBSS_LB_OVERFLOWED:
  532. STAT(mesq_noop_lb_overflow);
  533. ret = MQE_CONGESTION;
  534. break;
  535. case CBSS_QLIMIT_REACHED:
  536. STAT(mesq_noop_qlimit_reached);
  537. ret = 0;
  538. break;
  539. case CBSS_AMO_NACKED:
  540. STAT(mesq_noop_amo_nacked);
  541. ret = MQE_CONGESTION;
  542. break;
  543. case CBSS_PUT_NACKED:
  544. STAT(mesq_noop_put_nacked);
  545. m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6);
  546. gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, 1, 1,
  547. IMA);
  548. if (gru_wait(cb) == CBS_IDLE)
  549. ret = MQIE_AGAIN;
  550. else
  551. ret = MQE_UNEXPECTED_CB_ERR;
  552. break;
  553. case CBSS_PAGE_OVERFLOW:
  554. STAT(mesq_noop_page_overflow);
  555. fallthrough;
  556. default:
  557. BUG();
  558. }
  559. }
  560. *mhdr = save_mhdr;
  561. return ret;
  562. }
  563. /*
  564. * Handle a gru_mesq full.
  565. */
  566. static int send_message_queue_full(void *cb, struct gru_message_queue_desc *mqd,
  567. void *mesg, int lines)
  568. {
  569. union gru_mesqhead mqh;
  570. unsigned int limit, head;
  571. unsigned long avalue;
  572. int half, qlines;
  573. /* Determine if switching to first/second half of q */
  574. avalue = gru_get_amo_value(cb);
  575. head = gru_get_amo_value_head(cb);
  576. limit = gru_get_amo_value_limit(cb);
  577. qlines = mqd->qlines;
  578. half = (limit != qlines);
  579. if (half)
  580. mqh = gru_mesq_head(qlines / 2 + 1, qlines);
  581. else
  582. mqh = gru_mesq_head(2, qlines / 2 + 1);
  583. /* Try to get lock for switching head pointer */
  584. gru_gamir(cb, EOP_IR_CLR, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA);
  585. if (gru_wait(cb) != CBS_IDLE)
  586. goto cberr;
  587. if (!gru_get_amo_value(cb)) {
  588. STAT(mesq_qf_locked);
  589. return MQE_QUEUE_FULL;
  590. }
  591. /* Got the lock. Send optional NOP if queue not full, */
  592. if (head != limit) {
  593. if (send_noop_message(cb, mqd, mesg)) {
  594. gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half),
  595. XTYPE_DW, IMA);
  596. if (gru_wait(cb) != CBS_IDLE)
  597. goto cberr;
  598. STAT(mesq_qf_noop_not_full);
  599. return MQIE_AGAIN;
  600. }
  601. avalue++;
  602. }
  603. /* Then flip queuehead to other half of queue. */
  604. gru_gamer(cb, EOP_ERR_CSWAP, mqd->mq_gpa, XTYPE_DW, mqh.val, avalue,
  605. IMA);
  606. if (gru_wait(cb) != CBS_IDLE)
  607. goto cberr;
  608. /* If not successfully in swapping queue head, clear the hstatus lock */
  609. if (gru_get_amo_value(cb) != avalue) {
  610. STAT(mesq_qf_switch_head_failed);
  611. gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half), XTYPE_DW,
  612. IMA);
  613. if (gru_wait(cb) != CBS_IDLE)
  614. goto cberr;
  615. }
  616. return MQIE_AGAIN;
  617. cberr:
  618. STAT(mesq_qf_unexpected_error);
  619. return MQE_UNEXPECTED_CB_ERR;
  620. }
  621. /*
  622. * Handle a PUT failure. Note: if message was a 2-line message, one of the
  623. * lines might have successfully have been written. Before sending the
  624. * message, "present" must be cleared in BOTH lines to prevent the receiver
  625. * from prematurely seeing the full message.
  626. */
  627. static int send_message_put_nacked(void *cb, struct gru_message_queue_desc *mqd,
  628. void *mesg, int lines)
  629. {
  630. unsigned long m;
  631. int ret, loops = 200; /* experimentally determined */
  632. m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6);
  633. if (lines == 2) {
  634. gru_vset(cb, m, 0, XTYPE_CL, lines, 1, IMA);
  635. if (gru_wait(cb) != CBS_IDLE)
  636. return MQE_UNEXPECTED_CB_ERR;
  637. }
  638. gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, lines, 1, IMA);
  639. if (gru_wait(cb) != CBS_IDLE)
  640. return MQE_UNEXPECTED_CB_ERR;
  641. if (!mqd->interrupt_vector)
  642. return MQE_OK;
  643. /*
  644. * Send a noop message in order to deliver a cross-partition interrupt
  645. * to the SSI that contains the target message queue. Normally, the
  646. * interrupt is automatically delivered by hardware following mesq
  647. * operations, but some error conditions require explicit delivery.
  648. * The noop message will trigger delivery. Otherwise partition failures
  649. * could cause unrecovered errors.
  650. */
  651. do {
  652. ret = send_noop_message(cb, mqd, mesg);
  653. } while ((ret == MQIE_AGAIN || ret == MQE_CONGESTION) && (loops-- > 0));
  654. if (ret == MQIE_AGAIN || ret == MQE_CONGESTION) {
  655. /*
  656. * Don't indicate to the app to resend the message, as it's
  657. * already been successfully sent. We simply send an OK
  658. * (rather than fail the send with MQE_UNEXPECTED_CB_ERR),
  659. * assuming that the other side is receiving enough
  660. * interrupts to get this message processed anyway.
  661. */
  662. ret = MQE_OK;
  663. }
  664. return ret;
  665. }
  666. /*
  667. * Handle a gru_mesq failure. Some of these failures are software recoverable
  668. * or retryable.
  669. */
  670. static int send_message_failure(void *cb, struct gru_message_queue_desc *mqd,
  671. void *mesg, int lines)
  672. {
  673. int substatus, ret = 0;
  674. substatus = gru_get_cb_message_queue_substatus(cb);
  675. switch (substatus) {
  676. case CBSS_NO_ERROR:
  677. STAT(mesq_send_unexpected_error);
  678. ret = MQE_UNEXPECTED_CB_ERR;
  679. break;
  680. case CBSS_LB_OVERFLOWED:
  681. STAT(mesq_send_lb_overflow);
  682. ret = MQE_CONGESTION;
  683. break;
  684. case CBSS_QLIMIT_REACHED:
  685. STAT(mesq_send_qlimit_reached);
  686. ret = send_message_queue_full(cb, mqd, mesg, lines);
  687. break;
  688. case CBSS_AMO_NACKED:
  689. STAT(mesq_send_amo_nacked);
  690. ret = MQE_CONGESTION;
  691. break;
  692. case CBSS_PUT_NACKED:
  693. STAT(mesq_send_put_nacked);
  694. ret = send_message_put_nacked(cb, mqd, mesg, lines);
  695. break;
  696. case CBSS_PAGE_OVERFLOW:
  697. STAT(mesq_page_overflow);
  698. fallthrough;
  699. default:
  700. BUG();
  701. }
  702. return ret;
  703. }
  704. /*
  705. * Send a message to a message queue
  706. * mqd message queue descriptor
  707. * mesg message. ust be vaddr within a GSEG
  708. * bytes message size (<= 2 CL)
  709. */
  710. int gru_send_message_gpa(struct gru_message_queue_desc *mqd, void *mesg,
  711. unsigned int bytes)
  712. {
  713. struct message_header *mhdr;
  714. void *cb;
  715. void *dsr;
  716. int istatus, clines, ret;
  717. STAT(mesq_send);
  718. BUG_ON(bytes < sizeof(int) || bytes > 2 * GRU_CACHE_LINE_BYTES);
  719. clines = DIV_ROUND_UP(bytes, GRU_CACHE_LINE_BYTES);
  720. if (gru_get_cpu_resources(bytes, &cb, &dsr))
  721. return MQE_BUG_NO_RESOURCES;
  722. memcpy(dsr, mesg, bytes);
  723. mhdr = dsr;
  724. mhdr->present = MQS_FULL;
  725. mhdr->lines = clines;
  726. if (clines == 2) {
  727. mhdr->present2 = get_present2(mhdr);
  728. restore_present2(mhdr, MQS_FULL);
  729. }
  730. do {
  731. ret = MQE_OK;
  732. gru_mesq(cb, mqd->mq_gpa, gru_get_tri(mhdr), clines, IMA);
  733. istatus = gru_wait(cb);
  734. if (istatus != CBS_IDLE)
  735. ret = send_message_failure(cb, mqd, dsr, clines);
  736. } while (ret == MQIE_AGAIN);
  737. gru_free_cpu_resources(cb, dsr);
  738. if (ret)
  739. STAT(mesq_send_failed);
  740. return ret;
  741. }
  742. EXPORT_SYMBOL_GPL(gru_send_message_gpa);
  743. /*
  744. * Advance the receive pointer for the queue to the next message.
  745. */
  746. void gru_free_message(struct gru_message_queue_desc *mqd, void *mesg)
  747. {
  748. struct message_queue *mq = mqd->mq;
  749. struct message_header *mhdr = mq->next;
  750. void *next, *pnext;
  751. int half = -1;
  752. int lines = mhdr->lines;
  753. if (lines == 2)
  754. restore_present2(mhdr, MQS_EMPTY);
  755. mhdr->present = MQS_EMPTY;
  756. pnext = mq->next;
  757. next = pnext + GRU_CACHE_LINE_BYTES * lines;
  758. if (next == mq->limit) {
  759. next = mq->start;
  760. half = 1;
  761. } else if (pnext < mq->start2 && next >= mq->start2) {
  762. half = 0;
  763. }
  764. if (half >= 0)
  765. mq->hstatus[half] = 1;
  766. mq->next = next;
  767. }
  768. EXPORT_SYMBOL_GPL(gru_free_message);
  769. /*
  770. * Get next message from message queue. Return NULL if no message
  771. * present. User must call next_message() to move to next message.
  772. * rmq message queue
  773. */
  774. void *gru_get_next_message(struct gru_message_queue_desc *mqd)
  775. {
  776. struct message_queue *mq = mqd->mq;
  777. struct message_header *mhdr = mq->next;
  778. int present = mhdr->present;
  779. /* skip NOOP messages */
  780. while (present == MQS_NOOP) {
  781. gru_free_message(mqd, mhdr);
  782. mhdr = mq->next;
  783. present = mhdr->present;
  784. }
  785. /* Wait for both halves of 2 line messages */
  786. if (present == MQS_FULL && mhdr->lines == 2 &&
  787. get_present2(mhdr) == MQS_EMPTY)
  788. present = MQS_EMPTY;
  789. if (!present) {
  790. STAT(mesq_receive_none);
  791. return NULL;
  792. }
  793. if (mhdr->lines == 2)
  794. restore_present2(mhdr, mhdr->present2);
  795. STAT(mesq_receive);
  796. return mhdr;
  797. }
  798. EXPORT_SYMBOL_GPL(gru_get_next_message);
  799. /* ---------------------- GRU DATA COPY FUNCTIONS ---------------------------*/
  800. /*
  801. * Load a DW from a global GPA. The GPA can be a memory or MMR address.
  802. */
  803. int gru_read_gpa(unsigned long *value, unsigned long gpa)
  804. {
  805. void *cb;
  806. void *dsr;
  807. int ret, iaa;
  808. STAT(read_gpa);
  809. if (gru_get_cpu_resources(GRU_NUM_KERNEL_DSR_BYTES, &cb, &dsr))
  810. return MQE_BUG_NO_RESOURCES;
  811. iaa = gpa >> 62;
  812. gru_vload_phys(cb, gpa, gru_get_tri(dsr), iaa, IMA);
  813. ret = gru_wait(cb);
  814. if (ret == CBS_IDLE)
  815. *value = *(unsigned long *)dsr;
  816. gru_free_cpu_resources(cb, dsr);
  817. return ret;
  818. }
  819. EXPORT_SYMBOL_GPL(gru_read_gpa);
  820. /*
  821. * Copy a block of data using the GRU resources
  822. */
  823. int gru_copy_gpa(unsigned long dest_gpa, unsigned long src_gpa,
  824. unsigned int bytes)
  825. {
  826. void *cb;
  827. void *dsr;
  828. int ret;
  829. STAT(copy_gpa);
  830. if (gru_get_cpu_resources(GRU_NUM_KERNEL_DSR_BYTES, &cb, &dsr))
  831. return MQE_BUG_NO_RESOURCES;
  832. gru_bcopy(cb, src_gpa, dest_gpa, gru_get_tri(dsr),
  833. XTYPE_B, bytes, GRU_NUM_KERNEL_DSR_CL, IMA);
  834. ret = gru_wait(cb);
  835. gru_free_cpu_resources(cb, dsr);
  836. return ret;
  837. }
  838. EXPORT_SYMBOL_GPL(gru_copy_gpa);
  839. /* ------------------- KERNEL QUICKTESTS RUN AT STARTUP ----------------*/
  840. /* Temp - will delete after we gain confidence in the GRU */
  841. static int quicktest0(unsigned long arg)
  842. {
  843. unsigned long word0;
  844. unsigned long word1;
  845. void *cb;
  846. void *dsr;
  847. unsigned long *p;
  848. int ret = -EIO;
  849. if (gru_get_cpu_resources(GRU_CACHE_LINE_BYTES, &cb, &dsr))
  850. return MQE_BUG_NO_RESOURCES;
  851. p = dsr;
  852. word0 = MAGIC;
  853. word1 = 0;
  854. gru_vload(cb, uv_gpa(&word0), gru_get_tri(dsr), XTYPE_DW, 1, 1, IMA);
  855. if (gru_wait(cb) != CBS_IDLE) {
  856. printk(KERN_DEBUG "GRU:%d quicktest0: CBR failure 1\n", smp_processor_id());
  857. goto done;
  858. }
  859. if (*p != MAGIC) {
  860. printk(KERN_DEBUG "GRU:%d quicktest0 bad magic 0x%lx\n", smp_processor_id(), *p);
  861. goto done;
  862. }
  863. gru_vstore(cb, uv_gpa(&word1), gru_get_tri(dsr), XTYPE_DW, 1, 1, IMA);
  864. if (gru_wait(cb) != CBS_IDLE) {
  865. printk(KERN_DEBUG "GRU:%d quicktest0: CBR failure 2\n", smp_processor_id());
  866. goto done;
  867. }
  868. if (word0 != word1 || word1 != MAGIC) {
  869. printk(KERN_DEBUG
  870. "GRU:%d quicktest0 err: found 0x%lx, expected 0x%lx\n",
  871. smp_processor_id(), word1, MAGIC);
  872. goto done;
  873. }
  874. ret = 0;
  875. done:
  876. gru_free_cpu_resources(cb, dsr);
  877. return ret;
  878. }
  879. #define ALIGNUP(p, q) ((void *)(((unsigned long)(p) + (q) - 1) & ~(q - 1)))
  880. static int quicktest1(unsigned long arg)
  881. {
  882. struct gru_message_queue_desc mqd;
  883. void *p, *mq;
  884. int i, ret = -EIO;
  885. char mes[GRU_CACHE_LINE_BYTES], *m;
  886. /* Need 1K cacheline aligned that does not cross page boundary */
  887. p = kmalloc(4096, 0);
  888. if (p == NULL)
  889. return -ENOMEM;
  890. mq = ALIGNUP(p, 1024);
  891. memset(mes, 0xee, sizeof(mes));
  892. gru_create_message_queue(&mqd, mq, 8 * GRU_CACHE_LINE_BYTES, 0, 0, 0);
  893. for (i = 0; i < 6; i++) {
  894. mes[8] = i;
  895. do {
  896. ret = gru_send_message_gpa(&mqd, mes, sizeof(mes));
  897. } while (ret == MQE_CONGESTION);
  898. if (ret)
  899. break;
  900. }
  901. if (ret != MQE_QUEUE_FULL || i != 4) {
  902. printk(KERN_DEBUG "GRU:%d quicktest1: unexpect status %d, i %d\n",
  903. smp_processor_id(), ret, i);
  904. goto done;
  905. }
  906. for (i = 0; i < 6; i++) {
  907. m = gru_get_next_message(&mqd);
  908. if (!m || m[8] != i)
  909. break;
  910. gru_free_message(&mqd, m);
  911. }
  912. if (i != 4) {
  913. printk(KERN_DEBUG "GRU:%d quicktest2: bad message, i %d, m %p, m8 %d\n",
  914. smp_processor_id(), i, m, m ? m[8] : -1);
  915. goto done;
  916. }
  917. ret = 0;
  918. done:
  919. kfree(p);
  920. return ret;
  921. }
  922. static int quicktest2(unsigned long arg)
  923. {
  924. static DECLARE_COMPLETION(cmp);
  925. unsigned long han;
  926. int blade_id = 0;
  927. int numcb = 4;
  928. int ret = 0;
  929. unsigned long *buf;
  930. void *cb0, *cb;
  931. struct gru_control_block_status *gen;
  932. int i, k, istatus, bytes;
  933. bytes = numcb * 4 * 8;
  934. buf = kmalloc(bytes, GFP_KERNEL);
  935. if (!buf)
  936. return -ENOMEM;
  937. ret = -EBUSY;
  938. han = gru_reserve_async_resources(blade_id, numcb, 0, &cmp);
  939. if (!han)
  940. goto done;
  941. gru_lock_async_resource(han, &cb0, NULL);
  942. memset(buf, 0xee, bytes);
  943. for (i = 0; i < numcb; i++)
  944. gru_vset(cb0 + i * GRU_HANDLE_STRIDE, uv_gpa(&buf[i * 4]), 0,
  945. XTYPE_DW, 4, 1, IMA_INTERRUPT);
  946. ret = 0;
  947. k = numcb;
  948. do {
  949. gru_wait_async_cbr(han);
  950. for (i = 0; i < numcb; i++) {
  951. cb = cb0 + i * GRU_HANDLE_STRIDE;
  952. istatus = gru_check_status(cb);
  953. if (istatus != CBS_ACTIVE && istatus != CBS_CALL_OS)
  954. break;
  955. }
  956. if (i == numcb)
  957. continue;
  958. if (istatus != CBS_IDLE) {
  959. printk(KERN_DEBUG "GRU:%d quicktest2: cb %d, exception\n", smp_processor_id(), i);
  960. ret = -EFAULT;
  961. } else if (buf[4 * i] || buf[4 * i + 1] || buf[4 * i + 2] ||
  962. buf[4 * i + 3]) {
  963. printk(KERN_DEBUG "GRU:%d quicktest2:cb %d, buf 0x%lx, 0x%lx, 0x%lx, 0x%lx\n",
  964. smp_processor_id(), i, buf[4 * i], buf[4 * i + 1], buf[4 * i + 2], buf[4 * i + 3]);
  965. ret = -EIO;
  966. }
  967. k--;
  968. gen = cb;
  969. gen->istatus = CBS_CALL_OS; /* don't handle this CBR again */
  970. } while (k);
  971. BUG_ON(cmp.done);
  972. gru_unlock_async_resource(han);
  973. gru_release_async_resources(han);
  974. done:
  975. kfree(buf);
  976. return ret;
  977. }
  978. #define BUFSIZE 200
  979. static int quicktest3(unsigned long arg)
  980. {
  981. char buf1[BUFSIZE], buf2[BUFSIZE];
  982. int ret = 0;
  983. memset(buf2, 0, sizeof(buf2));
  984. memset(buf1, get_cycles() & 255, sizeof(buf1));
  985. gru_copy_gpa(uv_gpa(buf2), uv_gpa(buf1), BUFSIZE);
  986. if (memcmp(buf1, buf2, BUFSIZE)) {
  987. printk(KERN_DEBUG "GRU:%d quicktest3 error\n", smp_processor_id());
  988. ret = -EIO;
  989. }
  990. return ret;
  991. }
  992. /*
  993. * Debugging only. User hook for various kernel tests
  994. * of driver & gru.
  995. */
  996. int gru_ktest(unsigned long arg)
  997. {
  998. int ret = -EINVAL;
  999. switch (arg & 0xff) {
  1000. case 0:
  1001. ret = quicktest0(arg);
  1002. break;
  1003. case 1:
  1004. ret = quicktest1(arg);
  1005. break;
  1006. case 2:
  1007. ret = quicktest2(arg);
  1008. break;
  1009. case 3:
  1010. ret = quicktest3(arg);
  1011. break;
  1012. case 99:
  1013. ret = gru_free_kernel_contexts();
  1014. break;
  1015. }
  1016. return ret;
  1017. }
  1018. int gru_kservices_init(void)
  1019. {
  1020. return 0;
  1021. }
  1022. void gru_kservices_exit(void)
  1023. {
  1024. if (gru_free_kernel_contexts())
  1025. BUG();
  1026. }