guest.c 27 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Copyright 2015 IBM Corp.
  4. */
  5. #include <linux/spinlock.h>
  6. #include <linux/uaccess.h>
  7. #include <linux/delay.h>
  8. #include "cxl.h"
  9. #include "hcalls.h"
  10. #include "trace.h"
  11. #define CXL_ERROR_DETECTED_EVENT 1
  12. #define CXL_SLOT_RESET_EVENT 2
  13. #define CXL_RESUME_EVENT 3
  14. static void pci_error_handlers(struct cxl_afu *afu,
  15. int bus_error_event,
  16. pci_channel_state_t state)
  17. {
  18. struct pci_dev *afu_dev;
  19. if (afu->phb == NULL)
  20. return;
  21. list_for_each_entry(afu_dev, &afu->phb->bus->devices, bus_list) {
  22. if (!afu_dev->driver)
  23. continue;
  24. switch (bus_error_event) {
  25. case CXL_ERROR_DETECTED_EVENT:
  26. afu_dev->error_state = state;
  27. if (afu_dev->driver->err_handler &&
  28. afu_dev->driver->err_handler->error_detected)
  29. afu_dev->driver->err_handler->error_detected(afu_dev, state);
  30. break;
  31. case CXL_SLOT_RESET_EVENT:
  32. afu_dev->error_state = state;
  33. if (afu_dev->driver->err_handler &&
  34. afu_dev->driver->err_handler->slot_reset)
  35. afu_dev->driver->err_handler->slot_reset(afu_dev);
  36. break;
  37. case CXL_RESUME_EVENT:
  38. if (afu_dev->driver->err_handler &&
  39. afu_dev->driver->err_handler->resume)
  40. afu_dev->driver->err_handler->resume(afu_dev);
  41. break;
  42. }
  43. }
  44. }
  45. static irqreturn_t guest_handle_psl_slice_error(struct cxl_context *ctx, u64 dsisr,
  46. u64 errstat)
  47. {
  48. pr_devel("in %s\n", __func__);
  49. dev_crit(&ctx->afu->dev, "PSL ERROR STATUS: 0x%.16llx\n", errstat);
  50. return cxl_ops->ack_irq(ctx, 0, errstat);
  51. }
  52. static ssize_t guest_collect_vpd(struct cxl *adapter, struct cxl_afu *afu,
  53. void *buf, size_t len)
  54. {
  55. unsigned int entries, mod;
  56. unsigned long **vpd_buf = NULL;
  57. struct sg_list *le;
  58. int rc = 0, i, tocopy;
  59. u64 out = 0;
  60. if (buf == NULL)
  61. return -EINVAL;
  62. /* number of entries in the list */
  63. entries = len / SG_BUFFER_SIZE;
  64. mod = len % SG_BUFFER_SIZE;
  65. if (mod)
  66. entries++;
  67. if (entries > SG_MAX_ENTRIES) {
  68. entries = SG_MAX_ENTRIES;
  69. len = SG_MAX_ENTRIES * SG_BUFFER_SIZE;
  70. mod = 0;
  71. }
  72. vpd_buf = kcalloc(entries, sizeof(unsigned long *), GFP_KERNEL);
  73. if (!vpd_buf)
  74. return -ENOMEM;
  75. le = (struct sg_list *)get_zeroed_page(GFP_KERNEL);
  76. if (!le) {
  77. rc = -ENOMEM;
  78. goto err1;
  79. }
  80. for (i = 0; i < entries; i++) {
  81. vpd_buf[i] = (unsigned long *)get_zeroed_page(GFP_KERNEL);
  82. if (!vpd_buf[i]) {
  83. rc = -ENOMEM;
  84. goto err2;
  85. }
  86. le[i].phys_addr = cpu_to_be64(virt_to_phys(vpd_buf[i]));
  87. le[i].len = cpu_to_be64(SG_BUFFER_SIZE);
  88. if ((i == (entries - 1)) && mod)
  89. le[i].len = cpu_to_be64(mod);
  90. }
  91. if (adapter)
  92. rc = cxl_h_collect_vpd_adapter(adapter->guest->handle,
  93. virt_to_phys(le), entries, &out);
  94. else
  95. rc = cxl_h_collect_vpd(afu->guest->handle, 0,
  96. virt_to_phys(le), entries, &out);
  97. pr_devel("length of available (entries: %i), vpd: %#llx\n",
  98. entries, out);
  99. if (!rc) {
  100. /*
  101. * hcall returns in 'out' the size of available VPDs.
  102. * It fills the buffer with as much data as possible.
  103. */
  104. if (out < len)
  105. len = out;
  106. rc = len;
  107. if (out) {
  108. for (i = 0; i < entries; i++) {
  109. if (len < SG_BUFFER_SIZE)
  110. tocopy = len;
  111. else
  112. tocopy = SG_BUFFER_SIZE;
  113. memcpy(buf, vpd_buf[i], tocopy);
  114. buf += tocopy;
  115. len -= tocopy;
  116. }
  117. }
  118. }
  119. err2:
  120. for (i = 0; i < entries; i++) {
  121. if (vpd_buf[i])
  122. free_page((unsigned long) vpd_buf[i]);
  123. }
  124. free_page((unsigned long) le);
  125. err1:
  126. kfree(vpd_buf);
  127. return rc;
  128. }
  129. static int guest_get_irq_info(struct cxl_context *ctx, struct cxl_irq_info *info)
  130. {
  131. return cxl_h_collect_int_info(ctx->afu->guest->handle, ctx->process_token, info);
  132. }
  133. static irqreturn_t guest_psl_irq(int irq, void *data)
  134. {
  135. struct cxl_context *ctx = data;
  136. struct cxl_irq_info irq_info;
  137. int rc;
  138. pr_devel("%d: received PSL interrupt %i\n", ctx->pe, irq);
  139. rc = guest_get_irq_info(ctx, &irq_info);
  140. if (rc) {
  141. WARN(1, "Unable to get IRQ info: %i\n", rc);
  142. return IRQ_HANDLED;
  143. }
  144. rc = cxl_irq_psl8(irq, ctx, &irq_info);
  145. return rc;
  146. }
  147. static int afu_read_error_state(struct cxl_afu *afu, int *state_out)
  148. {
  149. u64 state;
  150. int rc = 0;
  151. if (!afu)
  152. return -EIO;
  153. rc = cxl_h_read_error_state(afu->guest->handle, &state);
  154. if (!rc) {
  155. WARN_ON(state != H_STATE_NORMAL &&
  156. state != H_STATE_DISABLE &&
  157. state != H_STATE_TEMP_UNAVAILABLE &&
  158. state != H_STATE_PERM_UNAVAILABLE);
  159. *state_out = state & 0xffffffff;
  160. }
  161. return rc;
  162. }
  163. static irqreturn_t guest_slice_irq_err(int irq, void *data)
  164. {
  165. struct cxl_afu *afu = data;
  166. int rc;
  167. u64 serr, afu_error, dsisr;
  168. rc = cxl_h_get_fn_error_interrupt(afu->guest->handle, &serr);
  169. if (rc) {
  170. dev_crit(&afu->dev, "Couldn't read PSL_SERR_An: %d\n", rc);
  171. return IRQ_HANDLED;
  172. }
  173. afu_error = cxl_p2n_read(afu, CXL_AFU_ERR_An);
  174. dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
  175. cxl_afu_decode_psl_serr(afu, serr);
  176. dev_crit(&afu->dev, "AFU_ERR_An: 0x%.16llx\n", afu_error);
  177. dev_crit(&afu->dev, "PSL_DSISR_An: 0x%.16llx\n", dsisr);
  178. rc = cxl_h_ack_fn_error_interrupt(afu->guest->handle, serr);
  179. if (rc)
  180. dev_crit(&afu->dev, "Couldn't ack slice error interrupt: %d\n",
  181. rc);
  182. return IRQ_HANDLED;
  183. }
  184. static int irq_alloc_range(struct cxl *adapter, int len, int *irq)
  185. {
  186. int i, n;
  187. struct irq_avail *cur;
  188. for (i = 0; i < adapter->guest->irq_nranges; i++) {
  189. cur = &adapter->guest->irq_avail[i];
  190. n = bitmap_find_next_zero_area(cur->bitmap, cur->range,
  191. 0, len, 0);
  192. if (n < cur->range) {
  193. bitmap_set(cur->bitmap, n, len);
  194. *irq = cur->offset + n;
  195. pr_devel("guest: allocate IRQs %#x->%#x\n",
  196. *irq, *irq + len - 1);
  197. return 0;
  198. }
  199. }
  200. return -ENOSPC;
  201. }
  202. static int irq_free_range(struct cxl *adapter, int irq, int len)
  203. {
  204. int i, n;
  205. struct irq_avail *cur;
  206. if (len == 0)
  207. return -ENOENT;
  208. for (i = 0; i < adapter->guest->irq_nranges; i++) {
  209. cur = &adapter->guest->irq_avail[i];
  210. if (irq >= cur->offset &&
  211. (irq + len) <= (cur->offset + cur->range)) {
  212. n = irq - cur->offset;
  213. bitmap_clear(cur->bitmap, n, len);
  214. pr_devel("guest: release IRQs %#x->%#x\n",
  215. irq, irq + len - 1);
  216. return 0;
  217. }
  218. }
  219. return -ENOENT;
  220. }
  221. static int guest_reset(struct cxl *adapter)
  222. {
  223. struct cxl_afu *afu = NULL;
  224. int i, rc;
  225. pr_devel("Adapter reset request\n");
  226. spin_lock(&adapter->afu_list_lock);
  227. for (i = 0; i < adapter->slices; i++) {
  228. if ((afu = adapter->afu[i])) {
  229. pci_error_handlers(afu, CXL_ERROR_DETECTED_EVENT,
  230. pci_channel_io_frozen);
  231. cxl_context_detach_all(afu);
  232. }
  233. }
  234. rc = cxl_h_reset_adapter(adapter->guest->handle);
  235. for (i = 0; i < adapter->slices; i++) {
  236. if (!rc && (afu = adapter->afu[i])) {
  237. pci_error_handlers(afu, CXL_SLOT_RESET_EVENT,
  238. pci_channel_io_normal);
  239. pci_error_handlers(afu, CXL_RESUME_EVENT, 0);
  240. }
  241. }
  242. spin_unlock(&adapter->afu_list_lock);
  243. return rc;
  244. }
  245. static int guest_alloc_one_irq(struct cxl *adapter)
  246. {
  247. int irq;
  248. spin_lock(&adapter->guest->irq_alloc_lock);
  249. if (irq_alloc_range(adapter, 1, &irq))
  250. irq = -ENOSPC;
  251. spin_unlock(&adapter->guest->irq_alloc_lock);
  252. return irq;
  253. }
  254. static void guest_release_one_irq(struct cxl *adapter, int irq)
  255. {
  256. spin_lock(&adapter->guest->irq_alloc_lock);
  257. irq_free_range(adapter, irq, 1);
  258. spin_unlock(&adapter->guest->irq_alloc_lock);
  259. }
  260. static int guest_alloc_irq_ranges(struct cxl_irq_ranges *irqs,
  261. struct cxl *adapter, unsigned int num)
  262. {
  263. int i, try, irq;
  264. memset(irqs, 0, sizeof(struct cxl_irq_ranges));
  265. spin_lock(&adapter->guest->irq_alloc_lock);
  266. for (i = 0; i < CXL_IRQ_RANGES && num; i++) {
  267. try = num;
  268. while (try) {
  269. if (irq_alloc_range(adapter, try, &irq) == 0)
  270. break;
  271. try /= 2;
  272. }
  273. if (!try)
  274. goto error;
  275. irqs->offset[i] = irq;
  276. irqs->range[i] = try;
  277. num -= try;
  278. }
  279. if (num)
  280. goto error;
  281. spin_unlock(&adapter->guest->irq_alloc_lock);
  282. return 0;
  283. error:
  284. for (i = 0; i < CXL_IRQ_RANGES; i++)
  285. irq_free_range(adapter, irqs->offset[i], irqs->range[i]);
  286. spin_unlock(&adapter->guest->irq_alloc_lock);
  287. return -ENOSPC;
  288. }
  289. static void guest_release_irq_ranges(struct cxl_irq_ranges *irqs,
  290. struct cxl *adapter)
  291. {
  292. int i;
  293. spin_lock(&adapter->guest->irq_alloc_lock);
  294. for (i = 0; i < CXL_IRQ_RANGES; i++)
  295. irq_free_range(adapter, irqs->offset[i], irqs->range[i]);
  296. spin_unlock(&adapter->guest->irq_alloc_lock);
  297. }
  298. static int guest_register_serr_irq(struct cxl_afu *afu)
  299. {
  300. afu->err_irq_name = kasprintf(GFP_KERNEL, "cxl-%s-err",
  301. dev_name(&afu->dev));
  302. if (!afu->err_irq_name)
  303. return -ENOMEM;
  304. if (!(afu->serr_virq = cxl_map_irq(afu->adapter, afu->serr_hwirq,
  305. guest_slice_irq_err, afu, afu->err_irq_name))) {
  306. kfree(afu->err_irq_name);
  307. afu->err_irq_name = NULL;
  308. return -ENOMEM;
  309. }
  310. return 0;
  311. }
  312. static void guest_release_serr_irq(struct cxl_afu *afu)
  313. {
  314. cxl_unmap_irq(afu->serr_virq, afu);
  315. cxl_ops->release_one_irq(afu->adapter, afu->serr_hwirq);
  316. kfree(afu->err_irq_name);
  317. }
  318. static int guest_ack_irq(struct cxl_context *ctx, u64 tfc, u64 psl_reset_mask)
  319. {
  320. return cxl_h_control_faults(ctx->afu->guest->handle, ctx->process_token,
  321. tfc >> 32, (psl_reset_mask != 0));
  322. }
  323. static void disable_afu_irqs(struct cxl_context *ctx)
  324. {
  325. irq_hw_number_t hwirq;
  326. unsigned int virq;
  327. int r, i;
  328. pr_devel("Disabling AFU(%d) interrupts\n", ctx->afu->slice);
  329. for (r = 0; r < CXL_IRQ_RANGES; r++) {
  330. hwirq = ctx->irqs.offset[r];
  331. for (i = 0; i < ctx->irqs.range[r]; hwirq++, i++) {
  332. virq = irq_find_mapping(NULL, hwirq);
  333. disable_irq(virq);
  334. }
  335. }
  336. }
  337. static void enable_afu_irqs(struct cxl_context *ctx)
  338. {
  339. irq_hw_number_t hwirq;
  340. unsigned int virq;
  341. int r, i;
  342. pr_devel("Enabling AFU(%d) interrupts\n", ctx->afu->slice);
  343. for (r = 0; r < CXL_IRQ_RANGES; r++) {
  344. hwirq = ctx->irqs.offset[r];
  345. for (i = 0; i < ctx->irqs.range[r]; hwirq++, i++) {
  346. virq = irq_find_mapping(NULL, hwirq);
  347. enable_irq(virq);
  348. }
  349. }
  350. }
  351. static int _guest_afu_cr_readXX(int sz, struct cxl_afu *afu, int cr_idx,
  352. u64 offset, u64 *val)
  353. {
  354. unsigned long cr;
  355. char c;
  356. int rc = 0;
  357. if (afu->crs_len < sz)
  358. return -ENOENT;
  359. if (unlikely(offset >= afu->crs_len))
  360. return -ERANGE;
  361. cr = get_zeroed_page(GFP_KERNEL);
  362. if (!cr)
  363. return -ENOMEM;
  364. rc = cxl_h_get_config(afu->guest->handle, cr_idx, offset,
  365. virt_to_phys((void *)cr), sz);
  366. if (rc)
  367. goto err;
  368. switch (sz) {
  369. case 1:
  370. c = *((char *) cr);
  371. *val = c;
  372. break;
  373. case 2:
  374. *val = in_le16((u16 *)cr);
  375. break;
  376. case 4:
  377. *val = in_le32((unsigned *)cr);
  378. break;
  379. case 8:
  380. *val = in_le64((u64 *)cr);
  381. break;
  382. default:
  383. WARN_ON(1);
  384. }
  385. err:
  386. free_page(cr);
  387. return rc;
  388. }
  389. static int guest_afu_cr_read32(struct cxl_afu *afu, int cr_idx, u64 offset,
  390. u32 *out)
  391. {
  392. int rc;
  393. u64 val;
  394. rc = _guest_afu_cr_readXX(4, afu, cr_idx, offset, &val);
  395. if (!rc)
  396. *out = (u32) val;
  397. return rc;
  398. }
  399. static int guest_afu_cr_read16(struct cxl_afu *afu, int cr_idx, u64 offset,
  400. u16 *out)
  401. {
  402. int rc;
  403. u64 val;
  404. rc = _guest_afu_cr_readXX(2, afu, cr_idx, offset, &val);
  405. if (!rc)
  406. *out = (u16) val;
  407. return rc;
  408. }
  409. static int guest_afu_cr_read8(struct cxl_afu *afu, int cr_idx, u64 offset,
  410. u8 *out)
  411. {
  412. int rc;
  413. u64 val;
  414. rc = _guest_afu_cr_readXX(1, afu, cr_idx, offset, &val);
  415. if (!rc)
  416. *out = (u8) val;
  417. return rc;
  418. }
  419. static int guest_afu_cr_read64(struct cxl_afu *afu, int cr_idx, u64 offset,
  420. u64 *out)
  421. {
  422. return _guest_afu_cr_readXX(8, afu, cr_idx, offset, out);
  423. }
  424. static int guest_afu_cr_write32(struct cxl_afu *afu, int cr, u64 off, u32 in)
  425. {
  426. /* config record is not writable from guest */
  427. return -EPERM;
  428. }
  429. static int guest_afu_cr_write16(struct cxl_afu *afu, int cr, u64 off, u16 in)
  430. {
  431. /* config record is not writable from guest */
  432. return -EPERM;
  433. }
  434. static int guest_afu_cr_write8(struct cxl_afu *afu, int cr, u64 off, u8 in)
  435. {
  436. /* config record is not writable from guest */
  437. return -EPERM;
  438. }
  439. static int attach_afu_directed(struct cxl_context *ctx, u64 wed, u64 amr)
  440. {
  441. struct cxl_process_element_hcall *elem;
  442. struct cxl *adapter = ctx->afu->adapter;
  443. const struct cred *cred;
  444. u32 pid, idx;
  445. int rc, r, i;
  446. u64 mmio_addr, mmio_size;
  447. __be64 flags = 0;
  448. /* Must be 8 byte aligned and cannot cross a 4096 byte boundary */
  449. if (!(elem = (struct cxl_process_element_hcall *)
  450. get_zeroed_page(GFP_KERNEL)))
  451. return -ENOMEM;
  452. elem->version = cpu_to_be64(CXL_PROCESS_ELEMENT_VERSION);
  453. if (ctx->kernel) {
  454. pid = 0;
  455. flags |= CXL_PE_TRANSLATION_ENABLED;
  456. flags |= CXL_PE_PRIVILEGED_PROCESS;
  457. if (mfmsr() & MSR_SF)
  458. flags |= CXL_PE_64_BIT;
  459. } else {
  460. pid = current->pid;
  461. flags |= CXL_PE_PROBLEM_STATE;
  462. flags |= CXL_PE_TRANSLATION_ENABLED;
  463. if (!test_tsk_thread_flag(current, TIF_32BIT))
  464. flags |= CXL_PE_64_BIT;
  465. cred = get_current_cred();
  466. if (uid_eq(cred->euid, GLOBAL_ROOT_UID))
  467. flags |= CXL_PE_PRIVILEGED_PROCESS;
  468. put_cred(cred);
  469. }
  470. elem->flags = cpu_to_be64(flags);
  471. elem->common.tid = cpu_to_be32(0); /* Unused */
  472. elem->common.pid = cpu_to_be32(pid);
  473. elem->common.csrp = cpu_to_be64(0); /* disable */
  474. elem->common.u.psl8.aurp0 = cpu_to_be64(0); /* disable */
  475. elem->common.u.psl8.aurp1 = cpu_to_be64(0); /* disable */
  476. cxl_prefault(ctx, wed);
  477. elem->common.u.psl8.sstp0 = cpu_to_be64(ctx->sstp0);
  478. elem->common.u.psl8.sstp1 = cpu_to_be64(ctx->sstp1);
  479. /*
  480. * Ensure we have at least one interrupt allocated to take faults for
  481. * kernel contexts that may not have allocated any AFU IRQs at all:
  482. */
  483. if (ctx->irqs.range[0] == 0) {
  484. rc = afu_register_irqs(ctx, 0);
  485. if (rc)
  486. goto out_free;
  487. }
  488. for (r = 0; r < CXL_IRQ_RANGES; r++) {
  489. for (i = 0; i < ctx->irqs.range[r]; i++) {
  490. if (r == 0 && i == 0) {
  491. elem->pslVirtualIsn = cpu_to_be32(ctx->irqs.offset[0]);
  492. } else {
  493. idx = ctx->irqs.offset[r] + i - adapter->guest->irq_base_offset;
  494. elem->applicationVirtualIsnBitmap[idx / 8] |= 0x80 >> (idx % 8);
  495. }
  496. }
  497. }
  498. elem->common.amr = cpu_to_be64(amr);
  499. elem->common.wed = cpu_to_be64(wed);
  500. disable_afu_irqs(ctx);
  501. rc = cxl_h_attach_process(ctx->afu->guest->handle, elem,
  502. &ctx->process_token, &mmio_addr, &mmio_size);
  503. if (rc == H_SUCCESS) {
  504. if (ctx->master || !ctx->afu->pp_psa) {
  505. ctx->psn_phys = ctx->afu->psn_phys;
  506. ctx->psn_size = ctx->afu->adapter->ps_size;
  507. } else {
  508. ctx->psn_phys = mmio_addr;
  509. ctx->psn_size = mmio_size;
  510. }
  511. if (ctx->afu->pp_psa && mmio_size &&
  512. ctx->afu->pp_size == 0) {
  513. /*
  514. * There's no property in the device tree to read the
  515. * pp_size. We only find out at the 1st attach.
  516. * Compared to bare-metal, it is too late and we
  517. * should really lock here. However, on powerVM,
  518. * pp_size is really only used to display in /sys.
  519. * Being discussed with pHyp for their next release.
  520. */
  521. ctx->afu->pp_size = mmio_size;
  522. }
  523. /* from PAPR: process element is bytes 4-7 of process token */
  524. ctx->external_pe = ctx->process_token & 0xFFFFFFFF;
  525. pr_devel("CXL pe=%i is known as %i for pHyp, mmio_size=%#llx",
  526. ctx->pe, ctx->external_pe, ctx->psn_size);
  527. ctx->pe_inserted = true;
  528. enable_afu_irqs(ctx);
  529. }
  530. out_free:
  531. free_page((u64)elem);
  532. return rc;
  533. }
  534. static int guest_attach_process(struct cxl_context *ctx, bool kernel, u64 wed, u64 amr)
  535. {
  536. pr_devel("in %s\n", __func__);
  537. ctx->kernel = kernel;
  538. if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
  539. return attach_afu_directed(ctx, wed, amr);
  540. /* dedicated mode not supported on FW840 */
  541. return -EINVAL;
  542. }
  543. static int detach_afu_directed(struct cxl_context *ctx)
  544. {
  545. if (!ctx->pe_inserted)
  546. return 0;
  547. if (cxl_h_detach_process(ctx->afu->guest->handle, ctx->process_token))
  548. return -1;
  549. return 0;
  550. }
  551. static int guest_detach_process(struct cxl_context *ctx)
  552. {
  553. pr_devel("in %s\n", __func__);
  554. trace_cxl_detach(ctx);
  555. if (!cxl_ops->link_ok(ctx->afu->adapter, ctx->afu))
  556. return -EIO;
  557. if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
  558. return detach_afu_directed(ctx);
  559. return -EINVAL;
  560. }
  561. static void guest_release_afu(struct device *dev)
  562. {
  563. struct cxl_afu *afu = to_cxl_afu(dev);
  564. pr_devel("%s\n", __func__);
  565. idr_destroy(&afu->contexts_idr);
  566. kfree(afu->guest);
  567. kfree(afu);
  568. }
  569. ssize_t cxl_guest_read_afu_vpd(struct cxl_afu *afu, void *buf, size_t len)
  570. {
  571. return guest_collect_vpd(NULL, afu, buf, len);
  572. }
  573. #define ERR_BUFF_MAX_COPY_SIZE PAGE_SIZE
  574. static ssize_t guest_afu_read_err_buffer(struct cxl_afu *afu, char *buf,
  575. loff_t off, size_t count)
  576. {
  577. void *tbuf = NULL;
  578. int rc = 0;
  579. tbuf = (void *) get_zeroed_page(GFP_KERNEL);
  580. if (!tbuf)
  581. return -ENOMEM;
  582. rc = cxl_h_get_afu_err(afu->guest->handle,
  583. off & 0x7,
  584. virt_to_phys(tbuf),
  585. count);
  586. if (rc)
  587. goto err;
  588. if (count > ERR_BUFF_MAX_COPY_SIZE)
  589. count = ERR_BUFF_MAX_COPY_SIZE - (off & 0x7);
  590. memcpy(buf, tbuf, count);
  591. err:
  592. free_page((u64)tbuf);
  593. return rc;
  594. }
  595. static int guest_afu_check_and_enable(struct cxl_afu *afu)
  596. {
  597. return 0;
  598. }
  599. static bool guest_support_attributes(const char *attr_name,
  600. enum cxl_attrs type)
  601. {
  602. switch (type) {
  603. case CXL_ADAPTER_ATTRS:
  604. if ((strcmp(attr_name, "base_image") == 0) ||
  605. (strcmp(attr_name, "load_image_on_perst") == 0) ||
  606. (strcmp(attr_name, "perst_reloads_same_image") == 0) ||
  607. (strcmp(attr_name, "image_loaded") == 0))
  608. return false;
  609. break;
  610. case CXL_AFU_MASTER_ATTRS:
  611. if ((strcmp(attr_name, "pp_mmio_off") == 0))
  612. return false;
  613. break;
  614. case CXL_AFU_ATTRS:
  615. break;
  616. default:
  617. break;
  618. }
  619. return true;
  620. }
  621. static int activate_afu_directed(struct cxl_afu *afu)
  622. {
  623. int rc;
  624. dev_info(&afu->dev, "Activating AFU(%d) directed mode\n", afu->slice);
  625. afu->current_mode = CXL_MODE_DIRECTED;
  626. afu->num_procs = afu->max_procs_virtualised;
  627. if ((rc = cxl_chardev_m_afu_add(afu)))
  628. return rc;
  629. if ((rc = cxl_sysfs_afu_m_add(afu)))
  630. goto err;
  631. if ((rc = cxl_chardev_s_afu_add(afu)))
  632. goto err1;
  633. return 0;
  634. err1:
  635. cxl_sysfs_afu_m_remove(afu);
  636. err:
  637. cxl_chardev_afu_remove(afu);
  638. return rc;
  639. }
  640. static int guest_afu_activate_mode(struct cxl_afu *afu, int mode)
  641. {
  642. if (!mode)
  643. return 0;
  644. if (!(mode & afu->modes_supported))
  645. return -EINVAL;
  646. if (mode == CXL_MODE_DIRECTED)
  647. return activate_afu_directed(afu);
  648. if (mode == CXL_MODE_DEDICATED)
  649. dev_err(&afu->dev, "Dedicated mode not supported\n");
  650. return -EINVAL;
  651. }
  652. static int deactivate_afu_directed(struct cxl_afu *afu)
  653. {
  654. dev_info(&afu->dev, "Deactivating AFU(%d) directed mode\n", afu->slice);
  655. afu->current_mode = 0;
  656. afu->num_procs = 0;
  657. cxl_sysfs_afu_m_remove(afu);
  658. cxl_chardev_afu_remove(afu);
  659. cxl_ops->afu_reset(afu);
  660. return 0;
  661. }
  662. static int guest_afu_deactivate_mode(struct cxl_afu *afu, int mode)
  663. {
  664. if (!mode)
  665. return 0;
  666. if (!(mode & afu->modes_supported))
  667. return -EINVAL;
  668. if (mode == CXL_MODE_DIRECTED)
  669. return deactivate_afu_directed(afu);
  670. return 0;
  671. }
  672. static int guest_afu_reset(struct cxl_afu *afu)
  673. {
  674. pr_devel("AFU(%d) reset request\n", afu->slice);
  675. return cxl_h_reset_afu(afu->guest->handle);
  676. }
  677. static int guest_map_slice_regs(struct cxl_afu *afu)
  678. {
  679. if (!(afu->p2n_mmio = ioremap(afu->guest->p2n_phys, afu->guest->p2n_size))) {
  680. dev_err(&afu->dev, "Error mapping AFU(%d) MMIO regions\n",
  681. afu->slice);
  682. return -ENOMEM;
  683. }
  684. return 0;
  685. }
  686. static void guest_unmap_slice_regs(struct cxl_afu *afu)
  687. {
  688. if (afu->p2n_mmio)
  689. iounmap(afu->p2n_mmio);
  690. }
  691. static int afu_update_state(struct cxl_afu *afu)
  692. {
  693. int rc, cur_state;
  694. rc = afu_read_error_state(afu, &cur_state);
  695. if (rc)
  696. return rc;
  697. if (afu->guest->previous_state == cur_state)
  698. return 0;
  699. pr_devel("AFU(%d) update state to %#x\n", afu->slice, cur_state);
  700. switch (cur_state) {
  701. case H_STATE_NORMAL:
  702. afu->guest->previous_state = cur_state;
  703. break;
  704. case H_STATE_DISABLE:
  705. pci_error_handlers(afu, CXL_ERROR_DETECTED_EVENT,
  706. pci_channel_io_frozen);
  707. cxl_context_detach_all(afu);
  708. if ((rc = cxl_ops->afu_reset(afu)))
  709. pr_devel("reset hcall failed %d\n", rc);
  710. rc = afu_read_error_state(afu, &cur_state);
  711. if (!rc && cur_state == H_STATE_NORMAL) {
  712. pci_error_handlers(afu, CXL_SLOT_RESET_EVENT,
  713. pci_channel_io_normal);
  714. pci_error_handlers(afu, CXL_RESUME_EVENT, 0);
  715. }
  716. afu->guest->previous_state = 0;
  717. break;
  718. case H_STATE_TEMP_UNAVAILABLE:
  719. afu->guest->previous_state = cur_state;
  720. break;
  721. case H_STATE_PERM_UNAVAILABLE:
  722. dev_err(&afu->dev, "AFU is in permanent error state\n");
  723. pci_error_handlers(afu, CXL_ERROR_DETECTED_EVENT,
  724. pci_channel_io_perm_failure);
  725. afu->guest->previous_state = cur_state;
  726. break;
  727. default:
  728. pr_err("Unexpected AFU(%d) error state: %#x\n",
  729. afu->slice, cur_state);
  730. return -EINVAL;
  731. }
  732. return rc;
  733. }
  734. static void afu_handle_errstate(struct work_struct *work)
  735. {
  736. struct cxl_afu_guest *afu_guest =
  737. container_of(to_delayed_work(work), struct cxl_afu_guest, work_err);
  738. if (!afu_update_state(afu_guest->parent) &&
  739. afu_guest->previous_state == H_STATE_PERM_UNAVAILABLE)
  740. return;
  741. if (afu_guest->handle_err)
  742. schedule_delayed_work(&afu_guest->work_err,
  743. msecs_to_jiffies(3000));
  744. }
  745. static bool guest_link_ok(struct cxl *cxl, struct cxl_afu *afu)
  746. {
  747. int state;
  748. if (afu && (!afu_read_error_state(afu, &state))) {
  749. if (state == H_STATE_NORMAL)
  750. return true;
  751. }
  752. return false;
  753. }
  754. static int afu_properties_look_ok(struct cxl_afu *afu)
  755. {
  756. if (afu->pp_irqs < 0) {
  757. dev_err(&afu->dev, "Unexpected per-process minimum interrupt value\n");
  758. return -EINVAL;
  759. }
  760. if (afu->max_procs_virtualised < 1) {
  761. dev_err(&afu->dev, "Unexpected max number of processes virtualised value\n");
  762. return -EINVAL;
  763. }
  764. return 0;
  765. }
  766. int cxl_guest_init_afu(struct cxl *adapter, int slice, struct device_node *afu_np)
  767. {
  768. struct cxl_afu *afu;
  769. bool free = true;
  770. int rc;
  771. pr_devel("in %s - AFU(%d)\n", __func__, slice);
  772. if (!(afu = cxl_alloc_afu(adapter, slice)))
  773. return -ENOMEM;
  774. if (!(afu->guest = kzalloc(sizeof(struct cxl_afu_guest), GFP_KERNEL))) {
  775. kfree(afu);
  776. return -ENOMEM;
  777. }
  778. if ((rc = dev_set_name(&afu->dev, "afu%i.%i",
  779. adapter->adapter_num,
  780. slice)))
  781. goto err1;
  782. adapter->slices++;
  783. if ((rc = cxl_of_read_afu_handle(afu, afu_np)))
  784. goto err1;
  785. if ((rc = cxl_ops->afu_reset(afu)))
  786. goto err1;
  787. if ((rc = cxl_of_read_afu_properties(afu, afu_np)))
  788. goto err1;
  789. if ((rc = afu_properties_look_ok(afu)))
  790. goto err1;
  791. if ((rc = guest_map_slice_regs(afu)))
  792. goto err1;
  793. if ((rc = guest_register_serr_irq(afu)))
  794. goto err2;
  795. /*
  796. * After we call this function we must not free the afu directly, even
  797. * if it returns an error!
  798. */
  799. if ((rc = cxl_register_afu(afu)))
  800. goto err_put1;
  801. if ((rc = cxl_sysfs_afu_add(afu)))
  802. goto err_put1;
  803. /*
  804. * pHyp doesn't expose the programming models supported by the
  805. * AFU. pHyp currently only supports directed mode. If it adds
  806. * dedicated mode later, this version of cxl has no way to
  807. * detect it. So we'll initialize the driver, but the first
  808. * attach will fail.
  809. * Being discussed with pHyp to do better (likely new property)
  810. */
  811. if (afu->max_procs_virtualised == 1)
  812. afu->modes_supported = CXL_MODE_DEDICATED;
  813. else
  814. afu->modes_supported = CXL_MODE_DIRECTED;
  815. if ((rc = cxl_afu_select_best_mode(afu)))
  816. goto err_put2;
  817. adapter->afu[afu->slice] = afu;
  818. afu->enabled = true;
  819. /*
  820. * wake up the cpu periodically to check the state
  821. * of the AFU using "afu" stored in the guest structure.
  822. */
  823. afu->guest->parent = afu;
  824. afu->guest->handle_err = true;
  825. INIT_DELAYED_WORK(&afu->guest->work_err, afu_handle_errstate);
  826. schedule_delayed_work(&afu->guest->work_err, msecs_to_jiffies(1000));
  827. if ((rc = cxl_pci_vphb_add(afu)))
  828. dev_info(&afu->dev, "Can't register vPHB\n");
  829. return 0;
  830. err_put2:
  831. cxl_sysfs_afu_remove(afu);
  832. err_put1:
  833. device_unregister(&afu->dev);
  834. free = false;
  835. guest_release_serr_irq(afu);
  836. err2:
  837. guest_unmap_slice_regs(afu);
  838. err1:
  839. if (free) {
  840. kfree(afu->guest);
  841. kfree(afu);
  842. }
  843. return rc;
  844. }
  845. void cxl_guest_remove_afu(struct cxl_afu *afu)
  846. {
  847. if (!afu)
  848. return;
  849. /* flush and stop pending job */
  850. afu->guest->handle_err = false;
  851. flush_delayed_work(&afu->guest->work_err);
  852. cxl_pci_vphb_remove(afu);
  853. cxl_sysfs_afu_remove(afu);
  854. spin_lock(&afu->adapter->afu_list_lock);
  855. afu->adapter->afu[afu->slice] = NULL;
  856. spin_unlock(&afu->adapter->afu_list_lock);
  857. cxl_context_detach_all(afu);
  858. cxl_ops->afu_deactivate_mode(afu, afu->current_mode);
  859. guest_release_serr_irq(afu);
  860. guest_unmap_slice_regs(afu);
  861. device_unregister(&afu->dev);
  862. }
  863. static void free_adapter(struct cxl *adapter)
  864. {
  865. struct irq_avail *cur;
  866. int i;
  867. if (adapter->guest) {
  868. if (adapter->guest->irq_avail) {
  869. for (i = 0; i < adapter->guest->irq_nranges; i++) {
  870. cur = &adapter->guest->irq_avail[i];
  871. kfree(cur->bitmap);
  872. }
  873. kfree(adapter->guest->irq_avail);
  874. }
  875. kfree(adapter->guest->status);
  876. kfree(adapter->guest);
  877. }
  878. cxl_remove_adapter_nr(adapter);
  879. kfree(adapter);
  880. }
  881. static int properties_look_ok(struct cxl *adapter)
  882. {
  883. /* The absence of this property means that the operational
  884. * status is unknown or okay
  885. */
  886. if (strlen(adapter->guest->status) &&
  887. strcmp(adapter->guest->status, "okay")) {
  888. pr_err("ABORTING:Bad operational status of the device\n");
  889. return -EINVAL;
  890. }
  891. return 0;
  892. }
  893. ssize_t cxl_guest_read_adapter_vpd(struct cxl *adapter, void *buf, size_t len)
  894. {
  895. return guest_collect_vpd(adapter, NULL, buf, len);
  896. }
  897. void cxl_guest_remove_adapter(struct cxl *adapter)
  898. {
  899. pr_devel("in %s\n", __func__);
  900. cxl_sysfs_adapter_remove(adapter);
  901. cxl_guest_remove_chardev(adapter);
  902. device_unregister(&adapter->dev);
  903. }
  904. static void release_adapter(struct device *dev)
  905. {
  906. free_adapter(to_cxl_adapter(dev));
  907. }
  908. struct cxl *cxl_guest_init_adapter(struct device_node *np, struct platform_device *pdev)
  909. {
  910. struct cxl *adapter;
  911. bool free = true;
  912. int rc;
  913. if (!(adapter = cxl_alloc_adapter()))
  914. return ERR_PTR(-ENOMEM);
  915. if (!(adapter->guest = kzalloc(sizeof(struct cxl_guest), GFP_KERNEL))) {
  916. free_adapter(adapter);
  917. return ERR_PTR(-ENOMEM);
  918. }
  919. adapter->slices = 0;
  920. adapter->guest->pdev = pdev;
  921. adapter->dev.parent = &pdev->dev;
  922. adapter->dev.release = release_adapter;
  923. dev_set_drvdata(&pdev->dev, adapter);
  924. /*
  925. * Hypervisor controls PSL timebase initialization (p1 register).
  926. * On FW840, PSL is initialized.
  927. */
  928. adapter->psl_timebase_synced = true;
  929. if ((rc = cxl_of_read_adapter_handle(adapter, np)))
  930. goto err1;
  931. if ((rc = cxl_of_read_adapter_properties(adapter, np)))
  932. goto err1;
  933. if ((rc = properties_look_ok(adapter)))
  934. goto err1;
  935. if ((rc = cxl_guest_add_chardev(adapter)))
  936. goto err1;
  937. /*
  938. * After we call this function we must not free the adapter directly,
  939. * even if it returns an error!
  940. */
  941. if ((rc = cxl_register_adapter(adapter)))
  942. goto err_put1;
  943. if ((rc = cxl_sysfs_adapter_add(adapter)))
  944. goto err_put1;
  945. /* release the context lock as the adapter is configured */
  946. cxl_adapter_context_unlock(adapter);
  947. return adapter;
  948. err_put1:
  949. device_unregister(&adapter->dev);
  950. free = false;
  951. cxl_guest_remove_chardev(adapter);
  952. err1:
  953. if (free)
  954. free_adapter(adapter);
  955. return ERR_PTR(rc);
  956. }
  957. void cxl_guest_reload_module(struct cxl *adapter)
  958. {
  959. struct platform_device *pdev;
  960. pdev = adapter->guest->pdev;
  961. cxl_guest_remove_adapter(adapter);
  962. cxl_of_probe(pdev);
  963. }
  964. const struct cxl_backend_ops cxl_guest_ops = {
  965. .module = THIS_MODULE,
  966. .adapter_reset = guest_reset,
  967. .alloc_one_irq = guest_alloc_one_irq,
  968. .release_one_irq = guest_release_one_irq,
  969. .alloc_irq_ranges = guest_alloc_irq_ranges,
  970. .release_irq_ranges = guest_release_irq_ranges,
  971. .setup_irq = NULL,
  972. .handle_psl_slice_error = guest_handle_psl_slice_error,
  973. .psl_interrupt = guest_psl_irq,
  974. .ack_irq = guest_ack_irq,
  975. .attach_process = guest_attach_process,
  976. .detach_process = guest_detach_process,
  977. .update_ivtes = NULL,
  978. .support_attributes = guest_support_attributes,
  979. .link_ok = guest_link_ok,
  980. .release_afu = guest_release_afu,
  981. .afu_read_err_buffer = guest_afu_read_err_buffer,
  982. .afu_check_and_enable = guest_afu_check_and_enable,
  983. .afu_activate_mode = guest_afu_activate_mode,
  984. .afu_deactivate_mode = guest_afu_deactivate_mode,
  985. .afu_reset = guest_afu_reset,
  986. .afu_cr_read8 = guest_afu_cr_read8,
  987. .afu_cr_read16 = guest_afu_cr_read16,
  988. .afu_cr_read32 = guest_afu_cr_read32,
  989. .afu_cr_read64 = guest_afu_cr_read64,
  990. .afu_cr_write8 = guest_afu_cr_write8,
  991. .afu_cr_write16 = guest_afu_cr_write16,
  992. .afu_cr_write32 = guest_afu_cr_write32,
  993. .read_adapter_vpd = cxl_guest_read_adapter_vpd,
  994. };