xen-front-pgdir-shbuf.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553
  1. // SPDX-License-Identifier: GPL-2.0 OR MIT
  2. /*
  3. * Xen frontend/backend page directory based shared buffer
  4. * helper module.
  5. *
  6. * Copyright (C) 2018 EPAM Systems Inc.
  7. *
  8. * Author: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>
  9. */
  10. #include <linux/module.h>
  11. #include <linux/errno.h>
  12. #include <linux/mm.h>
  13. #include <asm/xen/hypervisor.h>
  14. #include <xen/balloon.h>
  15. #include <xen/xen.h>
  16. #include <xen/xenbus.h>
  17. #include <xen/interface/io/ring.h>
  18. #include <xen/xen-front-pgdir-shbuf.h>
  19. #ifndef GRANT_INVALID_REF
  20. /*
  21. * FIXME: usage of grant reference 0 as invalid grant reference:
  22. * grant reference 0 is valid, but never exposed to a PV driver,
  23. * because of the fact it is already in use/reserved by the PV console.
  24. */
  25. #define GRANT_INVALID_REF 0
  26. #endif
  27. /**
  28. * This structure represents the structure of a shared page
  29. * that contains grant references to the pages of the shared
  30. * buffer. This structure is common to many Xen para-virtualized
  31. * protocols at include/xen/interface/io/
  32. */
  33. struct xen_page_directory {
  34. grant_ref_t gref_dir_next_page;
  35. grant_ref_t gref[1]; /* Variable length */
  36. };
  37. /**
  38. * Shared buffer ops which are differently implemented
  39. * depending on the allocation mode, e.g. if the buffer
  40. * is allocated by the corresponding backend or frontend.
  41. * Some of the operations.
  42. */
  43. struct xen_front_pgdir_shbuf_ops {
  44. /*
  45. * Calculate number of grefs required to handle this buffer,
  46. * e.g. if grefs are required for page directory only or the buffer
  47. * pages as well.
  48. */
  49. void (*calc_num_grefs)(struct xen_front_pgdir_shbuf *buf);
  50. /* Fill page directory according to para-virtual display protocol. */
  51. void (*fill_page_dir)(struct xen_front_pgdir_shbuf *buf);
  52. /* Claim grant references for the pages of the buffer. */
  53. int (*grant_refs_for_buffer)(struct xen_front_pgdir_shbuf *buf,
  54. grant_ref_t *priv_gref_head, int gref_idx);
  55. /* Map grant references of the buffer. */
  56. int (*map)(struct xen_front_pgdir_shbuf *buf);
  57. /* Unmap grant references of the buffer. */
  58. int (*unmap)(struct xen_front_pgdir_shbuf *buf);
  59. };
  60. /**
  61. * Get granted reference to the very first page of the
  62. * page directory. Usually this is passed to the backend,
  63. * so it can find/fill the grant references to the buffer's
  64. * pages.
  65. *
  66. * \param buf shared buffer which page directory is of interest.
  67. * \return granted reference to the very first page of the
  68. * page directory.
  69. */
  70. grant_ref_t
  71. xen_front_pgdir_shbuf_get_dir_start(struct xen_front_pgdir_shbuf *buf)
  72. {
  73. if (!buf->grefs)
  74. return GRANT_INVALID_REF;
  75. return buf->grefs[0];
  76. }
  77. EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_get_dir_start);
  78. /**
  79. * Map granted references of the shared buffer.
  80. *
  81. * Depending on the shared buffer mode of allocation
  82. * (be_alloc flag) this can either do nothing (for buffers
  83. * shared by the frontend itself) or map the provided granted
  84. * references onto the backing storage (buf->pages).
  85. *
  86. * \param buf shared buffer which grants to be maped.
  87. * \return zero on success or a negative number on failure.
  88. */
  89. int xen_front_pgdir_shbuf_map(struct xen_front_pgdir_shbuf *buf)
  90. {
  91. if (buf->ops && buf->ops->map)
  92. return buf->ops->map(buf);
  93. /* No need to map own grant references. */
  94. return 0;
  95. }
  96. EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_map);
  97. /**
  98. * Unmap granted references of the shared buffer.
  99. *
  100. * Depending on the shared buffer mode of allocation
  101. * (be_alloc flag) this can either do nothing (for buffers
  102. * shared by the frontend itself) or unmap the provided granted
  103. * references.
  104. *
  105. * \param buf shared buffer which grants to be unmaped.
  106. * \return zero on success or a negative number on failure.
  107. */
  108. int xen_front_pgdir_shbuf_unmap(struct xen_front_pgdir_shbuf *buf)
  109. {
  110. if (buf->ops && buf->ops->unmap)
  111. return buf->ops->unmap(buf);
  112. /* No need to unmap own grant references. */
  113. return 0;
  114. }
  115. EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_unmap);
  116. /**
  117. * Free all the resources of the shared buffer.
  118. *
  119. * \param buf shared buffer which resources to be freed.
  120. */
  121. void xen_front_pgdir_shbuf_free(struct xen_front_pgdir_shbuf *buf)
  122. {
  123. if (buf->grefs) {
  124. int i;
  125. for (i = 0; i < buf->num_grefs; i++)
  126. if (buf->grefs[i] != GRANT_INVALID_REF)
  127. gnttab_end_foreign_access(buf->grefs[i],
  128. 0, 0UL);
  129. }
  130. kfree(buf->grefs);
  131. kfree(buf->directory);
  132. }
  133. EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_free);
  134. /*
  135. * Number of grefs a page can hold with respect to the
  136. * struct xen_page_directory header.
  137. */
  138. #define XEN_NUM_GREFS_PER_PAGE ((PAGE_SIZE - \
  139. offsetof(struct xen_page_directory, \
  140. gref)) / sizeof(grant_ref_t))
  141. /**
  142. * Get the number of pages the page directory consumes itself.
  143. *
  144. * \param buf shared buffer.
  145. */
  146. static int get_num_pages_dir(struct xen_front_pgdir_shbuf *buf)
  147. {
  148. return DIV_ROUND_UP(buf->num_pages, XEN_NUM_GREFS_PER_PAGE);
  149. }
  150. /**
  151. * Calculate the number of grant references needed to share the buffer
  152. * and its pages when backend allocates the buffer.
  153. *
  154. * \param buf shared buffer.
  155. */
  156. static void backend_calc_num_grefs(struct xen_front_pgdir_shbuf *buf)
  157. {
  158. /* Only for pages the page directory consumes itself. */
  159. buf->num_grefs = get_num_pages_dir(buf);
  160. }
  161. /**
  162. * Calculate the number of grant references needed to share the buffer
  163. * and its pages when frontend allocates the buffer.
  164. *
  165. * \param buf shared buffer.
  166. */
  167. static void guest_calc_num_grefs(struct xen_front_pgdir_shbuf *buf)
  168. {
  169. /*
  170. * Number of pages the page directory consumes itself
  171. * plus grefs for the buffer pages.
  172. */
  173. buf->num_grefs = get_num_pages_dir(buf) + buf->num_pages;
  174. }
  175. #define xen_page_to_vaddr(page) \
  176. ((uintptr_t)pfn_to_kaddr(page_to_xen_pfn(page)))
  177. /**
  178. * Unmap the buffer previously mapped with grant references
  179. * provided by the backend.
  180. *
  181. * \param buf shared buffer.
  182. * \return zero on success or a negative number on failure.
  183. */
  184. static int backend_unmap(struct xen_front_pgdir_shbuf *buf)
  185. {
  186. struct gnttab_unmap_grant_ref *unmap_ops;
  187. int i, ret;
  188. if (!buf->pages || !buf->backend_map_handles || !buf->grefs)
  189. return 0;
  190. unmap_ops = kcalloc(buf->num_pages, sizeof(*unmap_ops),
  191. GFP_KERNEL);
  192. if (!unmap_ops)
  193. return -ENOMEM;
  194. for (i = 0; i < buf->num_pages; i++) {
  195. phys_addr_t addr;
  196. addr = xen_page_to_vaddr(buf->pages[i]);
  197. gnttab_set_unmap_op(&unmap_ops[i], addr, GNTMAP_host_map,
  198. buf->backend_map_handles[i]);
  199. }
  200. ret = gnttab_unmap_refs(unmap_ops, NULL, buf->pages,
  201. buf->num_pages);
  202. for (i = 0; i < buf->num_pages; i++) {
  203. if (unlikely(unmap_ops[i].status != GNTST_okay))
  204. dev_err(&buf->xb_dev->dev,
  205. "Failed to unmap page %d: %d\n",
  206. i, unmap_ops[i].status);
  207. }
  208. if (ret)
  209. dev_err(&buf->xb_dev->dev,
  210. "Failed to unmap grant references, ret %d", ret);
  211. kfree(unmap_ops);
  212. kfree(buf->backend_map_handles);
  213. buf->backend_map_handles = NULL;
  214. return ret;
  215. }
  216. /**
  217. * Map the buffer with grant references provided by the backend.
  218. *
  219. * \param buf shared buffer.
  220. * \return zero on success or a negative number on failure.
  221. */
  222. static int backend_map(struct xen_front_pgdir_shbuf *buf)
  223. {
  224. struct gnttab_map_grant_ref *map_ops = NULL;
  225. unsigned char *ptr;
  226. int ret, cur_gref, cur_dir_page, cur_page, grefs_left;
  227. map_ops = kcalloc(buf->num_pages, sizeof(*map_ops), GFP_KERNEL);
  228. if (!map_ops)
  229. return -ENOMEM;
  230. buf->backend_map_handles = kcalloc(buf->num_pages,
  231. sizeof(*buf->backend_map_handles),
  232. GFP_KERNEL);
  233. if (!buf->backend_map_handles) {
  234. kfree(map_ops);
  235. return -ENOMEM;
  236. }
  237. /*
  238. * Read page directory to get grefs from the backend: for external
  239. * buffer we only allocate buf->grefs for the page directory,
  240. * so buf->num_grefs has number of pages in the page directory itself.
  241. */
  242. ptr = buf->directory;
  243. grefs_left = buf->num_pages;
  244. cur_page = 0;
  245. for (cur_dir_page = 0; cur_dir_page < buf->num_grefs; cur_dir_page++) {
  246. struct xen_page_directory *page_dir =
  247. (struct xen_page_directory *)ptr;
  248. int to_copy = XEN_NUM_GREFS_PER_PAGE;
  249. if (to_copy > grefs_left)
  250. to_copy = grefs_left;
  251. for (cur_gref = 0; cur_gref < to_copy; cur_gref++) {
  252. phys_addr_t addr;
  253. addr = xen_page_to_vaddr(buf->pages[cur_page]);
  254. gnttab_set_map_op(&map_ops[cur_page], addr,
  255. GNTMAP_host_map,
  256. page_dir->gref[cur_gref],
  257. buf->xb_dev->otherend_id);
  258. cur_page++;
  259. }
  260. grefs_left -= to_copy;
  261. ptr += PAGE_SIZE;
  262. }
  263. ret = gnttab_map_refs(map_ops, NULL, buf->pages, buf->num_pages);
  264. /* Save handles even if error, so we can unmap. */
  265. for (cur_page = 0; cur_page < buf->num_pages; cur_page++) {
  266. buf->backend_map_handles[cur_page] = map_ops[cur_page].handle;
  267. if (unlikely(map_ops[cur_page].status != GNTST_okay))
  268. dev_err(&buf->xb_dev->dev,
  269. "Failed to map page %d: %d\n",
  270. cur_page, map_ops[cur_page].status);
  271. }
  272. if (ret) {
  273. dev_err(&buf->xb_dev->dev,
  274. "Failed to map grant references, ret %d", ret);
  275. backend_unmap(buf);
  276. }
  277. kfree(map_ops);
  278. return ret;
  279. }
  280. /**
  281. * Fill page directory with grant references to the pages of the
  282. * page directory itself.
  283. *
  284. * The grant references to the buffer pages are provided by the
  285. * backend in this case.
  286. *
  287. * \param buf shared buffer.
  288. */
  289. static void backend_fill_page_dir(struct xen_front_pgdir_shbuf *buf)
  290. {
  291. struct xen_page_directory *page_dir;
  292. unsigned char *ptr;
  293. int i, num_pages_dir;
  294. ptr = buf->directory;
  295. num_pages_dir = get_num_pages_dir(buf);
  296. /* Fill only grefs for the page directory itself. */
  297. for (i = 0; i < num_pages_dir - 1; i++) {
  298. page_dir = (struct xen_page_directory *)ptr;
  299. page_dir->gref_dir_next_page = buf->grefs[i + 1];
  300. ptr += PAGE_SIZE;
  301. }
  302. /* Last page must say there is no more pages. */
  303. page_dir = (struct xen_page_directory *)ptr;
  304. page_dir->gref_dir_next_page = GRANT_INVALID_REF;
  305. }
  306. /**
  307. * Fill page directory with grant references to the pages of the
  308. * page directory and the buffer we share with the backend.
  309. *
  310. * \param buf shared buffer.
  311. */
  312. static void guest_fill_page_dir(struct xen_front_pgdir_shbuf *buf)
  313. {
  314. unsigned char *ptr;
  315. int cur_gref, grefs_left, to_copy, i, num_pages_dir;
  316. ptr = buf->directory;
  317. num_pages_dir = get_num_pages_dir(buf);
  318. /*
  319. * While copying, skip grefs at start, they are for pages
  320. * granted for the page directory itself.
  321. */
  322. cur_gref = num_pages_dir;
  323. grefs_left = buf->num_pages;
  324. for (i = 0; i < num_pages_dir; i++) {
  325. struct xen_page_directory *page_dir =
  326. (struct xen_page_directory *)ptr;
  327. if (grefs_left <= XEN_NUM_GREFS_PER_PAGE) {
  328. to_copy = grefs_left;
  329. page_dir->gref_dir_next_page = GRANT_INVALID_REF;
  330. } else {
  331. to_copy = XEN_NUM_GREFS_PER_PAGE;
  332. page_dir->gref_dir_next_page = buf->grefs[i + 1];
  333. }
  334. memcpy(&page_dir->gref, &buf->grefs[cur_gref],
  335. to_copy * sizeof(grant_ref_t));
  336. ptr += PAGE_SIZE;
  337. grefs_left -= to_copy;
  338. cur_gref += to_copy;
  339. }
  340. }
  341. /**
  342. * Grant references to the frontend's buffer pages.
  343. *
  344. * These will be shared with the backend, so it can
  345. * access the buffer's data.
  346. *
  347. * \param buf shared buffer.
  348. * \return zero on success or a negative number on failure.
  349. */
  350. static int guest_grant_refs_for_buffer(struct xen_front_pgdir_shbuf *buf,
  351. grant_ref_t *priv_gref_head,
  352. int gref_idx)
  353. {
  354. int i, cur_ref, otherend_id;
  355. otherend_id = buf->xb_dev->otherend_id;
  356. for (i = 0; i < buf->num_pages; i++) {
  357. cur_ref = gnttab_claim_grant_reference(priv_gref_head);
  358. if (cur_ref < 0)
  359. return cur_ref;
  360. gnttab_grant_foreign_access_ref(cur_ref, otherend_id,
  361. xen_page_to_gfn(buf->pages[i]),
  362. 0);
  363. buf->grefs[gref_idx++] = cur_ref;
  364. }
  365. return 0;
  366. }
  367. /**
  368. * Grant all the references needed to share the buffer.
  369. *
  370. * Grant references to the page directory pages and, if
  371. * needed, also to the pages of the shared buffer data.
  372. *
  373. * \param buf shared buffer.
  374. * \return zero on success or a negative number on failure.
  375. */
  376. static int grant_references(struct xen_front_pgdir_shbuf *buf)
  377. {
  378. grant_ref_t priv_gref_head;
  379. int ret, i, j, cur_ref;
  380. int otherend_id, num_pages_dir;
  381. ret = gnttab_alloc_grant_references(buf->num_grefs, &priv_gref_head);
  382. if (ret < 0) {
  383. dev_err(&buf->xb_dev->dev,
  384. "Cannot allocate grant references\n");
  385. return ret;
  386. }
  387. otherend_id = buf->xb_dev->otherend_id;
  388. j = 0;
  389. num_pages_dir = get_num_pages_dir(buf);
  390. for (i = 0; i < num_pages_dir; i++) {
  391. unsigned long frame;
  392. cur_ref = gnttab_claim_grant_reference(&priv_gref_head);
  393. if (cur_ref < 0)
  394. return cur_ref;
  395. frame = xen_page_to_gfn(virt_to_page(buf->directory +
  396. PAGE_SIZE * i));
  397. gnttab_grant_foreign_access_ref(cur_ref, otherend_id, frame, 0);
  398. buf->grefs[j++] = cur_ref;
  399. }
  400. if (buf->ops->grant_refs_for_buffer) {
  401. ret = buf->ops->grant_refs_for_buffer(buf, &priv_gref_head, j);
  402. if (ret)
  403. return ret;
  404. }
  405. gnttab_free_grant_references(priv_gref_head);
  406. return 0;
  407. }
  408. /**
  409. * Allocate all required structures to mange shared buffer.
  410. *
  411. * \param buf shared buffer.
  412. * \return zero on success or a negative number on failure.
  413. */
  414. static int alloc_storage(struct xen_front_pgdir_shbuf *buf)
  415. {
  416. buf->grefs = kcalloc(buf->num_grefs, sizeof(*buf->grefs), GFP_KERNEL);
  417. if (!buf->grefs)
  418. return -ENOMEM;
  419. buf->directory = kcalloc(get_num_pages_dir(buf), PAGE_SIZE, GFP_KERNEL);
  420. if (!buf->directory)
  421. return -ENOMEM;
  422. return 0;
  423. }
  424. /*
  425. * For backend allocated buffers we don't need grant_refs_for_buffer
  426. * as those grant references are allocated at backend side.
  427. */
  428. static const struct xen_front_pgdir_shbuf_ops backend_ops = {
  429. .calc_num_grefs = backend_calc_num_grefs,
  430. .fill_page_dir = backend_fill_page_dir,
  431. .map = backend_map,
  432. .unmap = backend_unmap
  433. };
  434. /*
  435. * For locally granted references we do not need to map/unmap
  436. * the references.
  437. */
  438. static const struct xen_front_pgdir_shbuf_ops local_ops = {
  439. .calc_num_grefs = guest_calc_num_grefs,
  440. .fill_page_dir = guest_fill_page_dir,
  441. .grant_refs_for_buffer = guest_grant_refs_for_buffer,
  442. };
  443. /**
  444. * Allocate a new instance of a shared buffer.
  445. *
  446. * \param cfg configuration to be used while allocating a new shared buffer.
  447. * \return zero on success or a negative number on failure.
  448. */
  449. int xen_front_pgdir_shbuf_alloc(struct xen_front_pgdir_shbuf_cfg *cfg)
  450. {
  451. struct xen_front_pgdir_shbuf *buf = cfg->pgdir;
  452. int ret;
  453. if (cfg->be_alloc)
  454. buf->ops = &backend_ops;
  455. else
  456. buf->ops = &local_ops;
  457. buf->xb_dev = cfg->xb_dev;
  458. buf->num_pages = cfg->num_pages;
  459. buf->pages = cfg->pages;
  460. buf->ops->calc_num_grefs(buf);
  461. ret = alloc_storage(buf);
  462. if (ret)
  463. goto fail;
  464. ret = grant_references(buf);
  465. if (ret)
  466. goto fail;
  467. buf->ops->fill_page_dir(buf);
  468. return 0;
  469. fail:
  470. xen_front_pgdir_shbuf_free(buf);
  471. return ret;
  472. }
  473. EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_alloc);
  474. MODULE_DESCRIPTION("Xen frontend/backend page directory based "
  475. "shared buffer handling");
  476. MODULE_AUTHOR("Oleksandr Andrushchenko");
  477. MODULE_LICENSE("GPL");