vringh.c 37 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Helpers for the host side of a virtio ring.
  4. *
  5. * Since these may be in userspace, we use (inline) accessors.
  6. */
  7. #include <linux/compiler.h>
  8. #include <linux/module.h>
  9. #include <linux/vringh.h>
  10. #include <linux/virtio_ring.h>
  11. #include <linux/kernel.h>
  12. #include <linux/ratelimit.h>
  13. #include <linux/uaccess.h>
  14. #include <linux/slab.h>
  15. #include <linux/export.h>
  16. #if IS_REACHABLE(CONFIG_VHOST_IOTLB)
  17. #include <linux/bvec.h>
  18. #include <linux/highmem.h>
  19. #include <linux/vhost_iotlb.h>
  20. #endif
  21. #include <uapi/linux/virtio_config.h>
  22. static __printf(1,2) __cold void vringh_bad(const char *fmt, ...)
  23. {
  24. static DEFINE_RATELIMIT_STATE(vringh_rs,
  25. DEFAULT_RATELIMIT_INTERVAL,
  26. DEFAULT_RATELIMIT_BURST);
  27. if (__ratelimit(&vringh_rs)) {
  28. va_list ap;
  29. va_start(ap, fmt);
  30. printk(KERN_NOTICE "vringh:");
  31. vprintk(fmt, ap);
  32. va_end(ap);
  33. }
  34. }
  35. /* Returns vring->num if empty, -ve on error. */
  36. static inline int __vringh_get_head(const struct vringh *vrh,
  37. int (*getu16)(const struct vringh *vrh,
  38. u16 *val, const __virtio16 *p),
  39. u16 *last_avail_idx)
  40. {
  41. u16 avail_idx, i, head;
  42. int err;
  43. err = getu16(vrh, &avail_idx, &vrh->vring.avail->idx);
  44. if (err) {
  45. vringh_bad("Failed to access avail idx at %p",
  46. &vrh->vring.avail->idx);
  47. return err;
  48. }
  49. if (*last_avail_idx == avail_idx)
  50. return vrh->vring.num;
  51. /* Only get avail ring entries after they have been exposed by guest. */
  52. virtio_rmb(vrh->weak_barriers);
  53. i = *last_avail_idx & (vrh->vring.num - 1);
  54. err = getu16(vrh, &head, &vrh->vring.avail->ring[i]);
  55. if (err) {
  56. vringh_bad("Failed to read head: idx %d address %p",
  57. *last_avail_idx, &vrh->vring.avail->ring[i]);
  58. return err;
  59. }
  60. if (head >= vrh->vring.num) {
  61. vringh_bad("Guest says index %u > %u is available",
  62. head, vrh->vring.num);
  63. return -EINVAL;
  64. }
  65. (*last_avail_idx)++;
  66. return head;
  67. }
  68. /* Copy some bytes to/from the iovec. Returns num copied. */
  69. static inline ssize_t vringh_iov_xfer(struct vringh *vrh,
  70. struct vringh_kiov *iov,
  71. void *ptr, size_t len,
  72. int (*xfer)(const struct vringh *vrh,
  73. void *addr, void *ptr,
  74. size_t len))
  75. {
  76. int err, done = 0;
  77. while (len && iov->i < iov->used) {
  78. size_t partlen;
  79. partlen = min(iov->iov[iov->i].iov_len, len);
  80. err = xfer(vrh, iov->iov[iov->i].iov_base, ptr, partlen);
  81. if (err)
  82. return err;
  83. done += partlen;
  84. len -= partlen;
  85. ptr += partlen;
  86. iov->consumed += partlen;
  87. iov->iov[iov->i].iov_len -= partlen;
  88. iov->iov[iov->i].iov_base += partlen;
  89. if (!iov->iov[iov->i].iov_len) {
  90. /* Fix up old iov element then increment. */
  91. iov->iov[iov->i].iov_len = iov->consumed;
  92. iov->iov[iov->i].iov_base -= iov->consumed;
  93. iov->consumed = 0;
  94. iov->i++;
  95. }
  96. }
  97. return done;
  98. }
  99. /* May reduce *len if range is shorter. */
  100. static inline bool range_check(struct vringh *vrh, u64 addr, size_t *len,
  101. struct vringh_range *range,
  102. bool (*getrange)(struct vringh *,
  103. u64, struct vringh_range *))
  104. {
  105. if (addr < range->start || addr > range->end_incl) {
  106. if (!getrange(vrh, addr, range))
  107. return false;
  108. }
  109. BUG_ON(addr < range->start || addr > range->end_incl);
  110. /* To end of memory? */
  111. if (unlikely(addr + *len == 0)) {
  112. if (range->end_incl == -1ULL)
  113. return true;
  114. goto truncate;
  115. }
  116. /* Otherwise, don't wrap. */
  117. if (addr + *len < addr) {
  118. vringh_bad("Wrapping descriptor %zu@0x%llx",
  119. *len, (unsigned long long)addr);
  120. return false;
  121. }
  122. if (unlikely(addr + *len - 1 > range->end_incl))
  123. goto truncate;
  124. return true;
  125. truncate:
  126. *len = range->end_incl + 1 - addr;
  127. return true;
  128. }
  129. static inline bool no_range_check(struct vringh *vrh, u64 addr, size_t *len,
  130. struct vringh_range *range,
  131. bool (*getrange)(struct vringh *,
  132. u64, struct vringh_range *))
  133. {
  134. return true;
  135. }
  136. /* No reason for this code to be inline. */
  137. static int move_to_indirect(const struct vringh *vrh,
  138. int *up_next, u16 *i, void *addr,
  139. const struct vring_desc *desc,
  140. struct vring_desc **descs, int *desc_max)
  141. {
  142. u32 len;
  143. /* Indirect tables can't have indirect. */
  144. if (*up_next != -1) {
  145. vringh_bad("Multilevel indirect %u->%u", *up_next, *i);
  146. return -EINVAL;
  147. }
  148. len = vringh32_to_cpu(vrh, desc->len);
  149. if (unlikely(len % sizeof(struct vring_desc))) {
  150. vringh_bad("Strange indirect len %u", desc->len);
  151. return -EINVAL;
  152. }
  153. /* We will check this when we follow it! */
  154. if (desc->flags & cpu_to_vringh16(vrh, VRING_DESC_F_NEXT))
  155. *up_next = vringh16_to_cpu(vrh, desc->next);
  156. else
  157. *up_next = -2;
  158. *descs = addr;
  159. *desc_max = len / sizeof(struct vring_desc);
  160. /* Now, start at the first indirect. */
  161. *i = 0;
  162. return 0;
  163. }
  164. static int resize_iovec(struct vringh_kiov *iov, gfp_t gfp)
  165. {
  166. struct kvec *new;
  167. unsigned int flag, new_num = (iov->max_num & ~VRINGH_IOV_ALLOCATED) * 2;
  168. if (new_num < 8)
  169. new_num = 8;
  170. flag = (iov->max_num & VRINGH_IOV_ALLOCATED);
  171. if (flag)
  172. new = krealloc(iov->iov, new_num * sizeof(struct iovec), gfp);
  173. else {
  174. new = kmalloc_array(new_num, sizeof(struct iovec), gfp);
  175. if (new) {
  176. memcpy(new, iov->iov,
  177. iov->max_num * sizeof(struct iovec));
  178. flag = VRINGH_IOV_ALLOCATED;
  179. }
  180. }
  181. if (!new)
  182. return -ENOMEM;
  183. iov->iov = new;
  184. iov->max_num = (new_num | flag);
  185. return 0;
  186. }
  187. static u16 __cold return_from_indirect(const struct vringh *vrh, int *up_next,
  188. struct vring_desc **descs, int *desc_max)
  189. {
  190. u16 i = *up_next;
  191. *up_next = -1;
  192. *descs = vrh->vring.desc;
  193. *desc_max = vrh->vring.num;
  194. return i;
  195. }
  196. static int slow_copy(struct vringh *vrh, void *dst, const void *src,
  197. bool (*rcheck)(struct vringh *vrh, u64 addr, size_t *len,
  198. struct vringh_range *range,
  199. bool (*getrange)(struct vringh *vrh,
  200. u64,
  201. struct vringh_range *)),
  202. bool (*getrange)(struct vringh *vrh,
  203. u64 addr,
  204. struct vringh_range *r),
  205. struct vringh_range *range,
  206. int (*copy)(const struct vringh *vrh,
  207. void *dst, const void *src, size_t len))
  208. {
  209. size_t part, len = sizeof(struct vring_desc);
  210. do {
  211. u64 addr;
  212. int err;
  213. part = len;
  214. addr = (u64)(unsigned long)src - range->offset;
  215. if (!rcheck(vrh, addr, &part, range, getrange))
  216. return -EINVAL;
  217. err = copy(vrh, dst, src, part);
  218. if (err)
  219. return err;
  220. dst += part;
  221. src += part;
  222. len -= part;
  223. } while (len);
  224. return 0;
  225. }
  226. static inline int
  227. __vringh_iov(struct vringh *vrh, u16 i,
  228. struct vringh_kiov *riov,
  229. struct vringh_kiov *wiov,
  230. bool (*rcheck)(struct vringh *vrh, u64 addr, size_t *len,
  231. struct vringh_range *range,
  232. bool (*getrange)(struct vringh *, u64,
  233. struct vringh_range *)),
  234. bool (*getrange)(struct vringh *, u64, struct vringh_range *),
  235. gfp_t gfp,
  236. int (*copy)(const struct vringh *vrh,
  237. void *dst, const void *src, size_t len))
  238. {
  239. int err, count = 0, up_next, desc_max;
  240. struct vring_desc desc, *descs;
  241. struct vringh_range range = { -1ULL, 0 }, slowrange;
  242. bool slow = false;
  243. /* We start traversing vring's descriptor table. */
  244. descs = vrh->vring.desc;
  245. desc_max = vrh->vring.num;
  246. up_next = -1;
  247. /* You must want something! */
  248. if (WARN_ON(!riov && !wiov))
  249. return -EINVAL;
  250. if (riov)
  251. riov->i = riov->used = 0;
  252. if (wiov)
  253. wiov->i = wiov->used = 0;
  254. for (;;) {
  255. void *addr;
  256. struct vringh_kiov *iov;
  257. size_t len;
  258. if (unlikely(slow))
  259. err = slow_copy(vrh, &desc, &descs[i], rcheck, getrange,
  260. &slowrange, copy);
  261. else
  262. err = copy(vrh, &desc, &descs[i], sizeof(desc));
  263. if (unlikely(err))
  264. goto fail;
  265. if (unlikely(desc.flags &
  266. cpu_to_vringh16(vrh, VRING_DESC_F_INDIRECT))) {
  267. u64 a = vringh64_to_cpu(vrh, desc.addr);
  268. /* Make sure it's OK, and get offset. */
  269. len = vringh32_to_cpu(vrh, desc.len);
  270. if (!rcheck(vrh, a, &len, &range, getrange)) {
  271. err = -EINVAL;
  272. goto fail;
  273. }
  274. if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) {
  275. slow = true;
  276. /* We need to save this range to use offset */
  277. slowrange = range;
  278. }
  279. addr = (void *)(long)(a + range.offset);
  280. err = move_to_indirect(vrh, &up_next, &i, addr, &desc,
  281. &descs, &desc_max);
  282. if (err)
  283. goto fail;
  284. continue;
  285. }
  286. if (count++ == vrh->vring.num) {
  287. vringh_bad("Descriptor loop in %p", descs);
  288. err = -ELOOP;
  289. goto fail;
  290. }
  291. if (desc.flags & cpu_to_vringh16(vrh, VRING_DESC_F_WRITE))
  292. iov = wiov;
  293. else {
  294. iov = riov;
  295. if (unlikely(wiov && wiov->used)) {
  296. vringh_bad("Readable desc %p after writable",
  297. &descs[i]);
  298. err = -EINVAL;
  299. goto fail;
  300. }
  301. }
  302. if (!iov) {
  303. vringh_bad("Unexpected %s desc",
  304. !wiov ? "writable" : "readable");
  305. err = -EPROTO;
  306. goto fail;
  307. }
  308. again:
  309. /* Make sure it's OK, and get offset. */
  310. len = vringh32_to_cpu(vrh, desc.len);
  311. if (!rcheck(vrh, vringh64_to_cpu(vrh, desc.addr), &len, &range,
  312. getrange)) {
  313. err = -EINVAL;
  314. goto fail;
  315. }
  316. addr = (void *)(unsigned long)(vringh64_to_cpu(vrh, desc.addr) +
  317. range.offset);
  318. if (unlikely(iov->used == (iov->max_num & ~VRINGH_IOV_ALLOCATED))) {
  319. err = resize_iovec(iov, gfp);
  320. if (err)
  321. goto fail;
  322. }
  323. iov->iov[iov->used].iov_base = addr;
  324. iov->iov[iov->used].iov_len = len;
  325. iov->used++;
  326. if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) {
  327. desc.len = cpu_to_vringh32(vrh,
  328. vringh32_to_cpu(vrh, desc.len) - len);
  329. desc.addr = cpu_to_vringh64(vrh,
  330. vringh64_to_cpu(vrh, desc.addr) + len);
  331. goto again;
  332. }
  333. if (desc.flags & cpu_to_vringh16(vrh, VRING_DESC_F_NEXT)) {
  334. i = vringh16_to_cpu(vrh, desc.next);
  335. } else {
  336. /* Just in case we need to finish traversing above. */
  337. if (unlikely(up_next > 0)) {
  338. i = return_from_indirect(vrh, &up_next,
  339. &descs, &desc_max);
  340. slow = false;
  341. } else
  342. break;
  343. }
  344. if (i >= desc_max) {
  345. vringh_bad("Chained index %u > %u", i, desc_max);
  346. err = -EINVAL;
  347. goto fail;
  348. }
  349. }
  350. return 0;
  351. fail:
  352. return err;
  353. }
  354. static inline int __vringh_complete(struct vringh *vrh,
  355. const struct vring_used_elem *used,
  356. unsigned int num_used,
  357. int (*putu16)(const struct vringh *vrh,
  358. __virtio16 *p, u16 val),
  359. int (*putused)(const struct vringh *vrh,
  360. struct vring_used_elem *dst,
  361. const struct vring_used_elem
  362. *src, unsigned num))
  363. {
  364. struct vring_used *used_ring;
  365. int err;
  366. u16 used_idx, off;
  367. used_ring = vrh->vring.used;
  368. used_idx = vrh->last_used_idx + vrh->completed;
  369. off = used_idx % vrh->vring.num;
  370. /* Compiler knows num_used == 1 sometimes, hence extra check */
  371. if (num_used > 1 && unlikely(off + num_used >= vrh->vring.num)) {
  372. u16 part = vrh->vring.num - off;
  373. err = putused(vrh, &used_ring->ring[off], used, part);
  374. if (!err)
  375. err = putused(vrh, &used_ring->ring[0], used + part,
  376. num_used - part);
  377. } else
  378. err = putused(vrh, &used_ring->ring[off], used, num_used);
  379. if (err) {
  380. vringh_bad("Failed to write %u used entries %u at %p",
  381. num_used, off, &used_ring->ring[off]);
  382. return err;
  383. }
  384. /* Make sure buffer is written before we update index. */
  385. virtio_wmb(vrh->weak_barriers);
  386. err = putu16(vrh, &vrh->vring.used->idx, used_idx + num_used);
  387. if (err) {
  388. vringh_bad("Failed to update used index at %p",
  389. &vrh->vring.used->idx);
  390. return err;
  391. }
  392. vrh->completed += num_used;
  393. return 0;
  394. }
  395. static inline int __vringh_need_notify(struct vringh *vrh,
  396. int (*getu16)(const struct vringh *vrh,
  397. u16 *val,
  398. const __virtio16 *p))
  399. {
  400. bool notify;
  401. u16 used_event;
  402. int err;
  403. /* Flush out used index update. This is paired with the
  404. * barrier that the Guest executes when enabling
  405. * interrupts. */
  406. virtio_mb(vrh->weak_barriers);
  407. /* Old-style, without event indices. */
  408. if (!vrh->event_indices) {
  409. u16 flags;
  410. err = getu16(vrh, &flags, &vrh->vring.avail->flags);
  411. if (err) {
  412. vringh_bad("Failed to get flags at %p",
  413. &vrh->vring.avail->flags);
  414. return err;
  415. }
  416. return (!(flags & VRING_AVAIL_F_NO_INTERRUPT));
  417. }
  418. /* Modern: we know when other side wants to know. */
  419. err = getu16(vrh, &used_event, &vring_used_event(&vrh->vring));
  420. if (err) {
  421. vringh_bad("Failed to get used event idx at %p",
  422. &vring_used_event(&vrh->vring));
  423. return err;
  424. }
  425. /* Just in case we added so many that we wrap. */
  426. if (unlikely(vrh->completed > 0xffff))
  427. notify = true;
  428. else
  429. notify = vring_need_event(used_event,
  430. vrh->last_used_idx + vrh->completed,
  431. vrh->last_used_idx);
  432. vrh->last_used_idx += vrh->completed;
  433. vrh->completed = 0;
  434. return notify;
  435. }
  436. static inline bool __vringh_notify_enable(struct vringh *vrh,
  437. int (*getu16)(const struct vringh *vrh,
  438. u16 *val, const __virtio16 *p),
  439. int (*putu16)(const struct vringh *vrh,
  440. __virtio16 *p, u16 val))
  441. {
  442. u16 avail;
  443. if (!vrh->event_indices) {
  444. /* Old-school; update flags. */
  445. if (putu16(vrh, &vrh->vring.used->flags, 0) != 0) {
  446. vringh_bad("Clearing used flags %p",
  447. &vrh->vring.used->flags);
  448. return true;
  449. }
  450. } else {
  451. if (putu16(vrh, &vring_avail_event(&vrh->vring),
  452. vrh->last_avail_idx) != 0) {
  453. vringh_bad("Updating avail event index %p",
  454. &vring_avail_event(&vrh->vring));
  455. return true;
  456. }
  457. }
  458. /* They could have slipped one in as we were doing that: make
  459. * sure it's written, then check again. */
  460. virtio_mb(vrh->weak_barriers);
  461. if (getu16(vrh, &avail, &vrh->vring.avail->idx) != 0) {
  462. vringh_bad("Failed to check avail idx at %p",
  463. &vrh->vring.avail->idx);
  464. return true;
  465. }
  466. /* This is unlikely, so we just leave notifications enabled
  467. * (if we're using event_indices, we'll only get one
  468. * notification anyway). */
  469. return avail == vrh->last_avail_idx;
  470. }
  471. static inline void __vringh_notify_disable(struct vringh *vrh,
  472. int (*putu16)(const struct vringh *vrh,
  473. __virtio16 *p, u16 val))
  474. {
  475. if (!vrh->event_indices) {
  476. /* Old-school; update flags. */
  477. if (putu16(vrh, &vrh->vring.used->flags,
  478. VRING_USED_F_NO_NOTIFY)) {
  479. vringh_bad("Setting used flags %p",
  480. &vrh->vring.used->flags);
  481. }
  482. }
  483. }
  484. /* Userspace access helpers: in this case, addresses are really userspace. */
  485. static inline int getu16_user(const struct vringh *vrh, u16 *val, const __virtio16 *p)
  486. {
  487. __virtio16 v = 0;
  488. int rc = get_user(v, (__force __virtio16 __user *)p);
  489. *val = vringh16_to_cpu(vrh, v);
  490. return rc;
  491. }
  492. static inline int putu16_user(const struct vringh *vrh, __virtio16 *p, u16 val)
  493. {
  494. __virtio16 v = cpu_to_vringh16(vrh, val);
  495. return put_user(v, (__force __virtio16 __user *)p);
  496. }
  497. static inline int copydesc_user(const struct vringh *vrh,
  498. void *dst, const void *src, size_t len)
  499. {
  500. return copy_from_user(dst, (__force void __user *)src, len) ?
  501. -EFAULT : 0;
  502. }
  503. static inline int putused_user(const struct vringh *vrh,
  504. struct vring_used_elem *dst,
  505. const struct vring_used_elem *src,
  506. unsigned int num)
  507. {
  508. return copy_to_user((__force void __user *)dst, src,
  509. sizeof(*dst) * num) ? -EFAULT : 0;
  510. }
  511. static inline int xfer_from_user(const struct vringh *vrh, void *src,
  512. void *dst, size_t len)
  513. {
  514. return copy_from_user(dst, (__force void __user *)src, len) ?
  515. -EFAULT : 0;
  516. }
  517. static inline int xfer_to_user(const struct vringh *vrh,
  518. void *dst, void *src, size_t len)
  519. {
  520. return copy_to_user((__force void __user *)dst, src, len) ?
  521. -EFAULT : 0;
  522. }
  523. /**
  524. * vringh_init_user - initialize a vringh for a userspace vring.
  525. * @vrh: the vringh to initialize.
  526. * @features: the feature bits for this ring.
  527. * @num: the number of elements.
  528. * @weak_barriers: true if we only need memory barriers, not I/O.
  529. * @desc: the userpace descriptor pointer.
  530. * @avail: the userpace avail pointer.
  531. * @used: the userpace used pointer.
  532. *
  533. * Returns an error if num is invalid: you should check pointers
  534. * yourself!
  535. */
  536. int vringh_init_user(struct vringh *vrh, u64 features,
  537. unsigned int num, bool weak_barriers,
  538. vring_desc_t __user *desc,
  539. vring_avail_t __user *avail,
  540. vring_used_t __user *used)
  541. {
  542. /* Sane power of 2 please! */
  543. if (!num || num > 0xffff || (num & (num - 1))) {
  544. vringh_bad("Bad ring size %u", num);
  545. return -EINVAL;
  546. }
  547. vrh->little_endian = (features & (1ULL << VIRTIO_F_VERSION_1));
  548. vrh->event_indices = (features & (1 << VIRTIO_RING_F_EVENT_IDX));
  549. vrh->weak_barriers = weak_barriers;
  550. vrh->completed = 0;
  551. vrh->last_avail_idx = 0;
  552. vrh->last_used_idx = 0;
  553. vrh->vring.num = num;
  554. /* vring expects kernel addresses, but only used via accessors. */
  555. vrh->vring.desc = (__force struct vring_desc *)desc;
  556. vrh->vring.avail = (__force struct vring_avail *)avail;
  557. vrh->vring.used = (__force struct vring_used *)used;
  558. return 0;
  559. }
  560. EXPORT_SYMBOL(vringh_init_user);
  561. /**
  562. * vringh_getdesc_user - get next available descriptor from userspace ring.
  563. * @vrh: the userspace vring.
  564. * @riov: where to put the readable descriptors (or NULL)
  565. * @wiov: where to put the writable descriptors (or NULL)
  566. * @getrange: function to call to check ranges.
  567. * @head: head index we received, for passing to vringh_complete_user().
  568. *
  569. * Returns 0 if there was no descriptor, 1 if there was, or -errno.
  570. *
  571. * Note that on error return, you can tell the difference between an
  572. * invalid ring and a single invalid descriptor: in the former case,
  573. * *head will be vrh->vring.num. You may be able to ignore an invalid
  574. * descriptor, but there's not much you can do with an invalid ring.
  575. *
  576. * Note that you may need to clean up riov and wiov, even on error!
  577. */
  578. int vringh_getdesc_user(struct vringh *vrh,
  579. struct vringh_iov *riov,
  580. struct vringh_iov *wiov,
  581. bool (*getrange)(struct vringh *vrh,
  582. u64 addr, struct vringh_range *r),
  583. u16 *head)
  584. {
  585. int err;
  586. *head = vrh->vring.num;
  587. err = __vringh_get_head(vrh, getu16_user, &vrh->last_avail_idx);
  588. if (err < 0)
  589. return err;
  590. /* Empty... */
  591. if (err == vrh->vring.num)
  592. return 0;
  593. /* We need the layouts to be the identical for this to work */
  594. BUILD_BUG_ON(sizeof(struct vringh_kiov) != sizeof(struct vringh_iov));
  595. BUILD_BUG_ON(offsetof(struct vringh_kiov, iov) !=
  596. offsetof(struct vringh_iov, iov));
  597. BUILD_BUG_ON(offsetof(struct vringh_kiov, i) !=
  598. offsetof(struct vringh_iov, i));
  599. BUILD_BUG_ON(offsetof(struct vringh_kiov, used) !=
  600. offsetof(struct vringh_iov, used));
  601. BUILD_BUG_ON(offsetof(struct vringh_kiov, max_num) !=
  602. offsetof(struct vringh_iov, max_num));
  603. BUILD_BUG_ON(sizeof(struct iovec) != sizeof(struct kvec));
  604. BUILD_BUG_ON(offsetof(struct iovec, iov_base) !=
  605. offsetof(struct kvec, iov_base));
  606. BUILD_BUG_ON(offsetof(struct iovec, iov_len) !=
  607. offsetof(struct kvec, iov_len));
  608. BUILD_BUG_ON(sizeof(((struct iovec *)NULL)->iov_base)
  609. != sizeof(((struct kvec *)NULL)->iov_base));
  610. BUILD_BUG_ON(sizeof(((struct iovec *)NULL)->iov_len)
  611. != sizeof(((struct kvec *)NULL)->iov_len));
  612. *head = err;
  613. err = __vringh_iov(vrh, *head, (struct vringh_kiov *)riov,
  614. (struct vringh_kiov *)wiov,
  615. range_check, getrange, GFP_KERNEL, copydesc_user);
  616. if (err)
  617. return err;
  618. return 1;
  619. }
  620. EXPORT_SYMBOL(vringh_getdesc_user);
  621. /**
  622. * vringh_iov_pull_user - copy bytes from vring_iov.
  623. * @riov: the riov as passed to vringh_getdesc_user() (updated as we consume)
  624. * @dst: the place to copy.
  625. * @len: the maximum length to copy.
  626. *
  627. * Returns the bytes copied <= len or a negative errno.
  628. */
  629. ssize_t vringh_iov_pull_user(struct vringh_iov *riov, void *dst, size_t len)
  630. {
  631. return vringh_iov_xfer(NULL, (struct vringh_kiov *)riov,
  632. dst, len, xfer_from_user);
  633. }
  634. EXPORT_SYMBOL(vringh_iov_pull_user);
  635. /**
  636. * vringh_iov_push_user - copy bytes into vring_iov.
  637. * @wiov: the wiov as passed to vringh_getdesc_user() (updated as we consume)
  638. * @src: the place to copy from.
  639. * @len: the maximum length to copy.
  640. *
  641. * Returns the bytes copied <= len or a negative errno.
  642. */
  643. ssize_t vringh_iov_push_user(struct vringh_iov *wiov,
  644. const void *src, size_t len)
  645. {
  646. return vringh_iov_xfer(NULL, (struct vringh_kiov *)wiov,
  647. (void *)src, len, xfer_to_user);
  648. }
  649. EXPORT_SYMBOL(vringh_iov_push_user);
  650. /**
  651. * vringh_abandon_user - we've decided not to handle the descriptor(s).
  652. * @vrh: the vring.
  653. * @num: the number of descriptors to put back (ie. num
  654. * vringh_get_user() to undo).
  655. *
  656. * The next vringh_get_user() will return the old descriptor(s) again.
  657. */
  658. void vringh_abandon_user(struct vringh *vrh, unsigned int num)
  659. {
  660. /* We only update vring_avail_event(vr) when we want to be notified,
  661. * so we haven't changed that yet. */
  662. vrh->last_avail_idx -= num;
  663. }
  664. EXPORT_SYMBOL(vringh_abandon_user);
  665. /**
  666. * vringh_complete_user - we've finished with descriptor, publish it.
  667. * @vrh: the vring.
  668. * @head: the head as filled in by vringh_getdesc_user.
  669. * @len: the length of data we have written.
  670. *
  671. * You should check vringh_need_notify_user() after one or more calls
  672. * to this function.
  673. */
  674. int vringh_complete_user(struct vringh *vrh, u16 head, u32 len)
  675. {
  676. struct vring_used_elem used;
  677. used.id = cpu_to_vringh32(vrh, head);
  678. used.len = cpu_to_vringh32(vrh, len);
  679. return __vringh_complete(vrh, &used, 1, putu16_user, putused_user);
  680. }
  681. EXPORT_SYMBOL(vringh_complete_user);
  682. /**
  683. * vringh_complete_multi_user - we've finished with many descriptors.
  684. * @vrh: the vring.
  685. * @used: the head, length pairs.
  686. * @num_used: the number of used elements.
  687. *
  688. * You should check vringh_need_notify_user() after one or more calls
  689. * to this function.
  690. */
  691. int vringh_complete_multi_user(struct vringh *vrh,
  692. const struct vring_used_elem used[],
  693. unsigned num_used)
  694. {
  695. return __vringh_complete(vrh, used, num_used,
  696. putu16_user, putused_user);
  697. }
  698. EXPORT_SYMBOL(vringh_complete_multi_user);
  699. /**
  700. * vringh_notify_enable_user - we want to know if something changes.
  701. * @vrh: the vring.
  702. *
  703. * This always enables notifications, but returns false if there are
  704. * now more buffers available in the vring.
  705. */
  706. bool vringh_notify_enable_user(struct vringh *vrh)
  707. {
  708. return __vringh_notify_enable(vrh, getu16_user, putu16_user);
  709. }
  710. EXPORT_SYMBOL(vringh_notify_enable_user);
  711. /**
  712. * vringh_notify_disable_user - don't tell us if something changes.
  713. * @vrh: the vring.
  714. *
  715. * This is our normal running state: we disable and then only enable when
  716. * we're going to sleep.
  717. */
  718. void vringh_notify_disable_user(struct vringh *vrh)
  719. {
  720. __vringh_notify_disable(vrh, putu16_user);
  721. }
  722. EXPORT_SYMBOL(vringh_notify_disable_user);
  723. /**
  724. * vringh_need_notify_user - must we tell the other side about used buffers?
  725. * @vrh: the vring we've called vringh_complete_user() on.
  726. *
  727. * Returns -errno or 0 if we don't need to tell the other side, 1 if we do.
  728. */
  729. int vringh_need_notify_user(struct vringh *vrh)
  730. {
  731. return __vringh_need_notify(vrh, getu16_user);
  732. }
  733. EXPORT_SYMBOL(vringh_need_notify_user);
  734. /* Kernelspace access helpers. */
  735. static inline int getu16_kern(const struct vringh *vrh,
  736. u16 *val, const __virtio16 *p)
  737. {
  738. *val = vringh16_to_cpu(vrh, READ_ONCE(*p));
  739. return 0;
  740. }
  741. static inline int putu16_kern(const struct vringh *vrh, __virtio16 *p, u16 val)
  742. {
  743. WRITE_ONCE(*p, cpu_to_vringh16(vrh, val));
  744. return 0;
  745. }
  746. static inline int copydesc_kern(const struct vringh *vrh,
  747. void *dst, const void *src, size_t len)
  748. {
  749. memcpy(dst, src, len);
  750. return 0;
  751. }
  752. static inline int putused_kern(const struct vringh *vrh,
  753. struct vring_used_elem *dst,
  754. const struct vring_used_elem *src,
  755. unsigned int num)
  756. {
  757. memcpy(dst, src, num * sizeof(*dst));
  758. return 0;
  759. }
  760. static inline int xfer_kern(const struct vringh *vrh, void *src,
  761. void *dst, size_t len)
  762. {
  763. memcpy(dst, src, len);
  764. return 0;
  765. }
  766. static inline int kern_xfer(const struct vringh *vrh, void *dst,
  767. void *src, size_t len)
  768. {
  769. memcpy(dst, src, len);
  770. return 0;
  771. }
  772. /**
  773. * vringh_init_kern - initialize a vringh for a kernelspace vring.
  774. * @vrh: the vringh to initialize.
  775. * @features: the feature bits for this ring.
  776. * @num: the number of elements.
  777. * @weak_barriers: true if we only need memory barriers, not I/O.
  778. * @desc: the userpace descriptor pointer.
  779. * @avail: the userpace avail pointer.
  780. * @used: the userpace used pointer.
  781. *
  782. * Returns an error if num is invalid.
  783. */
  784. int vringh_init_kern(struct vringh *vrh, u64 features,
  785. unsigned int num, bool weak_barriers,
  786. struct vring_desc *desc,
  787. struct vring_avail *avail,
  788. struct vring_used *used)
  789. {
  790. /* Sane power of 2 please! */
  791. if (!num || num > 0xffff || (num & (num - 1))) {
  792. vringh_bad("Bad ring size %u", num);
  793. return -EINVAL;
  794. }
  795. vrh->little_endian = (features & (1ULL << VIRTIO_F_VERSION_1));
  796. vrh->event_indices = (features & (1 << VIRTIO_RING_F_EVENT_IDX));
  797. vrh->weak_barriers = weak_barriers;
  798. vrh->completed = 0;
  799. vrh->last_avail_idx = 0;
  800. vrh->last_used_idx = 0;
  801. vrh->vring.num = num;
  802. vrh->vring.desc = desc;
  803. vrh->vring.avail = avail;
  804. vrh->vring.used = used;
  805. return 0;
  806. }
  807. EXPORT_SYMBOL(vringh_init_kern);
  808. /**
  809. * vringh_getdesc_kern - get next available descriptor from kernelspace ring.
  810. * @vrh: the kernelspace vring.
  811. * @riov: where to put the readable descriptors (or NULL)
  812. * @wiov: where to put the writable descriptors (or NULL)
  813. * @head: head index we received, for passing to vringh_complete_kern().
  814. * @gfp: flags for allocating larger riov/wiov.
  815. *
  816. * Returns 0 if there was no descriptor, 1 if there was, or -errno.
  817. *
  818. * Note that on error return, you can tell the difference between an
  819. * invalid ring and a single invalid descriptor: in the former case,
  820. * *head will be vrh->vring.num. You may be able to ignore an invalid
  821. * descriptor, but there's not much you can do with an invalid ring.
  822. *
  823. * Note that you may need to clean up riov and wiov, even on error!
  824. */
  825. int vringh_getdesc_kern(struct vringh *vrh,
  826. struct vringh_kiov *riov,
  827. struct vringh_kiov *wiov,
  828. u16 *head,
  829. gfp_t gfp)
  830. {
  831. int err;
  832. err = __vringh_get_head(vrh, getu16_kern, &vrh->last_avail_idx);
  833. if (err < 0)
  834. return err;
  835. /* Empty... */
  836. if (err == vrh->vring.num)
  837. return 0;
  838. *head = err;
  839. err = __vringh_iov(vrh, *head, riov, wiov, no_range_check, NULL,
  840. gfp, copydesc_kern);
  841. if (err)
  842. return err;
  843. return 1;
  844. }
  845. EXPORT_SYMBOL(vringh_getdesc_kern);
  846. /**
  847. * vringh_iov_pull_kern - copy bytes from vring_iov.
  848. * @riov: the riov as passed to vringh_getdesc_kern() (updated as we consume)
  849. * @dst: the place to copy.
  850. * @len: the maximum length to copy.
  851. *
  852. * Returns the bytes copied <= len or a negative errno.
  853. */
  854. ssize_t vringh_iov_pull_kern(struct vringh_kiov *riov, void *dst, size_t len)
  855. {
  856. return vringh_iov_xfer(NULL, riov, dst, len, xfer_kern);
  857. }
  858. EXPORT_SYMBOL(vringh_iov_pull_kern);
  859. /**
  860. * vringh_iov_push_kern - copy bytes into vring_iov.
  861. * @wiov: the wiov as passed to vringh_getdesc_kern() (updated as we consume)
  862. * @src: the place to copy from.
  863. * @len: the maximum length to copy.
  864. *
  865. * Returns the bytes copied <= len or a negative errno.
  866. */
  867. ssize_t vringh_iov_push_kern(struct vringh_kiov *wiov,
  868. const void *src, size_t len)
  869. {
  870. return vringh_iov_xfer(NULL, wiov, (void *)src, len, kern_xfer);
  871. }
  872. EXPORT_SYMBOL(vringh_iov_push_kern);
  873. /**
  874. * vringh_abandon_kern - we've decided not to handle the descriptor(s).
  875. * @vrh: the vring.
  876. * @num: the number of descriptors to put back (ie. num
  877. * vringh_get_kern() to undo).
  878. *
  879. * The next vringh_get_kern() will return the old descriptor(s) again.
  880. */
  881. void vringh_abandon_kern(struct vringh *vrh, unsigned int num)
  882. {
  883. /* We only update vring_avail_event(vr) when we want to be notified,
  884. * so we haven't changed that yet. */
  885. vrh->last_avail_idx -= num;
  886. }
  887. EXPORT_SYMBOL(vringh_abandon_kern);
  888. /**
  889. * vringh_complete_kern - we've finished with descriptor, publish it.
  890. * @vrh: the vring.
  891. * @head: the head as filled in by vringh_getdesc_kern.
  892. * @len: the length of data we have written.
  893. *
  894. * You should check vringh_need_notify_kern() after one or more calls
  895. * to this function.
  896. */
  897. int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len)
  898. {
  899. struct vring_used_elem used;
  900. used.id = cpu_to_vringh32(vrh, head);
  901. used.len = cpu_to_vringh32(vrh, len);
  902. return __vringh_complete(vrh, &used, 1, putu16_kern, putused_kern);
  903. }
  904. EXPORT_SYMBOL(vringh_complete_kern);
  905. /**
  906. * vringh_notify_enable_kern - we want to know if something changes.
  907. * @vrh: the vring.
  908. *
  909. * This always enables notifications, but returns false if there are
  910. * now more buffers available in the vring.
  911. */
  912. bool vringh_notify_enable_kern(struct vringh *vrh)
  913. {
  914. return __vringh_notify_enable(vrh, getu16_kern, putu16_kern);
  915. }
  916. EXPORT_SYMBOL(vringh_notify_enable_kern);
  917. /**
  918. * vringh_notify_disable_kern - don't tell us if something changes.
  919. * @vrh: the vring.
  920. *
  921. * This is our normal running state: we disable and then only enable when
  922. * we're going to sleep.
  923. */
  924. void vringh_notify_disable_kern(struct vringh *vrh)
  925. {
  926. __vringh_notify_disable(vrh, putu16_kern);
  927. }
  928. EXPORT_SYMBOL(vringh_notify_disable_kern);
  929. /**
  930. * vringh_need_notify_kern - must we tell the other side about used buffers?
  931. * @vrh: the vring we've called vringh_complete_kern() on.
  932. *
  933. * Returns -errno or 0 if we don't need to tell the other side, 1 if we do.
  934. */
  935. int vringh_need_notify_kern(struct vringh *vrh)
  936. {
  937. return __vringh_need_notify(vrh, getu16_kern);
  938. }
  939. EXPORT_SYMBOL(vringh_need_notify_kern);
  940. #if IS_REACHABLE(CONFIG_VHOST_IOTLB)
  941. static int iotlb_translate(const struct vringh *vrh,
  942. u64 addr, u64 len, struct bio_vec iov[],
  943. int iov_size, u32 perm)
  944. {
  945. struct vhost_iotlb_map *map;
  946. struct vhost_iotlb *iotlb = vrh->iotlb;
  947. int ret = 0;
  948. u64 s = 0;
  949. while (len > s) {
  950. u64 size, pa, pfn;
  951. if (unlikely(ret >= iov_size)) {
  952. ret = -ENOBUFS;
  953. break;
  954. }
  955. map = vhost_iotlb_itree_first(iotlb, addr,
  956. addr + len - 1);
  957. if (!map || map->start > addr) {
  958. ret = -EINVAL;
  959. break;
  960. } else if (!(map->perm & perm)) {
  961. ret = -EPERM;
  962. break;
  963. }
  964. size = map->size - addr + map->start;
  965. pa = map->addr + addr - map->start;
  966. pfn = pa >> PAGE_SHIFT;
  967. iov[ret].bv_page = pfn_to_page(pfn);
  968. iov[ret].bv_len = min(len - s, size);
  969. iov[ret].bv_offset = pa & (PAGE_SIZE - 1);
  970. s += size;
  971. addr += size;
  972. ++ret;
  973. }
  974. return ret;
  975. }
  976. static inline int copy_from_iotlb(const struct vringh *vrh, void *dst,
  977. void *src, size_t len)
  978. {
  979. struct iov_iter iter;
  980. struct bio_vec iov[16];
  981. int ret;
  982. ret = iotlb_translate(vrh, (u64)(uintptr_t)src,
  983. len, iov, 16, VHOST_MAP_RO);
  984. if (ret < 0)
  985. return ret;
  986. iov_iter_bvec(&iter, READ, iov, ret, len);
  987. ret = copy_from_iter(dst, len, &iter);
  988. return ret;
  989. }
  990. static inline int copy_to_iotlb(const struct vringh *vrh, void *dst,
  991. void *src, size_t len)
  992. {
  993. struct iov_iter iter;
  994. struct bio_vec iov[16];
  995. int ret;
  996. ret = iotlb_translate(vrh, (u64)(uintptr_t)dst,
  997. len, iov, 16, VHOST_MAP_WO);
  998. if (ret < 0)
  999. return ret;
  1000. iov_iter_bvec(&iter, WRITE, iov, ret, len);
  1001. return copy_to_iter(src, len, &iter);
  1002. }
  1003. static inline int getu16_iotlb(const struct vringh *vrh,
  1004. u16 *val, const __virtio16 *p)
  1005. {
  1006. struct bio_vec iov;
  1007. void *kaddr, *from;
  1008. int ret;
  1009. /* Atomic read is needed for getu16 */
  1010. ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p),
  1011. &iov, 1, VHOST_MAP_RO);
  1012. if (ret < 0)
  1013. return ret;
  1014. kaddr = kmap_atomic(iov.bv_page);
  1015. from = kaddr + iov.bv_offset;
  1016. *val = vringh16_to_cpu(vrh, READ_ONCE(*(__virtio16 *)from));
  1017. kunmap_atomic(kaddr);
  1018. return 0;
  1019. }
  1020. static inline int putu16_iotlb(const struct vringh *vrh,
  1021. __virtio16 *p, u16 val)
  1022. {
  1023. struct bio_vec iov;
  1024. void *kaddr, *to;
  1025. int ret;
  1026. /* Atomic write is needed for putu16 */
  1027. ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p),
  1028. &iov, 1, VHOST_MAP_WO);
  1029. if (ret < 0)
  1030. return ret;
  1031. kaddr = kmap_atomic(iov.bv_page);
  1032. to = kaddr + iov.bv_offset;
  1033. WRITE_ONCE(*(__virtio16 *)to, cpu_to_vringh16(vrh, val));
  1034. kunmap_atomic(kaddr);
  1035. return 0;
  1036. }
  1037. static inline int copydesc_iotlb(const struct vringh *vrh,
  1038. void *dst, const void *src, size_t len)
  1039. {
  1040. int ret;
  1041. ret = copy_from_iotlb(vrh, dst, (void *)src, len);
  1042. if (ret != len)
  1043. return -EFAULT;
  1044. return 0;
  1045. }
  1046. static inline int xfer_from_iotlb(const struct vringh *vrh, void *src,
  1047. void *dst, size_t len)
  1048. {
  1049. int ret;
  1050. ret = copy_from_iotlb(vrh, dst, src, len);
  1051. if (ret != len)
  1052. return -EFAULT;
  1053. return 0;
  1054. }
  1055. static inline int xfer_to_iotlb(const struct vringh *vrh,
  1056. void *dst, void *src, size_t len)
  1057. {
  1058. int ret;
  1059. ret = copy_to_iotlb(vrh, dst, src, len);
  1060. if (ret != len)
  1061. return -EFAULT;
  1062. return 0;
  1063. }
  1064. static inline int putused_iotlb(const struct vringh *vrh,
  1065. struct vring_used_elem *dst,
  1066. const struct vring_used_elem *src,
  1067. unsigned int num)
  1068. {
  1069. int size = num * sizeof(*dst);
  1070. int ret;
  1071. ret = copy_to_iotlb(vrh, dst, (void *)src, num * sizeof(*dst));
  1072. if (ret != size)
  1073. return -EFAULT;
  1074. return 0;
  1075. }
  1076. /**
  1077. * vringh_init_iotlb - initialize a vringh for a ring with IOTLB.
  1078. * @vrh: the vringh to initialize.
  1079. * @features: the feature bits for this ring.
  1080. * @num: the number of elements.
  1081. * @weak_barriers: true if we only need memory barriers, not I/O.
  1082. * @desc: the userpace descriptor pointer.
  1083. * @avail: the userpace avail pointer.
  1084. * @used: the userpace used pointer.
  1085. *
  1086. * Returns an error if num is invalid.
  1087. */
  1088. int vringh_init_iotlb(struct vringh *vrh, u64 features,
  1089. unsigned int num, bool weak_barriers,
  1090. struct vring_desc *desc,
  1091. struct vring_avail *avail,
  1092. struct vring_used *used)
  1093. {
  1094. return vringh_init_kern(vrh, features, num, weak_barriers,
  1095. desc, avail, used);
  1096. }
  1097. EXPORT_SYMBOL(vringh_init_iotlb);
  1098. /**
  1099. * vringh_set_iotlb - initialize a vringh for a ring with IOTLB.
  1100. * @vrh: the vring
  1101. * @iotlb: iotlb associated with this vring
  1102. */
  1103. void vringh_set_iotlb(struct vringh *vrh, struct vhost_iotlb *iotlb)
  1104. {
  1105. vrh->iotlb = iotlb;
  1106. }
  1107. EXPORT_SYMBOL(vringh_set_iotlb);
  1108. /**
  1109. * vringh_getdesc_iotlb - get next available descriptor from ring with
  1110. * IOTLB.
  1111. * @vrh: the kernelspace vring.
  1112. * @riov: where to put the readable descriptors (or NULL)
  1113. * @wiov: where to put the writable descriptors (or NULL)
  1114. * @head: head index we received, for passing to vringh_complete_iotlb().
  1115. * @gfp: flags for allocating larger riov/wiov.
  1116. *
  1117. * Returns 0 if there was no descriptor, 1 if there was, or -errno.
  1118. *
  1119. * Note that on error return, you can tell the difference between an
  1120. * invalid ring and a single invalid descriptor: in the former case,
  1121. * *head will be vrh->vring.num. You may be able to ignore an invalid
  1122. * descriptor, but there's not much you can do with an invalid ring.
  1123. *
  1124. * Note that you may need to clean up riov and wiov, even on error!
  1125. */
  1126. int vringh_getdesc_iotlb(struct vringh *vrh,
  1127. struct vringh_kiov *riov,
  1128. struct vringh_kiov *wiov,
  1129. u16 *head,
  1130. gfp_t gfp)
  1131. {
  1132. int err;
  1133. err = __vringh_get_head(vrh, getu16_iotlb, &vrh->last_avail_idx);
  1134. if (err < 0)
  1135. return err;
  1136. /* Empty... */
  1137. if (err == vrh->vring.num)
  1138. return 0;
  1139. *head = err;
  1140. err = __vringh_iov(vrh, *head, riov, wiov, no_range_check, NULL,
  1141. gfp, copydesc_iotlb);
  1142. if (err)
  1143. return err;
  1144. return 1;
  1145. }
  1146. EXPORT_SYMBOL(vringh_getdesc_iotlb);
  1147. /**
  1148. * vringh_iov_pull_iotlb - copy bytes from vring_iov.
  1149. * @vrh: the vring.
  1150. * @riov: the riov as passed to vringh_getdesc_iotlb() (updated as we consume)
  1151. * @dst: the place to copy.
  1152. * @len: the maximum length to copy.
  1153. *
  1154. * Returns the bytes copied <= len or a negative errno.
  1155. */
  1156. ssize_t vringh_iov_pull_iotlb(struct vringh *vrh,
  1157. struct vringh_kiov *riov,
  1158. void *dst, size_t len)
  1159. {
  1160. return vringh_iov_xfer(vrh, riov, dst, len, xfer_from_iotlb);
  1161. }
  1162. EXPORT_SYMBOL(vringh_iov_pull_iotlb);
  1163. /**
  1164. * vringh_iov_push_iotlb - copy bytes into vring_iov.
  1165. * @vrh: the vring.
  1166. * @wiov: the wiov as passed to vringh_getdesc_iotlb() (updated as we consume)
  1167. * @src: the place to copy from.
  1168. * @len: the maximum length to copy.
  1169. *
  1170. * Returns the bytes copied <= len or a negative errno.
  1171. */
  1172. ssize_t vringh_iov_push_iotlb(struct vringh *vrh,
  1173. struct vringh_kiov *wiov,
  1174. const void *src, size_t len)
  1175. {
  1176. return vringh_iov_xfer(vrh, wiov, (void *)src, len, xfer_to_iotlb);
  1177. }
  1178. EXPORT_SYMBOL(vringh_iov_push_iotlb);
  1179. /**
  1180. * vringh_abandon_iotlb - we've decided not to handle the descriptor(s).
  1181. * @vrh: the vring.
  1182. * @num: the number of descriptors to put back (ie. num
  1183. * vringh_get_iotlb() to undo).
  1184. *
  1185. * The next vringh_get_iotlb() will return the old descriptor(s) again.
  1186. */
  1187. void vringh_abandon_iotlb(struct vringh *vrh, unsigned int num)
  1188. {
  1189. /* We only update vring_avail_event(vr) when we want to be notified,
  1190. * so we haven't changed that yet.
  1191. */
  1192. vrh->last_avail_idx -= num;
  1193. }
  1194. EXPORT_SYMBOL(vringh_abandon_iotlb);
  1195. /**
  1196. * vringh_complete_iotlb - we've finished with descriptor, publish it.
  1197. * @vrh: the vring.
  1198. * @head: the head as filled in by vringh_getdesc_iotlb.
  1199. * @len: the length of data we have written.
  1200. *
  1201. * You should check vringh_need_notify_iotlb() after one or more calls
  1202. * to this function.
  1203. */
  1204. int vringh_complete_iotlb(struct vringh *vrh, u16 head, u32 len)
  1205. {
  1206. struct vring_used_elem used;
  1207. used.id = cpu_to_vringh32(vrh, head);
  1208. used.len = cpu_to_vringh32(vrh, len);
  1209. return __vringh_complete(vrh, &used, 1, putu16_iotlb, putused_iotlb);
  1210. }
  1211. EXPORT_SYMBOL(vringh_complete_iotlb);
  1212. /**
  1213. * vringh_notify_enable_iotlb - we want to know if something changes.
  1214. * @vrh: the vring.
  1215. *
  1216. * This always enables notifications, but returns false if there are
  1217. * now more buffers available in the vring.
  1218. */
  1219. bool vringh_notify_enable_iotlb(struct vringh *vrh)
  1220. {
  1221. return __vringh_notify_enable(vrh, getu16_iotlb, putu16_iotlb);
  1222. }
  1223. EXPORT_SYMBOL(vringh_notify_enable_iotlb);
  1224. /**
  1225. * vringh_notify_disable_iotlb - don't tell us if something changes.
  1226. * @vrh: the vring.
  1227. *
  1228. * This is our normal running state: we disable and then only enable when
  1229. * we're going to sleep.
  1230. */
  1231. void vringh_notify_disable_iotlb(struct vringh *vrh)
  1232. {
  1233. __vringh_notify_disable(vrh, putu16_iotlb);
  1234. }
  1235. EXPORT_SYMBOL(vringh_notify_disable_iotlb);
  1236. /**
  1237. * vringh_need_notify_iotlb - must we tell the other side about used buffers?
  1238. * @vrh: the vring we've called vringh_complete_iotlb() on.
  1239. *
  1240. * Returns -errno or 0 if we don't need to tell the other side, 1 if we do.
  1241. */
  1242. int vringh_need_notify_iotlb(struct vringh *vrh)
  1243. {
  1244. return __vringh_need_notify(vrh, getu16_iotlb);
  1245. }
  1246. EXPORT_SYMBOL(vringh_need_notify_iotlb);
  1247. #endif
  1248. MODULE_LICENSE("GPL");