blkback.c 41 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479
  1. /******************************************************************************
  2. *
  3. * Back-end of the driver for virtual block devices. This portion of the
  4. * driver exports a 'unified' block-device interface that can be accessed
  5. * by any operating system that implements a compatible front end. A
  6. * reference front-end implementation can be found in:
  7. * drivers/block/xen-blkfront.c
  8. *
  9. * Copyright (c) 2003-2004, Keir Fraser & Steve Hand
  10. * Copyright (c) 2005, Christopher Clark
  11. *
  12. * This program is free software; you can redistribute it and/or
  13. * modify it under the terms of the GNU General Public License version 2
  14. * as published by the Free Software Foundation; or, when distributed
  15. * separately from the Linux kernel or incorporated into other
  16. * software packages, subject to the following license:
  17. *
  18. * Permission is hereby granted, free of charge, to any person obtaining a copy
  19. * of this source file (the "Software"), to deal in the Software without
  20. * restriction, including without limitation the rights to use, copy, modify,
  21. * merge, publish, distribute, sublicense, and/or sell copies of the Software,
  22. * and to permit persons to whom the Software is furnished to do so, subject to
  23. * the following conditions:
  24. *
  25. * The above copyright notice and this permission notice shall be included in
  26. * all copies or substantial portions of the Software.
  27. *
  28. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  29. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  30. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  31. * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  32. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  33. * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  34. * IN THE SOFTWARE.
  35. */
  36. #define pr_fmt(fmt) "xen-blkback: " fmt
  37. #include <linux/spinlock.h>
  38. #include <linux/kthread.h>
  39. #include <linux/list.h>
  40. #include <linux/delay.h>
  41. #include <linux/freezer.h>
  42. #include <linux/bitmap.h>
  43. #include <xen/events.h>
  44. #include <xen/page.h>
  45. #include <xen/xen.h>
  46. #include <asm/xen/hypervisor.h>
  47. #include <asm/xen/hypercall.h>
  48. #include <xen/balloon.h>
  49. #include <xen/grant_table.h>
  50. #include "common.h"
  51. /*
  52. * Maximum number of unused free pages to keep in the internal buffer.
  53. * Setting this to a value too low will reduce memory used in each backend,
  54. * but can have a performance penalty.
  55. *
  56. * A sane value is xen_blkif_reqs * BLKIF_MAX_SEGMENTS_PER_REQUEST, but can
  57. * be set to a lower value that might degrade performance on some intensive
  58. * IO workloads.
  59. */
  60. static int max_buffer_pages = 1024;
  61. module_param_named(max_buffer_pages, max_buffer_pages, int, 0644);
  62. MODULE_PARM_DESC(max_buffer_pages,
  63. "Maximum number of free pages to keep in each block backend buffer");
  64. /*
  65. * Maximum number of grants to map persistently in blkback. For maximum
  66. * performance this should be the total numbers of grants that can be used
  67. * to fill the ring, but since this might become too high, specially with
  68. * the use of indirect descriptors, we set it to a value that provides good
  69. * performance without using too much memory.
  70. *
  71. * When the list of persistent grants is full we clean it up using a LRU
  72. * algorithm.
  73. */
  74. static int max_pgrants = 1056;
  75. module_param_named(max_persistent_grants, max_pgrants, int, 0644);
  76. MODULE_PARM_DESC(max_persistent_grants,
  77. "Maximum number of grants to map persistently");
  78. /*
  79. * How long a persistent grant is allowed to remain allocated without being in
  80. * use. The time is in seconds, 0 means indefinitely long.
  81. */
  82. static unsigned int pgrant_timeout = 60;
  83. module_param_named(persistent_grant_unused_seconds, pgrant_timeout,
  84. uint, 0644);
  85. MODULE_PARM_DESC(persistent_grant_unused_seconds,
  86. "Time in seconds an unused persistent grant is allowed to "
  87. "remain allocated. Default is 60, 0 means unlimited.");
  88. /*
  89. * Maximum number of rings/queues blkback supports, allow as many queues as there
  90. * are CPUs if user has not specified a value.
  91. */
  92. unsigned int xenblk_max_queues;
  93. module_param_named(max_queues, xenblk_max_queues, uint, 0644);
  94. MODULE_PARM_DESC(max_queues,
  95. "Maximum number of hardware queues per virtual disk." \
  96. "By default it is the number of online CPUs.");
  97. /*
  98. * Maximum order of pages to be used for the shared ring between front and
  99. * backend, 4KB page granularity is used.
  100. */
  101. unsigned int xen_blkif_max_ring_order = XENBUS_MAX_RING_GRANT_ORDER;
  102. module_param_named(max_ring_page_order, xen_blkif_max_ring_order, int, 0444);
  103. MODULE_PARM_DESC(max_ring_page_order, "Maximum order of pages to be used for the shared ring");
  104. /*
  105. * The LRU mechanism to clean the lists of persistent grants needs to
  106. * be executed periodically. The time interval between consecutive executions
  107. * of the purge mechanism is set in ms.
  108. */
  109. #define LRU_INTERVAL 100
  110. /*
  111. * When the persistent grants list is full we will remove unused grants
  112. * from the list. The percent number of grants to be removed at each LRU
  113. * execution.
  114. */
  115. #define LRU_PERCENT_CLEAN 5
  116. /* Run-time switchable: /sys/module/blkback/parameters/ */
  117. static unsigned int log_stats;
  118. module_param(log_stats, int, 0644);
  119. #define BLKBACK_INVALID_HANDLE (~0)
  120. static inline bool persistent_gnt_timeout(struct persistent_gnt *persistent_gnt)
  121. {
  122. return pgrant_timeout && (jiffies - persistent_gnt->last_used >=
  123. HZ * pgrant_timeout);
  124. }
  125. #define vaddr(page) ((unsigned long)pfn_to_kaddr(page_to_pfn(page)))
  126. static int do_block_io_op(struct xen_blkif_ring *ring, unsigned int *eoi_flags);
  127. static int dispatch_rw_block_io(struct xen_blkif_ring *ring,
  128. struct blkif_request *req,
  129. struct pending_req *pending_req);
  130. static void make_response(struct xen_blkif_ring *ring, u64 id,
  131. unsigned short op, int st);
  132. #define foreach_grant_safe(pos, n, rbtree, node) \
  133. for ((pos) = container_of(rb_first((rbtree)), typeof(*(pos)), node), \
  134. (n) = (&(pos)->node != NULL) ? rb_next(&(pos)->node) : NULL; \
  135. &(pos)->node != NULL; \
  136. (pos) = container_of(n, typeof(*(pos)), node), \
  137. (n) = (&(pos)->node != NULL) ? rb_next(&(pos)->node) : NULL)
  138. /*
  139. * We don't need locking around the persistent grant helpers
  140. * because blkback uses a single-thread for each backend, so we
  141. * can be sure that this functions will never be called recursively.
  142. *
  143. * The only exception to that is put_persistent_grant, that can be called
  144. * from interrupt context (by xen_blkbk_unmap), so we have to use atomic
  145. * bit operations to modify the flags of a persistent grant and to count
  146. * the number of used grants.
  147. */
  148. static int add_persistent_gnt(struct xen_blkif_ring *ring,
  149. struct persistent_gnt *persistent_gnt)
  150. {
  151. struct rb_node **new = NULL, *parent = NULL;
  152. struct persistent_gnt *this;
  153. struct xen_blkif *blkif = ring->blkif;
  154. if (ring->persistent_gnt_c >= max_pgrants) {
  155. if (!blkif->vbd.overflow_max_grants)
  156. blkif->vbd.overflow_max_grants = 1;
  157. return -EBUSY;
  158. }
  159. /* Figure out where to put new node */
  160. new = &ring->persistent_gnts.rb_node;
  161. while (*new) {
  162. this = container_of(*new, struct persistent_gnt, node);
  163. parent = *new;
  164. if (persistent_gnt->gnt < this->gnt)
  165. new = &((*new)->rb_left);
  166. else if (persistent_gnt->gnt > this->gnt)
  167. new = &((*new)->rb_right);
  168. else {
  169. pr_alert_ratelimited("trying to add a gref that's already in the tree\n");
  170. return -EINVAL;
  171. }
  172. }
  173. persistent_gnt->active = true;
  174. /* Add new node and rebalance tree. */
  175. rb_link_node(&(persistent_gnt->node), parent, new);
  176. rb_insert_color(&(persistent_gnt->node), &ring->persistent_gnts);
  177. ring->persistent_gnt_c++;
  178. atomic_inc(&ring->persistent_gnt_in_use);
  179. return 0;
  180. }
  181. static struct persistent_gnt *get_persistent_gnt(struct xen_blkif_ring *ring,
  182. grant_ref_t gref)
  183. {
  184. struct persistent_gnt *data;
  185. struct rb_node *node = NULL;
  186. node = ring->persistent_gnts.rb_node;
  187. while (node) {
  188. data = container_of(node, struct persistent_gnt, node);
  189. if (gref < data->gnt)
  190. node = node->rb_left;
  191. else if (gref > data->gnt)
  192. node = node->rb_right;
  193. else {
  194. if (data->active) {
  195. pr_alert_ratelimited("requesting a grant already in use\n");
  196. return NULL;
  197. }
  198. data->active = true;
  199. atomic_inc(&ring->persistent_gnt_in_use);
  200. return data;
  201. }
  202. }
  203. return NULL;
  204. }
  205. static void put_persistent_gnt(struct xen_blkif_ring *ring,
  206. struct persistent_gnt *persistent_gnt)
  207. {
  208. if (!persistent_gnt->active)
  209. pr_alert_ratelimited("freeing a grant already unused\n");
  210. persistent_gnt->last_used = jiffies;
  211. persistent_gnt->active = false;
  212. atomic_dec(&ring->persistent_gnt_in_use);
  213. }
  214. static void free_persistent_gnts(struct xen_blkif_ring *ring, struct rb_root *root,
  215. unsigned int num)
  216. {
  217. struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
  218. struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
  219. struct persistent_gnt *persistent_gnt;
  220. struct rb_node *n;
  221. int segs_to_unmap = 0;
  222. struct gntab_unmap_queue_data unmap_data;
  223. unmap_data.pages = pages;
  224. unmap_data.unmap_ops = unmap;
  225. unmap_data.kunmap_ops = NULL;
  226. foreach_grant_safe(persistent_gnt, n, root, node) {
  227. BUG_ON(persistent_gnt->handle ==
  228. BLKBACK_INVALID_HANDLE);
  229. gnttab_set_unmap_op(&unmap[segs_to_unmap],
  230. (unsigned long) pfn_to_kaddr(page_to_pfn(
  231. persistent_gnt->page)),
  232. GNTMAP_host_map,
  233. persistent_gnt->handle);
  234. pages[segs_to_unmap] = persistent_gnt->page;
  235. if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST ||
  236. !rb_next(&persistent_gnt->node)) {
  237. unmap_data.count = segs_to_unmap;
  238. BUG_ON(gnttab_unmap_refs_sync(&unmap_data));
  239. gnttab_page_cache_put(&ring->free_pages, pages,
  240. segs_to_unmap);
  241. segs_to_unmap = 0;
  242. }
  243. rb_erase(&persistent_gnt->node, root);
  244. kfree(persistent_gnt);
  245. num--;
  246. }
  247. BUG_ON(num != 0);
  248. }
  249. void xen_blkbk_unmap_purged_grants(struct work_struct *work)
  250. {
  251. struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
  252. struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
  253. struct persistent_gnt *persistent_gnt;
  254. int segs_to_unmap = 0;
  255. struct xen_blkif_ring *ring = container_of(work, typeof(*ring), persistent_purge_work);
  256. struct gntab_unmap_queue_data unmap_data;
  257. unmap_data.pages = pages;
  258. unmap_data.unmap_ops = unmap;
  259. unmap_data.kunmap_ops = NULL;
  260. while(!list_empty(&ring->persistent_purge_list)) {
  261. persistent_gnt = list_first_entry(&ring->persistent_purge_list,
  262. struct persistent_gnt,
  263. remove_node);
  264. list_del(&persistent_gnt->remove_node);
  265. gnttab_set_unmap_op(&unmap[segs_to_unmap],
  266. vaddr(persistent_gnt->page),
  267. GNTMAP_host_map,
  268. persistent_gnt->handle);
  269. pages[segs_to_unmap] = persistent_gnt->page;
  270. if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST) {
  271. unmap_data.count = segs_to_unmap;
  272. BUG_ON(gnttab_unmap_refs_sync(&unmap_data));
  273. gnttab_page_cache_put(&ring->free_pages, pages,
  274. segs_to_unmap);
  275. segs_to_unmap = 0;
  276. }
  277. kfree(persistent_gnt);
  278. }
  279. if (segs_to_unmap > 0) {
  280. unmap_data.count = segs_to_unmap;
  281. BUG_ON(gnttab_unmap_refs_sync(&unmap_data));
  282. gnttab_page_cache_put(&ring->free_pages, pages, segs_to_unmap);
  283. }
  284. }
  285. static void purge_persistent_gnt(struct xen_blkif_ring *ring)
  286. {
  287. struct persistent_gnt *persistent_gnt;
  288. struct rb_node *n;
  289. unsigned int num_clean, total;
  290. bool scan_used = false;
  291. struct rb_root *root;
  292. if (work_busy(&ring->persistent_purge_work)) {
  293. pr_alert_ratelimited("Scheduled work from previous purge is still busy, cannot purge list\n");
  294. goto out;
  295. }
  296. if (ring->persistent_gnt_c < max_pgrants ||
  297. (ring->persistent_gnt_c == max_pgrants &&
  298. !ring->blkif->vbd.overflow_max_grants)) {
  299. num_clean = 0;
  300. } else {
  301. num_clean = (max_pgrants / 100) * LRU_PERCENT_CLEAN;
  302. num_clean = ring->persistent_gnt_c - max_pgrants + num_clean;
  303. num_clean = min(ring->persistent_gnt_c, num_clean);
  304. pr_debug("Going to purge at least %u persistent grants\n",
  305. num_clean);
  306. }
  307. /*
  308. * At this point, we can assure that there will be no calls
  309. * to get_persistent_grant (because we are executing this code from
  310. * xen_blkif_schedule), there can only be calls to put_persistent_gnt,
  311. * which means that the number of currently used grants will go down,
  312. * but never up, so we will always be able to remove the requested
  313. * number of grants.
  314. */
  315. total = 0;
  316. BUG_ON(!list_empty(&ring->persistent_purge_list));
  317. root = &ring->persistent_gnts;
  318. purge_list:
  319. foreach_grant_safe(persistent_gnt, n, root, node) {
  320. BUG_ON(persistent_gnt->handle ==
  321. BLKBACK_INVALID_HANDLE);
  322. if (persistent_gnt->active)
  323. continue;
  324. if (!scan_used && !persistent_gnt_timeout(persistent_gnt))
  325. continue;
  326. if (scan_used && total >= num_clean)
  327. continue;
  328. rb_erase(&persistent_gnt->node, root);
  329. list_add(&persistent_gnt->remove_node,
  330. &ring->persistent_purge_list);
  331. total++;
  332. }
  333. /*
  334. * Check whether we also need to start cleaning
  335. * grants that were used since last purge in order to cope
  336. * with the requested num
  337. */
  338. if (!scan_used && total < num_clean) {
  339. pr_debug("Still missing %u purged frames\n", num_clean - total);
  340. scan_used = true;
  341. goto purge_list;
  342. }
  343. if (total) {
  344. ring->persistent_gnt_c -= total;
  345. ring->blkif->vbd.overflow_max_grants = 0;
  346. /* We can defer this work */
  347. schedule_work(&ring->persistent_purge_work);
  348. pr_debug("Purged %u/%u\n", num_clean, total);
  349. }
  350. out:
  351. return;
  352. }
  353. /*
  354. * Retrieve from the 'pending_reqs' a free pending_req structure to be used.
  355. */
  356. static struct pending_req *alloc_req(struct xen_blkif_ring *ring)
  357. {
  358. struct pending_req *req = NULL;
  359. unsigned long flags;
  360. spin_lock_irqsave(&ring->pending_free_lock, flags);
  361. if (!list_empty(&ring->pending_free)) {
  362. req = list_entry(ring->pending_free.next, struct pending_req,
  363. free_list);
  364. list_del(&req->free_list);
  365. }
  366. spin_unlock_irqrestore(&ring->pending_free_lock, flags);
  367. return req;
  368. }
  369. /*
  370. * Return the 'pending_req' structure back to the freepool. We also
  371. * wake up the thread if it was waiting for a free page.
  372. */
  373. static void free_req(struct xen_blkif_ring *ring, struct pending_req *req)
  374. {
  375. unsigned long flags;
  376. int was_empty;
  377. spin_lock_irqsave(&ring->pending_free_lock, flags);
  378. was_empty = list_empty(&ring->pending_free);
  379. list_add(&req->free_list, &ring->pending_free);
  380. spin_unlock_irqrestore(&ring->pending_free_lock, flags);
  381. if (was_empty)
  382. wake_up(&ring->pending_free_wq);
  383. }
  384. /*
  385. * Routines for managing virtual block devices (vbds).
  386. */
  387. static int xen_vbd_translate(struct phys_req *req, struct xen_blkif *blkif,
  388. int operation)
  389. {
  390. struct xen_vbd *vbd = &blkif->vbd;
  391. int rc = -EACCES;
  392. if ((operation != REQ_OP_READ) && vbd->readonly)
  393. goto out;
  394. if (likely(req->nr_sects)) {
  395. blkif_sector_t end = req->sector_number + req->nr_sects;
  396. if (unlikely(end < req->sector_number))
  397. goto out;
  398. if (unlikely(end > vbd_sz(vbd)))
  399. goto out;
  400. }
  401. req->dev = vbd->pdevice;
  402. req->bdev = vbd->bdev;
  403. rc = 0;
  404. out:
  405. return rc;
  406. }
  407. static void xen_vbd_resize(struct xen_blkif *blkif)
  408. {
  409. struct xen_vbd *vbd = &blkif->vbd;
  410. struct xenbus_transaction xbt;
  411. int err;
  412. struct xenbus_device *dev = xen_blkbk_xenbus(blkif->be);
  413. unsigned long long new_size = vbd_sz(vbd);
  414. pr_info("VBD Resize: Domid: %d, Device: (%d, %d)\n",
  415. blkif->domid, MAJOR(vbd->pdevice), MINOR(vbd->pdevice));
  416. pr_info("VBD Resize: new size %llu\n", new_size);
  417. vbd->size = new_size;
  418. again:
  419. err = xenbus_transaction_start(&xbt);
  420. if (err) {
  421. pr_warn("Error starting transaction\n");
  422. return;
  423. }
  424. err = xenbus_printf(xbt, dev->nodename, "sectors", "%llu",
  425. (unsigned long long)vbd_sz(vbd));
  426. if (err) {
  427. pr_warn("Error writing new size\n");
  428. goto abort;
  429. }
  430. /*
  431. * Write the current state; we will use this to synchronize
  432. * the front-end. If the current state is "connected" the
  433. * front-end will get the new size information online.
  434. */
  435. err = xenbus_printf(xbt, dev->nodename, "state", "%d", dev->state);
  436. if (err) {
  437. pr_warn("Error writing the state\n");
  438. goto abort;
  439. }
  440. err = xenbus_transaction_end(xbt, 0);
  441. if (err == -EAGAIN)
  442. goto again;
  443. if (err)
  444. pr_warn("Error ending transaction\n");
  445. return;
  446. abort:
  447. xenbus_transaction_end(xbt, 1);
  448. }
  449. /*
  450. * Notification from the guest OS.
  451. */
  452. static void blkif_notify_work(struct xen_blkif_ring *ring)
  453. {
  454. ring->waiting_reqs = 1;
  455. wake_up(&ring->wq);
  456. }
  457. irqreturn_t xen_blkif_be_int(int irq, void *dev_id)
  458. {
  459. blkif_notify_work(dev_id);
  460. return IRQ_HANDLED;
  461. }
  462. /*
  463. * SCHEDULER FUNCTIONS
  464. */
  465. static void print_stats(struct xen_blkif_ring *ring)
  466. {
  467. pr_info("(%s): oo %3llu | rd %4llu | wr %4llu | f %4llu"
  468. " | ds %4llu | pg: %4u/%4d\n",
  469. current->comm, ring->st_oo_req,
  470. ring->st_rd_req, ring->st_wr_req,
  471. ring->st_f_req, ring->st_ds_req,
  472. ring->persistent_gnt_c, max_pgrants);
  473. ring->st_print = jiffies + msecs_to_jiffies(10 * 1000);
  474. ring->st_rd_req = 0;
  475. ring->st_wr_req = 0;
  476. ring->st_oo_req = 0;
  477. ring->st_ds_req = 0;
  478. }
  479. int xen_blkif_schedule(void *arg)
  480. {
  481. struct xen_blkif_ring *ring = arg;
  482. struct xen_blkif *blkif = ring->blkif;
  483. struct xen_vbd *vbd = &blkif->vbd;
  484. unsigned long timeout;
  485. int ret;
  486. bool do_eoi;
  487. unsigned int eoi_flags = XEN_EOI_FLAG_SPURIOUS;
  488. set_freezable();
  489. while (!kthread_should_stop()) {
  490. if (try_to_freeze())
  491. continue;
  492. if (unlikely(vbd->size != vbd_sz(vbd)))
  493. xen_vbd_resize(blkif);
  494. timeout = msecs_to_jiffies(LRU_INTERVAL);
  495. timeout = wait_event_interruptible_timeout(
  496. ring->wq,
  497. ring->waiting_reqs || kthread_should_stop(),
  498. timeout);
  499. if (timeout == 0)
  500. goto purge_gnt_list;
  501. timeout = wait_event_interruptible_timeout(
  502. ring->pending_free_wq,
  503. !list_empty(&ring->pending_free) ||
  504. kthread_should_stop(),
  505. timeout);
  506. if (timeout == 0)
  507. goto purge_gnt_list;
  508. do_eoi = ring->waiting_reqs;
  509. ring->waiting_reqs = 0;
  510. smp_mb(); /* clear flag *before* checking for work */
  511. ret = do_block_io_op(ring, &eoi_flags);
  512. if (ret > 0)
  513. ring->waiting_reqs = 1;
  514. if (ret == -EACCES)
  515. wait_event_interruptible(ring->shutdown_wq,
  516. kthread_should_stop());
  517. if (do_eoi && !ring->waiting_reqs) {
  518. xen_irq_lateeoi(ring->irq, eoi_flags);
  519. eoi_flags |= XEN_EOI_FLAG_SPURIOUS;
  520. }
  521. purge_gnt_list:
  522. if (blkif->vbd.feature_gnt_persistent &&
  523. time_after(jiffies, ring->next_lru)) {
  524. purge_persistent_gnt(ring);
  525. ring->next_lru = jiffies + msecs_to_jiffies(LRU_INTERVAL);
  526. }
  527. /* Shrink the free pages pool if it is too large. */
  528. if (time_before(jiffies, blkif->buffer_squeeze_end))
  529. gnttab_page_cache_shrink(&ring->free_pages, 0);
  530. else
  531. gnttab_page_cache_shrink(&ring->free_pages,
  532. max_buffer_pages);
  533. if (log_stats && time_after(jiffies, ring->st_print))
  534. print_stats(ring);
  535. }
  536. /* Drain pending purge work */
  537. flush_work(&ring->persistent_purge_work);
  538. if (log_stats)
  539. print_stats(ring);
  540. ring->xenblkd = NULL;
  541. return 0;
  542. }
  543. /*
  544. * Remove persistent grants and empty the pool of free pages
  545. */
  546. void xen_blkbk_free_caches(struct xen_blkif_ring *ring)
  547. {
  548. /* Free all persistent grant pages */
  549. if (!RB_EMPTY_ROOT(&ring->persistent_gnts))
  550. free_persistent_gnts(ring, &ring->persistent_gnts,
  551. ring->persistent_gnt_c);
  552. BUG_ON(!RB_EMPTY_ROOT(&ring->persistent_gnts));
  553. ring->persistent_gnt_c = 0;
  554. /* Since we are shutting down remove all pages from the buffer */
  555. gnttab_page_cache_shrink(&ring->free_pages, 0 /* All */);
  556. }
  557. static unsigned int xen_blkbk_unmap_prepare(
  558. struct xen_blkif_ring *ring,
  559. struct grant_page **pages,
  560. unsigned int num,
  561. struct gnttab_unmap_grant_ref *unmap_ops,
  562. struct page **unmap_pages)
  563. {
  564. unsigned int i, invcount = 0;
  565. for (i = 0; i < num; i++) {
  566. if (pages[i]->persistent_gnt != NULL) {
  567. put_persistent_gnt(ring, pages[i]->persistent_gnt);
  568. continue;
  569. }
  570. if (pages[i]->handle == BLKBACK_INVALID_HANDLE)
  571. continue;
  572. unmap_pages[invcount] = pages[i]->page;
  573. gnttab_set_unmap_op(&unmap_ops[invcount], vaddr(pages[i]->page),
  574. GNTMAP_host_map, pages[i]->handle);
  575. pages[i]->handle = BLKBACK_INVALID_HANDLE;
  576. invcount++;
  577. }
  578. return invcount;
  579. }
  580. static void xen_blkbk_unmap_and_respond_callback(int result, struct gntab_unmap_queue_data *data)
  581. {
  582. struct pending_req *pending_req = (struct pending_req *)(data->data);
  583. struct xen_blkif_ring *ring = pending_req->ring;
  584. struct xen_blkif *blkif = ring->blkif;
  585. /* BUG_ON used to reproduce existing behaviour,
  586. but is this the best way to deal with this? */
  587. BUG_ON(result);
  588. gnttab_page_cache_put(&ring->free_pages, data->pages, data->count);
  589. make_response(ring, pending_req->id,
  590. pending_req->operation, pending_req->status);
  591. free_req(ring, pending_req);
  592. /*
  593. * Make sure the request is freed before releasing blkif,
  594. * or there could be a race between free_req and the
  595. * cleanup done in xen_blkif_free during shutdown.
  596. *
  597. * NB: The fact that we might try to wake up pending_free_wq
  598. * before drain_complete (in case there's a drain going on)
  599. * it's not a problem with our current implementation
  600. * because we can assure there's no thread waiting on
  601. * pending_free_wq if there's a drain going on, but it has
  602. * to be taken into account if the current model is changed.
  603. */
  604. if (atomic_dec_and_test(&ring->inflight) && atomic_read(&blkif->drain)) {
  605. complete(&blkif->drain_complete);
  606. }
  607. xen_blkif_put(blkif);
  608. }
  609. static void xen_blkbk_unmap_and_respond(struct pending_req *req)
  610. {
  611. struct gntab_unmap_queue_data* work = &req->gnttab_unmap_data;
  612. struct xen_blkif_ring *ring = req->ring;
  613. struct grant_page **pages = req->segments;
  614. unsigned int invcount;
  615. invcount = xen_blkbk_unmap_prepare(ring, pages, req->nr_segs,
  616. req->unmap, req->unmap_pages);
  617. work->data = req;
  618. work->done = xen_blkbk_unmap_and_respond_callback;
  619. work->unmap_ops = req->unmap;
  620. work->kunmap_ops = NULL;
  621. work->pages = req->unmap_pages;
  622. work->count = invcount;
  623. gnttab_unmap_refs_async(&req->gnttab_unmap_data);
  624. }
  625. /*
  626. * Unmap the grant references.
  627. *
  628. * This could accumulate ops up to the batch size to reduce the number
  629. * of hypercalls, but since this is only used in error paths there's
  630. * no real need.
  631. */
  632. static void xen_blkbk_unmap(struct xen_blkif_ring *ring,
  633. struct grant_page *pages[],
  634. int num)
  635. {
  636. struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
  637. struct page *unmap_pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
  638. unsigned int invcount = 0;
  639. int ret;
  640. while (num) {
  641. unsigned int batch = min(num, BLKIF_MAX_SEGMENTS_PER_REQUEST);
  642. invcount = xen_blkbk_unmap_prepare(ring, pages, batch,
  643. unmap, unmap_pages);
  644. if (invcount) {
  645. ret = gnttab_unmap_refs(unmap, NULL, unmap_pages, invcount);
  646. BUG_ON(ret);
  647. gnttab_page_cache_put(&ring->free_pages, unmap_pages,
  648. invcount);
  649. }
  650. pages += batch;
  651. num -= batch;
  652. }
  653. }
  654. static int xen_blkbk_map(struct xen_blkif_ring *ring,
  655. struct grant_page *pages[],
  656. int num, bool ro)
  657. {
  658. struct gnttab_map_grant_ref map[BLKIF_MAX_SEGMENTS_PER_REQUEST];
  659. struct page *pages_to_gnt[BLKIF_MAX_SEGMENTS_PER_REQUEST];
  660. struct persistent_gnt *persistent_gnt = NULL;
  661. phys_addr_t addr = 0;
  662. int i, seg_idx, new_map_idx;
  663. int segs_to_map = 0;
  664. int ret = 0;
  665. int last_map = 0, map_until = 0;
  666. int use_persistent_gnts;
  667. struct xen_blkif *blkif = ring->blkif;
  668. use_persistent_gnts = (blkif->vbd.feature_gnt_persistent);
  669. /*
  670. * Fill out preq.nr_sects with proper amount of sectors, and setup
  671. * assign map[..] with the PFN of the page in our domain with the
  672. * corresponding grant reference for each page.
  673. */
  674. again:
  675. for (i = map_until; i < num; i++) {
  676. uint32_t flags;
  677. if (use_persistent_gnts) {
  678. persistent_gnt = get_persistent_gnt(
  679. ring,
  680. pages[i]->gref);
  681. }
  682. if (persistent_gnt) {
  683. /*
  684. * We are using persistent grants and
  685. * the grant is already mapped
  686. */
  687. pages[i]->page = persistent_gnt->page;
  688. pages[i]->persistent_gnt = persistent_gnt;
  689. } else {
  690. if (gnttab_page_cache_get(&ring->free_pages,
  691. &pages[i]->page)) {
  692. gnttab_page_cache_put(&ring->free_pages,
  693. pages_to_gnt,
  694. segs_to_map);
  695. ret = -ENOMEM;
  696. goto out;
  697. }
  698. addr = vaddr(pages[i]->page);
  699. pages_to_gnt[segs_to_map] = pages[i]->page;
  700. pages[i]->persistent_gnt = NULL;
  701. flags = GNTMAP_host_map;
  702. if (!use_persistent_gnts && ro)
  703. flags |= GNTMAP_readonly;
  704. gnttab_set_map_op(&map[segs_to_map++], addr,
  705. flags, pages[i]->gref,
  706. blkif->domid);
  707. }
  708. map_until = i + 1;
  709. if (segs_to_map == BLKIF_MAX_SEGMENTS_PER_REQUEST)
  710. break;
  711. }
  712. if (segs_to_map)
  713. ret = gnttab_map_refs(map, NULL, pages_to_gnt, segs_to_map);
  714. /*
  715. * Now swizzle the MFN in our domain with the MFN from the other domain
  716. * so that when we access vaddr(pending_req,i) it has the contents of
  717. * the page from the other domain.
  718. */
  719. for (seg_idx = last_map, new_map_idx = 0; seg_idx < map_until; seg_idx++) {
  720. if (!pages[seg_idx]->persistent_gnt) {
  721. /* This is a newly mapped grant */
  722. BUG_ON(new_map_idx >= segs_to_map);
  723. if (unlikely(map[new_map_idx].status != 0)) {
  724. pr_debug("invalid buffer -- could not remap it\n");
  725. gnttab_page_cache_put(&ring->free_pages,
  726. &pages[seg_idx]->page, 1);
  727. pages[seg_idx]->handle = BLKBACK_INVALID_HANDLE;
  728. ret |= !ret;
  729. goto next;
  730. }
  731. pages[seg_idx]->handle = map[new_map_idx].handle;
  732. } else {
  733. continue;
  734. }
  735. if (use_persistent_gnts &&
  736. ring->persistent_gnt_c < max_pgrants) {
  737. /*
  738. * We are using persistent grants, the grant is
  739. * not mapped but we might have room for it.
  740. */
  741. persistent_gnt = kmalloc(sizeof(struct persistent_gnt),
  742. GFP_KERNEL);
  743. if (!persistent_gnt) {
  744. /*
  745. * If we don't have enough memory to
  746. * allocate the persistent_gnt struct
  747. * map this grant non-persistenly
  748. */
  749. goto next;
  750. }
  751. persistent_gnt->gnt = map[new_map_idx].ref;
  752. persistent_gnt->handle = map[new_map_idx].handle;
  753. persistent_gnt->page = pages[seg_idx]->page;
  754. if (add_persistent_gnt(ring,
  755. persistent_gnt)) {
  756. kfree(persistent_gnt);
  757. persistent_gnt = NULL;
  758. goto next;
  759. }
  760. pages[seg_idx]->persistent_gnt = persistent_gnt;
  761. pr_debug("grant %u added to the tree of persistent grants, using %u/%u\n",
  762. persistent_gnt->gnt, ring->persistent_gnt_c,
  763. max_pgrants);
  764. goto next;
  765. }
  766. if (use_persistent_gnts && !blkif->vbd.overflow_max_grants) {
  767. blkif->vbd.overflow_max_grants = 1;
  768. pr_debug("domain %u, device %#x is using maximum number of persistent grants\n",
  769. blkif->domid, blkif->vbd.handle);
  770. }
  771. /*
  772. * We could not map this grant persistently, so use it as
  773. * a non-persistent grant.
  774. */
  775. next:
  776. new_map_idx++;
  777. }
  778. segs_to_map = 0;
  779. last_map = map_until;
  780. if (!ret && map_until != num)
  781. goto again;
  782. out:
  783. for (i = last_map; i < num; i++) {
  784. /* Don't zap current batch's valid persistent grants. */
  785. if(i >= map_until)
  786. pages[i]->persistent_gnt = NULL;
  787. pages[i]->handle = BLKBACK_INVALID_HANDLE;
  788. }
  789. return ret;
  790. }
  791. static int xen_blkbk_map_seg(struct pending_req *pending_req)
  792. {
  793. int rc;
  794. rc = xen_blkbk_map(pending_req->ring, pending_req->segments,
  795. pending_req->nr_segs,
  796. (pending_req->operation != BLKIF_OP_READ));
  797. return rc;
  798. }
  799. static int xen_blkbk_parse_indirect(struct blkif_request *req,
  800. struct pending_req *pending_req,
  801. struct seg_buf seg[],
  802. struct phys_req *preq)
  803. {
  804. struct grant_page **pages = pending_req->indirect_pages;
  805. struct xen_blkif_ring *ring = pending_req->ring;
  806. int indirect_grefs, rc, n, nseg, i;
  807. struct blkif_request_segment *segments = NULL;
  808. nseg = pending_req->nr_segs;
  809. indirect_grefs = INDIRECT_PAGES(nseg);
  810. BUG_ON(indirect_grefs > BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST);
  811. for (i = 0; i < indirect_grefs; i++)
  812. pages[i]->gref = req->u.indirect.indirect_grefs[i];
  813. rc = xen_blkbk_map(ring, pages, indirect_grefs, true);
  814. if (rc)
  815. goto unmap;
  816. for (n = 0, i = 0; n < nseg; n++) {
  817. uint8_t first_sect, last_sect;
  818. if ((n % SEGS_PER_INDIRECT_FRAME) == 0) {
  819. /* Map indirect segments */
  820. if (segments)
  821. kunmap_atomic(segments);
  822. segments = kmap_atomic(pages[n/SEGS_PER_INDIRECT_FRAME]->page);
  823. }
  824. i = n % SEGS_PER_INDIRECT_FRAME;
  825. pending_req->segments[n]->gref = segments[i].gref;
  826. first_sect = READ_ONCE(segments[i].first_sect);
  827. last_sect = READ_ONCE(segments[i].last_sect);
  828. if (last_sect >= (XEN_PAGE_SIZE >> 9) || last_sect < first_sect) {
  829. rc = -EINVAL;
  830. goto unmap;
  831. }
  832. seg[n].nsec = last_sect - first_sect + 1;
  833. seg[n].offset = first_sect << 9;
  834. preq->nr_sects += seg[n].nsec;
  835. }
  836. unmap:
  837. if (segments)
  838. kunmap_atomic(segments);
  839. xen_blkbk_unmap(ring, pages, indirect_grefs);
  840. return rc;
  841. }
  842. static int dispatch_discard_io(struct xen_blkif_ring *ring,
  843. struct blkif_request *req)
  844. {
  845. int err = 0;
  846. int status = BLKIF_RSP_OKAY;
  847. struct xen_blkif *blkif = ring->blkif;
  848. struct block_device *bdev = blkif->vbd.bdev;
  849. unsigned long secure;
  850. struct phys_req preq;
  851. xen_blkif_get(blkif);
  852. preq.sector_number = req->u.discard.sector_number;
  853. preq.nr_sects = req->u.discard.nr_sectors;
  854. err = xen_vbd_translate(&preq, blkif, REQ_OP_WRITE);
  855. if (err) {
  856. pr_warn("access denied: DISCARD [%llu->%llu] on dev=%04x\n",
  857. preq.sector_number,
  858. preq.sector_number + preq.nr_sects, blkif->vbd.pdevice);
  859. goto fail_response;
  860. }
  861. ring->st_ds_req++;
  862. secure = (blkif->vbd.discard_secure &&
  863. (req->u.discard.flag & BLKIF_DISCARD_SECURE)) ?
  864. BLKDEV_DISCARD_SECURE : 0;
  865. err = blkdev_issue_discard(bdev, req->u.discard.sector_number,
  866. req->u.discard.nr_sectors,
  867. GFP_KERNEL, secure);
  868. fail_response:
  869. if (err == -EOPNOTSUPP) {
  870. pr_debug("discard op failed, not supported\n");
  871. status = BLKIF_RSP_EOPNOTSUPP;
  872. } else if (err)
  873. status = BLKIF_RSP_ERROR;
  874. make_response(ring, req->u.discard.id, req->operation, status);
  875. xen_blkif_put(blkif);
  876. return err;
  877. }
  878. static int dispatch_other_io(struct xen_blkif_ring *ring,
  879. struct blkif_request *req,
  880. struct pending_req *pending_req)
  881. {
  882. free_req(ring, pending_req);
  883. make_response(ring, req->u.other.id, req->operation,
  884. BLKIF_RSP_EOPNOTSUPP);
  885. return -EIO;
  886. }
  887. static void xen_blk_drain_io(struct xen_blkif_ring *ring)
  888. {
  889. struct xen_blkif *blkif = ring->blkif;
  890. atomic_set(&blkif->drain, 1);
  891. do {
  892. if (atomic_read(&ring->inflight) == 0)
  893. break;
  894. wait_for_completion_interruptible_timeout(
  895. &blkif->drain_complete, HZ);
  896. if (!atomic_read(&blkif->drain))
  897. break;
  898. } while (!kthread_should_stop());
  899. atomic_set(&blkif->drain, 0);
  900. }
  901. static void __end_block_io_op(struct pending_req *pending_req,
  902. blk_status_t error)
  903. {
  904. /* An error fails the entire request. */
  905. if (pending_req->operation == BLKIF_OP_FLUSH_DISKCACHE &&
  906. error == BLK_STS_NOTSUPP) {
  907. pr_debug("flush diskcache op failed, not supported\n");
  908. xen_blkbk_flush_diskcache(XBT_NIL, pending_req->ring->blkif->be, 0);
  909. pending_req->status = BLKIF_RSP_EOPNOTSUPP;
  910. } else if (pending_req->operation == BLKIF_OP_WRITE_BARRIER &&
  911. error == BLK_STS_NOTSUPP) {
  912. pr_debug("write barrier op failed, not supported\n");
  913. xen_blkbk_barrier(XBT_NIL, pending_req->ring->blkif->be, 0);
  914. pending_req->status = BLKIF_RSP_EOPNOTSUPP;
  915. } else if (error) {
  916. pr_debug("Buffer not up-to-date at end of operation,"
  917. " error=%d\n", error);
  918. pending_req->status = BLKIF_RSP_ERROR;
  919. }
  920. /*
  921. * If all of the bio's have completed it is time to unmap
  922. * the grant references associated with 'request' and provide
  923. * the proper response on the ring.
  924. */
  925. if (atomic_dec_and_test(&pending_req->pendcnt))
  926. xen_blkbk_unmap_and_respond(pending_req);
  927. }
  928. /*
  929. * bio callback.
  930. */
  931. static void end_block_io_op(struct bio *bio)
  932. {
  933. __end_block_io_op(bio->bi_private, bio->bi_status);
  934. bio_put(bio);
  935. }
  936. /*
  937. * Function to copy the from the ring buffer the 'struct blkif_request'
  938. * (which has the sectors we want, number of them, grant references, etc),
  939. * and transmute it to the block API to hand it over to the proper block disk.
  940. */
  941. static int
  942. __do_block_io_op(struct xen_blkif_ring *ring, unsigned int *eoi_flags)
  943. {
  944. union blkif_back_rings *blk_rings = &ring->blk_rings;
  945. struct blkif_request req;
  946. struct pending_req *pending_req;
  947. RING_IDX rc, rp;
  948. int more_to_do = 0;
  949. rc = blk_rings->common.req_cons;
  950. rp = blk_rings->common.sring->req_prod;
  951. rmb(); /* Ensure we see queued requests up to 'rp'. */
  952. if (RING_REQUEST_PROD_OVERFLOW(&blk_rings->common, rp)) {
  953. rc = blk_rings->common.rsp_prod_pvt;
  954. pr_warn("Frontend provided bogus ring requests (%d - %d = %d). Halting ring processing on dev=%04x\n",
  955. rp, rc, rp - rc, ring->blkif->vbd.pdevice);
  956. return -EACCES;
  957. }
  958. while (rc != rp) {
  959. if (RING_REQUEST_CONS_OVERFLOW(&blk_rings->common, rc))
  960. break;
  961. /* We've seen a request, so clear spurious eoi flag. */
  962. *eoi_flags &= ~XEN_EOI_FLAG_SPURIOUS;
  963. if (kthread_should_stop()) {
  964. more_to_do = 1;
  965. break;
  966. }
  967. pending_req = alloc_req(ring);
  968. if (NULL == pending_req) {
  969. ring->st_oo_req++;
  970. more_to_do = 1;
  971. break;
  972. }
  973. switch (ring->blkif->blk_protocol) {
  974. case BLKIF_PROTOCOL_NATIVE:
  975. memcpy(&req, RING_GET_REQUEST(&blk_rings->native, rc), sizeof(req));
  976. break;
  977. case BLKIF_PROTOCOL_X86_32:
  978. blkif_get_x86_32_req(&req, RING_GET_REQUEST(&blk_rings->x86_32, rc));
  979. break;
  980. case BLKIF_PROTOCOL_X86_64:
  981. blkif_get_x86_64_req(&req, RING_GET_REQUEST(&blk_rings->x86_64, rc));
  982. break;
  983. default:
  984. BUG();
  985. }
  986. blk_rings->common.req_cons = ++rc; /* before make_response() */
  987. /* Apply all sanity checks to /private copy/ of request. */
  988. barrier();
  989. switch (req.operation) {
  990. case BLKIF_OP_READ:
  991. case BLKIF_OP_WRITE:
  992. case BLKIF_OP_WRITE_BARRIER:
  993. case BLKIF_OP_FLUSH_DISKCACHE:
  994. case BLKIF_OP_INDIRECT:
  995. if (dispatch_rw_block_io(ring, &req, pending_req))
  996. goto done;
  997. break;
  998. case BLKIF_OP_DISCARD:
  999. free_req(ring, pending_req);
  1000. if (dispatch_discard_io(ring, &req))
  1001. goto done;
  1002. break;
  1003. default:
  1004. if (dispatch_other_io(ring, &req, pending_req))
  1005. goto done;
  1006. break;
  1007. }
  1008. /* Yield point for this unbounded loop. */
  1009. cond_resched();
  1010. }
  1011. done:
  1012. return more_to_do;
  1013. }
  1014. static int
  1015. do_block_io_op(struct xen_blkif_ring *ring, unsigned int *eoi_flags)
  1016. {
  1017. union blkif_back_rings *blk_rings = &ring->blk_rings;
  1018. int more_to_do;
  1019. do {
  1020. more_to_do = __do_block_io_op(ring, eoi_flags);
  1021. if (more_to_do)
  1022. break;
  1023. RING_FINAL_CHECK_FOR_REQUESTS(&blk_rings->common, more_to_do);
  1024. } while (more_to_do);
  1025. return more_to_do;
  1026. }
  1027. /*
  1028. * Transmutation of the 'struct blkif_request' to a proper 'struct bio'
  1029. * and call the 'submit_bio' to pass it to the underlying storage.
  1030. */
  1031. static int dispatch_rw_block_io(struct xen_blkif_ring *ring,
  1032. struct blkif_request *req,
  1033. struct pending_req *pending_req)
  1034. {
  1035. struct phys_req preq;
  1036. struct seg_buf *seg = pending_req->seg;
  1037. unsigned int nseg;
  1038. struct bio *bio = NULL;
  1039. struct bio **biolist = pending_req->biolist;
  1040. int i, nbio = 0;
  1041. int operation;
  1042. int operation_flags = 0;
  1043. struct blk_plug plug;
  1044. bool drain = false;
  1045. struct grant_page **pages = pending_req->segments;
  1046. unsigned short req_operation;
  1047. req_operation = req->operation == BLKIF_OP_INDIRECT ?
  1048. req->u.indirect.indirect_op : req->operation;
  1049. if ((req->operation == BLKIF_OP_INDIRECT) &&
  1050. (req_operation != BLKIF_OP_READ) &&
  1051. (req_operation != BLKIF_OP_WRITE)) {
  1052. pr_debug("Invalid indirect operation (%u)\n", req_operation);
  1053. goto fail_response;
  1054. }
  1055. switch (req_operation) {
  1056. case BLKIF_OP_READ:
  1057. ring->st_rd_req++;
  1058. operation = REQ_OP_READ;
  1059. break;
  1060. case BLKIF_OP_WRITE:
  1061. ring->st_wr_req++;
  1062. operation = REQ_OP_WRITE;
  1063. operation_flags = REQ_SYNC | REQ_IDLE;
  1064. break;
  1065. case BLKIF_OP_WRITE_BARRIER:
  1066. drain = true;
  1067. fallthrough;
  1068. case BLKIF_OP_FLUSH_DISKCACHE:
  1069. ring->st_f_req++;
  1070. operation = REQ_OP_WRITE;
  1071. operation_flags = REQ_PREFLUSH;
  1072. break;
  1073. default:
  1074. operation = 0; /* make gcc happy */
  1075. goto fail_response;
  1076. break;
  1077. }
  1078. /* Check that the number of segments is sane. */
  1079. nseg = req->operation == BLKIF_OP_INDIRECT ?
  1080. req->u.indirect.nr_segments : req->u.rw.nr_segments;
  1081. if (unlikely(nseg == 0 && operation_flags != REQ_PREFLUSH) ||
  1082. unlikely((req->operation != BLKIF_OP_INDIRECT) &&
  1083. (nseg > BLKIF_MAX_SEGMENTS_PER_REQUEST)) ||
  1084. unlikely((req->operation == BLKIF_OP_INDIRECT) &&
  1085. (nseg > MAX_INDIRECT_SEGMENTS))) {
  1086. pr_debug("Bad number of segments in request (%d)\n", nseg);
  1087. /* Haven't submitted any bio's yet. */
  1088. goto fail_response;
  1089. }
  1090. preq.nr_sects = 0;
  1091. pending_req->ring = ring;
  1092. pending_req->id = req->u.rw.id;
  1093. pending_req->operation = req_operation;
  1094. pending_req->status = BLKIF_RSP_OKAY;
  1095. pending_req->nr_segs = nseg;
  1096. if (req->operation != BLKIF_OP_INDIRECT) {
  1097. preq.dev = req->u.rw.handle;
  1098. preq.sector_number = req->u.rw.sector_number;
  1099. for (i = 0; i < nseg; i++) {
  1100. pages[i]->gref = req->u.rw.seg[i].gref;
  1101. seg[i].nsec = req->u.rw.seg[i].last_sect -
  1102. req->u.rw.seg[i].first_sect + 1;
  1103. seg[i].offset = (req->u.rw.seg[i].first_sect << 9);
  1104. if ((req->u.rw.seg[i].last_sect >= (XEN_PAGE_SIZE >> 9)) ||
  1105. (req->u.rw.seg[i].last_sect <
  1106. req->u.rw.seg[i].first_sect))
  1107. goto fail_response;
  1108. preq.nr_sects += seg[i].nsec;
  1109. }
  1110. } else {
  1111. preq.dev = req->u.indirect.handle;
  1112. preq.sector_number = req->u.indirect.sector_number;
  1113. if (xen_blkbk_parse_indirect(req, pending_req, seg, &preq))
  1114. goto fail_response;
  1115. }
  1116. if (xen_vbd_translate(&preq, ring->blkif, operation) != 0) {
  1117. pr_debug("access denied: %s of [%llu,%llu] on dev=%04x\n",
  1118. operation == REQ_OP_READ ? "read" : "write",
  1119. preq.sector_number,
  1120. preq.sector_number + preq.nr_sects,
  1121. ring->blkif->vbd.pdevice);
  1122. goto fail_response;
  1123. }
  1124. /*
  1125. * This check _MUST_ be done after xen_vbd_translate as the preq.bdev
  1126. * is set there.
  1127. */
  1128. for (i = 0; i < nseg; i++) {
  1129. if (((int)preq.sector_number|(int)seg[i].nsec) &
  1130. ((bdev_logical_block_size(preq.bdev) >> 9) - 1)) {
  1131. pr_debug("Misaligned I/O request from domain %d\n",
  1132. ring->blkif->domid);
  1133. goto fail_response;
  1134. }
  1135. }
  1136. /* Wait on all outstanding I/O's and once that has been completed
  1137. * issue the flush.
  1138. */
  1139. if (drain)
  1140. xen_blk_drain_io(pending_req->ring);
  1141. /*
  1142. * If we have failed at this point, we need to undo the M2P override,
  1143. * set gnttab_set_unmap_op on all of the grant references and perform
  1144. * the hypercall to unmap the grants - that is all done in
  1145. * xen_blkbk_unmap.
  1146. */
  1147. if (xen_blkbk_map_seg(pending_req))
  1148. goto fail_flush;
  1149. /*
  1150. * This corresponding xen_blkif_put is done in __end_block_io_op, or
  1151. * below (in "!bio") if we are handling a BLKIF_OP_DISCARD.
  1152. */
  1153. xen_blkif_get(ring->blkif);
  1154. atomic_inc(&ring->inflight);
  1155. for (i = 0; i < nseg; i++) {
  1156. while ((bio == NULL) ||
  1157. (bio_add_page(bio,
  1158. pages[i]->page,
  1159. seg[i].nsec << 9,
  1160. seg[i].offset) == 0)) {
  1161. int nr_iovecs = min_t(int, (nseg-i), BIO_MAX_PAGES);
  1162. bio = bio_alloc(GFP_KERNEL, nr_iovecs);
  1163. if (unlikely(bio == NULL))
  1164. goto fail_put_bio;
  1165. biolist[nbio++] = bio;
  1166. bio_set_dev(bio, preq.bdev);
  1167. bio->bi_private = pending_req;
  1168. bio->bi_end_io = end_block_io_op;
  1169. bio->bi_iter.bi_sector = preq.sector_number;
  1170. bio_set_op_attrs(bio, operation, operation_flags);
  1171. }
  1172. preq.sector_number += seg[i].nsec;
  1173. }
  1174. /* This will be hit if the operation was a flush or discard. */
  1175. if (!bio) {
  1176. BUG_ON(operation_flags != REQ_PREFLUSH);
  1177. bio = bio_alloc(GFP_KERNEL, 0);
  1178. if (unlikely(bio == NULL))
  1179. goto fail_put_bio;
  1180. biolist[nbio++] = bio;
  1181. bio_set_dev(bio, preq.bdev);
  1182. bio->bi_private = pending_req;
  1183. bio->bi_end_io = end_block_io_op;
  1184. bio_set_op_attrs(bio, operation, operation_flags);
  1185. }
  1186. atomic_set(&pending_req->pendcnt, nbio);
  1187. blk_start_plug(&plug);
  1188. for (i = 0; i < nbio; i++)
  1189. submit_bio(biolist[i]);
  1190. /* Let the I/Os go.. */
  1191. blk_finish_plug(&plug);
  1192. if (operation == REQ_OP_READ)
  1193. ring->st_rd_sect += preq.nr_sects;
  1194. else if (operation == REQ_OP_WRITE)
  1195. ring->st_wr_sect += preq.nr_sects;
  1196. return 0;
  1197. fail_flush:
  1198. xen_blkbk_unmap(ring, pending_req->segments,
  1199. pending_req->nr_segs);
  1200. fail_response:
  1201. /* Haven't submitted any bio's yet. */
  1202. make_response(ring, req->u.rw.id, req_operation, BLKIF_RSP_ERROR);
  1203. free_req(ring, pending_req);
  1204. msleep(1); /* back off a bit */
  1205. return -EIO;
  1206. fail_put_bio:
  1207. for (i = 0; i < nbio; i++)
  1208. bio_put(biolist[i]);
  1209. atomic_set(&pending_req->pendcnt, 1);
  1210. __end_block_io_op(pending_req, BLK_STS_RESOURCE);
  1211. msleep(1); /* back off a bit */
  1212. return -EIO;
  1213. }
  1214. /*
  1215. * Put a response on the ring on how the operation fared.
  1216. */
  1217. static void make_response(struct xen_blkif_ring *ring, u64 id,
  1218. unsigned short op, int st)
  1219. {
  1220. struct blkif_response *resp;
  1221. unsigned long flags;
  1222. union blkif_back_rings *blk_rings;
  1223. int notify;
  1224. spin_lock_irqsave(&ring->blk_ring_lock, flags);
  1225. blk_rings = &ring->blk_rings;
  1226. /* Place on the response ring for the relevant domain. */
  1227. switch (ring->blkif->blk_protocol) {
  1228. case BLKIF_PROTOCOL_NATIVE:
  1229. resp = RING_GET_RESPONSE(&blk_rings->native,
  1230. blk_rings->native.rsp_prod_pvt);
  1231. break;
  1232. case BLKIF_PROTOCOL_X86_32:
  1233. resp = RING_GET_RESPONSE(&blk_rings->x86_32,
  1234. blk_rings->x86_32.rsp_prod_pvt);
  1235. break;
  1236. case BLKIF_PROTOCOL_X86_64:
  1237. resp = RING_GET_RESPONSE(&blk_rings->x86_64,
  1238. blk_rings->x86_64.rsp_prod_pvt);
  1239. break;
  1240. default:
  1241. BUG();
  1242. }
  1243. resp->id = id;
  1244. resp->operation = op;
  1245. resp->status = st;
  1246. blk_rings->common.rsp_prod_pvt++;
  1247. RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&blk_rings->common, notify);
  1248. spin_unlock_irqrestore(&ring->blk_ring_lock, flags);
  1249. if (notify)
  1250. notify_remote_via_irq(ring->irq);
  1251. }
  1252. static int __init xen_blkif_init(void)
  1253. {
  1254. int rc = 0;
  1255. if (!xen_domain())
  1256. return -ENODEV;
  1257. if (xen_blkif_max_ring_order > XENBUS_MAX_RING_GRANT_ORDER) {
  1258. pr_info("Invalid max_ring_order (%d), will use default max: %d.\n",
  1259. xen_blkif_max_ring_order, XENBUS_MAX_RING_GRANT_ORDER);
  1260. xen_blkif_max_ring_order = XENBUS_MAX_RING_GRANT_ORDER;
  1261. }
  1262. if (xenblk_max_queues == 0)
  1263. xenblk_max_queues = num_online_cpus();
  1264. rc = xen_blkif_interface_init();
  1265. if (rc)
  1266. goto failed_init;
  1267. rc = xen_blkif_xenbus_init();
  1268. if (rc)
  1269. goto failed_init;
  1270. failed_init:
  1271. return rc;
  1272. }
  1273. module_init(xen_blkif_init);
  1274. static void __exit xen_blkif_fini(void)
  1275. {
  1276. xen_blkif_xenbus_fini();
  1277. xen_blkif_interface_fini();
  1278. }
  1279. module_exit(xen_blkif_fini);
  1280. MODULE_LICENSE("Dual BSD/GPL");
  1281. MODULE_ALIAS("xen-backend:vbd");