null_blk_main.c 49 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Add configfs and memory store: Kyungchan Koh <kkc6196@fb.com> and
  4. * Shaohua Li <shli@fb.com>
  5. */
  6. #include <linux/module.h>
  7. #include <linux/moduleparam.h>
  8. #include <linux/sched.h>
  9. #include <linux/fs.h>
  10. #include <linux/init.h>
  11. #include "null_blk.h"
  12. #define PAGE_SECTORS_SHIFT (PAGE_SHIFT - SECTOR_SHIFT)
  13. #define PAGE_SECTORS (1 << PAGE_SECTORS_SHIFT)
  14. #define SECTOR_MASK (PAGE_SECTORS - 1)
  15. #define FREE_BATCH 16
  16. #define TICKS_PER_SEC 50ULL
  17. #define TIMER_INTERVAL (NSEC_PER_SEC / TICKS_PER_SEC)
  18. #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
  19. static DECLARE_FAULT_ATTR(null_timeout_attr);
  20. static DECLARE_FAULT_ATTR(null_requeue_attr);
  21. static DECLARE_FAULT_ATTR(null_init_hctx_attr);
  22. #endif
  23. static inline u64 mb_per_tick(int mbps)
  24. {
  25. return (1 << 20) / TICKS_PER_SEC * ((u64) mbps);
  26. }
  27. /*
  28. * Status flags for nullb_device.
  29. *
  30. * CONFIGURED: Device has been configured and turned on. Cannot reconfigure.
  31. * UP: Device is currently on and visible in userspace.
  32. * THROTTLED: Device is being throttled.
  33. * CACHE: Device is using a write-back cache.
  34. */
  35. enum nullb_device_flags {
  36. NULLB_DEV_FL_CONFIGURED = 0,
  37. NULLB_DEV_FL_UP = 1,
  38. NULLB_DEV_FL_THROTTLED = 2,
  39. NULLB_DEV_FL_CACHE = 3,
  40. };
  41. #define MAP_SZ ((PAGE_SIZE >> SECTOR_SHIFT) + 2)
  42. /*
  43. * nullb_page is a page in memory for nullb devices.
  44. *
  45. * @page: The page holding the data.
  46. * @bitmap: The bitmap represents which sector in the page has data.
  47. * Each bit represents one block size. For example, sector 8
  48. * will use the 7th bit
  49. * The highest 2 bits of bitmap are for special purpose. LOCK means the cache
  50. * page is being flushing to storage. FREE means the cache page is freed and
  51. * should be skipped from flushing to storage. Please see
  52. * null_make_cache_space
  53. */
  54. struct nullb_page {
  55. struct page *page;
  56. DECLARE_BITMAP(bitmap, MAP_SZ);
  57. };
  58. #define NULLB_PAGE_LOCK (MAP_SZ - 1)
  59. #define NULLB_PAGE_FREE (MAP_SZ - 2)
  60. static LIST_HEAD(nullb_list);
  61. static struct mutex lock;
  62. static int null_major;
  63. static DEFINE_IDA(nullb_indexes);
  64. static struct blk_mq_tag_set tag_set;
  65. enum {
  66. NULL_IRQ_NONE = 0,
  67. NULL_IRQ_SOFTIRQ = 1,
  68. NULL_IRQ_TIMER = 2,
  69. };
  70. enum {
  71. NULL_Q_BIO = 0,
  72. NULL_Q_RQ = 1,
  73. NULL_Q_MQ = 2,
  74. };
  75. static int g_no_sched;
  76. module_param_named(no_sched, g_no_sched, int, 0444);
  77. MODULE_PARM_DESC(no_sched, "No io scheduler");
  78. static int g_submit_queues = 1;
  79. module_param_named(submit_queues, g_submit_queues, int, 0444);
  80. MODULE_PARM_DESC(submit_queues, "Number of submission queues");
  81. static int g_home_node = NUMA_NO_NODE;
  82. module_param_named(home_node, g_home_node, int, 0444);
  83. MODULE_PARM_DESC(home_node, "Home node for the device");
  84. #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
  85. /*
  86. * For more details about fault injection, please refer to
  87. * Documentation/fault-injection/fault-injection.rst.
  88. */
  89. static char g_timeout_str[80];
  90. module_param_string(timeout, g_timeout_str, sizeof(g_timeout_str), 0444);
  91. MODULE_PARM_DESC(timeout, "Fault injection. timeout=<interval>,<probability>,<space>,<times>");
  92. static char g_requeue_str[80];
  93. module_param_string(requeue, g_requeue_str, sizeof(g_requeue_str), 0444);
  94. MODULE_PARM_DESC(requeue, "Fault injection. requeue=<interval>,<probability>,<space>,<times>");
  95. static char g_init_hctx_str[80];
  96. module_param_string(init_hctx, g_init_hctx_str, sizeof(g_init_hctx_str), 0444);
  97. MODULE_PARM_DESC(init_hctx, "Fault injection to fail hctx init. init_hctx=<interval>,<probability>,<space>,<times>");
  98. #endif
  99. static int g_queue_mode = NULL_Q_MQ;
  100. static int null_param_store_val(const char *str, int *val, int min, int max)
  101. {
  102. int ret, new_val;
  103. ret = kstrtoint(str, 10, &new_val);
  104. if (ret)
  105. return -EINVAL;
  106. if (new_val < min || new_val > max)
  107. return -EINVAL;
  108. *val = new_val;
  109. return 0;
  110. }
  111. static int null_set_queue_mode(const char *str, const struct kernel_param *kp)
  112. {
  113. return null_param_store_val(str, &g_queue_mode, NULL_Q_BIO, NULL_Q_MQ);
  114. }
  115. static const struct kernel_param_ops null_queue_mode_param_ops = {
  116. .set = null_set_queue_mode,
  117. .get = param_get_int,
  118. };
  119. device_param_cb(queue_mode, &null_queue_mode_param_ops, &g_queue_mode, 0444);
  120. MODULE_PARM_DESC(queue_mode, "Block interface to use (0=bio,1=rq,2=multiqueue)");
  121. static int g_gb = 250;
  122. module_param_named(gb, g_gb, int, 0444);
  123. MODULE_PARM_DESC(gb, "Size in GB");
  124. static int g_bs = 512;
  125. module_param_named(bs, g_bs, int, 0444);
  126. MODULE_PARM_DESC(bs, "Block size (in bytes)");
  127. static unsigned int nr_devices = 1;
  128. module_param(nr_devices, uint, 0444);
  129. MODULE_PARM_DESC(nr_devices, "Number of devices to register");
  130. static bool g_blocking;
  131. module_param_named(blocking, g_blocking, bool, 0444);
  132. MODULE_PARM_DESC(blocking, "Register as a blocking blk-mq driver device");
  133. static bool shared_tags;
  134. module_param(shared_tags, bool, 0444);
  135. MODULE_PARM_DESC(shared_tags, "Share tag set between devices for blk-mq");
  136. static bool g_shared_tag_bitmap;
  137. module_param_named(shared_tag_bitmap, g_shared_tag_bitmap, bool, 0444);
  138. MODULE_PARM_DESC(shared_tag_bitmap, "Use shared tag bitmap for all submission queues for blk-mq");
  139. static int g_irqmode = NULL_IRQ_SOFTIRQ;
  140. static int null_set_irqmode(const char *str, const struct kernel_param *kp)
  141. {
  142. return null_param_store_val(str, &g_irqmode, NULL_IRQ_NONE,
  143. NULL_IRQ_TIMER);
  144. }
  145. static const struct kernel_param_ops null_irqmode_param_ops = {
  146. .set = null_set_irqmode,
  147. .get = param_get_int,
  148. };
  149. device_param_cb(irqmode, &null_irqmode_param_ops, &g_irqmode, 0444);
  150. MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
  151. static unsigned long g_completion_nsec = 10000;
  152. module_param_named(completion_nsec, g_completion_nsec, ulong, 0444);
  153. MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns");
  154. static int g_hw_queue_depth = 64;
  155. module_param_named(hw_queue_depth, g_hw_queue_depth, int, 0444);
  156. MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64");
  157. static bool g_use_per_node_hctx;
  158. module_param_named(use_per_node_hctx, g_use_per_node_hctx, bool, 0444);
  159. MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: false");
  160. static bool g_zoned;
  161. module_param_named(zoned, g_zoned, bool, S_IRUGO);
  162. MODULE_PARM_DESC(zoned, "Make device as a host-managed zoned block device. Default: false");
  163. static unsigned long g_zone_size = 256;
  164. module_param_named(zone_size, g_zone_size, ulong, S_IRUGO);
  165. MODULE_PARM_DESC(zone_size, "Zone size in MB when block device is zoned. Must be power-of-two: Default: 256");
  166. static unsigned long g_zone_capacity;
  167. module_param_named(zone_capacity, g_zone_capacity, ulong, 0444);
  168. MODULE_PARM_DESC(zone_capacity, "Zone capacity in MB when block device is zoned. Can be less than or equal to zone size. Default: Zone size");
  169. static unsigned int g_zone_nr_conv;
  170. module_param_named(zone_nr_conv, g_zone_nr_conv, uint, 0444);
  171. MODULE_PARM_DESC(zone_nr_conv, "Number of conventional zones when block device is zoned. Default: 0");
  172. static unsigned int g_zone_max_open;
  173. module_param_named(zone_max_open, g_zone_max_open, uint, 0444);
  174. MODULE_PARM_DESC(zone_max_open, "Maximum number of open zones when block device is zoned. Default: 0 (no limit)");
  175. static unsigned int g_zone_max_active;
  176. module_param_named(zone_max_active, g_zone_max_active, uint, 0444);
  177. MODULE_PARM_DESC(zone_max_active, "Maximum number of active zones when block device is zoned. Default: 0 (no limit)");
  178. static struct nullb_device *null_alloc_dev(void);
  179. static void null_free_dev(struct nullb_device *dev);
  180. static void null_del_dev(struct nullb *nullb);
  181. static int null_add_dev(struct nullb_device *dev);
  182. static void null_free_device_storage(struct nullb_device *dev, bool is_cache);
  183. static inline struct nullb_device *to_nullb_device(struct config_item *item)
  184. {
  185. return item ? container_of(item, struct nullb_device, item) : NULL;
  186. }
  187. static inline ssize_t nullb_device_uint_attr_show(unsigned int val, char *page)
  188. {
  189. return snprintf(page, PAGE_SIZE, "%u\n", val);
  190. }
  191. static inline ssize_t nullb_device_ulong_attr_show(unsigned long val,
  192. char *page)
  193. {
  194. return snprintf(page, PAGE_SIZE, "%lu\n", val);
  195. }
  196. static inline ssize_t nullb_device_bool_attr_show(bool val, char *page)
  197. {
  198. return snprintf(page, PAGE_SIZE, "%u\n", val);
  199. }
  200. static ssize_t nullb_device_uint_attr_store(unsigned int *val,
  201. const char *page, size_t count)
  202. {
  203. unsigned int tmp;
  204. int result;
  205. result = kstrtouint(page, 0, &tmp);
  206. if (result < 0)
  207. return result;
  208. *val = tmp;
  209. return count;
  210. }
  211. static ssize_t nullb_device_ulong_attr_store(unsigned long *val,
  212. const char *page, size_t count)
  213. {
  214. int result;
  215. unsigned long tmp;
  216. result = kstrtoul(page, 0, &tmp);
  217. if (result < 0)
  218. return result;
  219. *val = tmp;
  220. return count;
  221. }
  222. static ssize_t nullb_device_bool_attr_store(bool *val, const char *page,
  223. size_t count)
  224. {
  225. bool tmp;
  226. int result;
  227. result = kstrtobool(page, &tmp);
  228. if (result < 0)
  229. return result;
  230. *val = tmp;
  231. return count;
  232. }
  233. /* The following macro should only be used with TYPE = {uint, ulong, bool}. */
  234. #define NULLB_DEVICE_ATTR(NAME, TYPE, APPLY) \
  235. static ssize_t \
  236. nullb_device_##NAME##_show(struct config_item *item, char *page) \
  237. { \
  238. return nullb_device_##TYPE##_attr_show( \
  239. to_nullb_device(item)->NAME, page); \
  240. } \
  241. static ssize_t \
  242. nullb_device_##NAME##_store(struct config_item *item, const char *page, \
  243. size_t count) \
  244. { \
  245. int (*apply_fn)(struct nullb_device *dev, TYPE new_value) = APPLY;\
  246. struct nullb_device *dev = to_nullb_device(item); \
  247. TYPE new_value = 0; \
  248. int ret; \
  249. \
  250. ret = nullb_device_##TYPE##_attr_store(&new_value, page, count);\
  251. if (ret < 0) \
  252. return ret; \
  253. if (apply_fn) \
  254. ret = apply_fn(dev, new_value); \
  255. else if (test_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags)) \
  256. ret = -EBUSY; \
  257. if (ret < 0) \
  258. return ret; \
  259. dev->NAME = new_value; \
  260. return count; \
  261. } \
  262. CONFIGFS_ATTR(nullb_device_, NAME);
  263. static int nullb_apply_submit_queues(struct nullb_device *dev,
  264. unsigned int submit_queues)
  265. {
  266. struct nullb *nullb = dev->nullb;
  267. struct blk_mq_tag_set *set;
  268. if (!nullb)
  269. return 0;
  270. /*
  271. * Make sure that null_init_hctx() does not access nullb->queues[] past
  272. * the end of that array.
  273. */
  274. if (submit_queues > nr_cpu_ids)
  275. return -EINVAL;
  276. set = nullb->tag_set;
  277. blk_mq_update_nr_hw_queues(set, submit_queues);
  278. return set->nr_hw_queues == submit_queues ? 0 : -ENOMEM;
  279. }
  280. NULLB_DEVICE_ATTR(size, ulong, NULL);
  281. NULLB_DEVICE_ATTR(completion_nsec, ulong, NULL);
  282. NULLB_DEVICE_ATTR(submit_queues, uint, nullb_apply_submit_queues);
  283. NULLB_DEVICE_ATTR(home_node, uint, NULL);
  284. NULLB_DEVICE_ATTR(queue_mode, uint, NULL);
  285. NULLB_DEVICE_ATTR(blocksize, uint, NULL);
  286. NULLB_DEVICE_ATTR(irqmode, uint, NULL);
  287. NULLB_DEVICE_ATTR(hw_queue_depth, uint, NULL);
  288. NULLB_DEVICE_ATTR(index, uint, NULL);
  289. NULLB_DEVICE_ATTR(blocking, bool, NULL);
  290. NULLB_DEVICE_ATTR(use_per_node_hctx, bool, NULL);
  291. NULLB_DEVICE_ATTR(memory_backed, bool, NULL);
  292. NULLB_DEVICE_ATTR(discard, bool, NULL);
  293. NULLB_DEVICE_ATTR(mbps, uint, NULL);
  294. NULLB_DEVICE_ATTR(cache_size, ulong, NULL);
  295. NULLB_DEVICE_ATTR(zoned, bool, NULL);
  296. NULLB_DEVICE_ATTR(zone_size, ulong, NULL);
  297. NULLB_DEVICE_ATTR(zone_capacity, ulong, NULL);
  298. NULLB_DEVICE_ATTR(zone_nr_conv, uint, NULL);
  299. NULLB_DEVICE_ATTR(zone_max_open, uint, NULL);
  300. NULLB_DEVICE_ATTR(zone_max_active, uint, NULL);
  301. static ssize_t nullb_device_power_show(struct config_item *item, char *page)
  302. {
  303. return nullb_device_bool_attr_show(to_nullb_device(item)->power, page);
  304. }
  305. static ssize_t nullb_device_power_store(struct config_item *item,
  306. const char *page, size_t count)
  307. {
  308. struct nullb_device *dev = to_nullb_device(item);
  309. bool newp = false;
  310. ssize_t ret;
  311. ret = nullb_device_bool_attr_store(&newp, page, count);
  312. if (ret < 0)
  313. return ret;
  314. if (!dev->power && newp) {
  315. if (test_and_set_bit(NULLB_DEV_FL_UP, &dev->flags))
  316. return count;
  317. if (null_add_dev(dev)) {
  318. clear_bit(NULLB_DEV_FL_UP, &dev->flags);
  319. return -ENOMEM;
  320. }
  321. set_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
  322. dev->power = newp;
  323. } else if (dev->power && !newp) {
  324. if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) {
  325. mutex_lock(&lock);
  326. dev->power = newp;
  327. null_del_dev(dev->nullb);
  328. mutex_unlock(&lock);
  329. }
  330. clear_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
  331. }
  332. return count;
  333. }
  334. CONFIGFS_ATTR(nullb_device_, power);
  335. static ssize_t nullb_device_badblocks_show(struct config_item *item, char *page)
  336. {
  337. struct nullb_device *t_dev = to_nullb_device(item);
  338. return badblocks_show(&t_dev->badblocks, page, 0);
  339. }
  340. static ssize_t nullb_device_badblocks_store(struct config_item *item,
  341. const char *page, size_t count)
  342. {
  343. struct nullb_device *t_dev = to_nullb_device(item);
  344. char *orig, *buf, *tmp;
  345. u64 start, end;
  346. int ret;
  347. orig = kstrndup(page, count, GFP_KERNEL);
  348. if (!orig)
  349. return -ENOMEM;
  350. buf = strstrip(orig);
  351. ret = -EINVAL;
  352. if (buf[0] != '+' && buf[0] != '-')
  353. goto out;
  354. tmp = strchr(&buf[1], '-');
  355. if (!tmp)
  356. goto out;
  357. *tmp = '\0';
  358. ret = kstrtoull(buf + 1, 0, &start);
  359. if (ret)
  360. goto out;
  361. ret = kstrtoull(tmp + 1, 0, &end);
  362. if (ret)
  363. goto out;
  364. ret = -EINVAL;
  365. if (start > end)
  366. goto out;
  367. /* enable badblocks */
  368. cmpxchg(&t_dev->badblocks.shift, -1, 0);
  369. if (buf[0] == '+')
  370. ret = badblocks_set(&t_dev->badblocks, start,
  371. end - start + 1, 1);
  372. else
  373. ret = badblocks_clear(&t_dev->badblocks, start,
  374. end - start + 1);
  375. if (ret == 0)
  376. ret = count;
  377. out:
  378. kfree(orig);
  379. return ret;
  380. }
  381. CONFIGFS_ATTR(nullb_device_, badblocks);
  382. static struct configfs_attribute *nullb_device_attrs[] = {
  383. &nullb_device_attr_size,
  384. &nullb_device_attr_completion_nsec,
  385. &nullb_device_attr_submit_queues,
  386. &nullb_device_attr_home_node,
  387. &nullb_device_attr_queue_mode,
  388. &nullb_device_attr_blocksize,
  389. &nullb_device_attr_irqmode,
  390. &nullb_device_attr_hw_queue_depth,
  391. &nullb_device_attr_index,
  392. &nullb_device_attr_blocking,
  393. &nullb_device_attr_use_per_node_hctx,
  394. &nullb_device_attr_power,
  395. &nullb_device_attr_memory_backed,
  396. &nullb_device_attr_discard,
  397. &nullb_device_attr_mbps,
  398. &nullb_device_attr_cache_size,
  399. &nullb_device_attr_badblocks,
  400. &nullb_device_attr_zoned,
  401. &nullb_device_attr_zone_size,
  402. &nullb_device_attr_zone_capacity,
  403. &nullb_device_attr_zone_nr_conv,
  404. &nullb_device_attr_zone_max_open,
  405. &nullb_device_attr_zone_max_active,
  406. NULL,
  407. };
  408. static void nullb_device_release(struct config_item *item)
  409. {
  410. struct nullb_device *dev = to_nullb_device(item);
  411. null_free_device_storage(dev, false);
  412. null_free_dev(dev);
  413. }
  414. static struct configfs_item_operations nullb_device_ops = {
  415. .release = nullb_device_release,
  416. };
  417. static const struct config_item_type nullb_device_type = {
  418. .ct_item_ops = &nullb_device_ops,
  419. .ct_attrs = nullb_device_attrs,
  420. .ct_owner = THIS_MODULE,
  421. };
  422. static struct
  423. config_item *nullb_group_make_item(struct config_group *group, const char *name)
  424. {
  425. struct nullb_device *dev;
  426. dev = null_alloc_dev();
  427. if (!dev)
  428. return ERR_PTR(-ENOMEM);
  429. config_item_init_type_name(&dev->item, name, &nullb_device_type);
  430. return &dev->item;
  431. }
  432. static void
  433. nullb_group_drop_item(struct config_group *group, struct config_item *item)
  434. {
  435. struct nullb_device *dev = to_nullb_device(item);
  436. if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) {
  437. mutex_lock(&lock);
  438. dev->power = false;
  439. null_del_dev(dev->nullb);
  440. mutex_unlock(&lock);
  441. }
  442. config_item_put(item);
  443. }
  444. static ssize_t memb_group_features_show(struct config_item *item, char *page)
  445. {
  446. return snprintf(page, PAGE_SIZE,
  447. "memory_backed,discard,bandwidth,cache,badblocks,zoned,zone_size,zone_capacity,zone_nr_conv,zone_max_open,zone_max_active\n");
  448. }
  449. CONFIGFS_ATTR_RO(memb_group_, features);
  450. static struct configfs_attribute *nullb_group_attrs[] = {
  451. &memb_group_attr_features,
  452. NULL,
  453. };
  454. static struct configfs_group_operations nullb_group_ops = {
  455. .make_item = nullb_group_make_item,
  456. .drop_item = nullb_group_drop_item,
  457. };
  458. static const struct config_item_type nullb_group_type = {
  459. .ct_group_ops = &nullb_group_ops,
  460. .ct_attrs = nullb_group_attrs,
  461. .ct_owner = THIS_MODULE,
  462. };
  463. static struct configfs_subsystem nullb_subsys = {
  464. .su_group = {
  465. .cg_item = {
  466. .ci_namebuf = "nullb",
  467. .ci_type = &nullb_group_type,
  468. },
  469. },
  470. };
  471. static inline int null_cache_active(struct nullb *nullb)
  472. {
  473. return test_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
  474. }
  475. static struct nullb_device *null_alloc_dev(void)
  476. {
  477. struct nullb_device *dev;
  478. dev = kzalloc(sizeof(*dev), GFP_KERNEL);
  479. if (!dev)
  480. return NULL;
  481. INIT_RADIX_TREE(&dev->data, GFP_ATOMIC);
  482. INIT_RADIX_TREE(&dev->cache, GFP_ATOMIC);
  483. if (badblocks_init(&dev->badblocks, 0)) {
  484. kfree(dev);
  485. return NULL;
  486. }
  487. dev->size = g_gb * 1024;
  488. dev->completion_nsec = g_completion_nsec;
  489. dev->submit_queues = g_submit_queues;
  490. dev->home_node = g_home_node;
  491. dev->queue_mode = g_queue_mode;
  492. dev->blocksize = g_bs;
  493. dev->irqmode = g_irqmode;
  494. dev->hw_queue_depth = g_hw_queue_depth;
  495. dev->blocking = g_blocking;
  496. dev->use_per_node_hctx = g_use_per_node_hctx;
  497. dev->zoned = g_zoned;
  498. dev->zone_size = g_zone_size;
  499. dev->zone_capacity = g_zone_capacity;
  500. dev->zone_nr_conv = g_zone_nr_conv;
  501. dev->zone_max_open = g_zone_max_open;
  502. dev->zone_max_active = g_zone_max_active;
  503. return dev;
  504. }
  505. static void null_free_dev(struct nullb_device *dev)
  506. {
  507. if (!dev)
  508. return;
  509. null_free_zoned_dev(dev);
  510. badblocks_exit(&dev->badblocks);
  511. kfree(dev);
  512. }
  513. static void put_tag(struct nullb_queue *nq, unsigned int tag)
  514. {
  515. clear_bit_unlock(tag, nq->tag_map);
  516. if (waitqueue_active(&nq->wait))
  517. wake_up(&nq->wait);
  518. }
  519. static unsigned int get_tag(struct nullb_queue *nq)
  520. {
  521. unsigned int tag;
  522. do {
  523. tag = find_first_zero_bit(nq->tag_map, nq->queue_depth);
  524. if (tag >= nq->queue_depth)
  525. return -1U;
  526. } while (test_and_set_bit_lock(tag, nq->tag_map));
  527. return tag;
  528. }
  529. static void free_cmd(struct nullb_cmd *cmd)
  530. {
  531. put_tag(cmd->nq, cmd->tag);
  532. }
  533. static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer);
  534. static struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq)
  535. {
  536. struct nullb_cmd *cmd;
  537. unsigned int tag;
  538. tag = get_tag(nq);
  539. if (tag != -1U) {
  540. cmd = &nq->cmds[tag];
  541. cmd->tag = tag;
  542. cmd->error = BLK_STS_OK;
  543. cmd->nq = nq;
  544. if (nq->dev->irqmode == NULL_IRQ_TIMER) {
  545. hrtimer_init(&cmd->timer, CLOCK_MONOTONIC,
  546. HRTIMER_MODE_REL);
  547. cmd->timer.function = null_cmd_timer_expired;
  548. }
  549. return cmd;
  550. }
  551. return NULL;
  552. }
  553. static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, int can_wait)
  554. {
  555. struct nullb_cmd *cmd;
  556. DEFINE_WAIT(wait);
  557. cmd = __alloc_cmd(nq);
  558. if (cmd || !can_wait)
  559. return cmd;
  560. do {
  561. prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE);
  562. cmd = __alloc_cmd(nq);
  563. if (cmd)
  564. break;
  565. io_schedule();
  566. } while (1);
  567. finish_wait(&nq->wait, &wait);
  568. return cmd;
  569. }
  570. static void end_cmd(struct nullb_cmd *cmd)
  571. {
  572. int queue_mode = cmd->nq->dev->queue_mode;
  573. switch (queue_mode) {
  574. case NULL_Q_MQ:
  575. blk_mq_end_request(cmd->rq, cmd->error);
  576. return;
  577. case NULL_Q_BIO:
  578. cmd->bio->bi_status = cmd->error;
  579. bio_endio(cmd->bio);
  580. break;
  581. }
  582. free_cmd(cmd);
  583. }
  584. static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer)
  585. {
  586. end_cmd(container_of(timer, struct nullb_cmd, timer));
  587. return HRTIMER_NORESTART;
  588. }
  589. static void null_cmd_end_timer(struct nullb_cmd *cmd)
  590. {
  591. ktime_t kt = cmd->nq->dev->completion_nsec;
  592. hrtimer_start(&cmd->timer, kt, HRTIMER_MODE_REL);
  593. }
  594. static void null_complete_rq(struct request *rq)
  595. {
  596. end_cmd(blk_mq_rq_to_pdu(rq));
  597. }
  598. static struct nullb_page *null_alloc_page(gfp_t gfp_flags)
  599. {
  600. struct nullb_page *t_page;
  601. t_page = kmalloc(sizeof(struct nullb_page), gfp_flags);
  602. if (!t_page)
  603. goto out;
  604. t_page->page = alloc_pages(gfp_flags, 0);
  605. if (!t_page->page)
  606. goto out_freepage;
  607. memset(t_page->bitmap, 0, sizeof(t_page->bitmap));
  608. return t_page;
  609. out_freepage:
  610. kfree(t_page);
  611. out:
  612. return NULL;
  613. }
  614. static void null_free_page(struct nullb_page *t_page)
  615. {
  616. __set_bit(NULLB_PAGE_FREE, t_page->bitmap);
  617. if (test_bit(NULLB_PAGE_LOCK, t_page->bitmap))
  618. return;
  619. __free_page(t_page->page);
  620. kfree(t_page);
  621. }
  622. static bool null_page_empty(struct nullb_page *page)
  623. {
  624. int size = MAP_SZ - 2;
  625. return find_first_bit(page->bitmap, size) == size;
  626. }
  627. static void null_free_sector(struct nullb *nullb, sector_t sector,
  628. bool is_cache)
  629. {
  630. unsigned int sector_bit;
  631. u64 idx;
  632. struct nullb_page *t_page, *ret;
  633. struct radix_tree_root *root;
  634. root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
  635. idx = sector >> PAGE_SECTORS_SHIFT;
  636. sector_bit = (sector & SECTOR_MASK);
  637. t_page = radix_tree_lookup(root, idx);
  638. if (t_page) {
  639. __clear_bit(sector_bit, t_page->bitmap);
  640. if (null_page_empty(t_page)) {
  641. ret = radix_tree_delete_item(root, idx, t_page);
  642. WARN_ON(ret != t_page);
  643. null_free_page(ret);
  644. if (is_cache)
  645. nullb->dev->curr_cache -= PAGE_SIZE;
  646. }
  647. }
  648. }
  649. static struct nullb_page *null_radix_tree_insert(struct nullb *nullb, u64 idx,
  650. struct nullb_page *t_page, bool is_cache)
  651. {
  652. struct radix_tree_root *root;
  653. root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
  654. if (radix_tree_insert(root, idx, t_page)) {
  655. null_free_page(t_page);
  656. t_page = radix_tree_lookup(root, idx);
  657. WARN_ON(!t_page || t_page->page->index != idx);
  658. } else if (is_cache)
  659. nullb->dev->curr_cache += PAGE_SIZE;
  660. return t_page;
  661. }
  662. static void null_free_device_storage(struct nullb_device *dev, bool is_cache)
  663. {
  664. unsigned long pos = 0;
  665. int nr_pages;
  666. struct nullb_page *ret, *t_pages[FREE_BATCH];
  667. struct radix_tree_root *root;
  668. root = is_cache ? &dev->cache : &dev->data;
  669. do {
  670. int i;
  671. nr_pages = radix_tree_gang_lookup(root,
  672. (void **)t_pages, pos, FREE_BATCH);
  673. for (i = 0; i < nr_pages; i++) {
  674. pos = t_pages[i]->page->index;
  675. ret = radix_tree_delete_item(root, pos, t_pages[i]);
  676. WARN_ON(ret != t_pages[i]);
  677. null_free_page(ret);
  678. }
  679. pos++;
  680. } while (nr_pages == FREE_BATCH);
  681. if (is_cache)
  682. dev->curr_cache = 0;
  683. }
  684. static struct nullb_page *__null_lookup_page(struct nullb *nullb,
  685. sector_t sector, bool for_write, bool is_cache)
  686. {
  687. unsigned int sector_bit;
  688. u64 idx;
  689. struct nullb_page *t_page;
  690. struct radix_tree_root *root;
  691. idx = sector >> PAGE_SECTORS_SHIFT;
  692. sector_bit = (sector & SECTOR_MASK);
  693. root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
  694. t_page = radix_tree_lookup(root, idx);
  695. WARN_ON(t_page && t_page->page->index != idx);
  696. if (t_page && (for_write || test_bit(sector_bit, t_page->bitmap)))
  697. return t_page;
  698. return NULL;
  699. }
  700. static struct nullb_page *null_lookup_page(struct nullb *nullb,
  701. sector_t sector, bool for_write, bool ignore_cache)
  702. {
  703. struct nullb_page *page = NULL;
  704. if (!ignore_cache)
  705. page = __null_lookup_page(nullb, sector, for_write, true);
  706. if (page)
  707. return page;
  708. return __null_lookup_page(nullb, sector, for_write, false);
  709. }
  710. static struct nullb_page *null_insert_page(struct nullb *nullb,
  711. sector_t sector, bool ignore_cache)
  712. __releases(&nullb->lock)
  713. __acquires(&nullb->lock)
  714. {
  715. u64 idx;
  716. struct nullb_page *t_page;
  717. t_page = null_lookup_page(nullb, sector, true, ignore_cache);
  718. if (t_page)
  719. return t_page;
  720. spin_unlock_irq(&nullb->lock);
  721. t_page = null_alloc_page(GFP_NOIO);
  722. if (!t_page)
  723. goto out_lock;
  724. if (radix_tree_preload(GFP_NOIO))
  725. goto out_freepage;
  726. spin_lock_irq(&nullb->lock);
  727. idx = sector >> PAGE_SECTORS_SHIFT;
  728. t_page->page->index = idx;
  729. t_page = null_radix_tree_insert(nullb, idx, t_page, !ignore_cache);
  730. radix_tree_preload_end();
  731. return t_page;
  732. out_freepage:
  733. null_free_page(t_page);
  734. out_lock:
  735. spin_lock_irq(&nullb->lock);
  736. return null_lookup_page(nullb, sector, true, ignore_cache);
  737. }
  738. static int null_flush_cache_page(struct nullb *nullb, struct nullb_page *c_page)
  739. {
  740. int i;
  741. unsigned int offset;
  742. u64 idx;
  743. struct nullb_page *t_page, *ret;
  744. void *dst, *src;
  745. idx = c_page->page->index;
  746. t_page = null_insert_page(nullb, idx << PAGE_SECTORS_SHIFT, true);
  747. __clear_bit(NULLB_PAGE_LOCK, c_page->bitmap);
  748. if (test_bit(NULLB_PAGE_FREE, c_page->bitmap)) {
  749. null_free_page(c_page);
  750. if (t_page && null_page_empty(t_page)) {
  751. ret = radix_tree_delete_item(&nullb->dev->data,
  752. idx, t_page);
  753. null_free_page(t_page);
  754. }
  755. return 0;
  756. }
  757. if (!t_page)
  758. return -ENOMEM;
  759. src = kmap_atomic(c_page->page);
  760. dst = kmap_atomic(t_page->page);
  761. for (i = 0; i < PAGE_SECTORS;
  762. i += (nullb->dev->blocksize >> SECTOR_SHIFT)) {
  763. if (test_bit(i, c_page->bitmap)) {
  764. offset = (i << SECTOR_SHIFT);
  765. memcpy(dst + offset, src + offset,
  766. nullb->dev->blocksize);
  767. __set_bit(i, t_page->bitmap);
  768. }
  769. }
  770. kunmap_atomic(dst);
  771. kunmap_atomic(src);
  772. ret = radix_tree_delete_item(&nullb->dev->cache, idx, c_page);
  773. null_free_page(ret);
  774. nullb->dev->curr_cache -= PAGE_SIZE;
  775. return 0;
  776. }
  777. static int null_make_cache_space(struct nullb *nullb, unsigned long n)
  778. {
  779. int i, err, nr_pages;
  780. struct nullb_page *c_pages[FREE_BATCH];
  781. unsigned long flushed = 0, one_round;
  782. again:
  783. if ((nullb->dev->cache_size * 1024 * 1024) >
  784. nullb->dev->curr_cache + n || nullb->dev->curr_cache == 0)
  785. return 0;
  786. nr_pages = radix_tree_gang_lookup(&nullb->dev->cache,
  787. (void **)c_pages, nullb->cache_flush_pos, FREE_BATCH);
  788. /*
  789. * nullb_flush_cache_page could unlock before using the c_pages. To
  790. * avoid race, we don't allow page free
  791. */
  792. for (i = 0; i < nr_pages; i++) {
  793. nullb->cache_flush_pos = c_pages[i]->page->index;
  794. /*
  795. * We found the page which is being flushed to disk by other
  796. * threads
  797. */
  798. if (test_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap))
  799. c_pages[i] = NULL;
  800. else
  801. __set_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap);
  802. }
  803. one_round = 0;
  804. for (i = 0; i < nr_pages; i++) {
  805. if (c_pages[i] == NULL)
  806. continue;
  807. err = null_flush_cache_page(nullb, c_pages[i]);
  808. if (err)
  809. return err;
  810. one_round++;
  811. }
  812. flushed += one_round << PAGE_SHIFT;
  813. if (n > flushed) {
  814. if (nr_pages == 0)
  815. nullb->cache_flush_pos = 0;
  816. if (one_round == 0) {
  817. /* give other threads a chance */
  818. spin_unlock_irq(&nullb->lock);
  819. spin_lock_irq(&nullb->lock);
  820. }
  821. goto again;
  822. }
  823. return 0;
  824. }
  825. static int copy_to_nullb(struct nullb *nullb, struct page *source,
  826. unsigned int off, sector_t sector, size_t n, bool is_fua)
  827. {
  828. size_t temp, count = 0;
  829. unsigned int offset;
  830. struct nullb_page *t_page;
  831. void *dst, *src;
  832. while (count < n) {
  833. temp = min_t(size_t, nullb->dev->blocksize, n - count);
  834. if (null_cache_active(nullb) && !is_fua)
  835. null_make_cache_space(nullb, PAGE_SIZE);
  836. offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
  837. t_page = null_insert_page(nullb, sector,
  838. !null_cache_active(nullb) || is_fua);
  839. if (!t_page)
  840. return -ENOSPC;
  841. src = kmap_atomic(source);
  842. dst = kmap_atomic(t_page->page);
  843. memcpy(dst + offset, src + off + count, temp);
  844. kunmap_atomic(dst);
  845. kunmap_atomic(src);
  846. __set_bit(sector & SECTOR_MASK, t_page->bitmap);
  847. if (is_fua)
  848. null_free_sector(nullb, sector, true);
  849. count += temp;
  850. sector += temp >> SECTOR_SHIFT;
  851. }
  852. return 0;
  853. }
  854. static int copy_from_nullb(struct nullb *nullb, struct page *dest,
  855. unsigned int off, sector_t sector, size_t n)
  856. {
  857. size_t temp, count = 0;
  858. unsigned int offset;
  859. struct nullb_page *t_page;
  860. void *dst, *src;
  861. while (count < n) {
  862. temp = min_t(size_t, nullb->dev->blocksize, n - count);
  863. offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
  864. t_page = null_lookup_page(nullb, sector, false,
  865. !null_cache_active(nullb));
  866. dst = kmap_atomic(dest);
  867. if (!t_page) {
  868. memset(dst + off + count, 0, temp);
  869. goto next;
  870. }
  871. src = kmap_atomic(t_page->page);
  872. memcpy(dst + off + count, src + offset, temp);
  873. kunmap_atomic(src);
  874. next:
  875. kunmap_atomic(dst);
  876. count += temp;
  877. sector += temp >> SECTOR_SHIFT;
  878. }
  879. return 0;
  880. }
  881. static void nullb_fill_pattern(struct nullb *nullb, struct page *page,
  882. unsigned int len, unsigned int off)
  883. {
  884. void *dst;
  885. dst = kmap_atomic(page);
  886. memset(dst + off, 0xFF, len);
  887. kunmap_atomic(dst);
  888. }
  889. static void null_handle_discard(struct nullb *nullb, sector_t sector, size_t n)
  890. {
  891. size_t temp;
  892. spin_lock_irq(&nullb->lock);
  893. while (n > 0) {
  894. temp = min_t(size_t, n, nullb->dev->blocksize);
  895. null_free_sector(nullb, sector, false);
  896. if (null_cache_active(nullb))
  897. null_free_sector(nullb, sector, true);
  898. sector += temp >> SECTOR_SHIFT;
  899. n -= temp;
  900. }
  901. spin_unlock_irq(&nullb->lock);
  902. }
  903. static int null_handle_flush(struct nullb *nullb)
  904. {
  905. int err;
  906. if (!null_cache_active(nullb))
  907. return 0;
  908. spin_lock_irq(&nullb->lock);
  909. while (true) {
  910. err = null_make_cache_space(nullb,
  911. nullb->dev->cache_size * 1024 * 1024);
  912. if (err || nullb->dev->curr_cache == 0)
  913. break;
  914. }
  915. WARN_ON(!radix_tree_empty(&nullb->dev->cache));
  916. spin_unlock_irq(&nullb->lock);
  917. return err;
  918. }
  919. static int null_transfer(struct nullb *nullb, struct page *page,
  920. unsigned int len, unsigned int off, bool is_write, sector_t sector,
  921. bool is_fua)
  922. {
  923. struct nullb_device *dev = nullb->dev;
  924. unsigned int valid_len = len;
  925. int err = 0;
  926. if (!is_write) {
  927. if (dev->zoned)
  928. valid_len = null_zone_valid_read_len(nullb,
  929. sector, len);
  930. if (valid_len) {
  931. err = copy_from_nullb(nullb, page, off,
  932. sector, valid_len);
  933. off += valid_len;
  934. len -= valid_len;
  935. }
  936. if (len)
  937. nullb_fill_pattern(nullb, page, len, off);
  938. flush_dcache_page(page);
  939. } else {
  940. flush_dcache_page(page);
  941. err = copy_to_nullb(nullb, page, off, sector, len, is_fua);
  942. }
  943. return err;
  944. }
  945. static int null_handle_rq(struct nullb_cmd *cmd)
  946. {
  947. struct request *rq = cmd->rq;
  948. struct nullb *nullb = cmd->nq->dev->nullb;
  949. int err;
  950. unsigned int len;
  951. sector_t sector;
  952. struct req_iterator iter;
  953. struct bio_vec bvec;
  954. sector = blk_rq_pos(rq);
  955. if (req_op(rq) == REQ_OP_DISCARD) {
  956. null_handle_discard(nullb, sector, blk_rq_bytes(rq));
  957. return 0;
  958. }
  959. spin_lock_irq(&nullb->lock);
  960. rq_for_each_segment(bvec, rq, iter) {
  961. len = bvec.bv_len;
  962. err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,
  963. op_is_write(req_op(rq)), sector,
  964. rq->cmd_flags & REQ_FUA);
  965. if (err) {
  966. spin_unlock_irq(&nullb->lock);
  967. return err;
  968. }
  969. sector += len >> SECTOR_SHIFT;
  970. }
  971. spin_unlock_irq(&nullb->lock);
  972. return 0;
  973. }
  974. static int null_handle_bio(struct nullb_cmd *cmd)
  975. {
  976. struct bio *bio = cmd->bio;
  977. struct nullb *nullb = cmd->nq->dev->nullb;
  978. int err;
  979. unsigned int len;
  980. sector_t sector;
  981. struct bio_vec bvec;
  982. struct bvec_iter iter;
  983. sector = bio->bi_iter.bi_sector;
  984. if (bio_op(bio) == REQ_OP_DISCARD) {
  985. null_handle_discard(nullb, sector,
  986. bio_sectors(bio) << SECTOR_SHIFT);
  987. return 0;
  988. }
  989. spin_lock_irq(&nullb->lock);
  990. bio_for_each_segment(bvec, bio, iter) {
  991. len = bvec.bv_len;
  992. err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,
  993. op_is_write(bio_op(bio)), sector,
  994. bio->bi_opf & REQ_FUA);
  995. if (err) {
  996. spin_unlock_irq(&nullb->lock);
  997. return err;
  998. }
  999. sector += len >> SECTOR_SHIFT;
  1000. }
  1001. spin_unlock_irq(&nullb->lock);
  1002. return 0;
  1003. }
  1004. static void null_stop_queue(struct nullb *nullb)
  1005. {
  1006. struct request_queue *q = nullb->q;
  1007. if (nullb->dev->queue_mode == NULL_Q_MQ)
  1008. blk_mq_stop_hw_queues(q);
  1009. }
  1010. static void null_restart_queue_async(struct nullb *nullb)
  1011. {
  1012. struct request_queue *q = nullb->q;
  1013. if (nullb->dev->queue_mode == NULL_Q_MQ)
  1014. blk_mq_start_stopped_hw_queues(q, true);
  1015. }
  1016. static inline blk_status_t null_handle_throttled(struct nullb_cmd *cmd)
  1017. {
  1018. struct nullb_device *dev = cmd->nq->dev;
  1019. struct nullb *nullb = dev->nullb;
  1020. blk_status_t sts = BLK_STS_OK;
  1021. struct request *rq = cmd->rq;
  1022. if (!hrtimer_active(&nullb->bw_timer))
  1023. hrtimer_restart(&nullb->bw_timer);
  1024. if (atomic_long_sub_return(blk_rq_bytes(rq), &nullb->cur_bytes) < 0) {
  1025. null_stop_queue(nullb);
  1026. /* race with timer */
  1027. if (atomic_long_read(&nullb->cur_bytes) > 0)
  1028. null_restart_queue_async(nullb);
  1029. /* requeue request */
  1030. sts = BLK_STS_DEV_RESOURCE;
  1031. }
  1032. return sts;
  1033. }
  1034. static inline blk_status_t null_handle_badblocks(struct nullb_cmd *cmd,
  1035. sector_t sector,
  1036. sector_t nr_sectors)
  1037. {
  1038. struct badblocks *bb = &cmd->nq->dev->badblocks;
  1039. sector_t first_bad;
  1040. int bad_sectors;
  1041. if (badblocks_check(bb, sector, nr_sectors, &first_bad, &bad_sectors))
  1042. return BLK_STS_IOERR;
  1043. return BLK_STS_OK;
  1044. }
  1045. static inline blk_status_t null_handle_memory_backed(struct nullb_cmd *cmd,
  1046. enum req_opf op)
  1047. {
  1048. struct nullb_device *dev = cmd->nq->dev;
  1049. int err;
  1050. if (dev->queue_mode == NULL_Q_BIO)
  1051. err = null_handle_bio(cmd);
  1052. else
  1053. err = null_handle_rq(cmd);
  1054. return errno_to_blk_status(err);
  1055. }
  1056. static void nullb_zero_read_cmd_buffer(struct nullb_cmd *cmd)
  1057. {
  1058. struct nullb_device *dev = cmd->nq->dev;
  1059. struct bio *bio;
  1060. if (dev->memory_backed)
  1061. return;
  1062. if (dev->queue_mode == NULL_Q_BIO && bio_op(cmd->bio) == REQ_OP_READ) {
  1063. zero_fill_bio(cmd->bio);
  1064. } else if (req_op(cmd->rq) == REQ_OP_READ) {
  1065. __rq_for_each_bio(bio, cmd->rq)
  1066. zero_fill_bio(bio);
  1067. }
  1068. }
  1069. static inline void nullb_complete_cmd(struct nullb_cmd *cmd)
  1070. {
  1071. /*
  1072. * Since root privileges are required to configure the null_blk
  1073. * driver, it is fine that this driver does not initialize the
  1074. * data buffers of read commands. Zero-initialize these buffers
  1075. * anyway if KMSAN is enabled to prevent that KMSAN complains
  1076. * about null_blk not initializing read data buffers.
  1077. */
  1078. if (IS_ENABLED(CONFIG_KMSAN))
  1079. nullb_zero_read_cmd_buffer(cmd);
  1080. /* Complete IO by inline, softirq or timer */
  1081. switch (cmd->nq->dev->irqmode) {
  1082. case NULL_IRQ_SOFTIRQ:
  1083. switch (cmd->nq->dev->queue_mode) {
  1084. case NULL_Q_MQ:
  1085. if (likely(!blk_should_fake_timeout(cmd->rq->q)))
  1086. blk_mq_complete_request(cmd->rq);
  1087. break;
  1088. case NULL_Q_BIO:
  1089. /*
  1090. * XXX: no proper submitting cpu information available.
  1091. */
  1092. end_cmd(cmd);
  1093. break;
  1094. }
  1095. break;
  1096. case NULL_IRQ_NONE:
  1097. end_cmd(cmd);
  1098. break;
  1099. case NULL_IRQ_TIMER:
  1100. null_cmd_end_timer(cmd);
  1101. break;
  1102. }
  1103. }
  1104. blk_status_t null_process_cmd(struct nullb_cmd *cmd,
  1105. enum req_opf op, sector_t sector,
  1106. unsigned int nr_sectors)
  1107. {
  1108. struct nullb_device *dev = cmd->nq->dev;
  1109. blk_status_t ret;
  1110. if (dev->badblocks.shift != -1) {
  1111. ret = null_handle_badblocks(cmd, sector, nr_sectors);
  1112. if (ret != BLK_STS_OK)
  1113. return ret;
  1114. }
  1115. if (dev->memory_backed)
  1116. return null_handle_memory_backed(cmd, op);
  1117. return BLK_STS_OK;
  1118. }
  1119. static blk_status_t null_handle_cmd(struct nullb_cmd *cmd, sector_t sector,
  1120. sector_t nr_sectors, enum req_opf op)
  1121. {
  1122. struct nullb_device *dev = cmd->nq->dev;
  1123. struct nullb *nullb = dev->nullb;
  1124. blk_status_t sts;
  1125. if (test_bit(NULLB_DEV_FL_THROTTLED, &dev->flags)) {
  1126. sts = null_handle_throttled(cmd);
  1127. if (sts != BLK_STS_OK)
  1128. return sts;
  1129. }
  1130. if (op == REQ_OP_FLUSH) {
  1131. cmd->error = errno_to_blk_status(null_handle_flush(nullb));
  1132. goto out;
  1133. }
  1134. if (dev->zoned)
  1135. sts = null_process_zoned_cmd(cmd, op, sector, nr_sectors);
  1136. else
  1137. sts = null_process_cmd(cmd, op, sector, nr_sectors);
  1138. /* Do not overwrite errors (e.g. timeout errors) */
  1139. if (cmd->error == BLK_STS_OK)
  1140. cmd->error = sts;
  1141. out:
  1142. nullb_complete_cmd(cmd);
  1143. return BLK_STS_OK;
  1144. }
  1145. static enum hrtimer_restart nullb_bwtimer_fn(struct hrtimer *timer)
  1146. {
  1147. struct nullb *nullb = container_of(timer, struct nullb, bw_timer);
  1148. ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
  1149. unsigned int mbps = nullb->dev->mbps;
  1150. if (atomic_long_read(&nullb->cur_bytes) == mb_per_tick(mbps))
  1151. return HRTIMER_NORESTART;
  1152. atomic_long_set(&nullb->cur_bytes, mb_per_tick(mbps));
  1153. null_restart_queue_async(nullb);
  1154. hrtimer_forward_now(&nullb->bw_timer, timer_interval);
  1155. return HRTIMER_RESTART;
  1156. }
  1157. static void nullb_setup_bwtimer(struct nullb *nullb)
  1158. {
  1159. ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
  1160. hrtimer_init(&nullb->bw_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  1161. nullb->bw_timer.function = nullb_bwtimer_fn;
  1162. atomic_long_set(&nullb->cur_bytes, mb_per_tick(nullb->dev->mbps));
  1163. hrtimer_start(&nullb->bw_timer, timer_interval, HRTIMER_MODE_REL);
  1164. }
  1165. static struct nullb_queue *nullb_to_queue(struct nullb *nullb)
  1166. {
  1167. int index = 0;
  1168. if (nullb->nr_queues != 1)
  1169. index = raw_smp_processor_id() / ((nr_cpu_ids + nullb->nr_queues - 1) / nullb->nr_queues);
  1170. return &nullb->queues[index];
  1171. }
  1172. static blk_qc_t null_submit_bio(struct bio *bio)
  1173. {
  1174. sector_t sector = bio->bi_iter.bi_sector;
  1175. sector_t nr_sectors = bio_sectors(bio);
  1176. struct nullb *nullb = bio->bi_disk->private_data;
  1177. struct nullb_queue *nq = nullb_to_queue(nullb);
  1178. struct nullb_cmd *cmd;
  1179. cmd = alloc_cmd(nq, 1);
  1180. cmd->bio = bio;
  1181. null_handle_cmd(cmd, sector, nr_sectors, bio_op(bio));
  1182. return BLK_QC_T_NONE;
  1183. }
  1184. static bool should_timeout_request(struct request *rq)
  1185. {
  1186. #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
  1187. if (g_timeout_str[0])
  1188. return should_fail(&null_timeout_attr, 1);
  1189. #endif
  1190. return false;
  1191. }
  1192. static bool should_requeue_request(struct request *rq)
  1193. {
  1194. #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
  1195. if (g_requeue_str[0])
  1196. return should_fail(&null_requeue_attr, 1);
  1197. #endif
  1198. return false;
  1199. }
  1200. static enum blk_eh_timer_return null_timeout_rq(struct request *rq, bool res)
  1201. {
  1202. struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
  1203. pr_info("rq %p timed out\n", rq);
  1204. /*
  1205. * If the device is marked as blocking (i.e. memory backed or zoned
  1206. * device), the submission path may be blocked waiting for resources
  1207. * and cause real timeouts. For these real timeouts, the submission
  1208. * path will complete the request using blk_mq_complete_request().
  1209. * Only fake timeouts need to execute blk_mq_complete_request() here.
  1210. */
  1211. cmd->error = BLK_STS_TIMEOUT;
  1212. if (cmd->fake_timeout)
  1213. blk_mq_complete_request(rq);
  1214. return BLK_EH_DONE;
  1215. }
  1216. static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
  1217. const struct blk_mq_queue_data *bd)
  1218. {
  1219. struct nullb_cmd *cmd = blk_mq_rq_to_pdu(bd->rq);
  1220. struct nullb_queue *nq = hctx->driver_data;
  1221. sector_t nr_sectors = blk_rq_sectors(bd->rq);
  1222. sector_t sector = blk_rq_pos(bd->rq);
  1223. might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
  1224. if (nq->dev->irqmode == NULL_IRQ_TIMER) {
  1225. hrtimer_init(&cmd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  1226. cmd->timer.function = null_cmd_timer_expired;
  1227. }
  1228. cmd->rq = bd->rq;
  1229. cmd->error = BLK_STS_OK;
  1230. cmd->nq = nq;
  1231. cmd->fake_timeout = should_timeout_request(bd->rq);
  1232. blk_mq_start_request(bd->rq);
  1233. if (should_requeue_request(bd->rq)) {
  1234. /*
  1235. * Alternate between hitting the core BUSY path, and the
  1236. * driver driven requeue path
  1237. */
  1238. nq->requeue_selection++;
  1239. if (nq->requeue_selection & 1)
  1240. return BLK_STS_RESOURCE;
  1241. else {
  1242. blk_mq_requeue_request(bd->rq, true);
  1243. return BLK_STS_OK;
  1244. }
  1245. }
  1246. if (cmd->fake_timeout)
  1247. return BLK_STS_OK;
  1248. return null_handle_cmd(cmd, sector, nr_sectors, req_op(bd->rq));
  1249. }
  1250. static void cleanup_queue(struct nullb_queue *nq)
  1251. {
  1252. kfree(nq->tag_map);
  1253. kfree(nq->cmds);
  1254. }
  1255. static void cleanup_queues(struct nullb *nullb)
  1256. {
  1257. int i;
  1258. for (i = 0; i < nullb->nr_queues; i++)
  1259. cleanup_queue(&nullb->queues[i]);
  1260. kfree(nullb->queues);
  1261. }
  1262. static void null_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
  1263. {
  1264. struct nullb_queue *nq = hctx->driver_data;
  1265. struct nullb *nullb = nq->dev->nullb;
  1266. nullb->nr_queues--;
  1267. }
  1268. static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)
  1269. {
  1270. init_waitqueue_head(&nq->wait);
  1271. nq->queue_depth = nullb->queue_depth;
  1272. nq->dev = nullb->dev;
  1273. }
  1274. static int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data,
  1275. unsigned int hctx_idx)
  1276. {
  1277. struct nullb *nullb = hctx->queue->queuedata;
  1278. struct nullb_queue *nq;
  1279. #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
  1280. if (g_init_hctx_str[0] && should_fail(&null_init_hctx_attr, 1))
  1281. return -EFAULT;
  1282. #endif
  1283. nq = &nullb->queues[hctx_idx];
  1284. hctx->driver_data = nq;
  1285. null_init_queue(nullb, nq);
  1286. nullb->nr_queues++;
  1287. return 0;
  1288. }
  1289. static const struct blk_mq_ops null_mq_ops = {
  1290. .queue_rq = null_queue_rq,
  1291. .complete = null_complete_rq,
  1292. .timeout = null_timeout_rq,
  1293. .init_hctx = null_init_hctx,
  1294. .exit_hctx = null_exit_hctx,
  1295. };
  1296. static void null_del_dev(struct nullb *nullb)
  1297. {
  1298. struct nullb_device *dev;
  1299. if (!nullb)
  1300. return;
  1301. dev = nullb->dev;
  1302. ida_simple_remove(&nullb_indexes, nullb->index);
  1303. list_del_init(&nullb->list);
  1304. del_gendisk(nullb->disk);
  1305. if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) {
  1306. hrtimer_cancel(&nullb->bw_timer);
  1307. atomic_long_set(&nullb->cur_bytes, LONG_MAX);
  1308. null_restart_queue_async(nullb);
  1309. }
  1310. blk_cleanup_queue(nullb->q);
  1311. if (dev->queue_mode == NULL_Q_MQ &&
  1312. nullb->tag_set == &nullb->__tag_set)
  1313. blk_mq_free_tag_set(nullb->tag_set);
  1314. put_disk(nullb->disk);
  1315. cleanup_queues(nullb);
  1316. if (null_cache_active(nullb))
  1317. null_free_device_storage(nullb->dev, true);
  1318. kfree(nullb);
  1319. dev->nullb = NULL;
  1320. }
  1321. static void null_config_discard(struct nullb *nullb)
  1322. {
  1323. if (nullb->dev->discard == false)
  1324. return;
  1325. if (nullb->dev->zoned) {
  1326. nullb->dev->discard = false;
  1327. pr_info("discard option is ignored in zoned mode\n");
  1328. return;
  1329. }
  1330. nullb->q->limits.discard_granularity = nullb->dev->blocksize;
  1331. nullb->q->limits.discard_alignment = nullb->dev->blocksize;
  1332. blk_queue_max_discard_sectors(nullb->q, UINT_MAX >> 9);
  1333. blk_queue_flag_set(QUEUE_FLAG_DISCARD, nullb->q);
  1334. }
  1335. static const struct block_device_operations null_bio_ops = {
  1336. .owner = THIS_MODULE,
  1337. .submit_bio = null_submit_bio,
  1338. .report_zones = null_report_zones,
  1339. };
  1340. static const struct block_device_operations null_rq_ops = {
  1341. .owner = THIS_MODULE,
  1342. .report_zones = null_report_zones,
  1343. };
  1344. static int setup_commands(struct nullb_queue *nq)
  1345. {
  1346. struct nullb_cmd *cmd;
  1347. int i, tag_size;
  1348. nq->cmds = kcalloc(nq->queue_depth, sizeof(*cmd), GFP_KERNEL);
  1349. if (!nq->cmds)
  1350. return -ENOMEM;
  1351. tag_size = ALIGN(nq->queue_depth, BITS_PER_LONG) / BITS_PER_LONG;
  1352. nq->tag_map = kcalloc(tag_size, sizeof(unsigned long), GFP_KERNEL);
  1353. if (!nq->tag_map) {
  1354. kfree(nq->cmds);
  1355. return -ENOMEM;
  1356. }
  1357. for (i = 0; i < nq->queue_depth; i++) {
  1358. cmd = &nq->cmds[i];
  1359. cmd->tag = -1U;
  1360. }
  1361. return 0;
  1362. }
  1363. static int setup_queues(struct nullb *nullb)
  1364. {
  1365. nullb->queues = kcalloc(nr_cpu_ids, sizeof(struct nullb_queue),
  1366. GFP_KERNEL);
  1367. if (!nullb->queues)
  1368. return -ENOMEM;
  1369. nullb->queue_depth = nullb->dev->hw_queue_depth;
  1370. return 0;
  1371. }
  1372. static int init_driver_queues(struct nullb *nullb)
  1373. {
  1374. struct nullb_queue *nq;
  1375. int i, ret = 0;
  1376. for (i = 0; i < nullb->dev->submit_queues; i++) {
  1377. nq = &nullb->queues[i];
  1378. null_init_queue(nullb, nq);
  1379. ret = setup_commands(nq);
  1380. if (ret)
  1381. return ret;
  1382. nullb->nr_queues++;
  1383. }
  1384. return 0;
  1385. }
  1386. static int null_gendisk_register(struct nullb *nullb)
  1387. {
  1388. sector_t size = ((sector_t)nullb->dev->size * SZ_1M) >> SECTOR_SHIFT;
  1389. struct gendisk *disk;
  1390. disk = nullb->disk = alloc_disk_node(1, nullb->dev->home_node);
  1391. if (!disk)
  1392. return -ENOMEM;
  1393. set_capacity(disk, size);
  1394. disk->flags |= GENHD_FL_EXT_DEVT | GENHD_FL_SUPPRESS_PARTITION_INFO;
  1395. disk->major = null_major;
  1396. disk->first_minor = nullb->index;
  1397. if (queue_is_mq(nullb->q))
  1398. disk->fops = &null_rq_ops;
  1399. else
  1400. disk->fops = &null_bio_ops;
  1401. disk->private_data = nullb;
  1402. disk->queue = nullb->q;
  1403. strncpy(disk->disk_name, nullb->disk_name, DISK_NAME_LEN);
  1404. if (nullb->dev->zoned) {
  1405. int ret = null_register_zoned_dev(nullb);
  1406. if (ret)
  1407. return ret;
  1408. }
  1409. add_disk(disk);
  1410. return 0;
  1411. }
  1412. static int null_init_tag_set(struct nullb *nullb, struct blk_mq_tag_set *set)
  1413. {
  1414. set->ops = &null_mq_ops;
  1415. set->nr_hw_queues = nullb ? nullb->dev->submit_queues :
  1416. g_submit_queues;
  1417. set->queue_depth = nullb ? nullb->dev->hw_queue_depth :
  1418. g_hw_queue_depth;
  1419. set->numa_node = nullb ? nullb->dev->home_node : g_home_node;
  1420. set->cmd_size = sizeof(struct nullb_cmd);
  1421. set->flags = BLK_MQ_F_SHOULD_MERGE;
  1422. if (g_no_sched)
  1423. set->flags |= BLK_MQ_F_NO_SCHED;
  1424. if (g_shared_tag_bitmap)
  1425. set->flags |= BLK_MQ_F_TAG_HCTX_SHARED;
  1426. set->driver_data = NULL;
  1427. if ((nullb && nullb->dev->blocking) || g_blocking)
  1428. set->flags |= BLK_MQ_F_BLOCKING;
  1429. return blk_mq_alloc_tag_set(set);
  1430. }
  1431. static int null_validate_conf(struct nullb_device *dev)
  1432. {
  1433. dev->blocksize = round_down(dev->blocksize, 512);
  1434. dev->blocksize = clamp_t(unsigned int, dev->blocksize, 512, 4096);
  1435. if (dev->queue_mode == NULL_Q_MQ && dev->use_per_node_hctx) {
  1436. if (dev->submit_queues != nr_online_nodes)
  1437. dev->submit_queues = nr_online_nodes;
  1438. } else if (dev->submit_queues > nr_cpu_ids)
  1439. dev->submit_queues = nr_cpu_ids;
  1440. else if (dev->submit_queues == 0)
  1441. dev->submit_queues = 1;
  1442. dev->queue_mode = min_t(unsigned int, dev->queue_mode, NULL_Q_MQ);
  1443. dev->irqmode = min_t(unsigned int, dev->irqmode, NULL_IRQ_TIMER);
  1444. /* Do memory allocation, so set blocking */
  1445. if (dev->memory_backed)
  1446. dev->blocking = true;
  1447. else /* cache is meaningless */
  1448. dev->cache_size = 0;
  1449. dev->cache_size = min_t(unsigned long, ULONG_MAX / 1024 / 1024,
  1450. dev->cache_size);
  1451. dev->mbps = min_t(unsigned int, 1024 * 40, dev->mbps);
  1452. /* can not stop a queue */
  1453. if (dev->queue_mode == NULL_Q_BIO)
  1454. dev->mbps = 0;
  1455. if (dev->zoned &&
  1456. (!dev->zone_size || !is_power_of_2(dev->zone_size))) {
  1457. pr_err("zone_size must be power-of-two\n");
  1458. return -EINVAL;
  1459. }
  1460. return 0;
  1461. }
  1462. #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
  1463. static bool __null_setup_fault(struct fault_attr *attr, char *str)
  1464. {
  1465. if (!str[0])
  1466. return true;
  1467. if (!setup_fault_attr(attr, str))
  1468. return false;
  1469. attr->verbose = 0;
  1470. return true;
  1471. }
  1472. #endif
  1473. static bool null_setup_fault(void)
  1474. {
  1475. #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
  1476. if (!__null_setup_fault(&null_timeout_attr, g_timeout_str))
  1477. return false;
  1478. if (!__null_setup_fault(&null_requeue_attr, g_requeue_str))
  1479. return false;
  1480. if (!__null_setup_fault(&null_init_hctx_attr, g_init_hctx_str))
  1481. return false;
  1482. #endif
  1483. return true;
  1484. }
  1485. static int null_add_dev(struct nullb_device *dev)
  1486. {
  1487. struct nullb *nullb;
  1488. int rv;
  1489. rv = null_validate_conf(dev);
  1490. if (rv)
  1491. return rv;
  1492. nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node);
  1493. if (!nullb) {
  1494. rv = -ENOMEM;
  1495. goto out;
  1496. }
  1497. nullb->dev = dev;
  1498. dev->nullb = nullb;
  1499. spin_lock_init(&nullb->lock);
  1500. rv = setup_queues(nullb);
  1501. if (rv)
  1502. goto out_free_nullb;
  1503. if (dev->queue_mode == NULL_Q_MQ) {
  1504. if (shared_tags) {
  1505. nullb->tag_set = &tag_set;
  1506. rv = 0;
  1507. } else {
  1508. nullb->tag_set = &nullb->__tag_set;
  1509. rv = null_init_tag_set(nullb, nullb->tag_set);
  1510. }
  1511. if (rv)
  1512. goto out_cleanup_queues;
  1513. if (!null_setup_fault())
  1514. goto out_cleanup_queues;
  1515. nullb->tag_set->timeout = 5 * HZ;
  1516. nullb->q = blk_mq_init_queue_data(nullb->tag_set, nullb);
  1517. if (IS_ERR(nullb->q)) {
  1518. rv = -ENOMEM;
  1519. goto out_cleanup_tags;
  1520. }
  1521. } else if (dev->queue_mode == NULL_Q_BIO) {
  1522. nullb->q = blk_alloc_queue(dev->home_node);
  1523. if (!nullb->q) {
  1524. rv = -ENOMEM;
  1525. goto out_cleanup_queues;
  1526. }
  1527. rv = init_driver_queues(nullb);
  1528. if (rv)
  1529. goto out_cleanup_blk_queue;
  1530. }
  1531. if (dev->mbps) {
  1532. set_bit(NULLB_DEV_FL_THROTTLED, &dev->flags);
  1533. nullb_setup_bwtimer(nullb);
  1534. }
  1535. if (dev->cache_size > 0) {
  1536. set_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
  1537. blk_queue_write_cache(nullb->q, true, true);
  1538. }
  1539. if (dev->zoned) {
  1540. rv = null_init_zoned_dev(dev, nullb->q);
  1541. if (rv)
  1542. goto out_cleanup_blk_queue;
  1543. }
  1544. nullb->q->queuedata = nullb;
  1545. blk_queue_flag_set(QUEUE_FLAG_NONROT, nullb->q);
  1546. blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, nullb->q);
  1547. mutex_lock(&lock);
  1548. nullb->index = ida_simple_get(&nullb_indexes, 0, 0, GFP_KERNEL);
  1549. dev->index = nullb->index;
  1550. mutex_unlock(&lock);
  1551. blk_queue_logical_block_size(nullb->q, dev->blocksize);
  1552. blk_queue_physical_block_size(nullb->q, dev->blocksize);
  1553. null_config_discard(nullb);
  1554. sprintf(nullb->disk_name, "nullb%d", nullb->index);
  1555. rv = null_gendisk_register(nullb);
  1556. if (rv)
  1557. goto out_cleanup_zone;
  1558. mutex_lock(&lock);
  1559. list_add_tail(&nullb->list, &nullb_list);
  1560. mutex_unlock(&lock);
  1561. return 0;
  1562. out_cleanup_zone:
  1563. null_free_zoned_dev(dev);
  1564. out_cleanup_blk_queue:
  1565. blk_cleanup_queue(nullb->q);
  1566. out_cleanup_tags:
  1567. if (dev->queue_mode == NULL_Q_MQ && nullb->tag_set == &nullb->__tag_set)
  1568. blk_mq_free_tag_set(nullb->tag_set);
  1569. out_cleanup_queues:
  1570. cleanup_queues(nullb);
  1571. out_free_nullb:
  1572. kfree(nullb);
  1573. dev->nullb = NULL;
  1574. out:
  1575. return rv;
  1576. }
  1577. static int __init null_init(void)
  1578. {
  1579. int ret = 0;
  1580. unsigned int i;
  1581. struct nullb *nullb;
  1582. struct nullb_device *dev;
  1583. if (g_bs > PAGE_SIZE) {
  1584. pr_warn("invalid block size\n");
  1585. pr_warn("defaults block size to %lu\n", PAGE_SIZE);
  1586. g_bs = PAGE_SIZE;
  1587. }
  1588. if (g_home_node != NUMA_NO_NODE && g_home_node >= nr_online_nodes) {
  1589. pr_err("invalid home_node value\n");
  1590. g_home_node = NUMA_NO_NODE;
  1591. }
  1592. if (g_queue_mode == NULL_Q_RQ) {
  1593. pr_err("legacy IO path no longer available\n");
  1594. return -EINVAL;
  1595. }
  1596. if (g_queue_mode == NULL_Q_MQ && g_use_per_node_hctx) {
  1597. if (g_submit_queues != nr_online_nodes) {
  1598. pr_warn("submit_queues param is set to %u.\n",
  1599. nr_online_nodes);
  1600. g_submit_queues = nr_online_nodes;
  1601. }
  1602. } else if (g_submit_queues > nr_cpu_ids)
  1603. g_submit_queues = nr_cpu_ids;
  1604. else if (g_submit_queues <= 0)
  1605. g_submit_queues = 1;
  1606. if (g_queue_mode == NULL_Q_MQ && shared_tags) {
  1607. ret = null_init_tag_set(NULL, &tag_set);
  1608. if (ret)
  1609. return ret;
  1610. }
  1611. config_group_init(&nullb_subsys.su_group);
  1612. mutex_init(&nullb_subsys.su_mutex);
  1613. ret = configfs_register_subsystem(&nullb_subsys);
  1614. if (ret)
  1615. goto err_tagset;
  1616. mutex_init(&lock);
  1617. null_major = register_blkdev(0, "nullb");
  1618. if (null_major < 0) {
  1619. ret = null_major;
  1620. goto err_conf;
  1621. }
  1622. for (i = 0; i < nr_devices; i++) {
  1623. dev = null_alloc_dev();
  1624. if (!dev) {
  1625. ret = -ENOMEM;
  1626. goto err_dev;
  1627. }
  1628. ret = null_add_dev(dev);
  1629. if (ret) {
  1630. null_free_dev(dev);
  1631. goto err_dev;
  1632. }
  1633. }
  1634. pr_info("module loaded\n");
  1635. return 0;
  1636. err_dev:
  1637. while (!list_empty(&nullb_list)) {
  1638. nullb = list_entry(nullb_list.next, struct nullb, list);
  1639. dev = nullb->dev;
  1640. null_del_dev(nullb);
  1641. null_free_dev(dev);
  1642. }
  1643. unregister_blkdev(null_major, "nullb");
  1644. err_conf:
  1645. configfs_unregister_subsystem(&nullb_subsys);
  1646. err_tagset:
  1647. if (g_queue_mode == NULL_Q_MQ && shared_tags)
  1648. blk_mq_free_tag_set(&tag_set);
  1649. return ret;
  1650. }
  1651. static void __exit null_exit(void)
  1652. {
  1653. struct nullb *nullb;
  1654. configfs_unregister_subsystem(&nullb_subsys);
  1655. unregister_blkdev(null_major, "nullb");
  1656. mutex_lock(&lock);
  1657. while (!list_empty(&nullb_list)) {
  1658. struct nullb_device *dev;
  1659. nullb = list_entry(nullb_list.next, struct nullb, list);
  1660. dev = nullb->dev;
  1661. null_del_dev(nullb);
  1662. null_free_dev(dev);
  1663. }
  1664. mutex_unlock(&lock);
  1665. if (g_queue_mode == NULL_Q_MQ && shared_tags)
  1666. blk_mq_free_tag_set(&tag_set);
  1667. }
  1668. module_init(null_init);
  1669. module_exit(null_exit);
  1670. MODULE_AUTHOR("Jens Axboe <axboe@kernel.dk>");
  1671. MODULE_LICENSE("GPL");