target_core_user.c 76 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 2013 Shaohua Li <shli@kernel.org>
  4. * Copyright (C) 2014 Red Hat, Inc.
  5. * Copyright (C) 2015 Arrikto, Inc.
  6. * Copyright (C) 2017 Chinamobile, Inc.
  7. */
  8. #include <linux/spinlock.h>
  9. #include <linux/module.h>
  10. #include <linux/idr.h>
  11. #include <linux/kernel.h>
  12. #include <linux/timer.h>
  13. #include <linux/parser.h>
  14. #include <linux/vmalloc.h>
  15. #include <linux/uio_driver.h>
  16. #include <linux/radix-tree.h>
  17. #include <linux/stringify.h>
  18. #include <linux/bitops.h>
  19. #include <linux/highmem.h>
  20. #include <linux/configfs.h>
  21. #include <linux/mutex.h>
  22. #include <linux/workqueue.h>
  23. #include <net/genetlink.h>
  24. #include <scsi/scsi_common.h>
  25. #include <scsi/scsi_proto.h>
  26. #include <target/target_core_base.h>
  27. #include <target/target_core_fabric.h>
  28. #include <target/target_core_backend.h>
  29. #include <linux/target_core_user.h>
  30. /**
  31. * DOC: Userspace I/O
  32. * Userspace I/O
  33. * -------------
  34. *
  35. * Define a shared-memory interface for LIO to pass SCSI commands and
  36. * data to userspace for processing. This is to allow backends that
  37. * are too complex for in-kernel support to be possible.
  38. *
  39. * It uses the UIO framework to do a lot of the device-creation and
  40. * introspection work for us.
  41. *
  42. * See the .h file for how the ring is laid out. Note that while the
  43. * command ring is defined, the particulars of the data area are
  44. * not. Offset values in the command entry point to other locations
  45. * internal to the mmap-ed area. There is separate space outside the
  46. * command ring for data buffers. This leaves maximum flexibility for
  47. * moving buffer allocations, or even page flipping or other
  48. * allocation techniques, without altering the command ring layout.
  49. *
  50. * SECURITY:
  51. * The user process must be assumed to be malicious. There's no way to
  52. * prevent it breaking the command ring protocol if it wants, but in
  53. * order to prevent other issues we must only ever read *data* from
  54. * the shared memory area, not offsets or sizes. This applies to
  55. * command ring entries as well as the mailbox. Extra code needed for
  56. * this may have a 'UAM' comment.
  57. */
  58. #define TCMU_TIME_OUT (30 * MSEC_PER_SEC)
  59. /* For cmd area, the size is fixed 8MB */
  60. #define CMDR_SIZE (8 * 1024 * 1024)
  61. /*
  62. * For data area, the block size is PAGE_SIZE and
  63. * the total size is 256K * PAGE_SIZE.
  64. */
  65. #define DATA_BLOCK_SIZE PAGE_SIZE
  66. #define DATA_BLOCK_SHIFT PAGE_SHIFT
  67. #define DATA_BLOCK_BITS_DEF (256 * 1024)
  68. #define TCMU_MBS_TO_BLOCKS(_mbs) (_mbs << (20 - DATA_BLOCK_SHIFT))
  69. #define TCMU_BLOCKS_TO_MBS(_blocks) (_blocks >> (20 - DATA_BLOCK_SHIFT))
  70. /*
  71. * Default number of global data blocks(512K * PAGE_SIZE)
  72. * when the unmap thread will be started.
  73. */
  74. #define TCMU_GLOBAL_MAX_BLOCKS_DEF (512 * 1024)
  75. static u8 tcmu_kern_cmd_reply_supported;
  76. static u8 tcmu_netlink_blocked;
  77. static struct device *tcmu_root_device;
  78. struct tcmu_hba {
  79. u32 host_id;
  80. };
  81. #define TCMU_CONFIG_LEN 256
  82. static DEFINE_MUTEX(tcmu_nl_cmd_mutex);
  83. static LIST_HEAD(tcmu_nl_cmd_list);
  84. struct tcmu_dev;
  85. struct tcmu_nl_cmd {
  86. /* wake up thread waiting for reply */
  87. struct completion complete;
  88. struct list_head nl_list;
  89. struct tcmu_dev *udev;
  90. int cmd;
  91. int status;
  92. };
  93. struct tcmu_dev {
  94. struct list_head node;
  95. struct kref kref;
  96. struct se_device se_dev;
  97. char *name;
  98. struct se_hba *hba;
  99. #define TCMU_DEV_BIT_OPEN 0
  100. #define TCMU_DEV_BIT_BROKEN 1
  101. #define TCMU_DEV_BIT_BLOCKED 2
  102. #define TCMU_DEV_BIT_TMR_NOTIFY 3
  103. unsigned long flags;
  104. struct uio_info uio_info;
  105. struct inode *inode;
  106. struct tcmu_mailbox *mb_addr;
  107. uint64_t dev_size;
  108. u32 cmdr_size;
  109. u32 cmdr_last_cleaned;
  110. /* Offset of data area from start of mb */
  111. /* Must add data_off and mb_addr to get the address */
  112. size_t data_off;
  113. size_t data_size;
  114. uint32_t max_blocks;
  115. size_t ring_size;
  116. struct mutex cmdr_lock;
  117. struct list_head qfull_queue;
  118. struct list_head tmr_queue;
  119. uint32_t dbi_max;
  120. uint32_t dbi_thresh;
  121. unsigned long *data_bitmap;
  122. struct radix_tree_root data_blocks;
  123. struct idr commands;
  124. struct timer_list cmd_timer;
  125. unsigned int cmd_time_out;
  126. struct list_head inflight_queue;
  127. struct timer_list qfull_timer;
  128. int qfull_time_out;
  129. struct list_head timedout_entry;
  130. struct tcmu_nl_cmd curr_nl_cmd;
  131. char dev_config[TCMU_CONFIG_LEN];
  132. int nl_reply_supported;
  133. };
  134. #define TCMU_DEV(_se_dev) container_of(_se_dev, struct tcmu_dev, se_dev)
  135. #define CMDR_OFF sizeof(struct tcmu_mailbox)
  136. struct tcmu_cmd {
  137. struct se_cmd *se_cmd;
  138. struct tcmu_dev *tcmu_dev;
  139. struct list_head queue_entry;
  140. uint16_t cmd_id;
  141. /* Can't use se_cmd when cleaning up expired cmds, because if
  142. cmd has been completed then accessing se_cmd is off limits */
  143. uint32_t dbi_cnt;
  144. uint32_t dbi_bidi_cnt;
  145. uint32_t dbi_cur;
  146. uint32_t *dbi;
  147. uint32_t data_len_bidi;
  148. unsigned long deadline;
  149. #define TCMU_CMD_BIT_EXPIRED 0
  150. unsigned long flags;
  151. };
  152. struct tcmu_tmr {
  153. struct list_head queue_entry;
  154. uint8_t tmr_type;
  155. uint32_t tmr_cmd_cnt;
  156. int16_t tmr_cmd_ids[];
  157. };
  158. /*
  159. * To avoid dead lock the mutex lock order should always be:
  160. *
  161. * mutex_lock(&root_udev_mutex);
  162. * ...
  163. * mutex_lock(&tcmu_dev->cmdr_lock);
  164. * mutex_unlock(&tcmu_dev->cmdr_lock);
  165. * ...
  166. * mutex_unlock(&root_udev_mutex);
  167. */
  168. static DEFINE_MUTEX(root_udev_mutex);
  169. static LIST_HEAD(root_udev);
  170. static DEFINE_SPINLOCK(timed_out_udevs_lock);
  171. static LIST_HEAD(timed_out_udevs);
  172. static struct kmem_cache *tcmu_cmd_cache;
  173. static atomic_t global_db_count = ATOMIC_INIT(0);
  174. static struct delayed_work tcmu_unmap_work;
  175. static int tcmu_global_max_blocks = TCMU_GLOBAL_MAX_BLOCKS_DEF;
  176. static int tcmu_set_global_max_data_area(const char *str,
  177. const struct kernel_param *kp)
  178. {
  179. int ret, max_area_mb;
  180. ret = kstrtoint(str, 10, &max_area_mb);
  181. if (ret)
  182. return -EINVAL;
  183. if (max_area_mb <= 0) {
  184. pr_err("global_max_data_area must be larger than 0.\n");
  185. return -EINVAL;
  186. }
  187. tcmu_global_max_blocks = TCMU_MBS_TO_BLOCKS(max_area_mb);
  188. if (atomic_read(&global_db_count) > tcmu_global_max_blocks)
  189. schedule_delayed_work(&tcmu_unmap_work, 0);
  190. else
  191. cancel_delayed_work_sync(&tcmu_unmap_work);
  192. return 0;
  193. }
  194. static int tcmu_get_global_max_data_area(char *buffer,
  195. const struct kernel_param *kp)
  196. {
  197. return sprintf(buffer, "%d\n", TCMU_BLOCKS_TO_MBS(tcmu_global_max_blocks));
  198. }
  199. static const struct kernel_param_ops tcmu_global_max_data_area_op = {
  200. .set = tcmu_set_global_max_data_area,
  201. .get = tcmu_get_global_max_data_area,
  202. };
  203. module_param_cb(global_max_data_area_mb, &tcmu_global_max_data_area_op, NULL,
  204. S_IWUSR | S_IRUGO);
  205. MODULE_PARM_DESC(global_max_data_area_mb,
  206. "Max MBs allowed to be allocated to all the tcmu device's "
  207. "data areas.");
  208. static int tcmu_get_block_netlink(char *buffer,
  209. const struct kernel_param *kp)
  210. {
  211. return sprintf(buffer, "%s\n", tcmu_netlink_blocked ?
  212. "blocked" : "unblocked");
  213. }
  214. static int tcmu_set_block_netlink(const char *str,
  215. const struct kernel_param *kp)
  216. {
  217. int ret;
  218. u8 val;
  219. ret = kstrtou8(str, 0, &val);
  220. if (ret < 0)
  221. return ret;
  222. if (val > 1) {
  223. pr_err("Invalid block netlink value %u\n", val);
  224. return -EINVAL;
  225. }
  226. tcmu_netlink_blocked = val;
  227. return 0;
  228. }
  229. static const struct kernel_param_ops tcmu_block_netlink_op = {
  230. .set = tcmu_set_block_netlink,
  231. .get = tcmu_get_block_netlink,
  232. };
  233. module_param_cb(block_netlink, &tcmu_block_netlink_op, NULL, S_IWUSR | S_IRUGO);
  234. MODULE_PARM_DESC(block_netlink, "Block new netlink commands.");
  235. static int tcmu_fail_netlink_cmd(struct tcmu_nl_cmd *nl_cmd)
  236. {
  237. struct tcmu_dev *udev = nl_cmd->udev;
  238. if (!tcmu_netlink_blocked) {
  239. pr_err("Could not reset device's netlink interface. Netlink is not blocked.\n");
  240. return -EBUSY;
  241. }
  242. if (nl_cmd->cmd != TCMU_CMD_UNSPEC) {
  243. pr_debug("Aborting nl cmd %d on %s\n", nl_cmd->cmd, udev->name);
  244. nl_cmd->status = -EINTR;
  245. list_del(&nl_cmd->nl_list);
  246. complete(&nl_cmd->complete);
  247. }
  248. return 0;
  249. }
  250. static int tcmu_set_reset_netlink(const char *str,
  251. const struct kernel_param *kp)
  252. {
  253. struct tcmu_nl_cmd *nl_cmd, *tmp_cmd;
  254. int ret;
  255. u8 val;
  256. ret = kstrtou8(str, 0, &val);
  257. if (ret < 0)
  258. return ret;
  259. if (val != 1) {
  260. pr_err("Invalid reset netlink value %u\n", val);
  261. return -EINVAL;
  262. }
  263. mutex_lock(&tcmu_nl_cmd_mutex);
  264. list_for_each_entry_safe(nl_cmd, tmp_cmd, &tcmu_nl_cmd_list, nl_list) {
  265. ret = tcmu_fail_netlink_cmd(nl_cmd);
  266. if (ret)
  267. break;
  268. }
  269. mutex_unlock(&tcmu_nl_cmd_mutex);
  270. return ret;
  271. }
  272. static const struct kernel_param_ops tcmu_reset_netlink_op = {
  273. .set = tcmu_set_reset_netlink,
  274. };
  275. module_param_cb(reset_netlink, &tcmu_reset_netlink_op, NULL, S_IWUSR);
  276. MODULE_PARM_DESC(reset_netlink, "Reset netlink commands.");
  277. /* multicast group */
  278. enum tcmu_multicast_groups {
  279. TCMU_MCGRP_CONFIG,
  280. };
  281. static const struct genl_multicast_group tcmu_mcgrps[] = {
  282. [TCMU_MCGRP_CONFIG] = { .name = "config", },
  283. };
  284. static struct nla_policy tcmu_attr_policy[TCMU_ATTR_MAX+1] = {
  285. [TCMU_ATTR_DEVICE] = { .type = NLA_STRING },
  286. [TCMU_ATTR_MINOR] = { .type = NLA_U32 },
  287. [TCMU_ATTR_CMD_STATUS] = { .type = NLA_S32 },
  288. [TCMU_ATTR_DEVICE_ID] = { .type = NLA_U32 },
  289. [TCMU_ATTR_SUPP_KERN_CMD_REPLY] = { .type = NLA_U8 },
  290. };
  291. static int tcmu_genl_cmd_done(struct genl_info *info, int completed_cmd)
  292. {
  293. struct tcmu_dev *udev = NULL;
  294. struct tcmu_nl_cmd *nl_cmd;
  295. int dev_id, rc, ret = 0;
  296. if (!info->attrs[TCMU_ATTR_CMD_STATUS] ||
  297. !info->attrs[TCMU_ATTR_DEVICE_ID]) {
  298. printk(KERN_ERR "TCMU_ATTR_CMD_STATUS or TCMU_ATTR_DEVICE_ID not set, doing nothing\n");
  299. return -EINVAL;
  300. }
  301. dev_id = nla_get_u32(info->attrs[TCMU_ATTR_DEVICE_ID]);
  302. rc = nla_get_s32(info->attrs[TCMU_ATTR_CMD_STATUS]);
  303. mutex_lock(&tcmu_nl_cmd_mutex);
  304. list_for_each_entry(nl_cmd, &tcmu_nl_cmd_list, nl_list) {
  305. if (nl_cmd->udev->se_dev.dev_index == dev_id) {
  306. udev = nl_cmd->udev;
  307. break;
  308. }
  309. }
  310. if (!udev) {
  311. pr_err("tcmu nl cmd %u/%d completion could not find device with dev id %u.\n",
  312. completed_cmd, rc, dev_id);
  313. ret = -ENODEV;
  314. goto unlock;
  315. }
  316. list_del(&nl_cmd->nl_list);
  317. pr_debug("%s genl cmd done got id %d curr %d done %d rc %d stat %d\n",
  318. udev->name, dev_id, nl_cmd->cmd, completed_cmd, rc,
  319. nl_cmd->status);
  320. if (nl_cmd->cmd != completed_cmd) {
  321. pr_err("Mismatched commands on %s (Expecting reply for %d. Current %d).\n",
  322. udev->name, completed_cmd, nl_cmd->cmd);
  323. ret = -EINVAL;
  324. goto unlock;
  325. }
  326. nl_cmd->status = rc;
  327. complete(&nl_cmd->complete);
  328. unlock:
  329. mutex_unlock(&tcmu_nl_cmd_mutex);
  330. return ret;
  331. }
  332. static int tcmu_genl_rm_dev_done(struct sk_buff *skb, struct genl_info *info)
  333. {
  334. return tcmu_genl_cmd_done(info, TCMU_CMD_REMOVED_DEVICE);
  335. }
  336. static int tcmu_genl_add_dev_done(struct sk_buff *skb, struct genl_info *info)
  337. {
  338. return tcmu_genl_cmd_done(info, TCMU_CMD_ADDED_DEVICE);
  339. }
  340. static int tcmu_genl_reconfig_dev_done(struct sk_buff *skb,
  341. struct genl_info *info)
  342. {
  343. return tcmu_genl_cmd_done(info, TCMU_CMD_RECONFIG_DEVICE);
  344. }
  345. static int tcmu_genl_set_features(struct sk_buff *skb, struct genl_info *info)
  346. {
  347. if (info->attrs[TCMU_ATTR_SUPP_KERN_CMD_REPLY]) {
  348. tcmu_kern_cmd_reply_supported =
  349. nla_get_u8(info->attrs[TCMU_ATTR_SUPP_KERN_CMD_REPLY]);
  350. printk(KERN_INFO "tcmu daemon: command reply support %u.\n",
  351. tcmu_kern_cmd_reply_supported);
  352. }
  353. return 0;
  354. }
  355. static const struct genl_small_ops tcmu_genl_ops[] = {
  356. {
  357. .cmd = TCMU_CMD_SET_FEATURES,
  358. .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
  359. .flags = GENL_ADMIN_PERM,
  360. .doit = tcmu_genl_set_features,
  361. },
  362. {
  363. .cmd = TCMU_CMD_ADDED_DEVICE_DONE,
  364. .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
  365. .flags = GENL_ADMIN_PERM,
  366. .doit = tcmu_genl_add_dev_done,
  367. },
  368. {
  369. .cmd = TCMU_CMD_REMOVED_DEVICE_DONE,
  370. .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
  371. .flags = GENL_ADMIN_PERM,
  372. .doit = tcmu_genl_rm_dev_done,
  373. },
  374. {
  375. .cmd = TCMU_CMD_RECONFIG_DEVICE_DONE,
  376. .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
  377. .flags = GENL_ADMIN_PERM,
  378. .doit = tcmu_genl_reconfig_dev_done,
  379. },
  380. };
  381. /* Our generic netlink family */
  382. static struct genl_family tcmu_genl_family __ro_after_init = {
  383. .module = THIS_MODULE,
  384. .hdrsize = 0,
  385. .name = "TCM-USER",
  386. .version = 2,
  387. .maxattr = TCMU_ATTR_MAX,
  388. .policy = tcmu_attr_policy,
  389. .mcgrps = tcmu_mcgrps,
  390. .n_mcgrps = ARRAY_SIZE(tcmu_mcgrps),
  391. .netnsok = true,
  392. .small_ops = tcmu_genl_ops,
  393. .n_small_ops = ARRAY_SIZE(tcmu_genl_ops),
  394. };
  395. #define tcmu_cmd_set_dbi_cur(cmd, index) ((cmd)->dbi_cur = (index))
  396. #define tcmu_cmd_reset_dbi_cur(cmd) tcmu_cmd_set_dbi_cur(cmd, 0)
  397. #define tcmu_cmd_set_dbi(cmd, index) ((cmd)->dbi[(cmd)->dbi_cur++] = (index))
  398. #define tcmu_cmd_get_dbi(cmd) ((cmd)->dbi[(cmd)->dbi_cur++])
  399. static void tcmu_cmd_free_data(struct tcmu_cmd *tcmu_cmd, uint32_t len)
  400. {
  401. struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;
  402. uint32_t i;
  403. for (i = 0; i < len; i++)
  404. clear_bit(tcmu_cmd->dbi[i], udev->data_bitmap);
  405. }
  406. static inline int tcmu_get_empty_block(struct tcmu_dev *udev,
  407. struct tcmu_cmd *tcmu_cmd,
  408. int prev_dbi, int *iov_cnt)
  409. {
  410. struct page *page;
  411. int ret, dbi;
  412. dbi = find_first_zero_bit(udev->data_bitmap, udev->dbi_thresh);
  413. if (dbi == udev->dbi_thresh)
  414. return -1;
  415. page = radix_tree_lookup(&udev->data_blocks, dbi);
  416. if (!page) {
  417. if (atomic_add_return(1, &global_db_count) >
  418. tcmu_global_max_blocks)
  419. schedule_delayed_work(&tcmu_unmap_work, 0);
  420. /* try to get new page from the mm */
  421. page = alloc_page(GFP_NOIO);
  422. if (!page)
  423. goto err_alloc;
  424. ret = radix_tree_insert(&udev->data_blocks, dbi, page);
  425. if (ret)
  426. goto err_insert;
  427. }
  428. if (dbi > udev->dbi_max)
  429. udev->dbi_max = dbi;
  430. set_bit(dbi, udev->data_bitmap);
  431. tcmu_cmd_set_dbi(tcmu_cmd, dbi);
  432. if (dbi != prev_dbi + 1)
  433. *iov_cnt += 1;
  434. return dbi;
  435. err_insert:
  436. __free_page(page);
  437. err_alloc:
  438. atomic_dec(&global_db_count);
  439. return -1;
  440. }
  441. static int tcmu_get_empty_blocks(struct tcmu_dev *udev,
  442. struct tcmu_cmd *tcmu_cmd, int dbi_cnt)
  443. {
  444. /* start value of dbi + 1 must not be a valid dbi */
  445. int dbi = -2;
  446. int i, iov_cnt = 0;
  447. for (i = 0; i < dbi_cnt; i++) {
  448. dbi = tcmu_get_empty_block(udev, tcmu_cmd, dbi, &iov_cnt);
  449. if (dbi < 0)
  450. return -1;
  451. }
  452. return iov_cnt;
  453. }
  454. static inline struct page *
  455. tcmu_get_block_page(struct tcmu_dev *udev, uint32_t dbi)
  456. {
  457. return radix_tree_lookup(&udev->data_blocks, dbi);
  458. }
  459. static inline void tcmu_free_cmd(struct tcmu_cmd *tcmu_cmd)
  460. {
  461. kfree(tcmu_cmd->dbi);
  462. kmem_cache_free(tcmu_cmd_cache, tcmu_cmd);
  463. }
  464. static inline void tcmu_cmd_set_block_cnts(struct tcmu_cmd *cmd)
  465. {
  466. int i, len;
  467. struct se_cmd *se_cmd = cmd->se_cmd;
  468. cmd->dbi_cnt = DIV_ROUND_UP(se_cmd->data_length, DATA_BLOCK_SIZE);
  469. if (se_cmd->se_cmd_flags & SCF_BIDI) {
  470. BUG_ON(!(se_cmd->t_bidi_data_sg && se_cmd->t_bidi_data_nents));
  471. for (i = 0, len = 0; i < se_cmd->t_bidi_data_nents; i++)
  472. len += se_cmd->t_bidi_data_sg[i].length;
  473. cmd->dbi_bidi_cnt = DIV_ROUND_UP(len, DATA_BLOCK_SIZE);
  474. cmd->dbi_cnt += cmd->dbi_bidi_cnt;
  475. cmd->data_len_bidi = len;
  476. }
  477. }
  478. static int new_block_to_iov(struct tcmu_dev *udev, struct tcmu_cmd *cmd,
  479. struct iovec **iov, int prev_dbi, int *remain)
  480. {
  481. /* Get the next dbi */
  482. int dbi = tcmu_cmd_get_dbi(cmd);
  483. /* Do not add more than DATA_BLOCK_SIZE to iov */
  484. int len = min_t(int, DATA_BLOCK_SIZE, *remain);
  485. *remain -= len;
  486. /*
  487. * The following code will gather and map the blocks to the same iovec
  488. * when the blocks are all next to each other.
  489. */
  490. if (dbi != prev_dbi + 1) {
  491. /* dbi is not next to previous dbi, so start new iov */
  492. if (prev_dbi >= 0)
  493. (*iov)++;
  494. /* write offset relative to mb_addr */
  495. (*iov)->iov_base = (void __user *)
  496. (udev->data_off + dbi * DATA_BLOCK_SIZE);
  497. }
  498. (*iov)->iov_len += len;
  499. return dbi;
  500. }
  501. static void tcmu_setup_iovs(struct tcmu_dev *udev, struct tcmu_cmd *cmd,
  502. struct iovec **iov, int data_length)
  503. {
  504. /* start value of dbi + 1 must not be a valid dbi */
  505. int dbi = -2;
  506. /* We prepare the IOVs for DMA_FROM_DEVICE transfer direction */
  507. while (data_length > 0)
  508. dbi = new_block_to_iov(udev, cmd, iov, dbi, &data_length);
  509. }
  510. static struct tcmu_cmd *tcmu_alloc_cmd(struct se_cmd *se_cmd)
  511. {
  512. struct se_device *se_dev = se_cmd->se_dev;
  513. struct tcmu_dev *udev = TCMU_DEV(se_dev);
  514. struct tcmu_cmd *tcmu_cmd;
  515. tcmu_cmd = kmem_cache_zalloc(tcmu_cmd_cache, GFP_NOIO);
  516. if (!tcmu_cmd)
  517. return NULL;
  518. INIT_LIST_HEAD(&tcmu_cmd->queue_entry);
  519. tcmu_cmd->se_cmd = se_cmd;
  520. tcmu_cmd->tcmu_dev = udev;
  521. tcmu_cmd_set_block_cnts(tcmu_cmd);
  522. tcmu_cmd->dbi = kcalloc(tcmu_cmd->dbi_cnt, sizeof(uint32_t),
  523. GFP_NOIO);
  524. if (!tcmu_cmd->dbi) {
  525. kmem_cache_free(tcmu_cmd_cache, tcmu_cmd);
  526. return NULL;
  527. }
  528. return tcmu_cmd;
  529. }
  530. static inline void tcmu_flush_dcache_range(void *vaddr, size_t size)
  531. {
  532. unsigned long offset = offset_in_page(vaddr);
  533. void *start = vaddr - offset;
  534. size = round_up(size+offset, PAGE_SIZE);
  535. while (size) {
  536. flush_dcache_page(vmalloc_to_page(start));
  537. start += PAGE_SIZE;
  538. size -= PAGE_SIZE;
  539. }
  540. }
  541. /*
  542. * Some ring helper functions. We don't assume size is a power of 2 so
  543. * we can't use circ_buf.h.
  544. */
  545. static inline size_t spc_used(size_t head, size_t tail, size_t size)
  546. {
  547. int diff = head - tail;
  548. if (diff >= 0)
  549. return diff;
  550. else
  551. return size + diff;
  552. }
  553. static inline size_t spc_free(size_t head, size_t tail, size_t size)
  554. {
  555. /* Keep 1 byte unused or we can't tell full from empty */
  556. return (size - spc_used(head, tail, size) - 1);
  557. }
  558. static inline size_t head_to_end(size_t head, size_t size)
  559. {
  560. return size - head;
  561. }
  562. #define UPDATE_HEAD(head, used, size) smp_store_release(&head, ((head % size) + used) % size)
  563. static void scatter_data_area(struct tcmu_dev *udev, struct tcmu_cmd *tcmu_cmd,
  564. struct iovec **iov)
  565. {
  566. struct se_cmd *se_cmd = tcmu_cmd->se_cmd;
  567. /* start value of dbi + 1 must not be a valid dbi */
  568. int i, dbi = -2;
  569. int block_remaining = 0;
  570. int data_len = se_cmd->data_length;
  571. void *from, *to = NULL;
  572. size_t copy_bytes, offset;
  573. struct scatterlist *sg;
  574. struct page *page = NULL;
  575. for_each_sg(se_cmd->t_data_sg, sg, se_cmd->t_data_nents, i) {
  576. int sg_remaining = sg->length;
  577. from = kmap_atomic(sg_page(sg)) + sg->offset;
  578. while (sg_remaining > 0) {
  579. if (block_remaining == 0) {
  580. if (to) {
  581. flush_dcache_page(page);
  582. kunmap_atomic(to);
  583. }
  584. /* get next dbi and add to IOVs */
  585. dbi = new_block_to_iov(udev, tcmu_cmd, iov, dbi,
  586. &data_len);
  587. page = tcmu_get_block_page(udev, dbi);
  588. to = kmap_atomic(page);
  589. block_remaining = DATA_BLOCK_SIZE;
  590. }
  591. copy_bytes = min_t(size_t, sg_remaining,
  592. block_remaining);
  593. offset = DATA_BLOCK_SIZE - block_remaining;
  594. memcpy(to + offset, from + sg->length - sg_remaining,
  595. copy_bytes);
  596. sg_remaining -= copy_bytes;
  597. block_remaining -= copy_bytes;
  598. }
  599. kunmap_atomic(from - sg->offset);
  600. }
  601. if (to) {
  602. flush_dcache_page(page);
  603. kunmap_atomic(to);
  604. }
  605. }
  606. static void gather_data_area(struct tcmu_dev *udev, struct tcmu_cmd *cmd,
  607. bool bidi, uint32_t read_len)
  608. {
  609. struct se_cmd *se_cmd = cmd->se_cmd;
  610. int i, dbi;
  611. int block_remaining = 0;
  612. void *from = NULL, *to;
  613. size_t copy_bytes, offset;
  614. struct scatterlist *sg, *data_sg;
  615. struct page *page;
  616. unsigned int data_nents;
  617. uint32_t count = 0;
  618. if (!bidi) {
  619. data_sg = se_cmd->t_data_sg;
  620. data_nents = se_cmd->t_data_nents;
  621. } else {
  622. /*
  623. * For bidi case, the first count blocks are for Data-Out
  624. * buffer blocks, and before gathering the Data-In buffer
  625. * the Data-Out buffer blocks should be skipped.
  626. */
  627. count = cmd->dbi_cnt - cmd->dbi_bidi_cnt;
  628. data_sg = se_cmd->t_bidi_data_sg;
  629. data_nents = se_cmd->t_bidi_data_nents;
  630. }
  631. tcmu_cmd_set_dbi_cur(cmd, count);
  632. for_each_sg(data_sg, sg, data_nents, i) {
  633. int sg_remaining = sg->length;
  634. to = kmap_atomic(sg_page(sg)) + sg->offset;
  635. while (sg_remaining > 0 && read_len > 0) {
  636. if (block_remaining == 0) {
  637. if (from)
  638. kunmap_atomic(from);
  639. block_remaining = DATA_BLOCK_SIZE;
  640. dbi = tcmu_cmd_get_dbi(cmd);
  641. page = tcmu_get_block_page(udev, dbi);
  642. from = kmap_atomic(page);
  643. flush_dcache_page(page);
  644. }
  645. copy_bytes = min_t(size_t, sg_remaining,
  646. block_remaining);
  647. if (read_len < copy_bytes)
  648. copy_bytes = read_len;
  649. offset = DATA_BLOCK_SIZE - block_remaining;
  650. memcpy(to + sg->length - sg_remaining, from + offset,
  651. copy_bytes);
  652. sg_remaining -= copy_bytes;
  653. block_remaining -= copy_bytes;
  654. read_len -= copy_bytes;
  655. }
  656. kunmap_atomic(to - sg->offset);
  657. if (read_len == 0)
  658. break;
  659. }
  660. if (from)
  661. kunmap_atomic(from);
  662. }
  663. static inline size_t spc_bitmap_free(unsigned long *bitmap, uint32_t thresh)
  664. {
  665. return thresh - bitmap_weight(bitmap, thresh);
  666. }
  667. /*
  668. * We can't queue a command until we have space available on the cmd ring.
  669. *
  670. * Called with ring lock held.
  671. */
  672. static bool is_ring_space_avail(struct tcmu_dev *udev, size_t cmd_size)
  673. {
  674. struct tcmu_mailbox *mb = udev->mb_addr;
  675. size_t space, cmd_needed;
  676. u32 cmd_head;
  677. tcmu_flush_dcache_range(mb, sizeof(*mb));
  678. cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */
  679. /*
  680. * If cmd end-of-ring space is too small then we need space for a NOP plus
  681. * original cmd - cmds are internally contiguous.
  682. */
  683. if (head_to_end(cmd_head, udev->cmdr_size) >= cmd_size)
  684. cmd_needed = cmd_size;
  685. else
  686. cmd_needed = cmd_size + head_to_end(cmd_head, udev->cmdr_size);
  687. space = spc_free(cmd_head, udev->cmdr_last_cleaned, udev->cmdr_size);
  688. if (space < cmd_needed) {
  689. pr_debug("no cmd space: %u %u %u\n", cmd_head,
  690. udev->cmdr_last_cleaned, udev->cmdr_size);
  691. return false;
  692. }
  693. return true;
  694. }
  695. /*
  696. * We have to allocate data buffers before we can queue a command.
  697. * Returns -1 on error (not enough space) or number of needed iovs on success
  698. *
  699. * Called with ring lock held.
  700. */
  701. static int tcmu_alloc_data_space(struct tcmu_dev *udev, struct tcmu_cmd *cmd,
  702. int *iov_bidi_cnt)
  703. {
  704. int space, iov_cnt = 0, ret = 0;
  705. if (!cmd->dbi_cnt)
  706. goto wr_iov_cnts;
  707. /* try to check and get the data blocks as needed */
  708. space = spc_bitmap_free(udev->data_bitmap, udev->dbi_thresh);
  709. if (space < cmd->dbi_cnt) {
  710. unsigned long blocks_left =
  711. (udev->max_blocks - udev->dbi_thresh) + space;
  712. if (blocks_left < cmd->dbi_cnt) {
  713. pr_debug("no data space: only %lu available, but ask for %lu\n",
  714. blocks_left * DATA_BLOCK_SIZE,
  715. cmd->dbi_cnt * DATA_BLOCK_SIZE);
  716. return -1;
  717. }
  718. udev->dbi_thresh += cmd->dbi_cnt;
  719. if (udev->dbi_thresh > udev->max_blocks)
  720. udev->dbi_thresh = udev->max_blocks;
  721. }
  722. iov_cnt = tcmu_get_empty_blocks(udev, cmd,
  723. cmd->dbi_cnt - cmd->dbi_bidi_cnt);
  724. if (iov_cnt < 0)
  725. return -1;
  726. if (cmd->dbi_bidi_cnt) {
  727. ret = tcmu_get_empty_blocks(udev, cmd, cmd->dbi_bidi_cnt);
  728. if (ret < 0)
  729. return -1;
  730. }
  731. wr_iov_cnts:
  732. *iov_bidi_cnt = ret;
  733. return iov_cnt + ret;
  734. }
  735. static inline size_t tcmu_cmd_get_base_cmd_size(size_t iov_cnt)
  736. {
  737. return max(offsetof(struct tcmu_cmd_entry, req.iov[iov_cnt]),
  738. sizeof(struct tcmu_cmd_entry));
  739. }
  740. static inline size_t tcmu_cmd_get_cmd_size(struct tcmu_cmd *tcmu_cmd,
  741. size_t base_command_size)
  742. {
  743. struct se_cmd *se_cmd = tcmu_cmd->se_cmd;
  744. size_t command_size;
  745. command_size = base_command_size +
  746. round_up(scsi_command_size(se_cmd->t_task_cdb),
  747. TCMU_OP_ALIGN_SIZE);
  748. WARN_ON(command_size & (TCMU_OP_ALIGN_SIZE-1));
  749. return command_size;
  750. }
  751. static void tcmu_setup_cmd_timer(struct tcmu_cmd *tcmu_cmd, unsigned int tmo,
  752. struct timer_list *timer)
  753. {
  754. if (!tmo)
  755. return;
  756. tcmu_cmd->deadline = round_jiffies_up(jiffies + msecs_to_jiffies(tmo));
  757. if (!timer_pending(timer))
  758. mod_timer(timer, tcmu_cmd->deadline);
  759. pr_debug("Timeout set up for cmd %p, dev = %s, tmo = %lu\n", tcmu_cmd,
  760. tcmu_cmd->tcmu_dev->name, tmo / MSEC_PER_SEC);
  761. }
  762. static int add_to_qfull_queue(struct tcmu_cmd *tcmu_cmd)
  763. {
  764. struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;
  765. unsigned int tmo;
  766. /*
  767. * For backwards compat if qfull_time_out is not set use
  768. * cmd_time_out and if that's not set use the default time out.
  769. */
  770. if (!udev->qfull_time_out)
  771. return -ETIMEDOUT;
  772. else if (udev->qfull_time_out > 0)
  773. tmo = udev->qfull_time_out;
  774. else if (udev->cmd_time_out)
  775. tmo = udev->cmd_time_out;
  776. else
  777. tmo = TCMU_TIME_OUT;
  778. tcmu_setup_cmd_timer(tcmu_cmd, tmo, &udev->qfull_timer);
  779. list_add_tail(&tcmu_cmd->queue_entry, &udev->qfull_queue);
  780. pr_debug("adding cmd %p on dev %s to ring space wait queue\n",
  781. tcmu_cmd, udev->name);
  782. return 0;
  783. }
  784. static uint32_t ring_insert_padding(struct tcmu_dev *udev, size_t cmd_size)
  785. {
  786. struct tcmu_cmd_entry_hdr *hdr;
  787. struct tcmu_mailbox *mb = udev->mb_addr;
  788. uint32_t cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */
  789. /* Insert a PAD if end-of-ring space is too small */
  790. if (head_to_end(cmd_head, udev->cmdr_size) < cmd_size) {
  791. size_t pad_size = head_to_end(cmd_head, udev->cmdr_size);
  792. hdr = (void *) mb + CMDR_OFF + cmd_head;
  793. tcmu_hdr_set_op(&hdr->len_op, TCMU_OP_PAD);
  794. tcmu_hdr_set_len(&hdr->len_op, pad_size);
  795. hdr->cmd_id = 0; /* not used for PAD */
  796. hdr->kflags = 0;
  797. hdr->uflags = 0;
  798. tcmu_flush_dcache_range(hdr, sizeof(*hdr));
  799. UPDATE_HEAD(mb->cmd_head, pad_size, udev->cmdr_size);
  800. tcmu_flush_dcache_range(mb, sizeof(*mb));
  801. cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */
  802. WARN_ON(cmd_head != 0);
  803. }
  804. return cmd_head;
  805. }
  806. /**
  807. * queue_cmd_ring - queue cmd to ring or internally
  808. * @tcmu_cmd: cmd to queue
  809. * @scsi_err: TCM error code if failure (-1) returned.
  810. *
  811. * Returns:
  812. * -1 we cannot queue internally or to the ring.
  813. * 0 success
  814. * 1 internally queued to wait for ring memory to free.
  815. */
  816. static int queue_cmd_ring(struct tcmu_cmd *tcmu_cmd, sense_reason_t *scsi_err)
  817. {
  818. struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;
  819. struct se_cmd *se_cmd = tcmu_cmd->se_cmd;
  820. size_t base_command_size, command_size;
  821. struct tcmu_mailbox *mb = udev->mb_addr;
  822. struct tcmu_cmd_entry *entry;
  823. struct iovec *iov;
  824. int iov_cnt, iov_bidi_cnt, cmd_id;
  825. uint32_t cmd_head;
  826. uint64_t cdb_off;
  827. /* size of data buffer needed */
  828. size_t data_length = (size_t)tcmu_cmd->dbi_cnt * DATA_BLOCK_SIZE;
  829. *scsi_err = TCM_NO_SENSE;
  830. if (test_bit(TCMU_DEV_BIT_BLOCKED, &udev->flags)) {
  831. *scsi_err = TCM_LUN_BUSY;
  832. return -1;
  833. }
  834. if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) {
  835. *scsi_err = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
  836. return -1;
  837. }
  838. if (!list_empty(&udev->qfull_queue))
  839. goto queue;
  840. if (data_length > udev->data_size) {
  841. pr_warn("TCMU: Request of size %zu is too big for %zu data area\n",
  842. data_length, udev->data_size);
  843. *scsi_err = TCM_INVALID_CDB_FIELD;
  844. return -1;
  845. }
  846. iov_cnt = tcmu_alloc_data_space(udev, tcmu_cmd, &iov_bidi_cnt);
  847. if (iov_cnt < 0)
  848. goto free_and_queue;
  849. /*
  850. * Must be a certain minimum size for response sense info, but
  851. * also may be larger if the iov array is large.
  852. */
  853. base_command_size = tcmu_cmd_get_base_cmd_size(iov_cnt);
  854. command_size = tcmu_cmd_get_cmd_size(tcmu_cmd, base_command_size);
  855. if (command_size > (udev->cmdr_size / 2)) {
  856. pr_warn("TCMU: Request of size %zu is too big for %u cmd ring\n",
  857. command_size, udev->cmdr_size);
  858. tcmu_cmd_free_data(tcmu_cmd, tcmu_cmd->dbi_cur);
  859. *scsi_err = TCM_INVALID_CDB_FIELD;
  860. return -1;
  861. }
  862. if (!is_ring_space_avail(udev, command_size))
  863. /*
  864. * Don't leave commands partially setup because the unmap
  865. * thread might need the blocks to make forward progress.
  866. */
  867. goto free_and_queue;
  868. cmd_id = idr_alloc(&udev->commands, tcmu_cmd, 1, USHRT_MAX, GFP_NOWAIT);
  869. if (cmd_id < 0) {
  870. pr_err("tcmu: Could not allocate cmd id.\n");
  871. tcmu_cmd_free_data(tcmu_cmd, tcmu_cmd->dbi_cnt);
  872. *scsi_err = TCM_OUT_OF_RESOURCES;
  873. return -1;
  874. }
  875. tcmu_cmd->cmd_id = cmd_id;
  876. pr_debug("allocated cmd id %u for cmd %p dev %s\n", tcmu_cmd->cmd_id,
  877. tcmu_cmd, udev->name);
  878. cmd_head = ring_insert_padding(udev, command_size);
  879. entry = (void *) mb + CMDR_OFF + cmd_head;
  880. memset(entry, 0, command_size);
  881. tcmu_hdr_set_op(&entry->hdr.len_op, TCMU_OP_CMD);
  882. /* prepare iov list and copy data to data area if necessary */
  883. tcmu_cmd_reset_dbi_cur(tcmu_cmd);
  884. iov = &entry->req.iov[0];
  885. if (se_cmd->data_direction == DMA_TO_DEVICE ||
  886. se_cmd->se_cmd_flags & SCF_BIDI)
  887. scatter_data_area(udev, tcmu_cmd, &iov);
  888. else
  889. tcmu_setup_iovs(udev, tcmu_cmd, &iov, se_cmd->data_length);
  890. entry->req.iov_cnt = iov_cnt - iov_bidi_cnt;
  891. /* Handle BIDI commands */
  892. if (se_cmd->se_cmd_flags & SCF_BIDI) {
  893. iov++;
  894. tcmu_setup_iovs(udev, tcmu_cmd, &iov, tcmu_cmd->data_len_bidi);
  895. entry->req.iov_bidi_cnt = iov_bidi_cnt;
  896. }
  897. tcmu_setup_cmd_timer(tcmu_cmd, udev->cmd_time_out, &udev->cmd_timer);
  898. entry->hdr.cmd_id = tcmu_cmd->cmd_id;
  899. tcmu_hdr_set_len(&entry->hdr.len_op, command_size);
  900. /* All offsets relative to mb_addr, not start of entry! */
  901. cdb_off = CMDR_OFF + cmd_head + base_command_size;
  902. memcpy((void *) mb + cdb_off, se_cmd->t_task_cdb, scsi_command_size(se_cmd->t_task_cdb));
  903. entry->req.cdb_off = cdb_off;
  904. tcmu_flush_dcache_range(entry, command_size);
  905. UPDATE_HEAD(mb->cmd_head, command_size, udev->cmdr_size);
  906. tcmu_flush_dcache_range(mb, sizeof(*mb));
  907. list_add_tail(&tcmu_cmd->queue_entry, &udev->inflight_queue);
  908. /* TODO: only if FLUSH and FUA? */
  909. uio_event_notify(&udev->uio_info);
  910. return 0;
  911. free_and_queue:
  912. tcmu_cmd_free_data(tcmu_cmd, tcmu_cmd->dbi_cur);
  913. tcmu_cmd_reset_dbi_cur(tcmu_cmd);
  914. queue:
  915. if (add_to_qfull_queue(tcmu_cmd)) {
  916. *scsi_err = TCM_OUT_OF_RESOURCES;
  917. return -1;
  918. }
  919. return 1;
  920. }
  921. /**
  922. * queue_tmr_ring - queue tmr info to ring or internally
  923. * @udev: related tcmu_dev
  924. * @tmr: tcmu_tmr containing tmr info to queue
  925. *
  926. * Returns:
  927. * 0 success
  928. * 1 internally queued to wait for ring memory to free.
  929. */
  930. static int
  931. queue_tmr_ring(struct tcmu_dev *udev, struct tcmu_tmr *tmr)
  932. {
  933. struct tcmu_tmr_entry *entry;
  934. int cmd_size;
  935. int id_list_sz;
  936. struct tcmu_mailbox *mb = udev->mb_addr;
  937. uint32_t cmd_head;
  938. if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags))
  939. goto out_free;
  940. id_list_sz = sizeof(tmr->tmr_cmd_ids[0]) * tmr->tmr_cmd_cnt;
  941. cmd_size = round_up(sizeof(*entry) + id_list_sz, TCMU_OP_ALIGN_SIZE);
  942. if (!list_empty(&udev->tmr_queue) ||
  943. !is_ring_space_avail(udev, cmd_size)) {
  944. list_add_tail(&tmr->queue_entry, &udev->tmr_queue);
  945. pr_debug("adding tmr %p on dev %s to TMR ring space wait queue\n",
  946. tmr, udev->name);
  947. return 1;
  948. }
  949. cmd_head = ring_insert_padding(udev, cmd_size);
  950. entry = (void *)mb + CMDR_OFF + cmd_head;
  951. memset(entry, 0, cmd_size);
  952. tcmu_hdr_set_op(&entry->hdr.len_op, TCMU_OP_TMR);
  953. tcmu_hdr_set_len(&entry->hdr.len_op, cmd_size);
  954. entry->tmr_type = tmr->tmr_type;
  955. entry->cmd_cnt = tmr->tmr_cmd_cnt;
  956. memcpy(&entry->cmd_ids[0], &tmr->tmr_cmd_ids[0], id_list_sz);
  957. tcmu_flush_dcache_range(entry, cmd_size);
  958. UPDATE_HEAD(mb->cmd_head, cmd_size, udev->cmdr_size);
  959. tcmu_flush_dcache_range(mb, sizeof(*mb));
  960. uio_event_notify(&udev->uio_info);
  961. out_free:
  962. kfree(tmr);
  963. return 0;
  964. }
  965. static sense_reason_t
  966. tcmu_queue_cmd(struct se_cmd *se_cmd)
  967. {
  968. struct se_device *se_dev = se_cmd->se_dev;
  969. struct tcmu_dev *udev = TCMU_DEV(se_dev);
  970. struct tcmu_cmd *tcmu_cmd;
  971. sense_reason_t scsi_ret = TCM_CHECK_CONDITION_ABORT_CMD;
  972. int ret = -1;
  973. tcmu_cmd = tcmu_alloc_cmd(se_cmd);
  974. if (!tcmu_cmd)
  975. return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
  976. mutex_lock(&udev->cmdr_lock);
  977. if (!(se_cmd->transport_state & CMD_T_ABORTED))
  978. ret = queue_cmd_ring(tcmu_cmd, &scsi_ret);
  979. if (ret < 0)
  980. tcmu_free_cmd(tcmu_cmd);
  981. else
  982. se_cmd->priv = tcmu_cmd;
  983. mutex_unlock(&udev->cmdr_lock);
  984. return scsi_ret;
  985. }
  986. static void tcmu_set_next_deadline(struct list_head *queue,
  987. struct timer_list *timer)
  988. {
  989. struct tcmu_cmd *cmd;
  990. if (!list_empty(queue)) {
  991. cmd = list_first_entry(queue, struct tcmu_cmd, queue_entry);
  992. mod_timer(timer, cmd->deadline);
  993. } else
  994. del_timer(timer);
  995. }
  996. static int
  997. tcmu_tmr_type(enum tcm_tmreq_table tmf)
  998. {
  999. switch (tmf) {
  1000. case TMR_ABORT_TASK: return TCMU_TMR_ABORT_TASK;
  1001. case TMR_ABORT_TASK_SET: return TCMU_TMR_ABORT_TASK_SET;
  1002. case TMR_CLEAR_ACA: return TCMU_TMR_CLEAR_ACA;
  1003. case TMR_CLEAR_TASK_SET: return TCMU_TMR_CLEAR_TASK_SET;
  1004. case TMR_LUN_RESET: return TCMU_TMR_LUN_RESET;
  1005. case TMR_TARGET_WARM_RESET: return TCMU_TMR_TARGET_WARM_RESET;
  1006. case TMR_TARGET_COLD_RESET: return TCMU_TMR_TARGET_COLD_RESET;
  1007. case TMR_LUN_RESET_PRO: return TCMU_TMR_LUN_RESET_PRO;
  1008. default: return TCMU_TMR_UNKNOWN;
  1009. }
  1010. }
  1011. static void
  1012. tcmu_tmr_notify(struct se_device *se_dev, enum tcm_tmreq_table tmf,
  1013. struct list_head *cmd_list)
  1014. {
  1015. int i = 0, cmd_cnt = 0;
  1016. bool unqueued = false;
  1017. uint16_t *cmd_ids = NULL;
  1018. struct tcmu_cmd *cmd;
  1019. struct se_cmd *se_cmd;
  1020. struct tcmu_tmr *tmr;
  1021. struct tcmu_dev *udev = TCMU_DEV(se_dev);
  1022. mutex_lock(&udev->cmdr_lock);
  1023. /* First we check for aborted commands in qfull_queue */
  1024. list_for_each_entry(se_cmd, cmd_list, state_list) {
  1025. i++;
  1026. if (!se_cmd->priv)
  1027. continue;
  1028. cmd = se_cmd->priv;
  1029. /* Commands on qfull queue have no id yet */
  1030. if (cmd->cmd_id) {
  1031. cmd_cnt++;
  1032. continue;
  1033. }
  1034. pr_debug("Removing aborted command %p from queue on dev %s.\n",
  1035. cmd, udev->name);
  1036. list_del_init(&cmd->queue_entry);
  1037. tcmu_free_cmd(cmd);
  1038. se_cmd->priv = NULL;
  1039. target_complete_cmd(se_cmd, SAM_STAT_TASK_ABORTED);
  1040. unqueued = true;
  1041. }
  1042. if (unqueued)
  1043. tcmu_set_next_deadline(&udev->qfull_queue, &udev->qfull_timer);
  1044. if (!test_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags))
  1045. goto unlock;
  1046. pr_debug("TMR event %d on dev %s, aborted cmds %d, afflicted cmd_ids %d\n",
  1047. tcmu_tmr_type(tmf), udev->name, i, cmd_cnt);
  1048. tmr = kmalloc(sizeof(*tmr) + cmd_cnt * sizeof(*cmd_ids), GFP_KERNEL);
  1049. if (!tmr)
  1050. goto unlock;
  1051. tmr->tmr_type = tcmu_tmr_type(tmf);
  1052. tmr->tmr_cmd_cnt = cmd_cnt;
  1053. if (cmd_cnt != 0) {
  1054. cmd_cnt = 0;
  1055. list_for_each_entry(se_cmd, cmd_list, state_list) {
  1056. if (!se_cmd->priv)
  1057. continue;
  1058. cmd = se_cmd->priv;
  1059. if (cmd->cmd_id)
  1060. tmr->tmr_cmd_ids[cmd_cnt++] = cmd->cmd_id;
  1061. }
  1062. }
  1063. queue_tmr_ring(udev, tmr);
  1064. unlock:
  1065. mutex_unlock(&udev->cmdr_lock);
  1066. }
  1067. static void tcmu_handle_completion(struct tcmu_cmd *cmd, struct tcmu_cmd_entry *entry)
  1068. {
  1069. struct se_cmd *se_cmd = cmd->se_cmd;
  1070. struct tcmu_dev *udev = cmd->tcmu_dev;
  1071. bool read_len_valid = false;
  1072. uint32_t read_len;
  1073. /*
  1074. * cmd has been completed already from timeout, just reclaim
  1075. * data area space and free cmd
  1076. */
  1077. if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) {
  1078. WARN_ON_ONCE(se_cmd);
  1079. goto out;
  1080. }
  1081. list_del_init(&cmd->queue_entry);
  1082. tcmu_cmd_reset_dbi_cur(cmd);
  1083. if (entry->hdr.uflags & TCMU_UFLAG_UNKNOWN_OP) {
  1084. pr_warn("TCMU: Userspace set UNKNOWN_OP flag on se_cmd %p\n",
  1085. cmd->se_cmd);
  1086. entry->rsp.scsi_status = SAM_STAT_CHECK_CONDITION;
  1087. goto done;
  1088. }
  1089. read_len = se_cmd->data_length;
  1090. if (se_cmd->data_direction == DMA_FROM_DEVICE &&
  1091. (entry->hdr.uflags & TCMU_UFLAG_READ_LEN) && entry->rsp.read_len) {
  1092. read_len_valid = true;
  1093. if (entry->rsp.read_len < read_len)
  1094. read_len = entry->rsp.read_len;
  1095. }
  1096. if (entry->rsp.scsi_status == SAM_STAT_CHECK_CONDITION) {
  1097. transport_copy_sense_to_cmd(se_cmd, entry->rsp.sense_buffer);
  1098. if (!read_len_valid )
  1099. goto done;
  1100. else
  1101. se_cmd->se_cmd_flags |= SCF_TREAT_READ_AS_NORMAL;
  1102. }
  1103. if (se_cmd->se_cmd_flags & SCF_BIDI) {
  1104. /* Get Data-In buffer before clean up */
  1105. gather_data_area(udev, cmd, true, read_len);
  1106. } else if (se_cmd->data_direction == DMA_FROM_DEVICE) {
  1107. gather_data_area(udev, cmd, false, read_len);
  1108. } else if (se_cmd->data_direction == DMA_TO_DEVICE) {
  1109. /* TODO: */
  1110. } else if (se_cmd->data_direction != DMA_NONE) {
  1111. pr_warn("TCMU: data direction was %d!\n",
  1112. se_cmd->data_direction);
  1113. }
  1114. done:
  1115. se_cmd->priv = NULL;
  1116. if (read_len_valid) {
  1117. pr_debug("read_len = %d\n", read_len);
  1118. target_complete_cmd_with_length(cmd->se_cmd,
  1119. entry->rsp.scsi_status, read_len);
  1120. } else
  1121. target_complete_cmd(cmd->se_cmd, entry->rsp.scsi_status);
  1122. out:
  1123. tcmu_cmd_free_data(cmd, cmd->dbi_cnt);
  1124. tcmu_free_cmd(cmd);
  1125. }
  1126. static int tcmu_run_tmr_queue(struct tcmu_dev *udev)
  1127. {
  1128. struct tcmu_tmr *tmr, *tmp;
  1129. LIST_HEAD(tmrs);
  1130. if (list_empty(&udev->tmr_queue))
  1131. return 1;
  1132. pr_debug("running %s's tmr queue\n", udev->name);
  1133. list_splice_init(&udev->tmr_queue, &tmrs);
  1134. list_for_each_entry_safe(tmr, tmp, &tmrs, queue_entry) {
  1135. list_del_init(&tmr->queue_entry);
  1136. pr_debug("removing tmr %p on dev %s from queue\n",
  1137. tmr, udev->name);
  1138. if (queue_tmr_ring(udev, tmr)) {
  1139. pr_debug("ran out of space during tmr queue run\n");
  1140. /*
  1141. * tmr was requeued, so just put all tmrs back in
  1142. * the queue
  1143. */
  1144. list_splice_tail(&tmrs, &udev->tmr_queue);
  1145. return 0;
  1146. }
  1147. }
  1148. return 1;
  1149. }
  1150. static bool tcmu_handle_completions(struct tcmu_dev *udev)
  1151. {
  1152. struct tcmu_mailbox *mb;
  1153. struct tcmu_cmd *cmd;
  1154. bool free_space = false;
  1155. if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) {
  1156. pr_err("ring broken, not handling completions\n");
  1157. return 0;
  1158. }
  1159. mb = udev->mb_addr;
  1160. tcmu_flush_dcache_range(mb, sizeof(*mb));
  1161. while (udev->cmdr_last_cleaned != READ_ONCE(mb->cmd_tail)) {
  1162. struct tcmu_cmd_entry *entry = (void *) mb + CMDR_OFF + udev->cmdr_last_cleaned;
  1163. /*
  1164. * Flush max. up to end of cmd ring since current entry might
  1165. * be a padding that is shorter than sizeof(*entry)
  1166. */
  1167. size_t ring_left = head_to_end(udev->cmdr_last_cleaned,
  1168. udev->cmdr_size);
  1169. tcmu_flush_dcache_range(entry, ring_left < sizeof(*entry) ?
  1170. ring_left : sizeof(*entry));
  1171. free_space = true;
  1172. if (tcmu_hdr_get_op(entry->hdr.len_op) == TCMU_OP_PAD ||
  1173. tcmu_hdr_get_op(entry->hdr.len_op) == TCMU_OP_TMR) {
  1174. UPDATE_HEAD(udev->cmdr_last_cleaned,
  1175. tcmu_hdr_get_len(entry->hdr.len_op),
  1176. udev->cmdr_size);
  1177. continue;
  1178. }
  1179. WARN_ON(tcmu_hdr_get_op(entry->hdr.len_op) != TCMU_OP_CMD);
  1180. cmd = idr_remove(&udev->commands, entry->hdr.cmd_id);
  1181. if (!cmd) {
  1182. pr_err("cmd_id %u not found, ring is broken\n",
  1183. entry->hdr.cmd_id);
  1184. set_bit(TCMU_DEV_BIT_BROKEN, &udev->flags);
  1185. return false;
  1186. }
  1187. tcmu_handle_completion(cmd, entry);
  1188. UPDATE_HEAD(udev->cmdr_last_cleaned,
  1189. tcmu_hdr_get_len(entry->hdr.len_op),
  1190. udev->cmdr_size);
  1191. }
  1192. if (free_space)
  1193. free_space = tcmu_run_tmr_queue(udev);
  1194. if (atomic_read(&global_db_count) > tcmu_global_max_blocks &&
  1195. idr_is_empty(&udev->commands) && list_empty(&udev->qfull_queue)) {
  1196. /*
  1197. * Allocated blocks exceeded global block limit, currently no
  1198. * more pending or waiting commands so try to reclaim blocks.
  1199. */
  1200. schedule_delayed_work(&tcmu_unmap_work, 0);
  1201. }
  1202. if (udev->cmd_time_out)
  1203. tcmu_set_next_deadline(&udev->inflight_queue, &udev->cmd_timer);
  1204. return free_space;
  1205. }
  1206. static void tcmu_check_expired_ring_cmd(struct tcmu_cmd *cmd)
  1207. {
  1208. struct se_cmd *se_cmd;
  1209. if (!time_after_eq(jiffies, cmd->deadline))
  1210. return;
  1211. set_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags);
  1212. list_del_init(&cmd->queue_entry);
  1213. se_cmd = cmd->se_cmd;
  1214. se_cmd->priv = NULL;
  1215. cmd->se_cmd = NULL;
  1216. pr_debug("Timing out inflight cmd %u on dev %s.\n",
  1217. cmd->cmd_id, cmd->tcmu_dev->name);
  1218. target_complete_cmd(se_cmd, SAM_STAT_CHECK_CONDITION);
  1219. }
  1220. static void tcmu_check_expired_queue_cmd(struct tcmu_cmd *cmd)
  1221. {
  1222. struct se_cmd *se_cmd;
  1223. if (!time_after_eq(jiffies, cmd->deadline))
  1224. return;
  1225. pr_debug("Timing out queued cmd %p on dev %s.\n",
  1226. cmd, cmd->tcmu_dev->name);
  1227. list_del_init(&cmd->queue_entry);
  1228. se_cmd = cmd->se_cmd;
  1229. tcmu_free_cmd(cmd);
  1230. se_cmd->priv = NULL;
  1231. target_complete_cmd(se_cmd, SAM_STAT_TASK_SET_FULL);
  1232. }
  1233. static void tcmu_device_timedout(struct tcmu_dev *udev)
  1234. {
  1235. spin_lock(&timed_out_udevs_lock);
  1236. if (list_empty(&udev->timedout_entry))
  1237. list_add_tail(&udev->timedout_entry, &timed_out_udevs);
  1238. spin_unlock(&timed_out_udevs_lock);
  1239. schedule_delayed_work(&tcmu_unmap_work, 0);
  1240. }
  1241. static void tcmu_cmd_timedout(struct timer_list *t)
  1242. {
  1243. struct tcmu_dev *udev = from_timer(udev, t, cmd_timer);
  1244. pr_debug("%s cmd timeout has expired\n", udev->name);
  1245. tcmu_device_timedout(udev);
  1246. }
  1247. static void tcmu_qfull_timedout(struct timer_list *t)
  1248. {
  1249. struct tcmu_dev *udev = from_timer(udev, t, qfull_timer);
  1250. pr_debug("%s qfull timeout has expired\n", udev->name);
  1251. tcmu_device_timedout(udev);
  1252. }
  1253. static int tcmu_attach_hba(struct se_hba *hba, u32 host_id)
  1254. {
  1255. struct tcmu_hba *tcmu_hba;
  1256. tcmu_hba = kzalloc(sizeof(struct tcmu_hba), GFP_KERNEL);
  1257. if (!tcmu_hba)
  1258. return -ENOMEM;
  1259. tcmu_hba->host_id = host_id;
  1260. hba->hba_ptr = tcmu_hba;
  1261. return 0;
  1262. }
  1263. static void tcmu_detach_hba(struct se_hba *hba)
  1264. {
  1265. kfree(hba->hba_ptr);
  1266. hba->hba_ptr = NULL;
  1267. }
  1268. static struct se_device *tcmu_alloc_device(struct se_hba *hba, const char *name)
  1269. {
  1270. struct tcmu_dev *udev;
  1271. udev = kzalloc(sizeof(struct tcmu_dev), GFP_KERNEL);
  1272. if (!udev)
  1273. return NULL;
  1274. kref_init(&udev->kref);
  1275. udev->name = kstrdup(name, GFP_KERNEL);
  1276. if (!udev->name) {
  1277. kfree(udev);
  1278. return NULL;
  1279. }
  1280. udev->hba = hba;
  1281. udev->cmd_time_out = TCMU_TIME_OUT;
  1282. udev->qfull_time_out = -1;
  1283. udev->max_blocks = DATA_BLOCK_BITS_DEF;
  1284. mutex_init(&udev->cmdr_lock);
  1285. INIT_LIST_HEAD(&udev->node);
  1286. INIT_LIST_HEAD(&udev->timedout_entry);
  1287. INIT_LIST_HEAD(&udev->qfull_queue);
  1288. INIT_LIST_HEAD(&udev->tmr_queue);
  1289. INIT_LIST_HEAD(&udev->inflight_queue);
  1290. idr_init(&udev->commands);
  1291. timer_setup(&udev->qfull_timer, tcmu_qfull_timedout, 0);
  1292. timer_setup(&udev->cmd_timer, tcmu_cmd_timedout, 0);
  1293. INIT_RADIX_TREE(&udev->data_blocks, GFP_KERNEL);
  1294. return &udev->se_dev;
  1295. }
  1296. static void run_qfull_queue(struct tcmu_dev *udev, bool fail)
  1297. {
  1298. struct tcmu_cmd *tcmu_cmd, *tmp_cmd;
  1299. LIST_HEAD(cmds);
  1300. sense_reason_t scsi_ret;
  1301. int ret;
  1302. if (list_empty(&udev->qfull_queue))
  1303. return;
  1304. pr_debug("running %s's cmdr queue forcefail %d\n", udev->name, fail);
  1305. list_splice_init(&udev->qfull_queue, &cmds);
  1306. list_for_each_entry_safe(tcmu_cmd, tmp_cmd, &cmds, queue_entry) {
  1307. list_del_init(&tcmu_cmd->queue_entry);
  1308. pr_debug("removing cmd %p on dev %s from queue\n",
  1309. tcmu_cmd, udev->name);
  1310. if (fail) {
  1311. /*
  1312. * We were not able to even start the command, so
  1313. * fail with busy to allow a retry in case runner
  1314. * was only temporarily down. If the device is being
  1315. * removed then LIO core will do the right thing and
  1316. * fail the retry.
  1317. */
  1318. tcmu_cmd->se_cmd->priv = NULL;
  1319. target_complete_cmd(tcmu_cmd->se_cmd, SAM_STAT_BUSY);
  1320. tcmu_free_cmd(tcmu_cmd);
  1321. continue;
  1322. }
  1323. ret = queue_cmd_ring(tcmu_cmd, &scsi_ret);
  1324. if (ret < 0) {
  1325. pr_debug("cmd %p on dev %s failed with %u\n",
  1326. tcmu_cmd, udev->name, scsi_ret);
  1327. /*
  1328. * Ignore scsi_ret for now. target_complete_cmd
  1329. * drops it.
  1330. */
  1331. tcmu_cmd->se_cmd->priv = NULL;
  1332. target_complete_cmd(tcmu_cmd->se_cmd,
  1333. SAM_STAT_CHECK_CONDITION);
  1334. tcmu_free_cmd(tcmu_cmd);
  1335. } else if (ret > 0) {
  1336. pr_debug("ran out of space during cmdr queue run\n");
  1337. /*
  1338. * cmd was requeued, so just put all cmds back in
  1339. * the queue
  1340. */
  1341. list_splice_tail(&cmds, &udev->qfull_queue);
  1342. break;
  1343. }
  1344. }
  1345. tcmu_set_next_deadline(&udev->qfull_queue, &udev->qfull_timer);
  1346. }
  1347. static int tcmu_irqcontrol(struct uio_info *info, s32 irq_on)
  1348. {
  1349. struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
  1350. mutex_lock(&udev->cmdr_lock);
  1351. if (tcmu_handle_completions(udev))
  1352. run_qfull_queue(udev, false);
  1353. mutex_unlock(&udev->cmdr_lock);
  1354. return 0;
  1355. }
  1356. /*
  1357. * mmap code from uio.c. Copied here because we want to hook mmap()
  1358. * and this stuff must come along.
  1359. */
  1360. static int tcmu_find_mem_index(struct vm_area_struct *vma)
  1361. {
  1362. struct tcmu_dev *udev = vma->vm_private_data;
  1363. struct uio_info *info = &udev->uio_info;
  1364. if (vma->vm_pgoff < MAX_UIO_MAPS) {
  1365. if (info->mem[vma->vm_pgoff].size == 0)
  1366. return -1;
  1367. return (int)vma->vm_pgoff;
  1368. }
  1369. return -1;
  1370. }
  1371. static struct page *tcmu_try_get_block_page(struct tcmu_dev *udev, uint32_t dbi)
  1372. {
  1373. struct page *page;
  1374. mutex_lock(&udev->cmdr_lock);
  1375. page = tcmu_get_block_page(udev, dbi);
  1376. if (likely(page)) {
  1377. get_page(page);
  1378. mutex_unlock(&udev->cmdr_lock);
  1379. return page;
  1380. }
  1381. /*
  1382. * Userspace messed up and passed in a address not in the
  1383. * data iov passed to it.
  1384. */
  1385. pr_err("Invalid addr to data block mapping (dbi %u) on device %s\n",
  1386. dbi, udev->name);
  1387. page = NULL;
  1388. mutex_unlock(&udev->cmdr_lock);
  1389. return page;
  1390. }
  1391. static vm_fault_t tcmu_vma_fault(struct vm_fault *vmf)
  1392. {
  1393. struct tcmu_dev *udev = vmf->vma->vm_private_data;
  1394. struct uio_info *info = &udev->uio_info;
  1395. struct page *page;
  1396. unsigned long offset;
  1397. void *addr;
  1398. int mi = tcmu_find_mem_index(vmf->vma);
  1399. if (mi < 0)
  1400. return VM_FAULT_SIGBUS;
  1401. /*
  1402. * We need to subtract mi because userspace uses offset = N*PAGE_SIZE
  1403. * to use mem[N].
  1404. */
  1405. offset = (vmf->pgoff - mi) << PAGE_SHIFT;
  1406. if (offset < udev->data_off) {
  1407. /* For the vmalloc()ed cmd area pages */
  1408. addr = (void *)(unsigned long)info->mem[mi].addr + offset;
  1409. page = vmalloc_to_page(addr);
  1410. get_page(page);
  1411. } else {
  1412. uint32_t dbi;
  1413. /* For the dynamically growing data area pages */
  1414. dbi = (offset - udev->data_off) / DATA_BLOCK_SIZE;
  1415. page = tcmu_try_get_block_page(udev, dbi);
  1416. if (!page)
  1417. return VM_FAULT_SIGBUS;
  1418. }
  1419. vmf->page = page;
  1420. return 0;
  1421. }
  1422. static const struct vm_operations_struct tcmu_vm_ops = {
  1423. .fault = tcmu_vma_fault,
  1424. };
  1425. static int tcmu_mmap(struct uio_info *info, struct vm_area_struct *vma)
  1426. {
  1427. struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
  1428. vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
  1429. vma->vm_ops = &tcmu_vm_ops;
  1430. vma->vm_private_data = udev;
  1431. /* Ensure the mmap is exactly the right size */
  1432. if (vma_pages(vma) != (udev->ring_size >> PAGE_SHIFT))
  1433. return -EINVAL;
  1434. return 0;
  1435. }
  1436. static int tcmu_open(struct uio_info *info, struct inode *inode)
  1437. {
  1438. struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
  1439. /* O_EXCL not supported for char devs, so fake it? */
  1440. if (test_and_set_bit(TCMU_DEV_BIT_OPEN, &udev->flags))
  1441. return -EBUSY;
  1442. udev->inode = inode;
  1443. kref_get(&udev->kref);
  1444. pr_debug("open\n");
  1445. return 0;
  1446. }
  1447. static void tcmu_dev_call_rcu(struct rcu_head *p)
  1448. {
  1449. struct se_device *dev = container_of(p, struct se_device, rcu_head);
  1450. struct tcmu_dev *udev = TCMU_DEV(dev);
  1451. kfree(udev->uio_info.name);
  1452. kfree(udev->name);
  1453. kfree(udev);
  1454. }
  1455. static int tcmu_check_and_free_pending_cmd(struct tcmu_cmd *cmd)
  1456. {
  1457. if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) {
  1458. kmem_cache_free(tcmu_cmd_cache, cmd);
  1459. return 0;
  1460. }
  1461. return -EINVAL;
  1462. }
  1463. static void tcmu_blocks_release(struct radix_tree_root *blocks,
  1464. int start, int end)
  1465. {
  1466. int i;
  1467. struct page *page;
  1468. for (i = start; i < end; i++) {
  1469. page = radix_tree_delete(blocks, i);
  1470. if (page) {
  1471. __free_page(page);
  1472. atomic_dec(&global_db_count);
  1473. }
  1474. }
  1475. }
  1476. static void tcmu_remove_all_queued_tmr(struct tcmu_dev *udev)
  1477. {
  1478. struct tcmu_tmr *tmr, *tmp;
  1479. list_for_each_entry_safe(tmr, tmp, &udev->tmr_queue, queue_entry) {
  1480. list_del_init(&tmr->queue_entry);
  1481. kfree(tmr);
  1482. }
  1483. }
  1484. static void tcmu_dev_kref_release(struct kref *kref)
  1485. {
  1486. struct tcmu_dev *udev = container_of(kref, struct tcmu_dev, kref);
  1487. struct se_device *dev = &udev->se_dev;
  1488. struct tcmu_cmd *cmd;
  1489. bool all_expired = true;
  1490. int i;
  1491. vfree(udev->mb_addr);
  1492. udev->mb_addr = NULL;
  1493. spin_lock_bh(&timed_out_udevs_lock);
  1494. if (!list_empty(&udev->timedout_entry))
  1495. list_del(&udev->timedout_entry);
  1496. spin_unlock_bh(&timed_out_udevs_lock);
  1497. /* Upper layer should drain all requests before calling this */
  1498. mutex_lock(&udev->cmdr_lock);
  1499. idr_for_each_entry(&udev->commands, cmd, i) {
  1500. if (tcmu_check_and_free_pending_cmd(cmd) != 0)
  1501. all_expired = false;
  1502. }
  1503. /* There can be left over TMR cmds. Remove them. */
  1504. tcmu_remove_all_queued_tmr(udev);
  1505. if (!list_empty(&udev->qfull_queue))
  1506. all_expired = false;
  1507. idr_destroy(&udev->commands);
  1508. WARN_ON(!all_expired);
  1509. tcmu_blocks_release(&udev->data_blocks, 0, udev->dbi_max + 1);
  1510. bitmap_free(udev->data_bitmap);
  1511. mutex_unlock(&udev->cmdr_lock);
  1512. call_rcu(&dev->rcu_head, tcmu_dev_call_rcu);
  1513. }
  1514. static int tcmu_release(struct uio_info *info, struct inode *inode)
  1515. {
  1516. struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
  1517. clear_bit(TCMU_DEV_BIT_OPEN, &udev->flags);
  1518. pr_debug("close\n");
  1519. /* release ref from open */
  1520. kref_put(&udev->kref, tcmu_dev_kref_release);
  1521. return 0;
  1522. }
  1523. static int tcmu_init_genl_cmd_reply(struct tcmu_dev *udev, int cmd)
  1524. {
  1525. struct tcmu_nl_cmd *nl_cmd = &udev->curr_nl_cmd;
  1526. if (!tcmu_kern_cmd_reply_supported)
  1527. return 0;
  1528. if (udev->nl_reply_supported <= 0)
  1529. return 0;
  1530. mutex_lock(&tcmu_nl_cmd_mutex);
  1531. if (tcmu_netlink_blocked) {
  1532. mutex_unlock(&tcmu_nl_cmd_mutex);
  1533. pr_warn("Failing nl cmd %d on %s. Interface is blocked.\n", cmd,
  1534. udev->name);
  1535. return -EAGAIN;
  1536. }
  1537. if (nl_cmd->cmd != TCMU_CMD_UNSPEC) {
  1538. mutex_unlock(&tcmu_nl_cmd_mutex);
  1539. pr_warn("netlink cmd %d already executing on %s\n",
  1540. nl_cmd->cmd, udev->name);
  1541. return -EBUSY;
  1542. }
  1543. memset(nl_cmd, 0, sizeof(*nl_cmd));
  1544. nl_cmd->cmd = cmd;
  1545. nl_cmd->udev = udev;
  1546. init_completion(&nl_cmd->complete);
  1547. INIT_LIST_HEAD(&nl_cmd->nl_list);
  1548. list_add_tail(&nl_cmd->nl_list, &tcmu_nl_cmd_list);
  1549. mutex_unlock(&tcmu_nl_cmd_mutex);
  1550. return 0;
  1551. }
  1552. static void tcmu_destroy_genl_cmd_reply(struct tcmu_dev *udev)
  1553. {
  1554. struct tcmu_nl_cmd *nl_cmd = &udev->curr_nl_cmd;
  1555. if (!tcmu_kern_cmd_reply_supported)
  1556. return;
  1557. if (udev->nl_reply_supported <= 0)
  1558. return;
  1559. mutex_lock(&tcmu_nl_cmd_mutex);
  1560. list_del(&nl_cmd->nl_list);
  1561. memset(nl_cmd, 0, sizeof(*nl_cmd));
  1562. mutex_unlock(&tcmu_nl_cmd_mutex);
  1563. }
  1564. static int tcmu_wait_genl_cmd_reply(struct tcmu_dev *udev)
  1565. {
  1566. struct tcmu_nl_cmd *nl_cmd = &udev->curr_nl_cmd;
  1567. int ret;
  1568. if (!tcmu_kern_cmd_reply_supported)
  1569. return 0;
  1570. if (udev->nl_reply_supported <= 0)
  1571. return 0;
  1572. pr_debug("sleeping for nl reply\n");
  1573. wait_for_completion(&nl_cmd->complete);
  1574. mutex_lock(&tcmu_nl_cmd_mutex);
  1575. nl_cmd->cmd = TCMU_CMD_UNSPEC;
  1576. ret = nl_cmd->status;
  1577. mutex_unlock(&tcmu_nl_cmd_mutex);
  1578. return ret;
  1579. }
  1580. static int tcmu_netlink_event_init(struct tcmu_dev *udev,
  1581. enum tcmu_genl_cmd cmd,
  1582. struct sk_buff **buf, void **hdr)
  1583. {
  1584. struct sk_buff *skb;
  1585. void *msg_header;
  1586. int ret = -ENOMEM;
  1587. skb = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
  1588. if (!skb)
  1589. return ret;
  1590. msg_header = genlmsg_put(skb, 0, 0, &tcmu_genl_family, 0, cmd);
  1591. if (!msg_header)
  1592. goto free_skb;
  1593. ret = nla_put_string(skb, TCMU_ATTR_DEVICE, udev->uio_info.name);
  1594. if (ret < 0)
  1595. goto free_skb;
  1596. ret = nla_put_u32(skb, TCMU_ATTR_MINOR, udev->uio_info.uio_dev->minor);
  1597. if (ret < 0)
  1598. goto free_skb;
  1599. ret = nla_put_u32(skb, TCMU_ATTR_DEVICE_ID, udev->se_dev.dev_index);
  1600. if (ret < 0)
  1601. goto free_skb;
  1602. *buf = skb;
  1603. *hdr = msg_header;
  1604. return ret;
  1605. free_skb:
  1606. nlmsg_free(skb);
  1607. return ret;
  1608. }
  1609. static int tcmu_netlink_event_send(struct tcmu_dev *udev,
  1610. enum tcmu_genl_cmd cmd,
  1611. struct sk_buff *skb, void *msg_header)
  1612. {
  1613. int ret;
  1614. genlmsg_end(skb, msg_header);
  1615. ret = tcmu_init_genl_cmd_reply(udev, cmd);
  1616. if (ret) {
  1617. nlmsg_free(skb);
  1618. return ret;
  1619. }
  1620. ret = genlmsg_multicast_allns(&tcmu_genl_family, skb, 0,
  1621. TCMU_MCGRP_CONFIG, GFP_KERNEL);
  1622. /* Wait during an add as the listener may not be up yet */
  1623. if (ret == 0 ||
  1624. (ret == -ESRCH && cmd == TCMU_CMD_ADDED_DEVICE))
  1625. return tcmu_wait_genl_cmd_reply(udev);
  1626. else
  1627. tcmu_destroy_genl_cmd_reply(udev);
  1628. return ret;
  1629. }
  1630. static int tcmu_send_dev_add_event(struct tcmu_dev *udev)
  1631. {
  1632. struct sk_buff *skb = NULL;
  1633. void *msg_header = NULL;
  1634. int ret = 0;
  1635. ret = tcmu_netlink_event_init(udev, TCMU_CMD_ADDED_DEVICE, &skb,
  1636. &msg_header);
  1637. if (ret < 0)
  1638. return ret;
  1639. return tcmu_netlink_event_send(udev, TCMU_CMD_ADDED_DEVICE, skb,
  1640. msg_header);
  1641. }
  1642. static int tcmu_send_dev_remove_event(struct tcmu_dev *udev)
  1643. {
  1644. struct sk_buff *skb = NULL;
  1645. void *msg_header = NULL;
  1646. int ret = 0;
  1647. ret = tcmu_netlink_event_init(udev, TCMU_CMD_REMOVED_DEVICE,
  1648. &skb, &msg_header);
  1649. if (ret < 0)
  1650. return ret;
  1651. return tcmu_netlink_event_send(udev, TCMU_CMD_REMOVED_DEVICE,
  1652. skb, msg_header);
  1653. }
  1654. static int tcmu_update_uio_info(struct tcmu_dev *udev)
  1655. {
  1656. struct tcmu_hba *hba = udev->hba->hba_ptr;
  1657. struct uio_info *info;
  1658. char *str;
  1659. info = &udev->uio_info;
  1660. if (udev->dev_config[0])
  1661. str = kasprintf(GFP_KERNEL, "tcm-user/%u/%s/%s", hba->host_id,
  1662. udev->name, udev->dev_config);
  1663. else
  1664. str = kasprintf(GFP_KERNEL, "tcm-user/%u/%s", hba->host_id,
  1665. udev->name);
  1666. if (!str)
  1667. return -ENOMEM;
  1668. /* If the old string exists, free it */
  1669. kfree(info->name);
  1670. info->name = str;
  1671. return 0;
  1672. }
  1673. static int tcmu_configure_device(struct se_device *dev)
  1674. {
  1675. struct tcmu_dev *udev = TCMU_DEV(dev);
  1676. struct uio_info *info;
  1677. struct tcmu_mailbox *mb;
  1678. int ret = 0;
  1679. ret = tcmu_update_uio_info(udev);
  1680. if (ret)
  1681. return ret;
  1682. info = &udev->uio_info;
  1683. mutex_lock(&udev->cmdr_lock);
  1684. udev->data_bitmap = bitmap_zalloc(udev->max_blocks, GFP_KERNEL);
  1685. mutex_unlock(&udev->cmdr_lock);
  1686. if (!udev->data_bitmap) {
  1687. ret = -ENOMEM;
  1688. goto err_bitmap_alloc;
  1689. }
  1690. udev->mb_addr = vzalloc(CMDR_SIZE);
  1691. if (!udev->mb_addr) {
  1692. ret = -ENOMEM;
  1693. goto err_vzalloc;
  1694. }
  1695. /* mailbox fits in first part of CMDR space */
  1696. udev->cmdr_size = CMDR_SIZE - CMDR_OFF;
  1697. udev->data_off = CMDR_SIZE;
  1698. udev->data_size = udev->max_blocks * DATA_BLOCK_SIZE;
  1699. udev->dbi_thresh = 0; /* Default in Idle state */
  1700. /* Initialise the mailbox of the ring buffer */
  1701. mb = udev->mb_addr;
  1702. mb->version = TCMU_MAILBOX_VERSION;
  1703. mb->flags = TCMU_MAILBOX_FLAG_CAP_OOOC |
  1704. TCMU_MAILBOX_FLAG_CAP_READ_LEN |
  1705. TCMU_MAILBOX_FLAG_CAP_TMR;
  1706. mb->cmdr_off = CMDR_OFF;
  1707. mb->cmdr_size = udev->cmdr_size;
  1708. WARN_ON(!PAGE_ALIGNED(udev->data_off));
  1709. WARN_ON(udev->data_size % PAGE_SIZE);
  1710. WARN_ON(udev->data_size % DATA_BLOCK_SIZE);
  1711. info->version = __stringify(TCMU_MAILBOX_VERSION);
  1712. info->mem[0].name = "tcm-user command & data buffer";
  1713. info->mem[0].addr = (phys_addr_t)(uintptr_t)udev->mb_addr;
  1714. info->mem[0].size = udev->ring_size = udev->data_size + CMDR_SIZE;
  1715. info->mem[0].memtype = UIO_MEM_NONE;
  1716. info->irqcontrol = tcmu_irqcontrol;
  1717. info->irq = UIO_IRQ_CUSTOM;
  1718. info->mmap = tcmu_mmap;
  1719. info->open = tcmu_open;
  1720. info->release = tcmu_release;
  1721. ret = uio_register_device(tcmu_root_device, info);
  1722. if (ret)
  1723. goto err_register;
  1724. /* User can set hw_block_size before enable the device */
  1725. if (dev->dev_attrib.hw_block_size == 0)
  1726. dev->dev_attrib.hw_block_size = 512;
  1727. /* Other attributes can be configured in userspace */
  1728. if (!dev->dev_attrib.hw_max_sectors)
  1729. dev->dev_attrib.hw_max_sectors = 128;
  1730. if (!dev->dev_attrib.emulate_write_cache)
  1731. dev->dev_attrib.emulate_write_cache = 0;
  1732. dev->dev_attrib.hw_queue_depth = 128;
  1733. /* If user didn't explicitly disable netlink reply support, use
  1734. * module scope setting.
  1735. */
  1736. if (udev->nl_reply_supported >= 0)
  1737. udev->nl_reply_supported = tcmu_kern_cmd_reply_supported;
  1738. /*
  1739. * Get a ref incase userspace does a close on the uio device before
  1740. * LIO has initiated tcmu_free_device.
  1741. */
  1742. kref_get(&udev->kref);
  1743. ret = tcmu_send_dev_add_event(udev);
  1744. if (ret)
  1745. goto err_netlink;
  1746. mutex_lock(&root_udev_mutex);
  1747. list_add(&udev->node, &root_udev);
  1748. mutex_unlock(&root_udev_mutex);
  1749. return 0;
  1750. err_netlink:
  1751. kref_put(&udev->kref, tcmu_dev_kref_release);
  1752. uio_unregister_device(&udev->uio_info);
  1753. err_register:
  1754. vfree(udev->mb_addr);
  1755. udev->mb_addr = NULL;
  1756. err_vzalloc:
  1757. bitmap_free(udev->data_bitmap);
  1758. udev->data_bitmap = NULL;
  1759. err_bitmap_alloc:
  1760. kfree(info->name);
  1761. info->name = NULL;
  1762. return ret;
  1763. }
  1764. static void tcmu_free_device(struct se_device *dev)
  1765. {
  1766. struct tcmu_dev *udev = TCMU_DEV(dev);
  1767. /* release ref from init */
  1768. kref_put(&udev->kref, tcmu_dev_kref_release);
  1769. }
  1770. static void tcmu_destroy_device(struct se_device *dev)
  1771. {
  1772. struct tcmu_dev *udev = TCMU_DEV(dev);
  1773. del_timer_sync(&udev->cmd_timer);
  1774. del_timer_sync(&udev->qfull_timer);
  1775. mutex_lock(&root_udev_mutex);
  1776. list_del(&udev->node);
  1777. mutex_unlock(&root_udev_mutex);
  1778. tcmu_send_dev_remove_event(udev);
  1779. uio_unregister_device(&udev->uio_info);
  1780. /* release ref from configure */
  1781. kref_put(&udev->kref, tcmu_dev_kref_release);
  1782. }
  1783. static void tcmu_unblock_dev(struct tcmu_dev *udev)
  1784. {
  1785. mutex_lock(&udev->cmdr_lock);
  1786. clear_bit(TCMU_DEV_BIT_BLOCKED, &udev->flags);
  1787. mutex_unlock(&udev->cmdr_lock);
  1788. }
  1789. static void tcmu_block_dev(struct tcmu_dev *udev)
  1790. {
  1791. mutex_lock(&udev->cmdr_lock);
  1792. if (test_and_set_bit(TCMU_DEV_BIT_BLOCKED, &udev->flags))
  1793. goto unlock;
  1794. /* complete IO that has executed successfully */
  1795. tcmu_handle_completions(udev);
  1796. /* fail IO waiting to be queued */
  1797. run_qfull_queue(udev, true);
  1798. unlock:
  1799. mutex_unlock(&udev->cmdr_lock);
  1800. }
  1801. static void tcmu_reset_ring(struct tcmu_dev *udev, u8 err_level)
  1802. {
  1803. struct tcmu_mailbox *mb;
  1804. struct tcmu_cmd *cmd;
  1805. int i;
  1806. mutex_lock(&udev->cmdr_lock);
  1807. idr_for_each_entry(&udev->commands, cmd, i) {
  1808. pr_debug("removing cmd %u on dev %s from ring (is expired %d)\n",
  1809. cmd->cmd_id, udev->name,
  1810. test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags));
  1811. idr_remove(&udev->commands, i);
  1812. if (!test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) {
  1813. WARN_ON(!cmd->se_cmd);
  1814. list_del_init(&cmd->queue_entry);
  1815. cmd->se_cmd->priv = NULL;
  1816. if (err_level == 1) {
  1817. /*
  1818. * Userspace was not able to start the
  1819. * command or it is retryable.
  1820. */
  1821. target_complete_cmd(cmd->se_cmd, SAM_STAT_BUSY);
  1822. } else {
  1823. /* hard failure */
  1824. target_complete_cmd(cmd->se_cmd,
  1825. SAM_STAT_CHECK_CONDITION);
  1826. }
  1827. }
  1828. tcmu_cmd_free_data(cmd, cmd->dbi_cnt);
  1829. tcmu_free_cmd(cmd);
  1830. }
  1831. mb = udev->mb_addr;
  1832. tcmu_flush_dcache_range(mb, sizeof(*mb));
  1833. pr_debug("mb last %u head %u tail %u\n", udev->cmdr_last_cleaned,
  1834. mb->cmd_tail, mb->cmd_head);
  1835. udev->cmdr_last_cleaned = 0;
  1836. mb->cmd_tail = 0;
  1837. mb->cmd_head = 0;
  1838. tcmu_flush_dcache_range(mb, sizeof(*mb));
  1839. clear_bit(TCMU_DEV_BIT_BROKEN, &udev->flags);
  1840. del_timer(&udev->cmd_timer);
  1841. /*
  1842. * ring is empty and qfull queue never contains aborted commands.
  1843. * So TMRs in tmr queue do not contain relevant cmd_ids.
  1844. * After a ring reset userspace should do a fresh start, so
  1845. * even LUN RESET message is no longer relevant.
  1846. * Therefore remove all TMRs from qfull queue
  1847. */
  1848. tcmu_remove_all_queued_tmr(udev);
  1849. run_qfull_queue(udev, false);
  1850. mutex_unlock(&udev->cmdr_lock);
  1851. }
  1852. enum {
  1853. Opt_dev_config, Opt_dev_size, Opt_hw_block_size, Opt_hw_max_sectors,
  1854. Opt_nl_reply_supported, Opt_max_data_area_mb, Opt_err,
  1855. };
  1856. static match_table_t tokens = {
  1857. {Opt_dev_config, "dev_config=%s"},
  1858. {Opt_dev_size, "dev_size=%s"},
  1859. {Opt_hw_block_size, "hw_block_size=%d"},
  1860. {Opt_hw_max_sectors, "hw_max_sectors=%d"},
  1861. {Opt_nl_reply_supported, "nl_reply_supported=%d"},
  1862. {Opt_max_data_area_mb, "max_data_area_mb=%d"},
  1863. {Opt_err, NULL}
  1864. };
  1865. static int tcmu_set_dev_attrib(substring_t *arg, u32 *dev_attrib)
  1866. {
  1867. int val, ret;
  1868. ret = match_int(arg, &val);
  1869. if (ret < 0) {
  1870. pr_err("match_int() failed for dev attrib. Error %d.\n",
  1871. ret);
  1872. return ret;
  1873. }
  1874. if (val <= 0) {
  1875. pr_err("Invalid dev attrib value %d. Must be greater than zero.\n",
  1876. val);
  1877. return -EINVAL;
  1878. }
  1879. *dev_attrib = val;
  1880. return 0;
  1881. }
  1882. static int tcmu_set_max_blocks_param(struct tcmu_dev *udev, substring_t *arg)
  1883. {
  1884. int val, ret;
  1885. ret = match_int(arg, &val);
  1886. if (ret < 0) {
  1887. pr_err("match_int() failed for max_data_area_mb=. Error %d.\n",
  1888. ret);
  1889. return ret;
  1890. }
  1891. if (val <= 0) {
  1892. pr_err("Invalid max_data_area %d.\n", val);
  1893. return -EINVAL;
  1894. }
  1895. mutex_lock(&udev->cmdr_lock);
  1896. if (udev->data_bitmap) {
  1897. pr_err("Cannot set max_data_area_mb after it has been enabled.\n");
  1898. ret = -EINVAL;
  1899. goto unlock;
  1900. }
  1901. udev->max_blocks = TCMU_MBS_TO_BLOCKS(val);
  1902. if (udev->max_blocks > tcmu_global_max_blocks) {
  1903. pr_err("%d is too large. Adjusting max_data_area_mb to global limit of %u\n",
  1904. val, TCMU_BLOCKS_TO_MBS(tcmu_global_max_blocks));
  1905. udev->max_blocks = tcmu_global_max_blocks;
  1906. }
  1907. unlock:
  1908. mutex_unlock(&udev->cmdr_lock);
  1909. return ret;
  1910. }
  1911. static ssize_t tcmu_set_configfs_dev_params(struct se_device *dev,
  1912. const char *page, ssize_t count)
  1913. {
  1914. struct tcmu_dev *udev = TCMU_DEV(dev);
  1915. char *orig, *ptr, *opts;
  1916. substring_t args[MAX_OPT_ARGS];
  1917. int ret = 0, token;
  1918. opts = kstrdup(page, GFP_KERNEL);
  1919. if (!opts)
  1920. return -ENOMEM;
  1921. orig = opts;
  1922. while ((ptr = strsep(&opts, ",\n")) != NULL) {
  1923. if (!*ptr)
  1924. continue;
  1925. token = match_token(ptr, tokens, args);
  1926. switch (token) {
  1927. case Opt_dev_config:
  1928. if (match_strlcpy(udev->dev_config, &args[0],
  1929. TCMU_CONFIG_LEN) == 0) {
  1930. ret = -EINVAL;
  1931. break;
  1932. }
  1933. pr_debug("TCMU: Referencing Path: %s\n", udev->dev_config);
  1934. break;
  1935. case Opt_dev_size:
  1936. ret = match_u64(&args[0], &udev->dev_size);
  1937. if (ret < 0)
  1938. pr_err("match_u64() failed for dev_size=. Error %d.\n",
  1939. ret);
  1940. break;
  1941. case Opt_hw_block_size:
  1942. ret = tcmu_set_dev_attrib(&args[0],
  1943. &(dev->dev_attrib.hw_block_size));
  1944. break;
  1945. case Opt_hw_max_sectors:
  1946. ret = tcmu_set_dev_attrib(&args[0],
  1947. &(dev->dev_attrib.hw_max_sectors));
  1948. break;
  1949. case Opt_nl_reply_supported:
  1950. ret = match_int(&args[0], &udev->nl_reply_supported);
  1951. if (ret < 0)
  1952. pr_err("match_int() failed for nl_reply_supported=. Error %d.\n",
  1953. ret);
  1954. break;
  1955. case Opt_max_data_area_mb:
  1956. ret = tcmu_set_max_blocks_param(udev, &args[0]);
  1957. break;
  1958. default:
  1959. break;
  1960. }
  1961. if (ret)
  1962. break;
  1963. }
  1964. kfree(orig);
  1965. return (!ret) ? count : ret;
  1966. }
  1967. static ssize_t tcmu_show_configfs_dev_params(struct se_device *dev, char *b)
  1968. {
  1969. struct tcmu_dev *udev = TCMU_DEV(dev);
  1970. ssize_t bl = 0;
  1971. bl = sprintf(b + bl, "Config: %s ",
  1972. udev->dev_config[0] ? udev->dev_config : "NULL");
  1973. bl += sprintf(b + bl, "Size: %llu ", udev->dev_size);
  1974. bl += sprintf(b + bl, "MaxDataAreaMB: %u\n",
  1975. TCMU_BLOCKS_TO_MBS(udev->max_blocks));
  1976. return bl;
  1977. }
  1978. static sector_t tcmu_get_blocks(struct se_device *dev)
  1979. {
  1980. struct tcmu_dev *udev = TCMU_DEV(dev);
  1981. return div_u64(udev->dev_size - dev->dev_attrib.block_size,
  1982. dev->dev_attrib.block_size);
  1983. }
  1984. static sense_reason_t
  1985. tcmu_parse_cdb(struct se_cmd *cmd)
  1986. {
  1987. return passthrough_parse_cdb(cmd, tcmu_queue_cmd);
  1988. }
  1989. static ssize_t tcmu_cmd_time_out_show(struct config_item *item, char *page)
  1990. {
  1991. struct se_dev_attrib *da = container_of(to_config_group(item),
  1992. struct se_dev_attrib, da_group);
  1993. struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
  1994. return snprintf(page, PAGE_SIZE, "%lu\n", udev->cmd_time_out / MSEC_PER_SEC);
  1995. }
  1996. static ssize_t tcmu_cmd_time_out_store(struct config_item *item, const char *page,
  1997. size_t count)
  1998. {
  1999. struct se_dev_attrib *da = container_of(to_config_group(item),
  2000. struct se_dev_attrib, da_group);
  2001. struct tcmu_dev *udev = container_of(da->da_dev,
  2002. struct tcmu_dev, se_dev);
  2003. u32 val;
  2004. int ret;
  2005. if (da->da_dev->export_count) {
  2006. pr_err("Unable to set tcmu cmd_time_out while exports exist\n");
  2007. return -EINVAL;
  2008. }
  2009. ret = kstrtou32(page, 0, &val);
  2010. if (ret < 0)
  2011. return ret;
  2012. udev->cmd_time_out = val * MSEC_PER_SEC;
  2013. return count;
  2014. }
  2015. CONFIGFS_ATTR(tcmu_, cmd_time_out);
  2016. static ssize_t tcmu_qfull_time_out_show(struct config_item *item, char *page)
  2017. {
  2018. struct se_dev_attrib *da = container_of(to_config_group(item),
  2019. struct se_dev_attrib, da_group);
  2020. struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
  2021. return snprintf(page, PAGE_SIZE, "%ld\n", udev->qfull_time_out <= 0 ?
  2022. udev->qfull_time_out :
  2023. udev->qfull_time_out / MSEC_PER_SEC);
  2024. }
  2025. static ssize_t tcmu_qfull_time_out_store(struct config_item *item,
  2026. const char *page, size_t count)
  2027. {
  2028. struct se_dev_attrib *da = container_of(to_config_group(item),
  2029. struct se_dev_attrib, da_group);
  2030. struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
  2031. s32 val;
  2032. int ret;
  2033. ret = kstrtos32(page, 0, &val);
  2034. if (ret < 0)
  2035. return ret;
  2036. if (val >= 0) {
  2037. udev->qfull_time_out = val * MSEC_PER_SEC;
  2038. } else if (val == -1) {
  2039. udev->qfull_time_out = val;
  2040. } else {
  2041. printk(KERN_ERR "Invalid qfull timeout value %d\n", val);
  2042. return -EINVAL;
  2043. }
  2044. return count;
  2045. }
  2046. CONFIGFS_ATTR(tcmu_, qfull_time_out);
  2047. static ssize_t tcmu_max_data_area_mb_show(struct config_item *item, char *page)
  2048. {
  2049. struct se_dev_attrib *da = container_of(to_config_group(item),
  2050. struct se_dev_attrib, da_group);
  2051. struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
  2052. return snprintf(page, PAGE_SIZE, "%u\n",
  2053. TCMU_BLOCKS_TO_MBS(udev->max_blocks));
  2054. }
  2055. CONFIGFS_ATTR_RO(tcmu_, max_data_area_mb);
  2056. static ssize_t tcmu_dev_config_show(struct config_item *item, char *page)
  2057. {
  2058. struct se_dev_attrib *da = container_of(to_config_group(item),
  2059. struct se_dev_attrib, da_group);
  2060. struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
  2061. return snprintf(page, PAGE_SIZE, "%s\n", udev->dev_config);
  2062. }
  2063. static int tcmu_send_dev_config_event(struct tcmu_dev *udev,
  2064. const char *reconfig_data)
  2065. {
  2066. struct sk_buff *skb = NULL;
  2067. void *msg_header = NULL;
  2068. int ret = 0;
  2069. ret = tcmu_netlink_event_init(udev, TCMU_CMD_RECONFIG_DEVICE,
  2070. &skb, &msg_header);
  2071. if (ret < 0)
  2072. return ret;
  2073. ret = nla_put_string(skb, TCMU_ATTR_DEV_CFG, reconfig_data);
  2074. if (ret < 0) {
  2075. nlmsg_free(skb);
  2076. return ret;
  2077. }
  2078. return tcmu_netlink_event_send(udev, TCMU_CMD_RECONFIG_DEVICE,
  2079. skb, msg_header);
  2080. }
  2081. static ssize_t tcmu_dev_config_store(struct config_item *item, const char *page,
  2082. size_t count)
  2083. {
  2084. struct se_dev_attrib *da = container_of(to_config_group(item),
  2085. struct se_dev_attrib, da_group);
  2086. struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
  2087. int ret, len;
  2088. len = strlen(page);
  2089. if (!len || len > TCMU_CONFIG_LEN - 1)
  2090. return -EINVAL;
  2091. /* Check if device has been configured before */
  2092. if (target_dev_configured(&udev->se_dev)) {
  2093. ret = tcmu_send_dev_config_event(udev, page);
  2094. if (ret) {
  2095. pr_err("Unable to reconfigure device\n");
  2096. return ret;
  2097. }
  2098. strlcpy(udev->dev_config, page, TCMU_CONFIG_LEN);
  2099. ret = tcmu_update_uio_info(udev);
  2100. if (ret)
  2101. return ret;
  2102. return count;
  2103. }
  2104. strlcpy(udev->dev_config, page, TCMU_CONFIG_LEN);
  2105. return count;
  2106. }
  2107. CONFIGFS_ATTR(tcmu_, dev_config);
  2108. static ssize_t tcmu_dev_size_show(struct config_item *item, char *page)
  2109. {
  2110. struct se_dev_attrib *da = container_of(to_config_group(item),
  2111. struct se_dev_attrib, da_group);
  2112. struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
  2113. return snprintf(page, PAGE_SIZE, "%llu\n", udev->dev_size);
  2114. }
  2115. static int tcmu_send_dev_size_event(struct tcmu_dev *udev, u64 size)
  2116. {
  2117. struct sk_buff *skb = NULL;
  2118. void *msg_header = NULL;
  2119. int ret = 0;
  2120. ret = tcmu_netlink_event_init(udev, TCMU_CMD_RECONFIG_DEVICE,
  2121. &skb, &msg_header);
  2122. if (ret < 0)
  2123. return ret;
  2124. ret = nla_put_u64_64bit(skb, TCMU_ATTR_DEV_SIZE,
  2125. size, TCMU_ATTR_PAD);
  2126. if (ret < 0) {
  2127. nlmsg_free(skb);
  2128. return ret;
  2129. }
  2130. return tcmu_netlink_event_send(udev, TCMU_CMD_RECONFIG_DEVICE,
  2131. skb, msg_header);
  2132. }
  2133. static ssize_t tcmu_dev_size_store(struct config_item *item, const char *page,
  2134. size_t count)
  2135. {
  2136. struct se_dev_attrib *da = container_of(to_config_group(item),
  2137. struct se_dev_attrib, da_group);
  2138. struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
  2139. u64 val;
  2140. int ret;
  2141. ret = kstrtou64(page, 0, &val);
  2142. if (ret < 0)
  2143. return ret;
  2144. /* Check if device has been configured before */
  2145. if (target_dev_configured(&udev->se_dev)) {
  2146. ret = tcmu_send_dev_size_event(udev, val);
  2147. if (ret) {
  2148. pr_err("Unable to reconfigure device\n");
  2149. return ret;
  2150. }
  2151. }
  2152. udev->dev_size = val;
  2153. return count;
  2154. }
  2155. CONFIGFS_ATTR(tcmu_, dev_size);
  2156. static ssize_t tcmu_nl_reply_supported_show(struct config_item *item,
  2157. char *page)
  2158. {
  2159. struct se_dev_attrib *da = container_of(to_config_group(item),
  2160. struct se_dev_attrib, da_group);
  2161. struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
  2162. return snprintf(page, PAGE_SIZE, "%d\n", udev->nl_reply_supported);
  2163. }
  2164. static ssize_t tcmu_nl_reply_supported_store(struct config_item *item,
  2165. const char *page, size_t count)
  2166. {
  2167. struct se_dev_attrib *da = container_of(to_config_group(item),
  2168. struct se_dev_attrib, da_group);
  2169. struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
  2170. s8 val;
  2171. int ret;
  2172. ret = kstrtos8(page, 0, &val);
  2173. if (ret < 0)
  2174. return ret;
  2175. udev->nl_reply_supported = val;
  2176. return count;
  2177. }
  2178. CONFIGFS_ATTR(tcmu_, nl_reply_supported);
  2179. static ssize_t tcmu_emulate_write_cache_show(struct config_item *item,
  2180. char *page)
  2181. {
  2182. struct se_dev_attrib *da = container_of(to_config_group(item),
  2183. struct se_dev_attrib, da_group);
  2184. return snprintf(page, PAGE_SIZE, "%i\n", da->emulate_write_cache);
  2185. }
  2186. static int tcmu_send_emulate_write_cache(struct tcmu_dev *udev, u8 val)
  2187. {
  2188. struct sk_buff *skb = NULL;
  2189. void *msg_header = NULL;
  2190. int ret = 0;
  2191. ret = tcmu_netlink_event_init(udev, TCMU_CMD_RECONFIG_DEVICE,
  2192. &skb, &msg_header);
  2193. if (ret < 0)
  2194. return ret;
  2195. ret = nla_put_u8(skb, TCMU_ATTR_WRITECACHE, val);
  2196. if (ret < 0) {
  2197. nlmsg_free(skb);
  2198. return ret;
  2199. }
  2200. return tcmu_netlink_event_send(udev, TCMU_CMD_RECONFIG_DEVICE,
  2201. skb, msg_header);
  2202. }
  2203. static ssize_t tcmu_emulate_write_cache_store(struct config_item *item,
  2204. const char *page, size_t count)
  2205. {
  2206. struct se_dev_attrib *da = container_of(to_config_group(item),
  2207. struct se_dev_attrib, da_group);
  2208. struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
  2209. u8 val;
  2210. int ret;
  2211. ret = kstrtou8(page, 0, &val);
  2212. if (ret < 0)
  2213. return ret;
  2214. /* Check if device has been configured before */
  2215. if (target_dev_configured(&udev->se_dev)) {
  2216. ret = tcmu_send_emulate_write_cache(udev, val);
  2217. if (ret) {
  2218. pr_err("Unable to reconfigure device\n");
  2219. return ret;
  2220. }
  2221. }
  2222. da->emulate_write_cache = val;
  2223. return count;
  2224. }
  2225. CONFIGFS_ATTR(tcmu_, emulate_write_cache);
  2226. static ssize_t tcmu_tmr_notification_show(struct config_item *item, char *page)
  2227. {
  2228. struct se_dev_attrib *da = container_of(to_config_group(item),
  2229. struct se_dev_attrib, da_group);
  2230. struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
  2231. return snprintf(page, PAGE_SIZE, "%i\n",
  2232. test_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags));
  2233. }
  2234. static ssize_t tcmu_tmr_notification_store(struct config_item *item,
  2235. const char *page, size_t count)
  2236. {
  2237. struct se_dev_attrib *da = container_of(to_config_group(item),
  2238. struct se_dev_attrib, da_group);
  2239. struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
  2240. u8 val;
  2241. int ret;
  2242. ret = kstrtou8(page, 0, &val);
  2243. if (ret < 0)
  2244. return ret;
  2245. if (val > 1)
  2246. return -EINVAL;
  2247. if (val)
  2248. set_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags);
  2249. else
  2250. clear_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags);
  2251. return count;
  2252. }
  2253. CONFIGFS_ATTR(tcmu_, tmr_notification);
  2254. static ssize_t tcmu_block_dev_show(struct config_item *item, char *page)
  2255. {
  2256. struct se_device *se_dev = container_of(to_config_group(item),
  2257. struct se_device,
  2258. dev_action_group);
  2259. struct tcmu_dev *udev = TCMU_DEV(se_dev);
  2260. if (test_bit(TCMU_DEV_BIT_BLOCKED, &udev->flags))
  2261. return snprintf(page, PAGE_SIZE, "%s\n", "blocked");
  2262. else
  2263. return snprintf(page, PAGE_SIZE, "%s\n", "unblocked");
  2264. }
  2265. static ssize_t tcmu_block_dev_store(struct config_item *item, const char *page,
  2266. size_t count)
  2267. {
  2268. struct se_device *se_dev = container_of(to_config_group(item),
  2269. struct se_device,
  2270. dev_action_group);
  2271. struct tcmu_dev *udev = TCMU_DEV(se_dev);
  2272. u8 val;
  2273. int ret;
  2274. if (!target_dev_configured(&udev->se_dev)) {
  2275. pr_err("Device is not configured.\n");
  2276. return -EINVAL;
  2277. }
  2278. ret = kstrtou8(page, 0, &val);
  2279. if (ret < 0)
  2280. return ret;
  2281. if (val > 1) {
  2282. pr_err("Invalid block value %d\n", val);
  2283. return -EINVAL;
  2284. }
  2285. if (!val)
  2286. tcmu_unblock_dev(udev);
  2287. else
  2288. tcmu_block_dev(udev);
  2289. return count;
  2290. }
  2291. CONFIGFS_ATTR(tcmu_, block_dev);
  2292. static ssize_t tcmu_reset_ring_store(struct config_item *item, const char *page,
  2293. size_t count)
  2294. {
  2295. struct se_device *se_dev = container_of(to_config_group(item),
  2296. struct se_device,
  2297. dev_action_group);
  2298. struct tcmu_dev *udev = TCMU_DEV(se_dev);
  2299. u8 val;
  2300. int ret;
  2301. if (!target_dev_configured(&udev->se_dev)) {
  2302. pr_err("Device is not configured.\n");
  2303. return -EINVAL;
  2304. }
  2305. ret = kstrtou8(page, 0, &val);
  2306. if (ret < 0)
  2307. return ret;
  2308. if (val != 1 && val != 2) {
  2309. pr_err("Invalid reset ring value %d\n", val);
  2310. return -EINVAL;
  2311. }
  2312. tcmu_reset_ring(udev, val);
  2313. return count;
  2314. }
  2315. CONFIGFS_ATTR_WO(tcmu_, reset_ring);
  2316. static struct configfs_attribute *tcmu_attrib_attrs[] = {
  2317. &tcmu_attr_cmd_time_out,
  2318. &tcmu_attr_qfull_time_out,
  2319. &tcmu_attr_max_data_area_mb,
  2320. &tcmu_attr_dev_config,
  2321. &tcmu_attr_dev_size,
  2322. &tcmu_attr_emulate_write_cache,
  2323. &tcmu_attr_tmr_notification,
  2324. &tcmu_attr_nl_reply_supported,
  2325. NULL,
  2326. };
  2327. static struct configfs_attribute **tcmu_attrs;
  2328. static struct configfs_attribute *tcmu_action_attrs[] = {
  2329. &tcmu_attr_block_dev,
  2330. &tcmu_attr_reset_ring,
  2331. NULL,
  2332. };
  2333. static struct target_backend_ops tcmu_ops = {
  2334. .name = "user",
  2335. .owner = THIS_MODULE,
  2336. .transport_flags_default = TRANSPORT_FLAG_PASSTHROUGH,
  2337. .transport_flags_changeable = TRANSPORT_FLAG_PASSTHROUGH_PGR |
  2338. TRANSPORT_FLAG_PASSTHROUGH_ALUA,
  2339. .attach_hba = tcmu_attach_hba,
  2340. .detach_hba = tcmu_detach_hba,
  2341. .alloc_device = tcmu_alloc_device,
  2342. .configure_device = tcmu_configure_device,
  2343. .destroy_device = tcmu_destroy_device,
  2344. .free_device = tcmu_free_device,
  2345. .parse_cdb = tcmu_parse_cdb,
  2346. .tmr_notify = tcmu_tmr_notify,
  2347. .set_configfs_dev_params = tcmu_set_configfs_dev_params,
  2348. .show_configfs_dev_params = tcmu_show_configfs_dev_params,
  2349. .get_device_type = sbc_get_device_type,
  2350. .get_blocks = tcmu_get_blocks,
  2351. .tb_dev_action_attrs = tcmu_action_attrs,
  2352. };
  2353. static void find_free_blocks(void)
  2354. {
  2355. struct tcmu_dev *udev;
  2356. loff_t off;
  2357. u32 start, end, block, total_freed = 0;
  2358. if (atomic_read(&global_db_count) <= tcmu_global_max_blocks)
  2359. return;
  2360. mutex_lock(&root_udev_mutex);
  2361. list_for_each_entry(udev, &root_udev, node) {
  2362. mutex_lock(&udev->cmdr_lock);
  2363. if (!target_dev_configured(&udev->se_dev)) {
  2364. mutex_unlock(&udev->cmdr_lock);
  2365. continue;
  2366. }
  2367. /* Try to complete the finished commands first */
  2368. if (tcmu_handle_completions(udev))
  2369. run_qfull_queue(udev, false);
  2370. /* Skip the udevs in idle */
  2371. if (!udev->dbi_thresh) {
  2372. mutex_unlock(&udev->cmdr_lock);
  2373. continue;
  2374. }
  2375. end = udev->dbi_max + 1;
  2376. block = find_last_bit(udev->data_bitmap, end);
  2377. if (block == udev->dbi_max) {
  2378. /*
  2379. * The last bit is dbi_max, so it is not possible
  2380. * reclaim any blocks.
  2381. */
  2382. mutex_unlock(&udev->cmdr_lock);
  2383. continue;
  2384. } else if (block == end) {
  2385. /* The current udev will goto idle state */
  2386. udev->dbi_thresh = start = 0;
  2387. udev->dbi_max = 0;
  2388. } else {
  2389. udev->dbi_thresh = start = block + 1;
  2390. udev->dbi_max = block;
  2391. }
  2392. /* Here will truncate the data area from off */
  2393. off = udev->data_off + start * DATA_BLOCK_SIZE;
  2394. unmap_mapping_range(udev->inode->i_mapping, off, 0, 1);
  2395. /* Release the block pages */
  2396. tcmu_blocks_release(&udev->data_blocks, start, end);
  2397. mutex_unlock(&udev->cmdr_lock);
  2398. total_freed += end - start;
  2399. pr_debug("Freed %u blocks (total %u) from %s.\n", end - start,
  2400. total_freed, udev->name);
  2401. }
  2402. mutex_unlock(&root_udev_mutex);
  2403. if (atomic_read(&global_db_count) > tcmu_global_max_blocks)
  2404. schedule_delayed_work(&tcmu_unmap_work, msecs_to_jiffies(5000));
  2405. }
  2406. static void check_timedout_devices(void)
  2407. {
  2408. struct tcmu_dev *udev, *tmp_dev;
  2409. struct tcmu_cmd *cmd, *tmp_cmd;
  2410. LIST_HEAD(devs);
  2411. spin_lock_bh(&timed_out_udevs_lock);
  2412. list_splice_init(&timed_out_udevs, &devs);
  2413. list_for_each_entry_safe(udev, tmp_dev, &devs, timedout_entry) {
  2414. list_del_init(&udev->timedout_entry);
  2415. spin_unlock_bh(&timed_out_udevs_lock);
  2416. mutex_lock(&udev->cmdr_lock);
  2417. /*
  2418. * If cmd_time_out is disabled but qfull is set deadline
  2419. * will only reflect the qfull timeout. Ignore it.
  2420. */
  2421. if (udev->cmd_time_out) {
  2422. list_for_each_entry_safe(cmd, tmp_cmd,
  2423. &udev->inflight_queue,
  2424. queue_entry) {
  2425. tcmu_check_expired_ring_cmd(cmd);
  2426. }
  2427. tcmu_set_next_deadline(&udev->inflight_queue,
  2428. &udev->cmd_timer);
  2429. }
  2430. list_for_each_entry_safe(cmd, tmp_cmd, &udev->qfull_queue,
  2431. queue_entry) {
  2432. tcmu_check_expired_queue_cmd(cmd);
  2433. }
  2434. tcmu_set_next_deadline(&udev->qfull_queue, &udev->qfull_timer);
  2435. mutex_unlock(&udev->cmdr_lock);
  2436. spin_lock_bh(&timed_out_udevs_lock);
  2437. }
  2438. spin_unlock_bh(&timed_out_udevs_lock);
  2439. }
  2440. static void tcmu_unmap_work_fn(struct work_struct *work)
  2441. {
  2442. check_timedout_devices();
  2443. find_free_blocks();
  2444. }
  2445. static int __init tcmu_module_init(void)
  2446. {
  2447. int ret, i, k, len = 0;
  2448. BUILD_BUG_ON((sizeof(struct tcmu_cmd_entry) % TCMU_OP_ALIGN_SIZE) != 0);
  2449. INIT_DELAYED_WORK(&tcmu_unmap_work, tcmu_unmap_work_fn);
  2450. tcmu_cmd_cache = kmem_cache_create("tcmu_cmd_cache",
  2451. sizeof(struct tcmu_cmd),
  2452. __alignof__(struct tcmu_cmd),
  2453. 0, NULL);
  2454. if (!tcmu_cmd_cache)
  2455. return -ENOMEM;
  2456. tcmu_root_device = root_device_register("tcm_user");
  2457. if (IS_ERR(tcmu_root_device)) {
  2458. ret = PTR_ERR(tcmu_root_device);
  2459. goto out_free_cache;
  2460. }
  2461. ret = genl_register_family(&tcmu_genl_family);
  2462. if (ret < 0) {
  2463. goto out_unreg_device;
  2464. }
  2465. for (i = 0; passthrough_attrib_attrs[i] != NULL; i++)
  2466. len += sizeof(struct configfs_attribute *);
  2467. for (i = 0; passthrough_pr_attrib_attrs[i] != NULL; i++)
  2468. len += sizeof(struct configfs_attribute *);
  2469. for (i = 0; tcmu_attrib_attrs[i] != NULL; i++)
  2470. len += sizeof(struct configfs_attribute *);
  2471. len += sizeof(struct configfs_attribute *);
  2472. tcmu_attrs = kzalloc(len, GFP_KERNEL);
  2473. if (!tcmu_attrs) {
  2474. ret = -ENOMEM;
  2475. goto out_unreg_genl;
  2476. }
  2477. for (i = 0; passthrough_attrib_attrs[i] != NULL; i++)
  2478. tcmu_attrs[i] = passthrough_attrib_attrs[i];
  2479. for (k = 0; passthrough_pr_attrib_attrs[k] != NULL; k++)
  2480. tcmu_attrs[i++] = passthrough_pr_attrib_attrs[k];
  2481. for (k = 0; tcmu_attrib_attrs[k] != NULL; k++)
  2482. tcmu_attrs[i++] = tcmu_attrib_attrs[k];
  2483. tcmu_ops.tb_dev_attrib_attrs = tcmu_attrs;
  2484. ret = transport_backend_register(&tcmu_ops);
  2485. if (ret)
  2486. goto out_attrs;
  2487. return 0;
  2488. out_attrs:
  2489. kfree(tcmu_attrs);
  2490. out_unreg_genl:
  2491. genl_unregister_family(&tcmu_genl_family);
  2492. out_unreg_device:
  2493. root_device_unregister(tcmu_root_device);
  2494. out_free_cache:
  2495. kmem_cache_destroy(tcmu_cmd_cache);
  2496. return ret;
  2497. }
  2498. static void __exit tcmu_module_exit(void)
  2499. {
  2500. cancel_delayed_work_sync(&tcmu_unmap_work);
  2501. target_backend_unregister(&tcmu_ops);
  2502. kfree(tcmu_attrs);
  2503. genl_unregister_family(&tcmu_genl_family);
  2504. root_device_unregister(tcmu_root_device);
  2505. kmem_cache_destroy(tcmu_cmd_cache);
  2506. }
  2507. MODULE_DESCRIPTION("TCM USER subsystem plugin");
  2508. MODULE_AUTHOR("Shaohua Li <shli@kernel.org>");
  2509. MODULE_AUTHOR("Andy Grover <agrover@redhat.com>");
  2510. MODULE_LICENSE("GPL");
  2511. module_init(tcmu_module_init);
  2512. module_exit(tcmu_module_exit);