target_core_transport.c 95 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*******************************************************************************
  3. * Filename: target_core_transport.c
  4. *
  5. * This file contains the Generic Target Engine Core.
  6. *
  7. * (c) Copyright 2002-2013 Datera, Inc.
  8. *
  9. * Nicholas A. Bellinger <nab@kernel.org>
  10. *
  11. ******************************************************************************/
  12. #include <linux/net.h>
  13. #include <linux/delay.h>
  14. #include <linux/string.h>
  15. #include <linux/timer.h>
  16. #include <linux/slab.h>
  17. #include <linux/spinlock.h>
  18. #include <linux/kthread.h>
  19. #include <linux/in.h>
  20. #include <linux/cdrom.h>
  21. #include <linux/module.h>
  22. #include <linux/ratelimit.h>
  23. #include <linux/vmalloc.h>
  24. #include <asm/unaligned.h>
  25. #include <net/sock.h>
  26. #include <net/tcp.h>
  27. #include <scsi/scsi_proto.h>
  28. #include <scsi/scsi_common.h>
  29. #include <target/target_core_base.h>
  30. #include <target/target_core_backend.h>
  31. #include <target/target_core_fabric.h>
  32. #include "target_core_internal.h"
  33. #include "target_core_alua.h"
  34. #include "target_core_pr.h"
  35. #include "target_core_ua.h"
  36. #define CREATE_TRACE_POINTS
  37. #include <trace/events/target.h>
  38. static struct workqueue_struct *target_completion_wq;
  39. static struct kmem_cache *se_sess_cache;
  40. struct kmem_cache *se_ua_cache;
  41. struct kmem_cache *t10_pr_reg_cache;
  42. struct kmem_cache *t10_alua_lu_gp_cache;
  43. struct kmem_cache *t10_alua_lu_gp_mem_cache;
  44. struct kmem_cache *t10_alua_tg_pt_gp_cache;
  45. struct kmem_cache *t10_alua_lba_map_cache;
  46. struct kmem_cache *t10_alua_lba_map_mem_cache;
  47. static void transport_complete_task_attr(struct se_cmd *cmd);
  48. static void translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason);
  49. static void transport_handle_queue_full(struct se_cmd *cmd,
  50. struct se_device *dev, int err, bool write_pending);
  51. static void target_complete_ok_work(struct work_struct *work);
  52. int init_se_kmem_caches(void)
  53. {
  54. se_sess_cache = kmem_cache_create("se_sess_cache",
  55. sizeof(struct se_session), __alignof__(struct se_session),
  56. 0, NULL);
  57. if (!se_sess_cache) {
  58. pr_err("kmem_cache_create() for struct se_session"
  59. " failed\n");
  60. goto out;
  61. }
  62. se_ua_cache = kmem_cache_create("se_ua_cache",
  63. sizeof(struct se_ua), __alignof__(struct se_ua),
  64. 0, NULL);
  65. if (!se_ua_cache) {
  66. pr_err("kmem_cache_create() for struct se_ua failed\n");
  67. goto out_free_sess_cache;
  68. }
  69. t10_pr_reg_cache = kmem_cache_create("t10_pr_reg_cache",
  70. sizeof(struct t10_pr_registration),
  71. __alignof__(struct t10_pr_registration), 0, NULL);
  72. if (!t10_pr_reg_cache) {
  73. pr_err("kmem_cache_create() for struct t10_pr_registration"
  74. " failed\n");
  75. goto out_free_ua_cache;
  76. }
  77. t10_alua_lu_gp_cache = kmem_cache_create("t10_alua_lu_gp_cache",
  78. sizeof(struct t10_alua_lu_gp), __alignof__(struct t10_alua_lu_gp),
  79. 0, NULL);
  80. if (!t10_alua_lu_gp_cache) {
  81. pr_err("kmem_cache_create() for t10_alua_lu_gp_cache"
  82. " failed\n");
  83. goto out_free_pr_reg_cache;
  84. }
  85. t10_alua_lu_gp_mem_cache = kmem_cache_create("t10_alua_lu_gp_mem_cache",
  86. sizeof(struct t10_alua_lu_gp_member),
  87. __alignof__(struct t10_alua_lu_gp_member), 0, NULL);
  88. if (!t10_alua_lu_gp_mem_cache) {
  89. pr_err("kmem_cache_create() for t10_alua_lu_gp_mem_"
  90. "cache failed\n");
  91. goto out_free_lu_gp_cache;
  92. }
  93. t10_alua_tg_pt_gp_cache = kmem_cache_create("t10_alua_tg_pt_gp_cache",
  94. sizeof(struct t10_alua_tg_pt_gp),
  95. __alignof__(struct t10_alua_tg_pt_gp), 0, NULL);
  96. if (!t10_alua_tg_pt_gp_cache) {
  97. pr_err("kmem_cache_create() for t10_alua_tg_pt_gp_"
  98. "cache failed\n");
  99. goto out_free_lu_gp_mem_cache;
  100. }
  101. t10_alua_lba_map_cache = kmem_cache_create(
  102. "t10_alua_lba_map_cache",
  103. sizeof(struct t10_alua_lba_map),
  104. __alignof__(struct t10_alua_lba_map), 0, NULL);
  105. if (!t10_alua_lba_map_cache) {
  106. pr_err("kmem_cache_create() for t10_alua_lba_map_"
  107. "cache failed\n");
  108. goto out_free_tg_pt_gp_cache;
  109. }
  110. t10_alua_lba_map_mem_cache = kmem_cache_create(
  111. "t10_alua_lba_map_mem_cache",
  112. sizeof(struct t10_alua_lba_map_member),
  113. __alignof__(struct t10_alua_lba_map_member), 0, NULL);
  114. if (!t10_alua_lba_map_mem_cache) {
  115. pr_err("kmem_cache_create() for t10_alua_lba_map_mem_"
  116. "cache failed\n");
  117. goto out_free_lba_map_cache;
  118. }
  119. target_completion_wq = alloc_workqueue("target_completion",
  120. WQ_MEM_RECLAIM, 0);
  121. if (!target_completion_wq)
  122. goto out_free_lba_map_mem_cache;
  123. return 0;
  124. out_free_lba_map_mem_cache:
  125. kmem_cache_destroy(t10_alua_lba_map_mem_cache);
  126. out_free_lba_map_cache:
  127. kmem_cache_destroy(t10_alua_lba_map_cache);
  128. out_free_tg_pt_gp_cache:
  129. kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
  130. out_free_lu_gp_mem_cache:
  131. kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
  132. out_free_lu_gp_cache:
  133. kmem_cache_destroy(t10_alua_lu_gp_cache);
  134. out_free_pr_reg_cache:
  135. kmem_cache_destroy(t10_pr_reg_cache);
  136. out_free_ua_cache:
  137. kmem_cache_destroy(se_ua_cache);
  138. out_free_sess_cache:
  139. kmem_cache_destroy(se_sess_cache);
  140. out:
  141. return -ENOMEM;
  142. }
  143. void release_se_kmem_caches(void)
  144. {
  145. destroy_workqueue(target_completion_wq);
  146. kmem_cache_destroy(se_sess_cache);
  147. kmem_cache_destroy(se_ua_cache);
  148. kmem_cache_destroy(t10_pr_reg_cache);
  149. kmem_cache_destroy(t10_alua_lu_gp_cache);
  150. kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
  151. kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
  152. kmem_cache_destroy(t10_alua_lba_map_cache);
  153. kmem_cache_destroy(t10_alua_lba_map_mem_cache);
  154. }
  155. /* This code ensures unique mib indexes are handed out. */
  156. static DEFINE_SPINLOCK(scsi_mib_index_lock);
  157. static u32 scsi_mib_index[SCSI_INDEX_TYPE_MAX];
  158. /*
  159. * Allocate a new row index for the entry type specified
  160. */
  161. u32 scsi_get_new_index(scsi_index_t type)
  162. {
  163. u32 new_index;
  164. BUG_ON((type < 0) || (type >= SCSI_INDEX_TYPE_MAX));
  165. spin_lock(&scsi_mib_index_lock);
  166. new_index = ++scsi_mib_index[type];
  167. spin_unlock(&scsi_mib_index_lock);
  168. return new_index;
  169. }
  170. void transport_subsystem_check_init(void)
  171. {
  172. int ret;
  173. static int sub_api_initialized;
  174. if (sub_api_initialized)
  175. return;
  176. ret = IS_ENABLED(CONFIG_TCM_IBLOCK) && request_module("target_core_iblock");
  177. if (ret != 0)
  178. pr_err("Unable to load target_core_iblock\n");
  179. ret = IS_ENABLED(CONFIG_TCM_FILEIO) && request_module("target_core_file");
  180. if (ret != 0)
  181. pr_err("Unable to load target_core_file\n");
  182. ret = IS_ENABLED(CONFIG_TCM_PSCSI) && request_module("target_core_pscsi");
  183. if (ret != 0)
  184. pr_err("Unable to load target_core_pscsi\n");
  185. ret = IS_ENABLED(CONFIG_TCM_USER2) && request_module("target_core_user");
  186. if (ret != 0)
  187. pr_err("Unable to load target_core_user\n");
  188. sub_api_initialized = 1;
  189. }
  190. static void target_release_sess_cmd_refcnt(struct percpu_ref *ref)
  191. {
  192. struct se_session *sess = container_of(ref, typeof(*sess), cmd_count);
  193. wake_up(&sess->cmd_list_wq);
  194. }
  195. /**
  196. * transport_init_session - initialize a session object
  197. * @se_sess: Session object pointer.
  198. *
  199. * The caller must have zero-initialized @se_sess before calling this function.
  200. */
  201. int transport_init_session(struct se_session *se_sess)
  202. {
  203. INIT_LIST_HEAD(&se_sess->sess_list);
  204. INIT_LIST_HEAD(&se_sess->sess_acl_list);
  205. INIT_LIST_HEAD(&se_sess->sess_cmd_list);
  206. spin_lock_init(&se_sess->sess_cmd_lock);
  207. init_waitqueue_head(&se_sess->cmd_list_wq);
  208. return percpu_ref_init(&se_sess->cmd_count,
  209. target_release_sess_cmd_refcnt, 0, GFP_KERNEL);
  210. }
  211. EXPORT_SYMBOL(transport_init_session);
  212. void transport_uninit_session(struct se_session *se_sess)
  213. {
  214. percpu_ref_exit(&se_sess->cmd_count);
  215. }
  216. /**
  217. * transport_alloc_session - allocate a session object and initialize it
  218. * @sup_prot_ops: bitmask that defines which T10-PI modes are supported.
  219. */
  220. struct se_session *transport_alloc_session(enum target_prot_op sup_prot_ops)
  221. {
  222. struct se_session *se_sess;
  223. int ret;
  224. se_sess = kmem_cache_zalloc(se_sess_cache, GFP_KERNEL);
  225. if (!se_sess) {
  226. pr_err("Unable to allocate struct se_session from"
  227. " se_sess_cache\n");
  228. return ERR_PTR(-ENOMEM);
  229. }
  230. ret = transport_init_session(se_sess);
  231. if (ret < 0) {
  232. kmem_cache_free(se_sess_cache, se_sess);
  233. return ERR_PTR(ret);
  234. }
  235. se_sess->sup_prot_ops = sup_prot_ops;
  236. return se_sess;
  237. }
  238. EXPORT_SYMBOL(transport_alloc_session);
  239. /**
  240. * transport_alloc_session_tags - allocate target driver private data
  241. * @se_sess: Session pointer.
  242. * @tag_num: Maximum number of in-flight commands between initiator and target.
  243. * @tag_size: Size in bytes of the private data a target driver associates with
  244. * each command.
  245. */
  246. int transport_alloc_session_tags(struct se_session *se_sess,
  247. unsigned int tag_num, unsigned int tag_size)
  248. {
  249. int rc;
  250. se_sess->sess_cmd_map = kvcalloc(tag_size, tag_num,
  251. GFP_KERNEL | __GFP_RETRY_MAYFAIL);
  252. if (!se_sess->sess_cmd_map) {
  253. pr_err("Unable to allocate se_sess->sess_cmd_map\n");
  254. return -ENOMEM;
  255. }
  256. rc = sbitmap_queue_init_node(&se_sess->sess_tag_pool, tag_num, -1,
  257. false, GFP_KERNEL, NUMA_NO_NODE);
  258. if (rc < 0) {
  259. pr_err("Unable to init se_sess->sess_tag_pool,"
  260. " tag_num: %u\n", tag_num);
  261. kvfree(se_sess->sess_cmd_map);
  262. se_sess->sess_cmd_map = NULL;
  263. return -ENOMEM;
  264. }
  265. return 0;
  266. }
  267. EXPORT_SYMBOL(transport_alloc_session_tags);
  268. /**
  269. * transport_init_session_tags - allocate a session and target driver private data
  270. * @tag_num: Maximum number of in-flight commands between initiator and target.
  271. * @tag_size: Size in bytes of the private data a target driver associates with
  272. * each command.
  273. * @sup_prot_ops: bitmask that defines which T10-PI modes are supported.
  274. */
  275. static struct se_session *
  276. transport_init_session_tags(unsigned int tag_num, unsigned int tag_size,
  277. enum target_prot_op sup_prot_ops)
  278. {
  279. struct se_session *se_sess;
  280. int rc;
  281. if (tag_num != 0 && !tag_size) {
  282. pr_err("init_session_tags called with percpu-ida tag_num:"
  283. " %u, but zero tag_size\n", tag_num);
  284. return ERR_PTR(-EINVAL);
  285. }
  286. if (!tag_num && tag_size) {
  287. pr_err("init_session_tags called with percpu-ida tag_size:"
  288. " %u, but zero tag_num\n", tag_size);
  289. return ERR_PTR(-EINVAL);
  290. }
  291. se_sess = transport_alloc_session(sup_prot_ops);
  292. if (IS_ERR(se_sess))
  293. return se_sess;
  294. rc = transport_alloc_session_tags(se_sess, tag_num, tag_size);
  295. if (rc < 0) {
  296. transport_free_session(se_sess);
  297. return ERR_PTR(-ENOMEM);
  298. }
  299. return se_sess;
  300. }
  301. /*
  302. * Called with spin_lock_irqsave(&struct se_portal_group->session_lock called.
  303. */
  304. void __transport_register_session(
  305. struct se_portal_group *se_tpg,
  306. struct se_node_acl *se_nacl,
  307. struct se_session *se_sess,
  308. void *fabric_sess_ptr)
  309. {
  310. const struct target_core_fabric_ops *tfo = se_tpg->se_tpg_tfo;
  311. unsigned char buf[PR_REG_ISID_LEN];
  312. unsigned long flags;
  313. se_sess->se_tpg = se_tpg;
  314. se_sess->fabric_sess_ptr = fabric_sess_ptr;
  315. /*
  316. * Used by struct se_node_acl's under ConfigFS to locate active se_session-t
  317. *
  318. * Only set for struct se_session's that will actually be moving I/O.
  319. * eg: *NOT* discovery sessions.
  320. */
  321. if (se_nacl) {
  322. /*
  323. *
  324. * Determine if fabric allows for T10-PI feature bits exposed to
  325. * initiators for device backends with !dev->dev_attrib.pi_prot_type.
  326. *
  327. * If so, then always save prot_type on a per se_node_acl node
  328. * basis and re-instate the previous sess_prot_type to avoid
  329. * disabling PI from below any previously initiator side
  330. * registered LUNs.
  331. */
  332. if (se_nacl->saved_prot_type)
  333. se_sess->sess_prot_type = se_nacl->saved_prot_type;
  334. else if (tfo->tpg_check_prot_fabric_only)
  335. se_sess->sess_prot_type = se_nacl->saved_prot_type =
  336. tfo->tpg_check_prot_fabric_only(se_tpg);
  337. /*
  338. * If the fabric module supports an ISID based TransportID,
  339. * save this value in binary from the fabric I_T Nexus now.
  340. */
  341. if (se_tpg->se_tpg_tfo->sess_get_initiator_sid != NULL) {
  342. memset(&buf[0], 0, PR_REG_ISID_LEN);
  343. se_tpg->se_tpg_tfo->sess_get_initiator_sid(se_sess,
  344. &buf[0], PR_REG_ISID_LEN);
  345. se_sess->sess_bin_isid = get_unaligned_be64(&buf[0]);
  346. }
  347. spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags);
  348. /*
  349. * The se_nacl->nacl_sess pointer will be set to the
  350. * last active I_T Nexus for each struct se_node_acl.
  351. */
  352. se_nacl->nacl_sess = se_sess;
  353. list_add_tail(&se_sess->sess_acl_list,
  354. &se_nacl->acl_sess_list);
  355. spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags);
  356. }
  357. list_add_tail(&se_sess->sess_list, &se_tpg->tpg_sess_list);
  358. pr_debug("TARGET_CORE[%s]: Registered fabric_sess_ptr: %p\n",
  359. se_tpg->se_tpg_tfo->fabric_name, se_sess->fabric_sess_ptr);
  360. }
  361. EXPORT_SYMBOL(__transport_register_session);
  362. void transport_register_session(
  363. struct se_portal_group *se_tpg,
  364. struct se_node_acl *se_nacl,
  365. struct se_session *se_sess,
  366. void *fabric_sess_ptr)
  367. {
  368. unsigned long flags;
  369. spin_lock_irqsave(&se_tpg->session_lock, flags);
  370. __transport_register_session(se_tpg, se_nacl, se_sess, fabric_sess_ptr);
  371. spin_unlock_irqrestore(&se_tpg->session_lock, flags);
  372. }
  373. EXPORT_SYMBOL(transport_register_session);
  374. struct se_session *
  375. target_setup_session(struct se_portal_group *tpg,
  376. unsigned int tag_num, unsigned int tag_size,
  377. enum target_prot_op prot_op,
  378. const char *initiatorname, void *private,
  379. int (*callback)(struct se_portal_group *,
  380. struct se_session *, void *))
  381. {
  382. struct se_session *sess;
  383. /*
  384. * If the fabric driver is using percpu-ida based pre allocation
  385. * of I/O descriptor tags, go ahead and perform that setup now..
  386. */
  387. if (tag_num != 0)
  388. sess = transport_init_session_tags(tag_num, tag_size, prot_op);
  389. else
  390. sess = transport_alloc_session(prot_op);
  391. if (IS_ERR(sess))
  392. return sess;
  393. sess->se_node_acl = core_tpg_check_initiator_node_acl(tpg,
  394. (unsigned char *)initiatorname);
  395. if (!sess->se_node_acl) {
  396. transport_free_session(sess);
  397. return ERR_PTR(-EACCES);
  398. }
  399. /*
  400. * Go ahead and perform any remaining fabric setup that is
  401. * required before transport_register_session().
  402. */
  403. if (callback != NULL) {
  404. int rc = callback(tpg, sess, private);
  405. if (rc) {
  406. transport_free_session(sess);
  407. return ERR_PTR(rc);
  408. }
  409. }
  410. transport_register_session(tpg, sess->se_node_acl, sess, private);
  411. return sess;
  412. }
  413. EXPORT_SYMBOL(target_setup_session);
  414. ssize_t target_show_dynamic_sessions(struct se_portal_group *se_tpg, char *page)
  415. {
  416. struct se_session *se_sess;
  417. ssize_t len = 0;
  418. spin_lock_bh(&se_tpg->session_lock);
  419. list_for_each_entry(se_sess, &se_tpg->tpg_sess_list, sess_list) {
  420. if (!se_sess->se_node_acl)
  421. continue;
  422. if (!se_sess->se_node_acl->dynamic_node_acl)
  423. continue;
  424. if (strlen(se_sess->se_node_acl->initiatorname) + 1 + len > PAGE_SIZE)
  425. break;
  426. len += snprintf(page + len, PAGE_SIZE - len, "%s\n",
  427. se_sess->se_node_acl->initiatorname);
  428. len += 1; /* Include NULL terminator */
  429. }
  430. spin_unlock_bh(&se_tpg->session_lock);
  431. return len;
  432. }
  433. EXPORT_SYMBOL(target_show_dynamic_sessions);
  434. static void target_complete_nacl(struct kref *kref)
  435. {
  436. struct se_node_acl *nacl = container_of(kref,
  437. struct se_node_acl, acl_kref);
  438. struct se_portal_group *se_tpg = nacl->se_tpg;
  439. if (!nacl->dynamic_stop) {
  440. complete(&nacl->acl_free_comp);
  441. return;
  442. }
  443. mutex_lock(&se_tpg->acl_node_mutex);
  444. list_del_init(&nacl->acl_list);
  445. mutex_unlock(&se_tpg->acl_node_mutex);
  446. core_tpg_wait_for_nacl_pr_ref(nacl);
  447. core_free_device_list_for_node(nacl, se_tpg);
  448. kfree(nacl);
  449. }
  450. void target_put_nacl(struct se_node_acl *nacl)
  451. {
  452. kref_put(&nacl->acl_kref, target_complete_nacl);
  453. }
  454. EXPORT_SYMBOL(target_put_nacl);
  455. void transport_deregister_session_configfs(struct se_session *se_sess)
  456. {
  457. struct se_node_acl *se_nacl;
  458. unsigned long flags;
  459. /*
  460. * Used by struct se_node_acl's under ConfigFS to locate active struct se_session
  461. */
  462. se_nacl = se_sess->se_node_acl;
  463. if (se_nacl) {
  464. spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags);
  465. if (!list_empty(&se_sess->sess_acl_list))
  466. list_del_init(&se_sess->sess_acl_list);
  467. /*
  468. * If the session list is empty, then clear the pointer.
  469. * Otherwise, set the struct se_session pointer from the tail
  470. * element of the per struct se_node_acl active session list.
  471. */
  472. if (list_empty(&se_nacl->acl_sess_list))
  473. se_nacl->nacl_sess = NULL;
  474. else {
  475. se_nacl->nacl_sess = container_of(
  476. se_nacl->acl_sess_list.prev,
  477. struct se_session, sess_acl_list);
  478. }
  479. spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags);
  480. }
  481. }
  482. EXPORT_SYMBOL(transport_deregister_session_configfs);
  483. void transport_free_session(struct se_session *se_sess)
  484. {
  485. struct se_node_acl *se_nacl = se_sess->se_node_acl;
  486. /*
  487. * Drop the se_node_acl->nacl_kref obtained from within
  488. * core_tpg_get_initiator_node_acl().
  489. */
  490. if (se_nacl) {
  491. struct se_portal_group *se_tpg = se_nacl->se_tpg;
  492. const struct target_core_fabric_ops *se_tfo = se_tpg->se_tpg_tfo;
  493. unsigned long flags;
  494. se_sess->se_node_acl = NULL;
  495. /*
  496. * Also determine if we need to drop the extra ->cmd_kref if
  497. * it had been previously dynamically generated, and
  498. * the endpoint is not caching dynamic ACLs.
  499. */
  500. mutex_lock(&se_tpg->acl_node_mutex);
  501. if (se_nacl->dynamic_node_acl &&
  502. !se_tfo->tpg_check_demo_mode_cache(se_tpg)) {
  503. spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags);
  504. if (list_empty(&se_nacl->acl_sess_list))
  505. se_nacl->dynamic_stop = true;
  506. spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags);
  507. if (se_nacl->dynamic_stop)
  508. list_del_init(&se_nacl->acl_list);
  509. }
  510. mutex_unlock(&se_tpg->acl_node_mutex);
  511. if (se_nacl->dynamic_stop)
  512. target_put_nacl(se_nacl);
  513. target_put_nacl(se_nacl);
  514. }
  515. if (se_sess->sess_cmd_map) {
  516. sbitmap_queue_free(&se_sess->sess_tag_pool);
  517. kvfree(se_sess->sess_cmd_map);
  518. }
  519. transport_uninit_session(se_sess);
  520. kmem_cache_free(se_sess_cache, se_sess);
  521. }
  522. EXPORT_SYMBOL(transport_free_session);
  523. static int target_release_res(struct se_device *dev, void *data)
  524. {
  525. struct se_session *sess = data;
  526. if (dev->reservation_holder == sess)
  527. target_release_reservation(dev);
  528. return 0;
  529. }
  530. void transport_deregister_session(struct se_session *se_sess)
  531. {
  532. struct se_portal_group *se_tpg = se_sess->se_tpg;
  533. unsigned long flags;
  534. if (!se_tpg) {
  535. transport_free_session(se_sess);
  536. return;
  537. }
  538. spin_lock_irqsave(&se_tpg->session_lock, flags);
  539. list_del(&se_sess->sess_list);
  540. se_sess->se_tpg = NULL;
  541. se_sess->fabric_sess_ptr = NULL;
  542. spin_unlock_irqrestore(&se_tpg->session_lock, flags);
  543. /*
  544. * Since the session is being removed, release SPC-2
  545. * reservations held by the session that is disappearing.
  546. */
  547. target_for_each_device(target_release_res, se_sess);
  548. pr_debug("TARGET_CORE[%s]: Deregistered fabric_sess\n",
  549. se_tpg->se_tpg_tfo->fabric_name);
  550. /*
  551. * If last kref is dropping now for an explicit NodeACL, awake sleeping
  552. * ->acl_free_comp caller to wakeup configfs se_node_acl->acl_group
  553. * removal context from within transport_free_session() code.
  554. *
  555. * For dynamic ACL, target_put_nacl() uses target_complete_nacl()
  556. * to release all remaining generate_node_acl=1 created ACL resources.
  557. */
  558. transport_free_session(se_sess);
  559. }
  560. EXPORT_SYMBOL(transport_deregister_session);
  561. void target_remove_session(struct se_session *se_sess)
  562. {
  563. transport_deregister_session_configfs(se_sess);
  564. transport_deregister_session(se_sess);
  565. }
  566. EXPORT_SYMBOL(target_remove_session);
  567. static void target_remove_from_state_list(struct se_cmd *cmd)
  568. {
  569. struct se_device *dev = cmd->se_dev;
  570. unsigned long flags;
  571. if (!dev)
  572. return;
  573. spin_lock_irqsave(&dev->execute_task_lock, flags);
  574. if (cmd->state_active) {
  575. list_del(&cmd->state_list);
  576. cmd->state_active = false;
  577. }
  578. spin_unlock_irqrestore(&dev->execute_task_lock, flags);
  579. }
  580. /*
  581. * This function is called by the target core after the target core has
  582. * finished processing a SCSI command or SCSI TMF. Both the regular command
  583. * processing code and the code for aborting commands can call this
  584. * function. CMD_T_STOP is set if and only if another thread is waiting
  585. * inside transport_wait_for_tasks() for t_transport_stop_comp.
  586. */
  587. static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd)
  588. {
  589. unsigned long flags;
  590. target_remove_from_state_list(cmd);
  591. /*
  592. * Clear struct se_cmd->se_lun before the handoff to FE.
  593. */
  594. cmd->se_lun = NULL;
  595. spin_lock_irqsave(&cmd->t_state_lock, flags);
  596. /*
  597. * Determine if frontend context caller is requesting the stopping of
  598. * this command for frontend exceptions.
  599. */
  600. if (cmd->transport_state & CMD_T_STOP) {
  601. pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n",
  602. __func__, __LINE__, cmd->tag);
  603. spin_unlock_irqrestore(&cmd->t_state_lock, flags);
  604. complete_all(&cmd->t_transport_stop_comp);
  605. return 1;
  606. }
  607. cmd->transport_state &= ~CMD_T_ACTIVE;
  608. spin_unlock_irqrestore(&cmd->t_state_lock, flags);
  609. /*
  610. * Some fabric modules like tcm_loop can release their internally
  611. * allocated I/O reference and struct se_cmd now.
  612. *
  613. * Fabric modules are expected to return '1' here if the se_cmd being
  614. * passed is released at this point, or zero if not being released.
  615. */
  616. return cmd->se_tfo->check_stop_free(cmd);
  617. }
  618. static void transport_lun_remove_cmd(struct se_cmd *cmd)
  619. {
  620. struct se_lun *lun = cmd->se_lun;
  621. if (!lun)
  622. return;
  623. if (cmpxchg(&cmd->lun_ref_active, true, false))
  624. percpu_ref_put(&lun->lun_ref);
  625. }
  626. static void target_complete_failure_work(struct work_struct *work)
  627. {
  628. struct se_cmd *cmd = container_of(work, struct se_cmd, work);
  629. transport_generic_request_failure(cmd,
  630. TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE);
  631. }
  632. /*
  633. * Used when asking transport to copy Sense Data from the underlying
  634. * Linux/SCSI struct scsi_cmnd
  635. */
  636. static unsigned char *transport_get_sense_buffer(struct se_cmd *cmd)
  637. {
  638. struct se_device *dev = cmd->se_dev;
  639. WARN_ON(!cmd->se_lun);
  640. if (!dev)
  641. return NULL;
  642. if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION)
  643. return NULL;
  644. cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER;
  645. pr_debug("HBA_[%u]_PLUG[%s]: Requesting sense for SAM STATUS: 0x%02x\n",
  646. dev->se_hba->hba_id, dev->transport->name, cmd->scsi_status);
  647. return cmd->sense_buffer;
  648. }
  649. void transport_copy_sense_to_cmd(struct se_cmd *cmd, unsigned char *sense)
  650. {
  651. unsigned char *cmd_sense_buf;
  652. unsigned long flags;
  653. spin_lock_irqsave(&cmd->t_state_lock, flags);
  654. cmd_sense_buf = transport_get_sense_buffer(cmd);
  655. if (!cmd_sense_buf) {
  656. spin_unlock_irqrestore(&cmd->t_state_lock, flags);
  657. return;
  658. }
  659. cmd->se_cmd_flags |= SCF_TRANSPORT_TASK_SENSE;
  660. memcpy(cmd_sense_buf, sense, cmd->scsi_sense_length);
  661. spin_unlock_irqrestore(&cmd->t_state_lock, flags);
  662. }
  663. EXPORT_SYMBOL(transport_copy_sense_to_cmd);
  664. static void target_handle_abort(struct se_cmd *cmd)
  665. {
  666. bool tas = cmd->transport_state & CMD_T_TAS;
  667. bool ack_kref = cmd->se_cmd_flags & SCF_ACK_KREF;
  668. int ret;
  669. pr_debug("tag %#llx: send_abort_response = %d\n", cmd->tag, tas);
  670. if (tas) {
  671. if (!(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) {
  672. cmd->scsi_status = SAM_STAT_TASK_ABORTED;
  673. pr_debug("Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x, ITT: 0x%08llx\n",
  674. cmd->t_task_cdb[0], cmd->tag);
  675. trace_target_cmd_complete(cmd);
  676. ret = cmd->se_tfo->queue_status(cmd);
  677. if (ret) {
  678. transport_handle_queue_full(cmd, cmd->se_dev,
  679. ret, false);
  680. return;
  681. }
  682. } else {
  683. cmd->se_tmr_req->response = TMR_FUNCTION_REJECTED;
  684. cmd->se_tfo->queue_tm_rsp(cmd);
  685. }
  686. } else {
  687. /*
  688. * Allow the fabric driver to unmap any resources before
  689. * releasing the descriptor via TFO->release_cmd().
  690. */
  691. cmd->se_tfo->aborted_task(cmd);
  692. if (ack_kref)
  693. WARN_ON_ONCE(target_put_sess_cmd(cmd) != 0);
  694. /*
  695. * To do: establish a unit attention condition on the I_T
  696. * nexus associated with cmd. See also the paragraph "Aborting
  697. * commands" in SAM.
  698. */
  699. }
  700. WARN_ON_ONCE(kref_read(&cmd->cmd_kref) == 0);
  701. transport_lun_remove_cmd(cmd);
  702. transport_cmd_check_stop_to_fabric(cmd);
  703. }
  704. static void target_abort_work(struct work_struct *work)
  705. {
  706. struct se_cmd *cmd = container_of(work, struct se_cmd, work);
  707. target_handle_abort(cmd);
  708. }
  709. static bool target_cmd_interrupted(struct se_cmd *cmd)
  710. {
  711. int post_ret;
  712. if (cmd->transport_state & CMD_T_ABORTED) {
  713. if (cmd->transport_complete_callback)
  714. cmd->transport_complete_callback(cmd, false, &post_ret);
  715. INIT_WORK(&cmd->work, target_abort_work);
  716. queue_work(target_completion_wq, &cmd->work);
  717. return true;
  718. } else if (cmd->transport_state & CMD_T_STOP) {
  719. if (cmd->transport_complete_callback)
  720. cmd->transport_complete_callback(cmd, false, &post_ret);
  721. complete_all(&cmd->t_transport_stop_comp);
  722. return true;
  723. }
  724. return false;
  725. }
  726. /* May be called from interrupt context so must not sleep. */
  727. void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status)
  728. {
  729. int success;
  730. unsigned long flags;
  731. if (target_cmd_interrupted(cmd))
  732. return;
  733. cmd->scsi_status = scsi_status;
  734. spin_lock_irqsave(&cmd->t_state_lock, flags);
  735. switch (cmd->scsi_status) {
  736. case SAM_STAT_CHECK_CONDITION:
  737. if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE)
  738. success = 1;
  739. else
  740. success = 0;
  741. break;
  742. default:
  743. success = 1;
  744. break;
  745. }
  746. cmd->t_state = TRANSPORT_COMPLETE;
  747. cmd->transport_state |= (CMD_T_COMPLETE | CMD_T_ACTIVE);
  748. spin_unlock_irqrestore(&cmd->t_state_lock, flags);
  749. INIT_WORK(&cmd->work, success ? target_complete_ok_work :
  750. target_complete_failure_work);
  751. if (cmd->se_cmd_flags & SCF_USE_CPUID)
  752. queue_work_on(cmd->cpuid, target_completion_wq, &cmd->work);
  753. else
  754. queue_work(target_completion_wq, &cmd->work);
  755. }
  756. EXPORT_SYMBOL(target_complete_cmd);
  757. void target_set_cmd_data_length(struct se_cmd *cmd, int length)
  758. {
  759. if (length < cmd->data_length) {
  760. if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) {
  761. cmd->residual_count += cmd->data_length - length;
  762. } else {
  763. cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
  764. cmd->residual_count = cmd->data_length - length;
  765. }
  766. cmd->data_length = length;
  767. }
  768. }
  769. EXPORT_SYMBOL(target_set_cmd_data_length);
  770. void target_complete_cmd_with_length(struct se_cmd *cmd, u8 scsi_status, int length)
  771. {
  772. if (scsi_status == SAM_STAT_GOOD ||
  773. cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL) {
  774. target_set_cmd_data_length(cmd, length);
  775. }
  776. target_complete_cmd(cmd, scsi_status);
  777. }
  778. EXPORT_SYMBOL(target_complete_cmd_with_length);
  779. static void target_add_to_state_list(struct se_cmd *cmd)
  780. {
  781. struct se_device *dev = cmd->se_dev;
  782. unsigned long flags;
  783. spin_lock_irqsave(&dev->execute_task_lock, flags);
  784. if (!cmd->state_active) {
  785. list_add_tail(&cmd->state_list, &dev->state_list);
  786. cmd->state_active = true;
  787. }
  788. spin_unlock_irqrestore(&dev->execute_task_lock, flags);
  789. }
  790. /*
  791. * Handle QUEUE_FULL / -EAGAIN and -ENOMEM status
  792. */
  793. static void transport_write_pending_qf(struct se_cmd *cmd);
  794. static void transport_complete_qf(struct se_cmd *cmd);
  795. void target_qf_do_work(struct work_struct *work)
  796. {
  797. struct se_device *dev = container_of(work, struct se_device,
  798. qf_work_queue);
  799. LIST_HEAD(qf_cmd_list);
  800. struct se_cmd *cmd, *cmd_tmp;
  801. spin_lock_irq(&dev->qf_cmd_lock);
  802. list_splice_init(&dev->qf_cmd_list, &qf_cmd_list);
  803. spin_unlock_irq(&dev->qf_cmd_lock);
  804. list_for_each_entry_safe(cmd, cmd_tmp, &qf_cmd_list, se_qf_node) {
  805. list_del(&cmd->se_qf_node);
  806. atomic_dec_mb(&dev->dev_qf_count);
  807. pr_debug("Processing %s cmd: %p QUEUE_FULL in work queue"
  808. " context: %s\n", cmd->se_tfo->fabric_name, cmd,
  809. (cmd->t_state == TRANSPORT_COMPLETE_QF_OK) ? "COMPLETE_OK" :
  810. (cmd->t_state == TRANSPORT_COMPLETE_QF_WP) ? "WRITE_PENDING"
  811. : "UNKNOWN");
  812. if (cmd->t_state == TRANSPORT_COMPLETE_QF_WP)
  813. transport_write_pending_qf(cmd);
  814. else if (cmd->t_state == TRANSPORT_COMPLETE_QF_OK ||
  815. cmd->t_state == TRANSPORT_COMPLETE_QF_ERR)
  816. transport_complete_qf(cmd);
  817. }
  818. }
  819. unsigned char *transport_dump_cmd_direction(struct se_cmd *cmd)
  820. {
  821. switch (cmd->data_direction) {
  822. case DMA_NONE:
  823. return "NONE";
  824. case DMA_FROM_DEVICE:
  825. return "READ";
  826. case DMA_TO_DEVICE:
  827. return "WRITE";
  828. case DMA_BIDIRECTIONAL:
  829. return "BIDI";
  830. default:
  831. break;
  832. }
  833. return "UNKNOWN";
  834. }
  835. void transport_dump_dev_state(
  836. struct se_device *dev,
  837. char *b,
  838. int *bl)
  839. {
  840. *bl += sprintf(b + *bl, "Status: ");
  841. if (dev->export_count)
  842. *bl += sprintf(b + *bl, "ACTIVATED");
  843. else
  844. *bl += sprintf(b + *bl, "DEACTIVATED");
  845. *bl += sprintf(b + *bl, " Max Queue Depth: %d", dev->queue_depth);
  846. *bl += sprintf(b + *bl, " SectorSize: %u HwMaxSectors: %u\n",
  847. dev->dev_attrib.block_size,
  848. dev->dev_attrib.hw_max_sectors);
  849. *bl += sprintf(b + *bl, " ");
  850. }
  851. void transport_dump_vpd_proto_id(
  852. struct t10_vpd *vpd,
  853. unsigned char *p_buf,
  854. int p_buf_len)
  855. {
  856. unsigned char buf[VPD_TMP_BUF_SIZE];
  857. int len;
  858. memset(buf, 0, VPD_TMP_BUF_SIZE);
  859. len = sprintf(buf, "T10 VPD Protocol Identifier: ");
  860. switch (vpd->protocol_identifier) {
  861. case 0x00:
  862. sprintf(buf+len, "Fibre Channel\n");
  863. break;
  864. case 0x10:
  865. sprintf(buf+len, "Parallel SCSI\n");
  866. break;
  867. case 0x20:
  868. sprintf(buf+len, "SSA\n");
  869. break;
  870. case 0x30:
  871. sprintf(buf+len, "IEEE 1394\n");
  872. break;
  873. case 0x40:
  874. sprintf(buf+len, "SCSI Remote Direct Memory Access"
  875. " Protocol\n");
  876. break;
  877. case 0x50:
  878. sprintf(buf+len, "Internet SCSI (iSCSI)\n");
  879. break;
  880. case 0x60:
  881. sprintf(buf+len, "SAS Serial SCSI Protocol\n");
  882. break;
  883. case 0x70:
  884. sprintf(buf+len, "Automation/Drive Interface Transport"
  885. " Protocol\n");
  886. break;
  887. case 0x80:
  888. sprintf(buf+len, "AT Attachment Interface ATA/ATAPI\n");
  889. break;
  890. default:
  891. sprintf(buf+len, "Unknown 0x%02x\n",
  892. vpd->protocol_identifier);
  893. break;
  894. }
  895. if (p_buf)
  896. strncpy(p_buf, buf, p_buf_len);
  897. else
  898. pr_debug("%s", buf);
  899. }
  900. void
  901. transport_set_vpd_proto_id(struct t10_vpd *vpd, unsigned char *page_83)
  902. {
  903. /*
  904. * Check if the Protocol Identifier Valid (PIV) bit is set..
  905. *
  906. * from spc3r23.pdf section 7.5.1
  907. */
  908. if (page_83[1] & 0x80) {
  909. vpd->protocol_identifier = (page_83[0] & 0xf0);
  910. vpd->protocol_identifier_set = 1;
  911. transport_dump_vpd_proto_id(vpd, NULL, 0);
  912. }
  913. }
  914. EXPORT_SYMBOL(transport_set_vpd_proto_id);
  915. int transport_dump_vpd_assoc(
  916. struct t10_vpd *vpd,
  917. unsigned char *p_buf,
  918. int p_buf_len)
  919. {
  920. unsigned char buf[VPD_TMP_BUF_SIZE];
  921. int ret = 0;
  922. int len;
  923. memset(buf, 0, VPD_TMP_BUF_SIZE);
  924. len = sprintf(buf, "T10 VPD Identifier Association: ");
  925. switch (vpd->association) {
  926. case 0x00:
  927. sprintf(buf+len, "addressed logical unit\n");
  928. break;
  929. case 0x10:
  930. sprintf(buf+len, "target port\n");
  931. break;
  932. case 0x20:
  933. sprintf(buf+len, "SCSI target device\n");
  934. break;
  935. default:
  936. sprintf(buf+len, "Unknown 0x%02x\n", vpd->association);
  937. ret = -EINVAL;
  938. break;
  939. }
  940. if (p_buf)
  941. strncpy(p_buf, buf, p_buf_len);
  942. else
  943. pr_debug("%s", buf);
  944. return ret;
  945. }
  946. int transport_set_vpd_assoc(struct t10_vpd *vpd, unsigned char *page_83)
  947. {
  948. /*
  949. * The VPD identification association..
  950. *
  951. * from spc3r23.pdf Section 7.6.3.1 Table 297
  952. */
  953. vpd->association = (page_83[1] & 0x30);
  954. return transport_dump_vpd_assoc(vpd, NULL, 0);
  955. }
  956. EXPORT_SYMBOL(transport_set_vpd_assoc);
  957. int transport_dump_vpd_ident_type(
  958. struct t10_vpd *vpd,
  959. unsigned char *p_buf,
  960. int p_buf_len)
  961. {
  962. unsigned char buf[VPD_TMP_BUF_SIZE];
  963. int ret = 0;
  964. int len;
  965. memset(buf, 0, VPD_TMP_BUF_SIZE);
  966. len = sprintf(buf, "T10 VPD Identifier Type: ");
  967. switch (vpd->device_identifier_type) {
  968. case 0x00:
  969. sprintf(buf+len, "Vendor specific\n");
  970. break;
  971. case 0x01:
  972. sprintf(buf+len, "T10 Vendor ID based\n");
  973. break;
  974. case 0x02:
  975. sprintf(buf+len, "EUI-64 based\n");
  976. break;
  977. case 0x03:
  978. sprintf(buf+len, "NAA\n");
  979. break;
  980. case 0x04:
  981. sprintf(buf+len, "Relative target port identifier\n");
  982. break;
  983. case 0x08:
  984. sprintf(buf+len, "SCSI name string\n");
  985. break;
  986. default:
  987. sprintf(buf+len, "Unsupported: 0x%02x\n",
  988. vpd->device_identifier_type);
  989. ret = -EINVAL;
  990. break;
  991. }
  992. if (p_buf) {
  993. if (p_buf_len < strlen(buf)+1)
  994. return -EINVAL;
  995. strncpy(p_buf, buf, p_buf_len);
  996. } else {
  997. pr_debug("%s", buf);
  998. }
  999. return ret;
  1000. }
  1001. int transport_set_vpd_ident_type(struct t10_vpd *vpd, unsigned char *page_83)
  1002. {
  1003. /*
  1004. * The VPD identifier type..
  1005. *
  1006. * from spc3r23.pdf Section 7.6.3.1 Table 298
  1007. */
  1008. vpd->device_identifier_type = (page_83[1] & 0x0f);
  1009. return transport_dump_vpd_ident_type(vpd, NULL, 0);
  1010. }
  1011. EXPORT_SYMBOL(transport_set_vpd_ident_type);
  1012. int transport_dump_vpd_ident(
  1013. struct t10_vpd *vpd,
  1014. unsigned char *p_buf,
  1015. int p_buf_len)
  1016. {
  1017. unsigned char buf[VPD_TMP_BUF_SIZE];
  1018. int ret = 0;
  1019. memset(buf, 0, VPD_TMP_BUF_SIZE);
  1020. switch (vpd->device_identifier_code_set) {
  1021. case 0x01: /* Binary */
  1022. snprintf(buf, sizeof(buf),
  1023. "T10 VPD Binary Device Identifier: %s\n",
  1024. &vpd->device_identifier[0]);
  1025. break;
  1026. case 0x02: /* ASCII */
  1027. snprintf(buf, sizeof(buf),
  1028. "T10 VPD ASCII Device Identifier: %s\n",
  1029. &vpd->device_identifier[0]);
  1030. break;
  1031. case 0x03: /* UTF-8 */
  1032. snprintf(buf, sizeof(buf),
  1033. "T10 VPD UTF-8 Device Identifier: %s\n",
  1034. &vpd->device_identifier[0]);
  1035. break;
  1036. default:
  1037. sprintf(buf, "T10 VPD Device Identifier encoding unsupported:"
  1038. " 0x%02x", vpd->device_identifier_code_set);
  1039. ret = -EINVAL;
  1040. break;
  1041. }
  1042. if (p_buf)
  1043. strncpy(p_buf, buf, p_buf_len);
  1044. else
  1045. pr_debug("%s", buf);
  1046. return ret;
  1047. }
  1048. int
  1049. transport_set_vpd_ident(struct t10_vpd *vpd, unsigned char *page_83)
  1050. {
  1051. static const char hex_str[] = "0123456789abcdef";
  1052. int j = 0, i = 4; /* offset to start of the identifier */
  1053. /*
  1054. * The VPD Code Set (encoding)
  1055. *
  1056. * from spc3r23.pdf Section 7.6.3.1 Table 296
  1057. */
  1058. vpd->device_identifier_code_set = (page_83[0] & 0x0f);
  1059. switch (vpd->device_identifier_code_set) {
  1060. case 0x01: /* Binary */
  1061. vpd->device_identifier[j++] =
  1062. hex_str[vpd->device_identifier_type];
  1063. while (i < (4 + page_83[3])) {
  1064. vpd->device_identifier[j++] =
  1065. hex_str[(page_83[i] & 0xf0) >> 4];
  1066. vpd->device_identifier[j++] =
  1067. hex_str[page_83[i] & 0x0f];
  1068. i++;
  1069. }
  1070. break;
  1071. case 0x02: /* ASCII */
  1072. case 0x03: /* UTF-8 */
  1073. while (i < (4 + page_83[3]))
  1074. vpd->device_identifier[j++] = page_83[i++];
  1075. break;
  1076. default:
  1077. break;
  1078. }
  1079. return transport_dump_vpd_ident(vpd, NULL, 0);
  1080. }
  1081. EXPORT_SYMBOL(transport_set_vpd_ident);
  1082. static sense_reason_t
  1083. target_check_max_data_sg_nents(struct se_cmd *cmd, struct se_device *dev,
  1084. unsigned int size)
  1085. {
  1086. u32 mtl;
  1087. if (!cmd->se_tfo->max_data_sg_nents)
  1088. return TCM_NO_SENSE;
  1089. /*
  1090. * Check if fabric enforced maximum SGL entries per I/O descriptor
  1091. * exceeds se_cmd->data_length. If true, set SCF_UNDERFLOW_BIT +
  1092. * residual_count and reduce original cmd->data_length to maximum
  1093. * length based on single PAGE_SIZE entry scatter-lists.
  1094. */
  1095. mtl = (cmd->se_tfo->max_data_sg_nents * PAGE_SIZE);
  1096. if (cmd->data_length > mtl) {
  1097. /*
  1098. * If an existing CDB overflow is present, calculate new residual
  1099. * based on CDB size minus fabric maximum transfer length.
  1100. *
  1101. * If an existing CDB underflow is present, calculate new residual
  1102. * based on original cmd->data_length minus fabric maximum transfer
  1103. * length.
  1104. *
  1105. * Otherwise, set the underflow residual based on cmd->data_length
  1106. * minus fabric maximum transfer length.
  1107. */
  1108. if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) {
  1109. cmd->residual_count = (size - mtl);
  1110. } else if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) {
  1111. u32 orig_dl = size + cmd->residual_count;
  1112. cmd->residual_count = (orig_dl - mtl);
  1113. } else {
  1114. cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
  1115. cmd->residual_count = (cmd->data_length - mtl);
  1116. }
  1117. cmd->data_length = mtl;
  1118. /*
  1119. * Reset sbc_check_prot() calculated protection payload
  1120. * length based upon the new smaller MTL.
  1121. */
  1122. if (cmd->prot_length) {
  1123. u32 sectors = (mtl / dev->dev_attrib.block_size);
  1124. cmd->prot_length = dev->prot_length * sectors;
  1125. }
  1126. }
  1127. return TCM_NO_SENSE;
  1128. }
  1129. /**
  1130. * target_cmd_size_check - Check whether there will be a residual.
  1131. * @cmd: SCSI command.
  1132. * @size: Data buffer size derived from CDB. The data buffer size provided by
  1133. * the SCSI transport driver is available in @cmd->data_length.
  1134. *
  1135. * Compare the data buffer size from the CDB with the data buffer limit from the transport
  1136. * header. Set @cmd->residual_count and SCF_OVERFLOW_BIT or SCF_UNDERFLOW_BIT if necessary.
  1137. *
  1138. * Note: target drivers set @cmd->data_length by calling transport_init_se_cmd().
  1139. *
  1140. * Return: TCM_NO_SENSE
  1141. */
  1142. sense_reason_t
  1143. target_cmd_size_check(struct se_cmd *cmd, unsigned int size)
  1144. {
  1145. struct se_device *dev = cmd->se_dev;
  1146. if (cmd->unknown_data_length) {
  1147. cmd->data_length = size;
  1148. } else if (size != cmd->data_length) {
  1149. pr_warn_ratelimited("TARGET_CORE[%s]: Expected Transfer Length:"
  1150. " %u does not match SCSI CDB Length: %u for SAM Opcode:"
  1151. " 0x%02x\n", cmd->se_tfo->fabric_name,
  1152. cmd->data_length, size, cmd->t_task_cdb[0]);
  1153. if (cmd->data_direction == DMA_TO_DEVICE) {
  1154. if (cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) {
  1155. pr_err_ratelimited("Rejecting underflow/overflow"
  1156. " for WRITE data CDB\n");
  1157. return TCM_INVALID_CDB_FIELD;
  1158. }
  1159. /*
  1160. * Some fabric drivers like iscsi-target still expect to
  1161. * always reject overflow writes. Reject this case until
  1162. * full fabric driver level support for overflow writes
  1163. * is introduced tree-wide.
  1164. */
  1165. if (size > cmd->data_length) {
  1166. pr_err_ratelimited("Rejecting overflow for"
  1167. " WRITE control CDB\n");
  1168. return TCM_INVALID_CDB_FIELD;
  1169. }
  1170. }
  1171. /*
  1172. * Reject READ_* or WRITE_* with overflow/underflow for
  1173. * type SCF_SCSI_DATA_CDB.
  1174. */
  1175. if (dev->dev_attrib.block_size != 512) {
  1176. pr_err("Failing OVERFLOW/UNDERFLOW for LBA op"
  1177. " CDB on non 512-byte sector setup subsystem"
  1178. " plugin: %s\n", dev->transport->name);
  1179. /* Returns CHECK_CONDITION + INVALID_CDB_FIELD */
  1180. return TCM_INVALID_CDB_FIELD;
  1181. }
  1182. /*
  1183. * For the overflow case keep the existing fabric provided
  1184. * ->data_length. Otherwise for the underflow case, reset
  1185. * ->data_length to the smaller SCSI expected data transfer
  1186. * length.
  1187. */
  1188. if (size > cmd->data_length) {
  1189. cmd->se_cmd_flags |= SCF_OVERFLOW_BIT;
  1190. cmd->residual_count = (size - cmd->data_length);
  1191. } else {
  1192. cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
  1193. cmd->residual_count = (cmd->data_length - size);
  1194. cmd->data_length = size;
  1195. }
  1196. }
  1197. return target_check_max_data_sg_nents(cmd, dev, size);
  1198. }
  1199. /*
  1200. * Used by fabric modules containing a local struct se_cmd within their
  1201. * fabric dependent per I/O descriptor.
  1202. *
  1203. * Preserves the value of @cmd->tag.
  1204. */
  1205. void transport_init_se_cmd(
  1206. struct se_cmd *cmd,
  1207. const struct target_core_fabric_ops *tfo,
  1208. struct se_session *se_sess,
  1209. u32 data_length,
  1210. int data_direction,
  1211. int task_attr,
  1212. unsigned char *sense_buffer, u64 unpacked_lun)
  1213. {
  1214. INIT_LIST_HEAD(&cmd->se_delayed_node);
  1215. INIT_LIST_HEAD(&cmd->se_qf_node);
  1216. INIT_LIST_HEAD(&cmd->se_cmd_list);
  1217. INIT_LIST_HEAD(&cmd->state_list);
  1218. init_completion(&cmd->t_transport_stop_comp);
  1219. cmd->free_compl = NULL;
  1220. cmd->abrt_compl = NULL;
  1221. spin_lock_init(&cmd->t_state_lock);
  1222. INIT_WORK(&cmd->work, NULL);
  1223. kref_init(&cmd->cmd_kref);
  1224. cmd->se_tfo = tfo;
  1225. cmd->se_sess = se_sess;
  1226. cmd->data_length = data_length;
  1227. cmd->data_direction = data_direction;
  1228. cmd->sam_task_attr = task_attr;
  1229. cmd->sense_buffer = sense_buffer;
  1230. cmd->orig_fe_lun = unpacked_lun;
  1231. cmd->state_active = false;
  1232. }
  1233. EXPORT_SYMBOL(transport_init_se_cmd);
  1234. static sense_reason_t
  1235. transport_check_alloc_task_attr(struct se_cmd *cmd)
  1236. {
  1237. struct se_device *dev = cmd->se_dev;
  1238. /*
  1239. * Check if SAM Task Attribute emulation is enabled for this
  1240. * struct se_device storage object
  1241. */
  1242. if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
  1243. return 0;
  1244. if (cmd->sam_task_attr == TCM_ACA_TAG) {
  1245. pr_debug("SAM Task Attribute ACA"
  1246. " emulation is not supported\n");
  1247. return TCM_INVALID_CDB_FIELD;
  1248. }
  1249. return 0;
  1250. }
  1251. sense_reason_t
  1252. target_cmd_init_cdb(struct se_cmd *cmd, unsigned char *cdb)
  1253. {
  1254. sense_reason_t ret;
  1255. cmd->t_task_cdb = &cmd->__t_task_cdb[0];
  1256. /*
  1257. * Ensure that the received CDB is less than the max (252 + 8) bytes
  1258. * for VARIABLE_LENGTH_CMD
  1259. */
  1260. if (scsi_command_size(cdb) > SCSI_MAX_VARLEN_CDB_SIZE) {
  1261. pr_err("Received SCSI CDB with command_size: %d that"
  1262. " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
  1263. scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE);
  1264. ret = TCM_INVALID_CDB_FIELD;
  1265. goto err;
  1266. }
  1267. /*
  1268. * If the received CDB is larger than TCM_MAX_COMMAND_SIZE,
  1269. * allocate the additional extended CDB buffer now.. Otherwise
  1270. * setup the pointer from __t_task_cdb to t_task_cdb.
  1271. */
  1272. if (scsi_command_size(cdb) > sizeof(cmd->__t_task_cdb)) {
  1273. cmd->t_task_cdb = kzalloc(scsi_command_size(cdb),
  1274. GFP_KERNEL);
  1275. if (!cmd->t_task_cdb) {
  1276. pr_err("Unable to allocate cmd->t_task_cdb"
  1277. " %u > sizeof(cmd->__t_task_cdb): %lu ops\n",
  1278. scsi_command_size(cdb),
  1279. (unsigned long)sizeof(cmd->__t_task_cdb));
  1280. ret = TCM_OUT_OF_RESOURCES;
  1281. goto err;
  1282. }
  1283. }
  1284. /*
  1285. * Copy the original CDB into cmd->
  1286. */
  1287. memcpy(cmd->t_task_cdb, cdb, scsi_command_size(cdb));
  1288. trace_target_sequencer_start(cmd);
  1289. return 0;
  1290. err:
  1291. /*
  1292. * Copy the CDB here to allow trace_target_cmd_complete() to
  1293. * print the cdb to the trace buffers.
  1294. */
  1295. memcpy(cmd->t_task_cdb, cdb, min(scsi_command_size(cdb),
  1296. (unsigned int)TCM_MAX_COMMAND_SIZE));
  1297. return ret;
  1298. }
  1299. EXPORT_SYMBOL(target_cmd_init_cdb);
  1300. sense_reason_t
  1301. target_cmd_parse_cdb(struct se_cmd *cmd)
  1302. {
  1303. struct se_device *dev = cmd->se_dev;
  1304. sense_reason_t ret;
  1305. ret = dev->transport->parse_cdb(cmd);
  1306. if (ret == TCM_UNSUPPORTED_SCSI_OPCODE)
  1307. pr_warn_ratelimited("%s/%s: Unsupported SCSI Opcode 0x%02x, sending CHECK_CONDITION.\n",
  1308. cmd->se_tfo->fabric_name,
  1309. cmd->se_sess->se_node_acl->initiatorname,
  1310. cmd->t_task_cdb[0]);
  1311. if (ret)
  1312. return ret;
  1313. ret = transport_check_alloc_task_attr(cmd);
  1314. if (ret)
  1315. return ret;
  1316. cmd->se_cmd_flags |= SCF_SUPPORTED_SAM_OPCODE;
  1317. atomic_long_inc(&cmd->se_lun->lun_stats.cmd_pdus);
  1318. return 0;
  1319. }
  1320. EXPORT_SYMBOL(target_cmd_parse_cdb);
  1321. /*
  1322. * Used by fabric module frontends to queue tasks directly.
  1323. * May only be used from process context.
  1324. */
  1325. int transport_handle_cdb_direct(
  1326. struct se_cmd *cmd)
  1327. {
  1328. sense_reason_t ret;
  1329. if (!cmd->se_lun) {
  1330. dump_stack();
  1331. pr_err("cmd->se_lun is NULL\n");
  1332. return -EINVAL;
  1333. }
  1334. if (in_interrupt()) {
  1335. dump_stack();
  1336. pr_err("transport_generic_handle_cdb cannot be called"
  1337. " from interrupt context\n");
  1338. return -EINVAL;
  1339. }
  1340. /*
  1341. * Set TRANSPORT_NEW_CMD state and CMD_T_ACTIVE to ensure that
  1342. * outstanding descriptors are handled correctly during shutdown via
  1343. * transport_wait_for_tasks()
  1344. *
  1345. * Also, we don't take cmd->t_state_lock here as we only expect
  1346. * this to be called for initial descriptor submission.
  1347. */
  1348. cmd->t_state = TRANSPORT_NEW_CMD;
  1349. cmd->transport_state |= CMD_T_ACTIVE;
  1350. /*
  1351. * transport_generic_new_cmd() is already handling QUEUE_FULL,
  1352. * so follow TRANSPORT_NEW_CMD processing thread context usage
  1353. * and call transport_generic_request_failure() if necessary..
  1354. */
  1355. ret = transport_generic_new_cmd(cmd);
  1356. if (ret)
  1357. transport_generic_request_failure(cmd, ret);
  1358. return 0;
  1359. }
  1360. EXPORT_SYMBOL(transport_handle_cdb_direct);
  1361. sense_reason_t
  1362. transport_generic_map_mem_to_cmd(struct se_cmd *cmd, struct scatterlist *sgl,
  1363. u32 sgl_count, struct scatterlist *sgl_bidi, u32 sgl_bidi_count)
  1364. {
  1365. if (!sgl || !sgl_count)
  1366. return 0;
  1367. /*
  1368. * Reject SCSI data overflow with map_mem_to_cmd() as incoming
  1369. * scatterlists already have been set to follow what the fabric
  1370. * passes for the original expected data transfer length.
  1371. */
  1372. if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) {
  1373. pr_warn("Rejecting SCSI DATA overflow for fabric using"
  1374. " SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC\n");
  1375. return TCM_INVALID_CDB_FIELD;
  1376. }
  1377. cmd->t_data_sg = sgl;
  1378. cmd->t_data_nents = sgl_count;
  1379. cmd->t_bidi_data_sg = sgl_bidi;
  1380. cmd->t_bidi_data_nents = sgl_bidi_count;
  1381. cmd->se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
  1382. return 0;
  1383. }
  1384. /**
  1385. * target_submit_cmd_map_sgls - lookup unpacked lun and submit uninitialized
  1386. * se_cmd + use pre-allocated SGL memory.
  1387. *
  1388. * @se_cmd: command descriptor to submit
  1389. * @se_sess: associated se_sess for endpoint
  1390. * @cdb: pointer to SCSI CDB
  1391. * @sense: pointer to SCSI sense buffer
  1392. * @unpacked_lun: unpacked LUN to reference for struct se_lun
  1393. * @data_length: fabric expected data transfer length
  1394. * @task_attr: SAM task attribute
  1395. * @data_dir: DMA data direction
  1396. * @flags: flags for command submission from target_sc_flags_tables
  1397. * @sgl: struct scatterlist memory for unidirectional mapping
  1398. * @sgl_count: scatterlist count for unidirectional mapping
  1399. * @sgl_bidi: struct scatterlist memory for bidirectional READ mapping
  1400. * @sgl_bidi_count: scatterlist count for bidirectional READ mapping
  1401. * @sgl_prot: struct scatterlist memory protection information
  1402. * @sgl_prot_count: scatterlist count for protection information
  1403. *
  1404. * Task tags are supported if the caller has set @se_cmd->tag.
  1405. *
  1406. * Returns non zero to signal active I/O shutdown failure. All other
  1407. * setup exceptions will be returned as a SCSI CHECK_CONDITION response,
  1408. * but still return zero here.
  1409. *
  1410. * This may only be called from process context, and also currently
  1411. * assumes internal allocation of fabric payload buffer by target-core.
  1412. */
  1413. int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess,
  1414. unsigned char *cdb, unsigned char *sense, u64 unpacked_lun,
  1415. u32 data_length, int task_attr, int data_dir, int flags,
  1416. struct scatterlist *sgl, u32 sgl_count,
  1417. struct scatterlist *sgl_bidi, u32 sgl_bidi_count,
  1418. struct scatterlist *sgl_prot, u32 sgl_prot_count)
  1419. {
  1420. struct se_portal_group *se_tpg;
  1421. sense_reason_t rc;
  1422. int ret;
  1423. se_tpg = se_sess->se_tpg;
  1424. BUG_ON(!se_tpg);
  1425. BUG_ON(se_cmd->se_tfo || se_cmd->se_sess);
  1426. BUG_ON(in_interrupt());
  1427. /*
  1428. * Initialize se_cmd for target operation. From this point
  1429. * exceptions are handled by sending exception status via
  1430. * target_core_fabric_ops->queue_status() callback
  1431. */
  1432. transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
  1433. data_length, data_dir, task_attr, sense,
  1434. unpacked_lun);
  1435. if (flags & TARGET_SCF_USE_CPUID)
  1436. se_cmd->se_cmd_flags |= SCF_USE_CPUID;
  1437. else
  1438. se_cmd->cpuid = WORK_CPU_UNBOUND;
  1439. if (flags & TARGET_SCF_UNKNOWN_SIZE)
  1440. se_cmd->unknown_data_length = 1;
  1441. /*
  1442. * Obtain struct se_cmd->cmd_kref reference and add new cmd to
  1443. * se_sess->sess_cmd_list. A second kref_get here is necessary
  1444. * for fabrics using TARGET_SCF_ACK_KREF that expect a second
  1445. * kref_put() to happen during fabric packet acknowledgement.
  1446. */
  1447. ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF);
  1448. if (ret)
  1449. return ret;
  1450. /*
  1451. * Signal bidirectional data payloads to target-core
  1452. */
  1453. if (flags & TARGET_SCF_BIDI_OP)
  1454. se_cmd->se_cmd_flags |= SCF_BIDI;
  1455. rc = target_cmd_init_cdb(se_cmd, cdb);
  1456. if (rc) {
  1457. transport_send_check_condition_and_sense(se_cmd, rc, 0);
  1458. target_put_sess_cmd(se_cmd);
  1459. return 0;
  1460. }
  1461. /*
  1462. * Locate se_lun pointer and attach it to struct se_cmd
  1463. */
  1464. rc = transport_lookup_cmd_lun(se_cmd);
  1465. if (rc) {
  1466. transport_send_check_condition_and_sense(se_cmd, rc, 0);
  1467. target_put_sess_cmd(se_cmd);
  1468. return 0;
  1469. }
  1470. rc = target_cmd_parse_cdb(se_cmd);
  1471. if (rc != 0) {
  1472. transport_generic_request_failure(se_cmd, rc);
  1473. return 0;
  1474. }
  1475. /*
  1476. * Save pointers for SGLs containing protection information,
  1477. * if present.
  1478. */
  1479. if (sgl_prot_count) {
  1480. se_cmd->t_prot_sg = sgl_prot;
  1481. se_cmd->t_prot_nents = sgl_prot_count;
  1482. se_cmd->se_cmd_flags |= SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC;
  1483. }
  1484. /*
  1485. * When a non zero sgl_count has been passed perform SGL passthrough
  1486. * mapping for pre-allocated fabric memory instead of having target
  1487. * core perform an internal SGL allocation..
  1488. */
  1489. if (sgl_count != 0) {
  1490. BUG_ON(!sgl);
  1491. /*
  1492. * A work-around for tcm_loop as some userspace code via
  1493. * scsi-generic do not memset their associated read buffers,
  1494. * so go ahead and do that here for type non-data CDBs. Also
  1495. * note that this is currently guaranteed to be a single SGL
  1496. * for this case by target core in target_setup_cmd_from_cdb()
  1497. * -> transport_generic_cmd_sequencer().
  1498. */
  1499. if (!(se_cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) &&
  1500. se_cmd->data_direction == DMA_FROM_DEVICE) {
  1501. unsigned char *buf = NULL;
  1502. if (sgl)
  1503. buf = kmap(sg_page(sgl)) + sgl->offset;
  1504. if (buf) {
  1505. memset(buf, 0, sgl->length);
  1506. kunmap(sg_page(sgl));
  1507. }
  1508. }
  1509. rc = transport_generic_map_mem_to_cmd(se_cmd, sgl, sgl_count,
  1510. sgl_bidi, sgl_bidi_count);
  1511. if (rc != 0) {
  1512. transport_generic_request_failure(se_cmd, rc);
  1513. return 0;
  1514. }
  1515. }
  1516. /*
  1517. * Check if we need to delay processing because of ALUA
  1518. * Active/NonOptimized primary access state..
  1519. */
  1520. core_alua_check_nonop_delay(se_cmd);
  1521. transport_handle_cdb_direct(se_cmd);
  1522. return 0;
  1523. }
  1524. EXPORT_SYMBOL(target_submit_cmd_map_sgls);
  1525. /**
  1526. * target_submit_cmd - lookup unpacked lun and submit uninitialized se_cmd
  1527. *
  1528. * @se_cmd: command descriptor to submit
  1529. * @se_sess: associated se_sess for endpoint
  1530. * @cdb: pointer to SCSI CDB
  1531. * @sense: pointer to SCSI sense buffer
  1532. * @unpacked_lun: unpacked LUN to reference for struct se_lun
  1533. * @data_length: fabric expected data transfer length
  1534. * @task_attr: SAM task attribute
  1535. * @data_dir: DMA data direction
  1536. * @flags: flags for command submission from target_sc_flags_tables
  1537. *
  1538. * Task tags are supported if the caller has set @se_cmd->tag.
  1539. *
  1540. * Returns non zero to signal active I/O shutdown failure. All other
  1541. * setup exceptions will be returned as a SCSI CHECK_CONDITION response,
  1542. * but still return zero here.
  1543. *
  1544. * This may only be called from process context, and also currently
  1545. * assumes internal allocation of fabric payload buffer by target-core.
  1546. *
  1547. * It also assumes interal target core SGL memory allocation.
  1548. */
  1549. int target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess,
  1550. unsigned char *cdb, unsigned char *sense, u64 unpacked_lun,
  1551. u32 data_length, int task_attr, int data_dir, int flags)
  1552. {
  1553. return target_submit_cmd_map_sgls(se_cmd, se_sess, cdb, sense,
  1554. unpacked_lun, data_length, task_attr, data_dir,
  1555. flags, NULL, 0, NULL, 0, NULL, 0);
  1556. }
  1557. EXPORT_SYMBOL(target_submit_cmd);
  1558. static void target_complete_tmr_failure(struct work_struct *work)
  1559. {
  1560. struct se_cmd *se_cmd = container_of(work, struct se_cmd, work);
  1561. se_cmd->se_tmr_req->response = TMR_LUN_DOES_NOT_EXIST;
  1562. se_cmd->se_tfo->queue_tm_rsp(se_cmd);
  1563. transport_lun_remove_cmd(se_cmd);
  1564. transport_cmd_check_stop_to_fabric(se_cmd);
  1565. }
  1566. static bool target_lookup_lun_from_tag(struct se_session *se_sess, u64 tag,
  1567. u64 *unpacked_lun)
  1568. {
  1569. struct se_cmd *se_cmd;
  1570. unsigned long flags;
  1571. bool ret = false;
  1572. spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
  1573. list_for_each_entry(se_cmd, &se_sess->sess_cmd_list, se_cmd_list) {
  1574. if (se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)
  1575. continue;
  1576. if (se_cmd->tag == tag) {
  1577. *unpacked_lun = se_cmd->orig_fe_lun;
  1578. ret = true;
  1579. break;
  1580. }
  1581. }
  1582. spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
  1583. return ret;
  1584. }
  1585. /**
  1586. * target_submit_tmr - lookup unpacked lun and submit uninitialized se_cmd
  1587. * for TMR CDBs
  1588. *
  1589. * @se_cmd: command descriptor to submit
  1590. * @se_sess: associated se_sess for endpoint
  1591. * @sense: pointer to SCSI sense buffer
  1592. * @unpacked_lun: unpacked LUN to reference for struct se_lun
  1593. * @fabric_tmr_ptr: fabric context for TMR req
  1594. * @tm_type: Type of TM request
  1595. * @gfp: gfp type for caller
  1596. * @tag: referenced task tag for TMR_ABORT_TASK
  1597. * @flags: submit cmd flags
  1598. *
  1599. * Callable from all contexts.
  1600. **/
  1601. int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess,
  1602. unsigned char *sense, u64 unpacked_lun,
  1603. void *fabric_tmr_ptr, unsigned char tm_type,
  1604. gfp_t gfp, u64 tag, int flags)
  1605. {
  1606. struct se_portal_group *se_tpg;
  1607. int ret;
  1608. se_tpg = se_sess->se_tpg;
  1609. BUG_ON(!se_tpg);
  1610. transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
  1611. 0, DMA_NONE, TCM_SIMPLE_TAG, sense, unpacked_lun);
  1612. /*
  1613. * FIXME: Currently expect caller to handle se_cmd->se_tmr_req
  1614. * allocation failure.
  1615. */
  1616. ret = core_tmr_alloc_req(se_cmd, fabric_tmr_ptr, tm_type, gfp);
  1617. if (ret < 0)
  1618. return -ENOMEM;
  1619. if (tm_type == TMR_ABORT_TASK)
  1620. se_cmd->se_tmr_req->ref_task_tag = tag;
  1621. /* See target_submit_cmd for commentary */
  1622. ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF);
  1623. if (ret) {
  1624. core_tmr_release_req(se_cmd->se_tmr_req);
  1625. return ret;
  1626. }
  1627. /*
  1628. * If this is ABORT_TASK with no explicit fabric provided LUN,
  1629. * go ahead and search active session tags for a match to figure
  1630. * out unpacked_lun for the original se_cmd.
  1631. */
  1632. if (tm_type == TMR_ABORT_TASK && (flags & TARGET_SCF_LOOKUP_LUN_FROM_TAG)) {
  1633. if (!target_lookup_lun_from_tag(se_sess, tag,
  1634. &se_cmd->orig_fe_lun))
  1635. goto failure;
  1636. }
  1637. ret = transport_lookup_tmr_lun(se_cmd);
  1638. if (ret)
  1639. goto failure;
  1640. transport_generic_handle_tmr(se_cmd);
  1641. return 0;
  1642. /*
  1643. * For callback during failure handling, push this work off
  1644. * to process context with TMR_LUN_DOES_NOT_EXIST status.
  1645. */
  1646. failure:
  1647. INIT_WORK(&se_cmd->work, target_complete_tmr_failure);
  1648. schedule_work(&se_cmd->work);
  1649. return 0;
  1650. }
  1651. EXPORT_SYMBOL(target_submit_tmr);
  1652. /*
  1653. * Handle SAM-esque emulation for generic transport request failures.
  1654. */
  1655. void transport_generic_request_failure(struct se_cmd *cmd,
  1656. sense_reason_t sense_reason)
  1657. {
  1658. int ret = 0, post_ret;
  1659. pr_debug("-----[ Storage Engine Exception; sense_reason %d\n",
  1660. sense_reason);
  1661. target_show_cmd("-----[ ", cmd);
  1662. /*
  1663. * For SAM Task Attribute emulation for failed struct se_cmd
  1664. */
  1665. transport_complete_task_attr(cmd);
  1666. if (cmd->transport_complete_callback)
  1667. cmd->transport_complete_callback(cmd, false, &post_ret);
  1668. if (cmd->transport_state & CMD_T_ABORTED) {
  1669. INIT_WORK(&cmd->work, target_abort_work);
  1670. queue_work(target_completion_wq, &cmd->work);
  1671. return;
  1672. }
  1673. switch (sense_reason) {
  1674. case TCM_NON_EXISTENT_LUN:
  1675. case TCM_UNSUPPORTED_SCSI_OPCODE:
  1676. case TCM_INVALID_CDB_FIELD:
  1677. case TCM_INVALID_PARAMETER_LIST:
  1678. case TCM_PARAMETER_LIST_LENGTH_ERROR:
  1679. case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE:
  1680. case TCM_UNKNOWN_MODE_PAGE:
  1681. case TCM_WRITE_PROTECTED:
  1682. case TCM_ADDRESS_OUT_OF_RANGE:
  1683. case TCM_CHECK_CONDITION_ABORT_CMD:
  1684. case TCM_CHECK_CONDITION_UNIT_ATTENTION:
  1685. case TCM_CHECK_CONDITION_NOT_READY:
  1686. case TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED:
  1687. case TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED:
  1688. case TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED:
  1689. case TCM_COPY_TARGET_DEVICE_NOT_REACHABLE:
  1690. case TCM_TOO_MANY_TARGET_DESCS:
  1691. case TCM_UNSUPPORTED_TARGET_DESC_TYPE_CODE:
  1692. case TCM_TOO_MANY_SEGMENT_DESCS:
  1693. case TCM_UNSUPPORTED_SEGMENT_DESC_TYPE_CODE:
  1694. break;
  1695. case TCM_OUT_OF_RESOURCES:
  1696. cmd->scsi_status = SAM_STAT_TASK_SET_FULL;
  1697. goto queue_status;
  1698. case TCM_LUN_BUSY:
  1699. cmd->scsi_status = SAM_STAT_BUSY;
  1700. goto queue_status;
  1701. case TCM_RESERVATION_CONFLICT:
  1702. /*
  1703. * No SENSE Data payload for this case, set SCSI Status
  1704. * and queue the response to $FABRIC_MOD.
  1705. *
  1706. * Uses linux/include/scsi/scsi.h SAM status codes defs
  1707. */
  1708. cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
  1709. /*
  1710. * For UA Interlock Code 11b, a RESERVATION CONFLICT will
  1711. * establish a UNIT ATTENTION with PREVIOUS RESERVATION
  1712. * CONFLICT STATUS.
  1713. *
  1714. * See spc4r17, section 7.4.6 Control Mode Page, Table 349
  1715. */
  1716. if (cmd->se_sess &&
  1717. cmd->se_dev->dev_attrib.emulate_ua_intlck_ctrl
  1718. == TARGET_UA_INTLCK_CTRL_ESTABLISH_UA) {
  1719. target_ua_allocate_lun(cmd->se_sess->se_node_acl,
  1720. cmd->orig_fe_lun, 0x2C,
  1721. ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS);
  1722. }
  1723. goto queue_status;
  1724. default:
  1725. pr_err("Unknown transport error for CDB 0x%02x: %d\n",
  1726. cmd->t_task_cdb[0], sense_reason);
  1727. sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE;
  1728. break;
  1729. }
  1730. ret = transport_send_check_condition_and_sense(cmd, sense_reason, 0);
  1731. if (ret)
  1732. goto queue_full;
  1733. check_stop:
  1734. transport_lun_remove_cmd(cmd);
  1735. transport_cmd_check_stop_to_fabric(cmd);
  1736. return;
  1737. queue_status:
  1738. trace_target_cmd_complete(cmd);
  1739. ret = cmd->se_tfo->queue_status(cmd);
  1740. if (!ret)
  1741. goto check_stop;
  1742. queue_full:
  1743. transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
  1744. }
  1745. EXPORT_SYMBOL(transport_generic_request_failure);
  1746. void __target_execute_cmd(struct se_cmd *cmd, bool do_checks)
  1747. {
  1748. sense_reason_t ret;
  1749. if (!cmd->execute_cmd) {
  1750. ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
  1751. goto err;
  1752. }
  1753. if (do_checks) {
  1754. /*
  1755. * Check for an existing UNIT ATTENTION condition after
  1756. * target_handle_task_attr() has done SAM task attr
  1757. * checking, and possibly have already defered execution
  1758. * out to target_restart_delayed_cmds() context.
  1759. */
  1760. ret = target_scsi3_ua_check(cmd);
  1761. if (ret)
  1762. goto err;
  1763. ret = target_alua_state_check(cmd);
  1764. if (ret)
  1765. goto err;
  1766. ret = target_check_reservation(cmd);
  1767. if (ret) {
  1768. cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
  1769. goto err;
  1770. }
  1771. }
  1772. ret = cmd->execute_cmd(cmd);
  1773. if (!ret)
  1774. return;
  1775. err:
  1776. spin_lock_irq(&cmd->t_state_lock);
  1777. cmd->transport_state &= ~CMD_T_SENT;
  1778. spin_unlock_irq(&cmd->t_state_lock);
  1779. transport_generic_request_failure(cmd, ret);
  1780. }
  1781. static int target_write_prot_action(struct se_cmd *cmd)
  1782. {
  1783. u32 sectors;
  1784. /*
  1785. * Perform WRITE_INSERT of PI using software emulation when backend
  1786. * device has PI enabled, if the transport has not already generated
  1787. * PI using hardware WRITE_INSERT offload.
  1788. */
  1789. switch (cmd->prot_op) {
  1790. case TARGET_PROT_DOUT_INSERT:
  1791. if (!(cmd->se_sess->sup_prot_ops & TARGET_PROT_DOUT_INSERT))
  1792. sbc_dif_generate(cmd);
  1793. break;
  1794. case TARGET_PROT_DOUT_STRIP:
  1795. if (cmd->se_sess->sup_prot_ops & TARGET_PROT_DOUT_STRIP)
  1796. break;
  1797. sectors = cmd->data_length >> ilog2(cmd->se_dev->dev_attrib.block_size);
  1798. cmd->pi_err = sbc_dif_verify(cmd, cmd->t_task_lba,
  1799. sectors, 0, cmd->t_prot_sg, 0);
  1800. if (unlikely(cmd->pi_err)) {
  1801. spin_lock_irq(&cmd->t_state_lock);
  1802. cmd->transport_state &= ~CMD_T_SENT;
  1803. spin_unlock_irq(&cmd->t_state_lock);
  1804. transport_generic_request_failure(cmd, cmd->pi_err);
  1805. return -1;
  1806. }
  1807. break;
  1808. default:
  1809. break;
  1810. }
  1811. return 0;
  1812. }
  1813. static bool target_handle_task_attr(struct se_cmd *cmd)
  1814. {
  1815. struct se_device *dev = cmd->se_dev;
  1816. if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
  1817. return false;
  1818. cmd->se_cmd_flags |= SCF_TASK_ATTR_SET;
  1819. /*
  1820. * Check for the existence of HEAD_OF_QUEUE, and if true return 1
  1821. * to allow the passed struct se_cmd list of tasks to the front of the list.
  1822. */
  1823. switch (cmd->sam_task_attr) {
  1824. case TCM_HEAD_TAG:
  1825. atomic_inc_mb(&dev->non_ordered);
  1826. pr_debug("Added HEAD_OF_QUEUE for CDB: 0x%02x\n",
  1827. cmd->t_task_cdb[0]);
  1828. return false;
  1829. case TCM_ORDERED_TAG:
  1830. atomic_inc_mb(&dev->delayed_cmd_count);
  1831. pr_debug("Added ORDERED for CDB: 0x%02x to ordered list\n",
  1832. cmd->t_task_cdb[0]);
  1833. break;
  1834. default:
  1835. /*
  1836. * For SIMPLE and UNTAGGED Task Attribute commands
  1837. */
  1838. atomic_inc_mb(&dev->non_ordered);
  1839. if (atomic_read(&dev->delayed_cmd_count) == 0)
  1840. return false;
  1841. break;
  1842. }
  1843. if (cmd->sam_task_attr != TCM_ORDERED_TAG) {
  1844. atomic_inc_mb(&dev->delayed_cmd_count);
  1845. /*
  1846. * We will account for this when we dequeue from the delayed
  1847. * list.
  1848. */
  1849. atomic_dec_mb(&dev->non_ordered);
  1850. }
  1851. spin_lock(&dev->delayed_cmd_lock);
  1852. list_add_tail(&cmd->se_delayed_node, &dev->delayed_cmd_list);
  1853. spin_unlock(&dev->delayed_cmd_lock);
  1854. pr_debug("Added CDB: 0x%02x Task Attr: 0x%02x to delayed CMD listn",
  1855. cmd->t_task_cdb[0], cmd->sam_task_attr);
  1856. /*
  1857. * We may have no non ordered cmds when this function started or we
  1858. * could have raced with the last simple/head cmd completing, so kick
  1859. * the delayed handler here.
  1860. */
  1861. schedule_work(&dev->delayed_cmd_work);
  1862. return true;
  1863. }
  1864. void target_execute_cmd(struct se_cmd *cmd)
  1865. {
  1866. /*
  1867. * Determine if frontend context caller is requesting the stopping of
  1868. * this command for frontend exceptions.
  1869. *
  1870. * If the received CDB has already been aborted stop processing it here.
  1871. */
  1872. if (target_cmd_interrupted(cmd))
  1873. return;
  1874. spin_lock_irq(&cmd->t_state_lock);
  1875. cmd->t_state = TRANSPORT_PROCESSING;
  1876. cmd->transport_state |= CMD_T_ACTIVE | CMD_T_SENT;
  1877. spin_unlock_irq(&cmd->t_state_lock);
  1878. if (target_write_prot_action(cmd))
  1879. return;
  1880. if (target_handle_task_attr(cmd)) {
  1881. spin_lock_irq(&cmd->t_state_lock);
  1882. cmd->transport_state &= ~CMD_T_SENT;
  1883. spin_unlock_irq(&cmd->t_state_lock);
  1884. return;
  1885. }
  1886. __target_execute_cmd(cmd, true);
  1887. }
  1888. EXPORT_SYMBOL(target_execute_cmd);
  1889. /*
  1890. * Process all commands up to the last received ORDERED task attribute which
  1891. * requires another blocking boundary
  1892. */
  1893. void target_do_delayed_work(struct work_struct *work)
  1894. {
  1895. struct se_device *dev = container_of(work, struct se_device,
  1896. delayed_cmd_work);
  1897. spin_lock(&dev->delayed_cmd_lock);
  1898. while (!dev->ordered_sync_in_progress) {
  1899. struct se_cmd *cmd;
  1900. if (list_empty(&dev->delayed_cmd_list))
  1901. break;
  1902. cmd = list_entry(dev->delayed_cmd_list.next,
  1903. struct se_cmd, se_delayed_node);
  1904. if (cmd->sam_task_attr == TCM_ORDERED_TAG) {
  1905. /*
  1906. * Check if we started with:
  1907. * [ordered] [simple] [ordered]
  1908. * and we are now at the last ordered so we have to wait
  1909. * for the simple cmd.
  1910. */
  1911. if (atomic_read(&dev->non_ordered) > 0)
  1912. break;
  1913. dev->ordered_sync_in_progress = true;
  1914. }
  1915. list_del(&cmd->se_delayed_node);
  1916. atomic_dec_mb(&dev->delayed_cmd_count);
  1917. spin_unlock(&dev->delayed_cmd_lock);
  1918. if (cmd->sam_task_attr != TCM_ORDERED_TAG)
  1919. atomic_inc_mb(&dev->non_ordered);
  1920. cmd->transport_state |= CMD_T_SENT;
  1921. __target_execute_cmd(cmd, true);
  1922. spin_lock(&dev->delayed_cmd_lock);
  1923. }
  1924. spin_unlock(&dev->delayed_cmd_lock);
  1925. }
  1926. /*
  1927. * Called from I/O completion to determine which dormant/delayed
  1928. * and ordered cmds need to have their tasks added to the execution queue.
  1929. */
  1930. static void transport_complete_task_attr(struct se_cmd *cmd)
  1931. {
  1932. struct se_device *dev = cmd->se_dev;
  1933. if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
  1934. return;
  1935. if (!(cmd->se_cmd_flags & SCF_TASK_ATTR_SET))
  1936. goto restart;
  1937. if (cmd->sam_task_attr == TCM_SIMPLE_TAG) {
  1938. atomic_dec_mb(&dev->non_ordered);
  1939. dev->dev_cur_ordered_id++;
  1940. } else if (cmd->sam_task_attr == TCM_HEAD_TAG) {
  1941. atomic_dec_mb(&dev->non_ordered);
  1942. dev->dev_cur_ordered_id++;
  1943. pr_debug("Incremented dev_cur_ordered_id: %u for HEAD_OF_QUEUE\n",
  1944. dev->dev_cur_ordered_id);
  1945. } else if (cmd->sam_task_attr == TCM_ORDERED_TAG) {
  1946. spin_lock(&dev->delayed_cmd_lock);
  1947. dev->ordered_sync_in_progress = false;
  1948. spin_unlock(&dev->delayed_cmd_lock);
  1949. dev->dev_cur_ordered_id++;
  1950. pr_debug("Incremented dev_cur_ordered_id: %u for ORDERED\n",
  1951. dev->dev_cur_ordered_id);
  1952. }
  1953. cmd->se_cmd_flags &= ~SCF_TASK_ATTR_SET;
  1954. restart:
  1955. if (atomic_read(&dev->delayed_cmd_count) > 0)
  1956. schedule_work(&dev->delayed_cmd_work);
  1957. }
  1958. static void transport_complete_qf(struct se_cmd *cmd)
  1959. {
  1960. int ret = 0;
  1961. transport_complete_task_attr(cmd);
  1962. /*
  1963. * If a fabric driver ->write_pending() or ->queue_data_in() callback
  1964. * has returned neither -ENOMEM or -EAGAIN, assume it's fatal and
  1965. * the same callbacks should not be retried. Return CHECK_CONDITION
  1966. * if a scsi_status is not already set.
  1967. *
  1968. * If a fabric driver ->queue_status() has returned non zero, always
  1969. * keep retrying no matter what..
  1970. */
  1971. if (cmd->t_state == TRANSPORT_COMPLETE_QF_ERR) {
  1972. if (cmd->scsi_status)
  1973. goto queue_status;
  1974. translate_sense_reason(cmd, TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE);
  1975. goto queue_status;
  1976. }
  1977. /*
  1978. * Check if we need to send a sense buffer from
  1979. * the struct se_cmd in question. We do NOT want
  1980. * to take this path of the IO has been marked as
  1981. * needing to be treated like a "normal read". This
  1982. * is the case if it's a tape read, and either the
  1983. * FM, EOM, or ILI bits are set, but there is no
  1984. * sense data.
  1985. */
  1986. if (!(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL) &&
  1987. cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE)
  1988. goto queue_status;
  1989. switch (cmd->data_direction) {
  1990. case DMA_FROM_DEVICE:
  1991. /* queue status if not treating this as a normal read */
  1992. if (cmd->scsi_status &&
  1993. !(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL))
  1994. goto queue_status;
  1995. trace_target_cmd_complete(cmd);
  1996. ret = cmd->se_tfo->queue_data_in(cmd);
  1997. break;
  1998. case DMA_TO_DEVICE:
  1999. if (cmd->se_cmd_flags & SCF_BIDI) {
  2000. ret = cmd->se_tfo->queue_data_in(cmd);
  2001. break;
  2002. }
  2003. fallthrough;
  2004. case DMA_NONE:
  2005. queue_status:
  2006. trace_target_cmd_complete(cmd);
  2007. ret = cmd->se_tfo->queue_status(cmd);
  2008. break;
  2009. default:
  2010. break;
  2011. }
  2012. if (ret < 0) {
  2013. transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
  2014. return;
  2015. }
  2016. transport_lun_remove_cmd(cmd);
  2017. transport_cmd_check_stop_to_fabric(cmd);
  2018. }
  2019. static void transport_handle_queue_full(struct se_cmd *cmd, struct se_device *dev,
  2020. int err, bool write_pending)
  2021. {
  2022. /*
  2023. * -EAGAIN or -ENOMEM signals retry of ->write_pending() and/or
  2024. * ->queue_data_in() callbacks from new process context.
  2025. *
  2026. * Otherwise for other errors, transport_complete_qf() will send
  2027. * CHECK_CONDITION via ->queue_status() instead of attempting to
  2028. * retry associated fabric driver data-transfer callbacks.
  2029. */
  2030. if (err == -EAGAIN || err == -ENOMEM) {
  2031. cmd->t_state = (write_pending) ? TRANSPORT_COMPLETE_QF_WP :
  2032. TRANSPORT_COMPLETE_QF_OK;
  2033. } else {
  2034. pr_warn_ratelimited("Got unknown fabric queue status: %d\n", err);
  2035. cmd->t_state = TRANSPORT_COMPLETE_QF_ERR;
  2036. }
  2037. spin_lock_irq(&dev->qf_cmd_lock);
  2038. list_add_tail(&cmd->se_qf_node, &cmd->se_dev->qf_cmd_list);
  2039. atomic_inc_mb(&dev->dev_qf_count);
  2040. spin_unlock_irq(&cmd->se_dev->qf_cmd_lock);
  2041. schedule_work(&cmd->se_dev->qf_work_queue);
  2042. }
  2043. static bool target_read_prot_action(struct se_cmd *cmd)
  2044. {
  2045. switch (cmd->prot_op) {
  2046. case TARGET_PROT_DIN_STRIP:
  2047. if (!(cmd->se_sess->sup_prot_ops & TARGET_PROT_DIN_STRIP)) {
  2048. u32 sectors = cmd->data_length >>
  2049. ilog2(cmd->se_dev->dev_attrib.block_size);
  2050. cmd->pi_err = sbc_dif_verify(cmd, cmd->t_task_lba,
  2051. sectors, 0, cmd->t_prot_sg,
  2052. 0);
  2053. if (cmd->pi_err)
  2054. return true;
  2055. }
  2056. break;
  2057. case TARGET_PROT_DIN_INSERT:
  2058. if (cmd->se_sess->sup_prot_ops & TARGET_PROT_DIN_INSERT)
  2059. break;
  2060. sbc_dif_generate(cmd);
  2061. break;
  2062. default:
  2063. break;
  2064. }
  2065. return false;
  2066. }
  2067. static void target_complete_ok_work(struct work_struct *work)
  2068. {
  2069. struct se_cmd *cmd = container_of(work, struct se_cmd, work);
  2070. int ret;
  2071. /*
  2072. * Check if we need to move delayed/dormant tasks from cmds on the
  2073. * delayed execution list after a HEAD_OF_QUEUE or ORDERED Task
  2074. * Attribute.
  2075. */
  2076. transport_complete_task_attr(cmd);
  2077. /*
  2078. * Check to schedule QUEUE_FULL work, or execute an existing
  2079. * cmd->transport_qf_callback()
  2080. */
  2081. if (atomic_read(&cmd->se_dev->dev_qf_count) != 0)
  2082. schedule_work(&cmd->se_dev->qf_work_queue);
  2083. /*
  2084. * Check if we need to send a sense buffer from
  2085. * the struct se_cmd in question. We do NOT want
  2086. * to take this path of the IO has been marked as
  2087. * needing to be treated like a "normal read". This
  2088. * is the case if it's a tape read, and either the
  2089. * FM, EOM, or ILI bits are set, but there is no
  2090. * sense data.
  2091. */
  2092. if (!(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL) &&
  2093. cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) {
  2094. WARN_ON(!cmd->scsi_status);
  2095. ret = transport_send_check_condition_and_sense(
  2096. cmd, 0, 1);
  2097. if (ret)
  2098. goto queue_full;
  2099. transport_lun_remove_cmd(cmd);
  2100. transport_cmd_check_stop_to_fabric(cmd);
  2101. return;
  2102. }
  2103. /*
  2104. * Check for a callback, used by amongst other things
  2105. * XDWRITE_READ_10 and COMPARE_AND_WRITE emulation.
  2106. */
  2107. if (cmd->transport_complete_callback) {
  2108. sense_reason_t rc;
  2109. bool caw = (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE);
  2110. bool zero_dl = !(cmd->data_length);
  2111. int post_ret = 0;
  2112. rc = cmd->transport_complete_callback(cmd, true, &post_ret);
  2113. if (!rc && !post_ret) {
  2114. if (caw && zero_dl)
  2115. goto queue_rsp;
  2116. return;
  2117. } else if (rc) {
  2118. ret = transport_send_check_condition_and_sense(cmd,
  2119. rc, 0);
  2120. if (ret)
  2121. goto queue_full;
  2122. transport_lun_remove_cmd(cmd);
  2123. transport_cmd_check_stop_to_fabric(cmd);
  2124. return;
  2125. }
  2126. }
  2127. queue_rsp:
  2128. switch (cmd->data_direction) {
  2129. case DMA_FROM_DEVICE:
  2130. /*
  2131. * if this is a READ-type IO, but SCSI status
  2132. * is set, then skip returning data and just
  2133. * return the status -- unless this IO is marked
  2134. * as needing to be treated as a normal read,
  2135. * in which case we want to go ahead and return
  2136. * the data. This happens, for example, for tape
  2137. * reads with the FM, EOM, or ILI bits set, with
  2138. * no sense data.
  2139. */
  2140. if (cmd->scsi_status &&
  2141. !(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL))
  2142. goto queue_status;
  2143. atomic_long_add(cmd->data_length,
  2144. &cmd->se_lun->lun_stats.tx_data_octets);
  2145. /*
  2146. * Perform READ_STRIP of PI using software emulation when
  2147. * backend had PI enabled, if the transport will not be
  2148. * performing hardware READ_STRIP offload.
  2149. */
  2150. if (target_read_prot_action(cmd)) {
  2151. ret = transport_send_check_condition_and_sense(cmd,
  2152. cmd->pi_err, 0);
  2153. if (ret)
  2154. goto queue_full;
  2155. transport_lun_remove_cmd(cmd);
  2156. transport_cmd_check_stop_to_fabric(cmd);
  2157. return;
  2158. }
  2159. trace_target_cmd_complete(cmd);
  2160. ret = cmd->se_tfo->queue_data_in(cmd);
  2161. if (ret)
  2162. goto queue_full;
  2163. break;
  2164. case DMA_TO_DEVICE:
  2165. atomic_long_add(cmd->data_length,
  2166. &cmd->se_lun->lun_stats.rx_data_octets);
  2167. /*
  2168. * Check if we need to send READ payload for BIDI-COMMAND
  2169. */
  2170. if (cmd->se_cmd_flags & SCF_BIDI) {
  2171. atomic_long_add(cmd->data_length,
  2172. &cmd->se_lun->lun_stats.tx_data_octets);
  2173. ret = cmd->se_tfo->queue_data_in(cmd);
  2174. if (ret)
  2175. goto queue_full;
  2176. break;
  2177. }
  2178. fallthrough;
  2179. case DMA_NONE:
  2180. queue_status:
  2181. trace_target_cmd_complete(cmd);
  2182. ret = cmd->se_tfo->queue_status(cmd);
  2183. if (ret)
  2184. goto queue_full;
  2185. break;
  2186. default:
  2187. break;
  2188. }
  2189. transport_lun_remove_cmd(cmd);
  2190. transport_cmd_check_stop_to_fabric(cmd);
  2191. return;
  2192. queue_full:
  2193. pr_debug("Handling complete_ok QUEUE_FULL: se_cmd: %p,"
  2194. " data_direction: %d\n", cmd, cmd->data_direction);
  2195. transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
  2196. }
  2197. void target_free_sgl(struct scatterlist *sgl, int nents)
  2198. {
  2199. sgl_free_n_order(sgl, nents, 0);
  2200. }
  2201. EXPORT_SYMBOL(target_free_sgl);
  2202. static inline void transport_reset_sgl_orig(struct se_cmd *cmd)
  2203. {
  2204. /*
  2205. * Check for saved t_data_sg that may be used for COMPARE_AND_WRITE
  2206. * emulation, and free + reset pointers if necessary..
  2207. */
  2208. if (!cmd->t_data_sg_orig)
  2209. return;
  2210. kfree(cmd->t_data_sg);
  2211. cmd->t_data_sg = cmd->t_data_sg_orig;
  2212. cmd->t_data_sg_orig = NULL;
  2213. cmd->t_data_nents = cmd->t_data_nents_orig;
  2214. cmd->t_data_nents_orig = 0;
  2215. }
  2216. static inline void transport_free_pages(struct se_cmd *cmd)
  2217. {
  2218. if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC)) {
  2219. target_free_sgl(cmd->t_prot_sg, cmd->t_prot_nents);
  2220. cmd->t_prot_sg = NULL;
  2221. cmd->t_prot_nents = 0;
  2222. }
  2223. if (cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
  2224. /*
  2225. * Release special case READ buffer payload required for
  2226. * SG_TO_MEM_NOALLOC to function with COMPARE_AND_WRITE
  2227. */
  2228. if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) {
  2229. target_free_sgl(cmd->t_bidi_data_sg,
  2230. cmd->t_bidi_data_nents);
  2231. cmd->t_bidi_data_sg = NULL;
  2232. cmd->t_bidi_data_nents = 0;
  2233. }
  2234. transport_reset_sgl_orig(cmd);
  2235. return;
  2236. }
  2237. transport_reset_sgl_orig(cmd);
  2238. target_free_sgl(cmd->t_data_sg, cmd->t_data_nents);
  2239. cmd->t_data_sg = NULL;
  2240. cmd->t_data_nents = 0;
  2241. target_free_sgl(cmd->t_bidi_data_sg, cmd->t_bidi_data_nents);
  2242. cmd->t_bidi_data_sg = NULL;
  2243. cmd->t_bidi_data_nents = 0;
  2244. }
  2245. void *transport_kmap_data_sg(struct se_cmd *cmd)
  2246. {
  2247. struct scatterlist *sg = cmd->t_data_sg;
  2248. struct page **pages;
  2249. int i;
  2250. /*
  2251. * We need to take into account a possible offset here for fabrics like
  2252. * tcm_loop who may be using a contig buffer from the SCSI midlayer for
  2253. * control CDBs passed as SGLs via transport_generic_map_mem_to_cmd()
  2254. */
  2255. if (!cmd->t_data_nents)
  2256. return NULL;
  2257. BUG_ON(!sg);
  2258. if (cmd->t_data_nents == 1)
  2259. return kmap(sg_page(sg)) + sg->offset;
  2260. /* >1 page. use vmap */
  2261. pages = kmalloc_array(cmd->t_data_nents, sizeof(*pages), GFP_KERNEL);
  2262. if (!pages)
  2263. return NULL;
  2264. /* convert sg[] to pages[] */
  2265. for_each_sg(cmd->t_data_sg, sg, cmd->t_data_nents, i) {
  2266. pages[i] = sg_page(sg);
  2267. }
  2268. cmd->t_data_vmap = vmap(pages, cmd->t_data_nents, VM_MAP, PAGE_KERNEL);
  2269. kfree(pages);
  2270. if (!cmd->t_data_vmap)
  2271. return NULL;
  2272. return cmd->t_data_vmap + cmd->t_data_sg[0].offset;
  2273. }
  2274. EXPORT_SYMBOL(transport_kmap_data_sg);
  2275. void transport_kunmap_data_sg(struct se_cmd *cmd)
  2276. {
  2277. if (!cmd->t_data_nents) {
  2278. return;
  2279. } else if (cmd->t_data_nents == 1) {
  2280. kunmap(sg_page(cmd->t_data_sg));
  2281. return;
  2282. }
  2283. vunmap(cmd->t_data_vmap);
  2284. cmd->t_data_vmap = NULL;
  2285. }
  2286. EXPORT_SYMBOL(transport_kunmap_data_sg);
  2287. int
  2288. target_alloc_sgl(struct scatterlist **sgl, unsigned int *nents, u32 length,
  2289. bool zero_page, bool chainable)
  2290. {
  2291. gfp_t gfp = GFP_KERNEL | (zero_page ? __GFP_ZERO : 0);
  2292. *sgl = sgl_alloc_order(length, 0, chainable, gfp, nents);
  2293. return *sgl ? 0 : -ENOMEM;
  2294. }
  2295. EXPORT_SYMBOL(target_alloc_sgl);
  2296. /*
  2297. * Allocate any required resources to execute the command. For writes we
  2298. * might not have the payload yet, so notify the fabric via a call to
  2299. * ->write_pending instead. Otherwise place it on the execution queue.
  2300. */
  2301. sense_reason_t
  2302. transport_generic_new_cmd(struct se_cmd *cmd)
  2303. {
  2304. unsigned long flags;
  2305. int ret = 0;
  2306. bool zero_flag = !(cmd->se_cmd_flags & SCF_SCSI_DATA_CDB);
  2307. if (cmd->prot_op != TARGET_PROT_NORMAL &&
  2308. !(cmd->se_cmd_flags & SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC)) {
  2309. ret = target_alloc_sgl(&cmd->t_prot_sg, &cmd->t_prot_nents,
  2310. cmd->prot_length, true, false);
  2311. if (ret < 0)
  2312. return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
  2313. }
  2314. /*
  2315. * Determine if the TCM fabric module has already allocated physical
  2316. * memory, and is directly calling transport_generic_map_mem_to_cmd()
  2317. * beforehand.
  2318. */
  2319. if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) &&
  2320. cmd->data_length) {
  2321. if ((cmd->se_cmd_flags & SCF_BIDI) ||
  2322. (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE)) {
  2323. u32 bidi_length;
  2324. if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE)
  2325. bidi_length = cmd->t_task_nolb *
  2326. cmd->se_dev->dev_attrib.block_size;
  2327. else
  2328. bidi_length = cmd->data_length;
  2329. ret = target_alloc_sgl(&cmd->t_bidi_data_sg,
  2330. &cmd->t_bidi_data_nents,
  2331. bidi_length, zero_flag, false);
  2332. if (ret < 0)
  2333. return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
  2334. }
  2335. ret = target_alloc_sgl(&cmd->t_data_sg, &cmd->t_data_nents,
  2336. cmd->data_length, zero_flag, false);
  2337. if (ret < 0)
  2338. return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
  2339. } else if ((cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) &&
  2340. cmd->data_length) {
  2341. /*
  2342. * Special case for COMPARE_AND_WRITE with fabrics
  2343. * using SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC.
  2344. */
  2345. u32 caw_length = cmd->t_task_nolb *
  2346. cmd->se_dev->dev_attrib.block_size;
  2347. ret = target_alloc_sgl(&cmd->t_bidi_data_sg,
  2348. &cmd->t_bidi_data_nents,
  2349. caw_length, zero_flag, false);
  2350. if (ret < 0)
  2351. return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
  2352. }
  2353. /*
  2354. * If this command is not a write we can execute it right here,
  2355. * for write buffers we need to notify the fabric driver first
  2356. * and let it call back once the write buffers are ready.
  2357. */
  2358. target_add_to_state_list(cmd);
  2359. if (cmd->data_direction != DMA_TO_DEVICE || cmd->data_length == 0) {
  2360. target_execute_cmd(cmd);
  2361. return 0;
  2362. }
  2363. spin_lock_irqsave(&cmd->t_state_lock, flags);
  2364. cmd->t_state = TRANSPORT_WRITE_PENDING;
  2365. /*
  2366. * Determine if frontend context caller is requesting the stopping of
  2367. * this command for frontend exceptions.
  2368. */
  2369. if (cmd->transport_state & CMD_T_STOP &&
  2370. !cmd->se_tfo->write_pending_must_be_called) {
  2371. pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n",
  2372. __func__, __LINE__, cmd->tag);
  2373. spin_unlock_irqrestore(&cmd->t_state_lock, flags);
  2374. complete_all(&cmd->t_transport_stop_comp);
  2375. return 0;
  2376. }
  2377. cmd->transport_state &= ~CMD_T_ACTIVE;
  2378. spin_unlock_irqrestore(&cmd->t_state_lock, flags);
  2379. ret = cmd->se_tfo->write_pending(cmd);
  2380. if (ret)
  2381. goto queue_full;
  2382. return 0;
  2383. queue_full:
  2384. pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n", cmd);
  2385. transport_handle_queue_full(cmd, cmd->se_dev, ret, true);
  2386. return 0;
  2387. }
  2388. EXPORT_SYMBOL(transport_generic_new_cmd);
  2389. static void transport_write_pending_qf(struct se_cmd *cmd)
  2390. {
  2391. unsigned long flags;
  2392. int ret;
  2393. bool stop;
  2394. spin_lock_irqsave(&cmd->t_state_lock, flags);
  2395. stop = (cmd->transport_state & (CMD_T_STOP | CMD_T_ABORTED));
  2396. spin_unlock_irqrestore(&cmd->t_state_lock, flags);
  2397. if (stop) {
  2398. pr_debug("%s:%d CMD_T_STOP|CMD_T_ABORTED for ITT: 0x%08llx\n",
  2399. __func__, __LINE__, cmd->tag);
  2400. complete_all(&cmd->t_transport_stop_comp);
  2401. return;
  2402. }
  2403. ret = cmd->se_tfo->write_pending(cmd);
  2404. if (ret) {
  2405. pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n",
  2406. cmd);
  2407. transport_handle_queue_full(cmd, cmd->se_dev, ret, true);
  2408. }
  2409. }
  2410. static bool
  2411. __transport_wait_for_tasks(struct se_cmd *, bool, bool *, bool *,
  2412. unsigned long *flags);
  2413. static void target_wait_free_cmd(struct se_cmd *cmd, bool *aborted, bool *tas)
  2414. {
  2415. unsigned long flags;
  2416. spin_lock_irqsave(&cmd->t_state_lock, flags);
  2417. __transport_wait_for_tasks(cmd, true, aborted, tas, &flags);
  2418. spin_unlock_irqrestore(&cmd->t_state_lock, flags);
  2419. }
  2420. /*
  2421. * Call target_put_sess_cmd() and wait until target_release_cmd_kref(@cmd) has
  2422. * finished.
  2423. */
  2424. void target_put_cmd_and_wait(struct se_cmd *cmd)
  2425. {
  2426. DECLARE_COMPLETION_ONSTACK(compl);
  2427. WARN_ON_ONCE(cmd->abrt_compl);
  2428. cmd->abrt_compl = &compl;
  2429. target_put_sess_cmd(cmd);
  2430. wait_for_completion(&compl);
  2431. }
  2432. /*
  2433. * This function is called by frontend drivers after processing of a command
  2434. * has finished.
  2435. *
  2436. * The protocol for ensuring that either the regular frontend command
  2437. * processing flow or target_handle_abort() code drops one reference is as
  2438. * follows:
  2439. * - Calling .queue_data_in(), .queue_status() or queue_tm_rsp() will cause
  2440. * the frontend driver to call this function synchronously or asynchronously.
  2441. * That will cause one reference to be dropped.
  2442. * - During regular command processing the target core sets CMD_T_COMPLETE
  2443. * before invoking one of the .queue_*() functions.
  2444. * - The code that aborts commands skips commands and TMFs for which
  2445. * CMD_T_COMPLETE has been set.
  2446. * - CMD_T_ABORTED is set atomically after the CMD_T_COMPLETE check for
  2447. * commands that will be aborted.
  2448. * - If the CMD_T_ABORTED flag is set but CMD_T_TAS has not been set
  2449. * transport_generic_free_cmd() skips its call to target_put_sess_cmd().
  2450. * - For aborted commands for which CMD_T_TAS has been set .queue_status() will
  2451. * be called and will drop a reference.
  2452. * - For aborted commands for which CMD_T_TAS has not been set .aborted_task()
  2453. * will be called. target_handle_abort() will drop the final reference.
  2454. */
  2455. int transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks)
  2456. {
  2457. DECLARE_COMPLETION_ONSTACK(compl);
  2458. int ret = 0;
  2459. bool aborted = false, tas = false;
  2460. if (wait_for_tasks)
  2461. target_wait_free_cmd(cmd, &aborted, &tas);
  2462. if (cmd->se_cmd_flags & SCF_SE_LUN_CMD) {
  2463. /*
  2464. * Handle WRITE failure case where transport_generic_new_cmd()
  2465. * has already added se_cmd to state_list, but fabric has
  2466. * failed command before I/O submission.
  2467. */
  2468. if (cmd->state_active)
  2469. target_remove_from_state_list(cmd);
  2470. if (cmd->se_lun)
  2471. transport_lun_remove_cmd(cmd);
  2472. }
  2473. if (aborted)
  2474. cmd->free_compl = &compl;
  2475. ret = target_put_sess_cmd(cmd);
  2476. if (aborted) {
  2477. pr_debug("Detected CMD_T_ABORTED for ITT: %llu\n", cmd->tag);
  2478. wait_for_completion(&compl);
  2479. ret = 1;
  2480. }
  2481. return ret;
  2482. }
  2483. EXPORT_SYMBOL(transport_generic_free_cmd);
  2484. /**
  2485. * target_get_sess_cmd - Add command to active ->sess_cmd_list
  2486. * @se_cmd: command descriptor to add
  2487. * @ack_kref: Signal that fabric will perform an ack target_put_sess_cmd()
  2488. */
  2489. int target_get_sess_cmd(struct se_cmd *se_cmd, bool ack_kref)
  2490. {
  2491. struct se_session *se_sess = se_cmd->se_sess;
  2492. unsigned long flags;
  2493. int ret = 0;
  2494. /*
  2495. * Add a second kref if the fabric caller is expecting to handle
  2496. * fabric acknowledgement that requires two target_put_sess_cmd()
  2497. * invocations before se_cmd descriptor release.
  2498. */
  2499. if (ack_kref) {
  2500. if (!kref_get_unless_zero(&se_cmd->cmd_kref))
  2501. return -EINVAL;
  2502. se_cmd->se_cmd_flags |= SCF_ACK_KREF;
  2503. }
  2504. spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
  2505. if (se_sess->sess_tearing_down) {
  2506. ret = -ESHUTDOWN;
  2507. goto out;
  2508. }
  2509. list_add_tail(&se_cmd->se_cmd_list, &se_sess->sess_cmd_list);
  2510. percpu_ref_get(&se_sess->cmd_count);
  2511. out:
  2512. spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
  2513. if (ret && ack_kref)
  2514. target_put_sess_cmd(se_cmd);
  2515. return ret;
  2516. }
  2517. EXPORT_SYMBOL(target_get_sess_cmd);
  2518. static void target_free_cmd_mem(struct se_cmd *cmd)
  2519. {
  2520. transport_free_pages(cmd);
  2521. if (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)
  2522. core_tmr_release_req(cmd->se_tmr_req);
  2523. if (cmd->t_task_cdb != cmd->__t_task_cdb)
  2524. kfree(cmd->t_task_cdb);
  2525. }
  2526. static void target_release_cmd_kref(struct kref *kref)
  2527. {
  2528. struct se_cmd *se_cmd = container_of(kref, struct se_cmd, cmd_kref);
  2529. struct se_session *se_sess = se_cmd->se_sess;
  2530. struct completion *free_compl = se_cmd->free_compl;
  2531. struct completion *abrt_compl = se_cmd->abrt_compl;
  2532. unsigned long flags;
  2533. if (se_sess) {
  2534. spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
  2535. list_del_init(&se_cmd->se_cmd_list);
  2536. spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
  2537. }
  2538. target_free_cmd_mem(se_cmd);
  2539. se_cmd->se_tfo->release_cmd(se_cmd);
  2540. if (free_compl)
  2541. complete(free_compl);
  2542. if (abrt_compl)
  2543. complete(abrt_compl);
  2544. percpu_ref_put(&se_sess->cmd_count);
  2545. }
  2546. /**
  2547. * target_put_sess_cmd - decrease the command reference count
  2548. * @se_cmd: command to drop a reference from
  2549. *
  2550. * Returns 1 if and only if this target_put_sess_cmd() call caused the
  2551. * refcount to drop to zero. Returns zero otherwise.
  2552. */
  2553. int target_put_sess_cmd(struct se_cmd *se_cmd)
  2554. {
  2555. return kref_put(&se_cmd->cmd_kref, target_release_cmd_kref);
  2556. }
  2557. EXPORT_SYMBOL(target_put_sess_cmd);
  2558. static const char *data_dir_name(enum dma_data_direction d)
  2559. {
  2560. switch (d) {
  2561. case DMA_BIDIRECTIONAL: return "BIDI";
  2562. case DMA_TO_DEVICE: return "WRITE";
  2563. case DMA_FROM_DEVICE: return "READ";
  2564. case DMA_NONE: return "NONE";
  2565. }
  2566. return "(?)";
  2567. }
  2568. static const char *cmd_state_name(enum transport_state_table t)
  2569. {
  2570. switch (t) {
  2571. case TRANSPORT_NO_STATE: return "NO_STATE";
  2572. case TRANSPORT_NEW_CMD: return "NEW_CMD";
  2573. case TRANSPORT_WRITE_PENDING: return "WRITE_PENDING";
  2574. case TRANSPORT_PROCESSING: return "PROCESSING";
  2575. case TRANSPORT_COMPLETE: return "COMPLETE";
  2576. case TRANSPORT_ISTATE_PROCESSING:
  2577. return "ISTATE_PROCESSING";
  2578. case TRANSPORT_COMPLETE_QF_WP: return "COMPLETE_QF_WP";
  2579. case TRANSPORT_COMPLETE_QF_OK: return "COMPLETE_QF_OK";
  2580. case TRANSPORT_COMPLETE_QF_ERR: return "COMPLETE_QF_ERR";
  2581. }
  2582. return "(?)";
  2583. }
  2584. static void target_append_str(char **str, const char *txt)
  2585. {
  2586. char *prev = *str;
  2587. *str = *str ? kasprintf(GFP_ATOMIC, "%s,%s", *str, txt) :
  2588. kstrdup(txt, GFP_ATOMIC);
  2589. kfree(prev);
  2590. }
  2591. /*
  2592. * Convert a transport state bitmask into a string. The caller is
  2593. * responsible for freeing the returned pointer.
  2594. */
  2595. static char *target_ts_to_str(u32 ts)
  2596. {
  2597. char *str = NULL;
  2598. if (ts & CMD_T_ABORTED)
  2599. target_append_str(&str, "aborted");
  2600. if (ts & CMD_T_ACTIVE)
  2601. target_append_str(&str, "active");
  2602. if (ts & CMD_T_COMPLETE)
  2603. target_append_str(&str, "complete");
  2604. if (ts & CMD_T_SENT)
  2605. target_append_str(&str, "sent");
  2606. if (ts & CMD_T_STOP)
  2607. target_append_str(&str, "stop");
  2608. if (ts & CMD_T_FABRIC_STOP)
  2609. target_append_str(&str, "fabric_stop");
  2610. return str;
  2611. }
  2612. static const char *target_tmf_name(enum tcm_tmreq_table tmf)
  2613. {
  2614. switch (tmf) {
  2615. case TMR_ABORT_TASK: return "ABORT_TASK";
  2616. case TMR_ABORT_TASK_SET: return "ABORT_TASK_SET";
  2617. case TMR_CLEAR_ACA: return "CLEAR_ACA";
  2618. case TMR_CLEAR_TASK_SET: return "CLEAR_TASK_SET";
  2619. case TMR_LUN_RESET: return "LUN_RESET";
  2620. case TMR_TARGET_WARM_RESET: return "TARGET_WARM_RESET";
  2621. case TMR_TARGET_COLD_RESET: return "TARGET_COLD_RESET";
  2622. case TMR_LUN_RESET_PRO: return "LUN_RESET_PRO";
  2623. case TMR_UNKNOWN: break;
  2624. }
  2625. return "(?)";
  2626. }
  2627. void target_show_cmd(const char *pfx, struct se_cmd *cmd)
  2628. {
  2629. char *ts_str = target_ts_to_str(cmd->transport_state);
  2630. const u8 *cdb = cmd->t_task_cdb;
  2631. struct se_tmr_req *tmf = cmd->se_tmr_req;
  2632. if (!(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) {
  2633. pr_debug("%scmd %#02x:%#02x with tag %#llx dir %s i_state %d t_state %s len %d refcnt %d transport_state %s\n",
  2634. pfx, cdb[0], cdb[1], cmd->tag,
  2635. data_dir_name(cmd->data_direction),
  2636. cmd->se_tfo->get_cmd_state(cmd),
  2637. cmd_state_name(cmd->t_state), cmd->data_length,
  2638. kref_read(&cmd->cmd_kref), ts_str);
  2639. } else {
  2640. pr_debug("%stmf %s with tag %#llx ref_task_tag %#llx i_state %d t_state %s refcnt %d transport_state %s\n",
  2641. pfx, target_tmf_name(tmf->function), cmd->tag,
  2642. tmf->ref_task_tag, cmd->se_tfo->get_cmd_state(cmd),
  2643. cmd_state_name(cmd->t_state),
  2644. kref_read(&cmd->cmd_kref), ts_str);
  2645. }
  2646. kfree(ts_str);
  2647. }
  2648. EXPORT_SYMBOL(target_show_cmd);
  2649. /**
  2650. * target_sess_cmd_list_set_waiting - Set sess_tearing_down so no new commands are queued.
  2651. * @se_sess: session to flag
  2652. */
  2653. void target_sess_cmd_list_set_waiting(struct se_session *se_sess)
  2654. {
  2655. unsigned long flags;
  2656. spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
  2657. se_sess->sess_tearing_down = 1;
  2658. spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
  2659. percpu_ref_kill(&se_sess->cmd_count);
  2660. }
  2661. EXPORT_SYMBOL(target_sess_cmd_list_set_waiting);
  2662. /**
  2663. * target_wait_for_sess_cmds - Wait for outstanding commands
  2664. * @se_sess: session to wait for active I/O
  2665. */
  2666. void target_wait_for_sess_cmds(struct se_session *se_sess)
  2667. {
  2668. struct se_cmd *cmd;
  2669. int ret;
  2670. WARN_ON_ONCE(!se_sess->sess_tearing_down);
  2671. do {
  2672. ret = wait_event_timeout(se_sess->cmd_list_wq,
  2673. percpu_ref_is_zero(&se_sess->cmd_count),
  2674. 180 * HZ);
  2675. list_for_each_entry(cmd, &se_sess->sess_cmd_list, se_cmd_list)
  2676. target_show_cmd("session shutdown: still waiting for ",
  2677. cmd);
  2678. } while (ret <= 0);
  2679. }
  2680. EXPORT_SYMBOL(target_wait_for_sess_cmds);
  2681. /*
  2682. * Prevent that new percpu_ref_tryget_live() calls succeed and wait until
  2683. * all references to the LUN have been released. Called during LUN shutdown.
  2684. */
  2685. void transport_clear_lun_ref(struct se_lun *lun)
  2686. {
  2687. percpu_ref_kill(&lun->lun_ref);
  2688. wait_for_completion(&lun->lun_shutdown_comp);
  2689. }
  2690. static bool
  2691. __transport_wait_for_tasks(struct se_cmd *cmd, bool fabric_stop,
  2692. bool *aborted, bool *tas, unsigned long *flags)
  2693. __releases(&cmd->t_state_lock)
  2694. __acquires(&cmd->t_state_lock)
  2695. {
  2696. lockdep_assert_held(&cmd->t_state_lock);
  2697. if (fabric_stop)
  2698. cmd->transport_state |= CMD_T_FABRIC_STOP;
  2699. if (cmd->transport_state & CMD_T_ABORTED)
  2700. *aborted = true;
  2701. if (cmd->transport_state & CMD_T_TAS)
  2702. *tas = true;
  2703. if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD) &&
  2704. !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB))
  2705. return false;
  2706. if (!(cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE) &&
  2707. !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB))
  2708. return false;
  2709. if (!(cmd->transport_state & CMD_T_ACTIVE))
  2710. return false;
  2711. if (fabric_stop && *aborted)
  2712. return false;
  2713. cmd->transport_state |= CMD_T_STOP;
  2714. target_show_cmd("wait_for_tasks: Stopping ", cmd);
  2715. spin_unlock_irqrestore(&cmd->t_state_lock, *flags);
  2716. while (!wait_for_completion_timeout(&cmd->t_transport_stop_comp,
  2717. 180 * HZ))
  2718. target_show_cmd("wait for tasks: ", cmd);
  2719. spin_lock_irqsave(&cmd->t_state_lock, *flags);
  2720. cmd->transport_state &= ~(CMD_T_ACTIVE | CMD_T_STOP);
  2721. pr_debug("wait_for_tasks: Stopped wait_for_completion(&cmd->"
  2722. "t_transport_stop_comp) for ITT: 0x%08llx\n", cmd->tag);
  2723. return true;
  2724. }
  2725. /**
  2726. * transport_wait_for_tasks - set CMD_T_STOP and wait for t_transport_stop_comp
  2727. * @cmd: command to wait on
  2728. */
  2729. bool transport_wait_for_tasks(struct se_cmd *cmd)
  2730. {
  2731. unsigned long flags;
  2732. bool ret, aborted = false, tas = false;
  2733. spin_lock_irqsave(&cmd->t_state_lock, flags);
  2734. ret = __transport_wait_for_tasks(cmd, false, &aborted, &tas, &flags);
  2735. spin_unlock_irqrestore(&cmd->t_state_lock, flags);
  2736. return ret;
  2737. }
  2738. EXPORT_SYMBOL(transport_wait_for_tasks);
  2739. struct sense_info {
  2740. u8 key;
  2741. u8 asc;
  2742. u8 ascq;
  2743. bool add_sector_info;
  2744. };
  2745. static const struct sense_info sense_info_table[] = {
  2746. [TCM_NO_SENSE] = {
  2747. .key = NOT_READY
  2748. },
  2749. [TCM_NON_EXISTENT_LUN] = {
  2750. .key = ILLEGAL_REQUEST,
  2751. .asc = 0x25 /* LOGICAL UNIT NOT SUPPORTED */
  2752. },
  2753. [TCM_UNSUPPORTED_SCSI_OPCODE] = {
  2754. .key = ILLEGAL_REQUEST,
  2755. .asc = 0x20, /* INVALID COMMAND OPERATION CODE */
  2756. },
  2757. [TCM_SECTOR_COUNT_TOO_MANY] = {
  2758. .key = ILLEGAL_REQUEST,
  2759. .asc = 0x20, /* INVALID COMMAND OPERATION CODE */
  2760. },
  2761. [TCM_UNKNOWN_MODE_PAGE] = {
  2762. .key = ILLEGAL_REQUEST,
  2763. .asc = 0x24, /* INVALID FIELD IN CDB */
  2764. },
  2765. [TCM_CHECK_CONDITION_ABORT_CMD] = {
  2766. .key = ABORTED_COMMAND,
  2767. .asc = 0x29, /* BUS DEVICE RESET FUNCTION OCCURRED */
  2768. .ascq = 0x03,
  2769. },
  2770. [TCM_INCORRECT_AMOUNT_OF_DATA] = {
  2771. .key = ABORTED_COMMAND,
  2772. .asc = 0x0c, /* WRITE ERROR */
  2773. .ascq = 0x0d, /* NOT ENOUGH UNSOLICITED DATA */
  2774. },
  2775. [TCM_INVALID_CDB_FIELD] = {
  2776. .key = ILLEGAL_REQUEST,
  2777. .asc = 0x24, /* INVALID FIELD IN CDB */
  2778. },
  2779. [TCM_INVALID_PARAMETER_LIST] = {
  2780. .key = ILLEGAL_REQUEST,
  2781. .asc = 0x26, /* INVALID FIELD IN PARAMETER LIST */
  2782. },
  2783. [TCM_TOO_MANY_TARGET_DESCS] = {
  2784. .key = ILLEGAL_REQUEST,
  2785. .asc = 0x26,
  2786. .ascq = 0x06, /* TOO MANY TARGET DESCRIPTORS */
  2787. },
  2788. [TCM_UNSUPPORTED_TARGET_DESC_TYPE_CODE] = {
  2789. .key = ILLEGAL_REQUEST,
  2790. .asc = 0x26,
  2791. .ascq = 0x07, /* UNSUPPORTED TARGET DESCRIPTOR TYPE CODE */
  2792. },
  2793. [TCM_TOO_MANY_SEGMENT_DESCS] = {
  2794. .key = ILLEGAL_REQUEST,
  2795. .asc = 0x26,
  2796. .ascq = 0x08, /* TOO MANY SEGMENT DESCRIPTORS */
  2797. },
  2798. [TCM_UNSUPPORTED_SEGMENT_DESC_TYPE_CODE] = {
  2799. .key = ILLEGAL_REQUEST,
  2800. .asc = 0x26,
  2801. .ascq = 0x09, /* UNSUPPORTED SEGMENT DESCRIPTOR TYPE CODE */
  2802. },
  2803. [TCM_PARAMETER_LIST_LENGTH_ERROR] = {
  2804. .key = ILLEGAL_REQUEST,
  2805. .asc = 0x1a, /* PARAMETER LIST LENGTH ERROR */
  2806. },
  2807. [TCM_UNEXPECTED_UNSOLICITED_DATA] = {
  2808. .key = ILLEGAL_REQUEST,
  2809. .asc = 0x0c, /* WRITE ERROR */
  2810. .ascq = 0x0c, /* UNEXPECTED_UNSOLICITED_DATA */
  2811. },
  2812. [TCM_SERVICE_CRC_ERROR] = {
  2813. .key = ABORTED_COMMAND,
  2814. .asc = 0x47, /* PROTOCOL SERVICE CRC ERROR */
  2815. .ascq = 0x05, /* N/A */
  2816. },
  2817. [TCM_SNACK_REJECTED] = {
  2818. .key = ABORTED_COMMAND,
  2819. .asc = 0x11, /* READ ERROR */
  2820. .ascq = 0x13, /* FAILED RETRANSMISSION REQUEST */
  2821. },
  2822. [TCM_WRITE_PROTECTED] = {
  2823. .key = DATA_PROTECT,
  2824. .asc = 0x27, /* WRITE PROTECTED */
  2825. },
  2826. [TCM_ADDRESS_OUT_OF_RANGE] = {
  2827. .key = ILLEGAL_REQUEST,
  2828. .asc = 0x21, /* LOGICAL BLOCK ADDRESS OUT OF RANGE */
  2829. },
  2830. [TCM_CHECK_CONDITION_UNIT_ATTENTION] = {
  2831. .key = UNIT_ATTENTION,
  2832. },
  2833. [TCM_CHECK_CONDITION_NOT_READY] = {
  2834. .key = NOT_READY,
  2835. },
  2836. [TCM_MISCOMPARE_VERIFY] = {
  2837. .key = MISCOMPARE,
  2838. .asc = 0x1d, /* MISCOMPARE DURING VERIFY OPERATION */
  2839. .ascq = 0x00,
  2840. },
  2841. [TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED] = {
  2842. .key = ABORTED_COMMAND,
  2843. .asc = 0x10,
  2844. .ascq = 0x01, /* LOGICAL BLOCK GUARD CHECK FAILED */
  2845. .add_sector_info = true,
  2846. },
  2847. [TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED] = {
  2848. .key = ABORTED_COMMAND,
  2849. .asc = 0x10,
  2850. .ascq = 0x02, /* LOGICAL BLOCK APPLICATION TAG CHECK FAILED */
  2851. .add_sector_info = true,
  2852. },
  2853. [TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED] = {
  2854. .key = ABORTED_COMMAND,
  2855. .asc = 0x10,
  2856. .ascq = 0x03, /* LOGICAL BLOCK REFERENCE TAG CHECK FAILED */
  2857. .add_sector_info = true,
  2858. },
  2859. [TCM_COPY_TARGET_DEVICE_NOT_REACHABLE] = {
  2860. .key = COPY_ABORTED,
  2861. .asc = 0x0d,
  2862. .ascq = 0x02, /* COPY TARGET DEVICE NOT REACHABLE */
  2863. },
  2864. [TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE] = {
  2865. /*
  2866. * Returning ILLEGAL REQUEST would cause immediate IO errors on
  2867. * Solaris initiators. Returning NOT READY instead means the
  2868. * operations will be retried a finite number of times and we
  2869. * can survive intermittent errors.
  2870. */
  2871. .key = NOT_READY,
  2872. .asc = 0x08, /* LOGICAL UNIT COMMUNICATION FAILURE */
  2873. },
  2874. [TCM_INSUFFICIENT_REGISTRATION_RESOURCES] = {
  2875. /*
  2876. * From spc4r22 section5.7.7,5.7.8
  2877. * If a PERSISTENT RESERVE OUT command with a REGISTER service action
  2878. * or a REGISTER AND IGNORE EXISTING KEY service action or
  2879. * REGISTER AND MOVE service actionis attempted,
  2880. * but there are insufficient device server resources to complete the
  2881. * operation, then the command shall be terminated with CHECK CONDITION
  2882. * status, with the sense key set to ILLEGAL REQUEST,and the additonal
  2883. * sense code set to INSUFFICIENT REGISTRATION RESOURCES.
  2884. */
  2885. .key = ILLEGAL_REQUEST,
  2886. .asc = 0x55,
  2887. .ascq = 0x04, /* INSUFFICIENT REGISTRATION RESOURCES */
  2888. },
  2889. };
  2890. /**
  2891. * translate_sense_reason - translate a sense reason into T10 key, asc and ascq
  2892. * @cmd: SCSI command in which the resulting sense buffer or SCSI status will
  2893. * be stored.
  2894. * @reason: LIO sense reason code. If this argument has the value
  2895. * TCM_CHECK_CONDITION_UNIT_ATTENTION, try to dequeue a unit attention. If
  2896. * dequeuing a unit attention fails due to multiple commands being processed
  2897. * concurrently, set the command status to BUSY.
  2898. *
  2899. * Return: 0 upon success or -EINVAL if the sense buffer is too small.
  2900. */
  2901. static void translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason)
  2902. {
  2903. const struct sense_info *si;
  2904. u8 *buffer = cmd->sense_buffer;
  2905. int r = (__force int)reason;
  2906. u8 key, asc, ascq;
  2907. bool desc_format = target_sense_desc_format(cmd->se_dev);
  2908. if (r < ARRAY_SIZE(sense_info_table) && sense_info_table[r].key)
  2909. si = &sense_info_table[r];
  2910. else
  2911. si = &sense_info_table[(__force int)
  2912. TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE];
  2913. key = si->key;
  2914. if (reason == TCM_CHECK_CONDITION_UNIT_ATTENTION) {
  2915. if (!core_scsi3_ua_for_check_condition(cmd, &key, &asc,
  2916. &ascq)) {
  2917. cmd->scsi_status = SAM_STAT_BUSY;
  2918. return;
  2919. }
  2920. } else if (si->asc == 0) {
  2921. WARN_ON_ONCE(cmd->scsi_asc == 0);
  2922. asc = cmd->scsi_asc;
  2923. ascq = cmd->scsi_ascq;
  2924. } else {
  2925. asc = si->asc;
  2926. ascq = si->ascq;
  2927. }
  2928. cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE;
  2929. cmd->scsi_status = SAM_STAT_CHECK_CONDITION;
  2930. cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER;
  2931. scsi_build_sense_buffer(desc_format, buffer, key, asc, ascq);
  2932. if (si->add_sector_info)
  2933. WARN_ON_ONCE(scsi_set_sense_information(buffer,
  2934. cmd->scsi_sense_length,
  2935. cmd->bad_sector) < 0);
  2936. }
  2937. int
  2938. transport_send_check_condition_and_sense(struct se_cmd *cmd,
  2939. sense_reason_t reason, int from_transport)
  2940. {
  2941. unsigned long flags;
  2942. WARN_ON_ONCE(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB);
  2943. spin_lock_irqsave(&cmd->t_state_lock, flags);
  2944. if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
  2945. spin_unlock_irqrestore(&cmd->t_state_lock, flags);
  2946. return 0;
  2947. }
  2948. cmd->se_cmd_flags |= SCF_SENT_CHECK_CONDITION;
  2949. spin_unlock_irqrestore(&cmd->t_state_lock, flags);
  2950. if (!from_transport)
  2951. translate_sense_reason(cmd, reason);
  2952. trace_target_cmd_complete(cmd);
  2953. return cmd->se_tfo->queue_status(cmd);
  2954. }
  2955. EXPORT_SYMBOL(transport_send_check_condition_and_sense);
  2956. /**
  2957. * target_send_busy - Send SCSI BUSY status back to the initiator
  2958. * @cmd: SCSI command for which to send a BUSY reply.
  2959. *
  2960. * Note: Only call this function if target_submit_cmd*() failed.
  2961. */
  2962. int target_send_busy(struct se_cmd *cmd)
  2963. {
  2964. WARN_ON_ONCE(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB);
  2965. cmd->scsi_status = SAM_STAT_BUSY;
  2966. trace_target_cmd_complete(cmd);
  2967. return cmd->se_tfo->queue_status(cmd);
  2968. }
  2969. EXPORT_SYMBOL(target_send_busy);
  2970. static void target_tmr_work(struct work_struct *work)
  2971. {
  2972. struct se_cmd *cmd = container_of(work, struct se_cmd, work);
  2973. struct se_device *dev = cmd->se_dev;
  2974. struct se_tmr_req *tmr = cmd->se_tmr_req;
  2975. int ret;
  2976. if (cmd->transport_state & CMD_T_ABORTED)
  2977. goto aborted;
  2978. switch (tmr->function) {
  2979. case TMR_ABORT_TASK:
  2980. core_tmr_abort_task(dev, tmr, cmd->se_sess);
  2981. break;
  2982. case TMR_ABORT_TASK_SET:
  2983. case TMR_CLEAR_ACA:
  2984. case TMR_CLEAR_TASK_SET:
  2985. tmr->response = TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED;
  2986. break;
  2987. case TMR_LUN_RESET:
  2988. ret = core_tmr_lun_reset(dev, tmr, NULL, NULL);
  2989. tmr->response = (!ret) ? TMR_FUNCTION_COMPLETE :
  2990. TMR_FUNCTION_REJECTED;
  2991. if (tmr->response == TMR_FUNCTION_COMPLETE) {
  2992. target_ua_allocate_lun(cmd->se_sess->se_node_acl,
  2993. cmd->orig_fe_lun, 0x29,
  2994. ASCQ_29H_BUS_DEVICE_RESET_FUNCTION_OCCURRED);
  2995. }
  2996. break;
  2997. case TMR_TARGET_WARM_RESET:
  2998. tmr->response = TMR_FUNCTION_REJECTED;
  2999. break;
  3000. case TMR_TARGET_COLD_RESET:
  3001. tmr->response = TMR_FUNCTION_REJECTED;
  3002. break;
  3003. default:
  3004. pr_err("Unknown TMR function: 0x%02x.\n",
  3005. tmr->function);
  3006. tmr->response = TMR_FUNCTION_REJECTED;
  3007. break;
  3008. }
  3009. if (cmd->transport_state & CMD_T_ABORTED)
  3010. goto aborted;
  3011. cmd->se_tfo->queue_tm_rsp(cmd);
  3012. transport_lun_remove_cmd(cmd);
  3013. transport_cmd_check_stop_to_fabric(cmd);
  3014. return;
  3015. aborted:
  3016. target_handle_abort(cmd);
  3017. }
  3018. int transport_generic_handle_tmr(
  3019. struct se_cmd *cmd)
  3020. {
  3021. unsigned long flags;
  3022. bool aborted = false;
  3023. spin_lock_irqsave(&cmd->t_state_lock, flags);
  3024. if (cmd->transport_state & CMD_T_ABORTED) {
  3025. aborted = true;
  3026. } else {
  3027. cmd->t_state = TRANSPORT_ISTATE_PROCESSING;
  3028. cmd->transport_state |= CMD_T_ACTIVE;
  3029. }
  3030. spin_unlock_irqrestore(&cmd->t_state_lock, flags);
  3031. if (aborted) {
  3032. pr_warn_ratelimited("handle_tmr caught CMD_T_ABORTED TMR %d ref_tag: %llu tag: %llu\n",
  3033. cmd->se_tmr_req->function,
  3034. cmd->se_tmr_req->ref_task_tag, cmd->tag);
  3035. target_handle_abort(cmd);
  3036. return 0;
  3037. }
  3038. INIT_WORK(&cmd->work, target_tmr_work);
  3039. schedule_work(&cmd->work);
  3040. return 0;
  3041. }
  3042. EXPORT_SYMBOL(transport_generic_handle_tmr);
  3043. bool
  3044. target_check_wce(struct se_device *dev)
  3045. {
  3046. bool wce = false;
  3047. if (dev->transport->get_write_cache)
  3048. wce = dev->transport->get_write_cache(dev);
  3049. else if (dev->dev_attrib.emulate_write_cache > 0)
  3050. wce = true;
  3051. return wce;
  3052. }
  3053. bool
  3054. target_check_fua(struct se_device *dev)
  3055. {
  3056. return target_check_wce(dev) && dev->dev_attrib.emulate_fua_write > 0;
  3057. }