iscsi_target.c 126 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*******************************************************************************
  3. * This file contains main functions related to the iSCSI Target Core Driver.
  4. *
  5. * (c) Copyright 2007-2013 Datera, Inc.
  6. *
  7. * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
  8. *
  9. ******************************************************************************/
  10. #include <crypto/hash.h>
  11. #include <linux/string.h>
  12. #include <linux/kthread.h>
  13. #include <linux/completion.h>
  14. #include <linux/module.h>
  15. #include <linux/vmalloc.h>
  16. #include <linux/idr.h>
  17. #include <linux/delay.h>
  18. #include <linux/sched/signal.h>
  19. #include <asm/unaligned.h>
  20. #include <linux/inet.h>
  21. #include <net/ipv6.h>
  22. #include <scsi/scsi_proto.h>
  23. #include <scsi/iscsi_proto.h>
  24. #include <scsi/scsi_tcq.h>
  25. #include <target/target_core_base.h>
  26. #include <target/target_core_fabric.h>
  27. #include <target/iscsi/iscsi_target_core.h>
  28. #include "iscsi_target_parameters.h"
  29. #include "iscsi_target_seq_pdu_list.h"
  30. #include "iscsi_target_datain_values.h"
  31. #include "iscsi_target_erl0.h"
  32. #include "iscsi_target_erl1.h"
  33. #include "iscsi_target_erl2.h"
  34. #include "iscsi_target_login.h"
  35. #include "iscsi_target_tmr.h"
  36. #include "iscsi_target_tpg.h"
  37. #include "iscsi_target_util.h"
  38. #include "iscsi_target.h"
  39. #include "iscsi_target_device.h"
  40. #include <target/iscsi/iscsi_target_stat.h>
  41. #include <target/iscsi/iscsi_transport.h>
  42. static LIST_HEAD(g_tiqn_list);
  43. static LIST_HEAD(g_np_list);
  44. static DEFINE_SPINLOCK(tiqn_lock);
  45. static DEFINE_MUTEX(np_lock);
  46. static struct idr tiqn_idr;
  47. DEFINE_IDA(sess_ida);
  48. struct mutex auth_id_lock;
  49. struct iscsit_global *iscsit_global;
  50. struct kmem_cache *lio_qr_cache;
  51. struct kmem_cache *lio_dr_cache;
  52. struct kmem_cache *lio_ooo_cache;
  53. struct kmem_cache *lio_r2t_cache;
  54. static int iscsit_handle_immediate_data(struct iscsi_cmd *,
  55. struct iscsi_scsi_req *, u32);
  56. struct iscsi_tiqn *iscsit_get_tiqn_for_login(unsigned char *buf)
  57. {
  58. struct iscsi_tiqn *tiqn = NULL;
  59. spin_lock(&tiqn_lock);
  60. list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) {
  61. if (!strcmp(tiqn->tiqn, buf)) {
  62. spin_lock(&tiqn->tiqn_state_lock);
  63. if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) {
  64. tiqn->tiqn_access_count++;
  65. spin_unlock(&tiqn->tiqn_state_lock);
  66. spin_unlock(&tiqn_lock);
  67. return tiqn;
  68. }
  69. spin_unlock(&tiqn->tiqn_state_lock);
  70. }
  71. }
  72. spin_unlock(&tiqn_lock);
  73. return NULL;
  74. }
  75. static int iscsit_set_tiqn_shutdown(struct iscsi_tiqn *tiqn)
  76. {
  77. spin_lock(&tiqn->tiqn_state_lock);
  78. if (tiqn->tiqn_state == TIQN_STATE_ACTIVE) {
  79. tiqn->tiqn_state = TIQN_STATE_SHUTDOWN;
  80. spin_unlock(&tiqn->tiqn_state_lock);
  81. return 0;
  82. }
  83. spin_unlock(&tiqn->tiqn_state_lock);
  84. return -1;
  85. }
  86. void iscsit_put_tiqn_for_login(struct iscsi_tiqn *tiqn)
  87. {
  88. spin_lock(&tiqn->tiqn_state_lock);
  89. tiqn->tiqn_access_count--;
  90. spin_unlock(&tiqn->tiqn_state_lock);
  91. }
  92. /*
  93. * Note that IQN formatting is expected to be done in userspace, and
  94. * no explict IQN format checks are done here.
  95. */
  96. struct iscsi_tiqn *iscsit_add_tiqn(unsigned char *buf)
  97. {
  98. struct iscsi_tiqn *tiqn = NULL;
  99. int ret;
  100. if (strlen(buf) >= ISCSI_IQN_LEN) {
  101. pr_err("Target IQN exceeds %d bytes\n",
  102. ISCSI_IQN_LEN);
  103. return ERR_PTR(-EINVAL);
  104. }
  105. tiqn = kzalloc(sizeof(*tiqn), GFP_KERNEL);
  106. if (!tiqn)
  107. return ERR_PTR(-ENOMEM);
  108. sprintf(tiqn->tiqn, "%s", buf);
  109. INIT_LIST_HEAD(&tiqn->tiqn_list);
  110. INIT_LIST_HEAD(&tiqn->tiqn_tpg_list);
  111. spin_lock_init(&tiqn->tiqn_state_lock);
  112. spin_lock_init(&tiqn->tiqn_tpg_lock);
  113. spin_lock_init(&tiqn->sess_err_stats.lock);
  114. spin_lock_init(&tiqn->login_stats.lock);
  115. spin_lock_init(&tiqn->logout_stats.lock);
  116. tiqn->tiqn_state = TIQN_STATE_ACTIVE;
  117. idr_preload(GFP_KERNEL);
  118. spin_lock(&tiqn_lock);
  119. ret = idr_alloc(&tiqn_idr, NULL, 0, 0, GFP_NOWAIT);
  120. if (ret < 0) {
  121. pr_err("idr_alloc() failed for tiqn->tiqn_index\n");
  122. spin_unlock(&tiqn_lock);
  123. idr_preload_end();
  124. kfree(tiqn);
  125. return ERR_PTR(ret);
  126. }
  127. tiqn->tiqn_index = ret;
  128. list_add_tail(&tiqn->tiqn_list, &g_tiqn_list);
  129. spin_unlock(&tiqn_lock);
  130. idr_preload_end();
  131. pr_debug("CORE[0] - Added iSCSI Target IQN: %s\n", tiqn->tiqn);
  132. return tiqn;
  133. }
  134. static void iscsit_wait_for_tiqn(struct iscsi_tiqn *tiqn)
  135. {
  136. /*
  137. * Wait for accesses to said struct iscsi_tiqn to end.
  138. */
  139. spin_lock(&tiqn->tiqn_state_lock);
  140. while (tiqn->tiqn_access_count != 0) {
  141. spin_unlock(&tiqn->tiqn_state_lock);
  142. msleep(10);
  143. spin_lock(&tiqn->tiqn_state_lock);
  144. }
  145. spin_unlock(&tiqn->tiqn_state_lock);
  146. }
  147. void iscsit_del_tiqn(struct iscsi_tiqn *tiqn)
  148. {
  149. /*
  150. * iscsit_set_tiqn_shutdown sets tiqn->tiqn_state = TIQN_STATE_SHUTDOWN
  151. * while holding tiqn->tiqn_state_lock. This means that all subsequent
  152. * attempts to access this struct iscsi_tiqn will fail from both transport
  153. * fabric and control code paths.
  154. */
  155. if (iscsit_set_tiqn_shutdown(tiqn) < 0) {
  156. pr_err("iscsit_set_tiqn_shutdown() failed\n");
  157. return;
  158. }
  159. iscsit_wait_for_tiqn(tiqn);
  160. spin_lock(&tiqn_lock);
  161. list_del(&tiqn->tiqn_list);
  162. idr_remove(&tiqn_idr, tiqn->tiqn_index);
  163. spin_unlock(&tiqn_lock);
  164. pr_debug("CORE[0] - Deleted iSCSI Target IQN: %s\n",
  165. tiqn->tiqn);
  166. kfree(tiqn);
  167. }
  168. int iscsit_access_np(struct iscsi_np *np, struct iscsi_portal_group *tpg)
  169. {
  170. int ret;
  171. /*
  172. * Determine if the network portal is accepting storage traffic.
  173. */
  174. spin_lock_bh(&np->np_thread_lock);
  175. if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) {
  176. spin_unlock_bh(&np->np_thread_lock);
  177. return -1;
  178. }
  179. spin_unlock_bh(&np->np_thread_lock);
  180. /*
  181. * Determine if the portal group is accepting storage traffic.
  182. */
  183. spin_lock_bh(&tpg->tpg_state_lock);
  184. if (tpg->tpg_state != TPG_STATE_ACTIVE) {
  185. spin_unlock_bh(&tpg->tpg_state_lock);
  186. return -1;
  187. }
  188. spin_unlock_bh(&tpg->tpg_state_lock);
  189. /*
  190. * Here we serialize access across the TIQN+TPG Tuple.
  191. */
  192. ret = down_interruptible(&tpg->np_login_sem);
  193. if (ret != 0)
  194. return -1;
  195. spin_lock_bh(&tpg->tpg_state_lock);
  196. if (tpg->tpg_state != TPG_STATE_ACTIVE) {
  197. spin_unlock_bh(&tpg->tpg_state_lock);
  198. up(&tpg->np_login_sem);
  199. return -1;
  200. }
  201. spin_unlock_bh(&tpg->tpg_state_lock);
  202. return 0;
  203. }
  204. void iscsit_login_kref_put(struct kref *kref)
  205. {
  206. struct iscsi_tpg_np *tpg_np = container_of(kref,
  207. struct iscsi_tpg_np, tpg_np_kref);
  208. complete(&tpg_np->tpg_np_comp);
  209. }
  210. int iscsit_deaccess_np(struct iscsi_np *np, struct iscsi_portal_group *tpg,
  211. struct iscsi_tpg_np *tpg_np)
  212. {
  213. struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
  214. up(&tpg->np_login_sem);
  215. if (tpg_np)
  216. kref_put(&tpg_np->tpg_np_kref, iscsit_login_kref_put);
  217. if (tiqn)
  218. iscsit_put_tiqn_for_login(tiqn);
  219. return 0;
  220. }
  221. bool iscsit_check_np_match(
  222. struct sockaddr_storage *sockaddr,
  223. struct iscsi_np *np,
  224. int network_transport)
  225. {
  226. struct sockaddr_in *sock_in, *sock_in_e;
  227. struct sockaddr_in6 *sock_in6, *sock_in6_e;
  228. bool ip_match = false;
  229. u16 port, port_e;
  230. if (sockaddr->ss_family == AF_INET6) {
  231. sock_in6 = (struct sockaddr_in6 *)sockaddr;
  232. sock_in6_e = (struct sockaddr_in6 *)&np->np_sockaddr;
  233. if (!memcmp(&sock_in6->sin6_addr.in6_u,
  234. &sock_in6_e->sin6_addr.in6_u,
  235. sizeof(struct in6_addr)))
  236. ip_match = true;
  237. port = ntohs(sock_in6->sin6_port);
  238. port_e = ntohs(sock_in6_e->sin6_port);
  239. } else {
  240. sock_in = (struct sockaddr_in *)sockaddr;
  241. sock_in_e = (struct sockaddr_in *)&np->np_sockaddr;
  242. if (sock_in->sin_addr.s_addr == sock_in_e->sin_addr.s_addr)
  243. ip_match = true;
  244. port = ntohs(sock_in->sin_port);
  245. port_e = ntohs(sock_in_e->sin_port);
  246. }
  247. if (ip_match && (port_e == port) &&
  248. (np->np_network_transport == network_transport))
  249. return true;
  250. return false;
  251. }
  252. static struct iscsi_np *iscsit_get_np(
  253. struct sockaddr_storage *sockaddr,
  254. int network_transport)
  255. {
  256. struct iscsi_np *np;
  257. bool match;
  258. lockdep_assert_held(&np_lock);
  259. list_for_each_entry(np, &g_np_list, np_list) {
  260. spin_lock_bh(&np->np_thread_lock);
  261. if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) {
  262. spin_unlock_bh(&np->np_thread_lock);
  263. continue;
  264. }
  265. match = iscsit_check_np_match(sockaddr, np, network_transport);
  266. if (match) {
  267. /*
  268. * Increment the np_exports reference count now to
  269. * prevent iscsit_del_np() below from being called
  270. * while iscsi_tpg_add_network_portal() is called.
  271. */
  272. np->np_exports++;
  273. spin_unlock_bh(&np->np_thread_lock);
  274. return np;
  275. }
  276. spin_unlock_bh(&np->np_thread_lock);
  277. }
  278. return NULL;
  279. }
  280. struct iscsi_np *iscsit_add_np(
  281. struct sockaddr_storage *sockaddr,
  282. int network_transport)
  283. {
  284. struct iscsi_np *np;
  285. int ret;
  286. mutex_lock(&np_lock);
  287. /*
  288. * Locate the existing struct iscsi_np if already active..
  289. */
  290. np = iscsit_get_np(sockaddr, network_transport);
  291. if (np) {
  292. mutex_unlock(&np_lock);
  293. return np;
  294. }
  295. np = kzalloc(sizeof(*np), GFP_KERNEL);
  296. if (!np) {
  297. mutex_unlock(&np_lock);
  298. return ERR_PTR(-ENOMEM);
  299. }
  300. np->np_flags |= NPF_IP_NETWORK;
  301. np->np_network_transport = network_transport;
  302. spin_lock_init(&np->np_thread_lock);
  303. init_completion(&np->np_restart_comp);
  304. INIT_LIST_HEAD(&np->np_list);
  305. timer_setup(&np->np_login_timer, iscsi_handle_login_thread_timeout, 0);
  306. ret = iscsi_target_setup_login_socket(np, sockaddr);
  307. if (ret != 0) {
  308. kfree(np);
  309. mutex_unlock(&np_lock);
  310. return ERR_PTR(ret);
  311. }
  312. np->np_thread = kthread_run(iscsi_target_login_thread, np, "iscsi_np");
  313. if (IS_ERR(np->np_thread)) {
  314. pr_err("Unable to create kthread: iscsi_np\n");
  315. ret = PTR_ERR(np->np_thread);
  316. kfree(np);
  317. mutex_unlock(&np_lock);
  318. return ERR_PTR(ret);
  319. }
  320. /*
  321. * Increment the np_exports reference count now to prevent
  322. * iscsit_del_np() below from being run while a new call to
  323. * iscsi_tpg_add_network_portal() for a matching iscsi_np is
  324. * active. We don't need to hold np->np_thread_lock at this
  325. * point because iscsi_np has not been added to g_np_list yet.
  326. */
  327. np->np_exports = 1;
  328. np->np_thread_state = ISCSI_NP_THREAD_ACTIVE;
  329. list_add_tail(&np->np_list, &g_np_list);
  330. mutex_unlock(&np_lock);
  331. pr_debug("CORE[0] - Added Network Portal: %pISpc on %s\n",
  332. &np->np_sockaddr, np->np_transport->name);
  333. return np;
  334. }
  335. int iscsit_reset_np_thread(
  336. struct iscsi_np *np,
  337. struct iscsi_tpg_np *tpg_np,
  338. struct iscsi_portal_group *tpg,
  339. bool shutdown)
  340. {
  341. spin_lock_bh(&np->np_thread_lock);
  342. if (np->np_thread_state == ISCSI_NP_THREAD_INACTIVE) {
  343. spin_unlock_bh(&np->np_thread_lock);
  344. return 0;
  345. }
  346. np->np_thread_state = ISCSI_NP_THREAD_RESET;
  347. atomic_inc(&np->np_reset_count);
  348. if (np->np_thread) {
  349. spin_unlock_bh(&np->np_thread_lock);
  350. send_sig(SIGINT, np->np_thread, 1);
  351. wait_for_completion(&np->np_restart_comp);
  352. spin_lock_bh(&np->np_thread_lock);
  353. }
  354. spin_unlock_bh(&np->np_thread_lock);
  355. if (tpg_np && shutdown) {
  356. kref_put(&tpg_np->tpg_np_kref, iscsit_login_kref_put);
  357. wait_for_completion(&tpg_np->tpg_np_comp);
  358. }
  359. return 0;
  360. }
  361. static void iscsit_free_np(struct iscsi_np *np)
  362. {
  363. if (np->np_socket)
  364. sock_release(np->np_socket);
  365. }
  366. int iscsit_del_np(struct iscsi_np *np)
  367. {
  368. spin_lock_bh(&np->np_thread_lock);
  369. np->np_exports--;
  370. if (np->np_exports) {
  371. np->enabled = true;
  372. spin_unlock_bh(&np->np_thread_lock);
  373. return 0;
  374. }
  375. np->np_thread_state = ISCSI_NP_THREAD_SHUTDOWN;
  376. spin_unlock_bh(&np->np_thread_lock);
  377. if (np->np_thread) {
  378. /*
  379. * We need to send the signal to wakeup Linux/Net
  380. * which may be sleeping in sock_accept()..
  381. */
  382. send_sig(SIGINT, np->np_thread, 1);
  383. kthread_stop(np->np_thread);
  384. np->np_thread = NULL;
  385. }
  386. np->np_transport->iscsit_free_np(np);
  387. mutex_lock(&np_lock);
  388. list_del(&np->np_list);
  389. mutex_unlock(&np_lock);
  390. pr_debug("CORE[0] - Removed Network Portal: %pISpc on %s\n",
  391. &np->np_sockaddr, np->np_transport->name);
  392. iscsit_put_transport(np->np_transport);
  393. kfree(np);
  394. return 0;
  395. }
  396. static void iscsit_get_rx_pdu(struct iscsi_conn *);
  397. int iscsit_queue_rsp(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
  398. {
  399. return iscsit_add_cmd_to_response_queue(cmd, cmd->conn, cmd->i_state);
  400. }
  401. EXPORT_SYMBOL(iscsit_queue_rsp);
  402. void iscsit_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
  403. {
  404. spin_lock_bh(&conn->cmd_lock);
  405. if (!list_empty(&cmd->i_conn_node))
  406. list_del_init(&cmd->i_conn_node);
  407. spin_unlock_bh(&conn->cmd_lock);
  408. __iscsit_free_cmd(cmd, true);
  409. }
  410. EXPORT_SYMBOL(iscsit_aborted_task);
  411. static void iscsit_do_crypto_hash_buf(struct ahash_request *, const void *,
  412. u32, u32, const void *, void *);
  413. static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *);
  414. static int
  415. iscsit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
  416. const void *data_buf, u32 data_buf_len)
  417. {
  418. struct iscsi_hdr *hdr = (struct iscsi_hdr *)cmd->pdu;
  419. struct kvec *iov;
  420. u32 niov = 0, tx_size = ISCSI_HDR_LEN;
  421. int ret;
  422. iov = &cmd->iov_misc[0];
  423. iov[niov].iov_base = cmd->pdu;
  424. iov[niov++].iov_len = ISCSI_HDR_LEN;
  425. if (conn->conn_ops->HeaderDigest) {
  426. u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
  427. iscsit_do_crypto_hash_buf(conn->conn_tx_hash, hdr,
  428. ISCSI_HDR_LEN, 0, NULL,
  429. header_digest);
  430. iov[0].iov_len += ISCSI_CRC_LEN;
  431. tx_size += ISCSI_CRC_LEN;
  432. pr_debug("Attaching CRC32C HeaderDigest"
  433. " to opcode 0x%x 0x%08x\n",
  434. hdr->opcode, *header_digest);
  435. }
  436. if (data_buf_len) {
  437. u32 padding = ((-data_buf_len) & 3);
  438. iov[niov].iov_base = (void *)data_buf;
  439. iov[niov++].iov_len = data_buf_len;
  440. tx_size += data_buf_len;
  441. if (padding != 0) {
  442. iov[niov].iov_base = &cmd->pad_bytes;
  443. iov[niov++].iov_len = padding;
  444. tx_size += padding;
  445. pr_debug("Attaching %u additional"
  446. " padding bytes.\n", padding);
  447. }
  448. if (conn->conn_ops->DataDigest) {
  449. iscsit_do_crypto_hash_buf(conn->conn_tx_hash,
  450. data_buf, data_buf_len,
  451. padding, &cmd->pad_bytes,
  452. &cmd->data_crc);
  453. iov[niov].iov_base = &cmd->data_crc;
  454. iov[niov++].iov_len = ISCSI_CRC_LEN;
  455. tx_size += ISCSI_CRC_LEN;
  456. pr_debug("Attached DataDigest for %u"
  457. " bytes opcode 0x%x, CRC 0x%08x\n",
  458. data_buf_len, hdr->opcode, cmd->data_crc);
  459. }
  460. }
  461. cmd->iov_misc_count = niov;
  462. cmd->tx_size = tx_size;
  463. ret = iscsit_send_tx_data(cmd, conn, 1);
  464. if (ret < 0) {
  465. iscsit_tx_thread_wait_for_tcp(conn);
  466. return ret;
  467. }
  468. return 0;
  469. }
  470. static int iscsit_map_iovec(struct iscsi_cmd *cmd, struct kvec *iov, int nvec,
  471. u32 data_offset, u32 data_length);
  472. static void iscsit_unmap_iovec(struct iscsi_cmd *);
  473. static u32 iscsit_do_crypto_hash_sg(struct ahash_request *, struct iscsi_cmd *,
  474. u32, u32, u32, u8 *);
  475. static int
  476. iscsit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
  477. const struct iscsi_datain *datain)
  478. {
  479. struct kvec *iov;
  480. u32 iov_count = 0, tx_size = 0;
  481. int ret, iov_ret;
  482. iov = &cmd->iov_data[0];
  483. iov[iov_count].iov_base = cmd->pdu;
  484. iov[iov_count++].iov_len = ISCSI_HDR_LEN;
  485. tx_size += ISCSI_HDR_LEN;
  486. if (conn->conn_ops->HeaderDigest) {
  487. u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN];
  488. iscsit_do_crypto_hash_buf(conn->conn_tx_hash, cmd->pdu,
  489. ISCSI_HDR_LEN, 0, NULL,
  490. header_digest);
  491. iov[0].iov_len += ISCSI_CRC_LEN;
  492. tx_size += ISCSI_CRC_LEN;
  493. pr_debug("Attaching CRC32 HeaderDigest for DataIN PDU 0x%08x\n",
  494. *header_digest);
  495. }
  496. iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[iov_count],
  497. cmd->orig_iov_data_count - (iov_count + 2),
  498. datain->offset, datain->length);
  499. if (iov_ret < 0)
  500. return -1;
  501. iov_count += iov_ret;
  502. tx_size += datain->length;
  503. cmd->padding = ((-datain->length) & 3);
  504. if (cmd->padding) {
  505. iov[iov_count].iov_base = cmd->pad_bytes;
  506. iov[iov_count++].iov_len = cmd->padding;
  507. tx_size += cmd->padding;
  508. pr_debug("Attaching %u padding bytes\n", cmd->padding);
  509. }
  510. if (conn->conn_ops->DataDigest) {
  511. cmd->data_crc = iscsit_do_crypto_hash_sg(conn->conn_tx_hash,
  512. cmd, datain->offset,
  513. datain->length,
  514. cmd->padding,
  515. cmd->pad_bytes);
  516. iov[iov_count].iov_base = &cmd->data_crc;
  517. iov[iov_count++].iov_len = ISCSI_CRC_LEN;
  518. tx_size += ISCSI_CRC_LEN;
  519. pr_debug("Attached CRC32C DataDigest %d bytes, crc 0x%08x\n",
  520. datain->length + cmd->padding, cmd->data_crc);
  521. }
  522. cmd->iov_data_count = iov_count;
  523. cmd->tx_size = tx_size;
  524. ret = iscsit_fe_sendpage_sg(cmd, conn);
  525. iscsit_unmap_iovec(cmd);
  526. if (ret < 0) {
  527. iscsit_tx_thread_wait_for_tcp(conn);
  528. return ret;
  529. }
  530. return 0;
  531. }
  532. static int iscsit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
  533. struct iscsi_datain_req *dr, const void *buf,
  534. u32 buf_len)
  535. {
  536. if (dr)
  537. return iscsit_xmit_datain_pdu(conn, cmd, buf);
  538. else
  539. return iscsit_xmit_nondatain_pdu(conn, cmd, buf, buf_len);
  540. }
  541. static enum target_prot_op iscsit_get_sup_prot_ops(struct iscsi_conn *conn)
  542. {
  543. return TARGET_PROT_NORMAL;
  544. }
  545. static struct iscsit_transport iscsi_target_transport = {
  546. .name = "iSCSI/TCP",
  547. .transport_type = ISCSI_TCP,
  548. .rdma_shutdown = false,
  549. .owner = NULL,
  550. .iscsit_setup_np = iscsit_setup_np,
  551. .iscsit_accept_np = iscsit_accept_np,
  552. .iscsit_free_np = iscsit_free_np,
  553. .iscsit_get_login_rx = iscsit_get_login_rx,
  554. .iscsit_put_login_tx = iscsit_put_login_tx,
  555. .iscsit_get_dataout = iscsit_build_r2ts_for_cmd,
  556. .iscsit_immediate_queue = iscsit_immediate_queue,
  557. .iscsit_response_queue = iscsit_response_queue,
  558. .iscsit_queue_data_in = iscsit_queue_rsp,
  559. .iscsit_queue_status = iscsit_queue_rsp,
  560. .iscsit_aborted_task = iscsit_aborted_task,
  561. .iscsit_xmit_pdu = iscsit_xmit_pdu,
  562. .iscsit_get_rx_pdu = iscsit_get_rx_pdu,
  563. .iscsit_get_sup_prot_ops = iscsit_get_sup_prot_ops,
  564. };
  565. static int __init iscsi_target_init_module(void)
  566. {
  567. int ret = 0, size;
  568. pr_debug("iSCSI-Target "ISCSIT_VERSION"\n");
  569. iscsit_global = kzalloc(sizeof(*iscsit_global), GFP_KERNEL);
  570. if (!iscsit_global)
  571. return -1;
  572. spin_lock_init(&iscsit_global->ts_bitmap_lock);
  573. mutex_init(&auth_id_lock);
  574. idr_init(&tiqn_idr);
  575. ret = target_register_template(&iscsi_ops);
  576. if (ret)
  577. goto out;
  578. size = BITS_TO_LONGS(ISCSIT_BITMAP_BITS) * sizeof(long);
  579. iscsit_global->ts_bitmap = vzalloc(size);
  580. if (!iscsit_global->ts_bitmap)
  581. goto configfs_out;
  582. lio_qr_cache = kmem_cache_create("lio_qr_cache",
  583. sizeof(struct iscsi_queue_req),
  584. __alignof__(struct iscsi_queue_req), 0, NULL);
  585. if (!lio_qr_cache) {
  586. pr_err("Unable to kmem_cache_create() for"
  587. " lio_qr_cache\n");
  588. goto bitmap_out;
  589. }
  590. lio_dr_cache = kmem_cache_create("lio_dr_cache",
  591. sizeof(struct iscsi_datain_req),
  592. __alignof__(struct iscsi_datain_req), 0, NULL);
  593. if (!lio_dr_cache) {
  594. pr_err("Unable to kmem_cache_create() for"
  595. " lio_dr_cache\n");
  596. goto qr_out;
  597. }
  598. lio_ooo_cache = kmem_cache_create("lio_ooo_cache",
  599. sizeof(struct iscsi_ooo_cmdsn),
  600. __alignof__(struct iscsi_ooo_cmdsn), 0, NULL);
  601. if (!lio_ooo_cache) {
  602. pr_err("Unable to kmem_cache_create() for"
  603. " lio_ooo_cache\n");
  604. goto dr_out;
  605. }
  606. lio_r2t_cache = kmem_cache_create("lio_r2t_cache",
  607. sizeof(struct iscsi_r2t), __alignof__(struct iscsi_r2t),
  608. 0, NULL);
  609. if (!lio_r2t_cache) {
  610. pr_err("Unable to kmem_cache_create() for"
  611. " lio_r2t_cache\n");
  612. goto ooo_out;
  613. }
  614. iscsit_register_transport(&iscsi_target_transport);
  615. if (iscsit_load_discovery_tpg() < 0)
  616. goto r2t_out;
  617. return ret;
  618. r2t_out:
  619. iscsit_unregister_transport(&iscsi_target_transport);
  620. kmem_cache_destroy(lio_r2t_cache);
  621. ooo_out:
  622. kmem_cache_destroy(lio_ooo_cache);
  623. dr_out:
  624. kmem_cache_destroy(lio_dr_cache);
  625. qr_out:
  626. kmem_cache_destroy(lio_qr_cache);
  627. bitmap_out:
  628. vfree(iscsit_global->ts_bitmap);
  629. configfs_out:
  630. /* XXX: this probably wants it to be it's own unwind step.. */
  631. if (iscsit_global->discovery_tpg)
  632. iscsit_tpg_disable_portal_group(iscsit_global->discovery_tpg, 1);
  633. target_unregister_template(&iscsi_ops);
  634. out:
  635. kfree(iscsit_global);
  636. return -ENOMEM;
  637. }
  638. static void __exit iscsi_target_cleanup_module(void)
  639. {
  640. iscsit_release_discovery_tpg();
  641. iscsit_unregister_transport(&iscsi_target_transport);
  642. kmem_cache_destroy(lio_qr_cache);
  643. kmem_cache_destroy(lio_dr_cache);
  644. kmem_cache_destroy(lio_ooo_cache);
  645. kmem_cache_destroy(lio_r2t_cache);
  646. /*
  647. * Shutdown discovery sessions and disable discovery TPG
  648. */
  649. if (iscsit_global->discovery_tpg)
  650. iscsit_tpg_disable_portal_group(iscsit_global->discovery_tpg, 1);
  651. target_unregister_template(&iscsi_ops);
  652. vfree(iscsit_global->ts_bitmap);
  653. kfree(iscsit_global);
  654. }
  655. int iscsit_add_reject(
  656. struct iscsi_conn *conn,
  657. u8 reason,
  658. unsigned char *buf)
  659. {
  660. struct iscsi_cmd *cmd;
  661. cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
  662. if (!cmd)
  663. return -1;
  664. cmd->iscsi_opcode = ISCSI_OP_REJECT;
  665. cmd->reject_reason = reason;
  666. cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL);
  667. if (!cmd->buf_ptr) {
  668. pr_err("Unable to allocate memory for cmd->buf_ptr\n");
  669. iscsit_free_cmd(cmd, false);
  670. return -1;
  671. }
  672. spin_lock_bh(&conn->cmd_lock);
  673. list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
  674. spin_unlock_bh(&conn->cmd_lock);
  675. cmd->i_state = ISTATE_SEND_REJECT;
  676. iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
  677. return -1;
  678. }
  679. EXPORT_SYMBOL(iscsit_add_reject);
  680. static int iscsit_add_reject_from_cmd(
  681. struct iscsi_cmd *cmd,
  682. u8 reason,
  683. bool add_to_conn,
  684. unsigned char *buf)
  685. {
  686. struct iscsi_conn *conn;
  687. const bool do_put = cmd->se_cmd.se_tfo != NULL;
  688. if (!cmd->conn) {
  689. pr_err("cmd->conn is NULL for ITT: 0x%08x\n",
  690. cmd->init_task_tag);
  691. return -1;
  692. }
  693. conn = cmd->conn;
  694. cmd->iscsi_opcode = ISCSI_OP_REJECT;
  695. cmd->reject_reason = reason;
  696. cmd->buf_ptr = kmemdup(buf, ISCSI_HDR_LEN, GFP_KERNEL);
  697. if (!cmd->buf_ptr) {
  698. pr_err("Unable to allocate memory for cmd->buf_ptr\n");
  699. iscsit_free_cmd(cmd, false);
  700. return -1;
  701. }
  702. if (add_to_conn) {
  703. spin_lock_bh(&conn->cmd_lock);
  704. list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
  705. spin_unlock_bh(&conn->cmd_lock);
  706. }
  707. cmd->i_state = ISTATE_SEND_REJECT;
  708. iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
  709. /*
  710. * Perform the kref_put now if se_cmd has already been setup by
  711. * scsit_setup_scsi_cmd()
  712. */
  713. if (do_put) {
  714. pr_debug("iscsi reject: calling target_put_sess_cmd >>>>>>\n");
  715. target_put_sess_cmd(&cmd->se_cmd);
  716. }
  717. return -1;
  718. }
  719. static int iscsit_add_reject_cmd(struct iscsi_cmd *cmd, u8 reason,
  720. unsigned char *buf)
  721. {
  722. return iscsit_add_reject_from_cmd(cmd, reason, true, buf);
  723. }
  724. int iscsit_reject_cmd(struct iscsi_cmd *cmd, u8 reason, unsigned char *buf)
  725. {
  726. return iscsit_add_reject_from_cmd(cmd, reason, false, buf);
  727. }
  728. EXPORT_SYMBOL(iscsit_reject_cmd);
  729. /*
  730. * Map some portion of the allocated scatterlist to an iovec, suitable for
  731. * kernel sockets to copy data in/out.
  732. */
  733. static int iscsit_map_iovec(struct iscsi_cmd *cmd, struct kvec *iov, int nvec,
  734. u32 data_offset, u32 data_length)
  735. {
  736. u32 i = 0, orig_data_length = data_length;
  737. struct scatterlist *sg;
  738. unsigned int page_off;
  739. /*
  740. * We know each entry in t_data_sg contains a page.
  741. */
  742. u32 ent = data_offset / PAGE_SIZE;
  743. if (!data_length)
  744. return 0;
  745. if (ent >= cmd->se_cmd.t_data_nents) {
  746. pr_err("Initial page entry out-of-bounds\n");
  747. goto overflow;
  748. }
  749. sg = &cmd->se_cmd.t_data_sg[ent];
  750. page_off = (data_offset % PAGE_SIZE);
  751. cmd->first_data_sg = sg;
  752. cmd->first_data_sg_off = page_off;
  753. while (data_length) {
  754. u32 cur_len;
  755. if (WARN_ON_ONCE(!sg || i >= nvec))
  756. goto overflow;
  757. cur_len = min_t(u32, data_length, sg->length - page_off);
  758. iov[i].iov_base = kmap(sg_page(sg)) + sg->offset + page_off;
  759. iov[i].iov_len = cur_len;
  760. data_length -= cur_len;
  761. page_off = 0;
  762. sg = sg_next(sg);
  763. i++;
  764. }
  765. cmd->kmapped_nents = i;
  766. return i;
  767. overflow:
  768. pr_err("offset %d + length %d overflow; %d/%d; sg-list:\n",
  769. data_offset, orig_data_length, i, nvec);
  770. for_each_sg(cmd->se_cmd.t_data_sg, sg,
  771. cmd->se_cmd.t_data_nents, i) {
  772. pr_err("[%d] off %d len %d\n",
  773. i, sg->offset, sg->length);
  774. }
  775. return -1;
  776. }
  777. static void iscsit_unmap_iovec(struct iscsi_cmd *cmd)
  778. {
  779. u32 i;
  780. struct scatterlist *sg;
  781. sg = cmd->first_data_sg;
  782. for (i = 0; i < cmd->kmapped_nents; i++)
  783. kunmap(sg_page(&sg[i]));
  784. }
  785. static void iscsit_ack_from_expstatsn(struct iscsi_conn *conn, u32 exp_statsn)
  786. {
  787. LIST_HEAD(ack_list);
  788. struct iscsi_cmd *cmd, *cmd_p;
  789. conn->exp_statsn = exp_statsn;
  790. if (conn->sess->sess_ops->RDMAExtensions)
  791. return;
  792. spin_lock_bh(&conn->cmd_lock);
  793. list_for_each_entry_safe(cmd, cmd_p, &conn->conn_cmd_list, i_conn_node) {
  794. spin_lock(&cmd->istate_lock);
  795. if ((cmd->i_state == ISTATE_SENT_STATUS) &&
  796. iscsi_sna_lt(cmd->stat_sn, exp_statsn)) {
  797. cmd->i_state = ISTATE_REMOVE;
  798. spin_unlock(&cmd->istate_lock);
  799. list_move_tail(&cmd->i_conn_node, &ack_list);
  800. continue;
  801. }
  802. spin_unlock(&cmd->istate_lock);
  803. }
  804. spin_unlock_bh(&conn->cmd_lock);
  805. list_for_each_entry_safe(cmd, cmd_p, &ack_list, i_conn_node) {
  806. list_del_init(&cmd->i_conn_node);
  807. iscsit_free_cmd(cmd, false);
  808. }
  809. }
  810. static int iscsit_allocate_iovecs(struct iscsi_cmd *cmd)
  811. {
  812. u32 iov_count = max(1UL, DIV_ROUND_UP(cmd->se_cmd.data_length, PAGE_SIZE));
  813. iov_count += ISCSI_IOV_DATA_BUFFER;
  814. cmd->iov_data = kcalloc(iov_count, sizeof(*cmd->iov_data), GFP_KERNEL);
  815. if (!cmd->iov_data)
  816. return -ENOMEM;
  817. cmd->orig_iov_data_count = iov_count;
  818. return 0;
  819. }
  820. int iscsit_setup_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
  821. unsigned char *buf)
  822. {
  823. int data_direction, payload_length;
  824. struct iscsi_scsi_req *hdr;
  825. int iscsi_task_attr;
  826. int sam_task_attr;
  827. atomic_long_inc(&conn->sess->cmd_pdus);
  828. hdr = (struct iscsi_scsi_req *) buf;
  829. payload_length = ntoh24(hdr->dlength);
  830. /* FIXME; Add checks for AdditionalHeaderSegment */
  831. if (!(hdr->flags & ISCSI_FLAG_CMD_WRITE) &&
  832. !(hdr->flags & ISCSI_FLAG_CMD_FINAL)) {
  833. pr_err("ISCSI_FLAG_CMD_WRITE & ISCSI_FLAG_CMD_FINAL"
  834. " not set. Bad iSCSI Initiator.\n");
  835. return iscsit_add_reject_cmd(cmd,
  836. ISCSI_REASON_BOOKMARK_INVALID, buf);
  837. }
  838. if (((hdr->flags & ISCSI_FLAG_CMD_READ) ||
  839. (hdr->flags & ISCSI_FLAG_CMD_WRITE)) && !hdr->data_length) {
  840. /*
  841. * From RFC-3720 Section 10.3.1:
  842. *
  843. * "Either or both of R and W MAY be 1 when either the
  844. * Expected Data Transfer Length and/or Bidirectional Read
  845. * Expected Data Transfer Length are 0"
  846. *
  847. * For this case, go ahead and clear the unnecssary bits
  848. * to avoid any confusion with ->data_direction.
  849. */
  850. hdr->flags &= ~ISCSI_FLAG_CMD_READ;
  851. hdr->flags &= ~ISCSI_FLAG_CMD_WRITE;
  852. pr_warn("ISCSI_FLAG_CMD_READ or ISCSI_FLAG_CMD_WRITE"
  853. " set when Expected Data Transfer Length is 0 for"
  854. " CDB: 0x%02x, Fixing up flags\n", hdr->cdb[0]);
  855. }
  856. if (!(hdr->flags & ISCSI_FLAG_CMD_READ) &&
  857. !(hdr->flags & ISCSI_FLAG_CMD_WRITE) && (hdr->data_length != 0)) {
  858. pr_err("ISCSI_FLAG_CMD_READ and/or ISCSI_FLAG_CMD_WRITE"
  859. " MUST be set if Expected Data Transfer Length is not 0."
  860. " Bad iSCSI Initiator\n");
  861. return iscsit_add_reject_cmd(cmd,
  862. ISCSI_REASON_BOOKMARK_INVALID, buf);
  863. }
  864. if ((hdr->flags & ISCSI_FLAG_CMD_READ) &&
  865. (hdr->flags & ISCSI_FLAG_CMD_WRITE)) {
  866. pr_err("Bidirectional operations not supported!\n");
  867. return iscsit_add_reject_cmd(cmd,
  868. ISCSI_REASON_BOOKMARK_INVALID, buf);
  869. }
  870. if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
  871. pr_err("Illegally set Immediate Bit in iSCSI Initiator"
  872. " Scsi Command PDU.\n");
  873. return iscsit_add_reject_cmd(cmd,
  874. ISCSI_REASON_BOOKMARK_INVALID, buf);
  875. }
  876. if (payload_length && !conn->sess->sess_ops->ImmediateData) {
  877. pr_err("ImmediateData=No but DataSegmentLength=%u,"
  878. " protocol error.\n", payload_length);
  879. return iscsit_add_reject_cmd(cmd,
  880. ISCSI_REASON_PROTOCOL_ERROR, buf);
  881. }
  882. if ((be32_to_cpu(hdr->data_length) == payload_length) &&
  883. (!(hdr->flags & ISCSI_FLAG_CMD_FINAL))) {
  884. pr_err("Expected Data Transfer Length and Length of"
  885. " Immediate Data are the same, but ISCSI_FLAG_CMD_FINAL"
  886. " bit is not set protocol error\n");
  887. return iscsit_add_reject_cmd(cmd,
  888. ISCSI_REASON_PROTOCOL_ERROR, buf);
  889. }
  890. if (payload_length > be32_to_cpu(hdr->data_length)) {
  891. pr_err("DataSegmentLength: %u is greater than"
  892. " EDTL: %u, protocol error.\n", payload_length,
  893. hdr->data_length);
  894. return iscsit_add_reject_cmd(cmd,
  895. ISCSI_REASON_PROTOCOL_ERROR, buf);
  896. }
  897. if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
  898. pr_err("DataSegmentLength: %u is greater than"
  899. " MaxXmitDataSegmentLength: %u, protocol error.\n",
  900. payload_length, conn->conn_ops->MaxXmitDataSegmentLength);
  901. return iscsit_add_reject_cmd(cmd,
  902. ISCSI_REASON_PROTOCOL_ERROR, buf);
  903. }
  904. if (payload_length > conn->sess->sess_ops->FirstBurstLength) {
  905. pr_err("DataSegmentLength: %u is greater than"
  906. " FirstBurstLength: %u, protocol error.\n",
  907. payload_length, conn->sess->sess_ops->FirstBurstLength);
  908. return iscsit_add_reject_cmd(cmd,
  909. ISCSI_REASON_BOOKMARK_INVALID, buf);
  910. }
  911. data_direction = (hdr->flags & ISCSI_FLAG_CMD_WRITE) ? DMA_TO_DEVICE :
  912. (hdr->flags & ISCSI_FLAG_CMD_READ) ? DMA_FROM_DEVICE :
  913. DMA_NONE;
  914. cmd->data_direction = data_direction;
  915. iscsi_task_attr = hdr->flags & ISCSI_FLAG_CMD_ATTR_MASK;
  916. /*
  917. * Figure out the SAM Task Attribute for the incoming SCSI CDB
  918. */
  919. if ((iscsi_task_attr == ISCSI_ATTR_UNTAGGED) ||
  920. (iscsi_task_attr == ISCSI_ATTR_SIMPLE))
  921. sam_task_attr = TCM_SIMPLE_TAG;
  922. else if (iscsi_task_attr == ISCSI_ATTR_ORDERED)
  923. sam_task_attr = TCM_ORDERED_TAG;
  924. else if (iscsi_task_attr == ISCSI_ATTR_HEAD_OF_QUEUE)
  925. sam_task_attr = TCM_HEAD_TAG;
  926. else if (iscsi_task_attr == ISCSI_ATTR_ACA)
  927. sam_task_attr = TCM_ACA_TAG;
  928. else {
  929. pr_debug("Unknown iSCSI Task Attribute: 0x%02x, using"
  930. " TCM_SIMPLE_TAG\n", iscsi_task_attr);
  931. sam_task_attr = TCM_SIMPLE_TAG;
  932. }
  933. cmd->iscsi_opcode = ISCSI_OP_SCSI_CMD;
  934. cmd->i_state = ISTATE_NEW_CMD;
  935. cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
  936. cmd->immediate_data = (payload_length) ? 1 : 0;
  937. cmd->unsolicited_data = ((!(hdr->flags & ISCSI_FLAG_CMD_FINAL) &&
  938. (hdr->flags & ISCSI_FLAG_CMD_WRITE)) ? 1 : 0);
  939. if (cmd->unsolicited_data)
  940. cmd->cmd_flags |= ICF_NON_IMMEDIATE_UNSOLICITED_DATA;
  941. conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
  942. if (hdr->flags & ISCSI_FLAG_CMD_READ)
  943. cmd->targ_xfer_tag = session_get_next_ttt(conn->sess);
  944. else
  945. cmd->targ_xfer_tag = 0xFFFFFFFF;
  946. cmd->cmd_sn = be32_to_cpu(hdr->cmdsn);
  947. cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn);
  948. cmd->first_burst_len = payload_length;
  949. if (!conn->sess->sess_ops->RDMAExtensions &&
  950. cmd->data_direction == DMA_FROM_DEVICE) {
  951. struct iscsi_datain_req *dr;
  952. dr = iscsit_allocate_datain_req();
  953. if (!dr)
  954. return iscsit_add_reject_cmd(cmd,
  955. ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
  956. iscsit_attach_datain_req(cmd, dr);
  957. }
  958. /*
  959. * Initialize struct se_cmd descriptor from target_core_mod infrastructure
  960. */
  961. transport_init_se_cmd(&cmd->se_cmd, &iscsi_ops,
  962. conn->sess->se_sess, be32_to_cpu(hdr->data_length),
  963. cmd->data_direction, sam_task_attr,
  964. cmd->sense_buffer + 2, scsilun_to_int(&hdr->lun));
  965. pr_debug("Got SCSI Command, ITT: 0x%08x, CmdSN: 0x%08x,"
  966. " ExpXferLen: %u, Length: %u, CID: %hu\n", hdr->itt,
  967. hdr->cmdsn, be32_to_cpu(hdr->data_length), payload_length,
  968. conn->cid);
  969. target_get_sess_cmd(&cmd->se_cmd, true);
  970. cmd->se_cmd.tag = (__force u32)cmd->init_task_tag;
  971. cmd->sense_reason = target_cmd_init_cdb(&cmd->se_cmd, hdr->cdb);
  972. if (cmd->sense_reason) {
  973. if (cmd->sense_reason == TCM_OUT_OF_RESOURCES) {
  974. return iscsit_add_reject_cmd(cmd,
  975. ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
  976. }
  977. goto attach_cmd;
  978. }
  979. cmd->sense_reason = transport_lookup_cmd_lun(&cmd->se_cmd);
  980. if (cmd->sense_reason)
  981. goto attach_cmd;
  982. cmd->sense_reason = target_cmd_parse_cdb(&cmd->se_cmd);
  983. if (cmd->sense_reason)
  984. goto attach_cmd;
  985. if (iscsit_build_pdu_and_seq_lists(cmd, payload_length) < 0) {
  986. return iscsit_add_reject_cmd(cmd,
  987. ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
  988. }
  989. attach_cmd:
  990. spin_lock_bh(&conn->cmd_lock);
  991. list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
  992. spin_unlock_bh(&conn->cmd_lock);
  993. /*
  994. * Check if we need to delay processing because of ALUA
  995. * Active/NonOptimized primary access state..
  996. */
  997. core_alua_check_nonop_delay(&cmd->se_cmd);
  998. return 0;
  999. }
  1000. EXPORT_SYMBOL(iscsit_setup_scsi_cmd);
  1001. void iscsit_set_unsolicited_dataout(struct iscsi_cmd *cmd)
  1002. {
  1003. iscsit_set_dataout_sequence_values(cmd);
  1004. spin_lock_bh(&cmd->dataout_timeout_lock);
  1005. iscsit_start_dataout_timer(cmd, cmd->conn);
  1006. spin_unlock_bh(&cmd->dataout_timeout_lock);
  1007. }
  1008. EXPORT_SYMBOL(iscsit_set_unsolicited_dataout);
  1009. int iscsit_process_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
  1010. struct iscsi_scsi_req *hdr)
  1011. {
  1012. int cmdsn_ret = 0;
  1013. /*
  1014. * Check the CmdSN against ExpCmdSN/MaxCmdSN here if
  1015. * the Immediate Bit is not set, and no Immediate
  1016. * Data is attached.
  1017. *
  1018. * A PDU/CmdSN carrying Immediate Data can only
  1019. * be processed after the DataCRC has passed.
  1020. * If the DataCRC fails, the CmdSN MUST NOT
  1021. * be acknowledged. (See below)
  1022. */
  1023. if (!cmd->immediate_data) {
  1024. cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
  1025. (unsigned char *)hdr, hdr->cmdsn);
  1026. if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
  1027. return -1;
  1028. else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
  1029. target_put_sess_cmd(&cmd->se_cmd);
  1030. return 0;
  1031. }
  1032. }
  1033. iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
  1034. /*
  1035. * If no Immediate Data is attached, it's OK to return now.
  1036. */
  1037. if (!cmd->immediate_data) {
  1038. if (!cmd->sense_reason && cmd->unsolicited_data)
  1039. iscsit_set_unsolicited_dataout(cmd);
  1040. if (!cmd->sense_reason)
  1041. return 0;
  1042. target_put_sess_cmd(&cmd->se_cmd);
  1043. return 0;
  1044. }
  1045. /*
  1046. * Early CHECK_CONDITIONs with ImmediateData never make it to command
  1047. * execution. These exceptions are processed in CmdSN order using
  1048. * iscsit_check_received_cmdsn() in iscsit_get_immediate_data() below.
  1049. */
  1050. if (cmd->sense_reason)
  1051. return 1;
  1052. /*
  1053. * Call directly into transport_generic_new_cmd() to perform
  1054. * the backend memory allocation.
  1055. */
  1056. cmd->sense_reason = transport_generic_new_cmd(&cmd->se_cmd);
  1057. if (cmd->sense_reason)
  1058. return 1;
  1059. return 0;
  1060. }
  1061. EXPORT_SYMBOL(iscsit_process_scsi_cmd);
  1062. static int
  1063. iscsit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
  1064. bool dump_payload)
  1065. {
  1066. int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
  1067. int rc;
  1068. /*
  1069. * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
  1070. */
  1071. if (dump_payload) {
  1072. u32 length = min(cmd->se_cmd.data_length - cmd->write_data_done,
  1073. cmd->first_burst_len);
  1074. pr_debug("Dumping min(%d - %d, %d) = %d bytes of immediate data\n",
  1075. cmd->se_cmd.data_length, cmd->write_data_done,
  1076. cmd->first_burst_len, length);
  1077. rc = iscsit_dump_data_payload(cmd->conn, length, 1);
  1078. pr_debug("Finished dumping immediate data\n");
  1079. if (rc < 0)
  1080. immed_ret = IMMEDIATE_DATA_CANNOT_RECOVER;
  1081. } else {
  1082. immed_ret = iscsit_handle_immediate_data(cmd, hdr,
  1083. cmd->first_burst_len);
  1084. }
  1085. if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
  1086. /*
  1087. * A PDU/CmdSN carrying Immediate Data passed
  1088. * DataCRC, check against ExpCmdSN/MaxCmdSN if
  1089. * Immediate Bit is not set.
  1090. */
  1091. cmdsn_ret = iscsit_sequence_cmd(cmd->conn, cmd,
  1092. (unsigned char *)hdr, hdr->cmdsn);
  1093. if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
  1094. return -1;
  1095. if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
  1096. target_put_sess_cmd(&cmd->se_cmd);
  1097. return 0;
  1098. } else if (cmd->unsolicited_data)
  1099. iscsit_set_unsolicited_dataout(cmd);
  1100. } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
  1101. /*
  1102. * Immediate Data failed DataCRC and ERL>=1,
  1103. * silently drop this PDU and let the initiator
  1104. * plug the CmdSN gap.
  1105. *
  1106. * FIXME: Send Unsolicited NOPIN with reserved
  1107. * TTT here to help the initiator figure out
  1108. * the missing CmdSN, although they should be
  1109. * intelligent enough to determine the missing
  1110. * CmdSN and issue a retry to plug the sequence.
  1111. */
  1112. cmd->i_state = ISTATE_REMOVE;
  1113. iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, cmd->i_state);
  1114. } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
  1115. return -1;
  1116. return 0;
  1117. }
  1118. static int
  1119. iscsit_handle_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
  1120. unsigned char *buf)
  1121. {
  1122. struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
  1123. int rc, immed_data;
  1124. bool dump_payload = false;
  1125. rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
  1126. if (rc < 0)
  1127. return 0;
  1128. /*
  1129. * Allocation iovecs needed for struct socket operations for
  1130. * traditional iSCSI block I/O.
  1131. */
  1132. if (iscsit_allocate_iovecs(cmd) < 0) {
  1133. return iscsit_reject_cmd(cmd,
  1134. ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
  1135. }
  1136. immed_data = cmd->immediate_data;
  1137. rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
  1138. if (rc < 0)
  1139. return rc;
  1140. else if (rc > 0)
  1141. dump_payload = true;
  1142. if (!immed_data)
  1143. return 0;
  1144. return iscsit_get_immediate_data(cmd, hdr, dump_payload);
  1145. }
  1146. static u32 iscsit_do_crypto_hash_sg(
  1147. struct ahash_request *hash,
  1148. struct iscsi_cmd *cmd,
  1149. u32 data_offset,
  1150. u32 data_length,
  1151. u32 padding,
  1152. u8 *pad_bytes)
  1153. {
  1154. u32 data_crc;
  1155. struct scatterlist *sg;
  1156. unsigned int page_off;
  1157. crypto_ahash_init(hash);
  1158. sg = cmd->first_data_sg;
  1159. page_off = cmd->first_data_sg_off;
  1160. if (data_length && page_off) {
  1161. struct scatterlist first_sg;
  1162. u32 len = min_t(u32, data_length, sg->length - page_off);
  1163. sg_init_table(&first_sg, 1);
  1164. sg_set_page(&first_sg, sg_page(sg), len, sg->offset + page_off);
  1165. ahash_request_set_crypt(hash, &first_sg, NULL, len);
  1166. crypto_ahash_update(hash);
  1167. data_length -= len;
  1168. sg = sg_next(sg);
  1169. }
  1170. while (data_length) {
  1171. u32 cur_len = min_t(u32, data_length, sg->length);
  1172. ahash_request_set_crypt(hash, sg, NULL, cur_len);
  1173. crypto_ahash_update(hash);
  1174. data_length -= cur_len;
  1175. /* iscsit_map_iovec has already checked for invalid sg pointers */
  1176. sg = sg_next(sg);
  1177. }
  1178. if (padding) {
  1179. struct scatterlist pad_sg;
  1180. sg_init_one(&pad_sg, pad_bytes, padding);
  1181. ahash_request_set_crypt(hash, &pad_sg, (u8 *)&data_crc,
  1182. padding);
  1183. crypto_ahash_finup(hash);
  1184. } else {
  1185. ahash_request_set_crypt(hash, NULL, (u8 *)&data_crc, 0);
  1186. crypto_ahash_final(hash);
  1187. }
  1188. return data_crc;
  1189. }
  1190. static void iscsit_do_crypto_hash_buf(struct ahash_request *hash,
  1191. const void *buf, u32 payload_length, u32 padding,
  1192. const void *pad_bytes, void *data_crc)
  1193. {
  1194. struct scatterlist sg[2];
  1195. sg_init_table(sg, ARRAY_SIZE(sg));
  1196. sg_set_buf(sg, buf, payload_length);
  1197. if (padding)
  1198. sg_set_buf(sg + 1, pad_bytes, padding);
  1199. ahash_request_set_crypt(hash, sg, data_crc, payload_length + padding);
  1200. crypto_ahash_digest(hash);
  1201. }
  1202. int
  1203. __iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf,
  1204. struct iscsi_cmd *cmd, u32 payload_length,
  1205. bool *success)
  1206. {
  1207. struct iscsi_data *hdr = buf;
  1208. struct se_cmd *se_cmd;
  1209. int rc;
  1210. /* iSCSI write */
  1211. atomic_long_add(payload_length, &conn->sess->rx_data_octets);
  1212. pr_debug("Got DataOut ITT: 0x%08x, TTT: 0x%08x,"
  1213. " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
  1214. hdr->itt, hdr->ttt, hdr->datasn, ntohl(hdr->offset),
  1215. payload_length, conn->cid);
  1216. if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) {
  1217. pr_err("Command ITT: 0x%08x received DataOUT after"
  1218. " last DataOUT received, dumping payload\n",
  1219. cmd->init_task_tag);
  1220. return iscsit_dump_data_payload(conn, payload_length, 1);
  1221. }
  1222. if (cmd->data_direction != DMA_TO_DEVICE) {
  1223. pr_err("Command ITT: 0x%08x received DataOUT for a"
  1224. " NON-WRITE command.\n", cmd->init_task_tag);
  1225. return iscsit_dump_data_payload(conn, payload_length, 1);
  1226. }
  1227. se_cmd = &cmd->se_cmd;
  1228. iscsit_mod_dataout_timer(cmd);
  1229. if ((be32_to_cpu(hdr->offset) + payload_length) > cmd->se_cmd.data_length) {
  1230. pr_err("DataOut Offset: %u, Length %u greater than iSCSI Command EDTL %u, protocol error.\n",
  1231. be32_to_cpu(hdr->offset), payload_length,
  1232. cmd->se_cmd.data_length);
  1233. return iscsit_reject_cmd(cmd, ISCSI_REASON_BOOKMARK_INVALID, buf);
  1234. }
  1235. if (cmd->unsolicited_data) {
  1236. int dump_unsolicited_data = 0;
  1237. if (conn->sess->sess_ops->InitialR2T) {
  1238. pr_err("Received unexpected unsolicited data"
  1239. " while InitialR2T=Yes, protocol error.\n");
  1240. transport_send_check_condition_and_sense(&cmd->se_cmd,
  1241. TCM_UNEXPECTED_UNSOLICITED_DATA, 0);
  1242. return -1;
  1243. }
  1244. /*
  1245. * Special case for dealing with Unsolicited DataOUT
  1246. * and Unsupported SAM WRITE Opcodes and SE resource allocation
  1247. * failures;
  1248. */
  1249. /* Something's amiss if we're not in WRITE_PENDING state... */
  1250. WARN_ON(se_cmd->t_state != TRANSPORT_WRITE_PENDING);
  1251. if (!(se_cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE))
  1252. dump_unsolicited_data = 1;
  1253. if (dump_unsolicited_data) {
  1254. /*
  1255. * Check if a delayed TASK_ABORTED status needs to
  1256. * be sent now if the ISCSI_FLAG_CMD_FINAL has been
  1257. * received with the unsolicited data out.
  1258. */
  1259. if (hdr->flags & ISCSI_FLAG_CMD_FINAL)
  1260. iscsit_stop_dataout_timer(cmd);
  1261. return iscsit_dump_data_payload(conn, payload_length, 1);
  1262. }
  1263. } else {
  1264. /*
  1265. * For the normal solicited data path:
  1266. *
  1267. * Check for a delayed TASK_ABORTED status and dump any
  1268. * incoming data out payload if one exists. Also, when the
  1269. * ISCSI_FLAG_CMD_FINAL is set to denote the end of the current
  1270. * data out sequence, we decrement outstanding_r2ts. Once
  1271. * outstanding_r2ts reaches zero, go ahead and send the delayed
  1272. * TASK_ABORTED status.
  1273. */
  1274. if (se_cmd->transport_state & CMD_T_ABORTED) {
  1275. if (hdr->flags & ISCSI_FLAG_CMD_FINAL &&
  1276. --cmd->outstanding_r2ts < 1)
  1277. iscsit_stop_dataout_timer(cmd);
  1278. return iscsit_dump_data_payload(conn, payload_length, 1);
  1279. }
  1280. }
  1281. /*
  1282. * Perform DataSN, DataSequenceInOrder, DataPDUInOrder, and
  1283. * within-command recovery checks before receiving the payload.
  1284. */
  1285. rc = iscsit_check_pre_dataout(cmd, buf);
  1286. if (rc == DATAOUT_WITHIN_COMMAND_RECOVERY)
  1287. return 0;
  1288. else if (rc == DATAOUT_CANNOT_RECOVER)
  1289. return -1;
  1290. *success = true;
  1291. return 0;
  1292. }
  1293. EXPORT_SYMBOL(__iscsit_check_dataout_hdr);
  1294. int
  1295. iscsit_check_dataout_hdr(struct iscsi_conn *conn, void *buf,
  1296. struct iscsi_cmd **out_cmd)
  1297. {
  1298. struct iscsi_data *hdr = buf;
  1299. struct iscsi_cmd *cmd;
  1300. u32 payload_length = ntoh24(hdr->dlength);
  1301. int rc;
  1302. bool success = false;
  1303. if (!payload_length) {
  1304. pr_warn_ratelimited("DataOUT payload is ZERO, ignoring.\n");
  1305. return 0;
  1306. }
  1307. if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
  1308. pr_err_ratelimited("DataSegmentLength: %u is greater than"
  1309. " MaxXmitDataSegmentLength: %u\n", payload_length,
  1310. conn->conn_ops->MaxXmitDataSegmentLength);
  1311. return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR, buf);
  1312. }
  1313. cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, payload_length);
  1314. if (!cmd)
  1315. return 0;
  1316. rc = __iscsit_check_dataout_hdr(conn, buf, cmd, payload_length, &success);
  1317. if (success)
  1318. *out_cmd = cmd;
  1319. return rc;
  1320. }
  1321. EXPORT_SYMBOL(iscsit_check_dataout_hdr);
  1322. static int
  1323. iscsit_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
  1324. struct iscsi_data *hdr)
  1325. {
  1326. struct kvec *iov;
  1327. u32 checksum, iov_count = 0, padding = 0, rx_got = 0, rx_size = 0;
  1328. u32 payload_length;
  1329. int iov_ret, data_crc_failed = 0;
  1330. payload_length = min_t(u32, cmd->se_cmd.data_length,
  1331. ntoh24(hdr->dlength));
  1332. rx_size += payload_length;
  1333. iov = &cmd->iov_data[0];
  1334. iov_ret = iscsit_map_iovec(cmd, iov, cmd->orig_iov_data_count - 2,
  1335. be32_to_cpu(hdr->offset), payload_length);
  1336. if (iov_ret < 0)
  1337. return -1;
  1338. iov_count += iov_ret;
  1339. padding = ((-payload_length) & 3);
  1340. if (padding != 0) {
  1341. iov[iov_count].iov_base = cmd->pad_bytes;
  1342. iov[iov_count++].iov_len = padding;
  1343. rx_size += padding;
  1344. pr_debug("Receiving %u padding bytes.\n", padding);
  1345. }
  1346. if (conn->conn_ops->DataDigest) {
  1347. iov[iov_count].iov_base = &checksum;
  1348. iov[iov_count++].iov_len = ISCSI_CRC_LEN;
  1349. rx_size += ISCSI_CRC_LEN;
  1350. }
  1351. WARN_ON_ONCE(iov_count > cmd->orig_iov_data_count);
  1352. rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size);
  1353. iscsit_unmap_iovec(cmd);
  1354. if (rx_got != rx_size)
  1355. return -1;
  1356. if (conn->conn_ops->DataDigest) {
  1357. u32 data_crc;
  1358. data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd,
  1359. be32_to_cpu(hdr->offset),
  1360. payload_length, padding,
  1361. cmd->pad_bytes);
  1362. if (checksum != data_crc) {
  1363. pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
  1364. " DataSN: 0x%08x, CRC32C DataDigest 0x%08x"
  1365. " does not match computed 0x%08x\n",
  1366. hdr->itt, hdr->offset, payload_length,
  1367. hdr->datasn, checksum, data_crc);
  1368. data_crc_failed = 1;
  1369. } else {
  1370. pr_debug("Got CRC32C DataDigest 0x%08x for"
  1371. " %u bytes of Data Out\n", checksum,
  1372. payload_length);
  1373. }
  1374. }
  1375. return data_crc_failed;
  1376. }
  1377. int
  1378. iscsit_check_dataout_payload(struct iscsi_cmd *cmd, struct iscsi_data *hdr,
  1379. bool data_crc_failed)
  1380. {
  1381. struct iscsi_conn *conn = cmd->conn;
  1382. int rc, ooo_cmdsn;
  1383. /*
  1384. * Increment post receive data and CRC values or perform
  1385. * within-command recovery.
  1386. */
  1387. rc = iscsit_check_post_dataout(cmd, (unsigned char *)hdr, data_crc_failed);
  1388. if ((rc == DATAOUT_NORMAL) || (rc == DATAOUT_WITHIN_COMMAND_RECOVERY))
  1389. return 0;
  1390. else if (rc == DATAOUT_SEND_R2T) {
  1391. iscsit_set_dataout_sequence_values(cmd);
  1392. conn->conn_transport->iscsit_get_dataout(conn, cmd, false);
  1393. } else if (rc == DATAOUT_SEND_TO_TRANSPORT) {
  1394. /*
  1395. * Handle extra special case for out of order
  1396. * Unsolicited Data Out.
  1397. */
  1398. spin_lock_bh(&cmd->istate_lock);
  1399. ooo_cmdsn = (cmd->cmd_flags & ICF_OOO_CMDSN);
  1400. cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
  1401. cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
  1402. spin_unlock_bh(&cmd->istate_lock);
  1403. iscsit_stop_dataout_timer(cmd);
  1404. if (ooo_cmdsn)
  1405. return 0;
  1406. target_execute_cmd(&cmd->se_cmd);
  1407. return 0;
  1408. } else /* DATAOUT_CANNOT_RECOVER */
  1409. return -1;
  1410. return 0;
  1411. }
  1412. EXPORT_SYMBOL(iscsit_check_dataout_payload);
  1413. static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf)
  1414. {
  1415. struct iscsi_cmd *cmd = NULL;
  1416. struct iscsi_data *hdr = (struct iscsi_data *)buf;
  1417. int rc;
  1418. bool data_crc_failed = false;
  1419. rc = iscsit_check_dataout_hdr(conn, buf, &cmd);
  1420. if (rc < 0)
  1421. return 0;
  1422. else if (!cmd)
  1423. return 0;
  1424. rc = iscsit_get_dataout(conn, cmd, hdr);
  1425. if (rc < 0)
  1426. return rc;
  1427. else if (rc > 0)
  1428. data_crc_failed = true;
  1429. return iscsit_check_dataout_payload(cmd, hdr, data_crc_failed);
  1430. }
  1431. int iscsit_setup_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
  1432. struct iscsi_nopout *hdr)
  1433. {
  1434. u32 payload_length = ntoh24(hdr->dlength);
  1435. if (!(hdr->flags & ISCSI_FLAG_CMD_FINAL)) {
  1436. pr_err("NopOUT Flag's, Left Most Bit not set, protocol error.\n");
  1437. if (!cmd)
  1438. return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
  1439. (unsigned char *)hdr);
  1440. return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
  1441. (unsigned char *)hdr);
  1442. }
  1443. if (hdr->itt == RESERVED_ITT && !(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
  1444. pr_err("NOPOUT ITT is reserved, but Immediate Bit is"
  1445. " not set, protocol error.\n");
  1446. if (!cmd)
  1447. return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
  1448. (unsigned char *)hdr);
  1449. return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
  1450. (unsigned char *)hdr);
  1451. }
  1452. if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
  1453. pr_err("NOPOUT Ping Data DataSegmentLength: %u is"
  1454. " greater than MaxXmitDataSegmentLength: %u, protocol"
  1455. " error.\n", payload_length,
  1456. conn->conn_ops->MaxXmitDataSegmentLength);
  1457. if (!cmd)
  1458. return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
  1459. (unsigned char *)hdr);
  1460. return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
  1461. (unsigned char *)hdr);
  1462. }
  1463. pr_debug("Got NOPOUT Ping %s ITT: 0x%08x, TTT: 0x%08x,"
  1464. " CmdSN: 0x%08x, ExpStatSN: 0x%08x, Length: %u\n",
  1465. hdr->itt == RESERVED_ITT ? "Response" : "Request",
  1466. hdr->itt, hdr->ttt, hdr->cmdsn, hdr->exp_statsn,
  1467. payload_length);
  1468. /*
  1469. * This is not a response to a Unsolicited NopIN, which means
  1470. * it can either be a NOPOUT ping request (with a valid ITT),
  1471. * or a NOPOUT not requesting a NOPIN (with a reserved ITT).
  1472. * Either way, make sure we allocate an struct iscsi_cmd, as both
  1473. * can contain ping data.
  1474. */
  1475. if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
  1476. cmd->iscsi_opcode = ISCSI_OP_NOOP_OUT;
  1477. cmd->i_state = ISTATE_SEND_NOPIN;
  1478. cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ?
  1479. 1 : 0);
  1480. conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
  1481. cmd->targ_xfer_tag = 0xFFFFFFFF;
  1482. cmd->cmd_sn = be32_to_cpu(hdr->cmdsn);
  1483. cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn);
  1484. cmd->data_direction = DMA_NONE;
  1485. }
  1486. return 0;
  1487. }
  1488. EXPORT_SYMBOL(iscsit_setup_nop_out);
  1489. int iscsit_process_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
  1490. struct iscsi_nopout *hdr)
  1491. {
  1492. struct iscsi_cmd *cmd_p = NULL;
  1493. int cmdsn_ret = 0;
  1494. /*
  1495. * Initiator is expecting a NopIN ping reply..
  1496. */
  1497. if (hdr->itt != RESERVED_ITT) {
  1498. if (!cmd)
  1499. return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
  1500. (unsigned char *)hdr);
  1501. spin_lock_bh(&conn->cmd_lock);
  1502. list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
  1503. spin_unlock_bh(&conn->cmd_lock);
  1504. iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
  1505. if (hdr->opcode & ISCSI_OP_IMMEDIATE) {
  1506. iscsit_add_cmd_to_response_queue(cmd, conn,
  1507. cmd->i_state);
  1508. return 0;
  1509. }
  1510. cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
  1511. (unsigned char *)hdr, hdr->cmdsn);
  1512. if (cmdsn_ret == CMDSN_LOWER_THAN_EXP)
  1513. return 0;
  1514. if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
  1515. return -1;
  1516. return 0;
  1517. }
  1518. /*
  1519. * This was a response to a unsolicited NOPIN ping.
  1520. */
  1521. if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
  1522. cmd_p = iscsit_find_cmd_from_ttt(conn, be32_to_cpu(hdr->ttt));
  1523. if (!cmd_p)
  1524. return -EINVAL;
  1525. iscsit_stop_nopin_response_timer(conn);
  1526. cmd_p->i_state = ISTATE_REMOVE;
  1527. iscsit_add_cmd_to_immediate_queue(cmd_p, conn, cmd_p->i_state);
  1528. iscsit_start_nopin_timer(conn);
  1529. return 0;
  1530. }
  1531. /*
  1532. * Otherwise, initiator is not expecting a NOPIN is response.
  1533. * Just ignore for now.
  1534. */
  1535. if (cmd)
  1536. iscsit_free_cmd(cmd, false);
  1537. return 0;
  1538. }
  1539. EXPORT_SYMBOL(iscsit_process_nop_out);
  1540. static int iscsit_handle_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
  1541. unsigned char *buf)
  1542. {
  1543. unsigned char *ping_data = NULL;
  1544. struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf;
  1545. struct kvec *iov = NULL;
  1546. u32 payload_length = ntoh24(hdr->dlength);
  1547. int ret;
  1548. ret = iscsit_setup_nop_out(conn, cmd, hdr);
  1549. if (ret < 0)
  1550. return 0;
  1551. /*
  1552. * Handle NOP-OUT payload for traditional iSCSI sockets
  1553. */
  1554. if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
  1555. u32 checksum, data_crc, padding = 0;
  1556. int niov = 0, rx_got, rx_size = payload_length;
  1557. ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
  1558. if (!ping_data) {
  1559. ret = -1;
  1560. goto out;
  1561. }
  1562. iov = &cmd->iov_misc[0];
  1563. iov[niov].iov_base = ping_data;
  1564. iov[niov++].iov_len = payload_length;
  1565. padding = ((-payload_length) & 3);
  1566. if (padding != 0) {
  1567. pr_debug("Receiving %u additional bytes"
  1568. " for padding.\n", padding);
  1569. iov[niov].iov_base = &cmd->pad_bytes;
  1570. iov[niov++].iov_len = padding;
  1571. rx_size += padding;
  1572. }
  1573. if (conn->conn_ops->DataDigest) {
  1574. iov[niov].iov_base = &checksum;
  1575. iov[niov++].iov_len = ISCSI_CRC_LEN;
  1576. rx_size += ISCSI_CRC_LEN;
  1577. }
  1578. WARN_ON_ONCE(niov > ARRAY_SIZE(cmd->iov_misc));
  1579. rx_got = rx_data(conn, &cmd->iov_misc[0], niov, rx_size);
  1580. if (rx_got != rx_size) {
  1581. ret = -1;
  1582. goto out;
  1583. }
  1584. if (conn->conn_ops->DataDigest) {
  1585. iscsit_do_crypto_hash_buf(conn->conn_rx_hash, ping_data,
  1586. payload_length, padding,
  1587. cmd->pad_bytes, &data_crc);
  1588. if (checksum != data_crc) {
  1589. pr_err("Ping data CRC32C DataDigest"
  1590. " 0x%08x does not match computed 0x%08x\n",
  1591. checksum, data_crc);
  1592. if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
  1593. pr_err("Unable to recover from"
  1594. " NOPOUT Ping DataCRC failure while in"
  1595. " ERL=0.\n");
  1596. ret = -1;
  1597. goto out;
  1598. } else {
  1599. /*
  1600. * Silently drop this PDU and let the
  1601. * initiator plug the CmdSN gap.
  1602. */
  1603. pr_debug("Dropping NOPOUT"
  1604. " Command CmdSN: 0x%08x due to"
  1605. " DataCRC error.\n", hdr->cmdsn);
  1606. ret = 0;
  1607. goto out;
  1608. }
  1609. } else {
  1610. pr_debug("Got CRC32C DataDigest"
  1611. " 0x%08x for %u bytes of ping data.\n",
  1612. checksum, payload_length);
  1613. }
  1614. }
  1615. ping_data[payload_length] = '\0';
  1616. /*
  1617. * Attach ping data to struct iscsi_cmd->buf_ptr.
  1618. */
  1619. cmd->buf_ptr = ping_data;
  1620. cmd->buf_ptr_size = payload_length;
  1621. pr_debug("Got %u bytes of NOPOUT ping"
  1622. " data.\n", payload_length);
  1623. pr_debug("Ping Data: \"%s\"\n", ping_data);
  1624. }
  1625. return iscsit_process_nop_out(conn, cmd, hdr);
  1626. out:
  1627. if (cmd)
  1628. iscsit_free_cmd(cmd, false);
  1629. kfree(ping_data);
  1630. return ret;
  1631. }
  1632. static enum tcm_tmreq_table iscsit_convert_tmf(u8 iscsi_tmf)
  1633. {
  1634. switch (iscsi_tmf) {
  1635. case ISCSI_TM_FUNC_ABORT_TASK:
  1636. return TMR_ABORT_TASK;
  1637. case ISCSI_TM_FUNC_ABORT_TASK_SET:
  1638. return TMR_ABORT_TASK_SET;
  1639. case ISCSI_TM_FUNC_CLEAR_ACA:
  1640. return TMR_CLEAR_ACA;
  1641. case ISCSI_TM_FUNC_CLEAR_TASK_SET:
  1642. return TMR_CLEAR_TASK_SET;
  1643. case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET:
  1644. return TMR_LUN_RESET;
  1645. case ISCSI_TM_FUNC_TARGET_WARM_RESET:
  1646. return TMR_TARGET_WARM_RESET;
  1647. case ISCSI_TM_FUNC_TARGET_COLD_RESET:
  1648. return TMR_TARGET_COLD_RESET;
  1649. default:
  1650. return TMR_UNKNOWN;
  1651. }
  1652. }
  1653. int
  1654. iscsit_handle_task_mgt_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
  1655. unsigned char *buf)
  1656. {
  1657. struct se_tmr_req *se_tmr;
  1658. struct iscsi_tmr_req *tmr_req;
  1659. struct iscsi_tm *hdr;
  1660. int out_of_order_cmdsn = 0, ret;
  1661. u8 function, tcm_function = TMR_UNKNOWN;
  1662. hdr = (struct iscsi_tm *) buf;
  1663. hdr->flags &= ~ISCSI_FLAG_CMD_FINAL;
  1664. function = hdr->flags;
  1665. pr_debug("Got Task Management Request ITT: 0x%08x, CmdSN:"
  1666. " 0x%08x, Function: 0x%02x, RefTaskTag: 0x%08x, RefCmdSN:"
  1667. " 0x%08x, CID: %hu\n", hdr->itt, hdr->cmdsn, function,
  1668. hdr->rtt, hdr->refcmdsn, conn->cid);
  1669. if ((function != ISCSI_TM_FUNC_ABORT_TASK) &&
  1670. ((function != ISCSI_TM_FUNC_TASK_REASSIGN) &&
  1671. hdr->rtt != RESERVED_ITT)) {
  1672. pr_err("RefTaskTag should be set to 0xFFFFFFFF.\n");
  1673. hdr->rtt = RESERVED_ITT;
  1674. }
  1675. if ((function == ISCSI_TM_FUNC_TASK_REASSIGN) &&
  1676. !(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
  1677. pr_err("Task Management Request TASK_REASSIGN not"
  1678. " issued as immediate command, bad iSCSI Initiator"
  1679. "implementation\n");
  1680. return iscsit_add_reject_cmd(cmd,
  1681. ISCSI_REASON_PROTOCOL_ERROR, buf);
  1682. }
  1683. if ((function != ISCSI_TM_FUNC_ABORT_TASK) &&
  1684. be32_to_cpu(hdr->refcmdsn) != ISCSI_RESERVED_TAG)
  1685. hdr->refcmdsn = cpu_to_be32(ISCSI_RESERVED_TAG);
  1686. cmd->data_direction = DMA_NONE;
  1687. cmd->tmr_req = kzalloc(sizeof(*cmd->tmr_req), GFP_KERNEL);
  1688. if (!cmd->tmr_req) {
  1689. return iscsit_add_reject_cmd(cmd,
  1690. ISCSI_REASON_BOOKMARK_NO_RESOURCES,
  1691. buf);
  1692. }
  1693. transport_init_se_cmd(&cmd->se_cmd, &iscsi_ops,
  1694. conn->sess->se_sess, 0, DMA_NONE,
  1695. TCM_SIMPLE_TAG, cmd->sense_buffer + 2,
  1696. scsilun_to_int(&hdr->lun));
  1697. target_get_sess_cmd(&cmd->se_cmd, true);
  1698. /*
  1699. * TASK_REASSIGN for ERL=2 / connection stays inside of
  1700. * LIO-Target $FABRIC_MOD
  1701. */
  1702. if (function != ISCSI_TM_FUNC_TASK_REASSIGN) {
  1703. tcm_function = iscsit_convert_tmf(function);
  1704. if (tcm_function == TMR_UNKNOWN) {
  1705. pr_err("Unknown iSCSI TMR Function:"
  1706. " 0x%02x\n", function);
  1707. return iscsit_add_reject_cmd(cmd,
  1708. ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
  1709. }
  1710. }
  1711. ret = core_tmr_alloc_req(&cmd->se_cmd, cmd->tmr_req, tcm_function,
  1712. GFP_KERNEL);
  1713. if (ret < 0)
  1714. return iscsit_add_reject_cmd(cmd,
  1715. ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
  1716. cmd->tmr_req->se_tmr_req = cmd->se_cmd.se_tmr_req;
  1717. cmd->iscsi_opcode = ISCSI_OP_SCSI_TMFUNC;
  1718. cmd->i_state = ISTATE_SEND_TASKMGTRSP;
  1719. cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
  1720. cmd->init_task_tag = hdr->itt;
  1721. cmd->targ_xfer_tag = 0xFFFFFFFF;
  1722. cmd->cmd_sn = be32_to_cpu(hdr->cmdsn);
  1723. cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn);
  1724. se_tmr = cmd->se_cmd.se_tmr_req;
  1725. tmr_req = cmd->tmr_req;
  1726. /*
  1727. * Locate the struct se_lun for all TMRs not related to ERL=2 TASK_REASSIGN
  1728. */
  1729. if (function != ISCSI_TM_FUNC_TASK_REASSIGN) {
  1730. ret = transport_lookup_tmr_lun(&cmd->se_cmd);
  1731. if (ret < 0) {
  1732. se_tmr->response = ISCSI_TMF_RSP_NO_LUN;
  1733. goto attach;
  1734. }
  1735. }
  1736. switch (function) {
  1737. case ISCSI_TM_FUNC_ABORT_TASK:
  1738. se_tmr->response = iscsit_tmr_abort_task(cmd, buf);
  1739. if (se_tmr->response)
  1740. goto attach;
  1741. break;
  1742. case ISCSI_TM_FUNC_ABORT_TASK_SET:
  1743. case ISCSI_TM_FUNC_CLEAR_ACA:
  1744. case ISCSI_TM_FUNC_CLEAR_TASK_SET:
  1745. case ISCSI_TM_FUNC_LOGICAL_UNIT_RESET:
  1746. break;
  1747. case ISCSI_TM_FUNC_TARGET_WARM_RESET:
  1748. if (iscsit_tmr_task_warm_reset(conn, tmr_req, buf) < 0) {
  1749. se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED;
  1750. goto attach;
  1751. }
  1752. break;
  1753. case ISCSI_TM_FUNC_TARGET_COLD_RESET:
  1754. if (iscsit_tmr_task_cold_reset(conn, tmr_req, buf) < 0) {
  1755. se_tmr->response = ISCSI_TMF_RSP_AUTH_FAILED;
  1756. goto attach;
  1757. }
  1758. break;
  1759. case ISCSI_TM_FUNC_TASK_REASSIGN:
  1760. se_tmr->response = iscsit_tmr_task_reassign(cmd, buf);
  1761. /*
  1762. * Perform sanity checks on the ExpDataSN only if the
  1763. * TASK_REASSIGN was successful.
  1764. */
  1765. if (se_tmr->response)
  1766. break;
  1767. if (iscsit_check_task_reassign_expdatasn(tmr_req, conn) < 0)
  1768. return iscsit_add_reject_cmd(cmd,
  1769. ISCSI_REASON_BOOKMARK_INVALID, buf);
  1770. break;
  1771. default:
  1772. pr_err("Unknown TMR function: 0x%02x, protocol"
  1773. " error.\n", function);
  1774. se_tmr->response = ISCSI_TMF_RSP_NOT_SUPPORTED;
  1775. goto attach;
  1776. }
  1777. if ((function != ISCSI_TM_FUNC_TASK_REASSIGN) &&
  1778. (se_tmr->response == ISCSI_TMF_RSP_COMPLETE))
  1779. se_tmr->call_transport = 1;
  1780. attach:
  1781. spin_lock_bh(&conn->cmd_lock);
  1782. list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
  1783. spin_unlock_bh(&conn->cmd_lock);
  1784. if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
  1785. int cmdsn_ret = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
  1786. if (cmdsn_ret == CMDSN_HIGHER_THAN_EXP) {
  1787. out_of_order_cmdsn = 1;
  1788. } else if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
  1789. target_put_sess_cmd(&cmd->se_cmd);
  1790. return 0;
  1791. } else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) {
  1792. return -1;
  1793. }
  1794. }
  1795. iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
  1796. if (out_of_order_cmdsn || !(hdr->opcode & ISCSI_OP_IMMEDIATE))
  1797. return 0;
  1798. /*
  1799. * Found the referenced task, send to transport for processing.
  1800. */
  1801. if (se_tmr->call_transport)
  1802. return transport_generic_handle_tmr(&cmd->se_cmd);
  1803. /*
  1804. * Could not find the referenced LUN, task, or Task Management
  1805. * command not authorized or supported. Change state and
  1806. * let the tx_thread send the response.
  1807. *
  1808. * For connection recovery, this is also the default action for
  1809. * TMR TASK_REASSIGN.
  1810. */
  1811. iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
  1812. target_put_sess_cmd(&cmd->se_cmd);
  1813. return 0;
  1814. }
  1815. EXPORT_SYMBOL(iscsit_handle_task_mgt_cmd);
  1816. /* #warning FIXME: Support Text Command parameters besides SendTargets */
  1817. int
  1818. iscsit_setup_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
  1819. struct iscsi_text *hdr)
  1820. {
  1821. u32 payload_length = ntoh24(hdr->dlength);
  1822. if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) {
  1823. pr_err("Unable to accept text parameter length: %u"
  1824. "greater than MaxXmitDataSegmentLength %u.\n",
  1825. payload_length, conn->conn_ops->MaxXmitDataSegmentLength);
  1826. return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
  1827. (unsigned char *)hdr);
  1828. }
  1829. if (!(hdr->flags & ISCSI_FLAG_CMD_FINAL) ||
  1830. (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE)) {
  1831. pr_err("Multi sequence text commands currently not supported\n");
  1832. return iscsit_reject_cmd(cmd, ISCSI_REASON_CMD_NOT_SUPPORTED,
  1833. (unsigned char *)hdr);
  1834. }
  1835. pr_debug("Got Text Request: ITT: 0x%08x, CmdSN: 0x%08x,"
  1836. " ExpStatSN: 0x%08x, Length: %u\n", hdr->itt, hdr->cmdsn,
  1837. hdr->exp_statsn, payload_length);
  1838. cmd->iscsi_opcode = ISCSI_OP_TEXT;
  1839. cmd->i_state = ISTATE_SEND_TEXTRSP;
  1840. cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
  1841. conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
  1842. cmd->targ_xfer_tag = 0xFFFFFFFF;
  1843. cmd->cmd_sn = be32_to_cpu(hdr->cmdsn);
  1844. cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn);
  1845. cmd->data_direction = DMA_NONE;
  1846. kfree(cmd->text_in_ptr);
  1847. cmd->text_in_ptr = NULL;
  1848. return 0;
  1849. }
  1850. EXPORT_SYMBOL(iscsit_setup_text_cmd);
  1851. int
  1852. iscsit_process_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
  1853. struct iscsi_text *hdr)
  1854. {
  1855. unsigned char *text_in = cmd->text_in_ptr, *text_ptr;
  1856. int cmdsn_ret;
  1857. if (!text_in) {
  1858. cmd->targ_xfer_tag = be32_to_cpu(hdr->ttt);
  1859. if (cmd->targ_xfer_tag == 0xFFFFFFFF) {
  1860. pr_err("Unable to locate text_in buffer for sendtargets"
  1861. " discovery\n");
  1862. goto reject;
  1863. }
  1864. goto empty_sendtargets;
  1865. }
  1866. if (strncmp("SendTargets=", text_in, 12) != 0) {
  1867. pr_err("Received Text Data that is not"
  1868. " SendTargets, cannot continue.\n");
  1869. goto reject;
  1870. }
  1871. /* '=' confirmed in strncmp */
  1872. text_ptr = strchr(text_in, '=');
  1873. BUG_ON(!text_ptr);
  1874. if (!strncmp("=All", text_ptr, 5)) {
  1875. cmd->cmd_flags |= ICF_SENDTARGETS_ALL;
  1876. } else if (!strncmp("=iqn.", text_ptr, 5) ||
  1877. !strncmp("=eui.", text_ptr, 5)) {
  1878. cmd->cmd_flags |= ICF_SENDTARGETS_SINGLE;
  1879. } else {
  1880. pr_err("Unable to locate valid SendTargets%s value\n",
  1881. text_ptr);
  1882. goto reject;
  1883. }
  1884. spin_lock_bh(&conn->cmd_lock);
  1885. list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
  1886. spin_unlock_bh(&conn->cmd_lock);
  1887. empty_sendtargets:
  1888. iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
  1889. if (!(hdr->opcode & ISCSI_OP_IMMEDIATE)) {
  1890. cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
  1891. (unsigned char *)hdr, hdr->cmdsn);
  1892. if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
  1893. return -1;
  1894. return 0;
  1895. }
  1896. return iscsit_execute_cmd(cmd, 0);
  1897. reject:
  1898. return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
  1899. (unsigned char *)hdr);
  1900. }
  1901. EXPORT_SYMBOL(iscsit_process_text_cmd);
  1902. static int
  1903. iscsit_handle_text_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
  1904. unsigned char *buf)
  1905. {
  1906. struct iscsi_text *hdr = (struct iscsi_text *)buf;
  1907. char *text_in = NULL;
  1908. u32 payload_length = ntoh24(hdr->dlength);
  1909. int rx_size, rc;
  1910. rc = iscsit_setup_text_cmd(conn, cmd, hdr);
  1911. if (rc < 0)
  1912. return 0;
  1913. rx_size = payload_length;
  1914. if (payload_length) {
  1915. u32 checksum = 0, data_crc = 0;
  1916. u32 padding = 0;
  1917. int niov = 0, rx_got;
  1918. struct kvec iov[2];
  1919. rx_size = ALIGN(payload_length, 4);
  1920. text_in = kzalloc(rx_size, GFP_KERNEL);
  1921. if (!text_in)
  1922. goto reject;
  1923. cmd->text_in_ptr = text_in;
  1924. memset(iov, 0, sizeof(iov));
  1925. iov[niov].iov_base = text_in;
  1926. iov[niov++].iov_len = rx_size;
  1927. padding = rx_size - payload_length;
  1928. if (padding)
  1929. pr_debug("Receiving %u additional bytes"
  1930. " for padding.\n", padding);
  1931. if (conn->conn_ops->DataDigest) {
  1932. iov[niov].iov_base = &checksum;
  1933. iov[niov++].iov_len = ISCSI_CRC_LEN;
  1934. rx_size += ISCSI_CRC_LEN;
  1935. }
  1936. WARN_ON_ONCE(niov > ARRAY_SIZE(iov));
  1937. rx_got = rx_data(conn, &iov[0], niov, rx_size);
  1938. if (rx_got != rx_size)
  1939. goto reject;
  1940. if (conn->conn_ops->DataDigest) {
  1941. iscsit_do_crypto_hash_buf(conn->conn_rx_hash,
  1942. text_in, rx_size, 0, NULL,
  1943. &data_crc);
  1944. if (checksum != data_crc) {
  1945. pr_err("Text data CRC32C DataDigest"
  1946. " 0x%08x does not match computed"
  1947. " 0x%08x\n", checksum, data_crc);
  1948. if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
  1949. pr_err("Unable to recover from"
  1950. " Text Data digest failure while in"
  1951. " ERL=0.\n");
  1952. goto reject;
  1953. } else {
  1954. /*
  1955. * Silently drop this PDU and let the
  1956. * initiator plug the CmdSN gap.
  1957. */
  1958. pr_debug("Dropping Text"
  1959. " Command CmdSN: 0x%08x due to"
  1960. " DataCRC error.\n", hdr->cmdsn);
  1961. kfree(text_in);
  1962. return 0;
  1963. }
  1964. } else {
  1965. pr_debug("Got CRC32C DataDigest"
  1966. " 0x%08x for %u bytes of text data.\n",
  1967. checksum, payload_length);
  1968. }
  1969. }
  1970. text_in[payload_length - 1] = '\0';
  1971. pr_debug("Successfully read %d bytes of text"
  1972. " data.\n", payload_length);
  1973. }
  1974. return iscsit_process_text_cmd(conn, cmd, hdr);
  1975. reject:
  1976. kfree(cmd->text_in_ptr);
  1977. cmd->text_in_ptr = NULL;
  1978. return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, buf);
  1979. }
  1980. int iscsit_logout_closesession(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
  1981. {
  1982. struct iscsi_conn *conn_p;
  1983. struct iscsi_session *sess = conn->sess;
  1984. pr_debug("Received logout request CLOSESESSION on CID: %hu"
  1985. " for SID: %u.\n", conn->cid, conn->sess->sid);
  1986. atomic_set(&sess->session_logout, 1);
  1987. atomic_set(&conn->conn_logout_remove, 1);
  1988. conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_SESSION;
  1989. iscsit_inc_conn_usage_count(conn);
  1990. iscsit_inc_session_usage_count(sess);
  1991. spin_lock_bh(&sess->conn_lock);
  1992. list_for_each_entry(conn_p, &sess->sess_conn_list, conn_list) {
  1993. if (conn_p->conn_state != TARG_CONN_STATE_LOGGED_IN)
  1994. continue;
  1995. pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
  1996. conn_p->conn_state = TARG_CONN_STATE_IN_LOGOUT;
  1997. }
  1998. spin_unlock_bh(&sess->conn_lock);
  1999. iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
  2000. return 0;
  2001. }
  2002. int iscsit_logout_closeconnection(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
  2003. {
  2004. struct iscsi_conn *l_conn;
  2005. struct iscsi_session *sess = conn->sess;
  2006. pr_debug("Received logout request CLOSECONNECTION for CID:"
  2007. " %hu on CID: %hu.\n", cmd->logout_cid, conn->cid);
  2008. /*
  2009. * A Logout Request with a CLOSECONNECTION reason code for a CID
  2010. * can arrive on a connection with a differing CID.
  2011. */
  2012. if (conn->cid == cmd->logout_cid) {
  2013. spin_lock_bh(&conn->state_lock);
  2014. pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
  2015. conn->conn_state = TARG_CONN_STATE_IN_LOGOUT;
  2016. atomic_set(&conn->conn_logout_remove, 1);
  2017. conn->conn_logout_reason = ISCSI_LOGOUT_REASON_CLOSE_CONNECTION;
  2018. iscsit_inc_conn_usage_count(conn);
  2019. spin_unlock_bh(&conn->state_lock);
  2020. } else {
  2021. /*
  2022. * Handle all different cid CLOSECONNECTION requests in
  2023. * iscsit_logout_post_handler_diffcid() as to give enough
  2024. * time for any non immediate command's CmdSN to be
  2025. * acknowledged on the connection in question.
  2026. *
  2027. * Here we simply make sure the CID is still around.
  2028. */
  2029. l_conn = iscsit_get_conn_from_cid(sess,
  2030. cmd->logout_cid);
  2031. if (!l_conn) {
  2032. cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND;
  2033. iscsit_add_cmd_to_response_queue(cmd, conn,
  2034. cmd->i_state);
  2035. return 0;
  2036. }
  2037. iscsit_dec_conn_usage_count(l_conn);
  2038. }
  2039. iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
  2040. return 0;
  2041. }
  2042. int iscsit_logout_removeconnforrecovery(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
  2043. {
  2044. struct iscsi_session *sess = conn->sess;
  2045. pr_debug("Received explicit REMOVECONNFORRECOVERY logout for"
  2046. " CID: %hu on CID: %hu.\n", cmd->logout_cid, conn->cid);
  2047. if (sess->sess_ops->ErrorRecoveryLevel != 2) {
  2048. pr_err("Received Logout Request REMOVECONNFORRECOVERY"
  2049. " while ERL!=2.\n");
  2050. cmd->logout_response = ISCSI_LOGOUT_RECOVERY_UNSUPPORTED;
  2051. iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
  2052. return 0;
  2053. }
  2054. if (conn->cid == cmd->logout_cid) {
  2055. pr_err("Received Logout Request REMOVECONNFORRECOVERY"
  2056. " with CID: %hu on CID: %hu, implementation error.\n",
  2057. cmd->logout_cid, conn->cid);
  2058. cmd->logout_response = ISCSI_LOGOUT_CLEANUP_FAILED;
  2059. iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
  2060. return 0;
  2061. }
  2062. iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
  2063. return 0;
  2064. }
  2065. int
  2066. iscsit_handle_logout_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
  2067. unsigned char *buf)
  2068. {
  2069. int cmdsn_ret, logout_remove = 0;
  2070. u8 reason_code = 0;
  2071. struct iscsi_logout *hdr;
  2072. struct iscsi_tiqn *tiqn = iscsit_snmp_get_tiqn(conn);
  2073. hdr = (struct iscsi_logout *) buf;
  2074. reason_code = (hdr->flags & 0x7f);
  2075. if (tiqn) {
  2076. spin_lock(&tiqn->logout_stats.lock);
  2077. if (reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION)
  2078. tiqn->logout_stats.normal_logouts++;
  2079. else
  2080. tiqn->logout_stats.abnormal_logouts++;
  2081. spin_unlock(&tiqn->logout_stats.lock);
  2082. }
  2083. pr_debug("Got Logout Request ITT: 0x%08x CmdSN: 0x%08x"
  2084. " ExpStatSN: 0x%08x Reason: 0x%02x CID: %hu on CID: %hu\n",
  2085. hdr->itt, hdr->cmdsn, hdr->exp_statsn, reason_code,
  2086. hdr->cid, conn->cid);
  2087. if (conn->conn_state != TARG_CONN_STATE_LOGGED_IN) {
  2088. pr_err("Received logout request on connection that"
  2089. " is not in logged in state, ignoring request.\n");
  2090. iscsit_free_cmd(cmd, false);
  2091. return 0;
  2092. }
  2093. cmd->iscsi_opcode = ISCSI_OP_LOGOUT;
  2094. cmd->i_state = ISTATE_SEND_LOGOUTRSP;
  2095. cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0);
  2096. conn->sess->init_task_tag = cmd->init_task_tag = hdr->itt;
  2097. cmd->targ_xfer_tag = 0xFFFFFFFF;
  2098. cmd->cmd_sn = be32_to_cpu(hdr->cmdsn);
  2099. cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn);
  2100. cmd->logout_cid = be16_to_cpu(hdr->cid);
  2101. cmd->logout_reason = reason_code;
  2102. cmd->data_direction = DMA_NONE;
  2103. /*
  2104. * We need to sleep in these cases (by returning 1) until the Logout
  2105. * Response gets sent in the tx thread.
  2106. */
  2107. if ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_SESSION) ||
  2108. ((reason_code == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION) &&
  2109. be16_to_cpu(hdr->cid) == conn->cid))
  2110. logout_remove = 1;
  2111. spin_lock_bh(&conn->cmd_lock);
  2112. list_add_tail(&cmd->i_conn_node, &conn->conn_cmd_list);
  2113. spin_unlock_bh(&conn->cmd_lock);
  2114. if (reason_code != ISCSI_LOGOUT_REASON_RECOVERY)
  2115. iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn));
  2116. /*
  2117. * Immediate commands are executed, well, immediately.
  2118. * Non-Immediate Logout Commands are executed in CmdSN order.
  2119. */
  2120. if (cmd->immediate_cmd) {
  2121. int ret = iscsit_execute_cmd(cmd, 0);
  2122. if (ret < 0)
  2123. return ret;
  2124. } else {
  2125. cmdsn_ret = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
  2126. if (cmdsn_ret == CMDSN_LOWER_THAN_EXP)
  2127. logout_remove = 0;
  2128. else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
  2129. return -1;
  2130. }
  2131. return logout_remove;
  2132. }
  2133. EXPORT_SYMBOL(iscsit_handle_logout_cmd);
  2134. int iscsit_handle_snack(
  2135. struct iscsi_conn *conn,
  2136. unsigned char *buf)
  2137. {
  2138. struct iscsi_snack *hdr;
  2139. hdr = (struct iscsi_snack *) buf;
  2140. hdr->flags &= ~ISCSI_FLAG_CMD_FINAL;
  2141. pr_debug("Got ISCSI_INIT_SNACK, ITT: 0x%08x, ExpStatSN:"
  2142. " 0x%08x, Type: 0x%02x, BegRun: 0x%08x, RunLength: 0x%08x,"
  2143. " CID: %hu\n", hdr->itt, hdr->exp_statsn, hdr->flags,
  2144. hdr->begrun, hdr->runlength, conn->cid);
  2145. if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
  2146. pr_err("Initiator sent SNACK request while in"
  2147. " ErrorRecoveryLevel=0.\n");
  2148. return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
  2149. buf);
  2150. }
  2151. /*
  2152. * SNACK_DATA and SNACK_R2T are both 0, so check which function to
  2153. * call from inside iscsi_send_recovery_datain_or_r2t().
  2154. */
  2155. switch (hdr->flags & ISCSI_FLAG_SNACK_TYPE_MASK) {
  2156. case 0:
  2157. return iscsit_handle_recovery_datain_or_r2t(conn, buf,
  2158. hdr->itt,
  2159. be32_to_cpu(hdr->ttt),
  2160. be32_to_cpu(hdr->begrun),
  2161. be32_to_cpu(hdr->runlength));
  2162. case ISCSI_FLAG_SNACK_TYPE_STATUS:
  2163. return iscsit_handle_status_snack(conn, hdr->itt,
  2164. be32_to_cpu(hdr->ttt),
  2165. be32_to_cpu(hdr->begrun), be32_to_cpu(hdr->runlength));
  2166. case ISCSI_FLAG_SNACK_TYPE_DATA_ACK:
  2167. return iscsit_handle_data_ack(conn, be32_to_cpu(hdr->ttt),
  2168. be32_to_cpu(hdr->begrun),
  2169. be32_to_cpu(hdr->runlength));
  2170. case ISCSI_FLAG_SNACK_TYPE_RDATA:
  2171. /* FIXME: Support R-Data SNACK */
  2172. pr_err("R-Data SNACK Not Supported.\n");
  2173. return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
  2174. buf);
  2175. default:
  2176. pr_err("Unknown SNACK type 0x%02x, protocol"
  2177. " error.\n", hdr->flags & 0x0f);
  2178. return iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
  2179. buf);
  2180. }
  2181. return 0;
  2182. }
  2183. EXPORT_SYMBOL(iscsit_handle_snack);
  2184. static void iscsit_rx_thread_wait_for_tcp(struct iscsi_conn *conn)
  2185. {
  2186. if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) ||
  2187. (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) {
  2188. wait_for_completion_interruptible_timeout(
  2189. &conn->rx_half_close_comp,
  2190. ISCSI_RX_THREAD_TCP_TIMEOUT * HZ);
  2191. }
  2192. }
  2193. static int iscsit_handle_immediate_data(
  2194. struct iscsi_cmd *cmd,
  2195. struct iscsi_scsi_req *hdr,
  2196. u32 length)
  2197. {
  2198. int iov_ret, rx_got = 0, rx_size = 0;
  2199. u32 checksum, iov_count = 0, padding = 0;
  2200. struct iscsi_conn *conn = cmd->conn;
  2201. struct kvec *iov;
  2202. void *overflow_buf = NULL;
  2203. BUG_ON(cmd->write_data_done > cmd->se_cmd.data_length);
  2204. rx_size = min(cmd->se_cmd.data_length - cmd->write_data_done, length);
  2205. iov_ret = iscsit_map_iovec(cmd, cmd->iov_data,
  2206. cmd->orig_iov_data_count - 2,
  2207. cmd->write_data_done, rx_size);
  2208. if (iov_ret < 0)
  2209. return IMMEDIATE_DATA_CANNOT_RECOVER;
  2210. iov_count = iov_ret;
  2211. iov = &cmd->iov_data[0];
  2212. if (rx_size < length) {
  2213. /*
  2214. * Special case: length of immediate data exceeds the data
  2215. * buffer size derived from the CDB.
  2216. */
  2217. overflow_buf = kmalloc(length - rx_size, GFP_KERNEL);
  2218. if (!overflow_buf) {
  2219. iscsit_unmap_iovec(cmd);
  2220. return IMMEDIATE_DATA_CANNOT_RECOVER;
  2221. }
  2222. cmd->overflow_buf = overflow_buf;
  2223. iov[iov_count].iov_base = overflow_buf;
  2224. iov[iov_count].iov_len = length - rx_size;
  2225. iov_count++;
  2226. rx_size = length;
  2227. }
  2228. padding = ((-length) & 3);
  2229. if (padding != 0) {
  2230. iov[iov_count].iov_base = cmd->pad_bytes;
  2231. iov[iov_count++].iov_len = padding;
  2232. rx_size += padding;
  2233. }
  2234. if (conn->conn_ops->DataDigest) {
  2235. iov[iov_count].iov_base = &checksum;
  2236. iov[iov_count++].iov_len = ISCSI_CRC_LEN;
  2237. rx_size += ISCSI_CRC_LEN;
  2238. }
  2239. WARN_ON_ONCE(iov_count > cmd->orig_iov_data_count);
  2240. rx_got = rx_data(conn, &cmd->iov_data[0], iov_count, rx_size);
  2241. iscsit_unmap_iovec(cmd);
  2242. if (rx_got != rx_size) {
  2243. iscsit_rx_thread_wait_for_tcp(conn);
  2244. return IMMEDIATE_DATA_CANNOT_RECOVER;
  2245. }
  2246. if (conn->conn_ops->DataDigest) {
  2247. u32 data_crc;
  2248. data_crc = iscsit_do_crypto_hash_sg(conn->conn_rx_hash, cmd,
  2249. cmd->write_data_done, length, padding,
  2250. cmd->pad_bytes);
  2251. if (checksum != data_crc) {
  2252. pr_err("ImmediateData CRC32C DataDigest 0x%08x"
  2253. " does not match computed 0x%08x\n", checksum,
  2254. data_crc);
  2255. if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
  2256. pr_err("Unable to recover from"
  2257. " Immediate Data digest failure while"
  2258. " in ERL=0.\n");
  2259. iscsit_reject_cmd(cmd,
  2260. ISCSI_REASON_DATA_DIGEST_ERROR,
  2261. (unsigned char *)hdr);
  2262. return IMMEDIATE_DATA_CANNOT_RECOVER;
  2263. } else {
  2264. iscsit_reject_cmd(cmd,
  2265. ISCSI_REASON_DATA_DIGEST_ERROR,
  2266. (unsigned char *)hdr);
  2267. return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
  2268. }
  2269. } else {
  2270. pr_debug("Got CRC32C DataDigest 0x%08x for"
  2271. " %u bytes of Immediate Data\n", checksum,
  2272. length);
  2273. }
  2274. }
  2275. cmd->write_data_done += length;
  2276. if (cmd->write_data_done == cmd->se_cmd.data_length) {
  2277. spin_lock_bh(&cmd->istate_lock);
  2278. cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
  2279. cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
  2280. spin_unlock_bh(&cmd->istate_lock);
  2281. }
  2282. return IMMEDIATE_DATA_NORMAL_OPERATION;
  2283. }
  2284. /* #warning iscsi_build_conn_drop_async_message() only sends out on connections
  2285. with active network interface */
  2286. static void iscsit_build_conn_drop_async_message(struct iscsi_conn *conn)
  2287. {
  2288. struct iscsi_cmd *cmd;
  2289. struct iscsi_conn *conn_p;
  2290. bool found = false;
  2291. lockdep_assert_held(&conn->sess->conn_lock);
  2292. /*
  2293. * Only send a Asynchronous Message on connections whos network
  2294. * interface is still functional.
  2295. */
  2296. list_for_each_entry(conn_p, &conn->sess->sess_conn_list, conn_list) {
  2297. if (conn_p->conn_state == TARG_CONN_STATE_LOGGED_IN) {
  2298. iscsit_inc_conn_usage_count(conn_p);
  2299. found = true;
  2300. break;
  2301. }
  2302. }
  2303. if (!found)
  2304. return;
  2305. cmd = iscsit_allocate_cmd(conn_p, TASK_RUNNING);
  2306. if (!cmd) {
  2307. iscsit_dec_conn_usage_count(conn_p);
  2308. return;
  2309. }
  2310. cmd->logout_cid = conn->cid;
  2311. cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT;
  2312. cmd->i_state = ISTATE_SEND_ASYNCMSG;
  2313. spin_lock_bh(&conn_p->cmd_lock);
  2314. list_add_tail(&cmd->i_conn_node, &conn_p->conn_cmd_list);
  2315. spin_unlock_bh(&conn_p->cmd_lock);
  2316. iscsit_add_cmd_to_response_queue(cmd, conn_p, cmd->i_state);
  2317. iscsit_dec_conn_usage_count(conn_p);
  2318. }
  2319. static int iscsit_send_conn_drop_async_message(
  2320. struct iscsi_cmd *cmd,
  2321. struct iscsi_conn *conn)
  2322. {
  2323. struct iscsi_async *hdr;
  2324. cmd->iscsi_opcode = ISCSI_OP_ASYNC_EVENT;
  2325. hdr = (struct iscsi_async *) cmd->pdu;
  2326. hdr->opcode = ISCSI_OP_ASYNC_EVENT;
  2327. hdr->flags = ISCSI_FLAG_CMD_FINAL;
  2328. cmd->init_task_tag = RESERVED_ITT;
  2329. cmd->targ_xfer_tag = 0xFFFFFFFF;
  2330. put_unaligned_be64(0xFFFFFFFFFFFFFFFFULL, &hdr->rsvd4[0]);
  2331. cmd->stat_sn = conn->stat_sn++;
  2332. hdr->statsn = cpu_to_be32(cmd->stat_sn);
  2333. hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
  2334. hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
  2335. hdr->async_event = ISCSI_ASYNC_MSG_DROPPING_CONNECTION;
  2336. hdr->param1 = cpu_to_be16(cmd->logout_cid);
  2337. hdr->param2 = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Wait);
  2338. hdr->param3 = cpu_to_be16(conn->sess->sess_ops->DefaultTime2Retain);
  2339. pr_debug("Sending Connection Dropped Async Message StatSN:"
  2340. " 0x%08x, for CID: %hu on CID: %hu\n", cmd->stat_sn,
  2341. cmd->logout_cid, conn->cid);
  2342. return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0);
  2343. }
  2344. static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *conn)
  2345. {
  2346. if ((conn->sock->sk->sk_shutdown & SEND_SHUTDOWN) ||
  2347. (conn->sock->sk->sk_shutdown & RCV_SHUTDOWN)) {
  2348. wait_for_completion_interruptible_timeout(
  2349. &conn->tx_half_close_comp,
  2350. ISCSI_TX_THREAD_TCP_TIMEOUT * HZ);
  2351. }
  2352. }
  2353. void
  2354. iscsit_build_datain_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
  2355. struct iscsi_datain *datain, struct iscsi_data_rsp *hdr,
  2356. bool set_statsn)
  2357. {
  2358. hdr->opcode = ISCSI_OP_SCSI_DATA_IN;
  2359. hdr->flags = datain->flags;
  2360. if (hdr->flags & ISCSI_FLAG_DATA_STATUS) {
  2361. if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) {
  2362. hdr->flags |= ISCSI_FLAG_DATA_OVERFLOW;
  2363. hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
  2364. } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) {
  2365. hdr->flags |= ISCSI_FLAG_DATA_UNDERFLOW;
  2366. hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
  2367. }
  2368. }
  2369. hton24(hdr->dlength, datain->length);
  2370. if (hdr->flags & ISCSI_FLAG_DATA_ACK)
  2371. int_to_scsilun(cmd->se_cmd.orig_fe_lun,
  2372. (struct scsi_lun *)&hdr->lun);
  2373. else
  2374. put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
  2375. hdr->itt = cmd->init_task_tag;
  2376. if (hdr->flags & ISCSI_FLAG_DATA_ACK)
  2377. hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag);
  2378. else
  2379. hdr->ttt = cpu_to_be32(0xFFFFFFFF);
  2380. if (set_statsn)
  2381. hdr->statsn = cpu_to_be32(cmd->stat_sn);
  2382. else
  2383. hdr->statsn = cpu_to_be32(0xFFFFFFFF);
  2384. hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
  2385. hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
  2386. hdr->datasn = cpu_to_be32(datain->data_sn);
  2387. hdr->offset = cpu_to_be32(datain->offset);
  2388. pr_debug("Built DataIN ITT: 0x%08x, StatSN: 0x%08x,"
  2389. " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n",
  2390. cmd->init_task_tag, ntohl(hdr->statsn), ntohl(hdr->datasn),
  2391. ntohl(hdr->offset), datain->length, conn->cid);
  2392. }
  2393. EXPORT_SYMBOL(iscsit_build_datain_pdu);
  2394. static int iscsit_send_datain(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
  2395. {
  2396. struct iscsi_data_rsp *hdr = (struct iscsi_data_rsp *)&cmd->pdu[0];
  2397. struct iscsi_datain datain;
  2398. struct iscsi_datain_req *dr;
  2399. int eodr = 0, ret;
  2400. bool set_statsn = false;
  2401. memset(&datain, 0, sizeof(struct iscsi_datain));
  2402. dr = iscsit_get_datain_values(cmd, &datain);
  2403. if (!dr) {
  2404. pr_err("iscsit_get_datain_values failed for ITT: 0x%08x\n",
  2405. cmd->init_task_tag);
  2406. return -1;
  2407. }
  2408. /*
  2409. * Be paranoid and double check the logic for now.
  2410. */
  2411. if ((datain.offset + datain.length) > cmd->se_cmd.data_length) {
  2412. pr_err("Command ITT: 0x%08x, datain.offset: %u and"
  2413. " datain.length: %u exceeds cmd->data_length: %u\n",
  2414. cmd->init_task_tag, datain.offset, datain.length,
  2415. cmd->se_cmd.data_length);
  2416. return -1;
  2417. }
  2418. atomic_long_add(datain.length, &conn->sess->tx_data_octets);
  2419. /*
  2420. * Special case for successfully execution w/ both DATAIN
  2421. * and Sense Data.
  2422. */
  2423. if ((datain.flags & ISCSI_FLAG_DATA_STATUS) &&
  2424. (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE))
  2425. datain.flags &= ~ISCSI_FLAG_DATA_STATUS;
  2426. else {
  2427. if ((dr->dr_complete == DATAIN_COMPLETE_NORMAL) ||
  2428. (dr->dr_complete == DATAIN_COMPLETE_CONNECTION_RECOVERY)) {
  2429. iscsit_increment_maxcmdsn(cmd, conn->sess);
  2430. cmd->stat_sn = conn->stat_sn++;
  2431. set_statsn = true;
  2432. } else if (dr->dr_complete ==
  2433. DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY)
  2434. set_statsn = true;
  2435. }
  2436. iscsit_build_datain_pdu(cmd, conn, &datain, hdr, set_statsn);
  2437. ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, dr, &datain, 0);
  2438. if (ret < 0)
  2439. return ret;
  2440. if (dr->dr_complete) {
  2441. eodr = (cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ?
  2442. 2 : 1;
  2443. iscsit_free_datain_req(cmd, dr);
  2444. }
  2445. return eodr;
  2446. }
  2447. int
  2448. iscsit_build_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
  2449. struct iscsi_logout_rsp *hdr)
  2450. {
  2451. struct iscsi_conn *logout_conn = NULL;
  2452. struct iscsi_conn_recovery *cr = NULL;
  2453. struct iscsi_session *sess = conn->sess;
  2454. /*
  2455. * The actual shutting down of Sessions and/or Connections
  2456. * for CLOSESESSION and CLOSECONNECTION Logout Requests
  2457. * is done in scsi_logout_post_handler().
  2458. */
  2459. switch (cmd->logout_reason) {
  2460. case ISCSI_LOGOUT_REASON_CLOSE_SESSION:
  2461. pr_debug("iSCSI session logout successful, setting"
  2462. " logout response to ISCSI_LOGOUT_SUCCESS.\n");
  2463. cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
  2464. break;
  2465. case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION:
  2466. if (cmd->logout_response == ISCSI_LOGOUT_CID_NOT_FOUND)
  2467. break;
  2468. /*
  2469. * For CLOSECONNECTION logout requests carrying
  2470. * a matching logout CID -> local CID, the reference
  2471. * for the local CID will have been incremented in
  2472. * iscsi_logout_closeconnection().
  2473. *
  2474. * For CLOSECONNECTION logout requests carrying
  2475. * a different CID than the connection it arrived
  2476. * on, the connection responding to cmd->logout_cid
  2477. * is stopped in iscsit_logout_post_handler_diffcid().
  2478. */
  2479. pr_debug("iSCSI CID: %hu logout on CID: %hu"
  2480. " successful.\n", cmd->logout_cid, conn->cid);
  2481. cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
  2482. break;
  2483. case ISCSI_LOGOUT_REASON_RECOVERY:
  2484. if ((cmd->logout_response == ISCSI_LOGOUT_RECOVERY_UNSUPPORTED) ||
  2485. (cmd->logout_response == ISCSI_LOGOUT_CLEANUP_FAILED))
  2486. break;
  2487. /*
  2488. * If the connection is still active from our point of view
  2489. * force connection recovery to occur.
  2490. */
  2491. logout_conn = iscsit_get_conn_from_cid_rcfr(sess,
  2492. cmd->logout_cid);
  2493. if (logout_conn) {
  2494. iscsit_connection_reinstatement_rcfr(logout_conn);
  2495. iscsit_dec_conn_usage_count(logout_conn);
  2496. }
  2497. cr = iscsit_get_inactive_connection_recovery_entry(
  2498. conn->sess, cmd->logout_cid);
  2499. if (!cr) {
  2500. pr_err("Unable to locate CID: %hu for"
  2501. " REMOVECONNFORRECOVERY Logout Request.\n",
  2502. cmd->logout_cid);
  2503. cmd->logout_response = ISCSI_LOGOUT_CID_NOT_FOUND;
  2504. break;
  2505. }
  2506. iscsit_discard_cr_cmds_by_expstatsn(cr, cmd->exp_stat_sn);
  2507. pr_debug("iSCSI REMOVECONNFORRECOVERY logout"
  2508. " for recovery for CID: %hu on CID: %hu successful.\n",
  2509. cmd->logout_cid, conn->cid);
  2510. cmd->logout_response = ISCSI_LOGOUT_SUCCESS;
  2511. break;
  2512. default:
  2513. pr_err("Unknown cmd->logout_reason: 0x%02x\n",
  2514. cmd->logout_reason);
  2515. return -1;
  2516. }
  2517. hdr->opcode = ISCSI_OP_LOGOUT_RSP;
  2518. hdr->flags |= ISCSI_FLAG_CMD_FINAL;
  2519. hdr->response = cmd->logout_response;
  2520. hdr->itt = cmd->init_task_tag;
  2521. cmd->stat_sn = conn->stat_sn++;
  2522. hdr->statsn = cpu_to_be32(cmd->stat_sn);
  2523. iscsit_increment_maxcmdsn(cmd, conn->sess);
  2524. hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
  2525. hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
  2526. pr_debug("Built Logout Response ITT: 0x%08x StatSN:"
  2527. " 0x%08x Response: 0x%02x CID: %hu on CID: %hu\n",
  2528. cmd->init_task_tag, cmd->stat_sn, hdr->response,
  2529. cmd->logout_cid, conn->cid);
  2530. return 0;
  2531. }
  2532. EXPORT_SYMBOL(iscsit_build_logout_rsp);
  2533. static int
  2534. iscsit_send_logout(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
  2535. {
  2536. int rc;
  2537. rc = iscsit_build_logout_rsp(cmd, conn,
  2538. (struct iscsi_logout_rsp *)&cmd->pdu[0]);
  2539. if (rc < 0)
  2540. return rc;
  2541. return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0);
  2542. }
  2543. void
  2544. iscsit_build_nopin_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
  2545. struct iscsi_nopin *hdr, bool nopout_response)
  2546. {
  2547. hdr->opcode = ISCSI_OP_NOOP_IN;
  2548. hdr->flags |= ISCSI_FLAG_CMD_FINAL;
  2549. hton24(hdr->dlength, cmd->buf_ptr_size);
  2550. if (nopout_response)
  2551. put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun);
  2552. hdr->itt = cmd->init_task_tag;
  2553. hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag);
  2554. cmd->stat_sn = (nopout_response) ? conn->stat_sn++ :
  2555. conn->stat_sn;
  2556. hdr->statsn = cpu_to_be32(cmd->stat_sn);
  2557. if (nopout_response)
  2558. iscsit_increment_maxcmdsn(cmd, conn->sess);
  2559. hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
  2560. hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
  2561. pr_debug("Built NOPIN %s Response ITT: 0x%08x, TTT: 0x%08x,"
  2562. " StatSN: 0x%08x, Length %u\n", (nopout_response) ?
  2563. "Solicited" : "Unsolicited", cmd->init_task_tag,
  2564. cmd->targ_xfer_tag, cmd->stat_sn, cmd->buf_ptr_size);
  2565. }
  2566. EXPORT_SYMBOL(iscsit_build_nopin_rsp);
  2567. /*
  2568. * Unsolicited NOPIN, either requesting a response or not.
  2569. */
  2570. static int iscsit_send_unsolicited_nopin(
  2571. struct iscsi_cmd *cmd,
  2572. struct iscsi_conn *conn,
  2573. int want_response)
  2574. {
  2575. struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0];
  2576. int ret;
  2577. iscsit_build_nopin_rsp(cmd, conn, hdr, false);
  2578. pr_debug("Sending Unsolicited NOPIN TTT: 0x%08x StatSN:"
  2579. " 0x%08x CID: %hu\n", hdr->ttt, cmd->stat_sn, conn->cid);
  2580. ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0);
  2581. if (ret < 0)
  2582. return ret;
  2583. spin_lock_bh(&cmd->istate_lock);
  2584. cmd->i_state = want_response ?
  2585. ISTATE_SENT_NOPIN_WANT_RESPONSE : ISTATE_SENT_STATUS;
  2586. spin_unlock_bh(&cmd->istate_lock);
  2587. return 0;
  2588. }
  2589. static int
  2590. iscsit_send_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
  2591. {
  2592. struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0];
  2593. iscsit_build_nopin_rsp(cmd, conn, hdr, true);
  2594. /*
  2595. * NOPOUT Ping Data is attached to struct iscsi_cmd->buf_ptr.
  2596. * NOPOUT DataSegmentLength is at struct iscsi_cmd->buf_ptr_size.
  2597. */
  2598. pr_debug("Echoing back %u bytes of ping data.\n", cmd->buf_ptr_size);
  2599. return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL,
  2600. cmd->buf_ptr,
  2601. cmd->buf_ptr_size);
  2602. }
  2603. static int iscsit_send_r2t(
  2604. struct iscsi_cmd *cmd,
  2605. struct iscsi_conn *conn)
  2606. {
  2607. struct iscsi_r2t *r2t;
  2608. struct iscsi_r2t_rsp *hdr;
  2609. int ret;
  2610. r2t = iscsit_get_r2t_from_list(cmd);
  2611. if (!r2t)
  2612. return -1;
  2613. hdr = (struct iscsi_r2t_rsp *) cmd->pdu;
  2614. memset(hdr, 0, ISCSI_HDR_LEN);
  2615. hdr->opcode = ISCSI_OP_R2T;
  2616. hdr->flags |= ISCSI_FLAG_CMD_FINAL;
  2617. int_to_scsilun(cmd->se_cmd.orig_fe_lun,
  2618. (struct scsi_lun *)&hdr->lun);
  2619. hdr->itt = cmd->init_task_tag;
  2620. if (conn->conn_transport->iscsit_get_r2t_ttt)
  2621. conn->conn_transport->iscsit_get_r2t_ttt(conn, cmd, r2t);
  2622. else
  2623. r2t->targ_xfer_tag = session_get_next_ttt(conn->sess);
  2624. hdr->ttt = cpu_to_be32(r2t->targ_xfer_tag);
  2625. hdr->statsn = cpu_to_be32(conn->stat_sn);
  2626. hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
  2627. hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
  2628. hdr->r2tsn = cpu_to_be32(r2t->r2t_sn);
  2629. hdr->data_offset = cpu_to_be32(r2t->offset);
  2630. hdr->data_length = cpu_to_be32(r2t->xfer_len);
  2631. pr_debug("Built %sR2T, ITT: 0x%08x, TTT: 0x%08x, StatSN:"
  2632. " 0x%08x, R2TSN: 0x%08x, Offset: %u, DDTL: %u, CID: %hu\n",
  2633. (!r2t->recovery_r2t) ? "" : "Recovery ", cmd->init_task_tag,
  2634. r2t->targ_xfer_tag, ntohl(hdr->statsn), r2t->r2t_sn,
  2635. r2t->offset, r2t->xfer_len, conn->cid);
  2636. spin_lock_bh(&cmd->r2t_lock);
  2637. r2t->sent_r2t = 1;
  2638. spin_unlock_bh(&cmd->r2t_lock);
  2639. ret = conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0);
  2640. if (ret < 0) {
  2641. return ret;
  2642. }
  2643. spin_lock_bh(&cmd->dataout_timeout_lock);
  2644. iscsit_start_dataout_timer(cmd, conn);
  2645. spin_unlock_bh(&cmd->dataout_timeout_lock);
  2646. return 0;
  2647. }
  2648. /*
  2649. * @recovery: If called from iscsi_task_reassign_complete_write() for
  2650. * connection recovery.
  2651. */
  2652. int iscsit_build_r2ts_for_cmd(
  2653. struct iscsi_conn *conn,
  2654. struct iscsi_cmd *cmd,
  2655. bool recovery)
  2656. {
  2657. int first_r2t = 1;
  2658. u32 offset = 0, xfer_len = 0;
  2659. spin_lock_bh(&cmd->r2t_lock);
  2660. if (cmd->cmd_flags & ICF_SENT_LAST_R2T) {
  2661. spin_unlock_bh(&cmd->r2t_lock);
  2662. return 0;
  2663. }
  2664. if (conn->sess->sess_ops->DataSequenceInOrder &&
  2665. !recovery)
  2666. cmd->r2t_offset = max(cmd->r2t_offset, cmd->write_data_done);
  2667. while (cmd->outstanding_r2ts < conn->sess->sess_ops->MaxOutstandingR2T) {
  2668. if (conn->sess->sess_ops->DataSequenceInOrder) {
  2669. offset = cmd->r2t_offset;
  2670. if (first_r2t && recovery) {
  2671. int new_data_end = offset +
  2672. conn->sess->sess_ops->MaxBurstLength -
  2673. cmd->next_burst_len;
  2674. if (new_data_end > cmd->se_cmd.data_length)
  2675. xfer_len = cmd->se_cmd.data_length - offset;
  2676. else
  2677. xfer_len =
  2678. conn->sess->sess_ops->MaxBurstLength -
  2679. cmd->next_burst_len;
  2680. } else {
  2681. int new_data_end = offset +
  2682. conn->sess->sess_ops->MaxBurstLength;
  2683. if (new_data_end > cmd->se_cmd.data_length)
  2684. xfer_len = cmd->se_cmd.data_length - offset;
  2685. else
  2686. xfer_len = conn->sess->sess_ops->MaxBurstLength;
  2687. }
  2688. if ((s32)xfer_len < 0) {
  2689. cmd->cmd_flags |= ICF_SENT_LAST_R2T;
  2690. break;
  2691. }
  2692. cmd->r2t_offset += xfer_len;
  2693. if (cmd->r2t_offset == cmd->se_cmd.data_length)
  2694. cmd->cmd_flags |= ICF_SENT_LAST_R2T;
  2695. } else {
  2696. struct iscsi_seq *seq;
  2697. seq = iscsit_get_seq_holder_for_r2t(cmd);
  2698. if (!seq) {
  2699. spin_unlock_bh(&cmd->r2t_lock);
  2700. return -1;
  2701. }
  2702. offset = seq->offset;
  2703. xfer_len = seq->xfer_len;
  2704. if (cmd->seq_send_order == cmd->seq_count)
  2705. cmd->cmd_flags |= ICF_SENT_LAST_R2T;
  2706. }
  2707. cmd->outstanding_r2ts++;
  2708. first_r2t = 0;
  2709. if (iscsit_add_r2t_to_list(cmd, offset, xfer_len, 0, 0) < 0) {
  2710. spin_unlock_bh(&cmd->r2t_lock);
  2711. return -1;
  2712. }
  2713. if (cmd->cmd_flags & ICF_SENT_LAST_R2T)
  2714. break;
  2715. }
  2716. spin_unlock_bh(&cmd->r2t_lock);
  2717. return 0;
  2718. }
  2719. EXPORT_SYMBOL(iscsit_build_r2ts_for_cmd);
  2720. void iscsit_build_rsp_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
  2721. bool inc_stat_sn, struct iscsi_scsi_rsp *hdr)
  2722. {
  2723. if (inc_stat_sn)
  2724. cmd->stat_sn = conn->stat_sn++;
  2725. atomic_long_inc(&conn->sess->rsp_pdus);
  2726. memset(hdr, 0, ISCSI_HDR_LEN);
  2727. hdr->opcode = ISCSI_OP_SCSI_CMD_RSP;
  2728. hdr->flags |= ISCSI_FLAG_CMD_FINAL;
  2729. if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) {
  2730. hdr->flags |= ISCSI_FLAG_CMD_OVERFLOW;
  2731. hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
  2732. } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) {
  2733. hdr->flags |= ISCSI_FLAG_CMD_UNDERFLOW;
  2734. hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count);
  2735. }
  2736. hdr->response = cmd->iscsi_response;
  2737. hdr->cmd_status = cmd->se_cmd.scsi_status;
  2738. hdr->itt = cmd->init_task_tag;
  2739. hdr->statsn = cpu_to_be32(cmd->stat_sn);
  2740. iscsit_increment_maxcmdsn(cmd, conn->sess);
  2741. hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
  2742. hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
  2743. pr_debug("Built SCSI Response, ITT: 0x%08x, StatSN: 0x%08x,"
  2744. " Response: 0x%02x, SAM Status: 0x%02x, CID: %hu\n",
  2745. cmd->init_task_tag, cmd->stat_sn, cmd->se_cmd.scsi_status,
  2746. cmd->se_cmd.scsi_status, conn->cid);
  2747. }
  2748. EXPORT_SYMBOL(iscsit_build_rsp_pdu);
  2749. static int iscsit_send_response(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
  2750. {
  2751. struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)&cmd->pdu[0];
  2752. bool inc_stat_sn = (cmd->i_state == ISTATE_SEND_STATUS);
  2753. void *data_buf = NULL;
  2754. u32 padding = 0, data_buf_len = 0;
  2755. iscsit_build_rsp_pdu(cmd, conn, inc_stat_sn, hdr);
  2756. /*
  2757. * Attach SENSE DATA payload to iSCSI Response PDU
  2758. */
  2759. if (cmd->se_cmd.sense_buffer &&
  2760. ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
  2761. (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
  2762. put_unaligned_be16(cmd->se_cmd.scsi_sense_length, cmd->sense_buffer);
  2763. cmd->se_cmd.scsi_sense_length += sizeof (__be16);
  2764. padding = -(cmd->se_cmd.scsi_sense_length) & 3;
  2765. hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length);
  2766. data_buf = cmd->sense_buffer;
  2767. data_buf_len = cmd->se_cmd.scsi_sense_length + padding;
  2768. if (padding) {
  2769. memset(cmd->sense_buffer +
  2770. cmd->se_cmd.scsi_sense_length, 0, padding);
  2771. pr_debug("Adding %u bytes of padding to"
  2772. " SENSE.\n", padding);
  2773. }
  2774. pr_debug("Attaching SENSE DATA: %u bytes to iSCSI"
  2775. " Response PDU\n",
  2776. cmd->se_cmd.scsi_sense_length);
  2777. }
  2778. return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, data_buf,
  2779. data_buf_len);
  2780. }
  2781. static u8 iscsit_convert_tcm_tmr_rsp(struct se_tmr_req *se_tmr)
  2782. {
  2783. switch (se_tmr->response) {
  2784. case TMR_FUNCTION_COMPLETE:
  2785. return ISCSI_TMF_RSP_COMPLETE;
  2786. case TMR_TASK_DOES_NOT_EXIST:
  2787. return ISCSI_TMF_RSP_NO_TASK;
  2788. case TMR_LUN_DOES_NOT_EXIST:
  2789. return ISCSI_TMF_RSP_NO_LUN;
  2790. case TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED:
  2791. return ISCSI_TMF_RSP_NOT_SUPPORTED;
  2792. case TMR_FUNCTION_REJECTED:
  2793. default:
  2794. return ISCSI_TMF_RSP_REJECTED;
  2795. }
  2796. }
  2797. void
  2798. iscsit_build_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
  2799. struct iscsi_tm_rsp *hdr)
  2800. {
  2801. struct se_tmr_req *se_tmr = cmd->se_cmd.se_tmr_req;
  2802. hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
  2803. hdr->flags = ISCSI_FLAG_CMD_FINAL;
  2804. hdr->response = iscsit_convert_tcm_tmr_rsp(se_tmr);
  2805. hdr->itt = cmd->init_task_tag;
  2806. cmd->stat_sn = conn->stat_sn++;
  2807. hdr->statsn = cpu_to_be32(cmd->stat_sn);
  2808. iscsit_increment_maxcmdsn(cmd, conn->sess);
  2809. hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
  2810. hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
  2811. pr_debug("Built Task Management Response ITT: 0x%08x,"
  2812. " StatSN: 0x%08x, Response: 0x%02x, CID: %hu\n",
  2813. cmd->init_task_tag, cmd->stat_sn, hdr->response, conn->cid);
  2814. }
  2815. EXPORT_SYMBOL(iscsit_build_task_mgt_rsp);
  2816. static int
  2817. iscsit_send_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
  2818. {
  2819. struct iscsi_tm_rsp *hdr = (struct iscsi_tm_rsp *)&cmd->pdu[0];
  2820. iscsit_build_task_mgt_rsp(cmd, conn, hdr);
  2821. return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL, NULL, 0);
  2822. }
  2823. #define SENDTARGETS_BUF_LIMIT 32768U
  2824. static int
  2825. iscsit_build_sendtargets_response(struct iscsi_cmd *cmd,
  2826. enum iscsit_transport_type network_transport,
  2827. int skip_bytes, bool *completed)
  2828. {
  2829. char *payload = NULL;
  2830. struct iscsi_conn *conn = cmd->conn;
  2831. struct iscsi_portal_group *tpg;
  2832. struct iscsi_tiqn *tiqn;
  2833. struct iscsi_tpg_np *tpg_np;
  2834. int buffer_len, end_of_buf = 0, len = 0, payload_len = 0;
  2835. int target_name_printed;
  2836. unsigned char buf[ISCSI_IQN_LEN+12]; /* iqn + "TargetName=" + \0 */
  2837. unsigned char *text_in = cmd->text_in_ptr, *text_ptr = NULL;
  2838. bool active;
  2839. buffer_len = min(conn->conn_ops->MaxRecvDataSegmentLength,
  2840. SENDTARGETS_BUF_LIMIT);
  2841. payload = kzalloc(buffer_len, GFP_KERNEL);
  2842. if (!payload)
  2843. return -ENOMEM;
  2844. /*
  2845. * Locate pointer to iqn./eui. string for ICF_SENDTARGETS_SINGLE
  2846. * explicit case..
  2847. */
  2848. if (cmd->cmd_flags & ICF_SENDTARGETS_SINGLE) {
  2849. text_ptr = strchr(text_in, '=');
  2850. if (!text_ptr) {
  2851. pr_err("Unable to locate '=' string in text_in:"
  2852. " %s\n", text_in);
  2853. kfree(payload);
  2854. return -EINVAL;
  2855. }
  2856. /*
  2857. * Skip over '=' character..
  2858. */
  2859. text_ptr += 1;
  2860. }
  2861. spin_lock(&tiqn_lock);
  2862. list_for_each_entry(tiqn, &g_tiqn_list, tiqn_list) {
  2863. if ((cmd->cmd_flags & ICF_SENDTARGETS_SINGLE) &&
  2864. strcmp(tiqn->tiqn, text_ptr)) {
  2865. continue;
  2866. }
  2867. target_name_printed = 0;
  2868. spin_lock(&tiqn->tiqn_tpg_lock);
  2869. list_for_each_entry(tpg, &tiqn->tiqn_tpg_list, tpg_list) {
  2870. /* If demo_mode_discovery=0 and generate_node_acls=0
  2871. * (demo mode dislabed) do not return
  2872. * TargetName+TargetAddress unless a NodeACL exists.
  2873. */
  2874. if ((tpg->tpg_attrib.generate_node_acls == 0) &&
  2875. (tpg->tpg_attrib.demo_mode_discovery == 0) &&
  2876. (!target_tpg_has_node_acl(&tpg->tpg_se_tpg,
  2877. cmd->conn->sess->sess_ops->InitiatorName))) {
  2878. continue;
  2879. }
  2880. spin_lock(&tpg->tpg_state_lock);
  2881. active = (tpg->tpg_state == TPG_STATE_ACTIVE);
  2882. spin_unlock(&tpg->tpg_state_lock);
  2883. if (!active && tpg->tpg_attrib.tpg_enabled_sendtargets)
  2884. continue;
  2885. spin_lock(&tpg->tpg_np_lock);
  2886. list_for_each_entry(tpg_np, &tpg->tpg_gnp_list,
  2887. tpg_np_list) {
  2888. struct iscsi_np *np = tpg_np->tpg_np;
  2889. struct sockaddr_storage *sockaddr;
  2890. if (np->np_network_transport != network_transport)
  2891. continue;
  2892. if (!target_name_printed) {
  2893. len = sprintf(buf, "TargetName=%s",
  2894. tiqn->tiqn);
  2895. len += 1;
  2896. if ((len + payload_len) > buffer_len) {
  2897. spin_unlock(&tpg->tpg_np_lock);
  2898. spin_unlock(&tiqn->tiqn_tpg_lock);
  2899. end_of_buf = 1;
  2900. goto eob;
  2901. }
  2902. if (skip_bytes && len <= skip_bytes) {
  2903. skip_bytes -= len;
  2904. } else {
  2905. memcpy(payload + payload_len, buf, len);
  2906. payload_len += len;
  2907. target_name_printed = 1;
  2908. if (len > skip_bytes)
  2909. skip_bytes = 0;
  2910. }
  2911. }
  2912. if (inet_addr_is_any((struct sockaddr *)&np->np_sockaddr))
  2913. sockaddr = &conn->local_sockaddr;
  2914. else
  2915. sockaddr = &np->np_sockaddr;
  2916. len = sprintf(buf, "TargetAddress="
  2917. "%pISpc,%hu",
  2918. sockaddr,
  2919. tpg->tpgt);
  2920. len += 1;
  2921. if ((len + payload_len) > buffer_len) {
  2922. spin_unlock(&tpg->tpg_np_lock);
  2923. spin_unlock(&tiqn->tiqn_tpg_lock);
  2924. end_of_buf = 1;
  2925. goto eob;
  2926. }
  2927. if (skip_bytes && len <= skip_bytes) {
  2928. skip_bytes -= len;
  2929. } else {
  2930. memcpy(payload + payload_len, buf, len);
  2931. payload_len += len;
  2932. if (len > skip_bytes)
  2933. skip_bytes = 0;
  2934. }
  2935. }
  2936. spin_unlock(&tpg->tpg_np_lock);
  2937. }
  2938. spin_unlock(&tiqn->tiqn_tpg_lock);
  2939. eob:
  2940. if (end_of_buf) {
  2941. *completed = false;
  2942. break;
  2943. }
  2944. if (cmd->cmd_flags & ICF_SENDTARGETS_SINGLE)
  2945. break;
  2946. }
  2947. spin_unlock(&tiqn_lock);
  2948. cmd->buf_ptr = payload;
  2949. return payload_len;
  2950. }
  2951. int
  2952. iscsit_build_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
  2953. struct iscsi_text_rsp *hdr,
  2954. enum iscsit_transport_type network_transport)
  2955. {
  2956. int text_length, padding;
  2957. bool completed = true;
  2958. text_length = iscsit_build_sendtargets_response(cmd, network_transport,
  2959. cmd->read_data_done,
  2960. &completed);
  2961. if (text_length < 0)
  2962. return text_length;
  2963. if (completed) {
  2964. hdr->flags = ISCSI_FLAG_CMD_FINAL;
  2965. } else {
  2966. hdr->flags = ISCSI_FLAG_TEXT_CONTINUE;
  2967. cmd->read_data_done += text_length;
  2968. if (cmd->targ_xfer_tag == 0xFFFFFFFF)
  2969. cmd->targ_xfer_tag = session_get_next_ttt(conn->sess);
  2970. }
  2971. hdr->opcode = ISCSI_OP_TEXT_RSP;
  2972. padding = ((-text_length) & 3);
  2973. hton24(hdr->dlength, text_length);
  2974. hdr->itt = cmd->init_task_tag;
  2975. hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag);
  2976. cmd->stat_sn = conn->stat_sn++;
  2977. hdr->statsn = cpu_to_be32(cmd->stat_sn);
  2978. iscsit_increment_maxcmdsn(cmd, conn->sess);
  2979. /*
  2980. * Reset maxcmdsn_inc in multi-part text payload exchanges to
  2981. * correctly increment MaxCmdSN for each response answering a
  2982. * non immediate text request with a valid CmdSN.
  2983. */
  2984. cmd->maxcmdsn_inc = 0;
  2985. hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
  2986. hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
  2987. pr_debug("Built Text Response: ITT: 0x%08x, TTT: 0x%08x, StatSN: 0x%08x,"
  2988. " Length: %u, CID: %hu F: %d C: %d\n", cmd->init_task_tag,
  2989. cmd->targ_xfer_tag, cmd->stat_sn, text_length, conn->cid,
  2990. !!(hdr->flags & ISCSI_FLAG_CMD_FINAL),
  2991. !!(hdr->flags & ISCSI_FLAG_TEXT_CONTINUE));
  2992. return text_length + padding;
  2993. }
  2994. EXPORT_SYMBOL(iscsit_build_text_rsp);
  2995. static int iscsit_send_text_rsp(
  2996. struct iscsi_cmd *cmd,
  2997. struct iscsi_conn *conn)
  2998. {
  2999. struct iscsi_text_rsp *hdr = (struct iscsi_text_rsp *)cmd->pdu;
  3000. int text_length;
  3001. text_length = iscsit_build_text_rsp(cmd, conn, hdr,
  3002. conn->conn_transport->transport_type);
  3003. if (text_length < 0)
  3004. return text_length;
  3005. return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL,
  3006. cmd->buf_ptr,
  3007. text_length);
  3008. }
  3009. void
  3010. iscsit_build_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
  3011. struct iscsi_reject *hdr)
  3012. {
  3013. hdr->opcode = ISCSI_OP_REJECT;
  3014. hdr->reason = cmd->reject_reason;
  3015. hdr->flags |= ISCSI_FLAG_CMD_FINAL;
  3016. hton24(hdr->dlength, ISCSI_HDR_LEN);
  3017. hdr->ffffffff = cpu_to_be32(0xffffffff);
  3018. cmd->stat_sn = conn->stat_sn++;
  3019. hdr->statsn = cpu_to_be32(cmd->stat_sn);
  3020. hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn);
  3021. hdr->max_cmdsn = cpu_to_be32((u32) atomic_read(&conn->sess->max_cmd_sn));
  3022. }
  3023. EXPORT_SYMBOL(iscsit_build_reject);
  3024. static int iscsit_send_reject(
  3025. struct iscsi_cmd *cmd,
  3026. struct iscsi_conn *conn)
  3027. {
  3028. struct iscsi_reject *hdr = (struct iscsi_reject *)&cmd->pdu[0];
  3029. iscsit_build_reject(cmd, conn, hdr);
  3030. pr_debug("Built Reject PDU StatSN: 0x%08x, Reason: 0x%02x,"
  3031. " CID: %hu\n", ntohl(hdr->statsn), hdr->reason, conn->cid);
  3032. return conn->conn_transport->iscsit_xmit_pdu(conn, cmd, NULL,
  3033. cmd->buf_ptr,
  3034. ISCSI_HDR_LEN);
  3035. }
  3036. void iscsit_thread_get_cpumask(struct iscsi_conn *conn)
  3037. {
  3038. int ord, cpu;
  3039. /*
  3040. * bitmap_id is assigned from iscsit_global->ts_bitmap from
  3041. * within iscsit_start_kthreads()
  3042. *
  3043. * Here we use bitmap_id to determine which CPU that this
  3044. * iSCSI connection's RX/TX threads will be scheduled to
  3045. * execute upon.
  3046. */
  3047. ord = conn->bitmap_id % cpumask_weight(cpu_online_mask);
  3048. for_each_online_cpu(cpu) {
  3049. if (ord-- == 0) {
  3050. cpumask_set_cpu(cpu, conn->conn_cpumask);
  3051. return;
  3052. }
  3053. }
  3054. /*
  3055. * This should never be reached..
  3056. */
  3057. dump_stack();
  3058. cpumask_setall(conn->conn_cpumask);
  3059. }
  3060. int
  3061. iscsit_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
  3062. {
  3063. int ret;
  3064. switch (state) {
  3065. case ISTATE_SEND_R2T:
  3066. ret = iscsit_send_r2t(cmd, conn);
  3067. if (ret < 0)
  3068. goto err;
  3069. break;
  3070. case ISTATE_REMOVE:
  3071. spin_lock_bh(&conn->cmd_lock);
  3072. list_del_init(&cmd->i_conn_node);
  3073. spin_unlock_bh(&conn->cmd_lock);
  3074. iscsit_free_cmd(cmd, false);
  3075. break;
  3076. case ISTATE_SEND_NOPIN_WANT_RESPONSE:
  3077. iscsit_mod_nopin_response_timer(conn);
  3078. ret = iscsit_send_unsolicited_nopin(cmd, conn, 1);
  3079. if (ret < 0)
  3080. goto err;
  3081. break;
  3082. case ISTATE_SEND_NOPIN_NO_RESPONSE:
  3083. ret = iscsit_send_unsolicited_nopin(cmd, conn, 0);
  3084. if (ret < 0)
  3085. goto err;
  3086. break;
  3087. default:
  3088. pr_err("Unknown Opcode: 0x%02x ITT:"
  3089. " 0x%08x, i_state: %d on CID: %hu\n",
  3090. cmd->iscsi_opcode, cmd->init_task_tag, state,
  3091. conn->cid);
  3092. goto err;
  3093. }
  3094. return 0;
  3095. err:
  3096. return -1;
  3097. }
  3098. EXPORT_SYMBOL(iscsit_immediate_queue);
  3099. static int
  3100. iscsit_handle_immediate_queue(struct iscsi_conn *conn)
  3101. {
  3102. struct iscsit_transport *t = conn->conn_transport;
  3103. struct iscsi_queue_req *qr;
  3104. struct iscsi_cmd *cmd;
  3105. u8 state;
  3106. int ret;
  3107. while ((qr = iscsit_get_cmd_from_immediate_queue(conn))) {
  3108. atomic_set(&conn->check_immediate_queue, 0);
  3109. cmd = qr->cmd;
  3110. state = qr->state;
  3111. kmem_cache_free(lio_qr_cache, qr);
  3112. ret = t->iscsit_immediate_queue(conn, cmd, state);
  3113. if (ret < 0)
  3114. return ret;
  3115. }
  3116. return 0;
  3117. }
  3118. int
  3119. iscsit_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
  3120. {
  3121. int ret;
  3122. check_rsp_state:
  3123. switch (state) {
  3124. case ISTATE_SEND_DATAIN:
  3125. ret = iscsit_send_datain(cmd, conn);
  3126. if (ret < 0)
  3127. goto err;
  3128. else if (!ret)
  3129. /* more drs */
  3130. goto check_rsp_state;
  3131. else if (ret == 1) {
  3132. /* all done */
  3133. spin_lock_bh(&cmd->istate_lock);
  3134. cmd->i_state = ISTATE_SENT_STATUS;
  3135. spin_unlock_bh(&cmd->istate_lock);
  3136. if (atomic_read(&conn->check_immediate_queue))
  3137. return 1;
  3138. return 0;
  3139. } else if (ret == 2) {
  3140. /* Still must send status,
  3141. SCF_TRANSPORT_TASK_SENSE was set */
  3142. spin_lock_bh(&cmd->istate_lock);
  3143. cmd->i_state = ISTATE_SEND_STATUS;
  3144. spin_unlock_bh(&cmd->istate_lock);
  3145. state = ISTATE_SEND_STATUS;
  3146. goto check_rsp_state;
  3147. }
  3148. break;
  3149. case ISTATE_SEND_STATUS:
  3150. case ISTATE_SEND_STATUS_RECOVERY:
  3151. ret = iscsit_send_response(cmd, conn);
  3152. break;
  3153. case ISTATE_SEND_LOGOUTRSP:
  3154. ret = iscsit_send_logout(cmd, conn);
  3155. break;
  3156. case ISTATE_SEND_ASYNCMSG:
  3157. ret = iscsit_send_conn_drop_async_message(
  3158. cmd, conn);
  3159. break;
  3160. case ISTATE_SEND_NOPIN:
  3161. ret = iscsit_send_nopin(cmd, conn);
  3162. break;
  3163. case ISTATE_SEND_REJECT:
  3164. ret = iscsit_send_reject(cmd, conn);
  3165. break;
  3166. case ISTATE_SEND_TASKMGTRSP:
  3167. ret = iscsit_send_task_mgt_rsp(cmd, conn);
  3168. if (ret != 0)
  3169. break;
  3170. ret = iscsit_tmr_post_handler(cmd, conn);
  3171. if (ret != 0)
  3172. iscsit_fall_back_to_erl0(conn->sess);
  3173. break;
  3174. case ISTATE_SEND_TEXTRSP:
  3175. ret = iscsit_send_text_rsp(cmd, conn);
  3176. break;
  3177. default:
  3178. pr_err("Unknown Opcode: 0x%02x ITT:"
  3179. " 0x%08x, i_state: %d on CID: %hu\n",
  3180. cmd->iscsi_opcode, cmd->init_task_tag,
  3181. state, conn->cid);
  3182. goto err;
  3183. }
  3184. if (ret < 0)
  3185. goto err;
  3186. switch (state) {
  3187. case ISTATE_SEND_LOGOUTRSP:
  3188. if (!iscsit_logout_post_handler(cmd, conn))
  3189. return -ECONNRESET;
  3190. fallthrough;
  3191. case ISTATE_SEND_STATUS:
  3192. case ISTATE_SEND_ASYNCMSG:
  3193. case ISTATE_SEND_NOPIN:
  3194. case ISTATE_SEND_STATUS_RECOVERY:
  3195. case ISTATE_SEND_TEXTRSP:
  3196. case ISTATE_SEND_TASKMGTRSP:
  3197. case ISTATE_SEND_REJECT:
  3198. spin_lock_bh(&cmd->istate_lock);
  3199. cmd->i_state = ISTATE_SENT_STATUS;
  3200. spin_unlock_bh(&cmd->istate_lock);
  3201. break;
  3202. default:
  3203. pr_err("Unknown Opcode: 0x%02x ITT:"
  3204. " 0x%08x, i_state: %d on CID: %hu\n",
  3205. cmd->iscsi_opcode, cmd->init_task_tag,
  3206. cmd->i_state, conn->cid);
  3207. goto err;
  3208. }
  3209. if (atomic_read(&conn->check_immediate_queue))
  3210. return 1;
  3211. return 0;
  3212. err:
  3213. return -1;
  3214. }
  3215. EXPORT_SYMBOL(iscsit_response_queue);
  3216. static int iscsit_handle_response_queue(struct iscsi_conn *conn)
  3217. {
  3218. struct iscsit_transport *t = conn->conn_transport;
  3219. struct iscsi_queue_req *qr;
  3220. struct iscsi_cmd *cmd;
  3221. u8 state;
  3222. int ret;
  3223. while ((qr = iscsit_get_cmd_from_response_queue(conn))) {
  3224. cmd = qr->cmd;
  3225. state = qr->state;
  3226. kmem_cache_free(lio_qr_cache, qr);
  3227. ret = t->iscsit_response_queue(conn, cmd, state);
  3228. if (ret == 1 || ret < 0)
  3229. return ret;
  3230. }
  3231. return 0;
  3232. }
  3233. int iscsi_target_tx_thread(void *arg)
  3234. {
  3235. int ret = 0;
  3236. struct iscsi_conn *conn = arg;
  3237. bool conn_freed = false;
  3238. /*
  3239. * Allow ourselves to be interrupted by SIGINT so that a
  3240. * connection recovery / failure event can be triggered externally.
  3241. */
  3242. allow_signal(SIGINT);
  3243. while (!kthread_should_stop()) {
  3244. /*
  3245. * Ensure that both TX and RX per connection kthreads
  3246. * are scheduled to run on the same CPU.
  3247. */
  3248. iscsit_thread_check_cpumask(conn, current, 1);
  3249. wait_event_interruptible(conn->queues_wq,
  3250. !iscsit_conn_all_queues_empty(conn));
  3251. if (signal_pending(current))
  3252. goto transport_err;
  3253. get_immediate:
  3254. ret = iscsit_handle_immediate_queue(conn);
  3255. if (ret < 0)
  3256. goto transport_err;
  3257. ret = iscsit_handle_response_queue(conn);
  3258. if (ret == 1) {
  3259. goto get_immediate;
  3260. } else if (ret == -ECONNRESET) {
  3261. conn_freed = true;
  3262. goto out;
  3263. } else if (ret < 0) {
  3264. goto transport_err;
  3265. }
  3266. }
  3267. transport_err:
  3268. /*
  3269. * Avoid the normal connection failure code-path if this connection
  3270. * is still within LOGIN mode, and iscsi_np process context is
  3271. * responsible for cleaning up the early connection failure.
  3272. */
  3273. if (conn->conn_state != TARG_CONN_STATE_IN_LOGIN)
  3274. iscsit_take_action_for_connection_exit(conn, &conn_freed);
  3275. out:
  3276. if (!conn_freed) {
  3277. while (!kthread_should_stop()) {
  3278. msleep(100);
  3279. }
  3280. }
  3281. return 0;
  3282. }
  3283. static int iscsi_target_rx_opcode(struct iscsi_conn *conn, unsigned char *buf)
  3284. {
  3285. struct iscsi_hdr *hdr = (struct iscsi_hdr *)buf;
  3286. struct iscsi_cmd *cmd;
  3287. int ret = 0;
  3288. switch (hdr->opcode & ISCSI_OPCODE_MASK) {
  3289. case ISCSI_OP_SCSI_CMD:
  3290. cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
  3291. if (!cmd)
  3292. goto reject;
  3293. ret = iscsit_handle_scsi_cmd(conn, cmd, buf);
  3294. break;
  3295. case ISCSI_OP_SCSI_DATA_OUT:
  3296. ret = iscsit_handle_data_out(conn, buf);
  3297. break;
  3298. case ISCSI_OP_NOOP_OUT:
  3299. cmd = NULL;
  3300. if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
  3301. cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
  3302. if (!cmd)
  3303. goto reject;
  3304. }
  3305. ret = iscsit_handle_nop_out(conn, cmd, buf);
  3306. break;
  3307. case ISCSI_OP_SCSI_TMFUNC:
  3308. cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
  3309. if (!cmd)
  3310. goto reject;
  3311. ret = iscsit_handle_task_mgt_cmd(conn, cmd, buf);
  3312. break;
  3313. case ISCSI_OP_TEXT:
  3314. if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
  3315. cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
  3316. if (!cmd)
  3317. goto reject;
  3318. } else {
  3319. cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
  3320. if (!cmd)
  3321. goto reject;
  3322. }
  3323. ret = iscsit_handle_text_cmd(conn, cmd, buf);
  3324. break;
  3325. case ISCSI_OP_LOGOUT:
  3326. cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
  3327. if (!cmd)
  3328. goto reject;
  3329. ret = iscsit_handle_logout_cmd(conn, cmd, buf);
  3330. if (ret > 0)
  3331. wait_for_completion_timeout(&conn->conn_logout_comp,
  3332. SECONDS_FOR_LOGOUT_COMP * HZ);
  3333. break;
  3334. case ISCSI_OP_SNACK:
  3335. ret = iscsit_handle_snack(conn, buf);
  3336. break;
  3337. default:
  3338. pr_err("Got unknown iSCSI OpCode: 0x%02x\n", hdr->opcode);
  3339. if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
  3340. pr_err("Cannot recover from unknown"
  3341. " opcode while ERL=0, closing iSCSI connection.\n");
  3342. return -1;
  3343. }
  3344. pr_err("Unable to recover from unknown opcode while OFMarker=No,"
  3345. " closing iSCSI connection.\n");
  3346. ret = -1;
  3347. break;
  3348. }
  3349. return ret;
  3350. reject:
  3351. return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf);
  3352. }
  3353. static bool iscsi_target_check_conn_state(struct iscsi_conn *conn)
  3354. {
  3355. bool ret;
  3356. spin_lock_bh(&conn->state_lock);
  3357. ret = (conn->conn_state != TARG_CONN_STATE_LOGGED_IN);
  3358. spin_unlock_bh(&conn->state_lock);
  3359. return ret;
  3360. }
  3361. static void iscsit_get_rx_pdu(struct iscsi_conn *conn)
  3362. {
  3363. int ret;
  3364. u8 *buffer, opcode;
  3365. u32 checksum = 0, digest = 0;
  3366. struct kvec iov;
  3367. buffer = kcalloc(ISCSI_HDR_LEN, sizeof(*buffer), GFP_KERNEL);
  3368. if (!buffer)
  3369. return;
  3370. while (!kthread_should_stop()) {
  3371. /*
  3372. * Ensure that both TX and RX per connection kthreads
  3373. * are scheduled to run on the same CPU.
  3374. */
  3375. iscsit_thread_check_cpumask(conn, current, 0);
  3376. memset(&iov, 0, sizeof(struct kvec));
  3377. iov.iov_base = buffer;
  3378. iov.iov_len = ISCSI_HDR_LEN;
  3379. ret = rx_data(conn, &iov, 1, ISCSI_HDR_LEN);
  3380. if (ret != ISCSI_HDR_LEN) {
  3381. iscsit_rx_thread_wait_for_tcp(conn);
  3382. break;
  3383. }
  3384. if (conn->conn_ops->HeaderDigest) {
  3385. iov.iov_base = &digest;
  3386. iov.iov_len = ISCSI_CRC_LEN;
  3387. ret = rx_data(conn, &iov, 1, ISCSI_CRC_LEN);
  3388. if (ret != ISCSI_CRC_LEN) {
  3389. iscsit_rx_thread_wait_for_tcp(conn);
  3390. break;
  3391. }
  3392. iscsit_do_crypto_hash_buf(conn->conn_rx_hash, buffer,
  3393. ISCSI_HDR_LEN, 0, NULL,
  3394. &checksum);
  3395. if (digest != checksum) {
  3396. pr_err("HeaderDigest CRC32C failed,"
  3397. " received 0x%08x, computed 0x%08x\n",
  3398. digest, checksum);
  3399. /*
  3400. * Set the PDU to 0xff so it will intentionally
  3401. * hit default in the switch below.
  3402. */
  3403. memset(buffer, 0xff, ISCSI_HDR_LEN);
  3404. atomic_long_inc(&conn->sess->conn_digest_errors);
  3405. } else {
  3406. pr_debug("Got HeaderDigest CRC32C"
  3407. " 0x%08x\n", checksum);
  3408. }
  3409. }
  3410. if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
  3411. break;
  3412. opcode = buffer[0] & ISCSI_OPCODE_MASK;
  3413. if (conn->sess->sess_ops->SessionType &&
  3414. ((!(opcode & ISCSI_OP_TEXT)) ||
  3415. (!(opcode & ISCSI_OP_LOGOUT)))) {
  3416. pr_err("Received illegal iSCSI Opcode: 0x%02x"
  3417. " while in Discovery Session, rejecting.\n", opcode);
  3418. iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
  3419. buffer);
  3420. break;
  3421. }
  3422. ret = iscsi_target_rx_opcode(conn, buffer);
  3423. if (ret < 0)
  3424. break;
  3425. }
  3426. kfree(buffer);
  3427. }
  3428. int iscsi_target_rx_thread(void *arg)
  3429. {
  3430. int rc;
  3431. struct iscsi_conn *conn = arg;
  3432. bool conn_freed = false;
  3433. /*
  3434. * Allow ourselves to be interrupted by SIGINT so that a
  3435. * connection recovery / failure event can be triggered externally.
  3436. */
  3437. allow_signal(SIGINT);
  3438. /*
  3439. * Wait for iscsi_post_login_handler() to complete before allowing
  3440. * incoming iscsi/tcp socket I/O, and/or failing the connection.
  3441. */
  3442. rc = wait_for_completion_interruptible(&conn->rx_login_comp);
  3443. if (rc < 0 || iscsi_target_check_conn_state(conn))
  3444. goto out;
  3445. if (!conn->conn_transport->iscsit_get_rx_pdu)
  3446. return 0;
  3447. conn->conn_transport->iscsit_get_rx_pdu(conn);
  3448. if (!signal_pending(current))
  3449. atomic_set(&conn->transport_failed, 1);
  3450. iscsit_take_action_for_connection_exit(conn, &conn_freed);
  3451. out:
  3452. if (!conn_freed) {
  3453. while (!kthread_should_stop()) {
  3454. msleep(100);
  3455. }
  3456. }
  3457. return 0;
  3458. }
  3459. static void iscsit_release_commands_from_conn(struct iscsi_conn *conn)
  3460. {
  3461. LIST_HEAD(tmp_list);
  3462. struct iscsi_cmd *cmd = NULL, *cmd_tmp = NULL;
  3463. struct iscsi_session *sess = conn->sess;
  3464. /*
  3465. * We expect this function to only ever be called from either RX or TX
  3466. * thread context via iscsit_close_connection() once the other context
  3467. * has been reset -> returned sleeping pre-handler state.
  3468. */
  3469. spin_lock_bh(&conn->cmd_lock);
  3470. list_splice_init(&conn->conn_cmd_list, &tmp_list);
  3471. list_for_each_entry_safe(cmd, cmd_tmp, &tmp_list, i_conn_node) {
  3472. struct se_cmd *se_cmd = &cmd->se_cmd;
  3473. if (se_cmd->se_tfo != NULL) {
  3474. spin_lock_irq(&se_cmd->t_state_lock);
  3475. if (se_cmd->transport_state & CMD_T_ABORTED) {
  3476. /*
  3477. * LIO's abort path owns the cleanup for this,
  3478. * so put it back on the list and let
  3479. * aborted_task handle it.
  3480. */
  3481. list_move_tail(&cmd->i_conn_node,
  3482. &conn->conn_cmd_list);
  3483. } else {
  3484. se_cmd->transport_state |= CMD_T_FABRIC_STOP;
  3485. }
  3486. spin_unlock_irq(&se_cmd->t_state_lock);
  3487. }
  3488. }
  3489. spin_unlock_bh(&conn->cmd_lock);
  3490. list_for_each_entry_safe(cmd, cmd_tmp, &tmp_list, i_conn_node) {
  3491. list_del_init(&cmd->i_conn_node);
  3492. iscsit_increment_maxcmdsn(cmd, sess);
  3493. iscsit_free_cmd(cmd, true);
  3494. }
  3495. }
  3496. static void iscsit_stop_timers_for_cmds(
  3497. struct iscsi_conn *conn)
  3498. {
  3499. struct iscsi_cmd *cmd;
  3500. spin_lock_bh(&conn->cmd_lock);
  3501. list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
  3502. if (cmd->data_direction == DMA_TO_DEVICE)
  3503. iscsit_stop_dataout_timer(cmd);
  3504. }
  3505. spin_unlock_bh(&conn->cmd_lock);
  3506. }
  3507. int iscsit_close_connection(
  3508. struct iscsi_conn *conn)
  3509. {
  3510. int conn_logout = (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT);
  3511. struct iscsi_session *sess = conn->sess;
  3512. pr_debug("Closing iSCSI connection CID %hu on SID:"
  3513. " %u\n", conn->cid, sess->sid);
  3514. /*
  3515. * Always up conn_logout_comp for the traditional TCP and HW_OFFLOAD
  3516. * case just in case the RX Thread in iscsi_target_rx_opcode() is
  3517. * sleeping and the logout response never got sent because the
  3518. * connection failed.
  3519. *
  3520. * However for iser-target, isert_wait4logout() is using conn_logout_comp
  3521. * to signal logout response TX interrupt completion. Go ahead and skip
  3522. * this for iser since isert_rx_opcode() does not wait on logout failure,
  3523. * and to avoid iscsi_conn pointer dereference in iser-target code.
  3524. */
  3525. if (!conn->conn_transport->rdma_shutdown)
  3526. complete(&conn->conn_logout_comp);
  3527. if (!strcmp(current->comm, ISCSI_RX_THREAD_NAME)) {
  3528. if (conn->tx_thread &&
  3529. cmpxchg(&conn->tx_thread_active, true, false)) {
  3530. send_sig(SIGINT, conn->tx_thread, 1);
  3531. kthread_stop(conn->tx_thread);
  3532. }
  3533. } else if (!strcmp(current->comm, ISCSI_TX_THREAD_NAME)) {
  3534. if (conn->rx_thread &&
  3535. cmpxchg(&conn->rx_thread_active, true, false)) {
  3536. send_sig(SIGINT, conn->rx_thread, 1);
  3537. kthread_stop(conn->rx_thread);
  3538. }
  3539. }
  3540. spin_lock(&iscsit_global->ts_bitmap_lock);
  3541. bitmap_release_region(iscsit_global->ts_bitmap, conn->bitmap_id,
  3542. get_order(1));
  3543. spin_unlock(&iscsit_global->ts_bitmap_lock);
  3544. iscsit_stop_timers_for_cmds(conn);
  3545. iscsit_stop_nopin_response_timer(conn);
  3546. iscsit_stop_nopin_timer(conn);
  3547. if (conn->conn_transport->iscsit_wait_conn)
  3548. conn->conn_transport->iscsit_wait_conn(conn);
  3549. /*
  3550. * During Connection recovery drop unacknowledged out of order
  3551. * commands for this connection, and prepare the other commands
  3552. * for reallegiance.
  3553. *
  3554. * During normal operation clear the out of order commands (but
  3555. * do not free the struct iscsi_ooo_cmdsn's) and release all
  3556. * struct iscsi_cmds.
  3557. */
  3558. if (atomic_read(&conn->connection_recovery)) {
  3559. iscsit_discard_unacknowledged_ooo_cmdsns_for_conn(conn);
  3560. iscsit_prepare_cmds_for_reallegiance(conn);
  3561. } else {
  3562. iscsit_clear_ooo_cmdsns_for_conn(conn);
  3563. iscsit_release_commands_from_conn(conn);
  3564. }
  3565. iscsit_free_queue_reqs_for_conn(conn);
  3566. /*
  3567. * Handle decrementing session or connection usage count if
  3568. * a logout response was not able to be sent because the
  3569. * connection failed. Fall back to Session Recovery here.
  3570. */
  3571. if (atomic_read(&conn->conn_logout_remove)) {
  3572. if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_SESSION) {
  3573. iscsit_dec_conn_usage_count(conn);
  3574. iscsit_dec_session_usage_count(sess);
  3575. }
  3576. if (conn->conn_logout_reason == ISCSI_LOGOUT_REASON_CLOSE_CONNECTION)
  3577. iscsit_dec_conn_usage_count(conn);
  3578. atomic_set(&conn->conn_logout_remove, 0);
  3579. atomic_set(&sess->session_reinstatement, 0);
  3580. atomic_set(&sess->session_fall_back_to_erl0, 1);
  3581. }
  3582. spin_lock_bh(&sess->conn_lock);
  3583. list_del(&conn->conn_list);
  3584. /*
  3585. * Attempt to let the Initiator know this connection failed by
  3586. * sending an Connection Dropped Async Message on another
  3587. * active connection.
  3588. */
  3589. if (atomic_read(&conn->connection_recovery))
  3590. iscsit_build_conn_drop_async_message(conn);
  3591. spin_unlock_bh(&sess->conn_lock);
  3592. /*
  3593. * If connection reinstatement is being performed on this connection,
  3594. * up the connection reinstatement semaphore that is being blocked on
  3595. * in iscsit_cause_connection_reinstatement().
  3596. */
  3597. spin_lock_bh(&conn->state_lock);
  3598. if (atomic_read(&conn->sleep_on_conn_wait_comp)) {
  3599. spin_unlock_bh(&conn->state_lock);
  3600. complete(&conn->conn_wait_comp);
  3601. wait_for_completion(&conn->conn_post_wait_comp);
  3602. spin_lock_bh(&conn->state_lock);
  3603. }
  3604. /*
  3605. * If connection reinstatement is being performed on this connection
  3606. * by receiving a REMOVECONNFORRECOVERY logout request, up the
  3607. * connection wait rcfr semaphore that is being blocked on
  3608. * an iscsit_connection_reinstatement_rcfr().
  3609. */
  3610. if (atomic_read(&conn->connection_wait_rcfr)) {
  3611. spin_unlock_bh(&conn->state_lock);
  3612. complete(&conn->conn_wait_rcfr_comp);
  3613. wait_for_completion(&conn->conn_post_wait_comp);
  3614. spin_lock_bh(&conn->state_lock);
  3615. }
  3616. atomic_set(&conn->connection_reinstatement, 1);
  3617. spin_unlock_bh(&conn->state_lock);
  3618. /*
  3619. * If any other processes are accessing this connection pointer we
  3620. * must wait until they have completed.
  3621. */
  3622. iscsit_check_conn_usage_count(conn);
  3623. ahash_request_free(conn->conn_tx_hash);
  3624. if (conn->conn_rx_hash) {
  3625. struct crypto_ahash *tfm;
  3626. tfm = crypto_ahash_reqtfm(conn->conn_rx_hash);
  3627. ahash_request_free(conn->conn_rx_hash);
  3628. crypto_free_ahash(tfm);
  3629. }
  3630. if (conn->sock)
  3631. sock_release(conn->sock);
  3632. if (conn->conn_transport->iscsit_free_conn)
  3633. conn->conn_transport->iscsit_free_conn(conn);
  3634. pr_debug("Moving to TARG_CONN_STATE_FREE.\n");
  3635. conn->conn_state = TARG_CONN_STATE_FREE;
  3636. iscsit_free_conn(conn);
  3637. spin_lock_bh(&sess->conn_lock);
  3638. atomic_dec(&sess->nconn);
  3639. pr_debug("Decremented iSCSI connection count to %hu from node:"
  3640. " %s\n", atomic_read(&sess->nconn),
  3641. sess->sess_ops->InitiatorName);
  3642. /*
  3643. * Make sure that if one connection fails in an non ERL=2 iSCSI
  3644. * Session that they all fail.
  3645. */
  3646. if ((sess->sess_ops->ErrorRecoveryLevel != 2) && !conn_logout &&
  3647. !atomic_read(&sess->session_logout))
  3648. atomic_set(&sess->session_fall_back_to_erl0, 1);
  3649. /*
  3650. * If this was not the last connection in the session, and we are
  3651. * performing session reinstatement or falling back to ERL=0, call
  3652. * iscsit_stop_session() without sleeping to shutdown the other
  3653. * active connections.
  3654. */
  3655. if (atomic_read(&sess->nconn)) {
  3656. if (!atomic_read(&sess->session_reinstatement) &&
  3657. !atomic_read(&sess->session_fall_back_to_erl0)) {
  3658. spin_unlock_bh(&sess->conn_lock);
  3659. return 0;
  3660. }
  3661. if (!atomic_read(&sess->session_stop_active)) {
  3662. atomic_set(&sess->session_stop_active, 1);
  3663. spin_unlock_bh(&sess->conn_lock);
  3664. iscsit_stop_session(sess, 0, 0);
  3665. return 0;
  3666. }
  3667. spin_unlock_bh(&sess->conn_lock);
  3668. return 0;
  3669. }
  3670. /*
  3671. * If this was the last connection in the session and one of the
  3672. * following is occurring:
  3673. *
  3674. * Session Reinstatement is not being performed, and are falling back
  3675. * to ERL=0 call iscsit_close_session().
  3676. *
  3677. * Session Logout was requested. iscsit_close_session() will be called
  3678. * elsewhere.
  3679. *
  3680. * Session Continuation is not being performed, start the Time2Retain
  3681. * handler and check if sleep_on_sess_wait_sem is active.
  3682. */
  3683. if (!atomic_read(&sess->session_reinstatement) &&
  3684. atomic_read(&sess->session_fall_back_to_erl0)) {
  3685. spin_unlock_bh(&sess->conn_lock);
  3686. complete_all(&sess->session_wait_comp);
  3687. iscsit_close_session(sess);
  3688. return 0;
  3689. } else if (atomic_read(&sess->session_logout)) {
  3690. pr_debug("Moving to TARG_SESS_STATE_FREE.\n");
  3691. sess->session_state = TARG_SESS_STATE_FREE;
  3692. if (atomic_read(&sess->session_close)) {
  3693. spin_unlock_bh(&sess->conn_lock);
  3694. complete_all(&sess->session_wait_comp);
  3695. iscsit_close_session(sess);
  3696. } else {
  3697. spin_unlock_bh(&sess->conn_lock);
  3698. }
  3699. return 0;
  3700. } else {
  3701. pr_debug("Moving to TARG_SESS_STATE_FAILED.\n");
  3702. sess->session_state = TARG_SESS_STATE_FAILED;
  3703. if (!atomic_read(&sess->session_continuation))
  3704. iscsit_start_time2retain_handler(sess);
  3705. if (atomic_read(&sess->session_close)) {
  3706. spin_unlock_bh(&sess->conn_lock);
  3707. complete_all(&sess->session_wait_comp);
  3708. iscsit_close_session(sess);
  3709. } else {
  3710. spin_unlock_bh(&sess->conn_lock);
  3711. }
  3712. return 0;
  3713. }
  3714. }
  3715. /*
  3716. * If the iSCSI Session for the iSCSI Initiator Node exists,
  3717. * forcefully shutdown the iSCSI NEXUS.
  3718. */
  3719. int iscsit_close_session(struct iscsi_session *sess)
  3720. {
  3721. struct iscsi_portal_group *tpg = sess->tpg;
  3722. struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
  3723. if (atomic_read(&sess->nconn)) {
  3724. pr_err("%d connection(s) still exist for iSCSI session"
  3725. " to %s\n", atomic_read(&sess->nconn),
  3726. sess->sess_ops->InitiatorName);
  3727. BUG();
  3728. }
  3729. spin_lock_bh(&se_tpg->session_lock);
  3730. atomic_set(&sess->session_logout, 1);
  3731. atomic_set(&sess->session_reinstatement, 1);
  3732. iscsit_stop_time2retain_timer(sess);
  3733. spin_unlock_bh(&se_tpg->session_lock);
  3734. /*
  3735. * transport_deregister_session_configfs() will clear the
  3736. * struct se_node_acl->nacl_sess pointer now as a iscsi_np process context
  3737. * can be setting it again with __transport_register_session() in
  3738. * iscsi_post_login_handler() again after the iscsit_stop_session()
  3739. * completes in iscsi_np context.
  3740. */
  3741. transport_deregister_session_configfs(sess->se_sess);
  3742. /*
  3743. * If any other processes are accessing this session pointer we must
  3744. * wait until they have completed. If we are in an interrupt (the
  3745. * time2retain handler) and contain and active session usage count we
  3746. * restart the timer and exit.
  3747. */
  3748. if (!in_interrupt()) {
  3749. iscsit_check_session_usage_count(sess);
  3750. } else {
  3751. if (iscsit_check_session_usage_count(sess) == 2) {
  3752. atomic_set(&sess->session_logout, 0);
  3753. iscsit_start_time2retain_handler(sess);
  3754. return 0;
  3755. }
  3756. }
  3757. transport_deregister_session(sess->se_sess);
  3758. if (sess->sess_ops->ErrorRecoveryLevel == 2)
  3759. iscsit_free_connection_recovery_entries(sess);
  3760. iscsit_free_all_ooo_cmdsns(sess);
  3761. spin_lock_bh(&se_tpg->session_lock);
  3762. pr_debug("Moving to TARG_SESS_STATE_FREE.\n");
  3763. sess->session_state = TARG_SESS_STATE_FREE;
  3764. pr_debug("Released iSCSI session from node: %s\n",
  3765. sess->sess_ops->InitiatorName);
  3766. tpg->nsessions--;
  3767. if (tpg->tpg_tiqn)
  3768. tpg->tpg_tiqn->tiqn_nsessions--;
  3769. pr_debug("Decremented number of active iSCSI Sessions on"
  3770. " iSCSI TPG: %hu to %u\n", tpg->tpgt, tpg->nsessions);
  3771. ida_free(&sess_ida, sess->session_index);
  3772. kfree(sess->sess_ops);
  3773. sess->sess_ops = NULL;
  3774. spin_unlock_bh(&se_tpg->session_lock);
  3775. kfree(sess);
  3776. return 0;
  3777. }
  3778. static void iscsit_logout_post_handler_closesession(
  3779. struct iscsi_conn *conn)
  3780. {
  3781. struct iscsi_session *sess = conn->sess;
  3782. int sleep = 1;
  3783. /*
  3784. * Traditional iscsi/tcp will invoke this logic from TX thread
  3785. * context during session logout, so clear tx_thread_active and
  3786. * sleep if iscsit_close_connection() has not already occured.
  3787. *
  3788. * Since iser-target invokes this logic from it's own workqueue,
  3789. * always sleep waiting for RX/TX thread shutdown to complete
  3790. * within iscsit_close_connection().
  3791. */
  3792. if (!conn->conn_transport->rdma_shutdown) {
  3793. sleep = cmpxchg(&conn->tx_thread_active, true, false);
  3794. if (!sleep)
  3795. return;
  3796. }
  3797. atomic_set(&conn->conn_logout_remove, 0);
  3798. complete(&conn->conn_logout_comp);
  3799. iscsit_dec_conn_usage_count(conn);
  3800. atomic_set(&sess->session_close, 1);
  3801. iscsit_stop_session(sess, sleep, sleep);
  3802. iscsit_dec_session_usage_count(sess);
  3803. }
  3804. static void iscsit_logout_post_handler_samecid(
  3805. struct iscsi_conn *conn)
  3806. {
  3807. int sleep = 1;
  3808. if (!conn->conn_transport->rdma_shutdown) {
  3809. sleep = cmpxchg(&conn->tx_thread_active, true, false);
  3810. if (!sleep)
  3811. return;
  3812. }
  3813. atomic_set(&conn->conn_logout_remove, 0);
  3814. complete(&conn->conn_logout_comp);
  3815. iscsit_cause_connection_reinstatement(conn, sleep);
  3816. iscsit_dec_conn_usage_count(conn);
  3817. }
  3818. static void iscsit_logout_post_handler_diffcid(
  3819. struct iscsi_conn *conn,
  3820. u16 cid)
  3821. {
  3822. struct iscsi_conn *l_conn;
  3823. struct iscsi_session *sess = conn->sess;
  3824. bool conn_found = false;
  3825. if (!sess)
  3826. return;
  3827. spin_lock_bh(&sess->conn_lock);
  3828. list_for_each_entry(l_conn, &sess->sess_conn_list, conn_list) {
  3829. if (l_conn->cid == cid) {
  3830. iscsit_inc_conn_usage_count(l_conn);
  3831. conn_found = true;
  3832. break;
  3833. }
  3834. }
  3835. spin_unlock_bh(&sess->conn_lock);
  3836. if (!conn_found)
  3837. return;
  3838. if (l_conn->sock)
  3839. l_conn->sock->ops->shutdown(l_conn->sock, RCV_SHUTDOWN);
  3840. spin_lock_bh(&l_conn->state_lock);
  3841. pr_debug("Moving to TARG_CONN_STATE_IN_LOGOUT.\n");
  3842. l_conn->conn_state = TARG_CONN_STATE_IN_LOGOUT;
  3843. spin_unlock_bh(&l_conn->state_lock);
  3844. iscsit_cause_connection_reinstatement(l_conn, 1);
  3845. iscsit_dec_conn_usage_count(l_conn);
  3846. }
  3847. /*
  3848. * Return of 0 causes the TX thread to restart.
  3849. */
  3850. int iscsit_logout_post_handler(
  3851. struct iscsi_cmd *cmd,
  3852. struct iscsi_conn *conn)
  3853. {
  3854. int ret = 0;
  3855. switch (cmd->logout_reason) {
  3856. case ISCSI_LOGOUT_REASON_CLOSE_SESSION:
  3857. switch (cmd->logout_response) {
  3858. case ISCSI_LOGOUT_SUCCESS:
  3859. case ISCSI_LOGOUT_CLEANUP_FAILED:
  3860. default:
  3861. iscsit_logout_post_handler_closesession(conn);
  3862. break;
  3863. }
  3864. break;
  3865. case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION:
  3866. if (conn->cid == cmd->logout_cid) {
  3867. switch (cmd->logout_response) {
  3868. case ISCSI_LOGOUT_SUCCESS:
  3869. case ISCSI_LOGOUT_CLEANUP_FAILED:
  3870. default:
  3871. iscsit_logout_post_handler_samecid(conn);
  3872. break;
  3873. }
  3874. } else {
  3875. switch (cmd->logout_response) {
  3876. case ISCSI_LOGOUT_SUCCESS:
  3877. iscsit_logout_post_handler_diffcid(conn,
  3878. cmd->logout_cid);
  3879. break;
  3880. case ISCSI_LOGOUT_CID_NOT_FOUND:
  3881. case ISCSI_LOGOUT_CLEANUP_FAILED:
  3882. default:
  3883. break;
  3884. }
  3885. ret = 1;
  3886. }
  3887. break;
  3888. case ISCSI_LOGOUT_REASON_RECOVERY:
  3889. switch (cmd->logout_response) {
  3890. case ISCSI_LOGOUT_SUCCESS:
  3891. case ISCSI_LOGOUT_CID_NOT_FOUND:
  3892. case ISCSI_LOGOUT_RECOVERY_UNSUPPORTED:
  3893. case ISCSI_LOGOUT_CLEANUP_FAILED:
  3894. default:
  3895. break;
  3896. }
  3897. ret = 1;
  3898. break;
  3899. default:
  3900. break;
  3901. }
  3902. return ret;
  3903. }
  3904. EXPORT_SYMBOL(iscsit_logout_post_handler);
  3905. void iscsit_fail_session(struct iscsi_session *sess)
  3906. {
  3907. struct iscsi_conn *conn;
  3908. spin_lock_bh(&sess->conn_lock);
  3909. list_for_each_entry(conn, &sess->sess_conn_list, conn_list) {
  3910. pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n");
  3911. conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT;
  3912. }
  3913. spin_unlock_bh(&sess->conn_lock);
  3914. pr_debug("Moving to TARG_SESS_STATE_FAILED.\n");
  3915. sess->session_state = TARG_SESS_STATE_FAILED;
  3916. }
  3917. void iscsit_stop_session(
  3918. struct iscsi_session *sess,
  3919. int session_sleep,
  3920. int connection_sleep)
  3921. {
  3922. u16 conn_count = atomic_read(&sess->nconn);
  3923. struct iscsi_conn *conn, *conn_tmp = NULL;
  3924. int is_last;
  3925. spin_lock_bh(&sess->conn_lock);
  3926. if (connection_sleep) {
  3927. list_for_each_entry_safe(conn, conn_tmp, &sess->sess_conn_list,
  3928. conn_list) {
  3929. if (conn_count == 0)
  3930. break;
  3931. if (list_is_last(&conn->conn_list, &sess->sess_conn_list)) {
  3932. is_last = 1;
  3933. } else {
  3934. iscsit_inc_conn_usage_count(conn_tmp);
  3935. is_last = 0;
  3936. }
  3937. iscsit_inc_conn_usage_count(conn);
  3938. spin_unlock_bh(&sess->conn_lock);
  3939. iscsit_cause_connection_reinstatement(conn, 1);
  3940. spin_lock_bh(&sess->conn_lock);
  3941. iscsit_dec_conn_usage_count(conn);
  3942. if (is_last == 0)
  3943. iscsit_dec_conn_usage_count(conn_tmp);
  3944. conn_count--;
  3945. }
  3946. } else {
  3947. list_for_each_entry(conn, &sess->sess_conn_list, conn_list)
  3948. iscsit_cause_connection_reinstatement(conn, 0);
  3949. }
  3950. if (session_sleep && atomic_read(&sess->nconn)) {
  3951. spin_unlock_bh(&sess->conn_lock);
  3952. wait_for_completion(&sess->session_wait_comp);
  3953. } else
  3954. spin_unlock_bh(&sess->conn_lock);
  3955. }
  3956. int iscsit_release_sessions_for_tpg(struct iscsi_portal_group *tpg, int force)
  3957. {
  3958. struct iscsi_session *sess;
  3959. struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
  3960. struct se_session *se_sess, *se_sess_tmp;
  3961. LIST_HEAD(free_list);
  3962. int session_count = 0;
  3963. spin_lock_bh(&se_tpg->session_lock);
  3964. if (tpg->nsessions && !force) {
  3965. spin_unlock_bh(&se_tpg->session_lock);
  3966. return -1;
  3967. }
  3968. list_for_each_entry_safe(se_sess, se_sess_tmp, &se_tpg->tpg_sess_list,
  3969. sess_list) {
  3970. sess = (struct iscsi_session *)se_sess->fabric_sess_ptr;
  3971. spin_lock(&sess->conn_lock);
  3972. if (atomic_read(&sess->session_fall_back_to_erl0) ||
  3973. atomic_read(&sess->session_logout) ||
  3974. atomic_read(&sess->session_close) ||
  3975. (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)) {
  3976. spin_unlock(&sess->conn_lock);
  3977. continue;
  3978. }
  3979. iscsit_inc_session_usage_count(sess);
  3980. atomic_set(&sess->session_reinstatement, 1);
  3981. atomic_set(&sess->session_fall_back_to_erl0, 1);
  3982. atomic_set(&sess->session_close, 1);
  3983. spin_unlock(&sess->conn_lock);
  3984. list_move_tail(&se_sess->sess_list, &free_list);
  3985. }
  3986. spin_unlock_bh(&se_tpg->session_lock);
  3987. list_for_each_entry_safe(se_sess, se_sess_tmp, &free_list, sess_list) {
  3988. sess = (struct iscsi_session *)se_sess->fabric_sess_ptr;
  3989. list_del_init(&se_sess->sess_list);
  3990. iscsit_stop_session(sess, 1, 1);
  3991. iscsit_dec_session_usage_count(sess);
  3992. session_count++;
  3993. }
  3994. pr_debug("Released %d iSCSI Session(s) from Target Portal"
  3995. " Group: %hu\n", session_count, tpg->tpgt);
  3996. return 0;
  3997. }
  3998. MODULE_DESCRIPTION("iSCSI-Target Driver for mainline target infrastructure");
  3999. MODULE_VERSION("4.1.x");
  4000. MODULE_AUTHOR("nab@Linux-iSCSI.org");
  4001. MODULE_LICENSE("GPL");
  4002. module_init(iscsi_target_init_module);
  4003. module_exit(iscsi_target_cleanup_module);